Haley V2 import and the NDI V2 object layer (both V_eta shapes)

.github/workflows/test-eta-migrate-e2e.yml

@@ -111,7 +111,10 @@ jobs:
       - name: Set up MATLAB
         uses: matlab-actions/setup-matlab@v2
         with:
-          release: latest
+          # PINNED: `latest` moved to R2026b (2026-10-01), whose
+          # matlab.addons.install no longer accepts the agreeToLicense argument
+          # MatBox's installFexPackage passes. Unpin when MatBox is updated.
+          release: R2026a
           cache: true
           products: |
             Statistics_and_Machine_Learning_Toolbox

.github/workflows/test-import-v2.yml

@@ -0,0 +1,253 @@
+name: V2 import (Haley) -- fast
+
+# Stage-by-stage tests of ndi.setup.conv.haley.import_V2 and the generic
+# ndi.setup.V2 makers it uses. They build V2 (V_eta) documents through
+# DID-matlab's did2.build, so this job needs DID-matlab V2 and a did-schema
+# that carries the classes the import uses.
+#
+# DID_SCHEMA_REF: main. The `study` entity, `instance_of`, `awarded_to` and
+# `directed_relation.roles` the import uses reached main with did-schema PR #78
+# (merged 2026-09-30).
+#
+# The tests FILTER themselves when the schema or did2.build is missing; this
+# job fails if any were filtered, so a green run always means they ran.
+
+on:
+  push:
+    branches: ['claude/**']
+    paths:
+      - 'src/ndi/+ndi/+setup/+V2/**'
+      - 'src/ndi/+ndi/+setup/+conv/+haley/import_V2*'
+      - 'src/ndi/+ndi/+setup/+conv/+haley/**'
+      - 'src/ndi/+ndi/+setup/+conv/+haley/sessionList.m'
+      - 'src/ndi/+ndi/+setup/+conv/+haley/subjectList.m'
+      - 'src/ndi/+ndi/+setup/+conv/+haley/idPrefix.m'
+      - 'src/ndi/+ndi/+setup/+conv/+haley/recordingList.m'
+      - 'src/ndi/+ndi/+setup/+conv/+haley/sessionDocuments.m'
+      - 'src/ndi/+ndi/+setup/+conv/+haley/relationDocuments.m'
+      - 'src/ndi/+ndi/+setup/+conv/+haley/transferTime.m'
+      - 'src/ndi/+ndi/+file/+navigator/bodies.m'
+      - 'src/ndi/+ndi/+database/+fun/externalFileLocation.m'
+      - 'src/ndi/+ndi/+database/+implementations/+database/did2sqlite.m'
+      - 'src/ndi/+ndi/+database/+internal/applyReadNormalization.m'
+      - 'tests/+ndi/+unittest/+setup/+V2/**'
+      - 'src/ndi/+ndi/+vintage/**'
+      - 'src/ndi/+ndi/+v2/**'
+      - 'src/ndi/+ndi/entity.m'
+      - 'src/ndi/+ndi/dataset.m'
+      - 'src/ndi/+ndi/+dataset/dir.m'
+      - 'src/ndi/+ndi/session.m'
+      - 'src/ndi/+ndi/subject.m'
+      - 'src/ndi/+ndi/statement.m'
+      - 'src/ndi/+ndi/assertion.m'
+      - 'src/ndi/+ndi/interaction.m'
+      - 'src/ndi/+ndi/observation.m'
+      - 'src/ndi/+ndi/manipulation.m'
+      - 'src/ndi/+ndi/calculation.m'
+      - 'src/ndi/+ndi/data_type.m'
+      - 'tests/+ndi/+unittest/+vintage/**'
+      - 'src/ndi/+ndi/+fun/+doc/+lightsheet/**'
+      - 'src/ndi/+ndi/+gui/+app/LightsheetZarrManager.m'
+      - 'src/ndi/+ndi/+database/+fun/extract_docs_files.m'
+      - 'src/ndi/+ndi/+database/+internal/list_binary_files.m'
+      - 'tests/+ndi/+unittest/+fun/+doc/+lightsheet/**'
+      - 'tests/+ndi/+unittest/+gui/+app/TestLightsheetZarrManager.m'
+      - 'src/ndi/+ndi/+database/+implementations/+database/didsqlite.m'
+      - 'tests/+ndi/+unittest/+database/TestV1SubjectSearch.m'
+      - 'src/ndi/+ndi/+cloud/+sync/+internal/reconstructSeriesIngestLocations.m'
+      - 'src/ndi/+ndi/+cloud/+sync/+internal/updateFileInfoForLocalFiles.m'
+      - 'src/ndi/+ndi/+cloud/+sync/+internal/updateFileInfoForRemoteFiles.m'
+      - 'tests/+ndi/+unittest/ReconstructSeriesIngestLocationsTest.m'
+      - '.github/workflows/test-import-v2.yml'
+  pull_request:
+    branches: [main, V2]
+    paths:
+      - 'src/ndi/+ndi/+setup/+V2/**'
+      - 'src/ndi/+ndi/+setup/+conv/+haley/import_V2*'
+      - 'src/ndi/+ndi/+setup/+conv/+haley/**'
+      - 'src/ndi/+ndi/+setup/+conv/+haley/sessionList.m'
+      - 'src/ndi/+ndi/+setup/+conv/+haley/subjectList.m'
+      - 'src/ndi/+ndi/+setup/+conv/+haley/idPrefix.m'
+      - 'src/ndi/+ndi/+setup/+conv/+haley/recordingList.m'
+      - 'src/ndi/+ndi/+setup/+conv/+haley/sessionDocuments.m'
+      - 'src/ndi/+ndi/+setup/+conv/+haley/relationDocuments.m'
+      - 'src/ndi/+ndi/+setup/+conv/+haley/transferTime.m'
+      - 'src/ndi/+ndi/+file/+navigator/bodies.m'
+      - 'src/ndi/+ndi/+database/+fun/externalFileLocation.m'
+      - 'src/ndi/+ndi/+database/+implementations/+database/did2sqlite.m'
+      - 'src/ndi/+ndi/+database/+internal/applyReadNormalization.m'
+      - 'tests/+ndi/+unittest/+setup/+V2/**'
+      - 'src/ndi/+ndi/+vintage/**'
+      - 'src/ndi/+ndi/+v2/**'
+      - 'src/ndi/+ndi/entity.m'
+      - 'src/ndi/+ndi/dataset.m'
+      - 'src/ndi/+ndi/+dataset/dir.m'
+      - 'src/ndi/+ndi/session.m'
+      - 'src/ndi/+ndi/subject.m'
+      - 'src/ndi/+ndi/statement.m'
+      - 'src/ndi/+ndi/assertion.m'
+      - 'src/ndi/+ndi/interaction.m'
+      - 'src/ndi/+ndi/observation.m'
+      - 'src/ndi/+ndi/manipulation.m'
+      - 'src/ndi/+ndi/calculation.m'
+      - 'src/ndi/+ndi/data_type.m'
+      - 'tests/+ndi/+unittest/+vintage/**'
+      - 'src/ndi/+ndi/+fun/+doc/+lightsheet/**'
+      - 'src/ndi/+ndi/+gui/+app/LightsheetZarrManager.m'
+      - 'src/ndi/+ndi/+database/+fun/extract_docs_files.m'
+      - 'src/ndi/+ndi/+database/+internal/list_binary_files.m'
+      - 'tests/+ndi/+unittest/+fun/+doc/+lightsheet/**'
+      - 'tests/+ndi/+unittest/+gui/+app/TestLightsheetZarrManager.m'
+      - 'src/ndi/+ndi/+database/+implementations/+database/didsqlite.m'
+      - 'tests/+ndi/+unittest/+database/TestV1SubjectSearch.m'
+      - 'src/ndi/+ndi/+cloud/+sync/+internal/reconstructSeriesIngestLocations.m'
+      - 'src/ndi/+ndi/+cloud/+sync/+internal/updateFileInfoForLocalFiles.m'
+      - 'src/ndi/+ndi/+cloud/+sync/+internal/updateFileInfoForRemoteFiles.m'
+      - 'tests/+ndi/+unittest/ReconstructSeriesIngestLocationsTest.m'
+      - '.github/workflows/test-import-v2.yml'
+  workflow_dispatch:
+
+concurrency:
+  group: test-import-v2-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  import-v2:
+    name: V2 import stages (${{ matrix.schema }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        # current: did-schema main with DID-matlab V2 (from requirements.txt).
+        # next: the one-entity-class / composition schema and the did2.build that
+        # reads it, both still on review branches (did-schema PR #88, DID-matlab
+        # PR #218). The import and the readers build and read both shapes, so both
+        # legs must pass. Drop `next` once both merge.
+        include:
+          - schema: current
+            did_schema_ref: main
+            did_matlab_ref: ''
+          - schema: next
+            did_schema_ref: claude/ecstatic-pascal-md91fw
+            did_matlab_ref: claude/ecstatic-pascal-md91fw
+    env:
+      DID_SCHEMA_REF: ${{ matrix.did_schema_ref }}
+    steps:
+      - name: Check out NDI-matlab
+        uses: actions/checkout@v4
+
+      - name: Check out did-schema
+        uses: actions/checkout@v4
+        with:
+          repository: Waltham-Data-Science/did-schema
+          ref: ${{ env.DID_SCHEMA_REF }}
+          path: did-schema
+
+      - name: Assemble the V2 schema set
+        run: |
+          mkdir -p "${GITHUB_WORKSPACE}/eta-schemas"
+          cp did-schema/schemas/V_eta/stable/*.json "${GITHUB_WORKSPACE}/eta-schemas/"
+          cp did-schema/schemas/V_eta/draft/*.json  "${GITHUB_WORKSPACE}/eta-schemas/"
+          cp did-schema/schemas/V_eta/deprecated/*.json "${GITHUB_WORKSPACE}/eta-schemas/"
+          echo "DENOMINATOR: $(ls "${GITHUB_WORKSPACE}/eta-schemas" | wc -l) schema file(s) from did-schema@$(git -C did-schema rev-parse --short HEAD)"
+          echo "DID_SCHEMA_PATH=${GITHUB_WORKSPACE}/eta-schemas" >> "$GITHUB_ENV"
+
+      - name: Check out DID-matlab (next leg only)
+        if: ${{ matrix.did_matlab_ref != '' }}
+        uses: actions/checkout@v4
+        with:
+          repository: VH-Lab/DID-matlab
+          ref: ${{ matrix.did_matlab_ref }}
+          path: did-matlab-next
+
+      - name: Set up MATLAB
+        uses: matlab-actions/setup-matlab@v2
+        with:
+          # PINNED: `latest` moved to R2026b (2026-10-01), whose
+          # matlab.addons.install no longer accepts the agreeToLicense argument
+          # MatBox's installFexPackage passes, so dependency install fails before
+          # any test runs. Unpin when MatBox is updated.
+          release: R2026a
+          cache: true
+
+      - name: Install MatBox
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        uses: ehennestad/matbox-actions/install-matbox@v1
+
+      - name: Install repo dependencies
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        uses: matlab-actions/run-command@v2
+        with:
+          command: |
+            addpath(genpath("src"));
+            addpath(genpath("tools"));
+            matbox.installRequirements(nditools.projectdir());
+            matbox.installRequirements(fullfile(pwd, "tests"));
+
+      - name: Run the V2 import tests
+        uses: matlab-actions/run-command@v2
+        with:
+          command: |
+            addpath(genpath("src"));
+            addpath(genpath("tests"));
+            if isfolder("did-matlab-next/src")
+                % ahead of the installed DID-matlab V2 on the path
+                addpath(genpath(fullfile(pwd, "did-matlab-next", "src")));
+                disp("DID-matlab from did-matlab-next (" + string(which("did2.build.document")) + ")");
+            end
+            import matlab.unittest.TestSuite;
+            suite = [TestSuite.fromPackage("ndi.unittest.setup.V2", "IncludingSubpackages", true), ...
+                     TestSuite.fromClass(?ndi.unittest.vintage.TestVintageMap), ...
+                     TestSuite.fromClass(?ndi.unittest.vintage.TestLightsheetVintage), ...
+                     TestSuite.fromClass(?ndi.unittest.vintage.TestLightsheetMigratedSession)];
+            % The lightsheet readers over the REAL migrator's output need
+            % DID-matlab's lightsheet migrators, which only the next leg's
+            % DID-matlab has. Selected there, not filtered here.
+            if isfolder("did-matlab-next/src")
+                suite = [suite, TestSuite.fromClass(?ndi.unittest.vintage.TestLightsheetMigrated)];
+            end
+            results = run(suite);
+            disp(table(results));
+            nRun = numel(results);
+            nFailed = sum([results.Failed]);
+            nIncomplete = sum([results.Incomplete]);
+            fprintf("VERDICT: %d run, %d failed, %d filtered\n", nRun, nFailed, nIncomplete);
+            assert(nRun > 0, "no V2 import tests were selected");
+            assert(nFailed == 0, sprintf("%d V2 import test(s) failed", nFailed));
+            assert(nIncomplete == 0, sprintf("%d V2 import test(s) were FILTERED -- did2.build or the schema is missing", nIncomplete));
+
+      - name: Run the v1 lightsheet reader tests (v1 must read exactly as before)
+        # The lightsheet readers now route through ndi.vintage so they also
+        # read a MIGRATED pyramid. These are the existing v1 tests (legacy
+        # session, documents makePyramid writes), otherwise run only by
+        # run-tests.yml on main/Vnext; here so a branch that touches the
+        # readers proves v1 unchanged. A filtered test (a missing optional
+        # dependency) is reported, not counted as a pass, and not a failure.
+        uses: matlab-actions/run-command@v2
+        with:
+          command: |
+            addpath(genpath("src"));
+            addpath(genpath("tests"));
+            if isfolder("did-matlab-next/src")
+                addpath(genpath(fullfile(pwd, "did-matlab-next", "src")));
+            end
+            % as run-tests.yml runs them: no V2 schema set
+            setenv("DID_SCHEMA_PATH", "");
+            import matlab.unittest.TestSuite;
+            suite = [TestSuite.fromClass(?ndi.unittest.fun.doc.lightsheet.TestLevelTable), ...
+                     TestSuite.fromClass(?ndi.unittest.fun.doc.lightsheet.TestChooseLevel), ...
+                     TestSuite.fromClass(?ndi.unittest.fun.doc.lightsheet.TestMakePyramid), ...
+                     TestSuite.fromClass(?ndi.unittest.fun.doc.lightsheet.TestFromSPIM), ...
+                     TestSuite.fromClass(?ndi.unittest.gui.app.TestLightsheetZarrManager), ...
+                     TestSuite.fromClass(?ndi.unittest.database.TestV1SubjectSearch), ...
+                     TestSuite.fromClass(?ndi.unittest.ReconstructSeriesIngestLocationsTest)];
+            results = run(suite);
+            disp(table(results));
+            nRun = numel(results);
+            nFailed = sum([results.Failed]);
+            nIncomplete = sum([results.Incomplete]);
+            fprintf("VERDICT (v1 lightsheet): %d run, %d failed, %d filtered\n", nRun, nFailed, nIncomplete);
+            assert(nRun > 0, "no v1 lightsheet tests were selected");
+            assert(nFailed == 0, sprintf("%d v1 lightsheet test(s) failed", nFailed));

find_napari_debug_log.sh

@@ -0,0 +1,65 @@
+#!/usr/bin/env zsh
+# find_napari_debug_log.sh - locate and print the napariViewLightsheet debug
+# log written by ndi.test.cloud.lightsheet_blob_cloud_roundtrip.
+#
+# The demo defaults DebugLogFile to fullfile(tempdir, 'NapariLightSheetDebugging.txt')
+# which on macOS resolves under $TMPDIR (typically /var/folders/xx/yyy/T/),
+# not /tmp. This script looks in the usual spots, picks the newest match,
+# and cats it.
+
+set -u
+setopt NULL_GLOB EXTENDED_GLOB
+
+name="NapariLightSheetDebugging.txt"
+
+# Places to check, ordered by likelihood on macOS.
+locations=(
+    "${TMPDIR:-}/$name"                 # matches MATLAB's tempdir
+    "/tmp/$name"                        # a user who set DebugLogFile explicitly
+    "$HOME/$name"                       # or dropped it in their home
+    "/private/tmp/$name"                # macOS's real /tmp target
+)
+
+# De-duplicate, drop empties, and keep only paths that actually exist.
+typeset -a found
+for p in "${locations[@]}"; do
+    [[ -n "$p" && -f "$p" ]] && found+=("$p")
+done
+
+# Wider net if the shortlist misses: recurse under /var/folders and $TMPDIR
+# (macOS parks per-user tempdirs there). Prune common noise; keep the
+# recent ones.
+if (( ${#found} == 0 )); then
+    for root in "${TMPDIR:-}" /var/folders /private/var/folders /tmp; do
+        [[ -n "$root" && -d "$root" ]] || continue
+        while IFS= read -r hit; do
+            [[ -n "$hit" ]] && found+=("$hit")
+        done < <(find "$root" -maxdepth 6 -type f -name "$name" 2>/dev/null)
+    done
+fi
+
+if (( ${#found} == 0 )); then
+    print -u2 "No $name found. If your demo ran with a custom DebugLogFile,"
+    print -u2 "point me at it:  $0  /full/path/to/logfile"
+    print -u2 "Or check the value of tempdir in MATLAB (>> tempdir)."
+    exit 1
+fi
+
+# Pick the newest match if multiple.
+newest="$found[1]"
+newest_mtime=$(stat -f %m "$newest" 2>/dev/null || stat -c %Y "$newest" 2>/dev/null)
+for f in "$found[@]"; do
+    m=$(stat -f %m "$f" 2>/dev/null || stat -c %Y "$f" 2>/dev/null)
+    if [[ -n "$m" && "$m" -gt "$newest_mtime" ]]; then
+        newest="$f"
+        newest_mtime="$m"
+    fi
+done
+
+print -u2 "=== Reading: $newest ==="
+if (( ${#found} > 1 )); then
+    print -u2 "(also found: ${(j:, :)found[2,-1]})"
+fi
+print -u2 ""
+
+cat -- "$newest"

lightsheet_test_blob_ingest_and_view.m

@@ -0,0 +1,57 @@
+%LIGHTSHEET_TEST_BLOB_INGEST_AND_VIEW - end-to-end smoke test on the synthetic blob
+%
+%   Builds a small OME-Zarr fixture on disk (a couple of MB), ingests it
+%   into a fresh NDI session with the blosc-zstd fast path, and launches
+%   the napari viewer on the resulting pyramid. Use this to sanity-check
+%   the ingest -> document -> viewer pipeline without touching the real
+%   SmartSPIM data. Runs in under a minute on a laptop.
+%
+%   Edit CONFIG below if you want to keep the fixture / session between
+%   runs, change the blob size, or point at a different napari launcher.
+%
+%   See also: ndi.test.lightsheet.makeBlobFixture,
+%             ndi.test.lightsheet.ingestBlobFixture,
+%             ndi.fun.doc.lightsheet.fromOMEZarr,
+%             ndi.fun.doc.lightsheet.view
+
+% -------------------- CONFIG -----------------------------------------
+% Leave scratch dirs empty to land under tempname (fresh every run).
+sessionDir  = '';                              % e.g. '~/ndi-sessions/blob-demo'
+zarrDir     = '';                              % e.g. '~/ndi-sessions/blob-demo/zarr'
+subjectName = 'blob@vhlab';
+
+% Blob size / shape. 128^3 uint16 across 2 channels ~= 16 MB raw; small
+% enough for a fast smoke test, big enough that the pyramid has real
+% levels to look at.
+shape       = [128 128 128];
+numChannels = 2;
+chunkShape  = [32 32 32];
+numLevels   = 3;
+
+% Ingest options. blosc-zstd exercises the encodeChunk fast path; keep
+% clevel modest for a smoke test.
+codec       = 'blosc-zstd';
+clevel      = 5;
+
+% Napari launcher. Same wrapper the real-data script uses.
+launcher    = '/usr/local/bin/napariViewLightsheet';
+% ---------------------------------------------------------------------
+
+fprintf('Building blob fixture and ingesting ...\n');
+[S, P, info] = ndi.test.lightsheet.ingestBlobFixture( ...
+    'sessionDir',  sessionDir, ...
+    'zarrDir',     zarrDir, ...
+    'subject',     subjectName, ...
+    'Shape',       shape, ...
+    'NumChannels', numChannels, ...
+    'ChunkShape',  chunkShape, ...
+    'NumLevels',   numLevels, ...
+    'codec',       codec, ...
+    'clevel',      clevel);
+
+fprintf('Session:  %s\n', char(S.path()));
+fprintf('Zarr:     %s\n', info.zarrPath);
+fprintf('Pyramid:  %s\n', P.id());
+
+fprintf('Launching napari viewer ...\n');
+ndi.fun.doc.lightsheet.view(S, P.id(), 'launcher', launcher);

requirements.txt

@@ -11,4 +11,4 @@ https://github.com/Waltham-Data-Science/NDI-compress-matlabp
 https://github.com/Waltham-Data-Science/ndi-ontology-matlab
 https://github.com/VH-Lab/pyraview
 https://github.com/bastibe/Violinplot-Matlab
-fex://134212-openminds-metadata-models-for-matlab/0.11.0
+https://github.com/openMetadataInitiative/openMINDS_MATLAB@v0.12.0