diff --git a/.github/workflows/test-eta-migrate-e2e.yml b/.github/workflows/test-eta-migrate-e2e.yml index 2e96a119cc..2651b8efa1 100644 --- a/.github/workflows/test-eta-migrate-e2e.yml +++ b/.github/workflows/test-eta-migrate-e2e.yml @@ -111,7 +111,10 @@ jobs: - name: Set up MATLAB uses: matlab-actions/setup-matlab@v2 with: - release: latest + # PINNED: `latest` moved to R2026b (2026-10-01), whose + # matlab.addons.install no longer accepts the agreeToLicense argument + # MatBox's installFexPackage passes. Unpin when MatBox is updated. + release: R2026a cache: true products: | Statistics_and_Machine_Learning_Toolbox diff --git a/.github/workflows/test-import-v2.yml b/.github/workflows/test-import-v2.yml new file mode 100644 index 0000000000..ec4227cd7f --- /dev/null +++ b/.github/workflows/test-import-v2.yml @@ -0,0 +1,253 @@ +name: V2 import (Haley) -- fast + +# Stage-by-stage tests of ndi.setup.conv.haley.import_V2 and the generic +# ndi.setup.V2 makers it uses. They build V2 (V_eta) documents through +# DID-matlab's did2.build, so this job needs DID-matlab V2 and a did-schema +# that carries the classes the import uses. +# +# DID_SCHEMA_REF: main. The `study` entity, `instance_of`, `awarded_to` and +# `directed_relation.roles` the import uses reached main with did-schema PR #78 +# (merged 2026-09-30). +# +# The tests FILTER themselves when the schema or did2.build is missing; this +# job fails if any were filtered, so a green run always means they ran. + +on: + push: + branches: ['claude/**'] + paths: + - 'src/ndi/+ndi/+setup/+V2/**' + - 'src/ndi/+ndi/+setup/+conv/+haley/import_V2*' + - 'src/ndi/+ndi/+setup/+conv/+haley/**' + - 'src/ndi/+ndi/+setup/+conv/+haley/sessionList.m' + - 'src/ndi/+ndi/+setup/+conv/+haley/subjectList.m' + - 'src/ndi/+ndi/+setup/+conv/+haley/idPrefix.m' + - 'src/ndi/+ndi/+setup/+conv/+haley/recordingList.m' + - 'src/ndi/+ndi/+setup/+conv/+haley/sessionDocuments.m' + - 'src/ndi/+ndi/+setup/+conv/+haley/relationDocuments.m' + - 'src/ndi/+ndi/+setup/+conv/+haley/transferTime.m' + - 'src/ndi/+ndi/+file/+navigator/bodies.m' + - 'src/ndi/+ndi/+database/+fun/externalFileLocation.m' + - 'src/ndi/+ndi/+database/+implementations/+database/did2sqlite.m' + - 'src/ndi/+ndi/+database/+internal/applyReadNormalization.m' + - 'tests/+ndi/+unittest/+setup/+V2/**' + - 'src/ndi/+ndi/+vintage/**' + - 'src/ndi/+ndi/+v2/**' + - 'src/ndi/+ndi/entity.m' + - 'src/ndi/+ndi/dataset.m' + - 'src/ndi/+ndi/+dataset/dir.m' + - 'src/ndi/+ndi/session.m' + - 'src/ndi/+ndi/subject.m' + - 'src/ndi/+ndi/statement.m' + - 'src/ndi/+ndi/assertion.m' + - 'src/ndi/+ndi/interaction.m' + - 'src/ndi/+ndi/observation.m' + - 'src/ndi/+ndi/manipulation.m' + - 'src/ndi/+ndi/calculation.m' + - 'src/ndi/+ndi/data_type.m' + - 'tests/+ndi/+unittest/+vintage/**' + - 'src/ndi/+ndi/+fun/+doc/+lightsheet/**' + - 'src/ndi/+ndi/+gui/+app/LightsheetZarrManager.m' + - 'src/ndi/+ndi/+database/+fun/extract_docs_files.m' + - 'src/ndi/+ndi/+database/+internal/list_binary_files.m' + - 'tests/+ndi/+unittest/+fun/+doc/+lightsheet/**' + - 'tests/+ndi/+unittest/+gui/+app/TestLightsheetZarrManager.m' + - 'src/ndi/+ndi/+database/+implementations/+database/didsqlite.m' + - 'tests/+ndi/+unittest/+database/TestV1SubjectSearch.m' + - 'src/ndi/+ndi/+cloud/+sync/+internal/reconstructSeriesIngestLocations.m' + - 'src/ndi/+ndi/+cloud/+sync/+internal/updateFileInfoForLocalFiles.m' + - 'src/ndi/+ndi/+cloud/+sync/+internal/updateFileInfoForRemoteFiles.m' + - 'tests/+ndi/+unittest/ReconstructSeriesIngestLocationsTest.m' + - '.github/workflows/test-import-v2.yml' + pull_request: + branches: [main, V2] + paths: + - 'src/ndi/+ndi/+setup/+V2/**' + - 'src/ndi/+ndi/+setup/+conv/+haley/import_V2*' + - 'src/ndi/+ndi/+setup/+conv/+haley/**' + - 'src/ndi/+ndi/+setup/+conv/+haley/sessionList.m' + - 'src/ndi/+ndi/+setup/+conv/+haley/subjectList.m' + - 'src/ndi/+ndi/+setup/+conv/+haley/idPrefix.m' + - 'src/ndi/+ndi/+setup/+conv/+haley/recordingList.m' + - 'src/ndi/+ndi/+setup/+conv/+haley/sessionDocuments.m' + - 'src/ndi/+ndi/+setup/+conv/+haley/relationDocuments.m' + - 'src/ndi/+ndi/+setup/+conv/+haley/transferTime.m' + - 'src/ndi/+ndi/+file/+navigator/bodies.m' + - 'src/ndi/+ndi/+database/+fun/externalFileLocation.m' + - 'src/ndi/+ndi/+database/+implementations/+database/did2sqlite.m' + - 'src/ndi/+ndi/+database/+internal/applyReadNormalization.m' + - 'tests/+ndi/+unittest/+setup/+V2/**' + - 'src/ndi/+ndi/+vintage/**' + - 'src/ndi/+ndi/+v2/**' + - 'src/ndi/+ndi/entity.m' + - 'src/ndi/+ndi/dataset.m' + - 'src/ndi/+ndi/+dataset/dir.m' + - 'src/ndi/+ndi/session.m' + - 'src/ndi/+ndi/subject.m' + - 'src/ndi/+ndi/statement.m' + - 'src/ndi/+ndi/assertion.m' + - 'src/ndi/+ndi/interaction.m' + - 'src/ndi/+ndi/observation.m' + - 'src/ndi/+ndi/manipulation.m' + - 'src/ndi/+ndi/calculation.m' + - 'src/ndi/+ndi/data_type.m' + - 'tests/+ndi/+unittest/+vintage/**' + - 'src/ndi/+ndi/+fun/+doc/+lightsheet/**' + - 'src/ndi/+ndi/+gui/+app/LightsheetZarrManager.m' + - 'src/ndi/+ndi/+database/+fun/extract_docs_files.m' + - 'src/ndi/+ndi/+database/+internal/list_binary_files.m' + - 'tests/+ndi/+unittest/+fun/+doc/+lightsheet/**' + - 'tests/+ndi/+unittest/+gui/+app/TestLightsheetZarrManager.m' + - 'src/ndi/+ndi/+database/+implementations/+database/didsqlite.m' + - 'tests/+ndi/+unittest/+database/TestV1SubjectSearch.m' + - 'src/ndi/+ndi/+cloud/+sync/+internal/reconstructSeriesIngestLocations.m' + - 'src/ndi/+ndi/+cloud/+sync/+internal/updateFileInfoForLocalFiles.m' + - 'src/ndi/+ndi/+cloud/+sync/+internal/updateFileInfoForRemoteFiles.m' + - 'tests/+ndi/+unittest/ReconstructSeriesIngestLocationsTest.m' + - '.github/workflows/test-import-v2.yml' + workflow_dispatch: + +concurrency: + group: test-import-v2-${{ github.ref }} + cancel-in-progress: true + +jobs: + import-v2: + name: V2 import stages (${{ matrix.schema }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + # current: did-schema main with DID-matlab V2 (from requirements.txt). + # next: the one-entity-class / composition schema and the did2.build that + # reads it, both still on review branches (did-schema PR #88, DID-matlab + # PR #218). The import and the readers build and read both shapes, so both + # legs must pass. Drop `next` once both merge. + include: + - schema: current + did_schema_ref: main + did_matlab_ref: '' + - schema: next + did_schema_ref: claude/ecstatic-pascal-md91fw + did_matlab_ref: claude/ecstatic-pascal-md91fw + env: + DID_SCHEMA_REF: ${{ matrix.did_schema_ref }} + steps: + - name: Check out NDI-matlab + uses: actions/checkout@v4 + + - name: Check out did-schema + uses: actions/checkout@v4 + with: + repository: Waltham-Data-Science/did-schema + ref: ${{ env.DID_SCHEMA_REF }} + path: did-schema + + - name: Assemble the V2 schema set + run: | + mkdir -p "${GITHUB_WORKSPACE}/eta-schemas" + cp did-schema/schemas/V_eta/stable/*.json "${GITHUB_WORKSPACE}/eta-schemas/" + cp did-schema/schemas/V_eta/draft/*.json "${GITHUB_WORKSPACE}/eta-schemas/" + cp did-schema/schemas/V_eta/deprecated/*.json "${GITHUB_WORKSPACE}/eta-schemas/" + echo "DENOMINATOR: $(ls "${GITHUB_WORKSPACE}/eta-schemas" | wc -l) schema file(s) from did-schema@$(git -C did-schema rev-parse --short HEAD)" + echo "DID_SCHEMA_PATH=${GITHUB_WORKSPACE}/eta-schemas" >> "$GITHUB_ENV" + + - name: Check out DID-matlab (next leg only) + if: ${{ matrix.did_matlab_ref != '' }} + uses: actions/checkout@v4 + with: + repository: VH-Lab/DID-matlab + ref: ${{ matrix.did_matlab_ref }} + path: did-matlab-next + + - name: Set up MATLAB + uses: matlab-actions/setup-matlab@v2 + with: + # PINNED: `latest` moved to R2026b (2026-10-01), whose + # matlab.addons.install no longer accepts the agreeToLicense argument + # MatBox's installFexPackage passes, so dependency install fails before + # any test runs. Unpin when MatBox is updated. + release: R2026a + cache: true + + - name: Install MatBox + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + uses: ehennestad/matbox-actions/install-matbox@v1 + + - name: Install repo dependencies + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + uses: matlab-actions/run-command@v2 + with: + command: | + addpath(genpath("src")); + addpath(genpath("tools")); + matbox.installRequirements(nditools.projectdir()); + matbox.installRequirements(fullfile(pwd, "tests")); + + - name: Run the V2 import tests + uses: matlab-actions/run-command@v2 + with: + command: | + addpath(genpath("src")); + addpath(genpath("tests")); + if isfolder("did-matlab-next/src") + % ahead of the installed DID-matlab V2 on the path + addpath(genpath(fullfile(pwd, "did-matlab-next", "src"))); + disp("DID-matlab from did-matlab-next (" + string(which("did2.build.document")) + ")"); + end + import matlab.unittest.TestSuite; + suite = [TestSuite.fromPackage("ndi.unittest.setup.V2", "IncludingSubpackages", true), ... + TestSuite.fromClass(?ndi.unittest.vintage.TestVintageMap), ... + TestSuite.fromClass(?ndi.unittest.vintage.TestLightsheetVintage), ... + TestSuite.fromClass(?ndi.unittest.vintage.TestLightsheetMigratedSession)]; + % The lightsheet readers over the REAL migrator's output need + % DID-matlab's lightsheet migrators, which only the next leg's + % DID-matlab has. Selected there, not filtered here. + if isfolder("did-matlab-next/src") + suite = [suite, TestSuite.fromClass(?ndi.unittest.vintage.TestLightsheetMigrated)]; + end + results = run(suite); + disp(table(results)); + nRun = numel(results); + nFailed = sum([results.Failed]); + nIncomplete = sum([results.Incomplete]); + fprintf("VERDICT: %d run, %d failed, %d filtered\n", nRun, nFailed, nIncomplete); + assert(nRun > 0, "no V2 import tests were selected"); + assert(nFailed == 0, sprintf("%d V2 import test(s) failed", nFailed)); + assert(nIncomplete == 0, sprintf("%d V2 import test(s) were FILTERED -- did2.build or the schema is missing", nIncomplete)); + + - name: Run the v1 lightsheet reader tests (v1 must read exactly as before) + # The lightsheet readers now route through ndi.vintage so they also + # read a MIGRATED pyramid. These are the existing v1 tests (legacy + # session, documents makePyramid writes), otherwise run only by + # run-tests.yml on main/Vnext; here so a branch that touches the + # readers proves v1 unchanged. A filtered test (a missing optional + # dependency) is reported, not counted as a pass, and not a failure. + uses: matlab-actions/run-command@v2 + with: + command: | + addpath(genpath("src")); + addpath(genpath("tests")); + if isfolder("did-matlab-next/src") + addpath(genpath(fullfile(pwd, "did-matlab-next", "src"))); + end + % as run-tests.yml runs them: no V2 schema set + setenv("DID_SCHEMA_PATH", ""); + import matlab.unittest.TestSuite; + suite = [TestSuite.fromClass(?ndi.unittest.fun.doc.lightsheet.TestLevelTable), ... + TestSuite.fromClass(?ndi.unittest.fun.doc.lightsheet.TestChooseLevel), ... + TestSuite.fromClass(?ndi.unittest.fun.doc.lightsheet.TestMakePyramid), ... + TestSuite.fromClass(?ndi.unittest.fun.doc.lightsheet.TestFromSPIM), ... + TestSuite.fromClass(?ndi.unittest.gui.app.TestLightsheetZarrManager), ... + TestSuite.fromClass(?ndi.unittest.database.TestV1SubjectSearch), ... + TestSuite.fromClass(?ndi.unittest.ReconstructSeriesIngestLocationsTest)]; + results = run(suite); + disp(table(results)); + nRun = numel(results); + nFailed = sum([results.Failed]); + nIncomplete = sum([results.Incomplete]); + fprintf("VERDICT (v1 lightsheet): %d run, %d failed, %d filtered\n", nRun, nFailed, nIncomplete); + assert(nRun > 0, "no v1 lightsheet tests were selected"); + assert(nFailed == 0, sprintf("%d v1 lightsheet test(s) failed", nFailed)); diff --git a/find_napari_debug_log.sh b/find_napari_debug_log.sh new file mode 100755 index 0000000000..967f935132 --- /dev/null +++ b/find_napari_debug_log.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env zsh +# find_napari_debug_log.sh - locate and print the napariViewLightsheet debug +# log written by ndi.test.cloud.lightsheet_blob_cloud_roundtrip. +# +# The demo defaults DebugLogFile to fullfile(tempdir, 'NapariLightSheetDebugging.txt') +# which on macOS resolves under $TMPDIR (typically /var/folders/xx/yyy/T/), +# not /tmp. This script looks in the usual spots, picks the newest match, +# and cats it. + +set -u +setopt NULL_GLOB EXTENDED_GLOB + +name="NapariLightSheetDebugging.txt" + +# Places to check, ordered by likelihood on macOS. +locations=( + "${TMPDIR:-}/$name" # matches MATLAB's tempdir + "/tmp/$name" # a user who set DebugLogFile explicitly + "$HOME/$name" # or dropped it in their home + "/private/tmp/$name" # macOS's real /tmp target +) + +# De-duplicate, drop empties, and keep only paths that actually exist. +typeset -a found +for p in "${locations[@]}"; do + [[ -n "$p" && -f "$p" ]] && found+=("$p") +done + +# Wider net if the shortlist misses: recurse under /var/folders and $TMPDIR +# (macOS parks per-user tempdirs there). Prune common noise; keep the +# recent ones. +if (( ${#found} == 0 )); then + for root in "${TMPDIR:-}" /var/folders /private/var/folders /tmp; do + [[ -n "$root" && -d "$root" ]] || continue + while IFS= read -r hit; do + [[ -n "$hit" ]] && found+=("$hit") + done < <(find "$root" -maxdepth 6 -type f -name "$name" 2>/dev/null) + done +fi + +if (( ${#found} == 0 )); then + print -u2 "No $name found. If your demo ran with a custom DebugLogFile," + print -u2 "point me at it: $0 /full/path/to/logfile" + print -u2 "Or check the value of tempdir in MATLAB (>> tempdir)." + exit 1 +fi + +# Pick the newest match if multiple. +newest="$found[1]" +newest_mtime=$(stat -f %m "$newest" 2>/dev/null || stat -c %Y "$newest" 2>/dev/null) +for f in "$found[@]"; do + m=$(stat -f %m "$f" 2>/dev/null || stat -c %Y "$f" 2>/dev/null) + if [[ -n "$m" && "$m" -gt "$newest_mtime" ]]; then + newest="$f" + newest_mtime="$m" + fi +done + +print -u2 "=== Reading: $newest ===" +if (( ${#found} > 1 )); then + print -u2 "(also found: ${(j:, :)found[2,-1]})" +fi +print -u2 "" + +cat -- "$newest" diff --git a/lightsheet_test_blob_ingest_and_view.m b/lightsheet_test_blob_ingest_and_view.m new file mode 100644 index 0000000000..974cd2fab4 --- /dev/null +++ b/lightsheet_test_blob_ingest_and_view.m @@ -0,0 +1,57 @@ +%LIGHTSHEET_TEST_BLOB_INGEST_AND_VIEW - end-to-end smoke test on the synthetic blob +% +% Builds a small OME-Zarr fixture on disk (a couple of MB), ingests it +% into a fresh NDI session with the blosc-zstd fast path, and launches +% the napari viewer on the resulting pyramid. Use this to sanity-check +% the ingest -> document -> viewer pipeline without touching the real +% SmartSPIM data. Runs in under a minute on a laptop. +% +% Edit CONFIG below if you want to keep the fixture / session between +% runs, change the blob size, or point at a different napari launcher. +% +% See also: ndi.test.lightsheet.makeBlobFixture, +% ndi.test.lightsheet.ingestBlobFixture, +% ndi.fun.doc.lightsheet.fromOMEZarr, +% ndi.fun.doc.lightsheet.view + +% -------------------- CONFIG ----------------------------------------- +% Leave scratch dirs empty to land under tempname (fresh every run). +sessionDir = ''; % e.g. '~/ndi-sessions/blob-demo' +zarrDir = ''; % e.g. '~/ndi-sessions/blob-demo/zarr' +subjectName = 'blob@vhlab'; + +% Blob size / shape. 128^3 uint16 across 2 channels ~= 16 MB raw; small +% enough for a fast smoke test, big enough that the pyramid has real +% levels to look at. +shape = [128 128 128]; +numChannels = 2; +chunkShape = [32 32 32]; +numLevels = 3; + +% Ingest options. blosc-zstd exercises the encodeChunk fast path; keep +% clevel modest for a smoke test. +codec = 'blosc-zstd'; +clevel = 5; + +% Napari launcher. Same wrapper the real-data script uses. +launcher = '/usr/local/bin/napariViewLightsheet'; +% --------------------------------------------------------------------- + +fprintf('Building blob fixture and ingesting ...\n'); +[S, P, info] = ndi.test.lightsheet.ingestBlobFixture( ... + 'sessionDir', sessionDir, ... + 'zarrDir', zarrDir, ... + 'subject', subjectName, ... + 'Shape', shape, ... + 'NumChannels', numChannels, ... + 'ChunkShape', chunkShape, ... + 'NumLevels', numLevels, ... + 'codec', codec, ... + 'clevel', clevel); + +fprintf('Session: %s\n', char(S.path())); +fprintf('Zarr: %s\n', info.zarrPath); +fprintf('Pyramid: %s\n', P.id()); + +fprintf('Launching napari viewer ...\n'); +ndi.fun.doc.lightsheet.view(S, P.id(), 'launcher', launcher); diff --git a/requirements.txt b/requirements.txt index 7d861f6c9e..19838a39c0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,4 +11,4 @@ https://github.com/Waltham-Data-Science/NDI-compress-matlabp https://github.com/Waltham-Data-Science/ndi-ontology-matlab https://github.com/VH-Lab/pyraview https://github.com/bastibe/Violinplot-Matlab -fex://134212-openminds-metadata-models-for-matlab/0.11.0 +https://github.com/openMetadataInitiative/openMINDS_MATLAB@v0.12.0 diff --git a/src/ndi/+ndi/+cloud/+api/+feedback/deleteFeedback.m b/src/ndi/+ndi/+cloud/+api/+feedback/deleteFeedback.m new file mode 100644 index 0000000000..42bd7a7e32 --- /dev/null +++ b/src/ndi/+ndi/+cloud/+api/+feedback/deleteFeedback.m @@ -0,0 +1,32 @@ +function [b, answer, apiResponse, apiURL] = deleteFeedback(feedbackID) +%DELETEFEEDBACK User-facing wrapper to withdraw one of your own problem reports. +% +% [B, ANSWER, APIRESPONSE, APIURL] = ndi.cloud.api.feedback.deleteFeedback(FEEDBACKID) +% +% Deletes a report previously sent with ndi.cloud.api.feedback.submitFeedback. +% Only the user who submitted a report can delete it; any other report +% (or one that does not exist) is answered with 404. +% +% Inputs: +% feedbackID - The id returned by submitFeedback. +% +% Outputs: +% b - True if the call succeeded, false otherwise. +% answer - A confirmation message on success. +% apiResponse - The full matlab.net.http.ResponseMessage object. +% apiURL - The URL that was called. +% +% See also: ndi.cloud.api.feedback.submitFeedback, +% ndi.cloud.api.implementation.feedback.DeleteFeedback + + arguments + feedbackID (1,1) string + end + + % 1. Create an instance of the implementation class. + api_call = ndi.cloud.api.implementation.feedback.DeleteFeedback('feedbackID', feedbackID); + + % 2. Call the execute method and return its outputs directly. + [b, answer, apiResponse, apiURL] = api_call.execute(); + +end diff --git a/src/ndi/+ndi/+cloud/+api/+feedback/submitFeedback.m b/src/ndi/+ndi/+cloud/+api/+feedback/submitFeedback.m new file mode 100644 index 0000000000..665a1aeda7 --- /dev/null +++ b/src/ndi/+ndi/+cloud/+api/+feedback/submitFeedback.m @@ -0,0 +1,75 @@ +function [b, feedbackID, apiResponse, apiURL] = submitFeedback(app, category, description, options) +%SUBMITFEEDBACK User-facing wrapper to submit an in-app problem report to NDI Cloud. +% +% [B, FEEDBACKID, APIRESPONSE, APIURL] = ndi.cloud.api.feedback.submitFeedback(APP, CATEGORY, DESCRIPTION, ...) +% +% Sends a problem report to POST /feedback. The reporter is the +% logged-in user. The server redacts anything that looks like a bearer +% token or a signed S3 URL credential before storing the report, but +% callers should still scrub logs before sending. +% +% Inputs: +% app - Name of the client sending the report (e.g., "lightsheetZarr-viewer"). +% category - One of "bug", "performance" or "other". +% description - The user's free-text description of the problem. +% +% Optional name-value inputs: +% appVersion - Version of the client (e.g., "NDI-matlab 2.0"). +% datasetId - Cloud dataset id the report concerns. +% documentId - Document id the report concerns. +% environment - Struct describing the client environment (os, versions, ...). +% state - Struct with client state useful for diagnosis. +% log - Recent log lines. +% traceback - Error stack, if any. +% +% Server limits: 1 MB per request; description 20,000 characters, log +% 500,000, traceback 50,000; environment 20 kB and state 200 kB as JSON; +% 10 reports per user per hour. +% +% Outputs: +% b - True if the call succeeded, false otherwise. +% feedbackID - The id of the stored report (use it with deleteFeedback). +% apiResponse - The full matlab.net.http.ResponseMessage object. +% apiURL - The URL that was called. +% +% See also: ndi.cloud.api.feedback.deleteFeedback, +% ndi.cloud.api.implementation.feedback.SubmitFeedback + + arguments + app (1,1) string {mustBeNonzeroLengthText} + category (1,1) string {mustBeMember(category, ["bug", "performance", "other"])} + description (1,1) string {mustBeNonzeroLengthText} + options.appVersion (1,1) string = missing + options.datasetId (1,1) string = missing + options.documentId (1,1) string = missing + options.environment struct {mustBeScalarOrEmpty} = struct.empty + options.state struct {mustBeScalarOrEmpty} = struct.empty + options.log (1,1) string = missing + options.traceback (1,1) string = missing + end + + feedbackStruct = struct( ... + 'schemaVersion', 1, ... + 'app', app, ... + 'category', category, ... + 'description', description); + + stringFields = ["appVersion", "datasetId", "documentId", "log", "traceback"]; + for fieldName = stringFields + if ~ismissing(options.(fieldName)) + feedbackStruct.(fieldName) = options.(fieldName); + end + end + for fieldName = ["environment", "state"] + if ~isempty(options.(fieldName)) + feedbackStruct.(fieldName) = options.(fieldName); + end + end + + % 1. Create an instance of the implementation class. + api_call = ndi.cloud.api.implementation.feedback.SubmitFeedback('feedbackStruct', feedbackStruct); + + % 2. Call the execute method and return its outputs directly. + [b, feedbackID, apiResponse, apiURL] = api_call.execute(); + +end diff --git a/src/ndi/+ndi/+cloud/+api/+files/getFileUploadURL.m b/src/ndi/+ndi/+cloud/+api/+files/getFileUploadURL.m index cb8824056f..4d781a0ae7 100644 --- a/src/ndi/+ndi/+cloud/+api/+files/getFileUploadURL.m +++ b/src/ndi/+ndi/+cloud/+api/+files/getFileUploadURL.m @@ -1,7 +1,8 @@ -function [b, answer, apiResponse, apiURL] = getFileUploadURL(cloudDatasetID, cloudFileUID) +function [b, answer, apiResponse, apiURL] = getFileUploadURL(cloudDatasetID, cloudFileUID, options) %GETFILEUPLOADURL Get a pre-signed URL for uploading a single file. % % [B, ANSWER, APIRESPONSE, APIURL] = ndi.cloud.api.files.getFileUploadURL(CLOUDDATASETID, CLOUDFILEUID) +% [...] = ndi.cloud.api.files.getFileUploadURL(..., 'checksumCRC32', CRC) % % Retrieves a URL that can be used to PUT the contents of a single file % for a dataset. @@ -10,6 +11,17 @@ % cloudDatasetID - The ID of the dataset. % cloudFileUID - The unique identifier that will be used for the file. % +% Name-Value Inputs: +% checksumCRC32 - (Optional) The file's CRC-32 as base64 of its +% 4-byte big-endian value (ndi.fun.file.crc32). +% When given, the URL signs the +% x-amz-checksum-crc32 header, so the upload must +% send it: pass the same value to putFiles as +% 'checksumCRC32'. S3 then rejects content that +% does not match, and the cloud records the value +% as the file's crc32. +% Default "" (no checksum). +% % Outputs: % b - True if the call succeeded, false otherwise. % answer - The pre-signed upload URL (string) on success, or an error struct on failure. @@ -19,17 +31,26 @@ % Example: % [success, upload_url] = ndi.cloud.api.files.getFileUploadURL('d-12345', 'f-newfile'); % -% See also: ndi.cloud.api.implementation.files.GetFileUploadURL +% % With a checksum, so a corrupted upload is refused: +% crc = ndi.fun.file.crc32('myfile.dat'); +% [success, upload_url] = ndi.cloud.api.files.getFileUploadURL('d-12345', 'f-newfile', ... +% 'checksumCRC32', crc); +% ndi.cloud.api.files.putFiles(upload_url, 'myfile.dat', 'checksumCRC32', crc); +% +% See also: ndi.cloud.api.implementation.files.GetFileUploadURL, +% ndi.fun.file.crc32 arguments cloudDatasetID (1,1) string cloudFileUID (1,1) string + options.checksumCRC32 (1,1) string = "" end % 1. Create an instance of the implementation class. api_call = ndi.cloud.api.implementation.files.GetFileUploadURL(... 'cloudDatasetID', cloudDatasetID, ... - 'cloudFileUID', cloudFileUID); + 'cloudFileUID', cloudFileUID, ... + 'checksumCRC32', options.checksumCRC32); % 2. Call the execute method and return its outputs directly. [b, answer, apiResponse, apiURL] = api_call.execute(); diff --git a/src/ndi/+ndi/+cloud/+api/+files/getSignedURLSet.m b/src/ndi/+ndi/+cloud/+api/+files/getSignedURLSet.m index 0a83d0f750..628cd76227 100644 --- a/src/ndi/+ndi/+cloud/+api/+files/getSignedURLSet.m +++ b/src/ndi/+ndi/+cloud/+api/+files/getSignedURLSet.m @@ -27,6 +27,9 @@ % (or the wrapped struct form MATLAB unpacks the % server's map into) and whose values are pre-signed % GET URLs. +% `fileInfo` is an Nx1 struct array (uid, uploaded, +% size, crc32) for this page's files; crc32 is '' when +% none was recorded. Empty if the server predates it. % On failure, the server error payload. % apiResponse - The full matlab.net.http.ResponseMessage object. % apiURL - The URL that was called. diff --git a/src/ndi/+ndi/+cloud/+api/+files/getSignedURLSetAll.m b/src/ndi/+ndi/+cloud/+api/+files/getSignedURLSetAll.m index 2caa3455f2..f9ad256bef 100644 --- a/src/ndi/+ndi/+cloud/+api/+files/getSignedURLSetAll.m +++ b/src/ndi/+ndi/+cloud/+api/+files/getSignedURLSetAll.m @@ -29,6 +29,9 @@ % pages, expiresAt, files. % `files` is a struct whose field names are file UIDs % and whose values are pre-signed GET URLs. +% `fileInfo` is an Nx1 struct array (uid, uploaded, +% size, crc32) across all pages; crc32 is '' when none +% was recorded. % apiResponse - The matlab.net.http.ResponseMessage from the last page. % apiURL - The URL of the last page. % diff --git a/src/ndi/+ndi/+cloud/+api/+files/listFiles.m b/src/ndi/+ndi/+cloud/+api/+files/listFiles.m index f452f81cd5..33755d2c0f 100644 --- a/src/ndi/+ndi/+cloud/+api/+files/listFiles.m +++ b/src/ndi/+ndi/+cloud/+api/+files/listFiles.m @@ -29,6 +29,9 @@ % uploaded - A 0/1 flag indicating if the file has been uploaded. % sourceDatasetId - The cloudDatasetId of the file's parent dataset. % size - The file size in bytes. +% crc32 - The base64 big-endian CRC-32 of the content, +% recorded at upload ('' when none was recorded). +% Compare with ndi.fun.file.crc32 of a local copy. % % Example: % % Get the first page of files diff --git a/src/ndi/+ndi/+cloud/+api/+files/listFilesAll.m b/src/ndi/+ndi/+cloud/+api/+files/listFilesAll.m index 30f2e1619b..97313b90da 100644 --- a/src/ndi/+ndi/+cloud/+api/+files/listFilesAll.m +++ b/src/ndi/+ndi/+cloud/+api/+files/listFilesAll.m @@ -30,6 +30,8 @@ % uploaded - 0/1 flag: whether the file is uploaded. % sourceDatasetId - The parent dataset's cloud id. % size - The file size in bytes. +% crc32 - Base64 big-endian CRC-32 recorded at +% upload ('' when none was recorded). % apiResponse - The matlab.net.http.ResponseMessage object from the LAST % page call. % apiURL - The URL that was called for the LAST page. diff --git a/src/ndi/+ndi/+cloud/+api/+files/lookupFiles.m b/src/ndi/+ndi/+cloud/+api/+files/lookupFiles.m new file mode 100644 index 0000000000..2e86a55b31 --- /dev/null +++ b/src/ndi/+ndi/+cloud/+api/+files/lookupFiles.m @@ -0,0 +1,80 @@ +function [b, answer, apiResponse, apiURL] = lookupFiles(cloudDatasetID, cloudFileUIDs) +%LOOKUPFILES Look up the upload state, size and CRC-32 of files by uid. +% +% [B, ANSWER, APIRESPONSE, APIURL] = ndi.cloud.api.files.lookupFiles(CLOUDDATASETID, CLOUDFILEUIDS) +% +% Asks NDI Cloud, for each uid, whether the dataset has the file, whether it +% finished uploading, its size, and the CRC-32 recorded at upload. Use it to +% decide which files of a document or import batch still need uploading +% (for example to resume an interrupted upload) without listing the whole +% dataset. No download URLs are signed. +% +% The server takes at most 5000 uids per call; longer lists are split into +% several calls and the results merged. Duplicate uids are looked up once. +% +% Inputs: +% cloudDatasetID - The ID of the dataset. +% cloudFileUIDs - String array (or cellstr) of file uids. +% +% Outputs: +% b - True if every call succeeded, false otherwise. +% answer - On success, a struct with fields: +% files - Nx1 struct array, one per uid the dataset +% has a record for, with fields +% uid - char +% uploaded - logical +% size - double, bytes +% crc32 - char, base64 big-endian CRC-32 +% of the content, '' when none +% was recorded +% missing - 1xM string array of uids with no record +% On failure, the error body from the failing call. +% apiResponse - The matlab.net.http.ResponseMessage of the last call +% ([] when CLOUDFILEUIDS is empty). +% apiURL - The URL of the last call ([] when CLOUDFILEUIDS is empty). +% +% A cloud copy matches a local file when it is uploaded and its crc32 equals +% ndi.fun.file.crc32 of the local file. When crc32 is '' (files uploaded +% without a checksum), compare size instead. +% +% Example: +% [ok, r] = ndi.cloud.api.files.lookupFiles(datasetId, uids); +% for i = 1:numel(r.files) +% f = r.files(i); +% localCrc = ndi.fun.file.crc32(fullfile(folder, f.uid)); +% needsUpload = ~f.uploaded || ~strcmp(f.crc32, localCrc); +% end +% % r.missing lists uids never uploaded at all. +% +% See also: ndi.cloud.api.implementation.files.LookupFiles, +% ndi.fun.file.crc32, ndi.cloud.api.files.getFileUploadURL + + arguments + cloudDatasetID (1,1) string + cloudFileUIDs string + end + + uids = unique(reshape(cloudFileUIDs, 1, []), 'stable'); + + b = true; + answer = struct('files', ndi.cloud.api.implementation.files.fileMetadataArray([]), ... + 'missing', strings(1, 0)); + apiResponse = []; + apiURL = []; + + maxUids = ndi.cloud.api.implementation.files.LookupFiles.MaxUids; + for first = 1:maxUids:numel(uids) + batch = uids(first:min(first + maxUids - 1, numel(uids))); + api_call = ndi.cloud.api.implementation.files.LookupFiles(... + 'cloudDatasetID', cloudDatasetID, ... + 'cloudFileUIDs', batch); + [ok, batchAnswer, apiResponse, apiURL] = api_call.execute(); + if ~ok + b = false; + answer = batchAnswer; + return + end + answer.files = [answer.files; batchAnswer.files]; + answer.missing = [answer.missing batchAnswer.missing]; + end +end diff --git a/src/ndi/+ndi/+cloud/+api/+files/putFiles.m b/src/ndi/+ndi/+cloud/+api/+files/putFiles.m index c0dc892dce..dcb6fc200b 100644 --- a/src/ndi/+ndi/+cloud/+api/+files/putFiles.m +++ b/src/ndi/+ndi/+cloud/+api/+files/putFiles.m @@ -37,6 +37,14 @@ % 'timeout' (double) - Overall wait-for-completion deadline, in % seconds. Default 60. % +% 'checksumCRC32' (string) - The file's CRC-32 (ndi.fun.file.crc32). +% Required when the URL came from +% getFileUploadURL(..., 'checksumCRC32', CRC): +% that URL signs the x-amz-checksum-crc32 +% header, so it must be sent, with the same +% value. S3 then rejects content that does not +% match. Default "" (no header). +% % Outputs: % b - True if the upload succeeded (and, when % waitForCompletion is true, the server-side bulk @@ -57,6 +65,13 @@ % [s2] = ndi.cloud.api.files.putFiles(url, 'localfile.dat'); % end % +% % Single-file upload with a checksum +% crc = ndi.fun.file.crc32('localfile.dat'); +% [s, url] = ndi.cloud.api.files.getFileUploadURL('d-123', 'f-abc', 'checksumCRC32', crc); +% if s +% [s2] = ndi.cloud.api.files.putFiles(url, 'localfile.dat', 'checksumCRC32', crc); +% end +% % % Bulk (zip) upload, waiting for server-side extraction to finish % [s, info] = ndi.cloud.api.files.getFileCollectionUploadURL('d-123'); % if s @@ -76,6 +91,7 @@ options.jobId (1,1) string = "" options.waitForCompletion (1,1) logical = false options.timeout (1,1) double {mustBePositive} = 60 + options.checksumCRC32 (1,1) string = "" end % 1. Create an instance of the implementation class, passing the options. api_call = ndi.cloud.api.implementation.files.PutFiles(... @@ -84,7 +100,8 @@ 'useCurl', options.useCurl, ... 'jobId', options.jobId, ... 'waitForCompletion', options.waitForCompletion, ... - 'timeout', options.timeout); + 'timeout', options.timeout, ... + 'checksumCRC32', options.checksumCRC32); % 2. Call the execute method and return its outputs directly. [b, answer, apiResponse, apiURL] = api_call.execute(); diff --git a/src/ndi/+ndi/+cloud/+api/+files/waitForFileUpload.m b/src/ndi/+ndi/+cloud/+api/+files/waitForFileUpload.m new file mode 100644 index 0000000000..dc578cc35d --- /dev/null +++ b/src/ndi/+ndi/+cloud/+api/+files/waitForFileUpload.m @@ -0,0 +1,69 @@ +function [b, answer, apiResponse, apiURL] = waitForFileUpload(cloudDatasetID, cloudFileUIDs, options) +%WAITFORFILEUPLOAD Wait until single-file uploads are recorded by the server. +% +% [B, ANSWER, APIRESPONSE, APIURL] = ndi.cloud.api.files.waitForFileUpload(CLOUDDATASETID, CLOUDFILEUIDS) +% [...] = ndi.cloud.api.files.waitForFileUpload(CLOUDDATASETID, CLOUDFILEUIDS, 'timeout', T, ...) +% +% A PUT to a signed upload URL returning 200 only means S3 has the object. +% The server then records the upload (uploaded=true, size, crc32) in an +% S3-triggered step that can lag the PUT by seconds or more when other +% uploads are queued on the same stage. This function polls +% ndi.cloud.api.files.lookupFiles at exponentially growing intervals until +% every uid is marked uploaded or the overall timeout elapses. +% +% This is the single-file counterpart to waitForBulkUpload; bulk (zip) +% uploads should use that function, or putFiles(..., 'waitForCompletion', +% true), instead. +% +% Inputs: +% cloudDatasetID - The cloud dataset id. +% cloudFileUIDs - String array (or cellstr) of file uids to wait for. +% +% Name-Value Pairs: +% 'timeout' (double) - Overall deadline in seconds. Default 120. +% 'initialInterval' (double) - First sleep between polls (s). Default 1. +% 'maxInterval' (double) - Cap on the per-poll sleep (s). Default 10. +% 'backoffFactor' (double) - Multiplier applied after each poll. Default 2. +% +% Outputs: +% b - True iff every uid was marked uploaded before the timeout. +% answer - Struct describing the final state: +% .state - 'complete' or 'timeout' +% .files - lookupFiles rows (uid, uploaded, size, +% crc32) for the uploaded uids, plus on +% timeout the last rows for pending ones +% .pending - string array of uids not yet uploaded +% (including uids with no record) +% .elapsed - seconds spent waiting +% apiResponse - The ResponseMessage from the last poll. +% apiURL - The URL of the last poll. +% +% Example: +% [s, url] = ndi.cloud.api.files.getFileUploadURL('d-123', 'f-abc'); +% ndi.cloud.api.files.putFiles(url, 'localfile.dat'); +% b = ndi.cloud.api.files.waitForFileUpload('d-123', 'f-abc'); +% +% See also: ndi.cloud.api.implementation.files.WaitForFileUpload, +% ndi.cloud.api.files.lookupFiles, +% ndi.cloud.api.files.putFiles, +% ndi.cloud.api.files.waitForBulkUpload + + arguments + cloudDatasetID (1,1) string + cloudFileUIDs (1,:) string + options.timeout (1,1) double {mustBePositive} = 120 + options.initialInterval (1,1) double {mustBePositive} = 1 + options.maxInterval (1,1) double {mustBePositive} = 10 + options.backoffFactor (1,1) double {mustBePositive} = 2 + end + + api_call = ndi.cloud.api.implementation.files.WaitForFileUpload(... + 'cloudDatasetID', cloudDatasetID, ... + 'cloudFileUIDs', cloudFileUIDs, ... + 'timeout', options.timeout, ... + 'initialInterval', options.initialInterval, ... + 'maxInterval', options.maxInterval, ... + 'backoffFactor', options.backoffFactor); + + [b, answer, apiResponse, apiURL] = api_call.execute(); +end diff --git a/src/ndi/+ndi/+cloud/+api/+implementation/+feedback/DeleteFeedback.m b/src/ndi/+ndi/+cloud/+api/+implementation/+feedback/DeleteFeedback.m new file mode 100644 index 0000000000..051810635e --- /dev/null +++ b/src/ndi/+ndi/+cloud/+api/+implementation/+feedback/DeleteFeedback.m @@ -0,0 +1,48 @@ +classdef DeleteFeedback < ndi.cloud.api.call + %DELETEFEEDBACK Implements the API call to withdraw one of your own reports. + % + % See also: ndi.cloud.api.feedback.deleteFeedback + + properties + feedbackID (1,1) string + end + + methods + function this = DeleteFeedback(args) + arguments + args.feedbackID (1,1) string + end + + this.feedbackID = args.feedbackID; + this.endpointName = "delete_feedback"; + end + + function [b, answer, apiResponse, apiURL] = execute(this) + % EXECUTE - Performs the API call to delete a feedback report. + % + % The server answers 204 on success. A report that does not + % exist, or that belongs to another user, is answered with 404. + + method = matlab.net.http.RequestMethod.DELETE; + + auth_token = ndi.cloud.authenticate(); + acceptField = matlab.net.http.HeaderField('accept','application/json'); + authorizationField = matlab.net.http.HeaderField('Authorization', ['Bearer ' auth_token]); + headers = [acceptField authorizationField]; + + request = matlab.net.http.RequestMessage(method, headers); + + apiURL = ndi.cloud.api.url(this.endpointName, 'feedback_id', this.feedbackID); + + apiResponse = request.send(apiURL); + + if (apiResponse.StatusCode == 204) + b = true; + answer = 'Feedback deleted successfully.'; + else + b = false; + answer = []; + end + end + end +end diff --git a/src/ndi/+ndi/+cloud/+api/+implementation/+feedback/SubmitFeedback.m b/src/ndi/+ndi/+cloud/+api/+implementation/+feedback/SubmitFeedback.m new file mode 100644 index 0000000000..fecc49f4ca --- /dev/null +++ b/src/ndi/+ndi/+cloud/+api/+implementation/+feedback/SubmitFeedback.m @@ -0,0 +1,54 @@ +classdef SubmitFeedback < ndi.cloud.api.call + %SUBMITFEEDBACK Implements the API call to submit an in-app problem report. + % + % Sends FEEDBACKSTRUCT as the JSON body of POST /feedback. The struct + % is sent as-is, so callers (and tests) can send any payload; use + % ndi.cloud.api.feedback.submitFeedback to build a valid one. + % + % See also: ndi.cloud.api.feedback.submitFeedback + + properties + feedbackStruct % The JSON body of the report + end + + methods + function this = SubmitFeedback(args) + arguments + args.feedbackStruct (1,1) struct + end + + this.feedbackStruct = args.feedbackStruct; + this.endpointName = "submit_feedback"; + end + + function [b, answer, apiResponse, apiURL] = execute(this) + % EXECUTE - Performs the API call to submit a feedback report. + % + % On success (201), ANSWER is the id of the stored report. + + method = matlab.net.http.RequestMethod.POST; + body = matlab.net.http.MessageBody(this.feedbackStruct); + + acceptField = matlab.net.http.HeaderField('accept','application/json'); + contentTypeField = matlab.net.http.field.ContentTypeField(matlab.net.http.MediaType('application/json')); + + auth_token = ndi.cloud.authenticate(); + authorizationField = matlab.net.http.HeaderField('Authorization', ['Bearer ' auth_token]); + headers = [acceptField contentTypeField authorizationField]; + + request = matlab.net.http.RequestMessage(method, headers, body); + + apiURL = ndi.cloud.api.url(this.endpointName); + + apiResponse = request.send(apiURL); + + if (apiResponse.StatusCode == 201) + b = true; + answer = apiResponse.Body.Data.id; + else + b = false; + answer = []; + end + end + end +end diff --git a/src/ndi/+ndi/+cloud/+api/+implementation/+files/GetFileUploadURL.m b/src/ndi/+ndi/+cloud/+api/+implementation/+files/GetFileUploadURL.m index eeff441d6d..72d4589857 100644 --- a/src/ndi/+ndi/+cloud/+api/+implementation/+files/GetFileUploadURL.m +++ b/src/ndi/+ndi/+cloud/+api/+implementation/+files/GetFileUploadURL.m @@ -3,6 +3,7 @@ properties cloudFileUID + checksumCRC32 (1,1) string = "" % base64 big-endian CRC-32, or "" for none end methods @@ -12,17 +13,40 @@ % Inputs: % 'cloudDatasetID' - The ID of the dataset. % 'cloudFileUID' - The unique identifier for the file. + % 'checksumCRC32' - (Optional) The file's CRC-32, base64 of + % its 4-byte big-endian value. Sent as the + % checksumCRC32 query parameter. % arguments args.cloudDatasetID (1,1) string args.cloudFileUID (1,1) string + args.checksumCRC32 (1,1) string = "" end this.cloudDatasetID = args.cloudDatasetID; this.cloudFileUID = args.cloudFileUID; + this.checksumCRC32 = args.checksumCRC32; this.endpointName = 'get_file_upload_url'; end + function apiURL = buildURL(this, organizationId) + %BUILDURL The upload URL request for ORGANIZATIONID, with the + % checksumCRC32 query parameter when a checksum was given. + apiURL = ndi.cloud.api.url('get_file_upload_url', ... + 'dataset_id', this.cloudDatasetID, ... + 'organization_id', organizationId, ... + 'file_uid', this.cloudFileUID); + if strlength(this.checksumCRC32) > 0 + % QueryParameter percent-encodes the base64 '+', '/' and '='. + q = matlab.net.QueryParameter('checksumCRC32', char(this.checksumCRC32)); + if isempty(apiURL.Query) + apiURL.Query = q; + else + apiURL.Query = [apiURL.Query q]; + end + end + end + function [b, answer, apiResponse, apiURL] = execute(this) %EXECUTE Performs the API call. @@ -41,10 +65,7 @@ return; end - apiURL = ndi.cloud.api.url('get_file_upload_url', ... - 'dataset_id', this.cloudDatasetID, ... - 'organization_id', dsetinfo.organizationId, ... - 'file_uid', this.cloudFileUID); + apiURL = this.buildURL(dsetinfo.organizationId); method = matlab.net.http.RequestMethod.GET; diff --git a/src/ndi/+ndi/+cloud/+api/+implementation/+files/GetSignedURLSetAll.m b/src/ndi/+ndi/+cloud/+api/+implementation/+files/GetSignedURLSetAll.m index 6d0245337f..b16576ff71 100644 --- a/src/ndi/+ndi/+cloud/+api/+implementation/+files/GetSignedURLSetAll.m +++ b/src/ndi/+ndi/+cloud/+api/+implementation/+files/GetSignedURLSetAll.m @@ -52,6 +52,7 @@ % A containers.Map, not a struct: the keys are DID file uids and % a struct would carry the JSONDECODE-renamed versions of them. merged.files = containers.Map('KeyType','char','ValueType','any'); + merged.fileInfo = ndi.cloud.api.implementation.files.fileMetadataArray([]); merged.pageCount = 0; merged.totalCount = 0; merged.pages = 0; @@ -70,6 +71,9 @@ end merged.pageCount = merged.pageCount + numel(pageKeys); end + if isstruct(page) && isfield(page, 'fileInfo') && isstruct(page.fileInfo) + merged.fileInfo = [merged.fileInfo; page.fileInfo(:)]; + end if isstruct(page) && isfield(page, 'totalCount') merged.totalCount = page.totalCount; end diff --git a/src/ndi/+ndi/+cloud/+api/+implementation/+files/ListFiles.m b/src/ndi/+ndi/+cloud/+api/+implementation/+files/ListFiles.m index 0bd3a084d2..f9c8197bb9 100644 --- a/src/ndi/+ndi/+cloud/+api/+implementation/+files/ListFiles.m +++ b/src/ndi/+ndi/+cloud/+api/+implementation/+files/ListFiles.m @@ -82,7 +82,7 @@ raw_answer = apiResponse.Body.Data; % Standardize the output format. - answer = struct('uid', {}, 'uploaded', {}, 'sourceDatasetId', {}, 'size', {}); + answer = struct('uid', {}, 'uploaded', {}, 'sourceDatasetId', {}, 'size', {}, 'crc32', {}); if ~isempty(raw_answer) && isfield(raw_answer, 'files') && ~isempty(raw_answer.files) files_from_api = raw_answer.files; @@ -107,7 +107,8 @@ 'uid', file_in.uid, ... 'uploaded', [], ... 'sourceDatasetId', '', ... - 'size', []); + 'size', [], ... + 'crc32', ''); if isfield(file_in, 'uploaded') file_out.uploaded = file_in.uploaded; @@ -118,6 +119,11 @@ if isfield(file_in, 'size') file_out.size = file_in.size; end + % Recorded at upload (base64 big-endian CRC-32); + % absent for files uploaded without a checksum. + if isfield(file_in, 'crc32') && ~isempty(file_in.crc32) + file_out.crc32 = char(file_in.crc32); + end answer(end+1) = file_out; end diff --git a/src/ndi/+ndi/+cloud/+api/+implementation/+files/ListFilesAll.m b/src/ndi/+ndi/+cloud/+api/+implementation/+files/ListFilesAll.m index b34b73e4f2..e077bcd668 100644 --- a/src/ndi/+ndi/+cloud/+api/+implementation/+files/ListFilesAll.m +++ b/src/ndi/+ndi/+cloud/+api/+implementation/+files/ListFilesAll.m @@ -58,13 +58,13 @@ % Outputs: % b - True if all pages were read successfully, false otherwise. % answer - A struct array of file summaries (uid, uploaded, - % sourceDatasetId, size) on success. + % sourceDatasetId, size, crc32) on success. % apiResponse - The matlab.net.http.ResponseMessage object from the LAST page call. % apiURL - The URL that was called for the LAST page. % Initialize outputs b = true; - answer = struct('uid', {}, 'uploaded', {}, 'sourceDatasetId', {}, 'size', {}); + answer = struct('uid', {}, 'uploaded', {}, 'sourceDatasetId', {}, 'size', {}, 'crc32', {}); apiResponse = matlab.net.http.ResponseMessage.empty; apiURL = matlab.net.URI.empty; diff --git a/src/ndi/+ndi/+cloud/+api/+implementation/+files/LookupFiles.m b/src/ndi/+ndi/+cloud/+api/+implementation/+files/LookupFiles.m new file mode 100644 index 0000000000..c54f019ed7 --- /dev/null +++ b/src/ndi/+ndi/+cloud/+api/+implementation/+files/LookupFiles.m @@ -0,0 +1,102 @@ +classdef LookupFiles < ndi.cloud.api.call +%LOOKUPFILES Implementation class for looking up file records by uid. +% +% One call to POST /datasets/{datasetId}/files/lookup, for at most +% MaxUids uids. ndi.cloud.api.files.lookupFiles splits longer lists into +% several calls. +% +% See also: ndi.cloud.api.files.lookupFiles + + properties (Constant) + MaxUids = 5000 % server limit per call + end + + properties + cloudFileUIDs (1,:) string + end + + methods + function this = LookupFiles(args) + %LOOKUPFILES Creates a new LookupFiles API call object. + % + % THIS = ndi.cloud.api.implementation.files.LookupFiles( ... + % 'cloudDatasetID', ID, 'cloudFileUIDs', UIDS) + % + % Inputs: + % 'cloudDatasetID' - The ID of the dataset. + % 'cloudFileUIDs' - String array of file uids, 1 to MaxUids + % entries. + arguments + args.cloudDatasetID (1,1) string + args.cloudFileUIDs (1,:) string + end + if isempty(args.cloudFileUIDs) + error('NDI:CloudApi:LookupFiles:NoUids', ... + 'At least one file uid is required.'); + end + if numel(args.cloudFileUIDs) > ndi.cloud.api.implementation.files.LookupFiles.MaxUids + error('NDI:CloudApi:LookupFiles:TooManyUids', ... + 'At most %d uids per call (got %d); use ndi.cloud.api.files.lookupFiles, which splits the list.', ... + ndi.cloud.api.implementation.files.LookupFiles.MaxUids, numel(args.cloudFileUIDs)); + end + this.cloudDatasetID = args.cloudDatasetID; + this.cloudFileUIDs = args.cloudFileUIDs; + this.endpointName = 'lookup_dataset_files'; + end + + function body = requestBody(this) + %REQUESTBODY The JSON body sent to the server, as a struct. + % + % When MATLAB encodes a scalar string to JSON it produces a bare + % string, not an array, so a single uid is sent twice (the server + % collapses duplicates), as BulkFetch does. + uids = this.cloudFileUIDs; + if isscalar(uids) + uids = [uids uids]; + end + body = struct('uids', uids); + end + + function [b, answer, apiResponse, apiURL] = execute(this) + %EXECUTE Performs the API call. + % + % Outputs: + % b - True if the call succeeded, false otherwise. + % answer - On success, a struct with fields `files` + % (Nx1 struct array: uid, uploaded, size, + % crc32) and `missing` (string array). On + % failure, the error body from the server. + % apiResponse - The matlab.net.http.ResponseMessage object. + % apiURL - The URL that was called. + + b = false; + + token = ndi.cloud.authenticate(); + + apiURL = ndi.cloud.api.url(this.endpointName, 'dataset_id', this.cloudDatasetID); + + method = matlab.net.http.RequestMethod.POST; + body = matlab.net.http.MessageBody(this.requestBody()); + + acceptField = matlab.net.http.HeaderField('accept','application/json'); + contentTypeField = matlab.net.http.field.ContentTypeField(matlab.net.http.MediaType('application/json')); + authorizationField = matlab.net.http.HeaderField('Authorization', ['Bearer ' token]); + headers = [acceptField contentTypeField authorizationField]; + + request = matlab.net.http.RequestMessage(method, headers, body); + + apiResponse = send(request, apiURL); + + if (apiResponse.StatusCode == 200) + b = true; + answer = ndi.cloud.api.implementation.files.lookupFilesResult(apiResponse.Body.Data); + else + if isprop(apiResponse.Body, 'Data') + answer = apiResponse.Body.Data; + else + answer = apiResponse.Body; + end + end + end + end +end diff --git a/src/ndi/+ndi/+cloud/+api/+implementation/+files/PutFiles.m b/src/ndi/+ndi/+cloud/+api/+implementation/+files/PutFiles.m index 02d791bb78..b3c62884ac 100644 --- a/src/ndi/+ndi/+cloud/+api/+implementation/+files/PutFiles.m +++ b/src/ndi/+ndi/+cloud/+api/+implementation/+files/PutFiles.m @@ -16,6 +16,7 @@ jobId (1,1) string waitForCompletion (1,1) logical timeout (1,1) double + checksumCRC32 (1,1) string % base64 big-endian CRC-32, or "" for none end methods @@ -28,7 +29,17 @@ args.jobId (1,1) string = "" args.waitForCompletion (1,1) logical = false args.timeout (1,1) double {mustBePositive} = 60 + args.checksumCRC32 (1,1) string = "" end + % A CRC-32 is 4 bytes: six base64 characters and '=='. Checking + % the shape also keeps the value safe to put in a curl command. + if strlength(args.checksumCRC32) > 0 && ... + isempty(regexp(char(args.checksumCRC32), '^[A-Za-z0-9+/]{6}==$', 'once')) + error('NDI:CloudApi:PutFiles:InvalidChecksum', ... + 'checksumCRC32 must be the base64 of a 4-byte CRC-32 (e.g. ''NhCmhg==''), got ''%s''.', ... + args.checksumCRC32); + end + this.checksumCRC32 = args.checksumCRC32; this.preSignedURL = args.preSignedURL; this.filePath = args.filePath; this.useCurl = args.useCurl; @@ -77,8 +88,12 @@ apiURL = matlab.net.URI(this.preSignedURL); method = matlab.net.http.RequestMethod.PUT; provider = matlab.net.http.io.FileProvider(this.filePath); - contentTypeField = matlab.net.http.HeaderField('Content-Type', 'application/octet-stream'); - request = matlab.net.http.RequestMessage(method, contentTypeField, provider); + headers = matlab.net.http.HeaderField('Content-Type', 'application/octet-stream'); + if strlength(this.checksumCRC32) > 0 + % The upload URL signs this header (NDI Cloud #162). + headers(end+1) = matlab.net.http.HeaderField('x-amz-checksum-crc32', char(this.checksumCRC32)); + end + request = matlab.net.http.RequestMessage(method, headers, provider); try apiResponse = send(request, apiURL); @@ -92,6 +107,21 @@ end end + function command = curlArguments(this) + %CURLARGUMENTS The curl command (without the environment prefix). + % When checksumCRC32 is set, the x-amz-checksum-crc32 header is + % sent: the upload URL signs it (NDI Cloud #162), so S3 refuses + % the PUT without it and rejects content that does not match. + checksumHeader = ''; + if strlength(this.checksumCRC32) > 0 + checksumHeader = sprintf('-H "x-amz-checksum-crc32: %s" ', char(this.checksumCRC32)); + end + command = sprintf(['curl -fsSL -X PUT --upload-file "%s" ' ... + '-H "Content-Type: application/octet-stream" ' ... + '-H "Accept-Encoding: identity" ' ... + '%s"%s"'], this.filePath, checksumHeader, this.preSignedURL); + end + function [b, answer, apiResponse, apiURL] = executeWithCurl(this) % Implementation using a system call to curl b = false; @@ -115,10 +145,7 @@ % sprintf so a '%' in the library path cannot corrupt the format % string. envPrefix = ndi.common.systemCurlEnvPrefix(); - command = [envPrefix sprintf(['curl -fsSL -X PUT --upload-file "%s" ' ... - '-H "Content-Type: application/octet-stream" ' ... - '-H "Accept-Encoding: identity" ' ... - '"%s"'], this.filePath, this.preSignedURL)]; + command = [envPrefix this.curlArguments()]; [status, result] = system(command); diff --git a/src/ndi/+ndi/+cloud/+api/+implementation/+files/WaitForFileUpload.m b/src/ndi/+ndi/+cloud/+api/+implementation/+files/WaitForFileUpload.m new file mode 100644 index 0000000000..85a8be23bf --- /dev/null +++ b/src/ndi/+ndi/+cloud/+api/+implementation/+files/WaitForFileUpload.m @@ -0,0 +1,109 @@ +classdef WaitForFileUpload < ndi.cloud.api.call +%WAITFORFILEUPLOAD Wait until single-file uploads are recorded by the server. +% +% Implementation class behind ndi.cloud.api.files.waitForFileUpload. +% Polls lookupFiles(datasetId, uids) with exponential backoff until every +% uid is marked uploaded or the overall timeout elapses. +% +% A signed PUT returning 200 only means S3 has the object; the server +% records the upload in an S3-triggered step afterwards. Use this between +% a single-file putFiles and anything that reads the file's record +% (listFiles, getFileDetails, a download URL). + + properties + cloudFileUIDs (1,:) string + timeout (1,1) double = 120 + initialInterval (1,1) double = 1 + maxInterval (1,1) double = 10 + backoffFactor (1,1) double = 2 + end + + methods + function this = WaitForFileUpload(args) + %WAITFORFILEUPLOAD Construct a new WaitForFileUpload call. + arguments + args.cloudDatasetID (1,1) string + args.cloudFileUIDs (1,:) string + args.timeout (1,1) double {mustBePositive} = 120 + args.initialInterval (1,1) double {mustBePositive} = 1 + args.maxInterval (1,1) double {mustBePositive} = 10 + args.backoffFactor (1,1) double {mustBePositive} = 2 + end + this.cloudDatasetID = args.cloudDatasetID; + this.cloudFileUIDs = unique(args.cloudFileUIDs, 'stable'); + this.timeout = args.timeout; + this.initialInterval = args.initialInterval; + this.maxInterval = args.maxInterval; + this.backoffFactor = args.backoffFactor; + this.endpointName = 'lookup_dataset_files'; + end + + function [b, answer, apiResponse, apiURL] = execute(this) + %EXECUTE Poll until every uid is uploaded or the timeout elapses. + % + % b - true iff every uid was marked uploaded before + % the timeout. false on timeout; a failing + % lookupFiles call is retried until then. + % answer - struct with state ('complete' or 'timeout'), + % files (the uploaded rows, plus on timeout the + % last poll's rows for uids still pending), + % pending (uids not yet uploaded) and elapsed. + % apiResponse - The ResponseMessage from the last poll. + % apiURL - The URL of the last poll. + + b = false; + apiResponse = []; + apiURL = []; + doneFiles = []; % lookupFiles rows already marked uploaded + pendingFiles = []; % latest rows for uids not yet uploaded + pending = this.cloudFileUIDs; + + deadline = tic; + interval = this.initialInterval; + + while ~isempty(pending) + [ok, result, apiResponse, apiURL] = this.pollStatus(pending); + + if ok && isstruct(result) && isfield(result, 'files') + pendingFiles = []; + if ~isempty(result.files) + result.files = result.files(:); + isUp = [result.files.uploaded]; + doneFiles = [doneFiles; result.files(isUp)]; %#ok + pendingFiles = result.files(~isUp); + pending = setdiff(pending, string({result.files(isUp).uid}), 'stable'); + end + if isempty(pending) + break; + end + end + + elapsed = toc(deadline); + if elapsed >= this.timeout + answer = struct('state', 'timeout', 'files', [doneFiles; pendingFiles], ... + 'pending', pending, 'elapsed', elapsed); + return; + end + + remaining = this.timeout - elapsed; + pause(min([interval, this.maxInterval, remaining])); + interval = min(interval * this.backoffFactor, this.maxInterval); + end + + b = true; + answer = struct('state', 'complete', 'files', doneFiles, ... + 'pending', strings(1, 0), 'elapsed', toc(deadline)); + end + end + + methods (Access = protected) + function [ok, result, apiResponse, apiURL] = pollStatus(this, uids) + %POLLSTATUS Look up the given uids once. Seam: a test subclass + % overrides this to script a sequence of lookupFiles results, + % so the completion, timeout and retry logic can be exercised + % without a server. + [ok, result, apiResponse, apiURL] = ... + ndi.cloud.api.files.lookupFiles(this.cloudDatasetID, uids); + end + end +end diff --git a/src/ndi/+ndi/+cloud/+api/+implementation/+files/fileMetadataArray.m b/src/ndi/+ndi/+cloud/+api/+implementation/+files/fileMetadataArray.m new file mode 100644 index 0000000000..20d71ffbd1 --- /dev/null +++ b/src/ndi/+ndi/+cloud/+api/+implementation/+files/fileMetadataArray.m @@ -0,0 +1,62 @@ +function files = fileMetadataArray(rawFiles) +%FILEMETADATAARRAY Normalize decoded {uid, uploaded, size, crc32} records. +% +% FILES = ndi.cloud.api.implementation.files.FILEMETADATAARRAY(RAWFILES) +% +% NDI Cloud returns per-file metadata as a JSON array of objects with fields +% uid, uploaded, size and crc32 (POST /files/lookup `files`, and the +% signed-url-set `fileInfo`). JSONDECODE turns that into a struct array, a +% cell array (when the objects differ in fields), or [] (when empty), and +% turns a null crc32 into []. This returns one consistent shape. +% +% Inputs: +% rawFiles - The decoded array: struct array, cell array of structs, +% or empty. +% +% Outputs: +% files - Nx1 struct array with fields: +% uid - char +% uploaded - logical +% size - double, bytes +% crc32 - char, base64 of the big-endian CRC-32 of the +% content; '' when the cloud has none (compare +% size instead) +% Entries without a uid are skipped. +% +% See also: ndi.cloud.api.files.lookupFiles, ndi.fun.file.crc32 + + arguments + rawFiles + end + + files = struct('uid', {}, 'uploaded', {}, 'size', {}, 'crc32', {}); + if isempty(rawFiles) + files = reshape(files, 0, 1); + return + end + + n = numel(rawFiles); + for i = 1:n + if iscell(rawFiles) + f = rawFiles{i}; + else + f = rawFiles(i); + end + if ~isstruct(f) || ~isfield(f, 'uid') || isempty(f.uid) + continue + end + + entry = struct('uid', char(f.uid), 'uploaded', false, 'size', 0, 'crc32', ''); + if isfield(f, 'uploaded') && ~isempty(f.uploaded) + entry.uploaded = logical(f.uploaded); + end + if isfield(f, 'size') && ~isempty(f.size) + entry.size = double(f.size); + end + if isfield(f, 'crc32') && ~isempty(f.crc32) + entry.crc32 = char(f.crc32); + end + files(end+1, 1) = entry; %#ok + end + files = reshape(files, [], 1); +end diff --git a/src/ndi/+ndi/+cloud/+api/+implementation/+files/lookupFilesResult.m b/src/ndi/+ndi/+cloud/+api/+implementation/+files/lookupFilesResult.m new file mode 100644 index 0000000000..189110ee02 --- /dev/null +++ b/src/ndi/+ndi/+cloud/+api/+implementation/+files/lookupFilesResult.m @@ -0,0 +1,43 @@ +function result = lookupFilesResult(decodedData) +%LOOKUPFILESRESULT Turn a POST /files/lookup response body into a result struct. +% +% RESULT = ndi.cloud.api.implementation.files.LOOKUPFILESRESULT(DECODEDDATA) +% +% Takes the decoded body rather than the ResponseMessage, so a test can +% exercise it without a server. +% +% Inputs: +% decodedData - The decoded response body (apiResponse.Body.Data): a +% struct with fields `files` and `missing`. +% +% Outputs: +% result - struct with fields: +% files - Nx1 struct array (uid, uploaded, size, crc32); +% see ndi.cloud.api.implementation.files.fileMetadataArray +% missing - 1xM string array of requested uids that have no +% file record in the dataset +% +% See also: ndi.cloud.api.files.lookupFiles + + arguments + decodedData + end + + rawFiles = []; + rawMissing = []; + if isstruct(decodedData) + if isfield(decodedData, 'files') + rawFiles = decodedData.files; + end + if isfield(decodedData, 'missing') + rawMissing = decodedData.missing; + end + end + + result.files = ndi.cloud.api.implementation.files.fileMetadataArray(rawFiles); + if isempty(rawMissing) + result.missing = strings(1, 0); + else + result.missing = reshape(string(rawMissing), 1, []); + end +end diff --git a/src/ndi/+ndi/+cloud/+api/+implementation/+files/signedURLSetPage.m b/src/ndi/+ndi/+cloud/+api/+implementation/+files/signedURLSetPage.m index e6935ccf31..f054a94c57 100644 --- a/src/ndi/+ndi/+cloud/+api/+implementation/+files/signedURLSetPage.m +++ b/src/ndi/+ndi/+cloud/+api/+implementation/+files/signedURLSetPage.m @@ -24,6 +24,10 @@ % the last page % expiresAt - char timestamp when the signed URLs expire, '' % if the server did not supply one +% fileInfo - Nx1 struct array (uid, uploaded, size, crc32) +% for this page's files; empty when the server +% did not send it. See +% ndi.cloud.api.implementation.files.fileMetadataArray % % See also: ndi.cloud.api.implementation.files.signedURLFileMap @@ -52,6 +56,14 @@ % Normalize the two fields whose absence is meaningful: jsondecode turns % a JSON null into [], and callers test these with isempty. + % fileInfo is an array of objects carrying the uid as a value, so + % JSONDECODE's key renaming does not touch it. + rawFileInfo = []; + if isstruct(decodedData) && isfield(decodedData, 'fileInfo') + rawFileInfo = decodedData.fileInfo; + end + page.fileInfo = ndi.cloud.api.implementation.files.fileMetadataArray(rawFileInfo); + page.nextCursor = localCharField(decodedData, 'nextCursor'); page.expiresAt = localCharField(decodedData, 'expiresAt'); end diff --git a/src/ndi/+ndi/+cloud/+api/url.m b/src/ndi/+ndi/+cloud/+api/url.m index 75120429cf..0a619afe01 100644 --- a/src/ndi/+ndi/+cloud/+api/url.m +++ b/src/ndi/+ndi/+cloud/+api/url.m @@ -14,6 +14,7 @@ options.session_id (1,1) string = "" options.stage_id (1,1) string = "" options.job_id (1,1) string = "" + options.feedback_id (1,1) string = "" options.state (1,1) string = "" options.page (1,1) double = 1 options.page_size (1,1) double = 20 @@ -69,6 +70,7 @@ endpointMap("get_file_upload_url") = "/datasets/{organizationId}/{datasetId}/files/{file_uid}"; endpointMap("get_file_collection_upload_url") = "/datasets/{organizationId}/{datasetId}/files/bulk"; endpointMap("get_file_details") = "/datasets/{datasetId}/files/{file_uid}/detail"; + endpointMap("lookup_dataset_files") = "/datasets/{datasetId}/files/lookup"; endpointMap("get_bulk_upload_status") = "/bulk-uploads/{jobId}"; endpointMap("list_dataset_bulk_uploads") = "/datasets/{datasetId}/bulk-uploads?state={state}"; % Signed download URLs for the files a document references. The @@ -129,6 +131,11 @@ % last stage finalizes the session. There is no /finalize route. endpointMap("advance_compute_session") = "/compute/{sessionId}/advance"; endpointMap("list_compute_sessions") = "/compute"; + + % In-app problem reports (ndi-cloud-node#158). Reporters can withdraw + % their own reports; triage happens in the admin site. + endpointMap("submit_feedback") = "/feedback"; + endpointMap("delete_feedback") = "/feedback/{feedbackId}"; end endpointPath = endpointMap(endpointName); @@ -169,6 +176,7 @@ options = renameStructField(options, 'session_id', 'sessionId'); options = renameStructField(options, 'stage_id', 'stageId'); options = renameStructField(options, 'job_id', 'jobId'); + options = renameStructField(options, 'feedback_id', 'feedbackId'); function s = renameStructField(s, oldname, newname) s.(newname) = s.(oldname); diff --git a/src/ndi/+ndi/+cloud/+download/+internal/batchSignedUrlLookup.m b/src/ndi/+ndi/+cloud/+download/+internal/batchSignedUrlLookup.m index d99c6a7c77..8bb97c67ac 100644 --- a/src/ndi/+ndi/+cloud/+download/+internal/batchSignedUrlLookup.m +++ b/src/ndi/+ndi/+cloud/+download/+internal/batchSignedUrlLookup.m @@ -66,21 +66,43 @@ % Default 20*3600 (the server currently signs URLs % for 24 h, leaving a buffer). A scope past its % TTL is refetched on the next miss. -% partialMapRetrySeconds - How long to wait, in seconds, before -% re-fetching a scope whose first answer was a -% populated map that did not name the requested uid. -% Default 0.5. The retry gives a lagged batch -% endpoint time to catch up after a bulk upload; a -% zero value skips the wait but still performs the -% single retry (used by tests). See -% VH-Lab/NDI-matlab#991 and Waltham-Data-Science/ -% NDI-python#309. +% partialMapRetryDelays - Row vector of waits, in seconds, between +% successive re-fetches of a scope whose previous +% answer was a populated map that did not name the +% requested uid. Default [1 3 9] -- one initial call +% plus up to three retries at 1 s / 3 s / 9 s, ~13 s +% of wall clock in the worst case. The retries give +% a lagged batch endpoint time to catch up after +% ``waitForAllBulkUploads`` returns; if the whole +% schedule is spent and the map still misses, the +% caller falls back per uid. See +% VH-Lab/NDI-matlab#991 and +% Waltham-Data-Science/NDI-python#309 and #320. +% partialMapRetrySeconds - Deprecated single-wait scalar. NaN +% (default) means "use partialMapRetryDelays". Any +% other value replaces the schedule with a +% one-element one containing that wait, matching the +% pre-schedule contract where 0 still fires one +% retry (with no wait). Kept so tests pinned to that +% contract keep passing. % sleepFcn - Function handle called as sleepFcn(seconds) to -% perform the pause before a partial-map retry. +% perform the pause before each partial-map retry. % Defaults to @pause. Present so tests can % short-circuit the wait without adding real % latency to a suite that exercises the retry many % times. +% retryBackoffSeconds - Row vector of delays, in seconds, +% between successive scope-fetch attempts after a +% raise. Default [1 4 16] -- one initial call, up +% to three retries at 1 s / 4 s / 16 s, ~21 s of +% wall clock in the worst case. An exception here +% is a transient network condition +% (MATLAB:webservices:ConnectionFailed and the +% like); a signer that reports ok=false is a +% server-side no and is NOT retried. Pass [] to +% disable retries, or a vector of zeros to keep +% the retry count without adding real latency +% (used by tests). See VH-Lab/NDI-matlab#1010. % % Outputs: % url - The pre-signed URL for uid (char), or "" (string @@ -115,6 +137,16 @@ % this uid" gets that answer after the % retry. See VH-Lab/NDI-matlab#991 and % Waltham-Data-Science/NDI-python#309. +% .transientRetries how many extra signer calls were +% spent recovering from a raise (a +% transient network error) before the +% scope was cached or given up on. A +% fresh scope fetch that raises is +% retried with backoff so one blip +% does not cascade to per-member +% fallback across a 156k-member series. +% See VH-Lab/NDI-matlab#1010 and +% Waltham-Data-Science/NDI-python#322. % % These exist to make the fallback VISIBLE. It is % deliberate at runtime -- a reader opening one file @@ -148,8 +180,33 @@ options.clearCache (1,1) logical = false options.ttlSeconds (1,1) double = 20*3600 options.failureTtlSeconds (1,1) double = 60 - options.partialMapRetrySeconds (1,1) double = 0.5 + % PARTIAL-MAP retry schedule (bounded exponential backoff). Each + % entry is a wait, in seconds, before the corresponding retry. + % A scope whose Nth wave still returns a partial map falls back + % per uid without another wave. Three waves at 1 s / 3 s / 9 s + % cover the User-1-prod tail observed on + % Waltham-Data-Science/NDI-python#320: one 0.5 s retry was not + % enough to bridge the settle between bulk-upload extraction and + % the signed-URL-set index catching up. See also + % VH-Lab/NDI-matlab#991. + options.partialMapRetryDelays (1,:) double = [1 3 9] + % Back-compat scalar knob. If passed, replaces the schedule with + % a single-element one containing that wait. NaN means "not + % set", so partialMapRetryDelays wins. Preserves the pre-schedule + % semantics where partialMapRetrySeconds=0 still fires one retry + % (with no wait). + options.partialMapRetrySeconds (1,1) double = NaN options.sleepFcn (1,1) function_handle = @pause + options.retryBackoffSeconds (1,:) double = [1 4 16] + % Disk cache off by default: only enable when we have a URL-set + % source that will actually reduce cost on reopen (the async + % signed-URL-set job, or a scope big enough to matter). Callers + % that DO want it -- the DID chunk-download path for large + % lightsheet series -- turn it on explicitly. Existing tests + % that inject scripted signers see no disk activity, which is + % what they want: the disk cache is a production optimisation, + % not part of the batch contract. + options.diskCache (1,1) logical = false end % The persistent cache. Keyed on 'datasetId/documentId/seriesName'. @@ -166,7 +223,7 @@ if isempty(STATS) || options.clearCache STATS = struct('signerCalls', 0, 'uidHits', 0, 'uidMisses', 0, ... 'lastMapSize', 0, 'lastMapUids', {{}}, 'lastFailureReason', "", ... - 'partialMapRetries', 0); + 'partialMapRetries', 0, 'transientRetries', 0); end if isempty(WARNED) || options.clearCache WARNED = containers.Map('KeyType','char','ValueType','logical'); @@ -175,7 +232,20 @@ FAILEDSCOPES = containers.Map('KeyType','char','ValueType','any'); end if isempty(RETRIEDSCOPES) || options.clearCache - RETRIEDSCOPES = containers.Map('KeyType','char','ValueType','logical'); + % Values are the number of retry waves already spent on this + % scope, bounded by numel(effectiveDelays). Was a logical + % "has been retried" set before the multi-wave rework. + RETRIEDSCOPES = containers.Map('KeyType','char','ValueType','double'); + end + + % Resolve the effective partial-map schedule. A caller passing the + % deprecated scalar partialMapRetrySeconds gets a one-element + % schedule with that wait (scalar=0 still fires ONE retry, matching + % the pre-schedule behaviour). + if isnan(options.partialMapRetrySeconds) + effectiveDelays = options.partialMapRetryDelays; + else + effectiveDelays = options.partialMapRetrySeconds; end url = ""; @@ -205,6 +275,10 @@ end end + if isempty(entry) && options.diskCache + entry = localTryDiskCache(); + end + if isempty(entry) entry = localFetchScope(cacheKey); if isempty(entry) @@ -241,14 +315,25 @@ % misses becomes a whole series' worth of hits from one retry. % See VH-Lab/NDI-matlab#991 and Waltham-Data-Science/ % NDI-python#309. - if ~isKey(RETRIEDSCOPES, cacheKey) - RETRIEDSCOPES(cacheKey) = true; - STATS.partialMapRetries = STATS.partialMapRetries + 1; - if options.partialMapRetrySeconds > 0 - options.sleepFcn(options.partialMapRetrySeconds); + % Multi-wave retry: each wave sleeps for effectiveDelays(n+1) + % where n is the number of waves already spent, then refetches. + % Stops as soon as a fresh map names the uid, or when the + % schedule is exhausted. + if isKey(RETRIEDSCOPES, cacheKey) + attemptsSpent = RETRIEDSCOPES(cacheKey); + else + attemptsSpent = 0; + end + while attemptsSpent < numel(effectiveDelays) + delay = effectiveDelays(attemptsSpent + 1); + if delay > 0 + options.sleepFcn(delay); end % Drop the stale entry so localFetchScope replaces it fresh. remove(CACHE, cacheKey); + STATS.partialMapRetries = STATS.partialMapRetries + 1; + attemptsSpent = attemptsSpent + 1; + RETRIEDSCOPES(cacheKey) = attemptsSpent; refreshed = localFetchScope(cacheKey); if isempty(refreshed) % localFetchScope already recorded the miss and warned; @@ -324,34 +409,71 @@ wantsFour = false; end + % Fetch the scope with backoff on a raise. A transient network + % blip (MATLAB:webservices:ConnectionFailed and the like) at the + % start of a run would otherwise cascade: this scope is marked + % failed, every subsequent uid in it falls back to a per-member + % getFileDetails call, and a 156k-member series takes hours. + % Retry the batch call itself first; the per-member fallback is + % still there below as a safety net, just not triggered by + % transients. See VH-Lab/NDI-matlab#1010 and Waltham-Data-Science/ + % NDI-python#322. + % + % Only a caught exception is retried. A signer that reports + % ok=false is a server-side no (auth, 404, business-logic + % refusal): retrying hammers a dead endpoint and adds cost with + % no chance of recovery. + ok = false; + answer = []; apiResponse = []; failureReason = ""; - try - if wantsFour - if strlength(seriesName) > 0 - [ok, answer, apiResponse] = options.signer(cloudDatasetId, ... - ndiDocumentId, 'idNamespace', "ndi", ... - 'fileSeries', seriesName); - else - [ok, answer, apiResponse] = options.signer(cloudDatasetId, ... - ndiDocumentId, 'idNamespace', "ndi"); - end - else - if strlength(seriesName) > 0 - [ok, answer] = options.signer(cloudDatasetId, ndiDocumentId, ... - 'idNamespace', "ndi", 'fileSeries', seriesName); + maxAttempts = 1 + numel(options.retryBackoffSeconds); + for attempt = 1:maxAttempts + apiResponse = []; + failureReason = ""; + try + if wantsFour + if strlength(seriesName) > 0 + [ok, answer, apiResponse] = options.signer(cloudDatasetId, ... + ndiDocumentId, 'idNamespace', "ndi", ... + 'fileSeries', seriesName); + else + [ok, answer, apiResponse] = options.signer(cloudDatasetId, ... + ndiDocumentId, 'idNamespace', "ndi"); + end else - [ok, answer] = options.signer(cloudDatasetId, ndiDocumentId, ... - 'idNamespace', "ndi"); + if strlength(seriesName) > 0 + [ok, answer] = options.signer(cloudDatasetId, ndiDocumentId, ... + 'idNamespace', "ndi", 'fileSeries', seriesName); + else + [ok, answer] = options.signer(cloudDatasetId, ndiDocumentId, ... + 'idNamespace', "ndi"); + end end + catch signerError + ok = false; + answer = []; + failureReason = "the call raised " + string(signerError.identifier) + ... + ": " + string(signerError.message); + end + STATS.signerCalls = STATS.signerCalls + 1; + if ok + break + end + % Retry only on a raise. An ok=false response is not + % transient; break out and let the fallback path handle it. + if strlength(failureReason) == 0 + break + end + if attempt >= maxAttempts + break + end + STATS.transientRetries = STATS.transientRetries + 1; + delay = options.retryBackoffSeconds(attempt); + if delay > 0 + options.sleepFcn(delay); end - catch signerError - ok = false; - answer = []; - failureReason = "the call raised " + string(signerError.identifier) + ... - ": " + string(signerError.message); end - STATS.signerCalls = STATS.signerCalls + 1; if ~ok || ~isstruct(answer) || ~isfield(answer,'files') || ... ~isa(answer.files,'containers.Map') % Any failure or unexpected shape -> no batch URL. Don't @@ -377,6 +499,19 @@ CACHE(theCacheKey) = e; STATS.lastMapSize = double(e.map.Count); STATS.lastMapUids = keys(e.map); + + % Persist to disk when the caller has opted in AND the payload + % carries a server-signed expiry we can age-check off of. A + % payload with neither expiresAt nor filesExpireAt is not + % cacheable -- the disk cache refuses to invent a TTL, on + % purpose (see signedUrlDiskCache.save). Best-effort: any I/O + % failure is swallowed so a full disk does not break a read. + if options.diskCache + try %#ok + ndi.cloud.download.internal.signedUrlDiskCache.save( ... + cloudDatasetId, ndiDocumentId, seriesName, answer); + end + end end function reason = localDescribeFailure(ok, answer, apiResponse) @@ -408,6 +543,35 @@ end end + function e = localTryDiskCache() + % Consult the persistent (on-disk) cache before running the async + % signed-URL-set job. On hit, populate the in-memory cache so + % every uid in the scope resolves without another disk read. On + % miss (no file, corrupt file, expired-or-within-safety-buffer), + % return [] and let the caller fetch fresh. + % + % The disk cache lives one layer above DID's file-bytes cache + % (which is a different, complementary concern) and its TTL is + % keyed to the SERVER'S filesExpireAt, not a hardcoded value -- + % which is why a scientist reopening the same dataset over a + % 24 h day pays the ~85 min sign cost once. + e = []; + try + disk = ndi.cloud.download.internal.signedUrlDiskCache.load( ... + cloudDatasetId, ndiDocumentId, seriesName); + catch + disk = []; + end + if isempty(disk) || ~isstruct(disk) || ~isfield(disk, 'files') || ... + ~isa(disk.files, 'containers.Map') + return + end + e = struct('map', disk.files, 'fetchedAt', now_utc); + CACHE(cacheKey) = e; + STATS.lastMapSize = double(e.map.Count); + STATS.lastMapUids = keys(e.map); + end + function localWarnOnce(scopeKey) % Say it ONCE per scope, then stay quiet. % diff --git a/src/ndi/+ndi/+cloud/+download/+internal/signedUrlDiskCache.m b/src/ndi/+ndi/+cloud/+download/+internal/signedUrlDiskCache.m new file mode 100644 index 0000000000..5acd4ba7f9 --- /dev/null +++ b/src/ndi/+ndi/+cloud/+download/+internal/signedUrlDiskCache.m @@ -0,0 +1,631 @@ +classdef signedUrlDiskCache +%SIGNEDURLDISKCACHE Persistent (on-disk) cache for signed-URL-set payloads. +% +% This is the disk-backed layer that sits beneath the in-process cache +% inside ndi.cloud.download.internal.batchSignedUrlLookup. It exists so a +% working scientist reopening the same dataset over a day pays the +% ~85-minute async signed-URL-set-job cost ONCE and reads from disk on +% every subsequent viewer open. NDI cloud infrastructure, not DID: DID's +% file-bytes cache is a separate, complementary concern. +% +% Scope key +% --------- +% The unit the cache stores and evicts is one (datasetId, documentId, +% seriesName) tuple -- the same tuple batchSignedUrlLookup uses. An +% empty seriesName marks a whole-document scope. +% +% Payload +% ------- +% What save() writes and load() returns is a struct with the fields the +% getSignedURLSetResult / getSignedURLSetAll answers already carry: +% +% .files containers.Map from uid (char) to signed URL (char). +% .filesExpireAt ISO-8601 UTC string; the server's authoritative +% expiry timestamp for the SIGNED URLs. Preferred. +% .expiresAt ISO-8601 UTC string; older/paging path's field. +% Used only when filesExpireAt is not present. +% .generatedAt ISO-8601 UTC string; when the server built the map. +% Optional, informational. +% .fileCount Numeric; how many uids the server reported. +% Optional. +% +% On-disk contract +% ---------------- +% Location: //.json.gz (whole-doc) +% //_.json.gz +% (scoped) +% +% cacheDir defaults to ~/.ndi/signed-url-cache, overridable via the +% NDI_SIGNED_URL_CACHE_DIR environment variable. The base directory is +% created with mode 0700 (POSIX) because a signed URL is effectively a +% bearer token for 24 h; Windows relies on the default per-user profile +% ACL. +% +% Filename escaping for seriesName: any byte that is not +% [A-Za-z0-9._-] becomes '%XX' (uppercase hex). A resulting string +% longer than 96 characters is replaced with 'hash-' (45 +% chars). NDI-python mirrors this rule byte-for-byte. +% +% Format: Standard gzip framing (RFC 1952), single member, UTF-8 +% JSON body. Unzipping yields a human-readable JSON object: +% +% { +% "schemaVersion": 1, +% "datasetId": "", +% "documentId": "", +% "seriesName": "" or "", +% "cachedAt": "YYYY-MM-DDTHH:MM:SSZ", +% "filesExpireAt": "YYYY-MM-DDTHH:MM:SSZ" or "", +% "expiresAt": "YYYY-MM-DDTHH:MM:SSZ" or "", +% "generatedAt": "YYYY-MM-DDTHH:MM:SSZ" or "", +% "fileCount": N, +% "files": { "": "", ... } +% } +% +% TTL and safety buffer +% --------------------- +% The cache does not invent a TTL. It reads the authoritative +% filesExpireAt (or expiresAt) from the payload; a payload with +% neither is treated as un-cacheable (save is a no-op) and, on load, +% as an immediate miss. A URL that would expire in less than the +% safety buffer (default 30 min, override with +% NDI_SIGNED_URL_CACHE_SAFETY_SECONDS) is also treated as a miss -- +% we never hand a caller a URL likely to 403 before they can use it. +% +% Concurrency +% ----------- +% Writes go to a per-writer temp file next to the target and are +% moved into place with an atomic rename (java.nio.file.Files.move +% with ATOMIC_MOVE, which uses rename(2) on POSIX and MoveFileEx on +% Windows). Two viewers signing the same scope concurrently is fine: +% whichever atomic move lands last wins, both readers see a +% consistent file afterwards. There is no cross-process lock; the +% contract is last-writer-wins, not exclusive. +% +% Invalidation +% ------------ +% forget(dsId, docId, seriesName) removes the file for one scope. The +% caller does this from the chunk-download path when it sees an S3 +% 403 -- the URL was in the cache but the object rotated or the +% token was revoked, so drop the scope and let the next lookup +% refetch. Failing to remove the file is not fatal. +% +% Public API +% ---------- +% payload = load(datasetId, documentId, seriesName) +% Returns a struct on cache hit, [] on any miss/expired/error. +% save(datasetId, documentId, seriesName, payload) +% Writes atomically; best-effort (silent on any I/O error). +% forget(datasetId, documentId, seriesName) +% Deletes the cache file for that scope; best-effort. +% location = cacheDir() +% Returns the absolute on-disk root, honoring the env-var +% override. Also creates it (with mode 0700 on POSIX) the +% first time it is asked for on a session. +% +% See also: ndi.cloud.download.internal.batchSignedUrlLookup, +% ndi.cloud.api.files.getSignedURLSetResult, +% ndi.cloud.api.files.getSignedURLSetAll + + properties (Constant, Access = private) + SchemaVersion = 1 + DefaultSafetySec = 1800 % 30 min; task-declared floor. + MaxEscapedSeriesLen = 96 % Beyond this we hash the series name. + end + + methods (Static) + + function payload = load(datasetId, documentId, seriesName) + %LOAD Read one scope's cached signed-URL set, or [] on any miss. + % + % PAYLOAD = LOAD(DATASETID, DOCUMENTID, SERIESNAME) returns a + % struct (fields: files, filesExpireAt, expiresAt, generatedAt, + % fileCount) on cache hit. Returns [] on any of: no cache file, + % corrupt file, JSON parse error, expired-or-within-safety-buffer. + % + % The safety buffer is 30 min by default, overridable via + % NDI_SIGNED_URL_CACHE_SAFETY_SECONDS. A URL that would expire + % inside that window is a miss so the caller refetches rather + % than handing bytes to a viewer that then 403s. + arguments + datasetId (1,1) string + documentId (1,1) string + seriesName (1,1) string = "" + end + payload = []; + + filePath = ndi.cloud.download.internal.signedUrlDiskCache.cachePath( ... + datasetId, documentId, seriesName); + if ~isfile(filePath) + return + end + + try + jsonTxt = ndi.cloud.download.internal.signedUrlDiskCache.gunzipToString(filePath); + catch + return + end + + try + decoded = ndi.cloud.download.internal.signedUrlDiskCache.parseJson(jsonTxt); + catch + return + end + + if isempty(decoded) + return + end + + % Enforce the safety-buffer floor against the authoritative + % server timestamp. A payload with no timestamp is a miss: + % better to refetch than hand out a URL we can't age-check. + safetySec = ndi.cloud.download.internal.signedUrlDiskCache.safetySeconds(); + expiryStr = decoded.filesExpireAt; + if strlength(expiryStr) == 0 + expiryStr = decoded.expiresAt; + end + if strlength(expiryStr) == 0 + return + end + + expiry = ndi.cloud.download.internal.signedUrlDiskCache.parseIsoUtc(expiryStr); + if isnat(expiry) + return + end + nowUtc = datetime('now', 'TimeZone', 'UTC'); + if seconds(expiry - nowUtc) < safetySec + return + end + + payload = decoded; + end + + function save(datasetId, documentId, seriesName, payload) + %SAVE Persist a scope's signed-URL set. Best-effort, atomic. + % + % SAVE(DATASETID, DOCUMENTID, SERIESNAME, PAYLOAD) writes the + % payload to the scope's cache path via a temp file + atomic + % rename. A payload without a filesExpireAt / expiresAt is + % NOT written -- the cache cannot age-check something it has + % no timestamp for, and we refuse to invent a TTL. + % + % Any I/O failure (missing dir, permission denied, disk full, + % another process racing us) is swallowed silently: the + % in-memory cache still works, so a viewer open still succeeds. + arguments + datasetId (1,1) string + documentId (1,1) string + seriesName (1,1) string + payload struct + end + + if ~isfield(payload, 'files') || ~isa(payload.files, 'containers.Map') + return + end + + % Refuse to persist an un-age-checkable payload. + filesExpireAt = ndi.cloud.download.internal.signedUrlDiskCache.stringField(payload, 'filesExpireAt'); + expiresAt = ndi.cloud.download.internal.signedUrlDiskCache.stringField(payload, 'expiresAt'); + if strlength(filesExpireAt) == 0 && strlength(expiresAt) == 0 + return + end + + generatedAt = ndi.cloud.download.internal.signedUrlDiskCache.stringField(payload, 'generatedAt'); + fileCount = ndi.cloud.download.internal.signedUrlDiskCache.numericField(payload, 'fileCount', ... + double(payload.files.Count)); + + filePath = ndi.cloud.download.internal.signedUrlDiskCache.cachePath( ... + datasetId, documentId, seriesName); + + try + ndi.cloud.download.internal.signedUrlDiskCache.ensureParentDir(filePath); + catch + return + end + + try + jsonTxt = ndi.cloud.download.internal.signedUrlDiskCache.buildJson(... + struct( ... + 'datasetId', char(datasetId), ... + 'documentId', char(documentId), ... + 'seriesName', char(seriesName), ... + 'cachedAt', ndi.cloud.download.internal.signedUrlDiskCache.isoNow(), ... + 'filesExpireAt', char(filesExpireAt), ... + 'expiresAt', char(expiresAt), ... + 'generatedAt', char(generatedAt), ... + 'fileCount', fileCount, ... + 'files', payload.files)); + catch + return + end + + tmpPath = [filePath '.tmp.' num2str(localProcessId()) '.' ... + num2str(randi(2^31 - 1))]; + try + ndi.cloud.download.internal.signedUrlDiskCache.gzipToFile(jsonTxt, tmpPath); + ndi.cloud.download.internal.signedUrlDiskCache.atomicRename(tmpPath, filePath); + catch + if isfile(tmpPath) + try %#ok + delete(tmpPath); + end + end + end + end + + function forget(datasetId, documentId, seriesName) + %FORGET Remove one scope's cache file. Best-effort. + % + % Call this from the chunk-download path when an S3 403 says + % the cached URL is no longer valid. If the file is not + % present -- or the delete itself fails -- it is not an error: + % the next lookup will refetch. + arguments + datasetId (1,1) string + documentId (1,1) string + seriesName (1,1) string = "" + end + filePath = ndi.cloud.download.internal.signedUrlDiskCache.cachePath( ... + datasetId, documentId, seriesName); + if isfile(filePath) + try + delete(filePath); + catch + % Best-effort: a failed delete is not fatal to the + % caller who is already recovering from a 403. + end + end + end + + function d = cacheDir() + %CACHEDIR Absolute path to the on-disk cache root. + % + % Honors NDI_SIGNED_URL_CACHE_DIR; otherwise + % ~/.ndi/signed-url-cache. Creates the directory the first + % time it is asked for, with mode 0700 on POSIX so a bearer + % token that is legibly a bearer token is not world-readable. + override = string(getenv('NDI_SIGNED_URL_CACHE_DIR')); + if strlength(override) > 0 + d = char(override); + else + home = char(java.lang.System.getProperty('user.home')); + d = fullfile(home, '.ndi', 'signed-url-cache'); + end + if ~isfolder(d) + try + mkdir(d); + ndi.cloud.download.internal.signedUrlDiskCache.chmodUserOnly(d); + catch + % Best-effort: batchSignedUrlLookup treats a missing + % cache dir as "no disk cache available". + end + end + end + end + + methods (Static, Access = private) + + function s = safetySeconds() + override = string(getenv('NDI_SIGNED_URL_CACHE_SAFETY_SECONDS')); + if strlength(override) > 0 + v = str2double(override); + if isfinite(v) && v >= 0 + s = v; + return + end + end + s = ndi.cloud.download.internal.signedUrlDiskCache.DefaultSafetySec; + end + + function p = cachePath(datasetId, documentId, seriesName) + % Series names can be arbitrary strings (paths, spaces, uni); + % filesystems can't hold them literally. Follow the escape + % rule NDI-python will mirror byte-for-byte. + base = ndi.cloud.download.internal.signedUrlDiskCache.cacheDir(); + dsDir = fullfile(base, char(datasetId)); + if strlength(seriesName) == 0 + fname = [char(documentId) '.json.gz']; + else + escaped = ndi.cloud.download.internal.signedUrlDiskCache.escapeSeriesName(char(seriesName)); + fname = [char(documentId) '_' escaped '.json.gz']; + end + p = fullfile(dsDir, fname); + end + + function s = escapeSeriesName(raw) + % Percent-encode any byte outside [A-Za-z0-9._-]. Uppercase hex. + % Above MaxEscapedSeriesLen fall back to 'hash-' + % so a pathologically long name never overflows a filesystem's + % 255-byte name limit. + bytes = uint8(unicode2native(raw, 'UTF-8')); + safe = (bytes >= uint8('A') & bytes <= uint8('Z')) | ... + (bytes >= uint8('a') & bytes <= uint8('z')) | ... + (bytes >= uint8('0') & bytes <= uint8('9')) | ... + (bytes == uint8('.')) | ... + (bytes == uint8('_')) | ... + (bytes == uint8('-')); + parts = cell(1, numel(bytes)); + for i = 1:numel(bytes) + if safe(i) + parts{i} = char(bytes(i)); + else + parts{i} = sprintf('%%%02X', bytes(i)); + end + end + s = strjoin(parts, ''); + if numel(s) > ndi.cloud.download.internal.signedUrlDiskCache.MaxEscapedSeriesLen + s = ['hash-' ndi.cloud.download.internal.signedUrlDiskCache.sha1Hex(raw)]; + end + end + + function hex = sha1Hex(raw) + md = java.security.MessageDigest.getInstance('SHA-1'); + md.update(int8(unicode2native(raw, 'UTF-8'))); + digest = typecast(md.digest(), 'uint8'); + hex = lower(reshape(dec2hex(digest, 2).', 1, [])); + end + + function ensureParentDir(filePath) + parent = fileparts(filePath); + if ~isfolder(parent) + mkdir(parent); + end + end + + function chmodUserOnly(dirPath) + % Bearer tokens for 24 h -- keep world-readable off on POSIX. + % Windows relies on the default per-user profile ACL for + % ~/.ndi/, which is roughly equivalent. + if isunix + try %#ok + if exist('filePermissions', 'builtin') == 5 || ... + exist('filePermissions', 'file') == 2 + p = filePermissions(dirPath); + p.User.Read = true; + p.User.Write = true; + p.User.Execute = true; + p.Group.Read = false; + p.Group.Write = false; + p.Group.Execute = false; + p.Other.Read = false; + p.Other.Write = false; + p.Other.Execute = false; + filePermissions(dirPath, p); + else + fileattrib(dirPath, '+w', 'u'); + fileattrib(dirPath, '-w', 'go'); + fileattrib(dirPath, '-r', 'go'); + fileattrib(dirPath, '-x', 'go'); + end + end + end + end + + function gzipToFile(txt, outPath) + % Write UTF-8 bytes to a plain temp file, then use MATLAB's + % builtin gzip to produce a valid single-member gzip that + % gunzip(1), tar -xzf and NDI-python's gzip module can all + % read. We avoid Java streams because the Java bridge's + % overload resolution for write(byte[]) versus write(int) is + % release- and JVM-dependent, and a mis-picked overload here + % writes silently corrupt output. + parent = fileparts(outPath); + stem = tempname(parent); + plainPath = [stem '.json']; + fid = fopen(plainPath, 'w'); + if fid < 0 + error('NDI:SignedUrlDiskCache:WriteOpenFailed', ... + 'Could not open %s for writing.', plainPath); + end + cleanupPlain = onCleanup(@() localDeleteIfExists(plainPath)); %#ok + bytes = unicode2native(char(txt), 'UTF-8'); + fwrite(fid, bytes, 'uint8'); + fclose(fid); + % gzip(FILE) writes FILE.gz next to FILE in the same dir. + gzip(plainPath); + gzPath = [plainPath '.gz']; + cleanupGz = onCleanup(@() localDeleteIfExists(gzPath)); %#ok + movefile(gzPath, outPath, 'f'); + end + + function txt = gunzipToString(inPath) + % gunzip writes into a temp dir, then we read the plain file. + % Same rationale as gzipToFile: avoid the Java-bridge + % overload traps by using MATLAB builtins end to end. + outDir = tempname; + mkdir(outDir); + cleanupDir = onCleanup(@() localCleanupDir(outDir)); %#ok + names = gunzip(inPath, outDir); + if isempty(names) + error('NDI:SignedUrlDiskCache:EmptyGunzip', ... + 'gunzip produced no files from %s', inPath); + end + fid = fopen(names{1}, 'rb'); + if fid < 0 + error('NDI:SignedUrlDiskCache:ReadOpenFailed', ... + 'Could not open %s for reading.', names{1}); + end + raw = fread(fid, inf, '*uint8')'; + fclose(fid); + txt = native2unicode(raw, 'UTF-8'); + end + + function atomicRename(srcPath, dstPath) + % movefile on the same directory delegates to a POSIX + % rename(2) or a Windows MoveFileEx replace, which is enough + % for the "no half-written file visible to a concurrent + % reader" invariant this cache needs. We used to go through + % java.nio.file.Files.move(ATOMIC_MOVE) but the javaArray + % path there is fragile across MATLAB releases -- an + % interface-typed array plus an enum-to-array assignment can + % throw silently, and save()'s outer try/catch swallows it. + movefile(srcPath, dstPath, 'f'); + end + + function s = isoNow() + dt = datetime('now', 'TimeZone', 'UTC'); + dt.Format = 'yyyy-MM-dd''T''HH:mm:ss''Z'''; + s = char(dt); + end + + function dt = parseIsoUtc(str) + % Server sends 'YYYY-MM-DDTHH:MM:SSZ' or 'YYYY-MM-DDTHH:MM:SS.SSSZ'. + % Strip a trailing 'Z' (or ' +00:00' / '+00:00') and parse as + % naive UTC. Simpler than juggling MATLAB's timezone format + % code, which differs across releases. + str = char(str); + dt = NaT('TimeZone','UTC'); + if isempty(str) + return + end + % Normalise the trailing UTC marker. + trimmed = strtrim(str); + if endsWith(trimmed, 'Z') + trimmed = trimmed(1:end-1); + elseif endsWith(trimmed, '+00:00') || endsWith(trimmed, '-00:00') + trimmed = trimmed(1:end-6); + end + trimmed = strrep(trimmed, ' ', 'T'); + candidates = { ... + 'yyyy-MM-dd''T''HH:mm:ss.SSS', ... + 'yyyy-MM-dd''T''HH:mm:ss'}; + for i = 1:numel(candidates) + try + dt = datetime(trimmed, 'InputFormat', candidates{i}, ... + 'TimeZone','UTC'); + if ~isnat(dt) + return + end + catch + % try next + end + end + end + + function v = stringField(s, name) + v = ""; + if isfield(s, name) && ~isempty(s.(name)) + v = string(s.(name)); + if strlength(v) == 0 + v = ""; + end + end + end + + function v = numericField(s, name, defaultValue) + v = defaultValue; + if isfield(s, name) && isnumeric(s.(name)) && ~isempty(s.(name)) + v = double(s.(name)); + end + end + + function txt = buildJson(rec) + % Hand-build the JSON so digit-prefixed uid keys survive + % verbatim -- jsonencode on a containers.Map is a moving + % target across MATLAB releases, and jsondecode on a struct + % with digit keys goes through the same rename dance that + % signedURLFileMap already had to work around. + E = @(v) jsonencode(string(v)); + parts = {}; + parts{end+1} = sprintf('{"schemaVersion":%d', ... + ndi.cloud.download.internal.signedUrlDiskCache.SchemaVersion); + parts{end+1} = sprintf(',"datasetId":%s', E(rec.datasetId)); + parts{end+1} = sprintf(',"documentId":%s', E(rec.documentId)); + parts{end+1} = sprintf(',"seriesName":%s', E(rec.seriesName)); + parts{end+1} = sprintf(',"cachedAt":%s', E(rec.cachedAt)); + parts{end+1} = sprintf(',"filesExpireAt":%s', E(rec.filesExpireAt)); + parts{end+1} = sprintf(',"expiresAt":%s', E(rec.expiresAt)); + parts{end+1} = sprintf(',"generatedAt":%s', E(rec.generatedAt)); + parts{end+1} = sprintf(',"fileCount":%d', rec.fileCount); + parts{end+1} = ',"files":'; + parts{end+1} = ndi.cloud.download.internal.signedUrlDiskCache.mapToJson(rec.files); + parts{end+1} = '}'; + txt = strjoin(parts, ''); + end + + function txt = mapToJson(m) + ks = keys(m); + n = numel(ks); + if n == 0 + txt = '{}'; + return + end + parts = cell(1, n); + for i = 1:n + k = ks{i}; + v = m(k); + parts{i} = [jsonencode(string(k)) ':' jsonencode(string(v))]; + end + txt = ['{' strjoin(parts, ',') '}']; + end + + function out = parseJson(txt) + out = []; + % jsondecode renames field names beginning with a digit, + % which every did.ido uid does. Reuse the same trick + % signedURLFileMap uses: decode the whole thing, then + % rebuild the files map by scanning the raw text for uid + % keys in payload order. Everything else on the top-level + % object survives decoding intact. + data = jsondecode(txt); + if ~isstruct(data) + return + end + files = containers.Map('KeyType','char','ValueType','any'); + if isfield(data, 'files') && ~isempty(data.files) + files = ndi.cloud.api.implementation.files.signedURLFileMap( ... + data.files, txt); + end + out = struct( ... + 'files', files, ... + 'filesExpireAt', ndi.cloud.download.internal.signedUrlDiskCache.readTopString(data, 'filesExpireAt'), ... + 'expiresAt', ndi.cloud.download.internal.signedUrlDiskCache.readTopString(data, 'expiresAt'), ... + 'generatedAt', ndi.cloud.download.internal.signedUrlDiskCache.readTopString(data, 'generatedAt'), ... + 'fileCount', ndi.cloud.download.internal.signedUrlDiskCache.readTopNumber(data, 'fileCount')); + end + + function v = readTopString(data, name) + v = ""; + if isfield(data, name) && ~isempty(data.(name)) + v = string(data.(name)); + end + end + + function v = readTopNumber(data, name) + v = 0; + if isfield(data, name) && isnumeric(data.(name)) && ~isempty(data.(name)) + v = double(data.(name)); + end + end + end +end + +function localDeleteIfExists(p) +% Best-effort delete for an onCleanup guard on a scratch file. + if ischar(p) && ~isempty(p) && isfile(p) + try %#ok + delete(p); + end + end +end + +function localCleanupDir(d) +% Best-effort recursive rmdir for a scratch temp directory used only for +% one gunzip. Missing directory is not an error. + if ischar(d) && ~isempty(d) && isfolder(d) + try %#ok + rmdir(d, 's'); + end + end +end + +function pid = localProcessId() +% matlabProcessID is the supported accessor in recent releases; feature +% is the old, still-shipping one. Wrap so a plain integer PID comes back +% either way and no code path relies on feature('getpid'). + try + pid = matlabProcessID(); + catch + pid = feature('getpid'); + end +end diff --git a/src/ndi/+ndi/+cloud/+internal/decodeJwt.m b/src/ndi/+ndi/+cloud/+internal/decodeJwt.m index c8e644ef62..48226ae257 100644 --- a/src/ndi/+ndi/+cloud/+internal/decodeJwt.m +++ b/src/ndi/+ndi/+cloud/+internal/decodeJwt.m @@ -40,7 +40,7 @@ if padding > 0 payloadBase64 = [payloadBase64, repmat('=', 1, 4 - padding)]; end - payloadBytes = javax.xml.bind.DatatypeConverter.parseBase64Binary(payloadBase64); + payloadBytes = matlab.net.base64decode(payloadBase64); payloadJSON = native2unicode(payloadBytes, 'UTF-8'); % Ensure json payload is a row vector diff --git a/src/ndi/+ndi/+cloud/+internal/resolveOrganizationId.m b/src/ndi/+ndi/+cloud/+internal/resolveOrganizationId.m new file mode 100644 index 0000000000..28b0acff01 --- /dev/null +++ b/src/ndi/+ndi/+cloud/+internal/resolveOrganizationId.m @@ -0,0 +1,82 @@ +function organizationId = resolveOrganizationId(options) +% RESOLVEORGANIZATIONID - Resolve an organization name or id to an id. +% +% ORGANIZATIONID = ndi.cloud.internal.resolveOrganizationId( ... +% 'organizationName', ) resolves the given name against the +% organizations the current user belongs to (via ndi.cloud.api.users.me). +% +% ORGANIZATIONID = ndi.cloud.internal.resolveOrganizationId( ... +% 'organizationID', ) verifies the given id is one the current +% user belongs to and returns it. +% +% Exactly one of the two must be non-empty. Names are matched +% case-sensitively; ambiguous or unknown values raise an error naming +% the organizations that ARE available, so a caller can print a useful +% diagnostic. +% +% See also: ndi.cloud.uploadDataset, ndi.cloud.ui.dialog.selectOrganization. + + arguments + options.organizationName (1,:) char = '' + options.organizationID (1,:) char = '' + end + + nameGiven = ~isempty(options.organizationName); + idGiven = ~isempty(options.organizationID); + if nameGiven == idGiven + error('NDI:Cloud:OrganizationArguments', ... + 'Exactly one of organizationName or organizationID must be given.'); + end + + [ok, userInfo] = ndi.cloud.api.users.me(); + if ~ok + error('NDI:Cloud:MeFailed', ... + 'Could not retrieve your NDI Cloud user information.'); + end + + orgIds = cellfun(@(x) char(string(x)), userInfo.organizationID, 'UniformOutput', false); + orgNames = cellfun(@(x) char(string(x)), userInfo.organizationName, 'UniformOutput', false); + + if isempty(orgIds) + error('NDI:Cloud:NoOrganizations', ... + 'Your NDI Cloud account does not belong to any organizations.'); + end + + if nameGiven + matches = find(strcmp(orgNames, options.organizationName)); + if isempty(matches) + error('NDI:Cloud:UnknownOrganization', ... + ['Organization name "%s" is not one of the organizations you belong ', ... + 'to. Available: %s.'], options.organizationName, ... + availableList(orgNames, orgIds)); + end + if numel(matches) > 1 + error('NDI:Cloud:AmbiguousOrganization', ... + ['Organization name "%s" matches %d organizations you belong to; ', ... + 'pass organizationID instead. Ids: %s.'], options.organizationName, ... + numel(matches), strjoin(orgIds(matches), ', ')); + end + organizationId = orgIds{matches}; + else + matches = find(strcmp(orgIds, options.organizationID)); + if isempty(matches) + error('NDI:Cloud:UnknownOrganization', ... + ['Organization id "%s" is not one of the organizations you belong ', ... + 'to. Available: %s.'], options.organizationID, ... + availableList(orgNames, orgIds)); + end + organizationId = orgIds{matches}; + end +end + +function s = availableList(names, ids) + parts = strings(1, numel(ids)); + for k = 1:numel(ids) + if k <= numel(names) && ~isempty(names{k}) + parts(k) = sprintf('"%s" (%s)', names{k}, ids{k}); + else + parts(k) = sprintf('(no name) (%s)', ids{k}); + end + end + s = char(strjoin(parts, ', ')); +end diff --git a/src/ndi/+ndi/+cloud/+sync/+internal/downloadNdiDocuments.m b/src/ndi/+ndi/+cloud/+sync/+internal/downloadNdiDocuments.m index 066616303e..418a3888e2 100644 --- a/src/ndi/+ndi/+cloud/+sync/+internal/downloadNdiDocuments.m +++ b/src/ndi/+ndi/+cloud/+sync/+internal/downloadNdiDocuments.m @@ -130,8 +130,16 @@ if syncOptions.Verbose fprintf('"SyncFiles" option is false. Updating document file info to reflect remote files.\n'); end + % Build the uploaded-uid map ONCE from the paginated whole-dataset + % listing (NDI-matlab#1004's listFilesAll). Threaded into + % updateFileInfoForRemoteFiles so per-doc rewrites can validate + % uids and per-series manifest fetches can skip uids the server + % does not report as uploaded. Empty map on failure preserves + % the previous no-check behavior. + uploadedUidMap = buildUploadedUidMap(cloudDatasetId, syncOptions.Verbose); documentUpdateFcn = @(doc) ... - ndi.cloud.sync.internal.updateFileInfoForRemoteFiles(doc, cloudDatasetId); + ndi.cloud.sync.internal.updateFileInfoForRemoteFiles(doc, cloudDatasetId, ... + 'uploadedUidMap', uploadedUidMap); end % 3. Update file info for documents based on local / remote location @@ -153,3 +161,40 @@ downloadedNdiDocuments = newNdiDocuments; end end + +function m = buildUploadedUidMap(cloudDatasetId, verbose) + % One paginated whole-dataset listing (listFilesAll from NDI-matlab#1004), + % into a containers.Map of uid -> uploaded logical. Any failure returns + % an empty map, which the downstream code treats as "no check" -- + % preserves the pre-#1007 behavior. Verbose mode reports the count so + % the "Updating document file info..." step has a denominator to show. + m = containers.Map('KeyType','char','ValueType','logical'); + try + if verbose + fprintf('Fetching cloud file listing for uid validation ...\n'); + end + [ok, listing] = ndi.cloud.api.files.listFilesAll(cloudDatasetId); + if ~ok + if verbose + fprintf(['Could not fetch cloud file listing; skipping ', ... + 'uid validation (per-doc opens will surface any misses).\n']); + end + return + end + for i = 1:numel(listing) + uidChar = char(string(listing(i).uid)); + if isempty(uidChar), continue, end + m(uidChar) = logical(listing(i).uploaded); + end + if verbose + uploadedCount = sum(cellfun(@logical, m.values)); + fprintf('Cloud dataset lists %d file(s); %d uploaded.\n', ... + m.Count, uploadedCount); + end + catch err + if verbose + fprintf('Uid map build failed: %s. Continuing without validation.\n', err.message); + end + m = containers.Map('KeyType','char','ValueType','logical'); + end +end diff --git a/src/ndi/+ndi/+cloud/+sync/+internal/fetchManifest.m b/src/ndi/+ndi/+cloud/+sync/+internal/fetchManifest.m new file mode 100644 index 0000000000..0ae6bfad18 --- /dev/null +++ b/src/ndi/+ndi/+cloud/+sync/+internal/fetchManifest.m @@ -0,0 +1,111 @@ +function fetchManifest(destPath, cloudDatasetId, manifestUid, documentId, ... + seriesName, customFileHandler, options) +%FETCHMANIFEST Download a single series manifest to a local path. +% +% Extracted from ndi.cloud.sync.internal.updateFileInfoForRemoteFiles +% so the manifest fetch can be exercised in a test without dragging +% the whole reconstruction path in behind it. +% +% A manifest is a SINGLE KNOWN uid, so the URL is resolved with one +% direct getFileDetails call. Going through the per-document batch +% cache (ndi.cloud.download.internal.batchSignedUrlLookup) here would +% force the batch to walk the whole document's signed-URL set to +% answer a question that has one correct answer: for a 121k-chunk +% lightsheet level that is ~243 pages at ~25 s each -- roughly +% 100 min per manifest fetch, seven levels ≈ 12 h before any bytes +% hit disk. The batch is worth its cost when many uids share the +% document; for a single uid it is pure overhead. See +% VH-Lab/NDI-matlab#1010 and Waltham-Data-Science/NDI-python +% commit 5ab9c04 (the Python analog fix). +% +% Inputs: +% destPath - Local file to write the manifest bytes to. +% cloudDatasetId - The cloud dataset id. +% manifestUid - The manifest file's uid. +% documentId - The NDI document id (passed to the +% customFileHandler context; ignored on the +% direct-fetch path). +% seriesName - The series' name (passed to the handler +% context so it can identify the series +% whose manifest is being fetched). +% customFileHandler - When non-empty, the fetch is dispatched +% through DID's customFileHandler contract +% (same shape DID#201's fetchSeriesManifest +% uses on the read side). Otherwise a signed +% URL is minted and the file downloaded +% directly, matching didsqlite.m's own +% download_file_from_cloud path. +% +% Name-Value Pairs: +% DetailsFetcher - Function handle called as +% [ok, answer] = fcn(datasetId, fileUid) to +% resolve one uid's signed URL. Defaults to +% @ndi.cloud.api.files.getFileDetails. Present +% so a test can inject a scripted answer and +% prove which single-uid function was called +% with which uid, without hitting the network. +% FileFetcher - Function handle called as +% [ok, msg] = fcn(url, path, 'useCurl', true) +% to write the manifest bytes to disk. Defaults +% to @ndi.cloud.api.files.getFile. Same +% injection point for tests. +% +% See also: ndi.cloud.api.files.getFileDetails, +% ndi.cloud.api.files.getFile, +% ndi.cloud.download.internal.batchSignedUrlLookup + + arguments + destPath (1,1) string + cloudDatasetId (1,1) string + manifestUid (1,1) string + documentId (1,1) string + seriesName (1,1) string + customFileHandler + options.DetailsFetcher (1,1) function_handle = ... + @ndi.cloud.api.files.getFileDetails + options.FileFetcher (1,1) function_handle = ... + @ndi.cloud.api.files.getFile + end + + % seriesName is deliberately '' in the ctx: this is the manifest + % itself, not a member. On the DID contract, a non-empty + % seriesName marks a MEMBER fetch, where sourcePath names the + % manifest and context.uid names the member -- the handler must + % then treat context.uid as the file to retrieve, not the manifest + % again. A manifest fetch has no such switch: sourcePath names the + % manifest and context.uid names the same manifest. + sourcePath = sprintf('ndic://%s/%s', char(cloudDatasetId), char(manifestUid)); + + if ~isempty(customFileHandler) + ctx = struct( ... + 'documentId', char(documentId), ... + 'filename', char(seriesName), ... + 'seriesName', '', ... + 'uid', char(manifestUid), ... + 'mode', 'open'); + did.implementations.sqlitedb.dispatchCustomFileHandler( ... + customFileHandler, char(destPath), sourcePath, ctx); + return + end + + [success, answer] = options.DetailsFetcher( ... + char(cloudDatasetId), char(manifestUid)); + if ~success + message = ''; + if isstruct(answer) && isfield(answer, 'message') + message = char(string(answer.message)); + end + error('NDI:cloud:sync:ManifestFetchFailed', ... + 'Failed to get file details for manifest %s: %s', ... + char(manifestUid), message); + end + fileUrl = answer.downloadUrl; + + [success2, answer2] = options.FileFetcher( ... + char(fileUrl), char(destPath), 'useCurl', true); + if ~success2 + error('NDI:cloud:sync:ManifestFetchFailed', ... + 'Failed to download manifest %s from cloud: %s', ... + char(manifestUid), char(string(answer2))); + end +end diff --git a/src/ndi/+ndi/+cloud/+sync/+internal/reconstructSeriesIngestLocations.m b/src/ndi/+ndi/+cloud/+sync/+internal/reconstructSeriesIngestLocations.m index 4204e72b2e..8290ee29dd 100644 --- a/src/ndi/+ndi/+cloud/+sync/+internal/reconstructSeriesIngestLocations.m +++ b/src/ndi/+ndi/+cloud/+sync/+internal/reconstructSeriesIngestLocations.m @@ -1,8 +1,10 @@ -function seriesInfo = reconstructSeriesIngestLocations(seriesInfo, fileInfo, fileDirectory, cloudDatasetId) +function seriesInfo = reconstructSeriesIngestLocations(seriesInfo, fileInfo, fileDirectory, cloudDatasetId, options) %RECONSTRUCTSERIESINGESTLOCATIONS Rebuild series ingest_locations from a downloaded manifest. % % SERIESINFO = ndi.cloud.sync.internal.reconstructSeriesIngestLocations( ... % SERIESINFO, FILEINFO, FILEDIRECTORY, CLOUDDATASETID) +% SERIESINFO = ndi.cloud.sync.internal.reconstructSeriesIngestLocations( ... +% ..., 'MemberBase', BASE) % % For every series in SERIESINFO that declares `n_present > 0` and % carries no `ingest_locations`, this reads the downloaded manifest file @@ -42,6 +44,17 @@ % `location`, matching the URI scheme % `download_file_from_cloud` understands. % +% Options: +% MemberBase - The number of the series' FIRST member, read off +% the DOCUMENT'S VINTAGE with +% ndi.vintage.seriesMemberBase(document): 1 for a v1 +% document (NAME_1 is manifest slot 0), 0 for a +% V_eta document (NAME_0 is slot 0; DID-schema #73 +% item 17, DID-matlab 1a6bd9f). Default 1, the v1 +% behaviour. Callers holding the document must pass +% it: the default names the wrong members of a V_eta +% document. +% % Outputs: % seriesInfo - The updated struct array. % @@ -65,6 +78,7 @@ fileInfo fileDirectory (1,1) string cloudDatasetId (1,1) string + options.MemberBase (1,1) double {mustBeMember(options.MemberBase, [0 1])} = 1 end if isempty(seriesInfo), return, end @@ -92,7 +106,7 @@ continue end - ingestLocations = buildIngestLocations(manifest, cloudDatasetId); + ingestLocations = buildIngestLocations(manifest, cloudDatasetId, options.MemberBase); if isempty(ingestLocations), continue, end seriesInfo(k).ingest_locations = ingestLocations; @@ -112,27 +126,42 @@ end end -function ingestLocations = buildIngestLocations(manifest, cloudDatasetId) +function ingestLocations = buildIngestLocations(manifest, cloudDatasetId, memberBase) % One ingest_locations entry per present slot of the manifest. -% Order matches DID's authoring shape and its `sprintf('%s_%d', name, index)` -% naming (index is 1-based, same as manifest.uids{index}). - ingestLocations = struct( ... - 'index', {}, ... - 'uid', {}, ... - 'location', {}, ... - 'location_type', {}, ... - 'ingest', {}, ... - 'delete_original', {}); +% `index` is the member NUMBER as named, not the manifest position: DID +% names the member `sprintf('%s_%d', name, index)` when it ingests it +% (did.implementations.sqlitedb/do_add_doc), and did.document/addFileSeries +% writes `index` as the number in the document's own base. So manifest +% position `slot` (1-based, manifest.uids{slot}) is member number +% slot - 1 + memberBase: slot for a v1 document, slot - 1 for a V_eta one. +% +% Preallocated + trimmed rather than grown: a lightsheet pyramid level's +% manifest names tens of thousands of members, and growing the struct +% array one entry at a time with `end+1` reallocates the whole array on +% every step -- classic O(N^2), turning a linear pass into minutes. +% See NDI-matlab lightsheet 2026-09-24 download hang. + prototype = struct( ... + 'index', 0, ... + 'uid', '', ... + 'location', '', ... + 'location_type', '', ... + 'ingest', 0, ... + 'delete_original', 0); + entries = repmat(prototype, 1, manifest.count); + kept = 0; + cloudDatasetIdChar = char(cloudDatasetId); for slot = 1 : manifest.count memberUid = manifest.uids{slot}; if isempty(memberUid), continue, end - ingestLocations(end+1) = struct( ... - 'index', slot, ... - 'uid', memberUid, ... - 'location', sprintf('ndic://%s/%s', char(cloudDatasetId), memberUid), ... - 'location_type', 'ndicloud', ... - 'ingest', 0, ... - 'delete_original', 0); %#ok + kept = kept + 1; + entries(kept).index = slot - 1 + memberBase; + entries(kept).uid = memberUid; + entries(kept).location = sprintf('ndic://%s/%s', cloudDatasetIdChar, memberUid); + entries(kept).location_type = 'ndicloud'; + entries(kept).ingest = 0; + entries(kept).delete_original = 0; end + + ingestLocations = entries(1:kept); end diff --git a/src/ndi/+ndi/+cloud/+sync/+internal/updateFileInfoForLocalFiles.m b/src/ndi/+ndi/+cloud/+sync/+internal/updateFileInfoForLocalFiles.m index 971b82f746..f89b653467 100644 --- a/src/ndi/+ndi/+cloud/+sync/+internal/updateFileInfoForLocalFiles.m +++ b/src/ndi/+ndi/+cloud/+sync/+internal/updateFileInfoForLocalFiles.m @@ -108,7 +108,8 @@ if hasSeriesInfo if strlength(cloudDatasetId) > 0 originalSeriesInfo = ndi.cloud.sync.internal.reconstructSeriesIngestLocations( ... - originalSeriesInfo, originalFileInfo, fileDirectory, cloudDatasetId); + originalSeriesInfo, originalFileInfo, fileDirectory, cloudDatasetId, ... + 'MemberBase', ndi.vintage.seriesMemberBase(document)); end document = document.setproperties('files.series_info', originalSeriesInfo); end diff --git a/src/ndi/+ndi/+cloud/+sync/+internal/updateFileInfoForRemoteFiles.m b/src/ndi/+ndi/+cloud/+sync/+internal/updateFileInfoForRemoteFiles.m index 3dbdbaac42..1e26a74173 100644 --- a/src/ndi/+ndi/+cloud/+sync/+internal/updateFileInfoForRemoteFiles.m +++ b/src/ndi/+ndi/+cloud/+sync/+internal/updateFileInfoForRemoteFiles.m @@ -61,12 +61,22 @@ document cloudDatasetId (1,1) string options.customFileHandler = [] + % A containers.Map of uid -> logical uploaded flag, precomputed + % once at the caller from ndi.cloud.api.files.listFilesAll (the + % paginated whole-dataset listing landed in main via NDI-matlab + % #1004). When supplied, this function uses it to skip the manifest + % fetch for series whose manifest uid isn't uploaded on the + % server, and to warn on file_info uids that name unuploaded + % files. Empty (the default) preserves the previous behavior. + options.uploadedUidMap = [] end if ~document.has_files(), return, end updatedFileInfo = document.document_properties.files.file_info; + missingFileUidCount = 0; + firstMissingFileUid = ''; for i = 1:numel(updatedFileInfo) % Replace/override 1st file location updatedFileInfo(i).locations(1).delete_original = 0; @@ -76,9 +86,28 @@ fileLocation = sprintf('ndic://%s/%s', cloudDatasetId, fileUid); updatedFileInfo(i).locations(1).location = fileLocation; updatedFileInfo(i).locations(1).location_type = 'ndicloud'; + + % Optional integrity check: a file_info entry pointing at a uid + % the server does not yet report as uploaded produces a broken + % ndic:// location -- DID's later member/file open will fail + % obscurely. Count them now and warn once at the end of the doc + % rather than spamming per file. + if ~isUploaded(options.uploadedUidMap, fileUid) + missingFileUidCount = missingFileUidCount + 1; + if isempty(firstMissingFileUid) + firstMissingFileUid = char(fileUid); + end + end end document = document.setproperties('files.file_info', updatedFileInfo); + if missingFileUidCount > 0 + warning('NDI:cloud:sync:UnuploadedFileUid', ... + ['%d file_info entry/entries in document reference uids the ', ... + 'cloud does not report as uploaded (first: %s). A member/file ', ... + 'open will fail on those.'], missingFileUidCount, firstMissingFileUid); + end + % Rebuild series ingest_locations for any series that came back with % n_present > 0 but empty ingest_locations. Without this, DID's % MembersNotLocatable guard (DID-matlab#185) refuses the document on @@ -96,7 +125,8 @@ catch end seriesInfo = reconstructFromCloud(seriesInfo, updatedFileInfo, ... - cloudDatasetId, documentId, options.customFileHandler); + cloudDatasetId, documentId, options.customFileHandler, ... + options.uploadedUidMap, ndi.vintage.seriesMemberBase(document)); document = document.setproperties('files.series_info', seriesInfo); end end @@ -125,7 +155,7 @@ function seriesInfo = reconstructFromCloud(seriesInfo, fileInfo, ... - cloudDatasetId, documentId, customFileHandler) + cloudDatasetId, documentId, customFileHandler, uploadedUidMap, memberBase) % Fetch each qualifying series' manifest bytes into a scratch dir, % reconstruct ingest_locations from them via reconstructSeriesIngestLocations, % then delete the scratch dir. Manifests do NOT land in the DID file @@ -139,6 +169,13 @@ if ~isfolder(tmpDir), mkdir(tmpDir); end cleanup = onCleanup(@() safeRmdir(tmpDir)); + % Count qualifying series up front so the progress bar has a + % denominator. A doc with 10 pyramid levels has 10 series that each + % require one HTTPS round trip for the manifest, then one linear pass + % over the manifest's members; without a bar the whole "Updating + % document file info to reflect remote files." step looks hung on a + % large lightsheet dataset. + qualifyingIdx = []; for k = 1:numel(seriesInfo) entry = seriesInfo(k); if ~isfield(entry, 'n_present') || isempty(entry.n_present) || ... @@ -148,80 +185,66 @@ if isfield(entry, 'ingest_locations') && ~isempty(entry.ingest_locations) continue end + qualifyingIdx(end+1) = k; %#ok + end + + progressApp = []; + fetchBarId = ''; + if numel(qualifyingIdx) > 0 + try + progressApp = ndi.gui.component.ProgressBarWindow('NDI tasks'); + fetchBarId = did.ido.unique_id(); + progressApp.addBar( ... + 'Label', sprintf('Fetching %d series manifest(s) from cloud', ... + numel(qualifyingIdx)), ... + 'tag', fetchBarId, ... + 'Auto', true, ... + 'Timeout', minutes(Inf)); + catch + progressApp = []; + end + end + + for j = 1:numel(qualifyingIdx) + k = qualifyingIdx(j); + entry = seriesInfo(k); manifestUid = lookupManifestUid(fileInfo, entry.name); - if isempty(manifestUid), continue, end + if isempty(manifestUid) + tickBar(progressApp, fetchBarId, j / numel(qualifyingIdx)); + continue + end + + % Skip the fetch when the caller-provided uploaded-uid map says + % the manifest itself is not uploaded on the server. Attempting + % the fetch would only turn into a wasted HTTPS round trip whose + % 404/403 landed in the catch below; no reconstruction happens + % either way. When no map was passed, this is a no-op. + if ~isUploaded(uploadedUidMap, manifestUid) + tickBar(progressApp, fetchBarId, j / numel(qualifyingIdx)); + continue + end destPath = fullfile(tmpDir, manifestUid); try - fetchManifest(destPath, cloudDatasetId, string(manifestUid), ... - documentId, char(entry.name), customFileHandler); + ndi.cloud.sync.internal.fetchManifest( ... + string(destPath), string(cloudDatasetId), string(manifestUid), ... + string(documentId), string(entry.name), customFileHandler); catch % A manifest that cannot be fetched leaves the entry % un-reconstructed; DID#185's guard will then fire on % add_docs with the document's own identity, which is the % signal a partial download deserves. No warning here -- % add_docs is the right voice. - continue end + tickBar(progressApp, fetchBarId, j / numel(qualifyingIdx)); end seriesInfo = ndi.cloud.sync.internal.reconstructSeriesIngestLocations( ... - seriesInfo, fileInfo, tmpDir, cloudDatasetId); + seriesInfo, fileInfo, tmpDir, cloudDatasetId, 'MemberBase', memberBase); end -function fetchManifest(destPath, cloudDatasetId, manifestUid, documentId, ... - seriesName, customFileHandler) - % Fetch one manifest by uid. If a customFileHandler was passed in, - % offer the fetch through the DID contract -- same shape DID#201's - % fetchSeriesManifestBytes uses on the read side -- so tests can - % inject a mock and the two sides share their fetcher. Otherwise - % mint a signed URL and fetch directly, matching didsqlite.m's own - % download_file_from_cloud for a non-member ndic:// file. - % - % seriesName is deliberately '' in the context: this is the manifest - % itself, not a member. On the DID contract, a non-empty seriesName - % marks a MEMBER fetch, where sourcePath names the manifest and - % context.uid names the member -- the handler must then treat - % context.uid as the file to retrieve, not the manifest again. A - % manifest fetch has no such switch: sourcePath names the manifest - % and context.uid names the same manifest. - - sourcePath = sprintf('ndic://%s/%s', char(cloudDatasetId), char(manifestUid)); - - if ~isempty(customFileHandler) - ctx = struct( ... - 'documentId', char(documentId), ... - 'filename', char(seriesName), ... - 'seriesName', '', ... - 'uid', char(manifestUid), ... - 'mode', 'open'); - did.implementations.sqlitedb.dispatchCustomFileHandler( ... - customFileHandler, destPath, sourcePath, ctx); - return - end - - fileUrl = ndi.cloud.download.internal.batchSignedUrlLookup( ... - string(cloudDatasetId), documentId, "", manifestUid); - if strlength(fileUrl) == 0 - [success, answer] = ndi.cloud.api.files.getFileDetails( ... - char(cloudDatasetId), char(manifestUid)); - if ~success - error('NDI:cloud:sync:ManifestFetchFailed', ... - 'Failed to get file details for manifest %s: %s', ... - char(manifestUid), answer.message); - end - fileUrl = answer.downloadUrl; - end - [success2, answer2] = ndi.cloud.api.files.getFile( ... - char(fileUrl), char(destPath), 'useCurl', true); - if ~success2 - error('NDI:cloud:sync:ManifestFetchFailed', ... - 'Failed to download manifest %s from cloud: %s', ... - char(manifestUid), answer2); - end -end function uid = lookupManifestUid(fileInfo, seriesName) @@ -246,3 +269,37 @@ function safeRmdir(d) end end end + +function tf = isUploaded(map, uid) + % True when the caller passed no map (default open answer -- preserves + % the previous no-check behavior), or when the map says this uid is + % on the server as uploaded=true. False only when the map is present + % AND names this uid as not uploaded (or does not name it at all). + if isempty(map) + tf = true; + return + end + try + key = char(uid); + if ~isKey(map, key) + tf = false; + return + end + tf = logical(map(key)); + catch + tf = false; + end +end + +function tickBar(app, tag, progress) + % Guarded progress update. If the bar / window has been closed or + % culled, drop the update silently rather than crash the download + % pipeline. Same defensive contract list_binary_files uses. + if isempty(app) || ~isvalid(app) || isempty(tag) + return + end + try + app.updateBar(tag, progress); + catch + end +end diff --git a/src/ndi/+ndi/+cloud/+sync/+internal/uploadFilesForDatasetDocuments.m b/src/ndi/+ndi/+cloud/+sync/+internal/uploadFilesForDatasetDocuments.m index 006c667a54..59e2068b34 100644 --- a/src/ndi/+ndi/+cloud/+sync/+internal/uploadFilesForDatasetDocuments.m +++ b/src/ndi/+ndi/+cloud/+sync/+internal/uploadFilesForDatasetDocuments.m @@ -48,9 +48,25 @@ end if options.onlyMissing + if options.Verbose + fprintf('Checking cloud dataset %s for files already uploaded ...\n', cloudDatasetId); + end [file_manifest, message] = ndi.cloud.sync.internal.filesNotYetUploaded(file_manifest, cloudDatasetId); if ~isempty(message) success = false; + % Do not swallow the diagnostic. A caller that inspects only + % `success` still gets a printed line explaining what failed + % and where to look next, so the pipeline does not appear to + % succeed-silently-fail-silently. + if options.Verbose + fprintf(['ERROR while checking which files are already ', ... + 'in the cloud: %s\n', ... + 'This often means the active NDI_CLOUD_ORGANIZATION_ID ', ... + 'does not own dataset %s. Pass ndi.cloud.uploadDataset ', ... + 'the ''organizationName'' or ''organizationID'' the ', ... + 'dataset was created under. Current active org: %s.\n'], ... + message, cloudDatasetId, getenv('NDI_CLOUD_ORGANIZATION_ID')); + end return; end end @@ -61,6 +77,9 @@ if isempty(file_manifest) message = 'All files are already on the remote.'; + if options.Verbose + fprintf('%s\n', message); + end return; end @@ -71,24 +90,50 @@ app = ndi.gui.component.ProgressBarWindow('NDI tasks'); uuid = did.ido.unique_id(); app.addBar('Label','Uploading document-associated binary files','tag',uuid,'Auto',true); + files_uploaded_count = 0; + sentUids = strings(1, 0); for i=1:numel(file_manifest) if file_manifest(i).is_uploaded==false + % Wait for the server to record the uploads once, below, + % rather than after every file. [upload_success, upload_message] = ndi.cloud.uploadSingleFile(cloudDatasetId, ... - file_manifest(i).uid, file_manifest(i).file_path); + file_manifest(i).uid, file_manifest(i).file_path, 'waitForUpload', false); if ~upload_success if success message = ['Failed to upload file ' file_manifest(i).uid ': ' upload_message]; end success = false; + else + files_uploaded_count = files_uploaded_count + 1; + sentUids(end+1) = string(file_manifest(i).uid); %#ok end end app.updateBar(uuid,i/numel(file_manifest)); end + if ~isempty(sentUids) + [wait_success, wait_answer] = ndi.cloud.api.files.waitForFileUpload(cloudDatasetId, sentUids); + if ~wait_success + if success + message = sprintf(['%d of %d uploaded files were not recorded by NDI Cloud ' ... + 'within %.0f s (first: %s).'], numel(wait_answer.pending), numel(sentUids), ... + wait_answer.elapsed, wait_answer.pending(1)); + end + success = false; + end + end + if options.Verbose + fprintf('Uploaded %d of %d files.\n', files_uploaded_count, numel(file_manifest)); + end case "batch" [batch_success, batch_message] = ndi.cloud.upload.zipForUpload(ndiDataset, file_manifest, totalSizeKb, cloudDatasetId); if ~batch_success success = false; message = batch_message; + if options.Verbose + fprintf('Batch upload failed: %s\n', batch_message); + end + elseif options.Verbose + fprintf('Uploaded %d files in batch mode.\n', numel(file_manifest)); end end end diff --git a/src/ndi/+ndi/+cloud/+upload/zipForUpload.m b/src/ndi/+ndi/+cloud/+upload/zipForUpload.m index f567884573..7787da2169 100644 --- a/src/ndi/+ndi/+cloud/+upload/zipForUpload.m +++ b/src/ndi/+ndi/+cloud/+upload/zipForUpload.m @@ -38,8 +38,7 @@ files_left = numel(files_to_process); files_uploaded_count = 0; processed_bytes = 0; -base_dir = fullfile(D.path, '.ndi', 'files'); -skipped_files = {}; +base_dir = fullfile(D.path, '.ndi', 'files'); %#ok if options.Verbose fprintf('Beginning upload process. %d files to upload.\n', files_left); @@ -50,10 +49,12 @@ log_folder = ndi.common.PathConstants.LogFolder; if ~isfolder(log_folder), mkdir(log_folder); end - % Erase previous logs by opening in write mode 'wt' and writing headers - log_files_to_clear = {'zip_log.csv', 'processed_log.csv', 'skipped_log.csv'}; - % **Updated header for zip_log.csv** - headers = {'ZipFile,ZippedFile,UncompressedBytes', 'TotalProcessedBytes', 'SkippedFile,UID'}; + % Erase previous logs by opening in write mode 'wt' and writing headers. + % skipped_log.csv is no longer written: the pre-zip isfile() check was + % removed as redundant with list_binary_files' manifest, so there is no + % "missing at zip time" bookkeeping to log. + log_files_to_clear = {'zip_log.csv', 'processed_log.csv'}; + headers = {'ZipFile,ZippedFile,UncompressedBytes', 'TotalProcessedBytes'}; for k = 1:numel(log_files_to_clear) fid = fopen(fullfile(log_folder, log_files_to_clear{k}), 'wt'); @@ -67,77 +68,96 @@ end % --- Progress Bar Setup --- -h = waitbar(0, 'Preparing to upload files...'); -cleanupObj = onCleanup(@() delete(h(ishandle(h)))); -size_limit = options.SizeLimit; -file_processed = false(1, numel(files_to_process)); % Track processed files - -% --- Main Loop: Continue until all files are processed --- -while ~all(file_processed) - current_batch_size = 0; - files_for_current_batch = {}; - indices_for_current_batch = []; +% Standard NDI progress bar, same "NDI tasks" window the serial upload +% branch uses so bars stack in one place. Auto=true removes it at 1.0. +% Every touch is guarded with isvalid(): if the user closes the window +% or updateBar's own timeout culls the bar mid-run, we go silent rather +% than crash. See list_binary_files for the same pattern. +progressApp = ndi.gui.component.ProgressBarWindow('NDI tasks'); +uploadBarId = did.ido.unique_id(); +try + progressApp.addBar( ... + 'Label', sprintf('Uploading document-associated binary files (0 of %d)', files_left), ... + 'tag', uploadBarId, ... + 'Auto', true); +catch + progressApp = []; +end - % --- Inner Loop: Find files that fit into the current batch --- - for i = 1:numel(files_to_process) - if ~file_processed(i) - current_file = files_to_process(i); - file_path = current_file.file_path; - file_bytes = current_file.bytes; +size_limit = options.SizeLimit; +n_files = numel(files_to_process); +file_bytes_all = [files_to_process.bytes]; % scalar array, indexed below - % --- Logic Check: Ensure file exists on disk --- - if ~isfile(file_path) - if options.Verbose - warning('File %s (UID: %s) not found on disk. Skipping.', current_file.file_path, current_file.uid); - end - skipped_files{end+1} = current_file; - file_processed(i) = true; % Mark as processed to skip - continue; - end +% NOTE: no per-file isfile() check here. list_binary_files already +% resolved each entry through database_existbinarydoc + dir() when the +% manifest was built moments ago, so a redundant stat per file (four +% total per file with the zip/log dir() calls below) costs real time on +% a slow drive and adds nothing. If a file vanishes between manifest +% and zip time, MATLAB's zip() raises inside zipAndUploadBatch's +% try/catch and the batch is reported as failed rather than silently +% skipped -- the guard against issue #805 that isfile+skipped_files +% used to enforce. - % --- Batching Logic --- - if (current_batch_size + file_bytes <= size_limit) || isempty(files_for_current_batch) - files_for_current_batch{end+1} = file_path; - current_batch_size = current_batch_size + file_bytes; - indices_for_current_batch(end+1) = i; - if current_batch_size >= size_limit - break; - end - end +% --- Main Loop: Single pass through the file list -------------------- +% Cursor-based, greedy pack in the input order. Each file is examined +% exactly once across all batches, so the total work is O(N_files) +% instead of the O(N_batches * N_files) the previous restart-from-top +% design imposed -- at 160k files and batches of ~50 MB, that mattered. +% A file too large for the remaining budget of the current batch closes +% the batch and starts the next one at that file; a single file bigger +% than size_limit still goes up alone (the first file in a batch is +% always accepted). +cursor = 1; +files_processed_so_far = 0; +while cursor <= n_files + batch_start = cursor; + batch_end = cursor; % inclusive; grown below + current_batch_size = file_bytes_all(cursor); + cursor = cursor + 1; + while cursor <= n_files + fb = file_bytes_all(cursor); + if current_batch_size + fb > size_limit + break; + end + current_batch_size = current_batch_size + fb; + batch_end = cursor; + cursor = cursor + 1; + if current_batch_size >= size_limit + break; end end - % --- Upload the batch if it contains any files --- - if ~isempty(files_for_current_batch) - [success, batch_msg, uploaded_count] = zipAndUploadBatch(files_for_current_batch, dataset_id, options.numberRetries, options); - files_uploaded_count = files_uploaded_count + uploaded_count; + % Slice out just this batch's paths (no cell growth in a loop). + batch_indices = batch_start:batch_end; + files_for_current_batch = {files_to_process(batch_indices).file_path}; - if success - file_processed(indices_for_current_batch) = true; % Mark files as processed - else - b = 0; - files_not_uploaded = files_left - sum(file_processed); - msg = sprintf('%s\n%d files were successfully uploaded. %d files were not uploaded.', ... - batch_msg, files_uploaded_count, files_not_uploaded); - return; - end - else - % No files could be added to a batch, break to avoid infinite loop - % If there are still unprocessed files, it's an unexpected state - if ~all(file_processed) - msg = 'An unexpected error occurred: unable to process remaining files.'; - b = 0; - return; + [success, batch_msg, uploaded_count] = zipAndUploadBatch( ... + files_for_current_batch, dataset_id, options.numberRetries, options); + files_uploaded_count = files_uploaded_count + uploaded_count; + + if ~success + b = 0; + files_not_uploaded = files_left - files_processed_so_far; + msg = sprintf('%s\n%d files were successfully uploaded. %d files were not uploaded.', ... + batch_msg, files_uploaded_count, files_not_uploaded); + return; + end + + files_processed_so_far = files_processed_so_far + numel(batch_indices); + if ~isempty(progressApp) && isvalid(progressApp) + try + progressApp.updateBar(uploadBarId, files_processed_so_far / files_left); + catch + progressApp = []; end end +end - % --- Update Progress Bar --- +if ~isempty(progressApp) && isvalid(progressApp) try - progress = sum(file_processed) / files_left; - message = sprintf('Processed %d of %d files...', sum(file_processed), files_left); - waitbar(progress, h, message); + progressApp.updateBar(uploadBarId, 1); % Auto=true removes it catch - b = 0; msg = 'Upload cancelled by user.'; return; + % Nothing left to do; the batch loop is finished either way. end end @@ -152,34 +172,6 @@ fprintf(fid, '%d\n', processed_bytes); fclose(fid); end - - % Log skipped files - if ~isempty(skipped_files) - skipped_log_file = fullfile(log_folder, 'skipped_log.csv'); - fid = fopen(skipped_log_file, 'at'); % Append skipped files - if fid ~= -1 - for k = 1:numel(skipped_files) - fprintf(fid, '"%s","%s"\n', skipped_files{k}.file_path, skipped_files{k}.uid); - end - fclose(fid); - end - end -end - -% --- Report skipped files as a failure --- -% Files that were missing from disk are skipped during batching above. A -% skipped file means a binary that the manifest expected to upload did not -% get uploaded, so report this as a failure rather than silently succeeding. -% Otherwise the caller would record the associated documents as fully synced -% when their binaries are not on the remote (issue #805). -if ~isempty(skipped_files) - b = 0; - skipped_uids = cellfun(@(s) string(s.uid), skipped_files); - msg = sprintf(['%d file(s) were not uploaded because they were missing ', ... - 'from disk (UIDs: %s).'], numel(skipped_files), strjoin(skipped_uids, ', ')); - if options.Verbose - fprintf('%s\n', msg); - end end if options.Verbose diff --git a/src/ndi/+ndi/+cloud/uploadDataset.m b/src/ndi/+ndi/+cloud/uploadDataset.m index 24f1f4930c..5d7753eeae 100644 --- a/src/ndi/+ndi/+cloud/uploadDataset.m +++ b/src/ndi/+ndi/+cloud/uploadDataset.m @@ -33,8 +33,17 @@ % | | 'remoteDatasetName'. | % | 'remoteDatasetName' | (char) The name to be assigned to the dataset on the remote server. This | % | | is *required* if 'skipMetadataEditorMetadata' is true. | + % | 'organizationName' | (char/string) The NAME of the NDI Cloud organization the dataset should | + % | | be created under. The current user must belong to it. Names are matched | + % | | case-sensitively against the list ndi.cloud.api.users.me returns. | + % | | Mutually exclusive with 'organizationID'. When neither is given, the | + % | | active NDI_CLOUD_ORGANIZATION_ID env var is used (set by login to the | + % | | first organization; see ndi.cloud.ui.dialog.selectOrganization). | + % | 'organizationID' | (char/string) The ID of the NDI Cloud organization to use. Mutually | + % | | exclusive with 'organizationName'. | % - % See also: ndi.cloud.sync.SyncOptions, ndi.cloud.downloadDataset + % See also: ndi.cloud.sync.SyncOptions, ndi.cloud.downloadDataset, + % ndi.cloud.ui.dialog.selectOrganization % arguments @@ -43,10 +52,36 @@ options.uploadAsNew (1,1) logical = false options.skipMetadataEditorMetadata (1,1) logical = false options.remoteDatasetName (1,:) char = '' + options.organizationName (1,:) char = '' + options.organizationID (1,:) char = '' end syncOptions = ndi.cloud.sync.SyncOptions(syncOptions); + % Route the request to a specific organization when the caller named one. + % Restore the previous NDI_CLOUD_ORGANIZATION_ID on function exit so a + % later call in the same MATLAB session inherits neither our resolution + % nor a partial state after an early return / error. + if ~isempty(options.organizationName) || ~isempty(options.organizationID) + if ~isempty(options.organizationName) && ~isempty(options.organizationID) + success = false; cloudDatasetId = ''; + message = 'organizationName and organizationID are mutually exclusive.'; + return; + end + previousOrgId = getenv('NDI_CLOUD_ORGANIZATION_ID'); + orgRestore = onCleanup(@() setenv('NDI_CLOUD_ORGANIZATION_ID', previousOrgId)); %#ok + try + resolvedOrgId = ndi.cloud.internal.resolveOrganizationId( ... + 'organizationName', options.organizationName, ... + 'organizationID', options.organizationID); + catch resolveErr + success = false; cloudDatasetId = ''; + message = resolveErr.message; + return; + end + setenv('NDI_CLOUD_ORGANIZATION_ID', resolvedOrgId); + end + success = false; cloudDatasetId = ''; message = ''; @@ -96,16 +131,30 @@ if syncOptions.Verbose, disp('Examining documents for upload...'); end dataset_documents = ndiDataset.database_search( ndi.query('','isa','base') ); ndi.cloud.upload.uploadDocumentCollection(cloudDatasetId, dataset_documents, "onlyUploadMissing", true); + if syncOptions.Verbose, disp('Document upload complete.'); end % Step 3: Upload files + if syncOptions.Verbose + disp('Looking for files (walks each document''s series members; can be slow on a spinning drive) ...'); + end [success_upload, message_upload] = ndi.cloud.sync.internal.uploadFilesForDatasetDocuments( ... cloudDatasetId, ndiDataset, dataset_documents, ... "Verbose", syncOptions.Verbose, ... "FileUploadStrategy", syncOptions.FileUploadStrategy, "onlyMissing", true); if ~success_upload message = message_upload; + % Print the failure so a caller who did not capture the message + % output still sees why success came back false. Otherwise the + % previous lines all say "success" and the return value is 0 with + % no visible reason. + if syncOptions.Verbose + fprintf('ndi.cloud.uploadDataset FAILED: %s\n', message); + end return; end success = true; + if syncOptions.Verbose + fprintf('ndi.cloud.uploadDataset completed successfully. Cloud dataset id: %s\n', cloudDatasetId); + end end diff --git a/src/ndi/+ndi/+cloud/uploadSingleFile.m b/src/ndi/+ndi/+cloud/uploadSingleFile.m index 3699525fcf..ad711e1ab3 100644 --- a/src/ndi/+ndi/+cloud/uploadSingleFile.m +++ b/src/ndi/+ndi/+cloud/uploadSingleFile.m @@ -13,6 +13,19 @@ % Name-Value Pairs: % 'useBulkUpload' (logical) - If true, the file will be zipped and uploaded % using the bulk upload mechanism. Defaults to false. +% 'checksum' (logical) - If true (default), compute the file's CRC-32 +% (ndi.fun.file.crc32) and send it with the +% single-file upload URL request, so S3 rejects a +% corrupted upload and NDI Cloud records the +% file's crc32. Ignored for bulk uploads, whose +% zip entries carry their own CRC-32. +% 'waitForUpload' (logical) - If true (default), return only once NDI Cloud +% has recorded the upload (ndi.cloud.api.files. +% waitForFileUpload; for bulk uploads, the zip +% extraction job). A PUT returning 200 only +% means S3 has the object; the server records +% it shortly after. Set false when uploading +% many files and wait for them all at once. % % Outputs: % B - true if the upload was successful, false otherwise. @@ -24,6 +37,8 @@ filePath (1,1) string {mustBeFile} options.useBulkUpload (1,1) logical = false options.useCurl (1,1) logical = true + options.checksum (1,1) logical = true + options.waitForUpload (1,1) logical = true end errormsg = ''; @@ -42,9 +57,20 @@ [b_put, put_or_error] = ndi.cloud.api.files.putFiles(info_or_error.url, zip_file, ... 'useCurl', options.useCurl, ... - 'jobId', info_or_error.jobId); + 'jobId', info_or_error.jobId, ... + 'waitForCompletion', options.waitForUpload); if ~b_put - error(['Could not upload zip file: ' put_or_error.message]); + if isstruct(put_or_error) && isfield(put_or_error, 'message') + detail = put_or_error.message; + elseif isstruct(put_or_error) && isfield(put_or_error, 'state') + % waitForCompletion's final bulk-upload status + detail = ['bulk upload job ended in state ' char(string(put_or_error.state))]; + elseif ischar(put_or_error) || isstring(put_or_error) + detail = char(put_or_error); + else + detail = jsonencode(put_or_error); + end + error(['Could not upload zip file: ' detail]); end catch e @@ -56,7 +82,12 @@ if isfile(zip_file), delete(zip_file); end else % Get the pre-signed URL for single file upload - [b_url, url_or_error] = ndi.cloud.api.files.getFileUploadURL(cloudDatasetID, cloudFileUID); + checksumCRC32 = ""; + if options.checksum + checksumCRC32 = string(ndi.fun.file.crc32(filePath)); + end + [b_url, url_or_error] = ndi.cloud.api.files.getFileUploadURL(cloudDatasetID, cloudFileUID, ... + 'checksumCRC32', checksumCRC32); if ~b_url b = false; @@ -65,13 +96,24 @@ end % Upload the file - [b_put, put_or_error] = ndi.cloud.api.files.putFiles(url_or_error, filePath, 'useCurl', options.useCurl); + [b_put, put_or_error] = ndi.cloud.api.files.putFiles(url_or_error, filePath, 'useCurl', options.useCurl, ... + 'checksumCRC32', checksumCRC32); if ~b_put b = false; errormsg = ['Could not upload file: ' put_or_error.message]; return; end + + if options.waitForUpload + [b_wait, wait_answer] = ndi.cloud.api.files.waitForFileUpload(cloudDatasetID, cloudFileUID); + if ~b_wait + b = false; + errormsg = sprintf(['File was sent but NDI Cloud did not record the upload ' ... + 'within %.0f s (state: %s).'], wait_answer.elapsed, wait_answer.state); + return; + end + end end b = true; diff --git a/src/ndi/+ndi/+daq/+reader/+image/ndr.m b/src/ndi/+ndi/+daq/+reader/+image/ndr.m index 78c4c61062..b316eda7f4 100644 --- a/src/ndi/+ndi/+daq/+reader/+image/ndr.m +++ b/src/ndi/+ndi/+daq/+reader/+image/ndr.m @@ -131,7 +131,19 @@ end elseif nargin==2 & isa(varargin{1},'ndi.session') & isa(varargin{2},'ndi.document') obj.identifier = varargin{2}.document_properties.base.id; - obj.ndr_reader_string = varargin{2}.document_properties.daqreader_ndr.ndr_reader_string; + % BOTH VINTAGES, as in ndi.daq.reader.mfdaq.ndr: a V2 + % document carries the reader string as + % acquisition_reader.reader_string, not in a daqreader_ndr block. + [v, found] = ndi.vintage.field(varargin{2}, 'ndr_reader_string'); + if ~found + error('NDI:daqreaderimagendr:noReaderString', ... + ['document %s carries no reader string under ' ... + 'either spelling (v1 ' ... + 'daqreader_ndr.ndr_reader_string, V2 ' ... + 'acquisition_reader.reader_string)'], ... + varargin{2}.document_properties.base.id); + end + obj.ndr_reader_string = v; finished = 1; else error('Unknown arguments.'); diff --git a/src/ndi/+ndi/+daq/system.m b/src/ndi/+ndi/+daq/system.m index b1b7b23a41..ad74f2391e 100755 --- a/src/ndi/+ndi/+daq/system.m +++ b/src/ndi/+ndi/+daq/system.m @@ -35,7 +35,8 @@ % EDGE NAMES MOVE, TARGET IDS DO NOT. V_eta renames these % three edges (daqreader_id -> reader_id, filenavigator_id -> % epoch_file_pattern_id, daqmetadatareader_id_# -> - % acquisition_metadata_reader_#) while every migrator in the + % acquisition_metadata_reader_#, later epoch_parameter_reader_id + % repeated) while every migrator in the % family preserves base.id on what it emits -- so the lookups % below by `base.id` are unchanged and only the names need % translating. ndi.vintage.edge is a no-op on a v1 document. @@ -66,7 +67,9 @@ obj.daqreader = ndi.database.fun.ndi_document2ndi_object(daqreader_doc, session); obj.filenavigator = ndi.database.fun.ndi_document2ndi_object(filenavigator_doc,session); - obj.name = daqsystem_doc.document_properties.base.name; + % v1 and older V_eta: base.name. did-schema #73 item 54: + % `acquisition_system.name`. ndi.vintage.objectName reads both. + obj.name = ndi.vintage.objectName(daqsystem_doc); obj.identifier = daqsystem_doc.document_properties.base.id; obj = obj.set_daqmetadatareader(thedaqmetadatareader); else diff --git a/src/ndi/+ndi/+database/+fun/externalFileLocation.m b/src/ndi/+ndi/+database/+fun/externalFileLocation.m new file mode 100644 index 0000000000..d377b57011 --- /dev/null +++ b/src/ndi/+ndi/+database/+fun/externalFileLocation.m @@ -0,0 +1,64 @@ +function [tf, location] = externalFileLocation(document_properties, filename) +%EXTERNALFILELOCATION Where a document's NOT-ingested file lives, if it says. +% +% [TF, LOCATION] = ndi.database.fun.externalFileLocation(PROPS, FILENAME) +% reads PROPS.files.file_info -- the per-file record ndi.document/add_file +% writes -- for the entry named FILENAME, and returns the first of its +% `locations` that is a local file (`location_type` 'file' or empty) NOT +% taken into the database (`ingest` false) and that exists on this +% computer. TF is false (LOCATION '') when there is no such location. +% +% This is how a document points at bytes it does not hold: a V2 +% `opaque_body` written at import names its recording this way, and the +% bytes stay where they are until ingestion (Haley import decision #34). +% A location that is recorded but missing on this computer gives TF false, +% never an error: a caller asks this speculatively. +% +% See also ndi.document/add_file, +% ndi.database.implementations.database.did2sqlite. + +tf = false; +location = ''; +if ~isfield(document_properties, 'files') || ~isstruct(document_properties.files) ... + || ~isfield(document_properties.files, 'file_info') + return; +end +info = asCell(document_properties.files.file_info); +for i = 1:numel(info) + fi = info{i}; + if ~isstruct(fi) || ~isfield(fi, 'name') || ~strcmp(char(fi.name), filename) ... + || ~isfield(fi, 'locations') + continue; + end + locs = asCell(fi.locations); + for j = 1:numel(locs) + l = locs{j}; + if ~isstruct(l) || ~isfield(l, 'location') || isempty(l.location) + continue; + end + if isfield(l, 'ingest') && ~isempty(l.ingest) && logical(l.ingest(1)) + continue; % held (or to be held) by the database, not read in place + end + if isfield(l, 'location_type') && ~isempty(l.location_type) ... + && ~strcmpi(char(l.location_type), 'file') + continue; % a url / cloud location is not a local path + end + p = char(l.location); + if isfile(p) + tf = true; + location = p; + return; + end + end +end +end + +function c = asCell(x) +if iscell(x) + c = x; +elseif isstruct(x) + c = num2cell(x); +else + c = {}; +end +end diff --git a/src/ndi/+ndi/+database/+fun/extract_docs_files.m b/src/ndi/+ndi/+database/+fun/extract_docs_files.m index a272546df3..c594dd7e34 100644 --- a/src/ndi/+ndi/+database/+fun/extract_docs_files.m +++ b/src/ndi/+ndi/+database/+fun/extract_docs_files.m @@ -49,7 +49,8 @@ % current_file_list does not cover: it returns the manifest name only, so % the members are enumerated from the series record instead and fetched by % their NAME_ names, which resolve through the manifest - % (VH-Lab/DID-matlab#173, #183). + % (VH-Lab/DID-matlab#173, #183). The first member is NAME_1 on a v1 + % document and NAME_0 on a V_eta one (ndi.vintage.seriesMemberBase). % % Each member is recorded in the extracted document's ingest_locations % UNDER ITS ORIGINAL UID. That is what lets the copy be stored somewhere @@ -201,6 +202,14 @@ % member is the source's own file and must never be deleted. seriesMemberFiles = repmat({{}},1,numel(seriesInfo)); + % The NUMBER of slot m depends on the document's vintage: a v1 + % document's members are NAME_1..NAME_N, a V_eta document's + % NAME_0..NAME_ (did.document/seriesMemberBase; a + % migrated lightsheet level is body_data_0..). Asking a V_eta + % document for NAME_1..NAME_N would skip its first member and + % ask for one past its last. + memberBase = ndi.vintage.seriesMemberBase(d{i}); + for s = 1:numel(seriesInfo) seriesName = seriesInfo(s).name; slotCount = 0; @@ -222,7 +231,8 @@ copiedCount = 0; for m = 1:slotCount - memberName = sprintf('%s_%d',seriesName,m); + memberNumber = m - 1 + memberBase; + memberName = sprintf('%s_%d',seriesName,memberNumber); % database_existbinarydoc reports the member's LOCAL % path (via the manifest) without retrieving anything, % so a miss here is a member whose bytes are not on @@ -271,7 +281,10 @@ % delete_original 0: whether this is our copy in % TARGET_PATH or a reference to the source, the caller % was promised the files would still be there. - memberEntries(memberCount) = struct('index',m, ... + % `index` is the number AS WRITTEN IN THE NAME, + % which is how DID turns an ingest record back into + % NAME_ (did.implementations.sqlitedb). + memberEntries(memberCount) = struct('index',memberNumber, ... 'uid',memberUid,'location',memberLocation, ... 'location_type','file','ingest',1, ... 'delete_original',0,'parameters',''); diff --git a/src/ndi/+ndi/+database/+implementations/+database/did2sqlite.m b/src/ndi/+ndi/+database/+implementations/+database/did2sqlite.m index 9e68aa9eff..ef0ef5f9d6 100644 --- a/src/ndi/+ndi/+database/+implementations/+database/did2sqlite.m +++ b/src/ndi/+ndi/+database/+implementations/+database/did2sqlite.m @@ -30,14 +30,17 @@ % Document add / read / remove / search / alldocids are implemented by % delegating to did2.database.sqlitedb. % - % BINARY (file) documents are NOT, and they error rather than - % returning something. `did2.database.sqlitedb` has no file store at - % all -- it persists the document body and the two query sidecars and - % nothing else -- so there is no path from a document id and a - % filename to bytes to hand back. Returning empty here would make a - % dataset with attached files read as a dataset with none, which is - % the hollow-document failure this project has paid for repeatedly. - % An error names the gap instead. + % BINARY (file) documents: a file the document INGESTED is read from + % the database's file store, <.ndi>/files/ (did2.database.sqlitedb + % hasFile, DID-matlab #215: the same layout as the legacy backend, so + % file_directory is unchanged). A file the document records BY + % LOCATION without ingesting it (files.file_info.locations, ingest 0; + % ndi.database.fun.externalFileLocation) is read where it is. Anything + % else errors rather than returning something: an empty answer would + % make a dataset with attached files read as a dataset with none, the + % hollow-document failure this project has paid for repeatedly. With a + % DID-matlab whose did2 database has no file store yet, only the + % by-location path exists. % % See also: ndi.database, did2.database.sqlitedb, % ndi.database.implementations.database.didsqlite, @@ -170,12 +173,25 @@ end ss = []; + D = 'ndi.database.implementations.database.did2sqlite'; for i = 1:numel(q) sHere = q(i).searchstructure; for j = 1:numel(sHere) try + p1 = sHere(j).param1; + p2 = sHere(j).param2; + op = char(sHere(j).operation); + if strcmp(op, 'or') + % each branch is a searchstructure array: convert it + % the same way (a nested query may sit inside) + p1 = feval([D '.toDid2Query'], did.query(p1)).searchstructure; + p2 = feval([D '.toDid2Query'], did.query(p2)).searchstructure; + elseif any(strcmp(op, {'depends_on', '~depends_on'})) && isa(p2, 'did.query') + % the edge's target is itself a query (did2 resolves it) + p2 = feval([D '.toDid2Query'], p2); + end term = did2.query.searchstruct(sHere(j).field, ... - sHere(j).operation, sHere(j).param1, sHere(j).param2); + sHere(j).operation, p1, p2); catch ME error('NDI:did2sqlite:unsupportedOperation', ... ['This database is backed by did2, which will ' ... @@ -245,44 +261,82 @@ function [ndi_document_objs] = do_search(obj, searchoptions, searchparams) %#ok q2 = ndi.database.implementations.database.did2sqlite.toDid2Query( ... searchparams); - doc_ids = obj.db.searchIds(q2); + % The search already decodes every matching document; each is + % normalised from there rather than fetched and decoded a second + % time by id (do_read), which cost a query and a decode per + % document (8 s of 51 s profiled over the 10,536 Haley V2 + % subjects). + docs = obj.db.search(q2); ndi_document_objs = {}; - for i = 1:numel(doc_ids) - ndi_document_objs{i} = obj.do_read(doc_ids{i}); %#ok + for i = 1:numel(docs) + ndi_document_objs{i} = ... + ndi.database.internal.applyReadNormalization(docs{i}); %#ok end end % do_search() - function [ndi_binarydoc_obj] = do_openbinarydoc(obj, ndi_document_id, filename) %#ok + function [ndi_binarydoc_obj] = do_openbinarydoc(obj, ndi_document_id, filename) + % An ingested file (the database's file store) or one the + % document records BY LOCATION is opened read-only where it is. + % Anything else errors: an empty answer would make a dataset + % with attached files read as a dataset with none. + [tf, file_path] = obj.check_exist_binarydoc(ndi_document_id, filename); + if tf + ndi_binarydoc_obj = did.file.readonly_fileobj('fullpathfilename', file_path); + ndi_binarydoc_obj.fopen(); + return; + end error('NDI:did2sqlite:noBinaryStore', ... - ['This database is backed by did2.database.sqlitedb, which ' ... - 'stores document bodies and the query sidecars and has NO ' ... - 'file store, so there is no way to produce the bytes of ' ... - '"%s" on document %s. This errors rather than returning ' ... - 'empty: a dataset with attached files must not read as a ' ... - 'dataset with none.'], filename, ndi_document_id); + ['Document %s has no file "%s" on this computer: it is neither ' ... + 'in this database''s file store nor at a location the document ' ... + 'records (files.file_info.locations). This errors rather than ' ... + 'returning empty: a dataset with attached files must not read ' ... + 'as a dataset with none.'], ndi_document_id, filename); end % do_openbinarydoc() - function [tf, file_path] = check_exist_binarydoc(obj, ndi_document_id, filename) %#ok - % A did2 database has no file store, so the honest answer is - % "no" with no path -- and unlike do_openbinarydoc this one is - % ASKED speculatively by callers deciding whether to read, so a - % false is a real answer rather than a swallowed failure. + function [tf, file_path] = check_exist_binarydoc(obj, ndi_document_id, filename) + % Asked speculatively by callers deciding whether to read, so + % "no" is a real answer: true for a file in the database's file + % store (ingested), or one the document records at a location + % that exists here (ndi.database.fun.externalFileLocation). tf = false; file_path = ''; + if ismethod(obj.db, 'hasFile') % DID-matlab #215 + try + [tf, file_path] = obj.db.hasFile(ndi_document_id, filename); + catch + tf = false; + end + if tf + return; + end + file_path = ''; + end + try + raw = obj.db.get(ndi_document_id); + catch + return; + end + if isa(raw, 'did2.document') + props = raw.documentProperties; + else + props = raw; + end + [tf, file_path] = ndi.database.fun.externalFileLocation(props, filename); end % check_exist_binarydoc() function [ndi_binarydoc_matfid_obj] = do_closebinarydoc(obj, ndi_binarydoc_matfid_obj) %#ok - % Nothing can have been opened (do_openbinarydoc always - % errors), so there is nothing to close. + if ~isempty(ndi_binarydoc_matfid_obj) + ndi_binarydoc_matfid_obj.fclose(); + end end % do_closebinarydoc() function [file_dir] = file_directory(obj) - % FILE_DIRECTORY - where ingested files would live + % FILE_DIRECTORY - where ingested files live % % Same layout as the legacy backend so a session migrated in % place keeps its existing `files/` directory rather than - % growing a second one. Nothing in this class reads or writes - % it yet -- see do_openbinarydoc. + % growing a second one. did2.database.sqlitedb ingests into + % /files, which is this folder. file_dir = fullfile(obj.path, 'files'); end % file_directory diff --git a/src/ndi/+ndi/+database/+implementations/+database/didsqlite.m b/src/ndi/+ndi/+database/+implementations/+database/didsqlite.m index 5aaede8834..0dc7ea3f4a 100644 --- a/src/ndi/+ndi/+database/+implementations/+database/didsqlite.m +++ b/src/ndi/+ndi/+database/+implementations/+database/didsqlite.m @@ -114,6 +114,7 @@ end ndi_document_objs = {}; + searchparams = v1SubjectSearch(searchparams); [doc_ids] = ndi_didsqlite_obj.db.search(searchparams,'a'); ndi_document_objs = {}; for i=1:numel(doc_ids) @@ -226,8 +227,35 @@ function download_file_from_cloud(destPath, sourcePath, context) if strlength(seriesName) > 0 && strlength(contextUid) > 0 ndiFileUid = char(contextUid); end + % A single-file fetch (a series manifest, or any doc-level + % attachment) arrives here with seriesName="" -- there is no + % member being asked for. Going through the per-document batch + % scope to answer one uid is pure loss: for a lightsheet-scale + % pyramid document that scope names 15k+ files, so the batch + % endpoint spends 60-90 s signing a set the caller has no use + % for, delaying the ONE URL the download actually needs by more + % than a minute. Same reasoning ndi.cloud.sync.internal + % .fetchManifest uses for the internal manifest fetch; the DID + % handler path has to make the same choice or the pyramid + % pathology reappears whenever DID reads a series member (DID + % fetches the manifest via the handler with seriesName="" before + % it fetches any members). See Waltham-Data-Science/ + % NDI-python#320 and NDI-matlab#1010. + if strlength(seriesName) == 0 + batchDocId = ""; + else + batchDocId = docId; + end + % diskCache=true because DID is the production reopen-a-dataset + % path: a scientist who reopens the same lightsheet the next + % morning must not pay the ~85 min signed-URL-set job again. + % Signals lookup only for a scoped call (empty documentId + % dispatches to the batch's early return and no disk read + % happens). See ndi.cloud.download.internal.signedUrlDiskCache. + fromDiskCache = strlength(batchDocId) > 0; fileUrl = ndi.cloud.download.internal.batchSignedUrlLookup( ... - string(cloudDatasetId), docId, seriesName, string(ndiFileUid)); + string(cloudDatasetId), batchDocId, seriesName, string(ndiFileUid), ... + 'diskCache', fromDiskCache); if strlength(fileUrl) == 0 % No batch URL for this uid -- either no document context, @@ -239,9 +267,23 @@ function download_file_from_cloud(destPath, sourcePath, context) error(['Failed to get file details: ' answer.message]); end fileUrl = answer.downloadUrl; + fromDiskCache = false; % per-uid mint; nothing to invalidate. end [success2, answer2] = ndi.cloud.api.files.getFile(char(fileUrl), destPath, 'useCurl', true); if ~success2 + % An S3 403 on a URL that WAS served from the disk cache + % says the cached scope has gone stale (token revoked, or + % the object rotated). Drop the scope so the next read + % refetches -- otherwise we'd 403 our way through every + % subsequent uid in the same scope. Mirrors the in-memory + % scope-forget the batch would do on refresh. Best-effort; + % a failed forget is not fatal to the error we're raising. + if fromDiskCache && localIsS3Forbidden(answer2) + try %#ok + ndi.cloud.download.internal.signedUrlDiskCache.forget( ... + string(cloudDatasetId), docId, seriesName); + end + end error(['Failed to download file from cloud: ' answer2]); end else @@ -272,3 +314,70 @@ function download_file_from_cloud(destPath, sourcePath, context) end end end + +function tf = localIsS3Forbidden(answer) + % S3 returns 403 on a rotated/revoked signed URL. curl -f surfaces + % that as an exit-22 message like + % curl: (22) The requested URL returned error: 403 + % (Or "403 Forbidden" on some curl builds.) websave's message + % contains "403" too when it makes it that far. Match \b403\b. + tf = false; + if isempty(answer) + return + end + txt = ''; + if ischar(answer) + txt = answer; + elseif isstring(answer) && isscalar(answer) + txt = char(answer); + elseif isstruct(answer) && isfield(answer, 'message') && ~isempty(answer.message) + txt = char(string(answer.message)); + end + if isempty(txt) + return + end + tf = ~isempty(regexp(txt, '\<403\>', 'once')); +end + +function q = v1SubjectSearch(q) + % V1SUBJECTSEARCH - put back plain `isa subject` on a v1 database + % + % ndi.query('', 'isa', 'subject') leaves out subjects whose subject.type + % is not biological (ndi.query.subjectSearch). This database searches in + % SQL, where '~exact_string' needs the field to exist, so it would also + % leave out every v1 subject, none of which has a subject.type. A v1 + % subject always was a biological subject, so here the plain search is + % the right one. V2 sessions are did2sqlite and keep the rewrite. + ss = q.searchstructure; + [ss, changed] = swapSubjectSearch(ss, ndi.query.subjectSearch(), ... + did.query('', 'isa', 'subject').searchstructure); + if changed + q = did.query(ss); + end +end + +function [ss, changed] = swapSubjectSearch(ss, rewritten, plain) + changed = false; + if ~isstruct(ss) + return + end + if isequal(ss, rewritten) + ss = plain; + changed = true; + return + end + for k = 1:numel(ss) + if isequal(ss(k), rewritten) + ss(k) = plain; + changed = true; + elseif strcmpi(strrep(ss(k).operation, '~', ''), 'or') + [p1, c1] = swapSubjectSearch(ss(k).param1, rewritten, plain); + [p2, c2] = swapSubjectSearch(ss(k).param2, rewritten, plain); + if c1 || c2 + ss(k).param1 = p1; + ss(k).param2 = p2; + changed = true; + end + end + end +end diff --git a/src/ndi/+ndi/+database/+internal/applyReadNormalization.m b/src/ndi/+ndi/+database/+internal/applyReadNormalization.m index 3f6356c55f..bf89313c97 100644 --- a/src/ndi/+ndi/+database/+internal/applyReadNormalization.m +++ b/src/ndi/+ndi/+database/+internal/applyReadNormalization.m @@ -108,8 +108,30 @@ % BEYOND this target, and isAlreadyTarget now compares rank rather than % string equality, so they pass through untouched instead of being pushed % back through migrators aimed at a version they have already passed. + % ALREADY AT THE TARGET (every V2 document): with validation off, the + % full pass below reduces to exactly did2.convert.ensureClassBlocks (the + % base-field rename does nothing at this target, and wrapping in a + % did2.document and unwrapping is the identity), so apply that alone + % instead of the per-document batch machinery: 1.2 s of 2.4 s listing + % the 10,536 Haley V2 subjects. DID-matlab's testV1ToV2Audits pins that + % the two give the same document; only when that DID-matlab is present + % (#218), otherwise the full pass runs as before. + if hasReadShortcut() && did2.convert.isAlreadyTarget(body, READ_TARGET_VERSION) + ndiDocumentObj = ndi.document(did2.convert.ensureClassBlocks(body, [])); + return; + end + + % The report-only census (silentLoss, fileList, timeReferenceFamilies) + % is skipped: this read discards it, and on one-document reads it cost + % more than the conversion (23.5 s of 51 s profiled over the 10,536 + % Haley V2 subjects). Only when DID-matlab's v1_to_v2 has the option + % (DID-matlab #218); an older one runs it as before. + auditArgs = {}; + if v1ToV2HasAudits() + auditArgs = {'Audits', false}; + end result = did2.convert.v1_to_v2(body, 'Validate', false, ... - 'RenameClassNames', false, 'TargetVersion', READ_TARGET_VERSION); + 'RenameClassNames', false, 'TargetVersion', READ_TARGET_VERSION, auditArgs{:}); if isempty(result.migrated) if ~isempty(result.quarantine) @@ -135,3 +157,31 @@ % would require. v = 'V_delta'; end + +function tf = v1ToV2HasAudits() +%V1TOV2HASAUDITS True when did2.convert.v1_to_v2 takes the 'Audits' option. +% Read once from its source and remembered; drop this test once every +% supported DID-matlab has the option (DID-matlab #218). +persistent has +if isempty(has) + has = false; + try + f = which('did2.convert.v1_to_v2'); + has = ~isempty(f) && contains(fileread(f), 'options.Audits'); + catch + end +end +tf = has; +end + +function tf = hasReadShortcut() +%HASREADSHORTCUT True when DID-matlab has did2.convert.isAlreadyTarget and +% ensureClassBlocks as public functions (DID-matlab #218). Remembered; +% drop this test once every supported DID-matlab has them. +persistent has +if isempty(has) + has = ~isempty(which('did2.convert.isAlreadyTarget')) && ... + ~isempty(which('did2.convert.ensureClassBlocks')); +end +tf = has; +end diff --git a/src/ndi/+ndi/+database/+internal/list_binary_files.m b/src/ndi/+ndi/+database/+internal/list_binary_files.m index 28d39d1bd8..e43650fd23 100644 --- a/src/ndi/+ndi/+database/+internal/list_binary_files.m +++ b/src/ndi/+ndi/+database/+internal/list_binary_files.m @@ -34,8 +34,9 @@ % NEW-STYLE SERIES (files.file_series) still need the separate pass % below: addFileSeries calls add_file once, for the series MANIFEST, so % file_info holds the manifest's name and not the members'. Those come - % from the per-series record, and are one-based by DID's own rule - % (VH-Lab/DID-matlab#183). + % from the per-series record, numbered by DID's own rule: from 1 on a v1 + % document (VH-Lab/DID-matlab#183), from 0 on a V_eta one + % (did.document/seriesMemberBase; ndi.vintage.seriesMemberBase). arguments ndi_dataset (1,1) ndi.dataset @@ -53,11 +54,36 @@ % keep it open until finished. [db_cleanup_obj, ~] = ndi_dataset.open_database(); %#ok - for i = 1:num_documents + % Standard NDI progress bar for the top-level document walk. A large + % lightsheet dataset's inventory pass is minutes of work on a spinning + % drive; without a bar, the whole pre-upload phase looks hung. Same + % "NDI tasks" window the serial upload branch uses, so bars stack in + % one place. Auto=true removes the bar when it reaches 1.0. + % + % The progressApp handle is captured in a nested state variable so the + % safe* helpers below can null it out if the window is closed / deleted + % mid-run (e.g. the user Xs it out, or updateBar's own checkTimeout + % culls a bar and then autoDelete tears the window down with it). + % Every touch of the bar goes through those helpers -- see the failure + % history for what happened before: updateBar's side-effect timeout + % check culled the top bar during a >1-min series scan, autoDelete + % then removed the window because no bars were left, and the next + % addBar crashed on a stale handle. See NDI-matlab lightsheet demo, + % 2026-09-20. + progressApp = []; + topBarId = ''; + if verbose + progressApp = ndi.gui.component.ProgressBarWindow('NDI tasks'); + topBarId = did.ido.unique_id(); + % Timeout=minutes(Inf): the top bar does not tick as often as the + % app-level 1-minute timeout, and we do not want it culled by a + % global sweep triggered by an unrelated bar's update. The nested + % renew below is belt-and-suspenders progress feedback. + safeAddBar(sprintf('Scanning documents for files (%d docs)', num_documents), ... + topBarId, minutes(Inf)); + end - if verbose && (mod(i, 1000)==0 || i == num_documents) - fprintf('Working on document %d of %d\n', i, num_documents) - end + for i = 1:num_documents ndi_document_id = database_documents{i}.document_properties.base.id; @@ -96,6 +122,10 @@ seriesInfo = database_documents{i}.document_properties.files.series_info; end + % NAME_1.. on a v1 document, NAME_0.. on a V_eta one (a migrated + % lightsheet level's body_data_0..): ndi.vintage.seriesMemberBase. + memberBase = ndi.vintage.seriesMemberBase(database_documents{i}); + for s = 1:numel(seriesInfo) slotCount = 0; if isfield(seriesInfo(s), 'count') && ~isempty(seriesInfo(s).count) @@ -110,8 +140,24 @@ 'docid', '', 'bytes', 0, 'file_path', ''), 1, slotCount); member_count = 0; + % Nested progress bar per large series, so a single doc + % whose scan takes minutes shows its own progress rather + % than freezing the top-level bar. Small series stay + % silent. Time-throttled updates (~5 s) keep the print + % itself from becoming the bottleneck at 100+ stat/s. + seriesBarId = ''; + hasSeriesBar = false; + progressInterval = 5; + lastReport = tic; + if verbose && slotCount >= 1000 + seriesBarId = did.ido.unique_id(); + safeAddBar(sprintf(' doc %d/%d: series ''%s'' (%d slots)', ... + i, num_documents, series_name, slotCount), seriesBarId); + hasSeriesBar = ~isempty(progressApp); + end + for m = 1:slotCount - member_name = sprintf('%s_%d', series_name, m); + member_name = sprintf('%s_%d', series_name, m - 1 + memberBase); [member_exists, member_path] = ... ndi_dataset.database_existbinarydoc(ndi_document_id, member_name); % An absent slot is normal: a sparse series is the case @@ -126,10 +172,69 @@ 'name', member_name, 'uid', member_uid, ... 'docid', ndi_document_id, 'bytes', member_info.bytes, ... 'file_path', member_path); + + if hasSeriesBar && toc(lastReport) >= progressInterval + safeUpdateBar(seriesBarId, m/slotCount); + % Also tick the top bar so it does not sit idle + % past ProgressBarWindow's 1-minute timeout during + % a long series scan. Fractional progress is fine: + % (i-1)/N + (m/slotCount)/N is monotone, and the + % update at end-of-doc pins it to i/N below. + safeUpdateBar(topBarId, ((i-1) + m/slotCount) / num_documents); + lastReport = tic; + end + end + + if hasSeriesBar + safeUpdateBar(seriesBarId, 1); % Auto=true removes it end file_manifest = [file_manifest member_entries(1:member_count)]; %#ok end end + + if verbose + safeUpdateBar(topBarId, i/num_documents); + end + end + + % --- Nested helpers ------------------------------------------------- + % Every touch of the progress window goes through these so a closed / + % deleted window does not crash the inventory pass. progressApp is a + % handle stored in the enclosing scope; when a call raises because the + % window was torn down, we null it out and go silent for the rest of + % the run rather than firing a warning per iteration. + + function safeAddBar(labelText, tag, timeoutOverride) + if nargin < 3 + timeoutOverride = duration.empty; % inherit app-level timeout + end + if isempty(progressApp) || ~isvalid(progressApp) + progressApp = []; + return + end + try + progressApp.addBar('Label', labelText, 'tag', tag, 'Auto', true, ... + 'Timeout', timeoutOverride); + catch err + % One warning, then quiet. A user who closed the window does + % not want us re-opening it or spamming diagnostics. + warning('NDI:list_binary_files:ProgressBarUnavailable', ... + 'Progress bar window is no longer available (%s); continuing silently.', ... + err.message); + progressApp = []; + end + end + + function safeUpdateBar(tag, progress) + if isempty(progressApp) || ~isvalid(progressApp) || isempty(tag) + return + end + try + progressApp.updateBar(tag, progress); + catch + % Bar or window is gone; drop it and stop touching it. + progressApp = []; + end end end diff --git a/src/ndi/+ndi/+database/+metadata_app/+fun/getCCByLicences.m b/src/ndi/+ndi/+database/+metadata_app/+fun/getCCByLicences.m index 0000b3766d..260d6e7ec4 100644 --- a/src/ndi/+ndi/+database/+metadata_app/+fun/getCCByLicences.m +++ b/src/ndi/+ndi/+database/+metadata_app/+fun/getCCByLicences.m @@ -12,9 +12,10 @@ isCcBy = contains(instanceTable.InstanceName, 'CC-BY'); names = instanceTable.InstanceName(isCcBy); + filePaths = instanceTable.Filepath(isCcBy); shortNames = repmat("", size(names)); for i = 1:numel(names) - S = openminds.internal.getControlledInstance(names{i}, 'license', 'core'); + S = openminds.internal.utility.json.decode(fileread(filePaths(i))); shortNames(i) = S.shortName; end end diff --git a/src/ndi/+ndi/+database/+metadata_app/+fun/getOpenMindsInstances.m b/src/ndi/+ndi/+database/+metadata_app/+fun/getOpenMindsInstances.m index 9565df75b7..6cf11eb583 100644 --- a/src/ndi/+ndi/+database/+metadata_app/+fun/getOpenMindsInstances.m +++ b/src/ndi/+ndi/+database/+metadata_app/+fun/getOpenMindsInstances.m @@ -20,7 +20,7 @@ labels = repmat("", size(names)); for i = 1:numel(names) - S = openminds.internal.getControlledInstance(names{i}, schemaName, 'controlledTerms'); + S = openminds.internal.utility.json.decode(fileread(instanceTable.Filepath(i))); labels(i) = S.name; end diff --git a/src/ndi/+ndi/+database/+metadata_app/+fun/loadOpenMindsInstanceCatalog.m b/src/ndi/+ndi/+database/+metadata_app/+fun/loadOpenMindsInstanceCatalog.m index a09ccdda1b..7339bdbe86 100644 --- a/src/ndi/+ndi/+database/+metadata_app/+fun/loadOpenMindsInstanceCatalog.m +++ b/src/ndi/+ndi/+database/+metadata_app/+fun/loadOpenMindsInstanceCatalog.m @@ -23,7 +23,7 @@ catalog.NameField = options.PrimaryField; for i = 1:numel(names) - S = openminds.internal.getControlledInstance(names{i}, openMindsType, 'controlledTerms'); + S = openminds.internal.utility.json.decode(fileread(instanceTable.Filepath(i))); catalog.add(S) end end diff --git a/src/ndi/+ndi/+database/+metadata_ds_core/convertFormDataToDocuments.m b/src/ndi/+ndi/+database/+metadata_ds_core/convertFormDataToDocuments.m index feb09a2ecd..738bd9e540 100644 --- a/src/ndi/+ndi/+database/+metadata_ds_core/convertFormDataToDocuments.m +++ b/src/ndi/+ndi/+database/+metadata_ds_core/convertFormDataToDocuments.m @@ -111,8 +111,7 @@ if isfield( appUserData, 'License') if appUserData.License ~= "" - S = openminds.internal.getControlledInstance( appUserData.License, 'License', 'core'); - datasetVersion.license = openminds.core.License().fromStruct(S); + datasetVersion.license = openminds.core.License.fromName(appUserData.License); end end diff --git a/src/ndi/+ndi/+dataset/dir.m b/src/ndi/+ndi/+dataset/dir.m index 15987459b0..e50fba997b 100644 --- a/src/ndi/+ndi/+dataset/dir.m +++ b/src/ndi/+ndi/+dataset/dir.m @@ -43,28 +43,42 @@ % Use the session.dir's path as the path for this object ndi_dataset_dir_obj.path = ndi_dataset_dir_obj.session.path; - dataset_session_info_docs = ndi_dataset_dir_obj.database_search(ndi.query('','isa','dataset_session_info')); + dataset_session_info_docs = ndi_dataset_dir_obj.session.database.search(ndi.query('','isa','dataset_session_info')); correctSessionId = ''; if ~isempty(dataset_session_info_docs) correctSessionId = dataset_session_info_docs{1}.document_properties.base.session_id; else q = ndi.query('','isa','session_in_a_dataset'); - session_in_a_dataset_docs = ndi_dataset_dir_obj.database_search(q); + session_in_a_dataset_docs = ndi_dataset_dir_obj.session.database.search(q); if ~isempty(session_in_a_dataset_docs) correctSessionId = session_in_a_dataset_docs{1}.document_properties.base.session_id; else - q_session = ndi.query('','isa','session'); - candidate_session_doc = ndi_dataset_dir_obj.database_search(q_session); - if isscalar(candidate_session_doc) - correctSessionId = candidate_session_doc{1}.document_properties.base.session_id; + % A V2 dataset (ndi.setup.V2.createDataset) has no + % session_in_a_dataset documents: its sessions are ingested + % `session` documents, so the dataset's own session is the + % one that shares its session_id with the V2 `dataset` + % document (a class v1 never had). Read unscoped: the + % folder's unique_reference.txt names whichever session was + % opened last, not necessarily the dataset. + datasetDocs = ndi_dataset_dir_obj.session.database.search(ndi.v2.isaQuery('dataset')); + datasetDocs = datasetDocs(cellfun(@(d) strcmp(ndi.v2.kindOf(d.document_properties), ... + 'dataset'), datasetDocs)); % an entity of type dataset since 2026-10-08 + if isscalar(datasetDocs) + correctSessionId = datasetDocs{1}.document_properties.base.session_id; + else + q_session = ndi.v2.isaQuery('session'); + candidate_session_doc = ndi_dataset_dir_obj.session.database.search(q_session); + if isscalar(candidate_session_doc) + correctSessionId = candidate_session_doc{1}.document_properties.base.session_id; + end end end end if ~isempty(correctSessionId) - q_session = ndi.query('','isa','session') & ndi.query('base.session_id','exact_string',correctSessionId); - candidate_session_doc = ndi_dataset_dir_obj.database_search(q_session); + q_session = ndi.v2.isaQuery('session') & ndi.query('base.session_id','exact_string',correctSessionId); + candidate_session_doc = ndi_dataset_dir_obj.session.database.search(q_session); if isscalar(candidate_session_doc) % BOTH VINTAGES. V_eta renamed session.reference -> % session.local_identifier (did-schema, signed 2026-08-13). @@ -72,14 +86,20 @@ % a query-shaped grep does not find it -- the daqsystem % lesson. A dataset can hold a session document of either % vintage, so both spellings are accepted. - session_blk = candidate_session_doc{1}.document_properties.session; - if isfield(session_blk,'local_identifier') - ref = session_blk.local_identifier; - else - ref = session_blk.reference; + % and from 2026-10-08 the session may be an `entity` of + % type session (ndi.v2.blockOf reads either block) + p_session = candidate_session_doc{1}.document_properties; + ref = ndi.v2.blockOf(p_session, 'session', 'local_identifier'); + if isempty(ref) + ref = ndi.v2.blockOf(p_session, 'session', 'reference'); end session_id = candidate_session_doc{1}.document_properties.base.session_id; ndi_dataset_dir_obj.session = ndi.session.dir(ref,ndi_dataset_dir_obj.session.path,session_id); + % the session list may have been built while the folder was + % open under another session's id (a V2 dataset's + % unique_reference.txt names the session opened last); rebuild + ndi_dataset_dir_obj.session_info = []; + ndi_dataset_dir_obj.session_array = []; else error('Could not find dataset session document.'); end @@ -90,7 +110,7 @@ if ~isempty(dataset_session_info_docs) %disp('updating dataset to new form'); % double-check we still need to do it - dataset_session_info_docs2 = ndi_dataset_dir_obj.database_search(ndi.query('','isa','dataset_session_info')); + dataset_session_info_docs2 = ndi_dataset_dir_obj.session.database.search(ndi.query('','isa','dataset_session_info')); if ~isempty(dataset_session_info_docs2) ndi_dataset_dir_obj.repairDatasetSessionInfo(ndi_dataset_dir_obj,dataset_session_info_docs2); end diff --git a/src/ndi/+ndi/+file/+navigator/bodies.m b/src/ndi/+ndi/+file/+navigator/bodies.m new file mode 100644 index 0000000000..deb2d91525 --- /dev/null +++ b/src/ndi/+ndi/+file/+navigator/bodies.m @@ -0,0 +1,225 @@ +classdef bodies < ndi.file.navigator + % ndi.file.navigator.bodies - find an acquisition system's epochs through its documents + % + % A navigator for a V2 session whose recordings are described by + % documents rather than by files laid out in the session folder. It + % scans no folder and writes no file: everything it reports is read + % from the session's database. + % + % For the acquisition system that names this navigator (its + % `epoch_file_pattern_id`), each epoch is one recording statement: + % + % acquisition_system <- acquisition_channels (acquisition_system_id) + % <- subject_statement (acquisition_channels_id) + % <- data_body (owner_id) + % + % epoch files each body's files, at the location the body records + % (files.file_info.locations, not ingested), resolved by + % the database (ndi.session/database_existbinarydoc) -- + % the raw data stays where it is + % epoch id the `local_identifier` of the `epoch` document the + % statement's relative time reference points at + % probe map one entry: name = the acquisition system's name, + % reference 1, type = the statement's `method` name, + % device = :image1, subject = the statement's + % subject (a document id) + % + % A statement none of whose files is found on this computer is left + % out (and reported once, with a warning), so a missing raw file reads + % as a missing epoch rather than an error deep inside a reader. + % + % The file pattern in the epoch_file_pattern document (fileparameters) + % filters the bodies by file name: a body none of whose file names + % matches the first pattern is not this system's. + % + % See also: ndi.file.navigator, ndi.database.fun.externalFileLocation, + % ndi.setup.conv.haley.sessionDocuments + + properties (Access = protected) + found % containers.Map: 'recordings' -> struct array (a cache) + end + + methods + function obj = bodies(varargin) + obj = obj@ndi.file.navigator(varargin{:}); + obj.found = containers.Map(); + end + + function id = epochid(obj, epoch_number, epochfiles) + % The epoch document's local_identifier; no hidden file is + % written next to the (raw) epoch files. + if nargin < 3 + epochfiles = obj.getepochfiles_number(epoch_number); + end + if ndi.file.navigator.isingested(epochfiles) + id = ndi.file.navigator.ingestedfiles_epochid(epochfiles); + return; + end + R = obj.recordings(); + for k = 1:numel(R) + if isequal(R(k).files, epochfiles) + id = R(k).epoch_id; + return; + end + end + error('NDI:navigator:bodies:unknownEpoch', ... + 'No recording statement of this acquisition system has the files %s.', ... + strjoin(cellstr(epochfiles), ', ')); + end + + function [epochfiles_disk] = selectfilegroups_disk(obj) + R = obj.recordings(); + epochfiles_disk = {R.files}; + end + + function [epochfiles, epochprobemaps] = selectfilegroups(obj) + R = obj.recordings(); + epochfiles = {R.files}; + epochprobemaps = {R.probemap}; + end + + function epochprobemap = getepochprobemap(obj, N, epochfiles) + if nargin < 3 + epochfiles = obj.getepochfiles_number(N); + end + R = obj.recordings(); + for k = 1:numel(R) + if isequal(R(k).files, epochfiles) + epochprobemap = R(k).probemap; + return; + end + end + error('NDI:navigator:bodies:unknownEpoch', 'No recording for epoch %d.', N); + end + + function R = recordings(obj) + % The system's recordings, read from the database once per object. + if isKey(obj.found, 'recordings') + R = obj.found('recordings'); + return; + end + R = obj.readRecordings(); + obj.found('recordings') = R; + end + end + + methods (Access = protected) + function R = readRecordings(obj) + R = struct('files', {}, 'epoch_id', {}, 'probemap', {}); + S = obj.session; + if isempty(S) + return; + end + inSession = ndi.query('base.session_id', 'exact_string', S.id(), ''); + pattern = ''; + if ~isempty(obj.fileparameters) && ~isempty(obj.fileparameters.filematch) + pattern = obj.fileparameters.filematch{1}; + end + missing = {}; + systems = S.database_search(ndi.query('', 'isa', 'acquisition_system', '') & ... + ndi.query('', 'depends_on', 'epoch_file_pattern_id', obj.id()) & inSession); + for a = 1:numel(systems) + sysName = systems{a}.document_properties.acquisition_system.name; + channels = S.database_search(ndi.query('', 'isa', 'acquisition_channels', '') & ... + ndi.query('', 'depends_on', 'acquisition_system_id', systems{a}.id()) & inSession); + for c = 1:numel(channels) + statements = S.database_search(ndi.v2.isaQuery('statement') & ... + ndi.query('', 'depends_on', 'acquisition_channels_id', channels{c}.id()) & inSession); + for t = 1:numel(statements) + st = statements{t}; + [files, names] = obj.statementFiles(st, pattern); + if isempty(names) + continue; % not this system's (by file name) + end + if isempty(files) + missing{end+1} = strjoin(names, ', '); %#ok + continue; + end + R(end+1) = struct('files', {files}, ... + 'epoch_id', obj.statementEpoch(st), ... + 'probemap', obj.statementProbemap(st, sysName)); %#ok + end + end + end + if ~isempty(missing) + warning('NDI:navigator:bodies:missingFiles', ... + ['%d recording(s) of this acquisition system have no file on ' ... + 'this computer at the location their document records, and are ' ... + 'left out: %s'], numel(missing), strjoin(missing, '; ')); + end + [~, order] = sort({R.epoch_id}); + R = R(order); + end + + function [files, names] = statementFiles(obj, st, pattern) + % FILES: local paths of the statement's bodies' files; NAMES: + % the bodies' recorded file names that match PATTERN. + files = {}; + names = {}; + S = obj.session; + bodies = S.database_search(ndi.query('', 'isa', 'data_body', '') & ... + ndi.query('', 'depends_on', 'owner_id', st.id()) & ... + ndi.query('base.session_id', 'exact_string', S.id(), '')); + for b = 1:numel(bodies) + p = bodies{b}.document_properties; + if isfield(p, 'data_body') && isfield(p.data_body, 'filename') + name = char(p.data_body.filename); + else + name = ''; + end + if ~isempty(pattern) && isempty(regexp(name, pattern, 'once')) + continue; + end + names{end+1} = name; %#ok + list = {}; + if isfield(p, 'files') && isfield(p.files, 'file_list') + list = cellstr(p.files.file_list); + end + for f = 1:numel(list) + [tf, where] = S.database_existbinarydoc(bodies{b}, list{f}); + if tf + files{end+1} = where; %#ok + end + end + end + end + + function id = statementEpoch(obj, st) + % The epoch a statement's relative time reference points at. + S = obj.session; + refs = ndi.vintage.edge_n(st, 'time_reference_id'); + for r = 1:numel(refs) + d = S.database_search(ndi.query('base.id', 'exact_string', refs{r}, '')); + if numel(d) ~= 1 || ~isfield(d{1}.document_properties, 'relative_time_reference') + continue; + end + referent = d{1}.dependency_value('referent_id', 'ErrorIfNotFound', 0); + e = S.database_search(ndi.query('base.id', 'exact_string', referent, '') & ... + ndi.v2.isaQuery('epoch')); + if numel(e) == 1 + id = ndi.v2.blockOf(e{1}.document_properties, 'epoch', 'local_identifier'); + return; + end + end + error('NDI:navigator:bodies:noEpoch', ... + 'Recording statement %s has no relative time reference to an epoch document.', st.id()); + end + + function pm = statementProbemap(~, st, sysName) + p = st.document_properties; + type = ''; + m = ndi.v2.blockOf(p, 'interaction', 'method', []); + if isstruct(m) && isfield(m, 'name') + type = char(m.name); + end + if isempty(type) + error('NDI:navigator:bodies:noMethod', ... + ['Recording statement %s has no `method`, which names the probe ' ... + 'type (e.g. brightfield-imaging).'], st.id()); + end + subject = ndi.v2.edgeIds(p, 'entity_id'); + if isempty(subject), subject = ''; else, subject = subject{1}; end + pm = ndi.epoch.epochprobemap_daqsystem(sysName, 1, type, [sysName ':image1'], subject); + end + end +end diff --git a/src/ndi/+ndi/+fun/+doc/+lightsheet/README-lightsheet-zarr.md b/src/ndi/+ndi/+fun/+doc/+lightsheet/README-lightsheet-zarr.md new file mode 100644 index 0000000000..2a45ecb553 --- /dev/null +++ b/src/ndi/+ndi/+fun/+doc/+lightsheet/README-lightsheet-zarr.md @@ -0,0 +1,132 @@ +# lightsheet OME-Zarr documents (design + status) + +This package (`+ndi/+fun/+doc/+lightsheet/`) lets a session hold a +lightsheet OME-Zarr (NGFF v0.4) multiscale pyramid as NDI documents, +and hands one to napari for viewing through NDI-python. + +The pattern mirrors `+ndi/+fun/+doc/+gene/` (spatial transcriptomics +Stereo-seq / GEF) so a reader of that package will recognise every +piece here. + +## Document shape + +Two new document classes: + +- **`lightsheetZarrPyramid`** - one per REDUCTION variant (`mean`, + `max`, ...). Holds the frame (axes, level-0 shape, voxel size, + translation, channels, dtype). Depends on subject + optional element + + optional `fileReference` for the source store. +- **`lightsheetZarrLevel`** - one per (pyramid, level). Owns the chunk + bytes as a `chunk.bin_#` file series and the per-level metadata + (shape, chunks, voxel size, translation, codec, ...). Depends on its + parent pyramid. + +NDI documents are immutable, so: +- A new reduction is a new pyramid document + its own level ladder. +- A deeper coarsening is a new level document; the parent's `n_levels` + is out of date until a new parent version is written for that count. + +## MATLAB API surface + +- `ndi.fun.doc.lightsheet.fromOMEZarr(session, zarrPath, ...)` - + probe an OME-Zarr store, write ONE pyramid document per multiscales + entry and ONE level document per level. Metadata-only; does not copy + chunk bytes. **Working.** +- `ndi.fun.doc.lightsheet.makePyramid(session, pyramidEntry, ...)` - + the per-multiscales-entry worker `fromOMEZarr` calls. Reusable when + the caller already has an `ndr.format.omezarr.listPyramids` struct + in hand. **Working.** +- `ndi.fun.doc.lightsheet.makeSourceFile(session, zarrPath)` - + fileReference for the store. **Working.** +- `ndi.fun.doc.lightsheet.levelTable(session, pyramidDoc)` - + finest-first table of a pyramid's levels; `Properties.UserData` + carries the parent frame. **Working.** +- `ndi.fun.doc.lightsheet.chooseLevel(levels, targetVoxelSize)` - + pick the coarsest level whose voxel size still meets the target. + **Working.** +- `ndi.fun.doc.lightsheet.chooseTileShape(shape, axesOrder, voxelSize, dtype, budgetBytes)` - + size a chunk to a byte budget with tiles isotropic in world space. + Default budget 32 MB uncompressed, sized to the measured + signing+fetch+decode sweet spot for a viewer on an ~200 MB/s link. + Used by `makePyramid` per level; the source store's chunk shape is + not preserved. **Working.** +- `ndi.fun.doc.lightsheet.viewCommand(launcher, sessionPath, pyramidID, ...)` + - pure command builder. Same pattern as + `ndi.gui.app.GEFManager.viewCommand`; the launcher is + `/usr/local/bin/napariViewLightsheet`, a shell wrapper around the + `napariViewLightsheet` console script that ships in + `Waltham-Data-Science/NDI-python`. **Working.** + +## Straight from SmartSPIM: `fromSPIM` + +`ndi.fun.doc.lightsheet.fromSPIM(S, rootDir, 'subjectID', id)` builds the +same pyramid + level documents as `fromOMEZarr`, but reads the SmartSPIM +acquisition directly -- no spim2zarr step. Level 0 (all channels, axes +`tczyx`) comes from the stitched, harmonized `All_Channels` slices +(`ndr.format.smartspim.readAllChannelsInfo`; only the needed rows and +columns of each slice are read). The mean and max ladders are built +here, each level by block-reducing the one just written +(`ndi.fun.doc.lightsheet.reduceBlocks`), halving x/y/z while the voxel +size allows (coarse-z stacks even out in x/y first) until every spatial +axis is at most `minLevelSize` (512) voxels. Chunking, encoding, +parallel compression and installation are `makePyramid`'s, through its +`levelSources` option. + +## GUI + +`ndi.gui.app.LightsheetZarrManager(session)` mirrors `GEFManager`: +lists pyramids in a session, and drives Add / View / Delete. `View` +builds the napari command from `viewCommand` and shells out. + +### Viewer logs + +Both `View` and `ndi.fun.doc.lightsheet.view` launch the viewer through +`ndi.fun.doc.lightsheet.launchViewer`, which turns on the viewer's +debugging lines (`NDI_LIGHTSHEET_DEBUG=1`) and appends its stdout and +stderr to `lightsheet_YYYYMMDDHHMMSS.txt` in +`ndi.common.PathConstants.TempFolder`. The path is printed in the +Command Window (and shown in the manager's status line); open it with +`edit` or `type` to see what the viewer fetched and what failed. + +## What still needs building + +1. **Chunk materialization** into `chunk.bin_#` on + `lightsheetZarrLevel`. Uses the same file-series upload path as + `ndi.cloud.sync.internal.uploadFilesForDatasetDocuments`, plus a + small `chunk_index` side-field so sparse pyramids can elide + fill-value chunks without breaking the 1-based `chunk.bin_#` + addressing. Land that side-field on the schema first, then a + `makePyramid` option (or a stand-alone `materializeChunks` + function) can write bytes. +2. **NDI-python napari reader.** The MATLAB `viewCommand` targets + `/usr/local/bin/napariViewLightsheet`, a shell wrapper around + `ndi.gui.app.lightsheetZarr.cli:main` in + `Waltham-Data-Science/NDI-python` (installed as + `[project.scripts].napariViewLightsheet`). The CLI, viewer, and + metadata reader are landed; the per-chunk dask fetcher that + `levelArrays` calls is a scaffold today. See that repo's + `src/ndi/gui/app/lightsheetZarr/README-lightsheet-zarr.md`. +3. **`readViewport` (MATLAB).** A MATLAB path for reading a rectangle + out of a pyramid at the appropriate level. Two backends are + plausible: read the source store via `ndr.format.omezarr.readArray` + (works today, but only when the store is on disk), or read the + materialized `chunk.bin_#` files via the cloud API (blocked on #1). + The scaffold from an earlier version of this PR was removed to + keep the package free of `notImplemented` bodies; add it back with + real code when #1 lands. +4. **NDR helpers.** `ndr.format.omezarr.probe` is small; when it lands + in NDR-matlab / NDR-python, `fromOMEZarr` here can call it instead + of `listPyramids` for cheaper metadata reads. (Landing on the same + branch name in the NDR repos.) +5. **Tests.** Once #1 lands, extend + `tests/+ndi/+unittest/+fun/+doc/+lightsheet/` with round-trip and + viewport tests analogous to `+gene/`'s. + +## Related + +- `ndi.gui.app.GEFManager` - the sibling manager for Stereo-seq GEF + spatial transcriptomics. Same launcher pattern (a shell wrapper + around an NDI-python console script). Read it first if this is your + first NDI file-format ingest. +- `ndr.format.omezarr.*` - the metadata / read primitives this + package builds on. diff --git a/src/ndi/+ndi/+fun/+doc/+lightsheet/chooseLevel.m b/src/ndi/+ndi/+fun/+doc/+lightsheet/chooseLevel.m new file mode 100644 index 0000000000..60cf89d891 --- /dev/null +++ b/src/ndi/+ndi/+fun/+doc/+lightsheet/chooseLevel.m @@ -0,0 +1,64 @@ +function [levelIndex, levelRow] = chooseLevel(levels, targetVoxelSize) +% NDI.FUN.DOC.LIGHTSHEET.CHOOSELEVEL - pick the coarsest level meeting a target +% +% [LEVELINDEX, LEVELROW] = NDI.FUN.DOC.LIGHTSHEET.CHOOSELEVEL(LEVELS, +% TARGETVOXELSIZE) +% +% Given LEVELS (a struct array or table with per-level `voxel_size` +% and `level` fields, finest-first, e.g. from LEVELTABLE), returns +% the coarsest level whose per-axis voxel size is still <= +% TARGETVOXELSIZE. If every level is coarser than the target, returns +% level 0. If every level is finer, returns the coarsest. +% +% LEVELINDEX is the 0-based level index (matches +% lightsheetZarrLevel.level). +% +% TARGETVOXELSIZE may be a scalar (applied to every spatial axis) or a +% vector aligned with the level's `voxel_size`. +% +% See also: ndi.fun.doc.lightsheet.levelTable + + arguments + levels + targetVoxelSize (1,:) double + end + + if istable(levels) + vs = levels.voxel_size; + li = levels.level; + n = height(levels); + else + n = numel(levels); + vs = cell(n, 1); + li = zeros(n, 1); + for k = 1:n + vs{k} = reshape(double(levels(k).voxel_size), 1, []); + li(k) = levels(k).level; + end + end + if n == 0 + error('NDI:lightsheet:chooseLevel:emptyLevels', ... + 'levels is empty.'); + end + + bestK = 1; + for k = 1:n + v = vs{k}; + if isscalar(targetVoxelSize) + ok = all(v <= targetVoxelSize); + else + m = min(numel(v), numel(targetVoxelSize)); + ok = all(v(1:m) <= targetVoxelSize(1:m)); + end + if ok + bestK = k; + end + end + + levelIndex = li(bestK); + if istable(levels) + levelRow = levels(bestK, :); + else + levelRow = levels(bestK); + end +end diff --git a/src/ndi/+ndi/+fun/+doc/+lightsheet/chooseTileShape.m b/src/ndi/+ndi/+fun/+doc/+lightsheet/chooseTileShape.m new file mode 100644 index 0000000000..de7c6c4a7a --- /dev/null +++ b/src/ndi/+ndi/+fun/+doc/+lightsheet/chooseTileShape.m @@ -0,0 +1,131 @@ +function chunks = chooseTileShape(shape, axesOrder, voxelSize, dtype, budgetBytes) +% NDI.FUN.DOC.LIGHTSHEET.CHOOSETILESHAPE - pick a chunk shape that hits a byte budget +% +% CHUNKS = NDI.FUN.DOC.LIGHTSHEET.CHOOSETILESHAPE( ... +% SHAPE, AXESORDER, VOXELSIZE, DTYPE, BUDGETBYTES) +% +% Sizes an OME-Zarr chunk to land near BUDGETBYTES (uncompressed) +% and to be ISOTROPIC IN WORLD SPACE: each spatial axis gets a +% chunk length inversely proportional to its voxel size, so a +% lightsheet volume with 1 um XY and 5 um Z produces pancake tiles +% rather than voxel cubes. +% +% SHAPE - row vector, axis lengths in AXESORDER +% AXESORDER - char row like 'tczyx' (one letter per axis) +% VOXELSIZE - row vector, one per axis, same order (may include +% zeros or NaNs on t/c) +% DTYPE - char, NGFF-style ('uint16', 'float32', ' [256 256 256] % ~256^3 * 2 bytes = 32.0 MB +% +% Example (uint16, 32 MB, anisotropic 1/1/5 um): +% chooseTileShape([1200 2000 2000], 'zyx', [5 1 1], 'uint16', ... +% 32*2^20) +% -> [88 438 438] % ~88*438*438*2 ~= 33.8 MB, world-cube +% +% See also: ndi.fun.doc.lightsheet.makePyramid + + arguments + shape (1,:) double {mustBePositive, mustBeInteger} + axesOrder (1,:) char + voxelSize double + dtype (1,:) char + budgetBytes (1,1) double {mustBePositive} = 32 * 2^20 + end + + shape = double(shape); + n = numel(shape); + if numel(axesOrder) ~= n + error('NDI:lightsheet:chooseTileShape:axesShapeMismatch', ... + 'axes_order length %d does not match shape length %d.', ... + numel(axesOrder), n); + end + + % ismember with a cellstr treats the char row as ONE string; use a + % char row on both sides so it compares element-wise. + isSpatial = ~ismember(lower(axesOrder(1:n)), 'tc'); + isSpatial = reshape(isSpatial, 1, n); + chunks = ones(1, n); + if ~any(isSpatial) + chunks = min(chunks, shape); + return; + end + + vs = reshape(double(voxelSize), 1, []); + if numel(vs) ~= n || any(vs(isSpatial) <= 0) || any(~isfinite(vs(isSpatial))) + % No usable voxel size on spatial axes -- fall back to + % isotropic in voxel space (a raw n-th root of the budget). + vs = ones(1, n); + end + + bpv = bytesPerVoxel(dtype); + voxelsPerTile = budgetBytes / bpv; + if voxelsPerTile < 1 + voxelsPerTile = 1; + end + + vsSpatial = vs(isSpatial); + ns = numel(vsSpatial); + s = (voxelsPerTile * prod(vsSpatial))^(1/ns); + chunks(isSpatial) = max(1, round(s ./ vsSpatial)); + chunks = min(chunks, shape); + + % One redistribution pass: if any spatial axis clamped to its + % shape, spread the leftover budget across the axes still below + % their shape. Cheap and useful at coarser pyramid levels where + % one dim collapses first. + slack = isSpatial & (chunks < shape); + stuck = isSpatial & ~slack; + if any(slack) && any(stuck) + used = prod(chunks(stuck)); + if used < 1, used = 1; end + remaining = voxelsPerTile / used; + vsSlack = vs(slack); + s2 = (remaining * prod(vsSlack))^(1/numel(vsSlack)); + chunks(slack) = max(1, round(s2 ./ vsSlack)); + chunks = min(chunks, shape); + end +end + +function b = bytesPerVoxel(dtype) + d = lower(strtrim(char(dtype))); + if ~isempty(d) && any(d(1) == '<>|=') + d = d(2:end); + end + switch d + case {'uint8','int8','u1','i1','bool','?'} + b = 1; + case {'uint16','int16','u2','i2','float16','f2','half'} + b = 2; + case {'uint32','int32','u4','i4','float32','f4','single'} + b = 4; + case {'uint64','int64','u8','i8','float64','f8','double'} + b = 8; + case {'complex64','c8'} + b = 8; + case {'complex128','c16'} + b = 16; + otherwise + % Unknown dtype: assume uint16 (2 bytes), the common + % lightsheet case. Better than raising for a novel probe. + b = 2; + end +end diff --git a/src/ndi/+ndi/+fun/+doc/+lightsheet/fromOMEZarr.m b/src/ndi/+ndi/+fun/+doc/+lightsheet/fromOMEZarr.m new file mode 100644 index 0000000000..801259af76 --- /dev/null +++ b/src/ndi/+ndi/+fun/+doc/+lightsheet/fromOMEZarr.m @@ -0,0 +1,308 @@ +function [pyramidDoc, levelDocs, info] = fromOMEZarr(session, zarrPath, options) +% NDI.FUN.DOC.LIGHTSHEET.FROMOMEZARR - build lightsheetZarrPyramid docs from an OME-Zarr store +% +% [PYRAMIDDOC, LEVELDOCS, INFO] = NDI.FUN.DOC.LIGHTSHEET.FROMOMEZARR(... +% SESSION, ZARRPATH) +% +% Walks an OME-Zarr (NGFF v0.4) store at ZARRPATH, enumerates its +% multiscale pyramids with NDR.FORMAT.OMEZARR.LISTPYRAMIDS, and writes +% ONE lightsheetZarrPyramid document (the source volume) plus one +% lightsheetZarrLevel document per unique level to SESSION. +% +% ONE DOCUMENT PER SOURCE VOLUME. The parent describes the source +% volume; reductions (mean, max, ...) live on the level children. +% A store with mean and max pyramids produces one parent, not two. +% +% LEVEL 0 IS DEDUPED. When two NGFF multiscales entries reference +% the same underlying array (typically level 0: their +% `datasets[0].path` is the same string), one level document is +% written -- with reduction_function='none' -- and both readers use +% it. Reduced levels are per-reduction and get +% reduction_function='mean'/'max'/... . +% +% READING IS METADATA-ONLY. No chunk bytes are read here; the +% source store's .zattrs and per-level .zarray are enough to fill +% the documents. Materializing chunk bytes into a level document's +% `chunk.bin_#` file series is follow-up work (see the package +% README). +% +% Optional Name-Value Arguments: +% subjectID - char/string, id of the subject depended on. If empty, +% the caller is refused: a lightsheetZarrPyramid without +% a subject is not a valid document. +% elementID - char/string, id of the imaging element the volume was +% captured on. May be empty. +% pyramidNames - cellstr/string, restrict to these multiscales.name +% values. Empty means all. +% reductionMap - containers.Map / struct mapping pyramid name (as +% reported by listPyramids) or type to a reduction +% label ('mean', 'max', ...). When a name is not in +% the map, `pyramid_type` is inspected first, then a +% fallback of 'mean' is used with a warning. +% pipelineVersion - char/string tag written into +% lightsheetZarrPyramid.pipeline_version. +% sourceFileID - id of an existing fileReference document for +% ZARRPATH. When empty, a new fileReference is created +% via NDI.FUN.DOC.LIGHTSHEET.MAKESOURCEFILE. +% label - human-readable label written into the parent pyramid. +% tileBudgetBytes - target uncompressed bytes per chunk when chunk +% shape is chosen automatically. Default 32*2^20 +% (32 MB). Forwarded to makePyramid. +% chunks - optional explicit chunk shape (row vector, axes_order). +% Empty (default) means auto-size from the budget. +% Forwarded to makePyramid. +% +% INFO returns a struct with fields: +% zarrPath - absolute path resolved +% pyramids - the raw output of ndr.format.omezarr.listPyramids +% reductions - cellstr, unique reductions present in the ladder +% sourceFileID - id of the fileReference document used +% sharedLevel0 - logical, true if level 0 was shared across +% reductions and deduped +% +% Example: +% S = ndi.session.dir('mysession','/path/to/session'); +% [P, L, info] = ndi.fun.doc.lightsheet.fromOMEZarr(S, ... +% '/cloudcache/subject42/volume.ome.zarr', ... +% 'subjectID', subj.id()); +% +% See also: ndi.fun.doc.lightsheet.makePyramid, +% ndi.fun.doc.lightsheet.chooseLevel, +% ndi.fun.doc.lightsheet.viewCommand, +% ndr.format.omezarr.listPyramids, +% ndr.format.omezarr.probe + + arguments + session (1,1) + zarrPath (1,:) char + options.subjectID char = '' + options.elementID char = '' + options.pyramidNames = {} + options.reductionMap = struct() + options.pipelineVersion char = '' + options.sourceFileID char = '' + options.label char = '' + options.tileBudgetBytes (1,1) double {mustBePositive} = 32 * 2^20 + options.chunks double = [] + options.materializeChunks (1,1) logical = false + options.codec (1,:) char {mustBeMember(options.codec, {'raw','blosc-zstd'})} = 'raw' + options.clevel (1,1) double {mustBeInteger, mustBeGreaterThanOrEqual(options.clevel, 1), mustBeLessThanOrEqual(options.clevel, 9)} = 5 + % progressFcn is one of: + % 'default' - the default. Open an NDI + % ProgressBarWindow and drive it. + % Headless MATLAB (CI, -batch) does + % silent bookkeeping only, no figure, + % so this is safe on servers. + % function_handle - progressFcn(fraction, text) is + % called before each level and after + % each chunk write. Use this to feed + % your own dialog or logger. + % [] - no reporting at all. + options.progressFcn = 'default' + % Number of parallel workers to use for chunk compression. + % -1 (default) - use any parpool already open, else serial. + % 0 / 1 - force serial. + % N > 1 - open a parpool of size N. + % Reads stay serial in all modes; compression fans out. + % See makePyramid for details. + options.numWorkers (1,1) double {mustBeInteger, mustBeGreaterThanOrEqual(options.numWorkers, -1)} = -1 + % How much source data to prefetch into RAM per readArray + % call. Bigger = fewer seeks + subprocess round trips at the + % cost of more RAM. See makePyramid. Default 512 MB. + options.prefetchBytes (1,1) double {mustBePositive} = 512 * 2^20 + % Overlap disk reads with parfor compression via parfeval. + % See makePyramid for details. Default true; ignored when + % numWorkers resolves to serial. + options.asyncPrefetch (1,1) logical = true + % Path to a log file the ingest appends per-super-block memory + % diagnostics to (timestamp, level index, super-block index, + % chunks done, MATLAB RSS, levelDoc property size before/after + % add_file, parpool live-future count). Empty (default) + % disables. Intended for diagnosing memory growth. See + % makePyramid for details. + options.debugLog (1,:) char = '' + end + + if isempty(strtrim(options.subjectID)) + error('NDI:lightsheet:fromOMEZarr:noSubject', ... + ['A lightsheetZarrPyramid depends_on a subject; pass ' ... + '''subjectID''.']); + end + + if ~isfolder(zarrPath) + error('NDI:lightsheet:fromOMEZarr:noSuchStore', ... + 'OME-Zarr store not found at %s.', zarrPath); + end + + if ~ndr.format.omezarr.isOMEZarr(zarrPath) + error('NDI:lightsheet:fromOMEZarr:notOMEZarr', ... + '%s does not look like an OME-Zarr store (.zattrs missing multiscales).', ... + zarrPath); + end + + pyramids = ndr.format.omezarr.listPyramids(zarrPath); + if isempty(pyramids) + error('NDI:lightsheet:fromOMEZarr:noPyramids', ... + '%s reports zero multiscales entries.', zarrPath); + end + + keep = true(numel(pyramids), 1); + if ~isempty(options.pyramidNames) + wanted = cellstr(options.pyramidNames); + for i = 1:numel(pyramids) + keep(i) = any(strcmp(pyramids(i).name, wanted)); + end + if ~any(keep) + error('NDI:lightsheet:fromOMEZarr:noNamedPyramid', ... + ['None of the requested pyramid names (%s) matched. ' ... + 'Available: %s.'], strjoin(wanted, ', '), ... + strjoin({pyramids.name}, ', ')); + end + end + pyramids = pyramids(keep); + + n = numel(pyramids); + perEntryReduction = cell(n, 1); + for i = 1:n + perEntryReduction{i} = pickReduction(pyramids(i), options.reductionMap); + end + + sourceFileID = options.sourceFileID; + if isempty(sourceFileID) + srcDoc = ndi.fun.doc.lightsheet.makeSourceFile(session, zarrPath); + session.database_add(srcDoc); + sourceFileID = srcDoc.id(); + end + + % Resolve the progress callback: 'default' -> open an NDI + % ProgressBarWindow and update it; a function handle -> use it as + % is; [] -> stay silent. The window's constructor detects headless + % MATLAB by itself and no-ops without a figure, so CI runs need no + % special casing here. + progressFcn = resolveProgressCallback(options.progressFcn, ... + options.label, char(zarrPath)); + + [pyramidDoc, levelDocs, sharedLevel0] = ndi.fun.doc.lightsheet.makePyramid( ... + session, pyramids, perEntryReduction, ... + 'subjectID', options.subjectID, ... + 'elementID', options.elementID, ... + 'sourceFileID', sourceFileID, ... + 'pipelineVersion', options.pipelineVersion, ... + 'label', options.label, ... + 'tileBudgetBytes', options.tileBudgetBytes, ... + 'chunks', options.chunks, ... + 'materializeChunks', options.materializeChunks, ... + 'sourceZarrPath', char(zarrPath), ... + 'codec', options.codec, ... + 'clevel', options.clevel, ... + 'progressFcn', progressFcn, ... + 'numWorkers', options.numWorkers, ... + 'prefetchBytes', options.prefetchBytes, ... + 'asyncPrefetch', options.asyncPrefetch, ... + 'debugLog', options.debugLog); + + info = struct( ... + 'zarrPath', char(zarrPath), ... + 'pyramids', pyramids, ... + 'reductions', {unique(perEntryReduction)}, ... + 'sourceFileID', sourceFileID, ... + 'sharedLevel0', sharedLevel0); +end + +function r = pickReduction(pyramid, reductionMap) +% pickReduction - choose 'mean' or 'max' for this pyramid entry +% +% Preference: explicit map by name, then by type, then a heuristic: +% pyramid.type == 'max' -> 'max', anything else -> 'mean'. Warns when +% falling back because pyramid.type is unreliable. + r = ''; + if ~isempty(reductionMap) + if isstruct(reductionMap) + if ~isempty(pyramid.name) && isfield(reductionMap, pyramid.name) + r = char(reductionMap.(pyramid.name)); + elseif ~isempty(pyramid.type) && isfield(reductionMap, pyramid.type) + r = char(reductionMap.(pyramid.type)); + end + elseif isa(reductionMap, 'containers.Map') + if ~isempty(pyramid.name) && isKey(reductionMap, pyramid.name) + r = char(reductionMap(pyramid.name)); + elseif ~isempty(pyramid.type) && isKey(reductionMap, pyramid.type) + r = char(reductionMap(pyramid.type)); + end + end + end + if isempty(r) + t = lower(strtrim(char(pyramid.type))); + n = lower(strtrim(char(pyramid.name))); + if strcmp(t, 'max') || contains(n, 'max') + r = 'max'; + else + r = 'mean'; + if ~isempty(t) && ~any(strcmp(t, {'mean','box','average'})) + warning('NDI:lightsheet:fromOMEZarr:reductionFallback', ... + ['Pyramid %s has type ''%s''; falling back to ' ... + '''mean''. Pass reductionMap to override.'], ... + pyramid.name, pyramid.type); + end + end + end +end + +function fcn = resolveProgressCallback(spec, label, zarrPath) +% RESOLVEPROGRESSCALLBACK - turn the options.progressFcn spec into a +% callable, opening an NDI ProgressBarWindow when the caller took the +% default. Failures to open the bar are swallowed and the ingest +% continues silently: a bar that could not draw itself must not stop +% a multi-hour ingest. + if isa(spec, 'function_handle') + fcn = spec; + return; + end + if isempty(spec) + fcn = []; + return; + end + if ~(ischar(spec) || (isstring(spec) && isscalar(spec))) + error('NDI:lightsheet:fromOMEZarr:badProgressFcn', ... + ['progressFcn must be ''default'', a function handle, ' ... + 'or []. Got a %s.'], class(spec)); + end + spec = char(string(spec)); + if ~any(strcmpi(spec, {'default', 'auto'})) + error('NDI:lightsheet:fromOMEZarr:badProgressFcn', ... + ['progressFcn keyword ''%s'' is not recognised. Use ' ... + '''default'', a function handle, or [].'], spec); + end + + barLabel = 'Building lightsheet pyramid'; + if ~isempty(label) + barLabel = sprintf('Building pyramid: %s', label); + elseif ~isempty(zarrPath) + [~, zn] = fileparts(char(zarrPath)); + if ~isempty(zn) + barLabel = sprintf('Building pyramid: %s', zn); + end + end + tag = sprintf('lightsheet-ingest-%s', localUniqueId()); + + fcn = []; + try + pbw = ndi.gui.component.ProgressBarWindow('NDI lightsheet ingest'); + pbw.addBar('Label', barLabel, 'Tag', tag, 'Auto', true); + fcn = @(frac, txt) pbw.updateBar(tag, min(1, max(0, frac))); + catch + % Fall through with fcn = [] -- silent. + end +end + +function u = localUniqueId() +% A short unique tag so a second ingest in the same MATLAB does not +% collide with a bar the first one is still holding. + try + u = did.ido.unique_id(); + u = char(u); + catch + u = char(java.util.UUID.randomUUID().toString()); + end + if numel(u) > 12, u = u(1:12); end +end diff --git a/src/ndi/+ndi/+fun/+doc/+lightsheet/fromSPIM.m b/src/ndi/+ndi/+fun/+doc/+lightsheet/fromSPIM.m new file mode 100644 index 0000000000..c687f4f5d8 --- /dev/null +++ b/src/ndi/+ndi/+fun/+doc/+lightsheet/fromSPIM.m @@ -0,0 +1,312 @@ +function [pyramidDoc, levelDocs, info] = fromSPIM(session, rootDir, options) +% NDI.FUN.DOC.LIGHTSHEET.FROMSPIM - build a lightsheet pyramid straight from a SmartSPIM acquisition +% +% [PYRAMIDDOC, LEVELDOCS, INFO] = ndi.fun.doc.lightsheet.FROMSPIM(SESSION, ROOTDIR, ... +% 'subjectID', SUBJECTID) +% +% Reads the stitched, harmonized volume in ROOTDIR/All_Channels -- the +% output of SmartSPIM AutoStitch, which ndr.format.smartspim reads in +% preference to the per-tile Ex_* folders -- and writes the same +% documents ndi.fun.doc.lightsheet.fromOMEZarr writes for an OME-Zarr +% store: one lightsheetZarrPyramid and its lightsheetZarrLevel +% children, with chunk bytes materialized. No intermediate OME-Zarr +% (spim2zarr) step is needed. +% +% THE LADDER IS BUILT HERE. An OME-Zarr store arrives with its coarser +% levels already computed; a SmartSPIM folder has only full resolution. +% Level 0 (all channels, axes t,c,z,y,x) is read from the All_Channels +% slices. Each coarser level is then made by block-reducing the level +% just written -- once with 'mean' and once with 'max', the two +% ladders spim2zarr makes -- so no level re-reads full resolution and +% memory stays near prefetchBytes. Level 0 is shared by both ladders +% (reduction_function 'none'), exactly as fromOMEZarr dedupes it. +% +% A spatial axis is halved at a level only while its voxel size is at +% most 1.5x the finest voxel size among the axes still longer than one +% voxel, so an anisotropic stack +% (coarse z) is first evened out in x and y. Levels are added until +% every spatial axis is at most minLevelSize voxels (or maxLevels). +% +% All the chunking, encoding, parallel compression and file-series +% installation is ndi.fun.doc.lightsheet.makePyramid's, via its +% levelSources option; see makePyramid for numWorkers, prefetchBytes, +% asyncPrefetch and debugLog. +% +% Optional Name-Value Arguments: +% subjectID - REQUIRED. id of the subject the volume depends on. +% elementID - id of the imaging element (may be empty). +% label - label for the pyramid. Default: ROOTDIR's folder name. +% channels - channel directory names (e.g. "Ex_561_Em_561F_Ch2") +% to include, in the order wanted. Default: all, in +% the All_Channels ch order. +% reductions - ladders to build: any of "mean", "max". Default both. +% voxelSizeUm - [z y x] voxel size in micrometers. Default: from the +% acquisition's metadata.json; required if it has none. +% minLevelSize - stop adding levels once every spatial axis is at +% most this many voxels. Default 512. +% maxLevels - most levels per ladder, level 0 included. Default 10. +% tileBudgetBytes, chunks, codec, clevel, progressFcn, numWorkers, +% prefetchBytes, asyncPrefetch, debugLog, pipelineVersion, sourceFileID +% - as in ndi.fun.doc.lightsheet.fromOMEZarr, except +% that codec defaults to 'blosc-zstd': chunks are +% always materialized here. +% +% INFO returns a struct with fields: +% rootDir - ROOTDIR +% channelNames - cellstr, the channels on the c axis, in order +% levels - struct array (path, reduction, shape, voxelSize, factor) +% sourceFileID - id of the fileReference document for ROOTDIR +% sharedLevel0 - logical, true when level 0 is shared by the ladders +% +% Example: +% S = ndi.session.dir('/path/to/session'); +% [P, L] = ndi.fun.doc.lightsheet.fromSPIM(S, '/data/ferret42_SmartSPIM', ... +% 'subjectID', subj.id(), 'numWorkers', 8); +% +% See also: ndi.fun.doc.lightsheet.fromOMEZarr, +% ndi.fun.doc.lightsheet.makePyramid, +% ndi.fun.doc.lightsheet.reduceBlocks, +% ndr.format.smartspim.readAllChannelsInfo, +% ndr.format.smartspim.hasAllChannels + + arguments + session (1,1) + rootDir (1,:) char + options.subjectID char = '' + options.elementID char = '' + options.label char = '' + options.channels = {} + options.reductions = {'mean', 'max'} + options.voxelSizeUm double = [] + options.minLevelSize (1,1) double {mustBePositive} = 512 + options.maxLevels (1,1) double {mustBeInteger, mustBePositive} = 10 + options.pipelineVersion char = '' + options.sourceFileID char = '' + options.tileBudgetBytes (1,1) double {mustBePositive} = 32 * 2^20 + options.chunks double = [] + options.codec (1,:) char {mustBeMember(options.codec, {'raw','blosc-zstd'})} = 'blosc-zstd' + options.clevel (1,1) double {mustBeInteger, mustBeGreaterThanOrEqual(options.clevel, 1), mustBeLessThanOrEqual(options.clevel, 9)} = 5 + options.progressFcn = 'default' + options.numWorkers (1,1) double {mustBeInteger, mustBeGreaterThanOrEqual(options.numWorkers, -1)} = -1 + options.prefetchBytes (1,1) double {mustBePositive} = 512 * 2^20 + options.asyncPrefetch (1,1) logical = true + options.debugLog (1,:) char = '' + end + + if isempty(strtrim(options.subjectID)) + error('NDI:lightsheet:fromSPIM:noSubject', ... + 'A lightsheetZarrPyramid depends_on a subject; pass ''subjectID''.'); + end + if ~isfolder(rootDir) + error('NDI:lightsheet:fromSPIM:noSuchFolder', ... + 'SmartSPIM folder not found at %s.', rootDir); + end + if ~ndr.format.smartspim.hasAllChannels(rootDir) + error('NDI:lightsheet:fromSPIM:noAllChannels', ... + ['%s has no usable All_Channels folder (SmartSPIM AutoStitch ' ... + 'output, with slices for every channel). Run AutoStitch, or ' ... + 'convert the per-tile data with spim2zarr and use fromOMEZarr.'], ... + rootDir); + end + reductions = cellstr(options.reductions); + reductions = reductions(:); + if isempty(reductions) || ~all(ismember(reductions, {'mean', 'max'})) + error('NDI:lightsheet:fromSPIM:badReductions', ... + 'reductions must be a non-empty subset of {"mean", "max"}.'); + end + + % ---- what is on disk --------------------------------------------- + spim = ndr.format.smartspim.readAllChannelsInfo(rootDir); + [chans, channelNames] = pickChannels(spim, options.channels); + nSlices = arrayfun(@(c) c.numSlices, chans); + nz = min(nSlices); + if any(nSlices ~= nz) + warning('NDI:lightsheet:fromSPIM:unevenSlices', ... + ['Channels have different slice counts (%s); using the first ' ... + '%d slices of each.'], mat2str(nSlices), nz); + end + slicePaths = arrayfun(@(c) c.slicePaths(1:nz), chans, 'UniformOutput', false); + slicePaths = slicePaths(:); + + voxel = options.voxelSizeUm; + if isempty(voxel) + voxel = spim.voxelSizeUm; + end + if numel(voxel) ~= 3 || any(~isfinite(voxel)) || any(voxel <= 0) + error('NDI:lightsheet:fromSPIM:noVoxelSize', ... + ['No [z y x] voxel size: the acquisition metadata.json does ' ... + 'not give one. Pass ''voxelSizeUm'', [z y x].']); + end + dtype = char(spim.dtype); + shape0 = [1, numel(chans), nz, spim.height, spim.width]; + scale0 = [1, 1, double(voxel(:).')]; + + % ---- the ladder ---------------------------------------------------- + steps = planLadder(shape0, scale0, options.minLevelSize, options.maxLevels); + + axesDesc = struct('name', {'t', 'c', 'z', 'y', 'x'}, ... + 'type', {'time', 'channel', 'space', 'space', 'space'}, ... + 'unit', {'', '', 'micrometer', 'micrometer', 'micrometer'}); + level0 = levelEntry('0', shape0, scale0, dtype); + + levelSources = containers.Map(); + levelSources('0') = struct('kind', 'smartspim', ... + 'slicePaths', {slicePaths}, 'dtype', dtype); + + pyramids = repmat(struct('name', '', 'type', '', 'axes', axesDesc, ... + 'levels', level0), numel(reductions), 1); + nSteps = numel(steps); + levelInfo = repmat(struct('path', '0', 'reduction', 'none', ... + 'shape', shape0, 'voxelSize', scale0, 'factor', ones(1, 5)), ... + 1, 1 + numel(reductions) * nSteps); + for r = 1:numel(reductions) + red = reductions{r}; + levels = repmat(level0, 1, 1 + nSteps); + prevPath = '0'; + for k = 1:nSteps + levelPath = sprintf('%s/%d', red, k); + levels(1 + k) = levelEntry(levelPath, steps(k).shape, steps(k).scale, dtype); + levelSources(levelPath) = struct('kind', 'reduce', 'fromPath', prevPath, ... + 'factor', steps(k).factor, 'reduction', red); + levelInfo(1 + (r - 1) * nSteps + k) = struct('path', levelPath, ... + 'reduction', red, 'shape', steps(k).shape, ... + 'voxelSize', steps(k).scale, 'factor', steps(k).factor); + prevPath = levelPath; + end + pyramids(r).name = red; + pyramids(r).type = red; + pyramids(r).levels = levels; + end + + % ---- source description + progress --------------------------------- + sourceFileID = options.sourceFileID; + if isempty(sourceFileID) + srcDoc = ndi.fun.doc.lightsheet.makeSourceFile(session, rootDir); + session.database_add(srcDoc); + sourceFileID = srcDoc.id(); + end + label = options.label; + if isempty(label) + [~, label] = fileparts(strip(rootDir, 'right', filesep)); + end + progressFcn = resolveProgressCallback(options.progressFcn, label); + + [pyramidDoc, levelDocs, sharedLevel0] = ndi.fun.doc.lightsheet.makePyramid( ... + session, pyramids, reductions, ... + 'subjectID', options.subjectID, ... + 'elementID', options.elementID, ... + 'sourceFileID', sourceFileID, ... + 'pipelineVersion', options.pipelineVersion, ... + 'label', label, ... + 'channelNames', strjoin(channelNames, ','), ... + 'tileBudgetBytes', options.tileBudgetBytes, ... + 'chunks', options.chunks, ... + 'materializeChunks', true, ... + 'levelSources', levelSources, ... + 'codec', options.codec, ... + 'clevel', options.clevel, ... + 'progressFcn', progressFcn, ... + 'numWorkers', options.numWorkers, ... + 'prefetchBytes', options.prefetchBytes, ... + 'asyncPrefetch', options.asyncPrefetch, ... + 'debugLog', options.debugLog); + + info = struct( ... + 'rootDir', char(rootDir), ... + 'channelNames', {channelNames}, ... + 'levels', levelInfo, ... + 'sourceFileID', sourceFileID, ... + 'sharedLevel0', sharedLevel0); +end + +function steps = planLadder(shape0, scale0, minLevelSize, maxLevels) +% PLANLADDER - the coarser levels below level 0: shape, voxel size, factor +% +% Axes t,c,z,y,x; only z, y, x are reduced. At each step a spatial +% axis is halved if it is longer than 1 and its voxel size is at most +% 1.5x the finest spatial voxel size (so coarse-z stacks catch up in +% x, y first; "finest" counts only axes longer than 1). Stops when +% every spatial axis is <= minLevelSize, when +% no axis can be halved, or at maxLevels levels (level 0 included). + steps = struct('shape', {}, 'scale', {}, 'factor', {}); + shape = shape0; + scale = scale0; + while numel(steps) + 1 < maxLevels && max(shape(3:5)) > minLevelSize + f = ones(1, 5); + % Finest among the axes that can still be halved: an axis + % already down to one voxel must not hold the others back. + live = find(shape(3:5) > 1) + 2; + if isempty(live) + break; + end + finest = min(scale(live)); + for a = 3:5 + if shape(a) > 1 && scale(a) <= 1.5 * finest + f(a) = 2; + end + end + if all(f == 1) + break; + end + shape = ceil(shape ./ f); + scale = scale .* f; + steps(end+1) = struct('shape', shape, 'scale', scale, 'factor', f); %#ok + end +end + +function L = levelEntry(path, shape, scale, dtype) +% One level in the shape ndr.format.omezarr.listPyramids reports. + L = struct('path', path, 'shape', double(shape), 'chunks', [], ... + 'dtype', dtype, 'scale', double(scale), 'translation', zeros(1, 5)); +end + +function [chans, names] = pickChannels(spim, wanted) +% The channels to put on the c axis, in order. + chans = spim.channels(:); + if isempty(wanted) + names = {chans.name}; + return; + end + wanted = cellstr(wanted); + have = {chans.name}; + idx = zeros(1, numel(wanted)); + for i = 1:numel(wanted) + j = find(strcmp(have, wanted{i}), 1); + if isempty(j) + error('NDI:lightsheet:fromSPIM:noSuchChannel', ... + 'No channel %s in All_Channels. Available: %s.', ... + wanted{i}, strjoin(have, ', ')); + end + idx(i) = j; + end + chans = chans(idx); + names = {chans.name}; +end + +function fcn = resolveProgressCallback(spec, label) +% 'default' -> an NDI ProgressBarWindow; a function handle -> as is; +% [] -> silent. Same contract as fromOMEZarr's; a bar that cannot draw +% itself never stops the ingest. + if isa(spec, 'function_handle') + fcn = spec; + return; + end + fcn = []; + if isempty(spec) + return; + end + if ~((ischar(spec) || (isstring(spec) && isscalar(spec))) ... + && any(strcmpi(char(spec), {'default', 'auto'}))) + error('NDI:lightsheet:fromSPIM:badProgressFcn', ... + 'progressFcn must be ''default'', a function handle, or [].'); + end + try + tag = sprintf('lightsheet-spim-%s', char(did.ido.unique_id())); + pbw = ndi.gui.component.ProgressBarWindow('NDI lightsheet ingest'); + pbw.addBar('Label', sprintf('Building pyramid: %s', label), ... + 'Tag', tag, 'Auto', true); + fcn = @(frac, txt) pbw.updateBar(tag, min(1, max(0, frac))); + catch + fcn = []; + end +end diff --git a/src/ndi/+ndi/+fun/+doc/+lightsheet/launchViewer.m b/src/ndi/+ndi/+fun/+doc/+lightsheet/launchViewer.m new file mode 100644 index 0000000000..5d323b0c42 --- /dev/null +++ b/src/ndi/+ndi/+fun/+doc/+lightsheet/launchViewer.m @@ -0,0 +1,75 @@ +function [status, logFile, fullCmd] = launchViewer(cmd, options) +% NDI.FUN.DOC.LIGHTSHEET.LAUNCHVIEWER - run a napari viewer command with its log kept +% +% [STATUS, LOGFILE, FULLCMD] = ndi.fun.doc.lightsheet.LAUNCHVIEWER(CMD) +% +% Runs CMD (a napariViewLightsheet command line, as built by +% ndi.fun.doc.lightsheet.viewCommand) through SYSTEM with the viewer's +% debugging lines switched on (NDI_LIGHTSHEET_DEBUG=1) and its stdout +% and stderr appended to a log file in NDI's temporary folder: +% +% /lightsheet_YYYYMMDDHHMMSS.txt +% +% A viewer launched from MATLAB has no terminal, so without this its +% log -- which tiles it fetched, what failed and why -- is lost. The +% file starts with a header (time, command) and is named by the launch +% time, so each viewer gets its own and they sort in launch order. +% +% STATUS is the SYSTEM status: for a detached launch (the default) it +% says whether the launch started, not how the viewer ended. LOGFILE is +% the full path of the log. FULLCMD is the exact shell command run. +% +% Optional Name-Value Arguments: +% wait - block until the viewer exits (default false, so MATLAB +% stays interactive while the viewer is open). +% debug - set NDI_LIGHTSHEET_DEBUG=1 for the viewer (default true). +% logFile - write here instead of a new timestamped file in the NDI +% temporary folder. +% verbose - print the command and the log path (default true). +% +% Example: +% cmd = ndi.fun.doc.lightsheet.viewCommand(launcher, S.path(), id); +% [~, logFile] = ndi.fun.doc.lightsheet.launchViewer(cmd); +% edit(logFile) % or: type(logFile) +% +% See also: ndi.fun.doc.lightsheet.view, ndi.fun.doc.lightsheet.viewCommand, +% ndi.fun.doc.lightsheet.viewerLogFile, +% ndi.fun.doc.lightsheet.viewerShellCommand, +% ndi.gui.app.LightsheetZarrManager + + arguments + cmd (1,:) char + options.wait (1,1) logical = false + options.debug (1,1) logical = true + options.logFile (1,:) char = '' + options.verbose (1,1) logical = true + end + + logFile = options.logFile; + if isempty(logFile) + logFile = ndi.fun.doc.lightsheet.viewerLogFile(); + end + logDir = fileparts(logFile); + if ~isempty(logDir) && ~isfolder(logDir) + mkdir(logDir); + end + + % Header first, so a log whose viewer printed nothing still says what + % was launched and when. + fid = fopen(logFile, 'a'); + if fid >= 0 + fprintf(fid, '=== napari lightsheet viewer: %s ===\n', ... + char(datetime('now', 'Format', 'yyyy-MM-dd HH:mm:ss'))); + fprintf(fid, 'command : %s\n', cmd); + fprintf(fid, 'debug : %d\n\n', options.debug); + fclose(fid); + end + + fullCmd = ndi.fun.doc.lightsheet.viewerShellCommand(cmd, logFile, ... + options.wait, options.debug, ispc); + if options.verbose + fprintf('Executing: %s\n', fullCmd); + fprintf('Viewer log: %s\n', logFile); + end + status = system(fullCmd); +end diff --git a/src/ndi/+ndi/+fun/+doc/+lightsheet/levelTable.m b/src/ndi/+ndi/+fun/+doc/+lightsheet/levelTable.m new file mode 100644 index 0000000000..992fd38447 --- /dev/null +++ b/src/ndi/+ndi/+fun/+doc/+lightsheet/levelTable.m @@ -0,0 +1,141 @@ +function t = levelTable(session, pyramidDoc, options) +% NDI.FUN.DOC.LIGHTSHEET.LEVELTABLE - table of a pyramid's levels, finest-first +% +% T = NDI.FUN.DOC.LIGHTSHEET.LEVELTABLE(SESSION, PYRAMIDDOC) +% T = NDI.FUN.DOC.LIGHTSHEET.LEVELTABLE(SESSION, PYRAMIDDOC, ... +% 'reduction', 'mean') +% +% Queries SESSION for every LIGHTSHEETZARRLEVEL document whose +% `lightsheetZarrPyramid_id` depends_on matches PYRAMIDDOC.id(), and +% returns a MATLAB table sorted by `level` ascending (finest-first). +% +% BOTH VINTAGES. PYRAMIDDOC may also be a MIGRATED (V_eta) pyramid -- the +% intensity observation DID-matlab's lightsheet migrator makes, its id +% kept -- and its levels are then the sampled_bodies it owns. The columns +% and UserData are the same, by v1 name, read through ndi.vintage +% (lightsheetLevelDocs / lightsheetLevels / lightsheetPyramid); see those +% for where each value comes from and for the one derived column, `level`. +% +% When the 'reduction' name-value is passed, the returned table only +% contains levels with reduction_function in {'none', } -- +% the shared raw level(s) plus the requested reduction. Omit to get +% every level (both mean and max in a store that ships both). +% +% Table columns: level, reduction_function, shape, chunks, chunk_grid, +% n_chunks_stored, voxel_size, translation, dtype, id. +% +% UserData holds the parent pyramid's frame: +% axes_order, voxel_size_level0, translation_level0, dtype, +% pyramid_name, id (parent id), reductions (cellstr of the +% reduction_function values present in the full ladder, minus +% 'none'). +% +% PYRAMIDDOC may be an ndi.document or an id char/string; either is +% accepted so this can be called from a GUI that only has the row's +% id. +% +% See also: ndi.fun.doc.lightsheet.chooseLevel + + arguments + session (1,1) + pyramidDoc + options.reduction char = '' + end + + if isa(pyramidDoc, 'ndi.document') + parentDoc = pyramidDoc; + parentID = parentDoc.id(); + else + parentID = char(pyramidDoc); + parentDoc = session.database_search(ndi.query('base.id', 'exact_string', parentID)); + if isempty(parentDoc) + error('NDI:lightsheet:levelTable:noParent', ... + 'No lightsheetZarrPyramid with id %s in session.', parentID); + end + parentDoc = parentDoc{1}; + end + + % Both vintages: a v1 pyramid's lightsheetZarrLevel documents, or the + % sampled_bodies a migrated (V_eta) pyramid observation owns. The v1 + % query is the one this function always ran (ndi.vintage.lightsheetLevelDocs). + docs = ndi.vintage.lightsheetLevelDocs(session, parentDoc); + if isempty(docs) + t = emptyTable(); + t.Properties.UserData = frame(session, parentDoc, {}, {}); + return; + end + % Each level's fields by their v1 names. A v1 level's record IS its + % lightsheetZarrLevel block; a V_eta level's is read back from its keys, + % conditions and series, with `level` derived from the set + % (ndi.vintage.lightsheetLevels). + recs = ndi.vintage.lightsheetLevels(docs, parentDoc); + + n = numel(docs); + level = zeros(n, 1); + reduction_function = cell(n, 1); + shape = cell(n, 1); + chunks = cell(n, 1); + chunk_grid = cell(n, 1); + n_chunks_stored = zeros(n, 1); + voxel_size = cell(n, 1); + translation = cell(n, 1); + dtype = cell(n, 1); + id = cell(n, 1); + + for k = 1:n + p = recs{k}; + level(k) = p.level; + reduction_function{k} = char(p.reduction_function); + shape{k} = reshape(double(p.shape), 1, []); + chunks{k} = reshape(double(p.chunks), 1, []); + chunk_grid{k} = reshape(double(p.chunk_grid), 1, []); + n_chunks_stored(k) = p.n_chunks_stored; + voxel_size{k} = reshape(double(p.voxel_size), 1, []); + translation{k} = reshape(double(p.translation), 1, []); + dtype{k} = char(p.dtype); + id{k} = docs{k}.id(); + end + + reductionsPresent = setdiff(unique(reduction_function), {'none'}); + + if ~isempty(options.reduction) + keep = strcmp(reduction_function, 'none') | strcmp(reduction_function, options.reduction); + level = level(keep); + reduction_function = reduction_function(keep); + shape = shape(keep); + chunks = chunks(keep); + chunk_grid = chunk_grid(keep); + n_chunks_stored = n_chunks_stored(keep); + voxel_size = voxel_size(keep); + translation = translation(keep); + dtype = dtype(keep); + id = id(keep); + end + + t = table(level, reduction_function, shape, chunks, chunk_grid, ... + n_chunks_stored, voxel_size, translation, dtype, id); + t = sortrows(t, 'level'); + t.Properties.UserData = frame(session, parentDoc, reductionsPresent, recs); +end + +function f = frame(session, parentDoc, reductionsPresent, recs) + % v1: the pyramid's own block. V_eta: the same fields, read back by + % ndi.vintage.lightsheetPyramid from the store body / level 0. + p = ndi.vintage.lightsheetPyramid(parentDoc, session, recs); + f = struct( ... + 'axes_order', char(p.axes_order), ... + 'voxel_size_level0', reshape(double(p.voxel_size_level0), 1, []), ... + 'translation_level0', reshape(double(p.translation_level0), 1, []), ... + 'dtype', char(p.dtype), ... + 'pyramid_name', char(p.pyramid_name), ... + 'id', parentDoc.id(), ... + 'reductions', {reductionsPresent}); +end + +function t = emptyTable() + t = table( ... + zeros(0,1), cell(0,1), cell(0,1), cell(0,1), cell(0,1), ... + zeros(0,1), cell(0,1), cell(0,1), cell(0,1), cell(0,1), ... + 'VariableNames', {'level','reduction_function','shape','chunks','chunk_grid', ... + 'n_chunks_stored','voxel_size','translation','dtype','id'}); +end diff --git a/src/ndi/+ndi/+fun/+doc/+lightsheet/makePyramid.m b/src/ndi/+ndi/+fun/+doc/+lightsheet/makePyramid.m new file mode 100644 index 0000000000..83504c110c --- /dev/null +++ b/src/ndi/+ndi/+fun/+doc/+lightsheet/makePyramid.m @@ -0,0 +1,1241 @@ +function [pyramidDoc, levelDocs, sharedLevel0] = makePyramid(session, pyramids, perEntryReduction, options) +% NDI.FUN.DOC.LIGHTSHEET.MAKEPYRAMID - write one pyramid parent and its deduped level children +% +% [PYRAMIDDOC, LEVELDOCS, SHAREDLEVEL0] = ... +% NDI.FUN.DOC.LIGHTSHEET.MAKEPYRAMID(SESSION, PYRAMIDS, ... +% PEREDUCTION, options) +% +% Given the multiscales entries reported by +% NDR.FORMAT.OMEZARR.LISTPYRAMIDS and a per-entry reduction label, +% writes ONE lightsheetZarrPyramid parent (the source volume) and +% one lightsheetZarrLevel per UNIQUE array path across all entries. +% +% Dedupe rule: two entries whose levels(k).path is the same NGFF +% string refer to the same underlying array. Typical case: level 0 +% shared across mean and max pyramids. One level document is +% written and its reduction_function is 'none'. +% +% Non-shared levels get reduction_function equal to the reduction +% of the (single) entry that names them. +% +% PYRAMIDS - struct array from listPyramids +% PEREDUCTION - cellstr, one label per entry ('mean','max',...) +% +% Name-Value: +% subjectID - char, depended-on subject id +% elementID - char, depended-on element id (may be '') +% sourceFileID - char, id of the fileReference doc for the store +% pipelineVersion - char, tag for the pyramid +% label - char, human-readable label for the pyramid +% tileBudgetBytes - target UNCOMPRESSED bytes per chunk when the +% chunk shape is chosen automatically. Default +% 32*2^20 (32 MB). Sized to the measured +% signing+fetch+decode sweet spot for a viewer +% on an ~200 MB/s link: large enough that the +% per-chunk round trip is amortised and the +% chunk count stays bounded (121k-chunk level 0 +% at 8 MB -> ~30k at 32 MB on a whole-brain +% lightsheet), small enough that per-chunk +% decode stays under ~200 ms and the renderer +% can stream chunks without stalls. Chunks land +% near this budget but are clamped by the +% level's own shape, so a small coarse level +% may end up as one whole chunk. +% chunks - optional explicit chunk shape (row vector in +% axes_order). Empty (default) means auto: pick +% a shape ISOTROPIC IN WORLD SPACE from the +% level's voxel size, the dtype and +% tileBudgetBytes. Non-empty overrides the +% budget entirely and is used verbatim on every +% level (clamped by shape). +% +% The chosen chunk shape is what a future materializer will +% re-tile source bytes to; it is stamped onto the level document +% here so a reader can compute chunk_grid and index chunk.bin_# +% without consulting the source store. Source-store chunk shape is +% NOT preserved. +% +% With materializeChunks=false this is metadata only and +% n_chunks_stored is 0 on every level document. With +% materializeChunks=true each level's bytes are read, re-tiled, +% encoded and installed as the `chunk.bin_#` file series. +% +% WHERE THE BYTES COME FROM. By default every level is read from the +% OME-Zarr store at sourceZarrPath (the level's own array in it). +% levelSources overrides that per level path, for sources that are +% not an OME-Zarr store -- ndi.fun.doc.lightsheet.fromSPIM uses it to +% read level 0 from SmartSPIM's stitched All_Channels slices and to +% build every coarser level by block-reducing the level just written: +% +% levelSources - containers.Map from level path (the levels(k).path +% string in PYRAMIDS) to a source struct: +% struct('kind','omezarr') the default, from sourceZarrPath +% struct('kind','smartspim', 'slicePaths', {C x 1 cell of +% Z x 1 cellstr}, 'dtype', CLASS) +% 2-D TIFF slices, one per (c, z), +% axes t,c,z,y,x +% struct('kind','reduce', 'fromPath', PATH, 'factor', F, +% 'reduction', 'mean'|'max') +% block-reduce the level at PATH +% (written earlier in this call) +% by per-axis factor F +% A path not in the map falls back to 'omezarr'. +% +% See also: ndi.fun.doc.lightsheet.fromOMEZarr, +% ndi.fun.doc.lightsheet.fromSPIM, +% ndi.fun.doc.lightsheet.reduceBlocks, +% ndi.fun.doc.lightsheet.levelTable, +% ndi.fun.doc.lightsheet.chooseLevel + + arguments + session (1,1) + pyramids (:,1) struct + perEntryReduction (:,1) cell + options.subjectID (1,:) char + options.elementID char = '' + options.sourceFileID char = '' + options.pipelineVersion char = '' + options.label char = '' + options.tileBudgetBytes (1,1) double {mustBePositive} = 32 * 2^20 + options.chunks double = [] + options.materializeChunks (1,1) logical = false + options.sourceZarrPath char = '' + options.codec (1,:) char {mustBeMember(options.codec, {'raw','blosc-zstd'})} = 'raw' + options.clevel (1,1) double {mustBeInteger, mustBeGreaterThanOrEqual(options.clevel, 1), mustBeLessThanOrEqual(options.clevel, 9)} = 5 + % progressFcn(fraction, text) is called before and after each + % chunk write. Empty = no reporting. Keep this cheap; it fires + % once per chunk on a real ingest, which is many times. + options.progressFcn = [] + % Number of parallel workers to use for compression. + % -1 (default) - use whatever parpool is already open; + % if none is open, run serially. This lets + % the caller size and manage the pool once + % at the top of their script, and every + % subsequent ingest just uses it. + % 0 / 1 - force serial (the original path, no + % toolbox dependency). + % N > 1 - open (or replace) a parpool of size N and + % use it. Requires the Parallel Computing + % Toolbox; falls back to serial with a + % warning if unavailable. + % Reads stay strictly serial in all modes; a batch of + % effective-pool-size chunks is read into memory, then + % compression fans out. Ingestion order into DID is preserved. + options.numWorkers (1,1) double {mustBeInteger, mustBeGreaterThanOrEqual(options.numWorkers, -1)} = -1 + % How much source data to pull into RAM per readArray call, in + % bytes uncompressed. One "super-block" of chunks is read at a + % time, then compressed by the worker pool in sub-batches. A + % bigger prefetch amortises the fixed cost of a seek + NDR + % subprocess round-trip over more voxels, which matters on any + % drive slow enough for I/O to dominate. Default 512 MB is a + % sensible compromise for a modern workstation; passing 1e9 or + % 2e9 is fine on a machine with plenty of RAM. Peak MATLAB + % memory is prefetchBytes (uncompressed). + options.prefetchBytes (1,1) double {mustBePositive} = 512 * 2^20 + % Overlap disk reads with compression via parfeval. The next + % super-block starts reading on a pool worker while the parfor + % is still compressing the current one, so total time + % approaches max(read_time, compress_time) instead of their + % sum. Requires a parpool (falls back to synchronous reads + % otherwise). Peak RAM is ~2 * prefetchBytes (two super- + % blocks in flight). + options.asyncPrefetch (1,1) logical = true + % Path to a text log file the ingest will append per-super-block + % memory diagnostics to (timestamp, level index, super-block + % index, chunks done, main-process RSS in MB, `whos` sizes of + % the big variables, parpool live-future count). Empty (default) + % disables logging. Intended for diagnosing memory growth + % without affecting the hot path -- one line per super-block is + % negligible on a multi-hour ingest. + options.debugLog (1,:) char = '' + % Per-level-path byte sources; see "WHERE THE BYTES COME FROM" + % above. Empty (default) reads every level from sourceZarrPath. + options.levelSources = [] + % Comma-separated channel names written to the pyramid document. + options.channelNames (1,:) char = '' + end + + if isempty(options.levelSources) + options.levelSources = containers.Map(); + end + if options.materializeChunks && isempty(options.sourceZarrPath) ... + && options.levelSources.Count == 0 + error('NDI:lightsheet:makePyramid:noSource', ... + ['materializeChunks=true requires sourceZarrPath (or ' ... + 'levelSources) so chunks can be read from a source.']); + end + + if numel(pyramids) ~= numel(perEntryReduction) + error('NDI:lightsheet:makePyramid:reductionArity', ... + 'perEntryReduction (%d) must have one entry per pyramid (%d).', ... + numel(perEntryReduction), numel(pyramids)); + end + + % Group levels by NGFF path. entries at the same path share bytes. + % pathReductions maps path -> set of reductions that use it. + pathIndex = containers.Map(); + pathOrder = {}; + pathReductions = containers.Map(); + representative = containers.Map(); % path -> struct(entryIdx, levelIdx) + for i = 1:numel(pyramids) + levels = pyramids(i).levels; + for k = 1:numel(levels) + p = char(levels(k).path); + if ~isKey(pathIndex, p) + pathIndex(p) = numel(pathOrder) + 1; + pathOrder{end+1} = p; %#ok + pathReductions(p) = perEntryReduction(i); + representative(p) = struct('entryIdx', i, 'levelIdx', k); + else + cur = pathReductions(p); + if ~any(strcmp(cur, perEntryReduction{i})) + cur{end+1} = perEntryReduction{i}; %#ok + pathReductions(p) = cur; + end + end + end + end + + % Pick the source volume's identity from the first entry, first level. + firstEntry = pyramids(1); + firstLevel = firstEntry.levels(1); + axesOrder = joinAxisNames(firstEntry.axes); + axesUnits = collectAxisUnits(firstEntry.axes); + label = options.label; + if isempty(label) + label = firstEntry.name; + end + + pyramidDoc = session.newdocument('lightsheetZarrPyramid', ... + 'lightsheetZarrPyramid', struct( ... + 'label', label, ... + 'pyramid_name', char(firstEntry.name), ... + 'pyramid_type', char(firstEntry.type), ... + 'axes_order', axesOrder, ... + 'axes_units', strjoin(axesUnits, ','), ... + 'channel_names', options.channelNames, ... + 'n_channels', channelCount(firstLevel.shape, axesOrder), ... + 'shape_level0', firstLevel.shape, ... + 'voxel_size_level0', ensureRow(firstLevel.scale), ... + 'voxel_size_units', 'micrometer', ... + 'translation_level0', ensureRow(firstLevel.translation), ... + 'dtype', char(firstLevel.dtype), ... + 'pipeline_version', options.pipelineVersion, ... + 'byte_order', 'little')); + + pyramidDoc = pyramidDoc.set_dependency_value('subject_id', options.subjectID); + if ~isempty(options.elementID) + pyramidDoc = pyramidDoc.set_dependency_value('element_id', options.elementID); + end + if ~isempty(options.sourceFileID) + pyramidDoc = pyramidDoc.set_dependency_value('source_file_id', options.sourceFileID); + end + + session.database_add(pyramidDoc); + + % PROGRESS ACCOUNTING. Chunk-count planning happens up front so + % progressFcn can report overall fraction, not per-level fraction: + % a viewer wants "3 of 24000 chunks done", not a bar that resets at + % every level. chooseTileShape is cheap (metadata only), so running + % it twice -- once here, once inside makeOneLevel -- costs nothing. + levelChunks = cell(numel(pathOrder), 1); + levelGrids = cell(numel(pathOrder), 1); + totalChunks = 0; + for idx = 1:numel(pathOrder) + p = pathOrder{idx}; + rep = representative(p); + entry = pyramids(rep.entryIdx); + level = entry.levels(rep.levelIdx); + axesOrder = joinAxisNames(entry.axes); + if isempty(options.chunks) + chunks_i = ndi.fun.doc.lightsheet.chooseTileShape(level.shape, ... + axesOrder, level.scale, char(level.dtype), ... + options.tileBudgetBytes); + else + chunks_i = clampChunksToShape(ensureRow(options.chunks), level.shape); + end + levelChunks{idx} = chunks_i; + levelGrids{idx} = ceil(level.shape ./ chunks_i); + if options.materializeChunks + totalChunks = totalChunks + prod(levelGrids{idx}); + end + end + + % Announce the run configuration. Users have hit "the parfor did + % nothing" because MATLAB had cached the old .m file, or because + % no parpool was open at call time. A single up-front print says + % which path is really running. + if options.materializeChunks + [effWorkers, workerSource] = describeParallelPool(options.numWorkers); + asyncStr = 'off'; + if options.asyncPrefetch && effWorkers > 1 + asyncStr = 'on (read overlaps compress)'; + end + fprintf(['[makePyramid] Parallel workers: %d (%s). ' ... + 'Prefetch budget: %.0f MB per read. Async prefetch: %s. ' ... + 'Total chunks: %d across %d level path(s). Codec: %s.\n'], ... + effWorkers, workerSource, ... + options.prefetchBytes / 2^20, asyncStr, ... + totalChunks, numel(pathOrder), options.codec); + end + + % Assign each unique path a 0-based level index in the ORDER it + % first appears (finest first, since listPyramids is finest-first). + levelDocs = cell(numel(pathOrder), 1); + docByPath = containers.Map(); + sharedLevel0 = false; + doneChunks = 0; + for idx = 1:numel(pathOrder) + p = pathOrder{idx}; + rep = representative(p); + entry = pyramids(rep.entryIdx); + level = entry.levels(rep.levelIdx); + rlist = pathReductions(p); + if numel(rlist) > 1 + reductionFn = 'none'; + if idx == 1 + sharedLevel0 = true; + end + else + reductionFn = rlist{1}; + end + if options.materializeChunks + nChunksLvl = prod(levelGrids{idx}); + fprintf(['[makePyramid] Level %d/%d (%s, %s): shape=%s, ' ... + 'chunks=%s, %d chunks to write.\n'], ... + idx, numel(pathOrder), char(entry.name), reductionFn, ... + mat2str(double(level.shape)), ... + mat2str(double(levelChunks{idx})), nChunksLvl); + end + reportProgress(options.progressFcn, doneChunks, totalChunks, ... + sprintf('Level %d/%d (%s, %s): starting', ... + idx, numel(pathOrder), char(entry.name), reductionFn)); + source = []; + if options.materializeChunks + source = resolveLevelSource(session, p, entry, level, ... + options, docByPath); + end + [levelDocs{idx}, doneChunks] = makeOneLevel(session, pyramidDoc, ... + entry, level, idx - 1, reductionFn, options, ... + levelChunks{idx}, levelGrids{idx}, ... + doneChunks, totalChunks, source); + docByPath(p) = levelDocs{idx}; + end + reportProgress(options.progressFcn, max(doneChunks, totalChunks), ... + max(totalChunks, 1), 'Done.'); +end + +function [levelDoc, doneChunks] = makeOneLevel(session, pyramidDoc, pyramidEntry, level, levelIndex, reductionFn, options, chunks, chunkGrid, doneChunks, totalChunks, source) + + if options.materializeChunks + nChunksStored = prod(chunkGrid); + else + nChunksStored = 0; + end + + props = struct( ... + 'label', sprintf('%s level %d (%s)', pyramidEntry.name, levelIndex, reductionFn), ... + 'level', levelIndex, ... + 'reduction_function', reductionFn, ... + 'axes_order', joinAxisNames(pyramidEntry.axes), ... + 'shape', level.shape, ... + 'chunks', chunks, ... + 'chunk_grid', chunkGrid, ... + 'n_chunks_stored', nChunksStored, ... + 'chunk_index_origin', 1, ... + 'chunk_order', 'C', ... + 'dtype', char(level.dtype), ... + 'fill_value', 0, ... + 'byte_order', 'little', ... + 'codec', options.codec, ... + 'codec_params', codecParamsJSON(options), ... + 'voxel_size', ensureRow(level.scale), ... + 'voxel_size_units', 'micrometer', ... + 'translation', ensureRow(level.translation)); + + levelDoc = session.newdocument('lightsheetZarrLevel', ... + 'lightsheetZarrLevel', props); + + levelDoc = levelDoc.set_dependency_value('lightsheetZarrPyramid_id', pyramidDoc.id()); + levelDoc = levelDoc.set_dependency_value('subject_id', options.subjectID); + if ~isempty(options.sourceFileID) + levelDoc = levelDoc.set_dependency_value('source_file_id', options.sourceFileID); + end + + tmpRoot = ''; + if options.materializeChunks + levelLabel = sprintf('Level %d (%s)', levelIndex, reductionFn); + [levelDoc, tmpRoot, doneChunks] = attachChunkFiles(levelDoc, ... + source, level, chunks, chunkGrid, ... + options.codec, options.clevel, ... + options.progressFcn, doneChunks, totalChunks, levelLabel, ... + options.numWorkers, options.prefetchBytes, options.asyncPrefetch, ... + options.debugLog); + end + tmpCleaner = onCleanup(@() cleanupTmp(tmpRoot)); + + session.database_add(levelDoc); +end + +function [levelDoc, tmpRoot, doneChunks] = attachChunkFiles(levelDoc, source, level, chunks, chunkGrid, codec, clevel, progressFcn, doneChunks, totalChunks, levelLabel, numWorkers, prefetchBytes, asyncPrefetch, debugLog) +% Read the source zarr level as a whole array, then re-tile into the +% level document's chunk shape and attach each chunk as chunk.bin_ +% (1-based, C-order over chunkGrid). Files are ingested into DID, which +% deletes the temp original as it copies -- so we build each chunk in a +% new tempname and hand it to add_file. SOURCE (see resolveLevelSource) +% says where the level's bytes are read from. + + % STREAMING per-chunk read. We call readArray with `Region` per + % target chunk instead of loading the whole level up front. Previous + % versions did `data = readArray(...)` and sliced `data(subs{:})` + % per chunk; that reads O(level bytes) into one MATLAB allocation + % and crashes on lab-scale volumes (a stitched SmartSPIM level 0 + % can be hundreds of GB). Reading a single target chunk touches + % only the source chunks that overlap that region, so peak memory + % scales with target chunk size, not level size. + shape = ensureRow(level.shape); + if numel(shape) < numel(chunks) + % Singleton trailing axes: pad shape to axis count. + shape(end+1:numel(chunks)) = 1; + end + + % 1-based linear index goes into the chunk.bin_# filename. + tmpRoot = tempname; + mkdir(tmpRoot); + % No onCleanup here: tmpRoot is returned so the CALLER can register + % the cleanup AFTER session.database_add has ingested the files. If + % we cleaned up on return, database_add would find the paths already + % deleted. + linearOrder = allChunkIndices(chunkGrid); % rows are (c1, c2, c3, ...) 1-based + nChunks = size(linearOrder, 1); + typesize = elementSizeBytes(char(level.dtype)); + + % Locations for the whole level. Filled in as each super-block + % completes; handed to did.document.addFileSeries ONCE at the end + % of the level. Per-chunk add_file returned a fresh levelDoc every + % call, and 40k value-semantics reassignments of a growing struct + % churned through many GB of allocations even though the final + % series entry is small -- the bulk method installs all locations + % in one struct-array assignment, and the observed memory blow-up + % goes away. + seriesLocations = cell(1, nChunks); + + % BATCHED SERIAL-READ / PARALLEL-COMPRESS. On a slow spinning drive + % we cannot afford concurrent seeks, so reads stay strictly serial; + % but Zstd at clevel=9 is single-core CPU-bound and easy to fan out + % across a parallel pool. A batch of `poolSize` chunks is read from + % the source, then handed to the workers to compress and write, in + % parallel; each chunk's tempfile path is parked in seriesLocations + % and installed into the level document via a SINGLE + % did.document.addFileSeries call after the super-block loop exits. + % poolSize == 1 is the original serial path with no toolbox + % dependency. + poolSize = resolveParallelPool(numWorkers); + + % How many target chunks fit inside the prefetch budget, sized + % by uncompressed chunk bytes. Never smaller than the parfor + % pool (a super-block below poolSize would leave workers idle); + % never larger than the level itself. + chunkBytes = prod(chunks) * typesize; + % A reduced level reads prod(factor) source voxels per output voxel + % (and holds them as single while reducing, 2x a uint16), so its + % read buffer is that much bigger than the output region. Shrink + % the budget to match, so peak memory stays near prefetchBytes. + prefetchBytes = double(prefetchBytes) / readAmplification(source, typesize); + prefetchChunkCount = max(poolSize, ... + floor(double(prefetchBytes) / max(chunkBytes, 1))); + prefetchChunkCount = min(prefetchChunkCount, nChunks); + + % Plan every super-block up front (chunk indices + per-chunk lo/hi + % + union bbox) so the async pipeline can dispatch the next read + % while the current one is still compressing. + % + % Packing rule: chunks are consumed from linearOrder in order, but + % a super-block is CLOSED early when adding the next chunk would + % blow up the union bounding box past bboxByteCap. Without this + % cap, a super-block that spans a wrap in the linear chunk order + % (e.g. the last chunks of one row plus the first of the next) + % has a bbox that spans the full range on the fastest-varying + % axis. When two wraps land inside the same super-block, the + % bbox covers a large fraction of the level, and + % readSuperBlockRegion allocates that whole region uncompressed: + % a 40k-chunk lightsheet ingest can jump from ~7 GB to >100 GB + % on a single ill-placed super-block, which we observed. Capping + % the bbox volume keeps peak read allocation bounded to + % ~prefetchBytes regardless of chunk-grid geometry. + bboxByteCap = max(double(prefetchBytes) * 4, chunkBytes * poolSize * 2); + % Preallocate to the worst-case super-block count (one chunk per + % super-block) so Code Analyzer sees a static-size cell rather + % than end+1 growth. Trim at the end. The `nChunks` slot count is + % ~156k pointers = ~1.2 MB, negligible. + superBlocks = cell(nChunks, 1); + nSuperBlocks = 0; + i = 1; + while i <= nChunks + [idxs, lo, hi] = growSuperBlock(linearOrder, i, ... + min(prefetchChunkCount, nChunks - i + 1), ... + chunks, chunkGrid, shape, typesize, bboxByteCap); + nSuperBlocks = nSuperBlocks + 1; + superBlocks{nSuperBlocks} = struct('idxs', idxs, ... + 'loRows', lo, 'hiRows', hi, ... + 'loBBox', min(lo, [], 1), 'hiBBox', max(hi, [], 1)); + i = i + numel(idxs); + end + superBlocks = superBlocks(1:nSuperBlocks); + + % ASYNC PIPELINE. If a parpool is available and asyncPrefetch is + % on, the next super-block's read runs concurrently on a pool + % worker while the parfor is compressing the current one. Total + % time approaches max(read_time, compress_time) instead of their + % sum. Peak RAM ~ 2 * prefetchBytes (two blocks in flight). + useAsync = asyncPrefetch && poolSize > 1 && nSuperBlocks > 1; + if useAsync + pool = gcp(); + nextFut = parfeval(pool, @readSuperBlockRegion, 1, ... + source, superBlocks{1}.loBBox, superBlocks{1}.hiBBox); + end + + for s = 1:nSuperBlocks + sb = superBlocks{s}; + bs = numel(sb.idxs); + + % ---- READ current super-block, DISPATCH the next ----------- + if useAsync + bigTile = fetchOutputs(nextFut); + if s < nSuperBlocks + nb = superBlocks{s + 1}; + nextFut = parfeval(pool, @readSuperBlockRegion, 1, ... + source, nb.loBBox, nb.hiBBox); + end + else + bigTile = readSuperBlockRegion(source, sb.loBBox, sb.hiBBox); + end + + % ---- STAGE bigTile INTO A PARFOR-SLICEABLE ND-ARRAY ------- + % An earlier version handed the parfor a cell array `srcs` + % where each entry held one chunk's sub-array. MATLAB parfor + % could not statically slice that cell (`srcs{j}` on a + % heterogeneous cell defeats its analyzer), so it BROADCAST + % the entire ~512 MB cell to every worker: 512 MB x 18 + % workers x 2 super-blocks in flight (async prefetch) = 18+ + % GB of peak RAM per level, which OOM-killed lightsheet + % ingests on 64 GB machines. Preallocate a plain typed + % ND-array `bigChunks` sized (chunks, bs) instead: assigning + % into each j-th slab pre-pads edge chunks with zeros (so + % the worker sees an already-full-shape tile and skips the + % pad branch of padEncodeAndWrite), and reshaping to + % (prod(chunks), bs) makes it a plain 2-D matrix that + % parfor's slice detector unambiguously slices along the + % second axis. Each worker receives ONLY its 4 MB column, + % not the whole super-block. + srcClass = class(bigTile); + bigChunks = zeros([chunks(:).' bs], srcClass); + chunkIdxs = zeros(1, bs); + for j = 1:bs + chunkIdxs(j) = sb.idxs(j); + srcSubs = cell(1, numel(shape)); + dstSubs = cell(1, numel(chunks)); + for a = 1:numel(shape) + srcSubs{a} = (sb.loRows(j, a) - sb.loBBox(a) + 1) : ... + (sb.hiRows(j, a) - sb.loBBox(a) + 1); + dstSubs{a} = 1:numel(srcSubs{a}); + end + % Write this chunk's occupied sub-region into the j-th + % slab; whatever is beyond it stays zero (that IS the + % pad the writer used to do inside padEncodeAndWrite). + slabAssign = [dstSubs, {j}]; + bigChunks(slabAssign{:}) = bigTile(srcSubs{:}); + end + clear bigTile; + + % Flatten to (prod(chunks), bs) so parfor sees a plain 2-D + % array and slices it along the loop-variable axis with no + % ambiguity. + nChunkElems = prod(chunks); + bigChunksFlat = reshape(bigChunks, nChunkElems, bs); + clear bigChunks; + + % ---- ONE PARFOR ACROSS THE ENTIRE SUPER-BLOCK ------------- + % Workers now do everything from the flat slice onwards -- + % reshape into the chunk shape, permute into C-order, + % typecast to bytes, Blosc encode, and write the tempfile. + chunkShape1 = chunks(:).'; + tempPaths = cell(1, bs); + if poolSize > 1 && bs > 1 + parfor j = 1:bs + col = bigChunksFlat(:, j); + tile = reshape(col, chunkShape1); + tempPaths{j} = padEncodeAndWrite(tile, chunks, ... + codec, clevel, typesize, tmpRoot, chunkIdxs(j)); + end + else + for j = 1:bs + col = bigChunksFlat(:, j); + tile = reshape(col, chunkShape1); + tempPaths{j} = padEncodeAndWrite(tile, chunks, ... + codec, clevel, typesize, tmpRoot, chunkIdxs(j)); + end + end + clear bigChunksFlat; + + % ---- SERIAL ADD_FILE + BATCHED PROGRESS ------------------ + % In-memory metadata registration; still ordered by chunk + % index so the on-disk chunk numbering is unchanged. Progress + % ticks fire ONCE at the end of each super-block instead of + % per chunk: 18 workers producing a burst of 18 nearly- + % simultaneous updateBar calls was making the bar's + % redraw/ETA-recompute path a measurable fraction of the + % gap. One tick per super-block is more than enough + % granularity on a multi-hour ingest. + % ---- ACCUMULATE, DO NOT MUTATE levelDoc PER SUPER-BLOCK --- + % The old code called `levelDoc = levelDoc.add_file(...)` + % 40k+ times per level. Each reassignment triggered a + % copy-on-write on the whole document_properties, churning + % through gigabytes of transient allocations even though the + % final file-series entry is small. Instead, park each + % chunk's tempfile path in seriesLocations{chunkIdx} and + % install them all in ONE addFileSeries call after the + % super-block loop exits. + preAddDocSize = levelDocPropertyBytes(levelDoc); + for j = 1:bs + seriesLocations{chunkIdxs(j)} = tempPaths{j}; + end + postAddDocSize = preAddDocSize; + doneChunks = doneChunks + bs; + reportProgress(progressFcn, doneChunks, totalChunks, ... + sprintf('%s: super-block %d/%d, chunk %d/%d', ... + levelLabel, s, nSuperBlocks, ... + chunkIdxs(end), nChunks)); + bboxMB = prod(double(sb.hiBBox - sb.loBBox + 1)) * typesize / 2^20; + appendDebugLog(debugLog, levelLabel, s, nSuperBlocks, doneChunks, ... + totalChunks, preAddDocSize, postAddDocSize, bboxMB); + end + + % ---- ONE BULK addFileSeries CALL FOR THE WHOLE LEVEL --------- + % did.document.addFileSeries installs the whole cellstr of source + % paths in a single struct-array assignment. The level document's + % file-series manifest names the series `chunk.bin_#`; `indices` + % supplies the 1..N chunk numbering that DID substitutes for `#` + % in each stored location. No per-chunk copy-on-write. + % DID convention: the series MANIFEST name is bare ("chunk.bin"), not + % the "_#" pattern. Members are named chunk.bin_1 ... chunk.bin_N + % automatically. See did.document.seriesNames docstring for the + % contract. The schema declares "chunk.bin" in both file_list and + % file_series. + levelDoc = levelDoc.addFileSeries('chunk.bin', seriesLocations, ... + 'indices', 1:nChunks); +end + +function bytes = levelDocPropertyBytes(levelDoc) +% Approximate size in bytes of levelDoc.document_properties. Uses `whos` +% on a local copy of the struct so the levelDoc object itself is not +% traversed (which would add its own overhead). Returns NaN on any +% error so instrumentation cannot break the ingest. + bytes = NaN; + try + props = levelDoc.document_properties; %#ok + w = whos('props'); + bytes = w.bytes; + catch + end +end + +function appendDebugLog(path, levelLabel, s, nSuperBlocks, doneChunks, totalChunks, preBytes, postBytes, bboxMB) +% Append one line of memory diagnostics to `path`. Failures are +% swallowed; instrumentation must never take down an ingest. + if isempty(path), return; end + if nargin < 9, bboxMB = NaN; end + try + rss = matlabRSSMB(); + futCount = poolFutureCount(); + line = sprintf(['%s | %s | sb=%d/%d | chunks=%d/%d | ' ... + 'rss_MB=%.0f | bbox_MB=%.0f | ' ... + 'doc_pre_MB=%.2f | doc_post_MB=%.2f | ' ... + 'doc_delta_MB=%.2f | live_futures=%d'], ... + char(datetime('now', 'Format', 'yyyy-MM-dd HH:mm:ss')), ... + levelLabel, s, nSuperBlocks, ... + doneChunks, totalChunks, rss, bboxMB, ... + preBytes/2^20, postBytes/2^20, ... + (postBytes - preBytes)/2^20, futCount); + fid = fopen(path, 'a'); + if fid < 0, return; end + cleanup = onCleanup(@() fclose(fid)); + fprintf(fid, '%s\n', line); + catch + end +end + +function mb = matlabRSSMB() +% MATLAB's own process RSS in MB. `memory` is Windows-only; on macOS / +% Linux we shell out to `ps -o rss= -p ` which prints kilobytes. +% +% PID lookup goes through Java's ManagementFactory (available in every +% MATLAB that has bundled Java): getRuntimeMXBean().getName() returns +% "PID@HOSTNAME" on every JVM implementation we care about. This +% sidesteps both matlabProcessID (R2023a+ only) and feature('getpid') +% (Code Analyzer flags as unsupported). + mb = NaN; + try + rt = java.lang.management.ManagementFactory.getRuntimeMXBean(); + pidText = char(rt.getName()); + pid = str2double(strtok(pidText, '@')); + if isnan(pid), return; end + [rc, out] = system(sprintf('ps -o rss= -p %d', pid)); + if rc == 0 + v = sscanf(strtrim(out), '%f'); + if ~isempty(v), mb = v / 1024; end + end + catch + end +end + +function n = poolFutureCount() +% Number of parfeval / parfor futures the pool currently holds. This +% surfaces leaks in the async-prefetch pipeline where a future's output +% never gets released, which retains its bigTile buffer inside the pool. + n = NaN; + try + p = gcp('nocreate'); + if isempty(p), n = 0; return; end + n = numel(p.FevalQueue.QueuedFutures) + numel(p.FevalQueue.RunningFutures); + catch + end +end + +function [idxs, lo, hi] = growSuperBlock(linearOrder, i0, budgetChunks, chunks, chunkGrid, shape, typesize, bboxByteCap) +% GROWSUPERBLOCK - pack chunks starting at index i0 into one super-block, +% closing early when the union bbox would exceed bboxByteCap. +% +% Adds chunks one at a time from linearOrder(i0:end); recomputes the +% union bbox after each. Stops when either budgetChunks chunks have +% been packed, or the NEXT chunk would push the union bbox's +% uncompressed byte cost above bboxByteCap. Guarantees at least one +% chunk in the super-block (the first is always accepted regardless +% of bbox cost, because a single chunk cannot cost more than +% chunkBytes). + idxs = zeros(1, budgetChunks); + lo = zeros(budgetChunks, numel(shape)); + hi = zeros(budgetChunks, numel(shape)); + curLo = inf(1, numel(shape)); + curHi = -inf(1, numel(shape)); + n = 0; + for k = 0:budgetChunks-1 + c = linearOrder(i0 + k, :); + pieceLo = zeros(1, numel(shape)); + pieceHi = zeros(1, numel(shape)); + for a = 1:numel(chunkGrid) + pieceLo(a) = (c(a) - 1) * chunks(a) + 1; + pieceHi(a) = min(pieceLo(a) + chunks(a) - 1, shape(a)); + end + newLo = min(curLo, pieceLo); + newHi = max(curHi, pieceHi); + if n > 0 + newBoxBytes = prod(double(newHi - newLo + 1)) * typesize; + if newBoxBytes > bboxByteCap + break; + end + end + n = n + 1; + idxs(n) = i0 + k; % 1-based linear chunk id + lo(n, :) = pieceLo; + hi(n, :) = pieceHi; + curLo = newLo; + curHi = newHi; + end + idxs = idxs(1:n); + lo = lo(1:n, :); + hi = hi(1:n, :); +end + +function bigTile = readSuperBlockRegion(source, loBBox, hiBBox) +% READSUPERBLOCKREGION - fetch one super-block region [loBBox, hiBBox] +% +% Dispatches on SOURCE.kind (see resolveLevelSource). Kept as a plain +% subfunction (not nested), and every source carries file paths +% rather than a session, so parfeval can run it on a pool worker +% without pulling the caller's workspace or a database handle. + switch source.kind + case 'omezarr' + bigTile = ndr.format.omezarr.readArray(source.zarrPath, ... + source.pyramidName, source.kInEntry, ... + 'Region', [loBBox; hiBBox]); + case 'smartspim' + bigTile = readSmartSpimRegion(source, loBBox, hiBBox); + case 'reduce' + bigTile = readReducedRegion(source, loBBox, hiBBox); + otherwise + error('NDI:lightsheet:makePyramid:unknownSourceKind', ... + 'Unknown level source kind ''%s''.', source.kind); + end +end + +function source = resolveLevelSource(session, p, entry, level, options, docByPath) +% RESOLVELEVELSOURCE - the self-contained read recipe for level path P +% +% Everything a pool worker needs to read any region of the level, +% with no session: an OME-Zarr store + array, SmartSPIM slice paths, +% or (for a reduced level) the on-disk chunk files of the level it +% is reduced from, which was written earlier in this same call. + if isKey(options.levelSources, p) + source = options.levelSources(p); + else + source = struct('kind', 'omezarr'); + end + switch source.kind + case 'omezarr' + if isempty(options.sourceZarrPath) + error('NDI:lightsheet:makePyramid:noSource', ... + 'Level %s has no levelSources entry and no sourceZarrPath.', p); + end + % Which position in this pyramid's datasets holds this + % level? Its 1-based index is what readArray takes. + kInEntry = find(arrayfun(@(L) strcmp(char(L.path), p), ... + entry.levels), 1); + if isempty(kInEntry) + error('NDI:lightsheet:makePyramid:missingLevelInEntry', ... + 'level path %s not found in pyramid %s.', p, char(entry.name)); + end + source = struct('kind', 'omezarr', ... + 'zarrPath', char(options.sourceZarrPath), ... + 'pyramidName', char(entry.name), 'kInEntry', kInEntry); + case 'smartspim' + % Self-contained already (slice paths + dtype). + case 'reduce' + if ~isKey(docByPath, source.fromPath) + error('NDI:lightsheet:makePyramid:reduceOrder', ... + ['Level %s reduces level %s, which has not been ' ... + 'written yet. List finer levels first.'], p, source.fromPath); + end + from = docByPath(source.fromPath); + % Read through ndi.vintage so the level may be either vintage: + % for the v1 lightsheetZarrLevel this call writes, FP is its + % block and chunk i is chunk.bin_i, exactly as before; a + % migrated (V_eta) level is a sampled_body whose chunk i is + % body_data_ (V_eta numbers series members from 0). + fp = ndi.vintage.lightsheetLevel(from); + nFrom = prod(double(fp.chunk_grid)); + paths = cell(1, nFrom); + % Resolve each stored chunk to its file ONCE, here on the + % main thread; workers then read plain files. + for i = 1:nFrom + f = session.database_openbinarydoc(from, ... + ndi.vintage.lightsheetChunkName(from, i)); + paths{i} = f.fullpathfilename; + session.database_closebinarydoc(f); + end + source.fromShape = double(fp.shape(:).'); + source.fromChunks = double(fp.chunks(:).'); + source.fromGrid = double(fp.chunk_grid(:).'); + source.fromCodec = char(fp.codec); + source.fromDtype = char(fp.dtype); + source.chunkPaths = paths; + source.dtype = char(level.dtype); + source.factor = double(source.factor(:).'); + otherwise + error('NDI:lightsheet:makePyramid:unknownSourceKind', ... + 'Unknown level source kind ''%s'' for level %s.', source.kind, p); + end +end + +function a = readAmplification(source, typesize) +% How many bytes the reader holds per output byte (see attachChunkFiles). + a = 1; + if strcmp(source.kind, 'reduce') + a = prod(source.factor); + if strcmp(source.reduction, 'mean') + a = a * max(1, 4 / typesize); % reduced in single precision + end + end +end + +function out = readSmartSpimRegion(source, lo, hi) +% READSMARTSPIMREGION - region [lo, hi] (axes t,c,z,y,x) from 2-D slices +% +% source.slicePaths{c}{z} is the TIFF of channel c at z. Each slice +% spans the whole stitched canvas, so only the (y, x) window is read +% (imread PixelRegion decodes just the strips that cover it). + out = zeros(double(hi - lo + 1), source.dtype); + rows = [lo(4) hi(4)]; + cols = [lo(5) hi(5)]; + for c = lo(2):hi(2) + for z = lo(3):hi(3) + tile = imread(source.slicePaths{c}{z}, 'PixelRegion', {rows, cols}); + if ~isa(tile, source.dtype) + tile = cast(tile, source.dtype); + end + out(1, c - lo(2) + 1, z - lo(3) + 1, :, :) = ... + reshape(tile, [1 1 1 size(tile, 1) size(tile, 2)]); + end + end +end + +function out = readReducedRegion(source, lo, hi) +% READREDUCEDREGION - region [lo, hi] of a level made by block-reducing +% the level below it. +% +% Output voxel i covers source voxels (i-1)*f+1 .. i*f along each axis +% (the last block may be partial), so the region needs source voxels +% (lo-1).*f+1 .. min(hi.*f, fromShape). + f = source.factor; + srcLo = (lo - 1) .* f + 1; + srcHi = min(hi .* f, source.fromShape); + src = readStoredLevelRegion(source, srcLo, srcHi); + out = ndi.fun.doc.lightsheet.reduceBlocks(src, f, source.reduction, source.dtype); +end + +function out = readStoredLevelRegion(source, lo, hi) +% READSTOREDLEVELREGION - region [lo, hi] of a level from its chunk files +% +% Chunks are C-order (last axis fastest), 1-based, each a full +% (zero-padded) `chunks`-shaped tile in C byte order, raw or Blosc. + nd = numel(source.fromShape); + chunks = source.fromChunks; + grid = source.fromGrid; + out = zeros(double(hi - lo + 1), source.fromDtype); + cLo = floor((lo - 1) ./ chunks) + 1; + cHi = floor((hi - 1) ./ chunks) + 1; + strides = ones(1, nd); + for a = nd-1:-1:1 + strides(a) = strides(a + 1) * grid(a + 1); + end + ranges = arrayfun(@(a) cLo(a):cHi(a), 1:nd, 'UniformOutput', false); + combos = cell(1, nd); + [combos{:}] = ndgrid(ranges{:}); + nTiles = numel(combos{1}); + for t = 1:nTiles + c = cellfun(@(g) g(t), combos); + linear = sum((c - 1) .* strides) + 1; + tile = readStoredChunk(source.chunkPaths{linear}, chunks, ... + source.fromCodec, source.fromDtype); + tileLo = (c - 1) .* chunks + 1; + oLo = max(tileLo, lo); + oHi = min(tileLo + chunks - 1, hi); + dst = arrayfun(@(a) (oLo(a) - lo(a) + 1):(oHi(a) - lo(a) + 1), ... + 1:nd, 'UniformOutput', false); + srcSub = arrayfun(@(a) (oLo(a) - tileLo(a) + 1):(oHi(a) - tileLo(a) + 1), ... + 1:nd, 'UniformOutput', false); + out(dst{:}) = tile(srcSub{:}); + end +end + +function tile = readStoredChunk(path, chunks, codec, dtype) +% READSTOREDCHUNK - one stored chunk file -> a `chunks`-shaped array +% +% Inverse of padEncodeAndWrite: bytes are C-order, so reshape to the +% reversed shape and permute back. + fid = fopen(path, 'r'); + if fid < 0 + error('NDI:lightsheet:makePyramid:readChunk', ... + 'Could not open stored chunk %s.', path); + end + raw = fread(fid, Inf, '*uint8'); + fclose(fid); + % 'blosc' is how a migrated (V_eta) level records v1's 'blosc-zstd': + % a Blosc chunk's header carries its own settings (#73 L4). + if any(strcmp(codec, {'blosc-zstd', 'blosc'})) + raw = ndr.format.blosc.decode(raw); + end + vals = typecast(raw(:), dtype); + nd = numel(chunks); + tile = permute(reshape(vals, fliplr(chunks)), nd:-1:1); +end + +function tmp = padEncodeAndWrite(srcTile, chunks, codec, clevel, typesize, tmpRoot, chunkIdx) +% PADENCODEANDWRITE - one chunk, end to end, on the worker +% +% Pure per-chunk work: whatever runs in a parfor iteration lives +% here so the parallel worker has no dependency on outer-function +% state beyond the arguments it receives. +% +% For codec='blosc-zstd', the fast path is a single blosc.encodeChunk +% MEX call from blosc-matlab v0.2.0: pad + permute (Fortran -> Zarr +% C-order) + typecast + Blosc encode happen inside the C code with +% no MATLAB-side intermediate allocations. On a lightsheet ingest +% that removes ~3 full memcpys per chunk (each ~ chunkBytes), which +% at 40k chunks x 5 pyramid levels is hundreds of GB of avoided +% main-thread memory traffic per worker. +% +% Fallback (v0.1.0 MEX, or codec='raw'): do the pad + permute + +% typecast in MATLAB, then either return the raw bytes or hand them +% to ndr.format.blosc.encode. Byte-identical output. + switch codec + case 'raw' + payload = padPermuteBytes(srcTile, chunks); + case 'blosc-zstd' + if canUseEncodeChunk() + payload = blosc.encodeChunk(srcTile, chunks(:).', ... + 'cname', 'zstd', ... + 'clevel', clevel, ... + 'shuffle', 1); + else + raw = padPermuteBytes(srcTile, chunks); + payload = ndr.format.blosc.encode(raw, ... + 'typesize', typesize, ... + 'cname', 'zstd', ... + 'clevel', clevel, ... + 'shuffle', 1); + end + otherwise + error('NDI:lightsheet:makePyramid:unknownCodec', ... + 'Unknown codec %s.', codec); + end + tmp = fullfile(tmpRoot, sprintf('chunk_%d.bin', chunkIdx)); + fid = fopen(tmp, 'w'); + fwrite(fid, payload); + fclose(fid); +end + +function tf = canUseEncodeChunk() +% CANUSEENCODECHUNK - does this worker have blosc.encodeChunk on its path? +% +% Runs on the parfor worker (padEncodeAndWrite is called inside the +% parfor). Each worker is a separate MATLAB process with its own +% `persistent` scope, so the first call in a worker triggers +% ensureMex (which downloads blosc-matlab v0.2.0 to prefdir() on +% first use in that worker's session) and every subsequent call in +% the same worker is a memoized flag read. +% +% Falls back to false on any error so a caller without NDR-matlab +% or without a working install still gets the pad-permute-typecast +% MATLAB path via ndr.format.blosc.encode. + persistent flag + if ~isempty(flag), tf = flag; return; end + flag = false; + try + info = ndr.util.blosc.ensureMex(); + flag = info.available && exist('blosc.encodeChunk', 'file') == 2; + catch + end + tf = flag; +end + +function bytes = padPermuteBytes(srcTile, chunks) +% PADPERMUTEBYTES - Fortran ND-array -> zero-padded C-order uint8 bytes. +% +% The MATLAB fallback for the Blosc fast path: pad the tile up to +% the target chunk shape with zeros, permute axes into reversed +% order so the row-major byte layout matches what Zarr writes, then +% typecast to bytes. Only used when the v0.2.0 blosc-matlab MEX is +% not available. + if isequal(size(srcTile), chunks) + permuted = permute(srcTile, numel(chunks):-1:1); + else + padded = zeros(chunks, class(srcTile)); + localSubs = arrayfun(@(k) 1:size(srcTile, k), ... + 1:numel(chunks), 'UniformOutput', false); + padded(localSubs{:}) = srcTile; + permuted = permute(padded, numel(chunks):-1:1); + end + bytes = typecast(permuted(:), 'uint8'); +end + +function [n, src] = describeParallelPool(numWorkers) +% DESCRIBEPARALLELPOOL - peek at the effective pool without side effects +% +% Returns [n, src] where n is the worker count that resolveParallelPool +% would end up with, and src is a short human string naming why. +% No pool is created; used only to print the run banner before the +% pyramid loop starts. + if numWorkers == 0 || numWorkers == 1 + n = 1; src = 'serial, forced by numWorkers'; return; + end + if isempty(ver('parallel')) + n = 1; src = 'serial, Parallel Computing Toolbox not installed'; return; + end + p = gcp('nocreate'); + if numWorkers < 0 + if isempty(p) + n = 1; src = 'serial, no parpool open (call parpool(N) to enable)'; + else + n = p.NumWorkers; src = 'from existing parpool'; + end + return; + end + if isempty(p) || p.NumWorkers ~= numWorkers + n = numWorkers; src = 'will open a new parpool'; + else + n = numWorkers; src = 'from existing parpool'; + end +end + +function n = resolveParallelPool(numWorkers) +% RESOLVEPARALLELPOOL - resolve numWorkers into an effective batch size +% +% -1 -> the size of any parpool already open, else 1 (serial). This +% is the default so a caller who ran `parpool(8)` once at the +% top of their session gets 8-way compression without having +% to name a number in every ingest call. +% 0 or 1 -> serial. +% N > 1 -> ensure a parpool of size N is running (create it or +% restart the existing one at a different size), returning N. +% +% In every case a missing PCT or a failed parpool startup degrades +% to serial with a warning: a broken pool must not stop an ingest +% that may already be hours in. + if numWorkers == 0 || numWorkers == 1 + n = 1; + return; + end + if isempty(ver('parallel')) + if numWorkers > 1 + warning('NDI:lightsheet:makePyramid:noPCT', ... + ['numWorkers=%d requested but Parallel Computing ' ... + 'Toolbox is not available. Running serially.'], ... + numWorkers); + end + n = 1; + return; + end + try + p = gcp('nocreate'); + if numWorkers < 0 % 'auto' -> reuse open pool, else serial + if isempty(p) + n = 1; + else + n = p.NumWorkers; + end + return; + end + if isempty(p) || p.NumWorkers ~= numWorkers + if ~isempty(p) + delete(p); + end + parpool('local', numWorkers); + end + n = numWorkers; + catch ME + warning('NDI:lightsheet:makePyramid:parpoolFailed', ... + ['Could not use a parpool of %d workers (%s). ' ... + 'Running serially.'], numWorkers, ME.message); + n = 1; + end +end + +function s = codecParamsJSON(options) +% JSON string stamped into the level document so a reader knows how to +% decode. For 'raw' the params are empty; for Blosc-Zstd they name the +% codec identity numcodecs.Blosc expects. + switch options.codec + case 'raw' + s = '{}'; + case 'blosc-zstd' + s = sprintf(['{"cname":"zstd","clevel":%d,"shuffle":1,' ... + '"blocksize":0}'], options.clevel); + otherwise + s = '{}'; + end +end + +function b = elementSizeBytes(className) + switch className + case {'uint8','int8','logical'} + b = 1; + case {'uint16','int16'} + b = 2; + case {'uint32','int32','single'} + b = 4; + case {'uint64','int64','double'} + b = 8; + otherwise + b = 2; + end +end + +function cleanupTmp(d) + if isfolder(d) + try + rmdir(d, 's'); + catch + end + end +end + +function reportProgress(fcn, done, total, text) +% REPORTPROGRESS - call the caller's progress callback, if any +% +% Fraction is done/total, clamped into [0,1]. Total==0 (no chunks +% materialized) reports 0 rather than dividing, so the caller's bar +% still gets its label refreshed. Any exception is swallowed: a bar +% that stopped drawing must not stop an ingest. + if isempty(fcn), return; end + frac = 0; + if total > 0 + frac = max(0, min(1, double(done) / double(total))); + end + try + fcn(frac, char(text)); + catch + end +end + +function idx = allChunkIndices(chunkGrid) +% Return a matrix whose rows enumerate every 1-based chunk index tuple +% in C-order (last axis fastest, so the LINEAR order matches +% index = c1*n2*n3*... + c2*n3*... + ... + cn + 1). + n = numel(chunkGrid); + counts = cumprod(fliplr(chunkGrid)); + total = counts(end); + idx = zeros(total, n); + for r = 1:total + rem = r - 1; + for a = 1:n + % last axis is fastest + stride = 1; + for b = a+1:n + stride = stride * chunkGrid(b); + end + idx(r, a) = floor(rem / stride) + 1; + rem = mod(rem, stride); + end + end +end + +function s = joinAxisNames(axes) + if isempty(axes) + s = ''; + return; + end + names = arrayfun(@(a) char(a.name), axes, 'UniformOutput', false); + s = strjoin(names, ''); +end + +function u = collectAxisUnits(axes) + if isempty(axes) + u = {}; + return; + end + u = arrayfun(@(a) char(a.unit), axes, 'UniformOutput', false); +end + +function n = channelCount(shape, axesOrder) + c = strfind(axesOrder, 'c'); + if isempty(c) || numel(shape) < c(1) + n = 1; + else + n = shape(c(1)); + end +end + +function v = ensureRow(x) + if isempty(x) + v = []; + else + v = reshape(double(x), 1, []); + end +end + +function chunks = clampChunksToShape(chunks, shape) + shape = double(shape(:).'); + chunks = double(chunks(:).'); + if numel(chunks) ~= numel(shape) + error('NDI:lightsheet:makePyramid:chunksArity', ... + 'Explicit chunks length %d does not match level shape length %d.', ... + numel(chunks), numel(shape)); + end + if any(chunks < 1) || any(chunks ~= fix(chunks)) + error('NDI:lightsheet:makePyramid:badChunks', ... + 'Explicit chunks must be positive integers.'); + end + chunks = min(chunks, shape); +end + diff --git a/src/ndi/+ndi/+fun/+doc/+lightsheet/makeSourceFile.m b/src/ndi/+ndi/+fun/+doc/+lightsheet/makeSourceFile.m new file mode 100644 index 0000000000..801234aacc --- /dev/null +++ b/src/ndi/+ndi/+fun/+doc/+lightsheet/makeSourceFile.m @@ -0,0 +1,148 @@ +function doc = makeSourceFile(session, filename, options) +% MAKESOURCEFILE - describe, or attach, the source OME-Zarr a pyramid came from +% +% DOC = ndi.fun.doc.lightsheet.MAKESOURCEFILE(SESSION, FILENAME) +% DOC = ndi.fun.doc.lightsheet.MAKESOURCEFILE(..., 'attachFile', true) +% +% Creates the document that lightsheetZarrPyramid points at through +% its source_file_id dependency, so a pyramid can name the store it +% was built from and a later reader can verify it is looking at the +% same bytes. +% +% DESCRIBING IS NOT ATTACHING, and the default is to describe. An +% OME-Zarr store is a directory tree of tens to hundreds of GB; +% ingesting a copy of it alongside the pyramid derived from it would +% double the storage to hold bytes the pyramid already summarises. +% The document records the name, the size, the checksum and the dates +% -- enough to identify the file, verify it if it is still around, +% and say what a pyramid came from. +% +% The two cases are two classes, mirroring the +gene package's +% makeSourceFile after issue #981 taught the same lesson: +% +% attachFile false -> fileReference, which holds no file +% attachFile true -> generic_file, which holds generic_file.ext +% +% generic_file's schema declares generic_file.ext with mustbenotempty +% 1, so a generic_file without its file is not a lighter version of +% one, it is an invalid one. +% +% THE CHECKSUM IS THE PART WORTH HAVING. A filename tells you what +% somebody called a file; a checksum tells you whether the file you +% have now is the file the pyramid was built from. +% +% FILENAME may be a file or a directory. For an OME-Zarr store +% (a directory tree) the checksum, when requested, is computed over +% the store's `.zattrs` file, which is small and identifies the +% multiscales layout uniquely; the alternative -- walking every +% chunk file -- would cost the full pyramid read that the caller +% is trying to avoid. +% +% Optional Name-Value Arguments: +% checksum (true) - compute the MD5 identifying fingerprint. For a +% directory (an OME-Zarr store) this is the MD5 +% of .zattrs; for a plain file it is the MD5 +% of the file. Pass false to skip; the field is +% left empty rather than filled with a +% placeholder. +% attachFile (false) - ingest a copy of the file itself, and return a +% generic_file document rather than a +% fileReference. Only valid for a plain file -- +% attaching a directory tree is not supported. +% formatOntology ('') - an ontology term for the format, if you have +% one. Left EMPTY rather than guessed. +% documentID ('') - a document this file is about, for the +% class's own document_id dependency. +% +% Outputs: +% DOC - the ndi.document -- fileReference, or generic_file when +% attachFile is true -- NOT yet added to the database. The +% caller adds it, because it is normally created as part of a +% set whose dependencies have to be wired before any of them is +% stored. +% +% Example: +% src = ndi.fun.doc.lightsheet.makeSourceFile(S, '/data/vol.ome.zarr'); +% S.database_add(src); +% +% See also: ndi.fun.doc.lightsheet.fromOMEZarr, +% ndi.fun.doc.gene.makeSourceFile, +% ndi.fun.file.MD5 + + arguments + session (1,1) + filename (1,:) char + options.checksum (1,1) logical = true + options.attachFile (1,1) logical = false + options.formatOntology (1,:) char = '' + options.documentID (1,:) char = '' + end + + isDir = isfolder(filename); + isFile = ~isDir && isfile(filename); + if ~isDir && ~isFile + error('NDI:lightsheet:makeSourceFile:noSuchPath', ... + 'No file or directory at %s.', filename); + end + if options.attachFile && isDir + error('NDI:lightsheet:makeSourceFile:cannotAttachDirectory', ... + ['attachFile is only valid for a plain file. An OME-Zarr ' ... + 'store is a directory tree and is described, not ingested.']); + end + + [~, base, ext] = fileparts(filename); + leafname = [base ext]; + + checksum = ''; + algorithm = ''; + if options.checksum + if isDir + zattrs = fullfile(filename, '.zattrs'); + if isfile(zattrs) + checksum = ndi.fun.file.MD5(zattrs); + end + else + checksum = ndi.fun.file.MD5(filename); + end + if ~isempty(checksum) + algorithm = 'MD5'; + end + end + + created = 0; + updated = 0; + try + created = convertTo(ndi.fun.file.dateCreated(filename), 'datenum'); + updated = convertTo(ndi.fun.file.dateUpdated(filename), 'datenum'); + catch + end + + if options.attachFile + s = struct('filename', leafname, ... + 'formatOntology', options.formatOntology, ... + 'dateCreated', created, 'dateUpdated', updated, ... + 'checksum', checksum); + doc = ndi.document('generic_file', 'generic_file', s) + session.newdocument(); + doc = doc.add_file('generic_file.ext', filename, 'delete_original', 0); + else + fileSize = 0; + if isFile + d = dir(filename); + if ~isempty(d) + fileSize = double(d(1).bytes); + end + end + s = struct('filename', leafname, ... + 'originalPath', filename, ... + 'formatOntology', options.formatOntology, ... + 'dateCreated', created, 'dateUpdated', updated, ... + 'fileSize', fileSize, ... + 'checksum', checksum, ... + 'checksumAlgorithm', algorithm); + doc = ndi.document('fileReference', 'fileReference', s) + session.newdocument(); + end + + if ~isempty(options.documentID) + doc = doc.set_dependency_value('document_id', options.documentID); + end +end diff --git a/src/ndi/+ndi/+fun/+doc/+lightsheet/reduceBlocks.m b/src/ndi/+ndi/+fun/+doc/+lightsheet/reduceBlocks.m new file mode 100644 index 0000000000..212cf6bd75 --- /dev/null +++ b/src/ndi/+ndi/+fun/+doc/+lightsheet/reduceBlocks.m @@ -0,0 +1,113 @@ +function out = reduceBlocks(arr, factor, reduction, outClass) +% NDI.FUN.DOC.LIGHTSHEET.REDUCEBLOCKS - block-reduce an N-D array, vectorized +% +% OUT = ndi.fun.doc.lightsheet.REDUCEBLOCKS(ARR, FACTOR, REDUCTION) +% OUT = ndi.fun.doc.lightsheet.REDUCEBLOCKS(ARR, FACTOR, REDUCTION, OUTCLASS) +% +% Downsamples ARR by the per-axis integer FACTOR (one entry per axis +% of ARR, trailing singleton axes included; a scalar applies to every +% axis) using REDUCTION, 'mean' or 'max'. Output element i along an +% axis covers input elements (i-1)*f+1 .. i*f; the trailing block may +% be partial and is reduced over the elements it has, so an axis of +% length 5 with factor 2 gives 3 outputs (blocks of 2, 2 and 1). These +% are the semantics of ndr.format.omezarr.reduce and of the NGFF +% writers whose pyramids NDI ingests, so a pyramid built here matches +% one built by spim2zarr. +% +% OUTCLASS (default: the class of ARR) is the class of OUT. A 'mean' +% into an integer class is rounded to nearest. +% +% Vectorized one axis at a time: a block's mean is the mean of the +% per-axis means (the block is a Cartesian product), and likewise for +% max, so this costs a handful of whole-array operations instead of +% one loop iteration per output voxel. The mean is accumulated in +% single precision for 8- and 16-bit inputs (exact sums for blocks up +% to 256 voxels) and in double otherwise. +% +% Example: +% out = ndi.fun.doc.lightsheet.reduceBlocks(uint16(magic(5)), [2 2], 'mean'); +% +% See also: ndr.format.omezarr.reduce, ndi.fun.doc.lightsheet.fromSPIM + + arguments + arr {mustBeNumeric} + factor (1,:) double {mustBePositive, mustBeInteger} + reduction (1,:) char {mustBeMember(reduction, {'mean','max'})} + outClass (1,:) char = class(arr) + end + + if isscalar(factor) + factor = repmat(factor, 1, ndims(arr)); + end + if numel(factor) < ndims(arr) + error('NDI:lightsheet:reduceBlocks:badFactor', ... + 'factor has %d entries for a %d-D array.', numel(factor), ndims(arr)); + end + sz = size(arr, 1:numel(factor)); + + if strcmp(reduction, 'mean') + if any(strcmp(class(arr), {'uint8','int8','uint16','int16','single'})) + work = single(arr); + else + work = double(arr); + end + else + work = arr; + end + + for d = 1:numel(factor) + f = factor(d); + if f == 1 || sz(d) == 1 + continue; % nothing to reduce along this axis + end + work = reduceAxis(work, d, f, reduction); + sz(d) = size(work, d); + end + + if strcmp(reduction, 'mean') && ~any(strcmp(outClass, {'single','double'})) + work = round(work); + end + out = cast(work, outClass); +end + +function A = reduceAxis(A, d, f, reduction) +% Reduce A along axis D by factor F, trailing partial block included. + n = size(A, d); + m = ceil(n / f); + pad = m * f - n; + nd = max(ndims(A), d); + sz = size(A, 1:nd); + if pad > 0 + padSz = sz; + padSz(d) = pad; + switch reduction + case 'mean' + % Zeros add nothing to the sum; the last block's divisor + % is corrected below. + A = cat(d, A, zeros(padSz, 'like', A)); + case 'max' + % Repeat the last element: it cannot change the max. + idx = repmat({':'}, 1, nd); + idx{d} = repmat(n, 1, pad); + A = cat(d, A, A(idx{:})); + end + sz(d) = m * f; + end + % Split axis d into (f, m) and reduce over the f part. + newSz = [sz(1:d-1), f, m, sz(d+1:end)]; + A = reshape(A, newSz); + switch reduction + case 'mean' + A = sum(A, d); + counts = repmat(f, 1, m); + counts(end) = n - (m - 1) * f; + cshape = ones(1, numel(newSz)); + cshape(d + 1) = m; + A = A ./ reshape(cast(counts, 'like', A), cshape); + case 'max' + A = max(A, [], d); + end + outSz = sz; + outSz(d) = m; + A = reshape(A, outSz); +end diff --git a/src/ndi/+ndi/+fun/+doc/+lightsheet/view.m b/src/ndi/+ndi/+fun/+doc/+lightsheet/view.m new file mode 100644 index 0000000000..961d2aa54d --- /dev/null +++ b/src/ndi/+ndi/+fun/+doc/+lightsheet/view.m @@ -0,0 +1,127 @@ +function view(session, pyramidID, options) +% NDI.FUN.DOC.LIGHTSHEET.VIEW - launch the napari viewer on a lightsheet pyramid +% +% ndi.fun.doc.lightsheet.VIEW(SESSION) +% ndi.fun.doc.lightsheet.VIEW(SESSION, PYRAMIDID) +% ndi.fun.doc.lightsheet.VIEW(..., 'launcher', '/path/to/napariViewLightsheet') +% +% Builds the napariViewLightsheet command line (via +% ndi.fun.doc.lightsheet.VIEWCOMMAND) and dispatches it through SYSTEM. +% The launcher path defaults to /usr/local/bin/napariViewLightsheet, +% the shell wrapper installed alongside NDI-python. +% +% PYRAMIDID is optional. When omitted, the launcher runs in --list +% mode: it enumerates every lightsheetZarrPyramid in the session and +% prints their ids on stdout so the caller can pick one for a second +% call. When PYRAMIDID is passed, the pyramid is opened in napari. +% +% Optional Name-Value Arguments: +% launcher - char, path to the shell wrapper. Default +% /usr/local/bin/napariViewLightsheet. +% sessionPath - char, override the session directory used for the +% viewer's --session argument. Default: session.path(). +% reduction - char, '' | 'mean' | 'max'. Passed through to +% viewCommand; empty leaves the choice to what the +% pyramid id names. +% channel - 1-based channel index, or -1 for all channels. +% Passed through to viewCommand. +% level - initial level; -1 lets the viewer choose. +% controls - dock the reduction / channel / level control panels +% (default true). +% name - napari layer name; empty uses the pyramid's own +% label. +% wait - block until the viewer exits (default false so +% MATLAB stays interactive; the command is +% backgrounded on POSIX shells). +% verbose - print the command before running it (default true). +% debug - run the viewer with NDI_LIGHTSHEET_DEBUG=1 (default +% true). +% logFile - where the viewer's stdout/stderr go. Default: a new +% lightsheet_YYYYMMDDHHMMSS.txt in +% ndi.common.PathConstants.TempFolder (the path is +% printed). See ndi.fun.doc.lightsheet.launchViewer. +% +% Example -- list, then open: +% ndi.fun.doc.lightsheet.view(S); % prints ids +% ndi.fun.doc.lightsheet.view(S, ''); +% +% See also: ndi.fun.doc.lightsheet.viewCommand, +% ndi.fun.doc.lightsheet.launchViewer, +% ndi.gui.app.LightsheetZarrManager + + arguments + session (1,1) + pyramidID char = '' + options.launcher (1,:) char = '/usr/local/bin/napariViewLightsheet' + options.sessionPath (1,:) char = '' + options.reduction (1,:) char {mustBeMember(options.reduction, {'', 'mean', 'max'})} = '' + options.channel (1,1) double {mustBeInteger} = -1 + options.level (1,1) double {mustBeInteger} = -1 + options.controls (1,1) logical = true + options.name (1,:) char = '' + options.wait (1,1) logical = false + options.verbose (1,1) logical = true + options.debug (1,1) logical = true + options.logFile (1,:) char = '' + end + + sessionPath = options.sessionPath; + if isempty(sessionPath) + try + sessionPath = char(session.path()); + catch + error('NDI:lightsheet:view:noSessionPath', ... + ['Could not read a directory from the session; pass ' ... + '''sessionPath'' explicitly.']); + end + end + + if isempty(pyramidID) + cmd = sprintf('%s %s --list', shq(options.launcher), shq(sessionPath)); + else + cmd = ndi.fun.doc.lightsheet.viewCommand(options.launcher, ... + sessionPath, pyramidID, ... + 'channel', options.channel, ... + 'reduction', options.reduction, ... + 'controls', options.controls, ... + 'name', options.name, ... + 'level', options.level); + end + + if isempty(pyramidID) + % --list prints the ids for the caller to read: keep it on screen. + if ~options.wait && ~ispc + % Background the process on POSIX so MATLAB does not block. + % On Windows fall back to a foreground wait (simple; no + % `start` handling here). + cmd = [cmd ' &']; + end + if options.verbose + fprintf('Executing: %s\n', cmd); + end + status = system(cmd); + else + % The viewer itself: debugging lines on, and stdout/stderr kept + % in a log in NDI's temporary folder -- launched from MATLAB it + % has no terminal, so this is the only record of what it did. + [status, logFile] = ndi.fun.doc.lightsheet.launchViewer(cmd, ... + 'wait', options.wait, 'debug', options.debug, ... + 'logFile', options.logFile, 'verbose', options.verbose); + end + if status ~= 0 && options.wait + if isempty(pyramidID) + warning('NDI:lightsheet:view:nonZeroExit', ... + 'Launcher returned status %d.', status); + else + warning('NDI:lightsheet:view:nonZeroExit', ... + 'Launcher returned status %d; see %s.', status, logFile); + end + end +end + +function s = shq(s) +% Minimal POSIX shell quoting: wrap in single quotes and escape any +% single quote inside. Enough for paths and identifiers; not a general +% shell escaper. + s = ['''' strrep(s, '''', '''\''''') '''']; +end diff --git a/src/ndi/+ndi/+fun/+doc/+lightsheet/viewCommand.m b/src/ndi/+ndi/+fun/+doc/+lightsheet/viewCommand.m new file mode 100644 index 0000000000..eb2bea5231 --- /dev/null +++ b/src/ndi/+ndi/+fun/+doc/+lightsheet/viewCommand.m @@ -0,0 +1,104 @@ +function cmd = viewCommand(launcher, sessionPath, pyramidID, options) +% NDI.FUN.DOC.LIGHTSHEET.VIEWCOMMAND - the command line that opens a pyramid in napari +% +% CMD = NDI.FUN.DOC.LIGHTSHEET.VIEWCOMMAND(LAUNCHER, SESSIONPATH, PYRAMIDID) +% returns the shell command, as a char row vector, that hands this +% lightsheetZarrPyramid to the napari viewer in NDI-python. +% +% PURE, AND THAT IS THE POINT. Nothing here launches anything or +% touches the filesystem, so the command can be built, shown to the +% user in a dialog, and asserted in a test with no display and no +% Python. When the launch then fails, the exact command is on screen +% to be copied into a terminal, which is the difference between a bug +% report and a shrug. +% +% THE PYRAMID IS ALWAYS NAMED. The viewer will refuse to open a +% session that holds more than one lightsheetZarrPyramid without being +% told which one, so this function refuses to build a command without +% PYRAMIDID. +% +% Optional Name-Value Arguments: +% channel (-1) - 1-based channel index to display. -1 opens all +% channels as separate napari layers. +% reduction ('') - request an alternate reduction: '' opens the +% pyramid the user picked; 'mean' or 'max' +% instructs the viewer to look for a sibling +% pyramid with that reduction and open that one +% instead. Empty leaves the choice to what the +% pyramid id names. +% controls (true) - dock the reduction / channel / level panels +% name ('') - image layer name. '' leaves the viewer to use +% the pyramid's own label. +% level (-1) - initial level to display. -1 lets the viewer +% pick based on the window size (levelTable is +% cheap; picking is not expensive). +% +% See also: ndi.gui.app.LightsheetZarrManager, +% ndi.fun.doc.lightsheet.chooseLevel + + arguments + launcher char + sessionPath char + pyramidID char + options.channel (1,1) double {mustBeInteger} = -1 + options.reduction char {mustBeMember(options.reduction, {'', 'mean', 'max'})} = '' + options.controls (1,1) logical = true + options.name char = '' + options.level (1,1) double {mustBeInteger} = -1 + end + + if isempty(strtrim(launcher)) + error('NDI:lightsheet:viewCommand:noLauncher', ... + ['No viewer launcher is set. It is normally ' ... + '/usr/local/bin/napariViewLightsheet, a shell wrapper ' ... + 'that scrubs MATLAB''s library paths before starting ' ... + 'Python.']); + end + if isempty(strtrim(sessionPath)) + error('NDI:lightsheet:viewCommand:noSessionPath', ... + ['This session has no directory on disk, so there is ' ... + 'nothing for the viewer to open. The napari viewer ' ... + 'reads an ndi.session.dir.']); + end + if isempty(strtrim(pyramidID)) + error('NDI:lightsheet:viewCommand:noPyramidID', ... + ['viewCommand refuses to build a command without a ' ... + 'lightsheetZarrPyramid id; the viewer will not guess ' ... + 'which pyramid to open when a session holds more than ' ... + 'one.']); + end + + parts = { ... + shellQuote(strtrim(launcher)), ... + shellQuote(sessionPath), ... + '--pyramid', shellQuote(pyramidID)}; + if ~isempty(options.reduction) + parts{end+1} = '--reduction'; + parts{end+1} = shellQuote(options.reduction); + end + if options.channel >= 1 + parts{end+1} = '--channel'; + parts{end+1} = sprintf('%d', options.channel); + end + if options.level >= 0 + parts{end+1} = '--level'; + parts{end+1} = sprintf('%d', options.level); + end + if ~options.controls + parts{end+1} = '--no-controls'; + end + if ~isempty(strtrim(options.name)) + parts{end+1} = '--name'; + parts{end+1} = shellQuote(strtrim(options.name)); + end + cmd = strjoin(parts, ' '); +end + +function s = shellQuote(str) + str = char(str); + if ispc + s = ['"' strrep(str, '"', '""') '"']; + else + s = ['''' strrep(str, '''', '''\''''') '''']; + end +end diff --git a/src/ndi/+ndi/+fun/+doc/+lightsheet/viewerLogFile.m b/src/ndi/+ndi/+fun/+doc/+lightsheet/viewerLogFile.m new file mode 100644 index 0000000000..21a5b537cf --- /dev/null +++ b/src/ndi/+ndi/+fun/+doc/+lightsheet/viewerLogFile.m @@ -0,0 +1,20 @@ +function logFile = viewerLogFile(when) +% NDI.FUN.DOC.LIGHTSHEET.VIEWERLOGFILE - path for a new napari viewer log +% +% LOGFILE = ndi.fun.doc.lightsheet.VIEWERLOGFILE() +% LOGFILE = ndi.fun.doc.lightsheet.VIEWERLOGFILE(WHEN) +% +% Returns /lightsheet_YYYYMMDDHHMMSS.txt +% for the time WHEN (a datetime; default now). Named by launch time so +% each viewer gets its own log and a folder listing sorts them in +% launch order. Does not create the file or the folder. +% +% See also: ndi.fun.doc.lightsheet.launchViewer + + arguments + when (1,1) datetime = datetime('now') + end + stamp = char(datetime(when, 'Format', 'yyyyMMddHHmmss')); + logFile = fullfile(ndi.common.PathConstants.TempFolder, ... + ['lightsheet_' stamp '.txt']); +end diff --git a/src/ndi/+ndi/+fun/+doc/+lightsheet/viewerShellCommand.m b/src/ndi/+ndi/+fun/+doc/+lightsheet/viewerShellCommand.m new file mode 100644 index 0000000000..e3d589ad5b --- /dev/null +++ b/src/ndi/+ndi/+fun/+doc/+lightsheet/viewerShellCommand.m @@ -0,0 +1,51 @@ +function fullCmd = viewerShellCommand(cmd, logFile, wait, debug, isWindows) +% NDI.FUN.DOC.LIGHTSHEET.VIEWERSHELLCOMMAND - the shell line that runs a viewer with a log +% +% FULLCMD = ndi.fun.doc.lightsheet.VIEWERSHELLCOMMAND(CMD, LOGFILE, WAIT, DEBUG, ISWINDOWS) +% +% Wraps the viewer command CMD so that, when passed to SYSTEM: +% * NDI_LIGHTSHEET_DEBUG=1 is set for the viewer process (if DEBUG), +% * its stdout and stderr are APPENDED to LOGFILE, and +% * it runs detached (unless WAIT), so MATLAB is not blocked for as +% long as the viewer is open. +% +% Pure: builds the string only, for either shell (ISWINDOWS picks +% cmd.exe syntax over POSIX sh), so it can be tested anywhere. +% ndi.fun.doc.lightsheet.launchViewer is the function that runs it. +% +% POSIX: NDI_LIGHTSHEET_DEBUG=1 CMD >>'LOGFILE' 2>&1 & +% Windows: start "napari" /min cmd /c "set NDI_LIGHTSHEET_DEBUG=1&& CMD >>"LOGFILE" 2>&1" +% +% See also: ndi.fun.doc.lightsheet.launchViewer + + arguments + cmd (1,:) char + logFile (1,:) char + wait (1,1) logical = false + debug (1,1) logical = true + isWindows (1,1) logical = ispc + end + + if isWindows + inner = sprintf('%s >>"%s" 2>&1', cmd, logFile); + if debug + % No space before &&: cmd.exe would keep it in the value. + inner = ['set NDI_LIGHTSHEET_DEBUG=1&& ' inner]; + end + if wait + fullCmd = inner; + else + % cmd /c strips the outer quotes and runs the rest as typed. + fullCmd = ['start "napari" /min cmd /c "' inner '"']; + end + else + quotedLog = ['''' strrep(logFile, '''', '''\''''') '''']; + fullCmd = [cmd ' >>' quotedLog ' 2>&1']; + if debug + fullCmd = ['NDI_LIGHTSHEET_DEBUG=1 ' fullCmd]; + end + if ~wait + fullCmd = [fullCmd ' &']; + end + end +end diff --git a/src/ndi/+ndi/+fun/+file/crc32.m b/src/ndi/+ndi/+fun/+file/crc32.m new file mode 100644 index 0000000000..421f424ed9 --- /dev/null +++ b/src/ndi/+ndi/+fun/+file/crc32.m @@ -0,0 +1,54 @@ +function checksum = crc32(fileName, options) +%CRC32 CRC-32 of a file, base64-encoded the way NDI Cloud and S3 store it. +% +% SYNTAX: +% checksum = ndi.fun.file.crc32(fileName) +% checksum = ndi.fun.file.crc32(fileName, 'chunkSize', BYTES) +% +% DESCRIPTION: +% Computes the CRC-32 (IEEE, the zlib / java.util.zip.CRC32 polynomial) of +% the file's bytes and returns the base64 of its 4-byte big-endian value. +% That is the encoding S3 uses for x-amz-checksum-crc32, the value +% ndi.cloud.api.files.getFileUploadURL sends as checksumCRC32, and the +% value NDI Cloud returns as a file's crc32 (from getFileDetails, listFiles, +% lookupFiles and getSignedURLSet). Comparing this to the cloud's crc32 +% tells whether the cloud copy has the same content as the local file. +% +% The file is read in chunks, so large files do not need to fit in memory. +% +% INPUTS: +% fileName - Path to the file. +% +% NAME-VALUE INPUTS: +% chunkSize - Bytes read per chunk (default 8 MiB). +% +% OUTPUTS: +% checksum - 1x8 char, e.g. 'NhCmhg==' for a file containing 'hello'. +% +% EXAMPLE: +% c = ndi.fun.file.crc32('chunk.bin_1'); +% +% See also: ndi.cloud.api.files.lookupFiles, ndi.cloud.api.files.getFileUploadURL + + arguments + fileName (1,1) string {mustBeFile} + options.chunkSize (1,1) double {mustBePositive, mustBeInteger} = 8*1024*1024 + end + + fid = fopen(fileName, 'r'); + if fid < 0 + error('NDI:fun:file:crc32:CannotOpen', 'Could not open file: %s', fileName); + end + closer = onCleanup(@() fclose(fid)); + + crc = java.util.zip.CRC32(); + while true + data = fread(fid, options.chunkSize, '*int8'); + if isempty(data) + break; + end + crc.update(data); + end + + checksum = ndi.fun.file.crc32ToBase64(crc.getValue()); +end diff --git a/src/ndi/+ndi/+fun/+file/crc32ToBase64.m b/src/ndi/+ndi/+fun/+file/crc32ToBase64.m new file mode 100644 index 0000000000..c16ba7824e --- /dev/null +++ b/src/ndi/+ndi/+fun/+file/crc32ToBase64.m @@ -0,0 +1,28 @@ +function checksum = crc32ToBase64(value) +%CRC32TOBASE64 Encode a numeric CRC-32 as base64 of its 4-byte big-endian value. +% +% SYNTAX: +% checksum = ndi.fun.file.crc32ToBase64(value) +% +% DESCRIPTION: +% NDI Cloud and S3 represent a CRC-32 as the base64 of its four bytes in +% big-endian order. This converts a numeric CRC-32 (for example from +% java.util.zip.CRC32.getValue) to that form. +% +% INPUTS: +% value - The CRC-32 as a number in [0, 4294967295]. +% +% OUTPUTS: +% checksum - 1x8 char base64 string, e.g. 'NhCmhg==' for 0x3610A686. +% +% See also: ndi.fun.file.crc32 + + arguments + value (1,1) {mustBeNumeric, mustBeInteger, mustBeNonnegative, ... + mustBeLessThanOrEqual(value, 4294967295)} + end + + v = uint32(value); + bytes = uint8(bitand(bitshift(v, [-24 -16 -8 0]), uint32(255))); + checksum = char(matlab.net.base64encode(bytes)); +end diff --git a/src/ndi/+ndi/+gui/+app/LightsheetZarrManager.m b/src/ndi/+ndi/+gui/+app/LightsheetZarrManager.m new file mode 100644 index 0000000000..c61cad6018 --- /dev/null +++ b/src/ndi/+ndi/+gui/+app/LightsheetZarrManager.m @@ -0,0 +1,710 @@ +classdef LightsheetZarrManager < ndi.gui.app.sessionApp +% NDI.GUI.APP.LIGHTSHEETZARRMANAGER - list, add, view and delete lightsheet OME-Zarr pyramids +% +% ndi.gui.app.LightsheetZarrManager(SESSION) +% +% Lists the lightsheetZarrPyramid documents in a session, paired with +% their subject and level counts, and lets the user add, open or +% remove one. Reached from the navigator's per-session Apps menu. +% +% WHAT IS LISTED IS PYRAMIDS, NOT ZARR STORES. An OME-Zarr store on +% disk is the input; what lands in the database is a +% lightsheetZarrPyramid plus one lightsheetZarrLevel per level. A +% single store frequently produces two parents (a mean pyramid and a +% max pyramid), listed as two rows. +% +% VIEWING IS SOMEONE ELSE'S JOB. The viewer is napari, which is +% Python, so View does not draw anything here: it builds a command +% line via NDI.FUN.DOC.LIGHTSHEET.VIEWCOMMAND and hands it to a +% launcher, by default /usr/local/bin/napariViewLightsheet. That +% launcher is a shell wrapper rather than the console script itself +% because MATLAB exports library paths of its own, and a Python +% process started from MATLAB picks up MATLAB's copies of libraries +% it must not use; the wrapper scrubs the environment before exec'ing +% the real entry point. viewCommand builds the command and is pure, +% so what will run can be shown in the dialog before it runs and can +% be copied into a terminal when it does not. +% +% DELETING IS A CASCADE. Levels depend on the pyramid, so removing a +% pyramid orphans everything beneath it. deletionPlan enumerates that +% first and the confirmation names the counts, because "delete 1 +% document" and "delete 32 documents" should not look the same. +% +% Example: +% S = ndi.session.dir('mysession','/path/to/session'); +% ndi.gui.app.LightsheetZarrManager(S); +% +% See also: ndi.fun.doc.lightsheet.fromOMEZarr, +% ndi.fun.doc.lightsheet.viewCommand, +% ndi.gui.app.GEFManager (the pattern this app mirrors) + + properties (Constant) + Name = "Lightsheet Zarr Manager" + Category = "Lightsheet microscopy" + + % Where the napari launcher lives when nobody has said otherwise. + DefaultViewerLauncher = "/usr/local/bin/napariViewLightsheet" + end + + properties (Access = private) + session + fig + table + statusLabel + rows = struct([]) + % The View dialog, kept so a second click RAISES the open one + % rather than stacking another. Non-modal (see onView), so + % without this the button would build one dialog per press. + viewDialog = matlab.ui.Figure.empty + end + + methods + function obj = LightsheetZarrManager(sessionObj, options) + arguments + sessionObj (1,1) + % build=false constructs the model without a figure, so + % every decision this app makes is checkable with no + % display. Same arrangement GEFManager uses. + options.build (1,1) logical = true + end + obj.session = sessionObj; + if options.build + obj.buildUI(); + obj.reload(); + end + end + + function reload(obj) + obj.rows = ndi.gui.app.LightsheetZarrManager.pyramidRows(obj.session); + if ~isempty(obj.fig) && isvalid(obj.fig) + obj.table.Data = ndi.gui.app.LightsheetZarrManager.rowsToCell(obj.rows); + obj.setStatus(sprintf('%d pyramid(s).', numel(obj.rows))); + end + end + + function refresh(obj) + % REFRESH - backward-compatible alias for reload + obj.reload(); + end + end + + % ===================================================================== + % The model. Static and pure: no figure, no state, so it is testable + % headlessly and a test failure names a decision rather than a widget. + % ===================================================================== + methods (Static) + + function rows = pyramidRows(session) + % PYRAMIDROWS - one row per lightsheetZarrPyramid, with subject names + % + % The subject column shows local_identifier, not the raw id: an + % id names the subject uniquely and tells the reader nothing. + % The lookup is done ONCE for the whole session rather than + % per row -- a session with 20 pyramids would otherwise make + % 20 subject queries to answer one question. + % + % BOTH VINTAGES. A migrated (V_eta) pyramid is listed too: the + % intensity observation DID-matlab's lightsheet migrator makes of + % it, its id kept (ndi.vintage.lightsheetPyramidDocs). Its fields + % are read by their v1 names through ndi.vintage.lightsheetPyramid, + % its subject is its `entity_id`, and its levels are the + % sampled_bodies it owns (ndi.vintage.lightsheetLevelDocs). + docs = ndi.vintage.lightsheetPyramidDocs(session); + subjectNames = ndi.gui.app.LightsheetZarrManager.subjectNameMap(session); + rows = struct('id',{},'label',{},'reduction',{},'subject',{}, ... + 'subjectID',{},'nLevels',{},'shapeLevel0',{},'dtype',{},'doc',{}); + for i = 1:numel(docs) + d = docs{i}; + [p, vintage] = ndi.vintage.lightsheetPyramid(d, session); + k = numel(rows) + 1; + rows(k).id = d.id(); + rows(k).doc = d; + rows(k).label = ndi.gui.app.LightsheetZarrManager.field(p,'label',''); + if strcmp(vintage, 'v1') + rows(k).subjectID = ndi.gui.app.LightsheetZarrManager.dependency(d,'subject_id'); + else + rows(k).subjectID = ndi.gui.app.LightsheetZarrManager.dependency(d,'entity_id'); + if isempty(rows(k).subjectID) % before #73's rename + rows(k).subjectID = ndi.gui.app.LightsheetZarrManager.dependency(d,'subject_id'); + end + end + rows(k).subject = ndi.gui.app.LightsheetZarrManager.lookup( ... + subjectNames, rows(k).subjectID, rows(k).subjectID); + + % Level count and reduction come from the child level + % docs, not the parent -- the parent schema has neither + % an n_levels nor a reduction field. Reductions are per + % level (reduction_function on lightsheetZarrLevel); + % the parent's pyramid_type names the source volume, not + % the reduction. See NDI-matlab#979 review 2026-09-25. + levelDocs = ndi.vintage.lightsheetLevelDocs(session, d); + rows(k).nLevels = numel(levelDocs); + rows(k).reduction = ndi.gui.app.LightsheetZarrManager.joinReductions(levelDocs); + + shp = ndi.gui.app.LightsheetZarrManager.field(p,'shape_level0',[]); + rows(k).shapeLevel0 = shp; + rows(k).dtype = ndi.gui.app.LightsheetZarrManager.field(p,'dtype',''); + end + end + + function s = joinReductions(levelDocs) + % joinReductions - unique non-'none' reduction_function values + % Reads reduction_function from each lightsheetZarrLevel doc + % and returns them joined by "/", with 'none' (the raw level 0 + % marker) filtered out. When only 'none' levels exist, returns + % the string 'none' so the column doesn't lie about a missing + % ladder. + seen = strings(1, numel(levelDocs)); + for j = 1:numel(levelDocs) + % v1: the lightsheetZarrLevel block itself; V_eta: the + % reduction read off the body's `summary statistic` + % condition ('none' for level 0). + lp = ndi.vintage.lightsheetLevel(levelDocs{j}); + seen(j) = string(ndi.gui.app.LightsheetZarrManager.field( ... + lp, 'reduction_function', 'none')); + end + reductions = unique(seen); + reductions = reductions(reductions ~= "none"); + if isempty(reductions) + s = 'none'; + else + s = char(strjoin(reductions, '/')); + end + end + + function m = subjectNameMap(session) + % SUBJECTNAMEMAP - subject document id -> local_identifier + % + % containers.Map so the lookup is one query for the session + % rather than one per pyramid. A subject with no + % local_identifier is left out rather than mapped to '': the + % caller's default is the id, and an empty cell in the Subject + % column would say the pyramid has no subject, which is a + % different and worse claim than showing the id. + m = containers.Map('KeyType','char','ValueType','char'); + docs = session.database_search(ndi.query('','isa','subject')); + for i = 1:numel(docs) + p = docs{i}.document_properties; + if isfield(p,'subject') + nm = ndi.gui.app.LightsheetZarrManager.field(p.subject,'local_identifier',''); + else + % the one-entity-class schema keeps it in `entity` + nm = ndi.v2.blockOf(p, 'subject', 'local_identifier', ''); + end + if ~isempty(nm) + m(docs{i}.id()) = char(nm); + end + end + end + + function plan = deletionPlan(session, pyramidID) + % DELETIONPLAN - the pyramid and every level that depends on it + % + % Public and static so a caller without a display can preview + % what a delete would remove. + % + % A MIGRATED (V_eta) pyramid's levels are the sampled_bodies its + % observation owns, and the plan finds them through + % ndi.vintage.lightsheetLevelDocs. Its store record (the unheld + % opaque_body the migration made of the v1 fileReference) ALSO + % names the observation as its owner, where the v1 fileReference + % was named BY the pyramid -- so deleting the pyramid alone would + % leave that record pointing at nothing. It is listed in the plan + % (nStoreRecords) and removed with the pyramid; the stored bytes + % are not held by the database and are not touched. v1 has no + % such record and its plan is unchanged. + plan = struct(); + plan.pyramidID = char(pyramidID); + parentDoc = session.database_search(ndi.query('base.id', ... + 'exact_string', plan.pyramidID)); + storeDocs = {}; + if ~isempty(parentDoc) && ... + ~isfield(parentDoc{1}.document_properties, 'lightsheetZarrPyramid') + levelDocs = ndi.vintage.lightsheetLevelDocs(session, parentDoc{1}); + storeDocs = session.database_search( ... + ndi.query('', 'isa', 'opaque_body', '') & ... + ndi.query('', 'depends_on', 'owner_id', plan.pyramidID)); + else + q1 = ndi.query('', 'isa', 'lightsheetZarrLevel'); + q2 = ndi.query('depends_on', 'depends_on', ... + 'lightsheetZarrPyramid_id', plan.pyramidID); + levelDocs = session.database_search(q1 & q2); + end + plan.nLevels = numel(levelDocs); + plan.nStoreRecords = numel(storeDocs); + plan.docsToRemove = [levelDocs(:); storeDocs(:); parentDoc(:)]; + end + + function msg = deletionMessage(plan, label) + % DELETIONMESSAGE - the confirmation text for a deletion + % + % Names the pyramid and the level count so "delete 1 document" + % and "delete 32 documents" do not read the same. + if nargin < 2, label = ''; end + if isempty(label), label = plan.pyramidID; end + nStore = 0; + if isfield(plan, 'nStoreRecords'), nStore = plan.nStoreRecords; end + storeLine = ''; + if nStore > 0 + storeLine = sprintf([' %d store record(s) (the store itself ' ... + 'is not in the database and is not deleted)\n'], nStore); + end + msg = sprintf(['This removes %d document(s):\n\n' ... + ' 1 pyramid (%s)\n %d level(s)\n%s\n' ... + 'The levels go because the pyramid they belong to goes. ' ... + 'This cannot be undone.'], ... + 1 + plan.nLevels + nStore, label, plan.nLevels, storeLine); + end + + function p = launcherPath() + % LAUNCHERPATH - the viewer launcher, from preferences or default + % + % A missing or unreadable preference falls back to the default + % rather than raising: the dialog can still open, and the + % field it opens with is editable. + p = char(ndi.gui.app.LightsheetZarrManager.DefaultViewerLauncher); + try + v = ndi.preferences.get('GUI.LightsheetZarrManager.ViewerLauncher'); + if ~isempty(char(v)), p = char(v); end + catch + end + end + + function setLauncherPath(p) + % SETLAUNCHERPATH - remember the launcher for next time + % + % Failing to persist must not cost the launch about to happen. + try + ndi.preferences.set('GUI.LightsheetZarrManager.ViewerLauncher', string(p)); + catch + end + end + + function paper(h, c) + % PAPER - put a control on the cloud palette's white body + % + % The figure and its layouts are on c.offWhite, but the + % CONTROLS default to MATLAB's grey chrome, which reads as a + % grey app with a tinted border. Defensive: a missing handle + % or a control without a BackgroundColor is a no-op. + if isempty(h) || ~isvalid(h) + return; + end + try %#ok + h.BackgroundColor = c.white; + end + try %#ok + h.FontColor = c.darkBlue; + end + end + + function accent(btn, c) + % ACCENT - style a button in the NDI Cloud accent + % + % Light blue on navy, matching ndi.util.ListDialog and the + % cloud .mlapp apps. A missing or deleted handle is a no-op + % rather than an error: this is styling, and losing the + % window over a colour would be a poor trade. + if isempty(btn) || ~isvalid(btn) + return; + end + btn.BackgroundColor = c.lightBlue; + btn.FontColor = c.darkBlue; + btn.FontWeight = 'bold'; + end + + end + + % ===================================================================== + methods (Static, Access = private) + + function v = field(s, name, dflt) + if isstruct(s) && isfield(s, name), v = s.(name); else, v = dflt; end + end + + function v = dependency(d, name) + try + v = d.dependency_value(name); + catch + v = ''; + end + if isempty(v), v = ''; end + end + + function v = lookup(m, key, dflt) + v = dflt; + if isempty(key) || ~isKey(m, char(key)), return; end + v = m(char(key)); + end + + function c = rowsToCell(rows) + c = cell(numel(rows), 6); + for i = 1:numel(rows) + c{i,1} = rows(i).label; + c{i,2} = rows(i).subject; + c{i,3} = rows(i).reduction; + c{i,4} = rows(i).nLevels; + if isempty(rows(i).shapeLevel0) + c{i,5} = ''; + else + c{i,5} = mat2str(double(rows(i).shapeLevel0)); + end + c{i,6} = rows(i).dtype; + end + end + end + + % ===================================================================== + methods (Access = private) + + function buildUI(obj) + c = ndi.gui.cloudColors(); + obj.fig = uifigure('Name','Lightsheet Zarr Manager', ... + 'Position',[100 100 900 460], 'Color', c.offWhite); + % The View dialog is a separate top-level window; closing + % the manager has to take it along, or an orphaned dialog + % would report its result to a status bar that no longer + % exists. + obj.fig.CloseRequestFcn = @(~,~) obj.onClose(); + g = uigridlayout(obj.fig,[4 5]); + g.RowHeight = {28, '1x', 30, 22}; + g.ColumnWidth = {'1x', 90, 90, 90, 90}; + g.BackgroundColor = c.offWhite; + + % Navy header bar with white text, matching the cloud apps + % and ndi.util.ListDialog. + hb = uigridlayout(g,[1 1]); + hb.Layout.Row = 1; hb.Layout.Column = [1 5]; + hb.Padding = [8 0 8 0]; + hb.BackgroundColor = c.darkBlue; + uilabel(hb,'Text','Lightsheet OME-Zarr pyramids', ... + 'FontColor', c.white, 'FontWeight','bold', 'FontSize',14, ... + 'VerticalAlignment','center'); + + obj.table = uitable(g, 'ColumnName', ... + {'Label','Subject','Reduction','Levels','Shape (level 0)','dtype'}); + obj.table.Layout.Row = 2; obj.table.Layout.Column = [1 5]; + ndi.gui.app.LightsheetZarrManager.paper(obj.table, c); + + b = uibutton(g,'Text','Reload','ButtonPushedFcn',@(~,~) obj.reload()); + b.Layout.Row = 3; b.Layout.Column = 2; + ndi.gui.app.LightsheetZarrManager.accent(b, c); + b = uibutton(g,'Text','View...','ButtonPushedFcn',@(~,~) obj.onView()); + b.Layout.Row = 3; b.Layout.Column = 3; + ndi.gui.app.LightsheetZarrManager.accent(b, c); + b = uibutton(g,'Text','Add...','ButtonPushedFcn',@(~,~) obj.onAdd()); + b.Layout.Row = 3; b.Layout.Column = 4; + ndi.gui.app.LightsheetZarrManager.accent(b, c); + b = uibutton(g,'Text','Delete','ButtonPushedFcn',@(~,~) obj.onDelete()); + b.Layout.Row = 3; b.Layout.Column = 5; + ndi.gui.app.LightsheetZarrManager.accent(b, c); + + obj.statusLabel = uilabel(g,'Text','','FontColor', c.darkBlue); + obj.statusLabel.Layout.Row = 4; obj.statusLabel.Layout.Column = [1 5]; + end + + function setStatus(obj, msg) + if ~isempty(obj.statusLabel) && isvalid(obj.statusLabel) + obj.statusLabel.Text = msg; + end + end + + function r = selectedRow(obj) + r = []; + sel = obj.table.Selection; + if isempty(sel) || isempty(obj.rows), return; end + r = obj.rows(sel(1)); + end + + function onView(obj) + % ONVIEW - the launcher dialog, and then the launch + % + % A dialog rather than a button that just runs, because the + % things it asks about cannot be recovered afterwards from + % the window that opens: which channel, which reduction, + % which level, and what the layer is called. And because the + % launcher is a path on the user's machine that NDI cannot + % know, only default. + % + % THE COMMAND IS ON SCREEN before it runs, and stays readable + % afterwards. Launching another program is the step most + % likely to fail for reasons NDI cannot see -- a wrapper not + % installed, an environment that is not what it looks like -- + % and a command the user can copy into a terminal turns that + % from a shrug into something they can act on. + r = obj.selectedRow(); + if isempty(r) + uialert(obj.fig,'Select a pyramid first.','Nothing selected'); + return; + end + sessionPath = ''; + try + sessionPath = obj.session.getpath(); + catch + try + sessionPath = obj.session.path; + catch + end + end + if isempty(sessionPath) + uialert(obj.fig, ... + ['This session has no directory on disk. The napari ' ... + 'viewer opens an ndi.session.dir, and there is ' ... + 'nothing here for it to open.'], 'Nothing to view'); + return; + end + + % One dialog at a time. Non-modal, so nothing stops the + % button being pressed again; without this each press would + % build another and the user would be editing whichever + % landed on top while an older one held a different pyramid. + if ~isempty(obj.viewDialog) && isvalid(obj.viewDialog) + figure(obj.viewDialog); + return; + end + + c = ndi.gui.cloudColors(); + d = uifigure('Name','View in napari','Position',[120 120 660 430], ... + 'Color', c.offWhite); + gl = uigridlayout(d,[9 3]); + gl.RowHeight = {28, 40, 24, 24, 24, 24, 24, 24, 60}; + gl.ColumnWidth = {140, '1x', 100}; + gl.BackgroundColor = c.offWhite; + + hb = uigridlayout(gl,[1 1]); + hb.Layout.Row = 1; hb.Layout.Column = [1 3]; + hb.Padding = [8 0 8 0]; + hb.BackgroundColor = c.darkBlue; + uilabel(hb,'Text','View in napari', ... + 'FontColor', c.white, 'FontWeight','bold', 'FontSize',14, ... + 'VerticalAlignment','center'); + + intro = uilabel(gl,'WordWrap','on','Text', ... + ['Opens this pyramid in napari. The viewer is a separate ' ... + 'Python program: MATLAB starts it and does not wait for ' ... + 'it. The command that will run is shown at the bottom.']); + intro.Layout.Row = 2; intro.Layout.Column = [1 3]; + + lab = uilabel(gl,'Text','Launcher'); + lab.Layout.Row = 3; lab.Layout.Column = 1; + ed = uieditfield(gl,'text','Value', ... + ndi.gui.app.LightsheetZarrManager.launcherPath()); + ed.Layout.Row = 3; ed.Layout.Column = 2; + ndi.gui.app.LightsheetZarrManager.paper(ed, c); + br = uibutton(gl,'Text','Browse...'); + br.Layout.Row = 3; br.Layout.Column = 3; + ndi.gui.app.LightsheetZarrManager.accent(br, c); + + lab = uilabel(gl,'Text','Layer name'); + lab.Layout.Row = 4; lab.Layout.Column = 1; + nameEd = uieditfield(gl,'text','Value', ... + ndi.gui.app.LightsheetZarrManager.orEmpty(r.label)); + nameEd.Layout.Row = 4; nameEd.Layout.Column = [2 3]; + ndi.gui.app.LightsheetZarrManager.paper(nameEd, c); + + lab = uilabel(gl,'Text','Reduction'); + lab.Layout.Row = 5; lab.Layout.Column = 1; + redDd = uidropdown(gl, 'Items', {'(as selected)','mean','max'}, ... + 'Value','(as selected)'); + redDd.Layout.Row = 5; redDd.Layout.Column = [2 3]; + ndi.gui.app.LightsheetZarrManager.paper(redDd, c); + + lab = uilabel(gl,'Text','Channel'); + lab.Layout.Row = 6; lab.Layout.Column = 1; + chSpin = uispinner(gl,'Limits',[-1 32],'Value',-1,'Step',1); + chSpin.Layout.Row = 6; chSpin.Layout.Column = [2 3]; + chSpin.Tooltip = '-1 opens all channels as separate napari layers'; + ndi.gui.app.LightsheetZarrManager.paper(chSpin, c); + + lab = uilabel(gl,'Text','Initial level'); + lab.Layout.Row = 7; lab.Layout.Column = 1; + lvSpin = uispinner(gl,'Limits',[-1 32],'Value',-1,'Step',1); + lvSpin.Layout.Row = 7; lvSpin.Layout.Column = [2 3]; + lvSpin.Tooltip = '-1 lets the viewer choose based on window size'; + ndi.gui.app.LightsheetZarrManager.paper(lvSpin, c); + + cbPanels = uicheckbox(gl,'Text','Control panels','Value',true); + cbPanels.Layout.Row = 8; cbPanels.Layout.Column = [1 3]; + + cmdArea = uitextarea(gl,'Editable','off','Value',''); + cmdArea.Layout.Row = 9; cmdArea.Layout.Column = [1 2]; + ndi.gui.app.LightsheetZarrManager.paper(cmdArea, c); + go = uibutton(gl,'Text','Launch'); + ndi.gui.app.LightsheetZarrManager.accent(go, c); + go.Layout.Row = 9; go.Layout.Column = 3; + + br.ButtonPushedFcn = @(~,~) localBrowse(); + go.ButtonPushedFcn = @(~,~) localGo(); + everything = {ed, nameEd, redDd, chSpin, lvSpin, cbPanels}; + for i = 1:numel(everything) + everything{i}.ValueChangedFcn = @localRefresh; + end + + % One log per launch, named now so the command shown is the + % exact line that will run (and can be pasted in a terminal). + logFile = ndi.fun.doc.lightsheet.viewerLogFile(); + + localRefresh(); + obj.viewDialog = d; + + function [cmd, ok] = localCommand() + ok = true; + try + red = ''; + if ~strcmp(redDd.Value, '(as selected)') + red = redDd.Value; + end + cmd = ndi.fun.doc.lightsheet.viewCommand( ... + ed.Value, sessionPath, r.id, ... + 'reduction', red, ... + 'channel', chSpin.Value, ... + 'level', lvSpin.Value, ... + 'controls', cbPanels.Value, ... + 'name', nameEd.Value); + catch ME + cmd = ME.message; + ok = false; + end + end + + function localRefresh(~,~) + % Show the full shell line: debugging on and output + % appended to the log, as launchViewer will run it. + [cmd, ok] = localCommand(); + if ok + cmd = ndi.fun.doc.lightsheet.viewerShellCommand( ... + cmd, logFile, false, true, ispc); + end + cmdArea.Value = cmd; + end + + function localBrowse() + [f,pth] = uigetfile('*','Select the napari viewer launcher'); + figure(d); + if isequal(f,0), return; end + ed.Value = fullfile(pth,f); + localRefresh(); + end + + function localGo() + cmd = localCommand(); + target = strtrim(ed.Value); + if ~isempty(regexp(target,'[/\\]','once')) && ~isfile(target) + uialert(d, sprintf(['No launcher at\n %s\n\nThat ' ... + 'path is normally a small shell wrapper that ' ... + 'scrubs MATLAB''s library paths and then runs ' ... + 'napariViewLightsheet from your NDI-python ' ... + 'environment. Install it there, or point this ' ... + 'at the console script yourself.'], target), ... + 'Launcher not found'); + return; + end + ndi.gui.app.LightsheetZarrManager.setLauncherPath(target); + % Debugging lines on, and the viewer's output kept in a + % log in NDI's temporary folder: launched from here it + % has no terminal, so the log is the only record of what + % it fetched and what failed. + status = ndi.fun.doc.lightsheet.launchViewer(cmd, 'logFile', logFile); + delete(d); + if status == 0 + obj.setStatus(sprintf(['Launched the viewer for %s. ' ... + 'Its log: %s'], obj.rowLabel(r), logFile)); + else + obj.setStatus(sprintf(['The launcher exited %d. ' ... + 'The command was: %s (log: %s)'], status, cmd, logFile)); + end + end + end + + function onClose(obj) + % ONCLOSE - shut the manager, and anything it opened + % + % Only the View dialog is closed. The napari process is + % deliberately NOT touched: it was detached on purpose, it is + % somebody's picture, and closing the window that launched it + % is not a request to close it. + if ~isempty(obj.viewDialog) && isvalid(obj.viewDialog) + delete(obj.viewDialog); + end + delete(obj.fig); + end + + function onDelete(obj) + r = obj.selectedRow(); + if isempty(r) + uialert(obj.fig,'Select a pyramid first.','Nothing selected'); + return; + end + plan = ndi.gui.app.LightsheetZarrManager.deletionPlan( ... + obj.session, r.id); + msg = ndi.gui.app.LightsheetZarrManager.deletionMessage( ... + plan, obj.rowLabel(r)); + choice = uiconfirm(obj.fig, msg, 'Confirm delete', ... + 'Options',{'Delete','Cancel'}, ... + 'DefaultOption',2,'CancelOption',2); + if ~strcmp(choice,'Delete'), return; end + for k = 1:numel(plan.docsToRemove) + obj.session.database_rm(plan.docsToRemove{k}); + end + obj.setStatus(sprintf('Deleted %d document(s).', ... + numel(plan.docsToRemove))); + obj.reload(); + end + + function onAdd(obj) + [f, p] = uigetfile({'.zattrs', 'OME-Zarr .zattrs'; ... + '*', 'All Files (*)'}, ... + 'Select the OME-Zarr store .zattrs (or its containing folder)'); + if isequal(f, 0), return; end + if strcmp(f, '.zattrs') + zarrPath = p; + else + zarrPath = fullfile(p, f); + end + zarrPath = strip(zarrPath, 'right', filesep); + + subjectID = obj.pickSubject(); + if isempty(subjectID), return; end + + obj.setStatus('Ingesting...'); + drawnow; + try + ndi.fun.doc.lightsheet.fromOMEZarr(obj.session, zarrPath, ... + 'subjectID', subjectID); + obj.setStatus('Ingest complete.'); + obj.reload(); + catch ME + obj.setStatus('Ingest failed; see the message.'); + uialert(obj.fig, ME.message, 'Add failed'); + end + end + + function id = pickSubject(obj) + subs = obj.session.database_search(ndi.query('', 'isa', 'subject')); + if isempty(subs) + uialert(obj.fig, 'No subject documents in this session.', ... + 'No subject'); + id = ''; + return; + end + labels = cellfun(@(x) char( ... + x.document_properties.subject.local_identifier), ... + subs, 'UniformOutput', false); + [idx, ok] = listdlg('PromptString', 'Subject:', ... + 'SelectionMode', 'single', 'ListString', labels); + if ~ok, id = ''; return; end + id = subs{idx}.id(); + end + + function s = rowLabel(~, r) + s = r.label; + if isempty(s), s = r.id; end + end + end + + methods (Static, Access = private) + function s = orEmpty(v) + if isempty(v), s = ''; else, s = char(v); end + end + end +end diff --git a/src/ndi/+ndi/+gui/+component/ProgressBarWindow.m b/src/ndi/+ndi/+gui/+component/ProgressBarWindow.m index ac7c4b316c..b2937e56dd 100644 --- a/src/ndi/+ndi/+gui/+component/ProgressBarWindow.m +++ b/src/ndi/+ndi/+gui/+component/ProgressBarWindow.m @@ -45,6 +45,10 @@ % ndi.gui.component.ProgressBarWindow.isHeadless, % ndi.gui.component.ProgressBarWindow.silentModeDefault + properties (Constant, Hidden) + BarMargin double = 5 % Pixels between a bar and the edges of its row (as R2024b drew it). + end + properties (Hidden) ScreenFrac double = 0.025 % Fraction of screen height used per bar row. IconClose char = fullfile(ndi.common.PathConstants.RootFolder,... @@ -265,6 +269,15 @@ options.Tag {mustBeTextScalar(options.Tag)} = '' options.Color (1,3) double {mustBeInRange(options.Color,0,1)} = [1 1 1] options.Auto logical = false + % Per-bar timeout. Overrides app.Timeout for this bar only. + % duration.empty (the default) means "inherit app.Timeout". + % Pass minutes(Inf) to opt this bar out of the timeout entirely + % -- appropriate for a top-level bar that ticks less often + % than the app-level timeout (typically 1 min), which used to + % get culled by updateBar's global timeout sweep while its + % owner was still using it. See NDI-matlab lightsheet demo + % 2026-09-20. + options.Timeout duration = duration.empty end % Bring figure to front @@ -307,6 +320,9 @@ app.ProgressBars(barNum).Auto = options.Auto; app.ProgressBars(barNum).Progress = 0; app.ProgressBars(barNum).Clock(1:2) = {datetime('now')}; + % Per-bar timeout: duration.empty means inherit app.Timeout at + % checkTimeout time. minutes(Inf) means "never time out". + app.ProgressBars(barNum).Timeout = options.Timeout; % Everything from here on builds this bar's graphics. When % silent there is no grid and no figure, so the bar exists as @@ -346,13 +362,31 @@ app.ProgressBars(barNum).Timer.Layout.Row = rowNum - 1; app.ProgressBars(barNum).Timer.Layout.Column = 1:2; - % Add bar axes (background) - app.ProgressBars(barNum).Axes = uiaxes(app.ProgressGrid,... + % Add bar axes (background), inside a borderless panel that + % fills the grid cell. In a grid layout MATLAB places a uiaxes + % by its OUTER box and keeps room inside it for tick labels and + % a title, even hidden ones; R2026a keeps about 25 px, which + % left a 28 px row with a 2.7 px bar. Inside a plain panel the + % axes' inner box is placed exactly, BarMargin px in from each + % side -- the margin R2024b drew -- and kept there when the + % panel resizes. The panel is the axes' parent: it is what the + % grid lays out (removeBar moves and deletes it through + % Axes.Parent). + barPanel = uipanel(app.ProgressGrid,'BorderType','none',... + 'BackgroundColor',app.ProgressGrid.BackgroundColor,... + 'AutoResizeChildren','off'); % else SizeChangedFcn never runs + barPanel.Layout.Row = rowNum; + barPanel.Layout.Column = 1; + app.ProgressBars(barNum).Axes = uiaxes(barPanel,... 'XLim',[0 1],'YLim',[0 1],'XTick',[],'YTick',[],'Box','off',... 'XColor','none','YColor','none','Color','w','Interactions',[]); app.ProgressBars(barNum).Axes.Toolbar.Visible = 'off'; - app.ProgressBars(barNum).Axes.Layout.Row = rowNum; - app.ProgressBars(barNum).Axes.Layout.Column = 1; + app.ProgressBars(barNum).Axes.Units = 'pixels'; + app.ProgressBars(barNum).Axes.PositionConstraint = 'innerposition'; + barAxes = app.ProgressBars(barNum).Axes; + barPanel.SizeChangedFcn = @(panel,~) ... + ndi.gui.component.ProgressBarWindow.fitBarAxes(barAxes, panel); + ndi.gui.component.ProgressBarWindow.fitBarAxes(barAxes, barPanel); % Add bar patch (foreground) app.ProgressBars(barNum).Patch = patch(app.ProgressBars(barNum).Axes, ... @@ -513,7 +547,7 @@ rowNum = app.ProgressBars(barNum).Label.Layout.Row + [0 1]; % Remove progress bar - delete([app.ProgressBars(barNum).Axes,... + delete([app.ProgressBars(barNum).Axes.Parent,... % the bar's panel, and its axes app.ProgressBars(barNum).Percent,... app.ProgressBars(barNum).Button,... app.ProgressBars(barNum).Label,... @@ -524,7 +558,7 @@ for i = 1:numel(openBars) app.ProgressBars(openBars(i)).Label.Layout.Row = 2*i - 1; app.ProgressBars(openBars(i)).Timer.Layout.Row = 2*i - 1; - app.ProgressBars(openBars(i)).Axes.Layout.Row = 2*i; + app.ProgressBars(openBars(i)).Axes.Parent.Layout.Row = 2*i; app.ProgressBars(openBars(i)).Percent.Layout.Row = 2*i; app.ProgressBars(openBars(i)).Button.Layout.Row = 2*i; end @@ -663,11 +697,23 @@ % Get duration of time since last update timeout = datetime('now') - app.ProgressBars(i).Clock{2}; - if timeout >= app.Timeout & ... + % Per-bar timeout wins over the app default. A bar that + % was created with Timeout=minutes(Inf) never times out + % here -- this is what long-running top-level bars need + % so that a global sweep (fired as a side effect of ANY + % updateBar call, including one on a different, actively + % updating bar) does not cull them from behind. + thisTimeout = app.Timeout; + if isfield(app.ProgressBars(i), 'Timeout') && ... + ~isempty(app.ProgressBars(i).Timeout) + thisTimeout = app.ProgressBars(i).Timeout; + end + + if timeout >= thisTimeout & ... ~strcmpi(app.ProgressBars(i).State,'Closed') && ... ~strcmpi(app.ProgressBars(i).State,'Complete') && ... app.ProgressBars(i).Progress < 1 - + % Set icon to error and state to 'Timeout' app.setErrorIconForButton(app.ProgressBars(i).Button) app.ProgressBars(i).State = 'Timeout'; @@ -1019,6 +1065,21 @@ function bringToFront(app) end methods (Static) + function fitBarAxes(ax, panel) + %fitBarAxes Place a bar's axes BarMargin px inside its panel. + % + % ndi.gui.component.ProgressBarWindow.fitBarAxes(AX, PANEL) + % sets AX's inner box (pixels) to PANEL's area less BarMargin + % on each side, at least 1 x 1. addBar calls it once and as + % PANEL's SizeChangedFcn. + if ~isvalid(ax) || ~isvalid(panel) + return + end + m = ndi.gui.component.ProgressBarWindow.BarMargin; + pos = getpixelposition(panel); + ax.InnerPosition = [1 + m, 1 + m, max(pos(3) - 2*m, 1), max(pos(4) - 2*m, 1)]; + end + function tf = isHeadless() %isHeadless True when this MATLAB has no usable display. % diff --git a/src/ndi/+ndi/+session/dir.m b/src/ndi/+ndi/+session/dir.m index 9e2837771e..a10149bbff 100755 --- a/src/ndi/+ndi/+session/dir.m +++ b/src/ndi/+ndi/+session/dir.m @@ -104,7 +104,7 @@ read_from_database = 0; if should_we_try_to_read_from_database - session_doc = ndi_session_dir_obj.database_search(ndi.query('','isa','session')); + session_doc = ndi_session_dir_obj.database_search(ndi.v2.isaQuery('session')); if ~isempty(session_doc) % use the oldest time_diff_max = 0; @@ -140,7 +140,10 @@ % 2026-08-13). The OBJECT property stays `reference`: that is % ndi.session's own public API, read at 20 call sites, and it % is not what the migration renamed. - session_blk = session_doc.document_properties.session; + % and since 2026-10-08 the session may be an `entity` of type + % session, whose block is `entity` (ndi.v2.kindOf) + [~, session_blk_name] = ndi.v2.kindOf(session_doc.document_properties); + session_blk = session_doc.document_properties.(session_blk_name); if isfield(session_blk,'local_identifier') ndi_session_dir_obj.reference = session_blk.local_identifier; else diff --git a/src/ndi/+ndi/+setup/+V2/createDataset.m b/src/ndi/+ndi/+setup/+V2/createDataset.m new file mode 100644 index 0000000000..1bd9ddaf5d --- /dev/null +++ b/src/ndi/+ndi/+setup/+V2/createDataset.m @@ -0,0 +1,223 @@ +function [dataset, report] = createDataset(path, reference, datasetSessionId, datasetDocs, sessionDocs, options) +%CREATEDATASET Create a V2 dataset: ONE did2 database holding every document. +% +% [DATASET, REPORT] = ndi.setup.V2.createDataset(PATH, REFERENCE, +% DATASETSESSIONID, DATASETDOCS, SESSIONDOCS) writes, into PATH/.ndi's V2 +% database (did2), the dataset's own `session` document, DATASETDOCS (the +% dataset-level documents: the dataset, people, studies, ...; each with +% base.session_id = DATASETSESSIONID) and SESSIONDOCS (a cell, one cell of +% documents per session, each holding that session's `session` document, +% with its own session_id), then opens it as an ndi.dataset.dir. Every +% session is INGESTED: its documents live in the dataset's database and +% ndi.dataset opens it there (ndi.dataset/build_session_info lists a V2 +% dataset's sessions from their `session` documents). +% +% Files: a body recorded with `ingest` 1 is copied into PATH/.ndi/files; one +% recorded by location (`ingest` 0, the raw recordings) stays where it is. +% +% Before writing, it CHECKS the whole set, and REPORT says what it found +% (nothing is written when 'Strict' is true and a check fails): +% documents how many documents, per class and per session +% edges every depends_on value that names a document: how many, +% and those naming no document in the set (dangling), by +% class and edge name +% duplicateIds ids carried by more than one document +% sessions session documents found, and those with no `part_of` +% relation to a study that is itself `part_of` the dataset +% +% Options: +% 'Name' the dataset session's display name +% 'Overwrite' default false; true deletes PATH/.ndi first +% 'Strict' default false; true refuses to write when an edge +% dangles, an id repeats or a session is not in the dataset +% 'BatchSize' documents per write batch (default 2000); progress is +% printed (and shown in a ProgressBarWindow) per batch +% 'Validate' default true: validate each document again as it is +% inserted. Documents built with did2.build were validated +% when built, so false skips only a repeat. +% 'Progress' default true +% 'External' a cell of documents CHECKED with the dataset but not +% written into it: the documents of its LINKED sessions, +% written to their own folders (V_eta_linked_session_plan.md). +% Their edges, ids and membership are checked as one set with +% the dataset's, and REPORT.documents counts them. +% See ndi.setup.V2.writeDocuments for how a failed batch is reported. +% +% See also ndi.setup.V2.createSession, ndi.dataset.dir. + +arguments + path (1,:) char + reference (1,:) char {mustBeNonzeroLengthText} + datasetSessionId (1,:) char {mustBeNonzeroLengthText} + datasetDocs cell + sessionDocs cell + options.Name (1,:) char = '' + options.Overwrite (1,1) logical = false + options.Strict (1,1) logical = false + options.BatchSize (1,1) double {mustBePositive, mustBeInteger} = 2000 + options.Validate (1,1) logical = true + options.Progress (1,1) logical = true + options.External cell = {} +end + +if ~isfolder(path) + mkdir(path); +end +ndiDir = fullfile(path, '.ndi'); +if isfolder(ndiDir) + existing = [dir(fullfile(ndiDir, '*.sqlite')); dir(fullfile(ndiDir, '*.json'))]; + if ~isempty(existing) && ~options.Overwrite + error('ndi:setup:V2:datasetExists', ... + '%s already holds an NDI database (%s). Pass ''Overwrite'', true to replace it.', ... + ndiDir, strjoin({existing.name}, ', ')); + end + if options.Overwrite + rmdir(ndiDir, 's'); + end +end + +f = struct('local_identifier', reference); +if ~isempty(options.Name), f.name = options.Name; end +own = ndi.setup.V2.entityDocuments('session', f, struct(), 'SessionId', datasetSessionId); +own = own{1}; +for k = 1:numel(datasetDocs) + if ~strcmp(datasetDocs{k}.base.session_id, datasetSessionId) + error('ndi:setup:V2:wrongSession', ... + 'Dataset document %d (%s) has session_id %s, not the dataset''s %s.', k, ... + datasetDocs{k}.document_class.class_name, datasetDocs{k}.base.session_id, datasetSessionId); + end +end +docsAll = [{own}, reshape(datasetDocs, 1, [])]; +for k = 1:numel(sessionDocs) + docsAll = [docsAll, reshape(sessionDocs{k}, 1, [])]; %#ok +end + +report = check([docsAll, reshape(options.External, 1, [])], datasetSessionId); +fprintf(['DENOMINATOR: %d document(s) (%d dataset-level, %d session(s); %d written here, %d ' ... + 'in linked session folders); %d edge value(s) naming a document checked\n'], ... + numel(docsAll) + numel(options.External), numel(datasetDocs) + 1, numel(sessionDocs), ... + numel(docsAll), numel(options.External), report.edges.checked); +fprintf(' dangling edges: %d repeated ids: %d sessions outside the dataset: %d\n', ... + height(report.edges.dangling), numel(report.duplicateIds), numel(report.sessions.notInDataset)); +bad = height(report.edges.dangling) > 0 || ~isempty(report.duplicateIds) || ... + ~isempty(report.sessions.notInDataset); +if bad && options.Strict + error('ndi:setup:V2:datasetCheckFailed', ... + 'The dataset did not pass its checks (see the report); nothing was written.'); +end + +mkdir(ndiDir); +report.write = ndi.setup.V2.writeDocuments(fullfile(ndiDir, ... + ndi.database.implementations.database.did2sqlite.DEFAULTFILENAME()), docsAll, ... + 'BatchSize', options.BatchSize, 'Validate', options.Validate, ... + 'Progress', options.Progress, 'Label', 'dataset documents'); +vlt.file.str2text(fullfile(ndiDir, 'unique_reference.txt'), datasetSessionId); +vlt.file.str2text(fullfile(ndiDir, 'reference.txt'), reference); +% mark the folder a dataset (ndi.session.dir.directorytype), as ndi.dataset.dir +% would on first open; without it ndi.dataset.dir refuses a folder that only +% looks like a session, and a dataset marker is never downgraded +vlt.file.str2text(fullfile(ndiDir, ndi.session.dir.objecttypemarkerfilename()), 'dataset'); + +dataset = ndi.dataset.dir(path); +if ~strcmp(dataset.id(), datasetSessionId) + error('ndi:setup:V2:datasetMismatch', ... + 'The dataset opened with id %s, not the id just written (%s).', dataset.id(), datasetSessionId); +end +end + +% ----------------------------------------------------------------------------- +function report = check(docs, dsid) +n = numel(docs); +ids = cellfun(@(d) d.base.id, docs, 'UniformOutput', false); +cls = cellfun(@ndi.setup.V2.kindOf, docs, 'UniformOutput', false); % an entity's type since 2026-10-08 +sids = cellfun(@(d) d.base.session_id, docs, 'UniformOutput', false); +[u, ~, j] = unique(ids); +report.duplicateIds = u(accumarray(j(:), 1) > 1); +% counted by the class each document is STORED as (an `entity` is `entity`), +% so the census compares with what the database holds (verifyDataset reads +% its classname column); KNOWN below names an entity by its type +stored = cellfun(@(d) char(d.document_class.class_name), docs, 'UniformOutput', false); +report.documents.byClass = groupsummary(table(stored(:), 'VariableNames', {'class'}), 'class'); +report.documents.bySession = groupsummary(table(sids(:), 'VariableNames', {'session_id'}), 'session_id'); + +known = containers.Map(ids, cls); +dang = {}; +checked = 0; +for i = 1:n + if ~isfield(docs{i}, 'depends_on'), continue; end + dep = docs{i}.depends_on; + if isstruct(dep), dep = num2cell(dep); end + for e = 1:numel(dep) + v = edgeTarget(dep{e}); + if isempty(v), continue; end + checked = checked + 1; + if ~isKey(known, v) + dang(end+1, :) = {cls{i}, dep{e}.name}; %#ok + end + end +end +report.edges.checked = checked; +if isempty(dang) + report.edges.dangling = table(cell(0, 1), cell(0, 1), zeros(0, 1), ... + 'VariableNames', {'class', 'edge', 'GroupCount'}); +else + report.edges.dangling = groupsummary(cell2table(dang, 'VariableNames', {'class', 'edge'}), ... + {'class', 'edge'}); +end + +% sessions: part_of a study that is part_of the dataset +isRel = strcmp(cls, 'directed_relation'); +parentsOf = containers.Map(); +for i = find(isRel) + d = docs{i}; + if ~strcmp(relationName(d), 'part_of'), continue; end + c = edgeOf(d, 'child_id'); p = edgeOf(d, 'parent_id'); + if isKey(parentsOf, c), parentsOf(c) = [parentsOf(c), {p}]; else, parentsOf(c) = {p}; end +end +datasetIds = ids(strcmp(cls, 'dataset')); +sess = find(strcmp(cls, 'session') & ~strcmp(sids, dsid)); +report.sessions.found = numel(sess); +report.sessions.notInDataset = {}; +for i = sess + ok = false; + if isKey(parentsOf, ids{i}) + for p = parentsOf(ids{i}) + if isKey(parentsOf, p{1}) && any(ismember(parentsOf(p{1}), datasetIds)) + ok = true; + end + end + end + if ~ok + [~, blk] = ndi.v2.kindOf(docs{i}); + report.sessions.notInDataset{end+1} = docs{i}.(blk).local_identifier; + end +end +end + +function r = relationName(d) +r = ''; +if isfield(d, 'directed_relation') && isfield(d.directed_relation, 'relation') + x = d.directed_relation.relation; + if isstruct(x) && isfield(x, 'name'), r = x.name; else, r = char(x); end +end +end + +function v = edgeOf(d, name) +v = ''; +dep = d.depends_on; +if isstruct(dep), dep = num2cell(dep); end +for e = 1:numel(dep) + if strcmp(dep{e}.name, name), v = edgeTarget(dep{e}); return; end +end +end + +function v = edgeTarget(e) +% V2 (did2.build) stores an edge as {name, document_id}; v1 as {name, value} +v = ''; +if isfield(e, 'document_id') + v = e.document_id; +elseif isfield(e, 'value') + v = e.value; +end +v = char(v); +end diff --git a/src/ndi/+ndi/+setup/+V2/createSession.m b/src/ndi/+ndi/+setup/+V2/createSession.m new file mode 100644 index 0000000000..e0556adf6e --- /dev/null +++ b/src/ndi/+ndi/+setup/+V2/createSession.m @@ -0,0 +1,135 @@ +function [session, docs] = createSession(path, reference, options) +%CREATESESSION Create a new NDI session whose database is V2 (did2). +% +% [SESSION, DOCS] = ndi.setup.V2.createSession(PATH, REFERENCE) creates a +% session in directory PATH, backed by a did2 database +% (PATH/.ndi/V_eta.sqlite) instead of the legacy one, and opens it as an +% ordinary ndi.session.dir. REFERENCE becomes the session's +% `local_identifier` (the object's `reference` property). +% +% WHY THIS EXISTS. ndi.session.dir can OPEN a V2 session -- it tries each +% entry of the database hierarchy and `V_eta.sqlite` is entry 2 -- but it +% can only CREATE a legacy one: only entry 1 (didsqlite) has creation code +% (ndi.database.fun.databasehierarchyinit). This function does the creating +% and leaves ndi.session.dir unchanged, so nothing that makes or opens a +% legacy session behaves any differently (backwards compatible). +% +% It writes exactly one document before opening, the V2 `session` document +% built with did2.build, so that ndi.session.dir's own lookup (the oldest +% `isa session` document) finds it and takes the session id and reference +% from it. +% +% Options: +% 'SessionId' the session id (default: a new ndi.ido id) +% 'Name' the session's display name (session.name) +% 'Description' free text about the session (session.description) +% 'StudyIds' cellstr: studies this session is `part_of` (a relation +% document each, stored in this session) +% 'SessionDocId' base.id of the session document (default: a new id). +% Given when other documents must point at the session +% document before it exists (a time reference's referent). +% 'TimeReferenceId' the session's own time reference +% (session.time_reference_id), e.g. its UTC extent +% 'Documents' cell array of further documents (structs, built with the +% same SessionId) written in the same transaction, before +% the session is opened +% 'Overwrite' default false. When false, PATH must not already hold an +% NDI database of either kind. When true, an existing +% PATH/.ndi is DELETED first. +% 'BatchSize', 'Validate', 'Progress' passed to +% ndi.setup.V2.writeDocuments (defaults 2000, true, false) +% +% DOCS returns the documents written (the session document, any relations +% and 'Documents'), as structs. +% +% Errors ndi:setup:V2:sessionExists when PATH already has a database and +% 'Overwrite' is false. +% +% See also ndi.session.dir, did2.build.document, ndi.setup.conv.haley.import_V2. + +arguments + path (1,:) char {mustBeFolder} + reference (1,:) char {mustBeNonzeroLengthText} + options.SessionId (1,:) char = '' + options.Name (1,:) char = '' + options.Description (1,:) char = '' + options.StudyIds = {} + options.SessionDocId (1,:) char = '' + options.TimeReferenceId (1,:) char = '' + options.Documents = {} + options.Overwrite (1,1) logical = false + options.BatchSize (1,1) double {mustBePositive, mustBeInteger} = 2000 + options.Validate (1,1) logical = true + options.Progress (1,1) logical = false +end + +ndiDir = fullfile(path, '.ndi'); +if isfolder(ndiDir) + existing = [dir(fullfile(ndiDir, '*.sqlite')); dir(fullfile(ndiDir, '*.json'))]; + if ~isempty(existing) && ~options.Overwrite + error('ndi:setup:V2:sessionExists', ... + ['%s already holds an NDI database (%s). Pass ''Overwrite'', true to ' ... + 'replace it.'], ndiDir, strjoin({existing.name}, ', ')); + end + if options.Overwrite + rmdir(ndiDir, 's'); + end +end +mkdir(ndiDir); + +sid = options.SessionId; +if isempty(sid) + sid = ndi.ido.unique_id(); +end + +fields = struct('local_identifier', reference); +if ~isempty(options.Name) + fields.name = options.Name; +end +if ~isempty(options.Description) + fields.description = options.Description; +end +edges = struct(); +if ~isempty(options.TimeReferenceId) + edges.time_reference_id = options.TimeReferenceId; +end +docs = ndi.setup.V2.entityDocuments('session', fields, edges, 'SessionId', sid, ... + 'Id', options.SessionDocId); +studies = options.StudyIds; +if ischar(studies) || isstring(studies) + studies = cellstr(studies); +end +for k = 1:numel(studies) + docs{end+1} = did2.build.directedRelation(docs{1}.base.id, studies{k}, ... + 'part_of', 'SessionId', sid); %#ok +end + +docs = [docs, reshape(options.Documents, 1, [])]; +for k = 1:numel(docs) + if ~strcmp(docs{k}.base.session_id, sid) + error('ndi:setup:V2:wrongSession', ... + 'Document %d (%s) has session_id %s, not this session''s %s.', ... + k, docs{k}.document_class.class_name, docs{k}.base.session_id, sid); + end +end + +ndi.setup.V2.writeDocuments(fullfile(ndiDir, ... + ndi.database.implementations.database.did2sqlite.DEFAULTFILENAME()), docs, ... + 'BatchSize', options.BatchSize, 'Validate', options.Validate, ... + 'Progress', options.Progress, 'Label', [reference ' documents']); + +% ndi.session.dir scopes EVERY search to its session id (ndi.session/ +% database_search ANDs base.session_id), and on a one-argument open it takes +% that id from .ndi/unique_reference.txt -- or, if the file is absent, makes a +% provisional random one, which then finds nothing and fails "Could not load +% the REFERENCE field". So the two files ndi.session.dir itself writes are +% written here first (CI run 36752118828 found this). +vlt.file.str2text(fullfile(ndiDir, 'unique_reference.txt'), sid); +vlt.file.str2text(fullfile(ndiDir, 'reference.txt'), reference); + +session = ndi.session.dir(path); +if ~strcmp(session.id(), sid) + error('ndi:setup:V2:sessionMismatch', ... + 'The session opened with id %s, not the id just written (%s).', session.id(), sid); +end +end diff --git a/src/ndi/+ndi/+setup/+V2/curie.m b/src/ndi/+ndi/+setup/+V2/curie.m new file mode 100644 index 0000000000..f9f6cde61d --- /dev/null +++ b/src/ndi/+ndi/+setup/+V2/curie.m @@ -0,0 +1,67 @@ +function [out, local] = curie(identifiers) +%CURIE Spec identifiers as V_eta global identifiers (CURIE or IRI strings). +% +% [OUT, LOCAL] = ndi.setup.V2.curie(IDENTIFIERS) turns a spec's +% `identifiers`, a list of {scheme, value}, into the strings +% `entity.global_identifier` holds on a schema built on or after 2026-10-08: +% a CURIE whose prefix is registered in CURIE_lookups_meta.json +% ('orcid:0000-0001-6282-7124', 'doi:10.7554/eLife...', 'ror:03xez1567', +% 'rrid:SCR_015773'), or the full IRI for a URL. A value given as its +% resolver's address ('https://orcid.org/...') is reduced to the local id. +% +% An award number is not a global identifier -- it is unique only within its +% funder -- so it is returned as LOCAL ('' if none), for the funding entity's +% local_identifier (V_eta_entity_composition_plan.md sec. 6). +% +% Errors ndi:setup:V2:unknownScheme for a scheme with no prefix. + +out = {}; +local = ''; +if isempty(identifiers) + return; +end +if isstruct(identifiers) + identifiers = num2cell(identifiers); +end +resolvers = {'orcid', 'https://orcid.org/'; 'ror', 'https://ror.org/'; ... + 'doi', 'https://doi.org/'; 'doi', 'http://dx.doi.org/'; ... + 'pubmed', 'https://pubmed.ncbi.nlm.nih.gov/'; ... + 'wikidata', 'https://www.wikidata.org/wiki/'}; +for k = 1:numel(identifiers) + id = identifiers{k}; + scheme = id.scheme; + if isstruct(scheme) + scheme = scheme.name; + end + scheme = lower(char(scheme)); + value = strtrim(char(id.value)); + switch scheme + case 'url' + out{end+1} = value; %#ok + continue; + case 'awardnumber' + local = value; + continue; + case {'orcid', 'ror', 'doi', 'rrid', 'wikidata', 'swh', 'ndicloud', 'wormbase', 'ncbitaxon'} + prefix = scheme; + case 'pmid' + prefix = 'pubmed'; + case 'pmcid' + prefix = 'pmc'; + case 'swhid' + prefix = 'swh'; + otherwise + error('ndi:setup:V2:unknownScheme', ... + 'Identifier scheme "%s" has no registered prefix.', scheme); + end + for r = 1:size(resolvers, 1) + if strcmp(resolvers{r, 1}, prefix) && startsWith(value, resolvers{r, 2}, 'IgnoreCase', true) + value = value(numel(resolvers{r, 2}) + 1:end); + end + end + if startsWith(value, [prefix ':'], 'IgnoreCase', true) + value = value(numel(prefix) + 2:end); + end + out{end+1} = [prefix ':' value]; %#ok +end +end diff --git a/src/ndi/+ndi/+setup/+V2/datasetDatabaseFiles.m b/src/ndi/+ndi/+setup/+V2/datasetDatabaseFiles.m new file mode 100644 index 0000000000..3f40ecbba3 --- /dev/null +++ b/src/ndi/+ndi/+setup/+V2/datasetDatabaseFiles.m @@ -0,0 +1,45 @@ +function [files, folders, problems] = datasetDatabaseFiles(path) +%DATASETDATABASEFILES The V2 database files of a dataset: its own, then its linked sessions'. +% +% [FILES, FOLDERS, PROBLEMS] = ndi.setup.V2.datasetDatabaseFiles(PATH) reads +% the V2 dataset at PATH (the folder holding .ndi) and returns, as cellstrs: +% +% FILES the dataset's own database file, then that of each LINKED +% session (a `linked_session` document names its folder; +% V_eta_linked_session_plan.md) +% FOLDERS the folder each file's documents are in (PATH first) +% PROBLEMS each linked session whose folder holds no V2 database +% +% For tools that read a dataset's documents directly (a census, an edge +% check, a term inventory): an INGESTED session is in the first file; a +% linked one in its own. It reads the database files and opens no session. +% +% See also ndi.v2.datasetSessions, ndi.setup.conv.haley.verifyDataset. + +arguments + path (1,:) char {mustBeFolder} +end +name = ndi.database.implementations.database.did2sqlite.DEFAULTFILENAME(); +own = fullfile(path, '.ndi', name); +if ~isfile(own) + error('ndi:setup:V2:noDataset', 'No V2 database at %s.', own); +end +files = {own}; +folders = {path}; +problems = {}; +db = did2.database.sqlitedb(own); +closer = onCleanup(@() db.close()); +rows = mksqlite(db.testHookDbId(), ... + 'SELECT body FROM documents WHERE classname = ''linked_session'' ORDER BY id'); +for k = 1:numel(rows) + p = jsondecode(rows(k).body); + folder = ndi.v2.resolveLinkPath(char(p.linked_session.path), path); + f = fullfile(folder, '.ndi', name); + if isfile(f) + files{end+1} = f; %#ok + folders{end+1} = folder; %#ok + else + problems{end+1} = sprintf('linked session folder %s holds no V2 database', folder); %#ok + end +end +end diff --git a/src/ndi/+ndi/+setup/+V2/datasetMetadata.m b/src/ndi/+ndi/+setup/+V2/datasetMetadata.m new file mode 100644 index 0000000000..25d0866bfa --- /dev/null +++ b/src/ndi/+ndi/+setup/+V2/datasetMetadata.m @@ -0,0 +1,576 @@ +function result = datasetMetadata(spec, sessionId, options) +%DATASETMETADATA Build the dataset-level V2 documents described by a spec. +% +% RESULT = ndi.setup.V2.datasetMetadata(SPEC, SESSIONID) turns the dataset +% metadata section of an import spec into V2 (V_eta) documents, built and +% validated with did2.build. SPEC is a struct (jsondecode of the spec file) +% or the path to a JSON spec file. SESSIONID is the session the documents +% belong to: for dataset-level entities, the dataset's own session. +% +% The spec lists entries by kind -- organizations, people, funding, +% publications, web_resources, software, products, strains, chemicals, +% formulations, instruments, +% studies and one dataset -- each with a local `key`. References between +% entries use keys (a person's `affiliations`, a funding's `funder`, ...); +% this function resolves them to document ids and emits the relations. +% See +ndi/+setup/+conv/+haley/import_V2_spec.json for a complete example. +% +% RESULT fields: +% documents cell array of document structs, in creation order +% ids containers.Map, spec key -> document id +% census table: one row per class, with the count built +% unrepresented struct array {kind, key, field, why}: spec content that +% has NO place in the V2 schema and was therefore NOT +% recorded. Reported, never silently dropped. +% +% Options: +% 'DatasetId' the dataset document's id (default: a new one) +% 'Validate' passed to did2.build (default true) +% 'Studies' "include" (default): everything, studies too; "exclude": +% everything but the studies; "only": the studies and their +% `part_of` relations alone, which needs 'DatasetId' (an +% import mints studies beside the sessions they group -- +% Haley decision #50 -- and the rest in its metadata stage) +% +% Errors did2:build:* when an entry does not fit the schema, and +% ndi:setup:V2:* for spec problems (a duplicate key, a key that names +% nothing, an entry of the wrong kind). +% +% See also did2.build.document, ndi.setup.conv.haley.import_V2. + +arguments + spec + sessionId (1,:) char + options.DatasetId (1,:) char = '' + options.Validate (1,1) logical = true + options.Studies (1,1) string {mustBeMember(options.Studies, ["include", "exclude", "only"])} = "include" +end + +if ischar(spec) || isstring(spec) + spec = jsondecode(fileread(char(spec))); +end +state = struct(); +state.docs = {}; +state.ids = containers.Map(); +state.kinds = containers.Map(); +state.unrep = struct('kind', {}, 'key', {}, 'field', {}, 'why', {}); +state.sid = sessionId; +state.validate = options.Validate; + +if ~strcmp(options.Studies, 'only') + % ---- entities, in dependency order --------------------------------------- + for e = entries(spec, 'organizations') + o = e{1}; + state = checkKnown(state, 'organization', o, {'name', 'short_name', 'identifiers', 'parent'}); + f = struct('name', o.name); + f = putIf(f, 'short_name', o, 'short_name'); + f = putIds(f, o); + state = add(state, 'organization', o.key, f, struct()); + end + for e = entries(spec, 'organizations') + o = e{1}; + if isfield(o, 'parent') && ~isempty(o.parent) + state = relate(state, o.key, o.parent, 'suborganization_of', {'organization'}, {'organization'}); + end + end + + for e = entries(spec, 'people') + p = e{1}; + state = checkKnown(state, 'person', p, {'given_name', 'family_name', 'email', 'identifiers', 'affiliations'}); + f = struct(); + f = putIf(f, 'given_name', p, 'given_name'); + f = putIf(f, 'family_name', p, 'family_name'); + f = putIf(f, 'email', p, 'email'); + f = putIds(f, p); + state = add(state, 'person', p.key, f, struct()); + for a = asCell(getOr(p, 'affiliations', {})) + state = relate(state, p.key, a{1}, 'affiliated_with', {'person'}, {'organization'}); + end + end + + for e = entries(spec, 'funding') + g = e{1}; + state = checkKnown(state, 'funding', g, {'name', 'identifiers', 'funder', 'recipients'}); + f = putIds(struct('name', g.name), g); + state = add(state, 'funding', g.key, f, struct()); + if isfield(g, 'funder') + state = relate(state, g.key, g.funder, 'issued_by', {'funding'}, {'organization'}); + end + for r = asCell(getOr(g, 'recipients', {})) + state = relate(state, g.key, r{1}, 'awarded_to', {'funding'}, {'person', 'organization'}); + end + end + + for e = entries(spec, 'publications') + p = e{1}; + state = checkKnown(state, 'publication', p, {'name', 'publication_date', 'authors', 'identifiers'}); + f = struct('name', p.name); + f = putIf(f, 'publication_date', p, 'publication_date'); + f = putIf(f, 'authors', p, 'authors'); + f = putIds(f, p); + state = add(state, 'publication', p.key, f, struct()); + end + + % On a schema with one entity class (2026-10-08) there is no web_resource: + % a URL is an address for the thing it names. A resource the dataset is + % `stored_at` becomes one of the dataset's own identifiers; any other one is + % an entity of the `type` the spec gives it (software, publication, ...). + merged = ndi.setup.V2.mergedEntities(); + storedAt = asCell(getOr(getOr(spec, 'dataset', struct()), 'stored_at', {})); + state.storedIds = {}; + state.storedHosts = {}; + for e = entries(spec, 'web_resources') + w = e{1}; + if ~merged + state = checkKnown(state, 'web_resource', w, {'name', 'identifiers', 'host'}, ... + struct('type', '')); + state = add(state, 'web_resource', w.key, putIds(struct('name', w.name), w), struct()); + if isfield(w, 'host') + state = relate(state, w.key, w.host, 'hosted_by', {'web_resource'}, {'organization'}); + end + elseif any(strcmp(w.key, storedAt)) + % a stored copy is one of the dataset's identifiers; its host is the + % DATASET's `hosted_by` (did-schema 9bedceb, Jess 2026-10-15) + if ndi.setup.V2.schemaHasRelation('hosted_by') + state = checkKnown(state, 'web_resource', w, {'name', 'identifiers', 'host'}, ... + struct('type', '')); + if isfield(w, 'host') + state.storedHosts{end+1} = w.host; + end + else + state = checkKnown(state, 'web_resource', w, {'name', 'identifiers'}, ... + struct('host', 'a stored copy is an identifier of the dataset; its host is not recorded', ... + 'type', '')); + end + state.storedIds = [state.storedIds, storedIdentifiers(w)]; + state.ids(w.key) = ''; + state.kinds(w.key) = 'web_resource'; + else + state = checkKnown(state, 'web_resource', w, {'name', 'identifiers', 'type'}, ... + struct('host', 'the schema in use has no relation from a resource to its host')); + if ~isfield(w, 'type') + error('ndi:setup:V2:noType', ['web resource `%s` needs a `type` ' ... + '(software, publication, protocol, ...) on a schema with one entity class.'], w.key); + end + state = add(state, 'web_resource', w.key, putIds(struct('name', w.name), w), ... + struct(), '', char(w.type)); + end + end + + for e = entries(spec, 'software') + s = e{1}; + % its maker: `developed_by` (did-schema 9bedceb, Jess 2026-10-15), not + % `sold_by` -- free and open-source software was never sold + hasDeveloper = ndi.setup.V2.schemaHasRelation('developed_by'); + if hasDeveloper + state = checkKnown(state, 'software', s, {'name', 'version', 'identifiers', 'vendor'}); + else + state = checkKnown(state, 'software', s, {'name', 'version', 'identifiers'}, ... + struct('vendor', 'the schema in use has no relation from software to its developer')); + end + f = putIf(struct('name', s.name), 'version', s, 'version'); + state = add(state, 'software', s.key, putIds(f, s), struct()); + if hasDeveloper && isfield(s, 'vendor') + state = relate(state, s.key, s.vendor, 'developed_by', {'software'}, {'organization'}); + end + end + + for e = entries(spec, 'products') + p = e{1}; + state = checkKnown(state, 'product', p, {'name', 'catalog_number', 'lot_number', 'vendor'}); + f = struct('name', p.name); + f = putIf(f, 'catalog_number', p, 'catalog_number'); + f = putIf(f, 'lot_number', p, 'lot_number'); + edges = struct(); + if isfield(p, 'vendor') + edges.vendor_id = idOf(state, p.vendor, {'organization'}); + end + state = add(state, 'product', p.key, f, edges); + end + + for e = entries(spec, 'strains') + s = e{1}; + % `node` (the strain's ontology term) and `biological_sex` are stated by + % the import's assertions about the subjects of the strain, not held + % on the strain entity (V2 `strain` has neither field). + state = checkKnown(state, 'strain', s, {'name', 'species', 'genetic_strain_type', ... + 'description', 'genotype', 'identifiers', 'background', 'source'}, ... + struct('node', '', 'biological_sex', '')); + f = struct('name', s.name, 'species', s.species, ... + 'genetic_strain_type', asTerm(s.genetic_strain_type)); + % V2 `strain` has no genotype field (nor does openMINDS Strain); the + % genotype string is kept in the description rather than dropped. + desc = strjoin(cellfun(@char, [asCell(getOr(s, 'genotype', {})), ... + asCell(getOr(s, 'description', {}))], 'UniformOutput', false), '. '); + if ~isempty(desc) + f.description = desc; + end + if ndi.setup.V2.mergedEntities() && isfield(s, 'node') ... + && startsWith(char(s.node), 'WBStrain:') + % the strain's WormBase id is one of its identifiers (one entity class; + % `wormbase:WBStrain00000001`, V_eta_entity_composition_plan.md sec. 6), + % since a subject is `instance_of` the strain rather than asserting it + wb = struct('scheme', 'WormBase', 'value', strrep(char(s.node), ':', '')); + s.identifiers = [asCell(getOr(s, 'identifiers', {})), {wb}]; + end + f = putIds(f, s); + edges = struct(); + if isfield(s, 'source') + % The source repository's stock, as the strain's product (the current + % schema's strain.product_id; see decision log entry 17). + state = add(state, 'product', [s.key '#stock'], ... + struct('name', sprintf('%s (stock)', s.name), 'catalog_number', s.name), ... + struct('vendor_id', idOf(state, s.source, {'organization'}))); + edges.product_id = state.ids([s.key '#stock']); + end + bg = asCell(getOr(s, 'background', {})); + if ~isempty(bg) + edges.background_strain_id = cellfun(@(k) idOf(state, k, {'strain'}), bg, ... + 'UniformOutput', false); + end + state = add(state, 'strain', s.key, f, edges); + end + + % Chemicals and formulations (Haley decision #56): what plates and seeding + % suspensions are made of, dataset-level so every session's doses can point + % at one copy. A chemical's `substance` is a term (a name until the ontology + % lookup); its `concentration` is the bottle's own strength. A formulation's + % `ingredients` name chemicals, formulations or strains by key, each with + % what the source gave (an added `mass` / `volume`, or a final + % `concentration`), in order; `type` names the standard recipe it is + % (did-schema #84; left out on a schema without it); `documented_by` cites + % where the recipe is written down. + for e = entries(spec, 'chemicals') + c = e{1}; + state = checkKnown(state, 'chemical', c, {'substance', 'concentration', 'product'}); + v = struct('substance', asTerm(c.substance)); + v = putIf(v, 'concentration', c, 'concentration'); + edges = struct(); + if isfield(c, 'product') + edges.product_id = idOf(state, c.product, {'product'}); + end + state = add(state, 'chemical', c.key, struct('value', v), edges); + end + formulationHasType = ndi.setup.V2.schemaHasField('formulation', 'value.type'); + for e = entries(spec, 'formulations') + c = e{1}; + if formulationHasType + state = checkKnown(state, 'formulation', c, {'type', 'ingredients', 'ph', 'product', ... + 'documented_by'}); + else + state = checkKnown(state, 'formulation', c, {'ingredients', 'ph', 'product', ... + 'documented_by'}, struct('type', ... + 'the V2 schema in use has no formulation.value.type (did-schema PR #84)')); + end + v = struct(); + if formulationHasType && isfield(c, 'type') + v.type = asTerm(c.type); + end + ingredients = asCell(getOr(c, 'ingredients', {})); + ids = cell(1, numel(ingredients)); + amounts = cell(1, numel(ingredients)); + for k = 1:numel(ingredients) + g = ingredients{k}; + ids{k} = idOf(state, g.ingredient, {'chemical', 'formulation', 'strain'}); + amounts{k} = rmfield(g, 'ingredient'); + end + if ~isempty(amounts) + v.ingredients = amounts; + end + if isfield(c, 'ph') + v.ph = struct('ph', c.ph, 'source_value', c.ph, 'source_unit', 'pH'); + end + edges = struct(); + if ~isempty(ids) + edges.ingredient_id = ids; + end + if isfield(c, 'product') + edges.product_id = idOf(state, c.product, {'product'}); + end + state = add(state, 'formulation', c.key, struct('value', v), edges); + for t = asCell(getOr(c, 'documented_by', {})) + state = relate(state, c.key, t{1}, 'documented_by', {'formulation'}, {'web_resource'}); + end + end + + % `subject.name` arrived in did-schema PR #80; until the schema in use declares + % it, an instrument's name is reported as unrepresented rather than built. + subjectHasName = ndi.setup.V2.schemaHasField('subject', 'name'); + for e = entries(spec, 'instruments') + i = e{1}; + if subjectHasName + state = checkKnown(state, 'subject', i, {'local_identifier', 'name', 'description', 'product'}); + else + state = checkKnown(state, 'subject', i, {'local_identifier', 'description', 'product'}, ... + struct('name', 'the V2 schema in use has no subject.name (did-schema PR #80)')); + end + f = putIf(struct('local_identifier', i.local_identifier), 'description', i, 'description'); + if subjectHasName + f = putIf(f, 'name', i, 'name'); + end + state = add(state, 'subject', i.key, f, struct(), '', 'device'); + if isfield(i, 'product') + state = relate(state, i.key, i.product, 'instance_of', {'subject'}, {'product'}); + end + end + + % ---- the dataset, then everything that points at it ----------------------- + if ~isfield(spec, 'dataset') + error('ndi:setup:V2:noDataset', 'The spec has no `dataset` section.'); + end + d = spec.dataset; + state = checkKnown(state, 'dataset', d, {'name', 'short_name', 'version', ... + 'version_innovation', 'description', 'how_to_cite', 'keyword', 'license', ... + 'accessibility', 'ethics_assessment', 'experimental_approach', 'support_channel', ... + 'release_date', 'copyright_year', 'identifiers', 'authors', 'funding', 'cites', ... + 'stored_at', 'documented_by'}); + f = struct(); + for fn = {'name', 'short_name', 'version', 'version_innovation', 'description', ... + 'how_to_cite', 'keyword', 'license', 'accessibility', 'ethics_assessment', ... + 'experimental_approach', 'support_channel', 'release_date', 'copyright_year'} + f = putIf(f, fn{1}, d, fn{1}); + end + f = putIds(f, d); + if isfield(state, 'storedIds') && ~isempty(state.storedIds) + % a stored copy is one of the dataset's addresses (one entity class only) + extra = cellfun(@(v) struct('scheme', 'URL', 'value', v), state.storedIds, ... + 'UniformOutput', false); + if isfield(f, 'global_identifier') + extra = [num2cell(f.global_identifier(:)'), extra]; + end + f.global_identifier = did2.build.list(extra{:}); + end + key = '#dataset'; + state = add(state, 'dataset', key, f, struct(), options.DatasetId); + + authors = asCell(getOr(d, 'authors', {})); + for k = 1:numel(authors) + a = authors{k}; + extra = struct('sequence', k); + if isfield(a, 'roles') + extra.roles = did2.build.label(asCell(a.roles)); + end + state = relate(state, key, a.person, 'has_author', {'dataset'}, {'person'}, extra); + end + pairs = {'funding', 'funded_by', {'funding'}; 'cites', 'cites', {'publication'}; ... + 'stored_at', 'stored_at', {'web_resource'}; 'documented_by', 'documented_by', {'web_resource'}}'; + if ndi.setup.V2.mergedEntities() + pairs = pairs(:, ~strcmp(pairs(1, :), 'stored_at')); % now the dataset's identifiers + end + for pair = pairs + for t = asCell(getOr(d, pair{1}, {})) + state = relate(state, key, t{1}, pair{2}, {'dataset'}, pair{3}); + end + end + if isfield(state, 'storedHosts') && ~isempty(state.storedHosts) + % by index: `for` over an empty cell still runs once (on its 0x1 column) + hosts = unique(state.storedHosts, 'stable'); + for k = 1:numel(hosts) + state = relate(state, key, hosts{k}, 'hosted_by', {'dataset'}, {'organization'}); + end + end +else + % Studies alone (stage 3 of an import): they are part_of the dataset, + % whose document another call builds, so its id is given, not made. + if isempty(options.DatasetId) + error('ndi:setup:V2:noDatasetId', ['''Studies'', "only" needs ''DatasetId'': ' ... + 'the dataset document the studies are part_of.']); + end + key = '#dataset'; + state.ids(key) = options.DatasetId; + state.kinds(key) = 'dataset'; +end + +if ~strcmp(options.Studies, 'exclude') + for e = entries(spec, 'studies') + s = e{1}; + state = checkKnown(state, 'study', s, {'name', 'short_name', 'description', ... + 'factors', 'design', 'identifiers'}, ... + struct('source_folder', '', 'source_condition', '')); % import directives + f = struct('name', s.name); + f = putIf(f, 'short_name', s, 'short_name'); + f = putIf(f, 'description', s, 'description'); + f = putIf(f, 'factors', s, 'factors'); + f = putIf(f, 'design', s, 'design'); + state = add(state, 'study', s.key, putIds(f, s), struct()); + state = relate(state, s.key, key, 'part_of', {'study'}, {'dataset'}); + end +end + +% ---- result ----------------------------------------------------------------- +result = struct(); +result.documents = state.docs; +result.ids = state.ids; +% counted under the per-leaf names (ndi.v2.leafName), so an `entity` counts as +% its type and the census reads the same on either schema +classes = cellfun(@ndi.v2.leafName, state.docs, 'UniformOutput', false); +[u, ~, j] = unique(classes); +result.census = table(u(:), accumarray(j(:), 1), 'VariableNames', {'class', 'count'}); +result.unrepresented = state.unrep; +end + +% ============================================================================= + +function c = entries(spec, name) +if ~isfield(spec, name) || isempty(spec.(name)) + c = {}; + return; +end +c = asCell(spec.(name)); +for k = 1:numel(c) + if ~isstruct(c{k}) || ~isfield(c{k}, 'key') || isempty(c{k}.key) + error('ndi:setup:V2:noKey', 'Entry %d of `%s` has no `key`.', k, name); + end +end +end + +function c = asCell(v) +if isempty(v) + c = {}; +elseif iscell(v) + c = reshape(v, 1, []); +elseif isstruct(v) + c = num2cell(reshape(v, 1, [])); +elseif ischar(v) || isstring(v) + c = reshape(cellstr(v), 1, []); +else + c = {v}; +end +end + +function v = getOr(s, name, default) +if isfield(s, name) + v = s.(name); +else + v = default; +end +end + +function f = putIf(f, fieldName, src, srcName) +if isfield(src, srcName) && ~isempty(src.(srcName)) + f.(fieldName) = src.(srcName); +end +end + +function f = putIds(f, src) +% A spec entry's `identifiers` [{scheme, value}] -> entity.global_identifier. +if ~isfield(src, 'identifiers') || isempty(src.identifiers) + return; +end +ids = asCell(src.identifiers); +g = cell(1, numel(ids)); +for k = 1:numel(ids) + g{k} = struct('scheme', did2.build.label(ids{k}.scheme), 'value', ids{k}.value); +end +f.global_identifier = did2.build.list(g{:}); +end + +function t = asTerm(v) +if isstruct(v) + t = v; +else + t = did2.build.label(v); +end +end + +function state = checkKnown(state, kind, entry, known, noted) +% Record every spec field this kind cannot hold, instead of dropping it silently. +if nargin < 5 + noted = struct(); +end +% A spec key starting with `_` (`_verify`, `_note`) is an ANNOTATION for the +% reader, not data. jsondecode cannot keep a leading underscore in a field name +% and renames it `x_...`, so both spellings are recognised. +skip = [{'key'}, known]; +names = fieldnames(entry); +for k = 1:numel(names) + n = names{k}; + if any(strcmp(n, skip)) || startsWith(n, '_') || startsWith(n, 'x_') + continue; + end + if isfield(noted, n) + if isempty(noted.(n)) + continue; % an import directive, not metadata + end + why = noted.(n); + else + why = sprintf('`%s` has no field or relation for it', kind); + end + state.unrep(end+1) = struct('kind', kind, 'key', char(string(getOr(entry, 'key', ''))), ... + 'field', n, 'why', why); +end +end + +function id = idOf(state, key, kinds) +key = char(key); +if ~isKey(state.ids, key) + error('ndi:setup:V2:unknownKey', 'The spec refers to `%s`, which no entry defines.', key); +end +if ~any(strcmp(state.kinds(key), kinds)) + error('ndi:setup:V2:wrongKind', '`%s` is a %s; expected %s.', key, ... + state.kinds(key), strjoin(kinds, ' or ')); +end +id = state.ids(key); +end + +function state = add(state, className, key, fields, edges, id, type) +% TYPE: the entity type, for a subject (or web resource) on a schema with one +% entity class; ndi.setup.V2.entityDocuments. +if nargin < 6 + id = ''; +end +if nargin < 7 + type = ''; +end +if isKey(state.ids, key) + error('ndi:setup:V2:duplicateKey', 'The spec defines `%s` twice.', key); +end +entityKinds = {'organization', 'person', 'funding', 'publication', 'web_resource', ... + 'software', 'product', 'strain', 'subject', 'dataset', 'study'}; +if any(strcmp(className, entityKinds)) + buildAs = className; + if strcmp(className, 'web_resource') && ~isempty(type) + buildAs = type; % on a schema with one entity class, what the resource is + end + docs = ndi.setup.V2.entityDocuments(buildAs, fields, edges, 'SessionId', state.sid, ... + 'Id', id, 'Validate', state.validate, 'Type', type); +else + docs = {did2.build.document(className, fields, 'SessionId', state.sid, ... + 'Edges', edges, 'Id', id, 'Validate', state.validate)}; +end +state.docs = [state.docs, docs]; +state.ids(key) = docs{1}.base.id; +state.kinds(key) = className; +end + +function ids = storedIdentifiers(w) +% A stored copy's addresses: an NDI Cloud dataset page is `ndicloud:`. +ids = {}; +for x = asCell(getOr(w, 'identifiers', {})) + v = char(x{1}.value); + tok = regexp(v, 'ndi-cloud\.com/datasets/([0-9a-f]{24})', 'tokens', 'once'); + if ~isempty(tok) + ids{end+1} = ['ndicloud:' tok{1}]; %#ok + else + ids{end+1} = v; %#ok + end +end +end + +function state = relate(state, childKey, parentKey, relation, childKinds, parentKinds, extra) +if nargin < 7 + extra = struct(); +end +child = idOf(state, childKey, childKinds); +parent = idOf(state, parentKey, parentKinds); +opts = {}; +if isfield(extra, 'sequence') + opts = [opts, {'Sequence', extra.sequence}]; +end +if isfield(extra, 'roles') + opts = [opts, {'Fields', struct('roles', extra.roles)}]; +end +doc = did2.build.directedRelation(child, parent, relation, 'SessionId', state.sid, ... + 'Validate', state.validate, opts{:}); +state.docs{end+1} = doc; +end diff --git a/src/ndi/+ndi/+setup/+V2/discover.m b/src/ndi/+ndi/+setup/+V2/discover.m new file mode 100644 index 0000000000..16001f3239 --- /dev/null +++ b/src/ndi/+ndi/+setup/+V2/discover.m @@ -0,0 +1,88 @@ +function files = discover(root, options) +%DISCOVER Stage 0 of a V2 import: list the source files under ROOT. +% +% FILES = ndi.setup.V2.discover(ROOT) walks ROOT and returns a table with +% one row per file: path (relative to ROOT), folder, name, ext, bytes and +% modified. Nothing is read or assumed about what the files mean -- that is +% what the later stages ask about. +% +% An import's OWN earlier output is skipped by default, because profiling +% it would describe the last import instead of the data: NDI databases +% (`.ndi`, and the `.ndi_` copies doImport makes), dataset folders (any +% folder holding a `.ndi` of its own is still skipped by that rule), +% archives, and OS clutter. Pass 'Skip' to change the list; every skipped +% path is COUNTED and reported, never silently dropped. +% +% Options: +% 'Skip' cellstr of folder names or extensions/names to skip +% (default {'.ndi', '.ndi_', '.DS_Store', '__MACOSX', '.zip'}) +% 'SkipFolders' cellstr of additional folder paths, relative to ROOT +% 'Verbose' default true: print the denominator +% +% The denominator is printed first (Operating Rule 5): folders walked, +% files seen, files kept, and files skipped by which rule. +% +% See also ndi.setup.V2.profileTable. + +arguments + root (1,:) char {mustBeFolder} + options.Skip cell = {'.ndi', '.ndi_', '.DS_Store', '__MACOSX', '.zip'} + options.SkipFolders cell = {} + options.Verbose (1,1) logical = true +end + +listing = dir(fullfile(root, '**', '*')); +isDir = [listing.isdir]; +names = {listing.name}; +dirs = listing(isDir & ~ismember(names, {'.', '..'})); +listing = listing(~isDir); + +path = cell(numel(listing), 1); +keep = true(numel(listing), 1); +skippedBy = containers.Map(); +for k = 1:numel(listing) + full = fullfile(listing(k).folder, listing(k).name); + rel = full(numel(root) + 2:end); + path{k} = rel; + parts = strsplit(rel, filesep); + [~, ~, ext] = fileparts(listing(k).name); + rule = ''; + hit = intersect(parts(1:end-1), options.Skip); + if ~isempty(hit) + rule = ['folder ' hit{1}]; + elseif any(strcmp(ext, options.Skip)) + rule = ['ext ' ext]; + elseif any(strcmp(listing(k).name, options.Skip)) + rule = ['name ' listing(k).name]; + else + for s = 1:numel(options.SkipFolders) + if startsWith(rel, [options.SkipFolders{s} filesep]) + rule = ['folder ' options.SkipFolders{s}]; + break; + end + end + end + if ~isempty(rule) + keep(k) = false; + if isKey(skippedBy, rule), skippedBy(rule) = skippedBy(rule) + 1; + else, skippedBy(rule) = 1; end + end +end + +listing = listing(keep); +path = path(keep); +[folder, name, ext] = cellfun(@fileparts, path, 'UniformOutput', false); +files = table(path, folder, name, ext, [listing.bytes]', ... + datetime([listing.datenum]', 'ConvertFrom', 'datenum'), ... + 'VariableNames', {'path', 'folder', 'name', 'ext', 'bytes', 'modified'}); +files = sortrows(files, 'path'); + +if options.Verbose + fprintf('DENOMINATOR: %d folder(s) walked under %s; %d file(s) seen, %d kept, %d skipped\n', ... + numel(dirs), root, numel(keep), sum(keep), sum(~keep)); + rules = keys(skippedBy); + for r = 1:numel(rules) + fprintf(' skipped %6d by %s\n', skippedBy(rules{r}), rules{r}); + end +end +end diff --git a/src/ndi/+ndi/+setup/+V2/entityDocuments.m b/src/ndi/+ndi/+setup/+V2/entityDocuments.m new file mode 100644 index 0000000000..f3d0cff02d --- /dev/null +++ b/src/ndi/+ndi/+setup/+V2/entityDocuments.m @@ -0,0 +1,179 @@ +function docs = entityDocuments(className, fields, edges, options) +%ENTITYDOCUMENTS The documents that record one entity, for the schema in use. +% +% DOCS = ndi.setup.V2.entityDocuments(CLASSNAME, FIELDS, EDGES, 'SessionId', SID) +% builds an entity written for the pre-2026-10-08 classes (CLASSNAME +% 'subject', 'strain', 'person', 'dataset', 'session', 'epoch', ...; FIELDS +% and EDGES as that class declared them) and returns a cell array of +% documents, the entity's own first. +% +% On a schema that still has CLASSNAME, DOCS is that one document. +% +% On a schema with one `entity` class (did-schema +% V_eta_entity_composition_plan.md, 2026-10-08), DOCS is: +% 1. an `entity` with `type` CLASSNAME (for a subject, its own `type`, or the +% 'Type' option) and the identity fields -- name, local_identifier, +% description, global_identifier (as CURIEs, ndi.setup.V2.curie) -- and +% the time reference edge; +% 2. one assertion per other field (a term, text or date assertion whose +% `variable` is the field's name with spaces); +% 3. one relation per other edge: vendor_id -> sold_by, background_strain_id +% -> derived_from, product_id -> distributed_as (derived_from on a schema +% without it). +% A field or edge with no home there is an error (ndi:setup:V2:noHome), never +% dropped. +% +% Options: 'SessionId' (required), 'Id', 'Validate' (default true), +% 'Type' (the entity type of a subject that gives none). +% +% See also ndi.setup.V2.mergedEntities, ndi.setup.V2.curie. + +arguments + className (1,:) char + fields (1,1) struct + edges (1,1) struct = struct() + options.SessionId (1,:) char + options.Id (1,:) char = '' + options.Validate (1,1) logical = true + options.Type (1,:) char = '' +end + +common = {'SessionId', options.SessionId, 'Validate', options.Validate}; +if ~ndi.setup.V2.mergedEntities() + docs = {did2.build.document(className, fields, 'Edges', edges, ... + 'Id', options.Id, common{:})}; + return; +end + +% ---- 1. the entity ------------------------------------------------------------ +type = className; +if strcmp(className, 'subject') + if isfield(fields, 'type') && ~isempty(fields.type) + type = termName(fields.type); + elseif ~isempty(options.Type) + type = options.Type; + else + error('ndi:setup:V2:noType', ['A subject needs a type on a schema with ' ... + 'one entity class; pass ''Type'' (organism, device, ...).']); + end +end +ent = struct('type', did2.build.term('', type)); +if strcmp(className, 'person') && ~isfield(fields, 'name') + % the type registry asks a person for a display name; the spec gives parts + parts = {}; + for n = {'given_name', 'family_name'} + if isfield(fields, n{1}) && ~isempty(fields.(n{1})) + parts{end+1} = char(fields.(n{1})); %#ok + end + end + if ~isempty(parts) + fields.name = strjoin(parts, ' '); + end +end +for n = {'name', 'local_identifier', 'description'} + if isfield(fields, n{1}) && ~isempty(fields.(n{1})) + ent.(n{1}) = fields.(n{1}); + end +end +if isfield(fields, 'global_identifier') && ~isempty(fields.global_identifier) + [ids, award] = ndi.setup.V2.curie(fields.global_identifier); + if ~isempty(ids) + ent.global_identifier = ids; + end + if ~isempty(award) && ~isfield(ent, 'local_identifier') + ent.local_identifier = award; + end +end +entEdges = struct(); +if isfield(edges, 'time_reference_id') + entEdges.time_reference_id = edges.time_reference_id; +end +docs = {did2.build.document('entity', ent, 'Edges', entEdges, 'Id', options.Id, common{:})}; +eid = docs{1}.base.id; + +% ---- 2. the other fields, as assertions ------------------------------------------ +identity = {'type', 'name', 'local_identifier', 'description', 'global_identifier'}; +terms = {'species', 'genetic_strain_type', 'phenotype', 'breeding_type', 'disease_model', ... + 'keyword', 'license', 'accessibility', 'ethics_assessment', 'experimental_approach', ... + 'factors', 'design'}; +dates = {'release_date', 'publication_date'}; +texts = {'short_name', 'version', 'version_innovation', 'how_to_cite', 'support_channel', ... + 'copyright_year', 'given_name', 'family_name', 'alternate_name', 'email', ... + 'catalog_number', 'lot_number', 'laboratory_code', 'synonym', 'authors'}; +names = setdiff(fieldnames(fields), identity, 'stable'); +for k = 1:numel(names) + f = names{k}; + v = fields.(f); + if isempty(v) + continue; + end + variable = strrep(f, '_', ' '); + if any(strcmp(f, terms)) + docs{end+1} = did2.build.statement('term_assertion', eid, variable, asTerms(v), common{:}); %#ok + elseif any(strcmp(f, dates)) + d = cellstr(v); + docs{end+1} = did2.build.statement('date_assertion', eid, variable, ... + did2.build.valueCell('date', d, 'Fields', struct('precision', precisionOf(d{1}))), ... + common{:}); %#ok + elseif any(strcmp(f, texts)) + docs{end+1} = did2.build.statement('text_assertion', eid, variable, ... + did2.build.valueCell('text', cellstr(string(v))), common{:}); %#ok + else + error('ndi:setup:V2:noHome', ['`%s.%s` has no home on a schema with one ' ... + 'entity class: name it in ndi.setup.V2.entityDocuments.'], className, f); + end +end + +% ---- 3. the other edges, as relations --------------------------------------------- +relationOf = struct('vendor_id', 'sold_by', 'background_strain_id', 'derived_from', ... + 'product_id', 'derived_from'); +% a strain is DISTRIBUTED AS its product (a stock-centre or vendor listing), not +% derived from it (Haley import decision #17; did-schema 033a22d, 2026-10-15); +% `derived_from` stays only on a schema copy without `distributed_as` +if ndi.setup.V2.schemaHasRelation('distributed_as') + relationOf.product_id = 'distributed_as'; +end +enames = setdiff(fieldnames(edges), {'time_reference_id'}, 'stable'); +for k = 1:numel(enames) + e = enames{k}; + if ~isfield(relationOf, e) + error('ndi:setup:V2:noHome', ['Edge `%s.%s` has no home on a schema with ' ... + 'one entity class: name it in ndi.setup.V2.entityDocuments.'], className, e); + end + parents = cellstr(edges.(e)); + for j = 1:numel(parents) + docs{end+1} = did2.build.directedRelation(eid, parents{j}, ... + did2.build.term('', relationOf.(e)), common{:}); %#ok + end +end +end + +function n = termName(t) +if isstruct(t) + n = char(t.name); +else + n = char(t); +end +end + +function t = asTerms(v) +% one term, or a list of them, from terms or plain names +if isstruct(v) + t = v; + return; +end +c = cellstr(string(v)); +t = did2.build.label(c{1}); +for k = 2:numel(c) + t(k) = did2.build.label(c{k}); +end +end + +function p = precisionOf(s) +% a partial-precision date: YYYY, YYYY-MM or YYYY-MM-DD +switch numel(regexp(s, '\d+', 'match')) + case 1, p = 'year'; + case 2, p = 'month'; + otherwise, p = 'day'; +end +end diff --git a/src/ndi/+ndi/+setup/+V2/kindOf.m b/src/ndi/+ndi/+setup/+V2/kindOf.m new file mode 100644 index 0000000000..a7a4610149 --- /dev/null +++ b/src/ndi/+ndi/+setup/+V2/kindOf.m @@ -0,0 +1,9 @@ +function [kind, block] = kindOf(doc) +%KINDOF What a built V2 document is: its class, or an entity's type. +% +% [KIND, BLOCK] = ndi.setup.V2.kindOf(DOC) is ndi.v2.kindOf(DOC). +% +% See also ndi.v2.kindOf. + +[kind, block] = ndi.v2.kindOf(doc); +end diff --git a/src/ndi/+ndi/+setup/+V2/makeSessions.m b/src/ndi/+ndi/+setup/+V2/makeSessions.m new file mode 100644 index 0000000000..275dc28830 --- /dev/null +++ b/src/ndi/+ndi/+setup/+V2/makeSessions.m @@ -0,0 +1,96 @@ +function [T, sessions] = makeSessions(T, options) +%MAKESESSIONS Stage 3 of a V2 import: create one session per row of T. +% +% [T, SESSIONS] = ndi.setup.V2.makeSessions(T) creates a V2 session for +% each row of the table T (with ndi.setup.V2.createSession) and returns T +% with a `session_id` column filled in, plus the opened ndi.session.dir +% objects in the same order. +% +% T's columns: +% local_identifier the stable, spaceless handle (required, unique) +% path the session's directory; created if missing (required) +% name display name (optional column) +% description free text (optional column) +% study_ids a cellstr per row: the studies it is part_of (optional) +% session_id a preassigned session id (optional; '' = new) +% session_doc_id a preassigned session-document id (optional) +% time_reference_id the session's time reference (optional) +% documents a cell per row: further documents for that session, +% built with its session_id (optional) +% +% The table is checked BEFORE anything is written: a missing required +% column, an empty or duplicated local_identifier, or two rows sharing a +% path is an error (ndi:setup:V2:badSessionTable), so a bad table creates +% no sessions at all. +% +% Options: +% 'Overwrite' passed to createSession (default false) +% 'Validate', 'Progress', 'BatchSize' passed to createSession +% (defaults true, false, 2000) +% 'Verbose' default true: print the denominator +% +% See also ndi.setup.V2.createSession, ndi.setup.conv.haley.sessionList. + +arguments + T table + options.Overwrite (1,1) logical = false + options.Validate (1,1) logical = true + options.Progress (1,1) logical = false + options.BatchSize (1,1) double {mustBePositive, mustBeInteger} = 2000 + options.Verbose (1,1) logical = true +end + +vars = T.Properties.VariableNames; +for need = {'local_identifier', 'path'} + if ~any(strcmp(vars, need{1})) + error('ndi:setup:V2:badSessionTable', 'The session table has no `%s` column.', need{1}); + end +end +ids = cellstr(T.local_identifier); +if any(cellfun(@isempty, ids)) + error('ndi:setup:V2:badSessionTable', 'A row has an empty local_identifier.'); +end +[u, ~, j] = unique(ids); +dup = u(accumarray(j(:), 1) > 1); +if ~isempty(dup) + error('ndi:setup:V2:badSessionTable', 'local_identifier repeated: %s.', strjoin(dup, ', ')); +end +paths = cellstr(T.path); +[u, ~, j] = unique(paths); +dup = u(accumarray(j(:), 1) > 1); +if ~isempty(dup) + error('ndi:setup:V2:badSessionTable', 'Two sessions share a directory: %s.', strjoin(dup, ', ')); +end + +if ~any(strcmp(vars, 'session_id')) + T.session_id = repmat({''}, height(T), 1); +end +sessions = cell(height(T), 1); +for k = 1:height(T) + if ~isfolder(paths{k}) + mkdir(paths{k}); + end + args = {'Overwrite', options.Overwrite, 'Validate', options.Validate, ... + 'Progress', options.Progress, 'BatchSize', options.BatchSize}; + if any(strcmp(vars, 'name')), args = [args, {'Name', char(T.name{k})}]; end %#ok + if any(strcmp(vars, 'description')), args = [args, {'Description', char(T.description{k})}]; end %#ok + if any(strcmp(vars, 'study_ids')), args = [args, {'StudyIds', T.study_ids{k}}]; end %#ok + if ~isempty(T.session_id{k}), args = [args, {'SessionId', char(T.session_id{k})}]; end %#ok + for opt = {'session_doc_id', 'SessionDocId'; 'time_reference_id', 'TimeReferenceId'; ... + 'documents', 'Documents'}' + if any(strcmp(vars, opt{1})) && ~isempty(T.(opt{1}){k}) + args = [args, {opt{2}, T.(opt{1}){k}}]; %#ok + end + end + if options.Progress + fprintf('session %d of %d: %s -> %s\n', k, height(T), ids{k}, paths{k}); + end + sessions{k} = ndi.setup.V2.createSession(paths{k}, ids{k}, args{:}); + T.session_id{k} = sessions{k}.id(); +end + +if options.Verbose + fprintf('DENOMINATOR: %d session row(s) read, %d session(s) created\n', ... + height(T), sum(~cellfun(@isempty, T.session_id))); +end +end diff --git a/src/ndi/+ndi/+setup/+V2/mergedEntities.m b/src/ndi/+ndi/+setup/+V2/mergedEntities.m new file mode 100644 index 0000000000..7ed9be5447 --- /dev/null +++ b/src/ndi/+ndi/+setup/+V2/mergedEntities.m @@ -0,0 +1,15 @@ +function tf = mergedEntities() +%MERGEDENTITIES True when the V2 schema in use has one `entity` class. +% +% TF = ndi.setup.V2.mergedEntities() is true for a V_eta schema built on or +% after 2026-10-08, where subject, strain, product, software, person, +% organization, funding, web_resource, dataset, study, publication, session and +% epoch merged into one concrete `entity` with a bound `type` (did-schema +% V_eta_entity_composition_plan.md sec. 4), and false for one built before. +% The import's writers ask, so they build valid documents against either. +% +% See also ndi.setup.V2.entityDocuments. + +c = did2.schema.cache.shared(); +tf = c.hasClass('entity') && ~c.hasClass('subject'); +end diff --git a/src/ndi/+ndi/+setup/+V2/preflight.m b/src/ndi/+ndi/+setup/+V2/preflight.m new file mode 100644 index 0000000000..b7150d3981 --- /dev/null +++ b/src/ndi/+ndi/+setup/+V2/preflight.m @@ -0,0 +1,79 @@ +function preflight(options) +%PREFLIGHT Check that a V2 import can run, and say how to fix it if not. +% +% ndi.setup.V2.preflight() errors (ndi:setup:V2:preflight) with ONE message +% listing every missing requirement, before any stage runs: +% +% * DID-matlab's did2.build is on the MATLAB path (DID-matlab `V2`, since +% DID-matlab PR #210); +% * DID_SCHEMA_PATH names a folder holding the V2 (V_eta) schema, assembled +% flat from did-schema main's schemas/V_eta/{stable,draft,deprecated}; +% * that schema carries each class listed in 'Classes'. +% +% A missing requirement otherwise surfaces deep inside a stage as "Unable +% to resolve the name 'did2.build.label'" or a schema error, which says +% nothing about the fix. +% +% Options: +% 'Classes' cellstr of schema classes the import needs +% (default {'study', 'session'}) + +arguments + options.Classes cell = {'study', 'session'} +end + +problems = {}; +if isempty(which('did2.build.document')) + problems{end+1} = sprintf(['did2.build is not on the MATLAB path. Update DID-matlab to its ' ... + '`V2` branch (git checkout V2; git pull) and add it to the path.']); +end +sp = getenv('DID_SCHEMA_PATH'); +if isempty(sp) || ~isfolder(sp) + problems{end+1} = sprintf(['DID_SCHEMA_PATH is not set to a folder (it is "%s"). Copy did-schema ' ... + 'main''s schemas/V_eta/{stable,draft,deprecated}/*.json into one folder and ' ... + 'setenv(''DID_SCHEMA_PATH'', ), then did2.schema.cache.resetSingleton().'], sp); +else + % a class merged into `entity` on 2026-10-08 counts as present (schemaHasClass) + missing = options.Classes(~cellfun(@(c) ndi.setup.V2.schemaHasClass(c, sp), options.Classes)); + if ~isempty(missing) + problems{end+1} = sprintf(['The schema in DID_SCHEMA_PATH (%s) has no %s. It predates ' ... + 'the V2 classes this import uses: re-copy it from did-schema main.'], ... + sp, strjoin(strcat('`', missing, '`'), ', ')); + elseif any(strcmp(options.Classes, 'session')) + if ~ndi.setup.V2.schemaHasField('session', 'name') + problems{end+1} = sprintf(['The schema in DID_SCHEMA_PATH (%s) has no session.name ' ... + '(did-schema PR #79): re-copy it from did-schema main.'], sp); + end + end +end +% The schema cache is a SINGLETON: whichever path it saw first (DID_SCHEMA_PATH, +% or DID's own default -- a V_delta folder -- when the variable was unset) it +% keeps until reset. Setting DID_SCHEMA_PATH afterwards changes nothing, which +% is how a correct setenv still produced "No schema file ... V_delta/stable". +% So compare the path the cache is ACTUALLY using, and reset it when stale. +if isempty(problems) + cache = did2.schema.cache.shared(); + if ~strcmp(canonical(cache.schemaPath), canonical(sp)) + fprintf(['preflight: the schema cache was loaded from %s; resetting it to ' ... + 'DID_SCHEMA_PATH (%s).\n'], cache.schemaPath, sp); + did2.schema.cache.resetSingleton(); + cache = did2.schema.cache.shared(); + if ~strcmp(canonical(cache.schemaPath), canonical(sp)) + problems{end+1} = sprintf(['The schema cache still reads %s after a reset, ' ... + 'not DID_SCHEMA_PATH (%s).'], cache.schemaPath, sp); + end + end +end +if ~isempty(problems) + error('ndi:setup:V2:preflight', 'The V2 import cannot run yet:\n - %s', ... + strjoin(problems, sprintf('\n - '))); +end +end + +function p = canonical(p) +% Compare folders, not spellings: drop a trailing separator. +p = char(p); +while numel(p) > 1 && any(p(end) == '/\\') + p = p(1:end-1); +end +end diff --git a/src/ndi/+ndi/+setup/+V2/profileTable.m b/src/ndi/+ndi/+setup/+V2/profileTable.m new file mode 100644 index 0000000000..4347a6cbc4 --- /dev/null +++ b/src/ndi/+ndi/+setup/+V2/profileTable.m @@ -0,0 +1,62 @@ +function P = profileTable(T, options) +%PROFILETABLE Describe every column of a table, without interpreting it. +% +% P = ndi.setup.V2.profileTable(T) returns one row per column of table T: +% column, class (for a cell column, the classes it holds), cellSize (the +% sizes of the cells, from the first 50), nRows, nUnique (NaN where +% uniqueness is not defined, e.g. image arrays), example (the first value, +% or its class and size when it is large) and description +% (T.Properties.VariableDescriptions, when the source set any). +% +% This is the raw material for an import's questions -- "which column +% identifies a worm?", "what unit is OD600 in?" -- and deliberately +% answers none of them. A column that looks like a key has nUnique equal +% to nRows; a column that looks like a condition has few unique values. +% +% Options: +% 'MaxExample' numel above which the example is summarised (default 10) + +arguments + T table + options.MaxExample (1,1) double = 10 +end + +n = width(T); +P = table('Size', [n 7], ... + 'VariableTypes', {'string', 'string', 'string', 'double', 'double', 'string', 'string'}, ... + 'VariableNames', {'column', 'class', 'cellSize', 'nRows', 'nUnique', 'example', 'description'}); +desc = T.Properties.VariableDescriptions; +for c = 1:n + x = T.(c); + P.column(c) = T.Properties.VariableNames{c}; + P.class(c) = class(x); + P.nRows(c) = height(T); + if iscell(x) + P.class(c) = "cell of " + strjoin(unique(cellfun(@class, x, 'UniformOutput', false)), '/'); + sample = x(1:min(end, 50)); + P.cellSize(c) = strjoin(unique(cellfun(@(e) mat2str(size(e)), sample, ... + 'UniformOutput', false)), ' '); + else + P.cellSize(c) = mat2str(size(x, 2:ndims(x))); + end + try + P.nUnique(c) = height(unique(x)); + catch + P.nUnique(c) = NaN; + end + try + e = x(1, :); + if iscell(e), e = e{1}; end + if numel(e) > options.MaxExample + P.example(c) = sprintf('<%s %s>', class(e), mat2str(size(e))); + else + P.example(c) = strtrim(evalc('disp(e)')); + end + catch + P.example(c) = "?"; + end + if numel(desc) >= c + P.description(c) = desc{c}; + end +end +end diff --git a/src/ndi/+ndi/+setup/+V2/schemaHasClass.m b/src/ndi/+ndi/+setup/+V2/schemaHasClass.m new file mode 100644 index 0000000000..bc0f2c30ec --- /dev/null +++ b/src/ndi/+ndi/+setup/+V2/schemaHasClass.m @@ -0,0 +1,23 @@ +function tf = schemaHasClass(className, schemaPath) +%SCHEMAHASCLASS True when the V2 schema in SCHEMAPATH can hold CLASSNAME. +% +% TF = ndi.setup.V2.schemaHasClass(CLASSNAME, SCHEMAPATH) (SCHEMAPATH defaults +% to DID_SCHEMA_PATH) is true when the schema has CLASSNAME.json, or -- on a +% schema built on or after 2026-10-08 -- when CLASSNAME is one of the entity +% classes merged into `entity` (study, session, subject, ...; +% ndi.v2.entityTypesFor) and entity.json is there. Asked by file, so it works +% before the schema cache is loaded. +% +% See also ndi.setup.V2.schemaHasField, ndi.setup.V2.schemaHasLeaf. + +if nargin < 2 + schemaPath = getenv('DID_SCHEMA_PATH'); +end +tf = false; +if isempty(schemaPath) + return; +end +tf = isfile(fullfile(schemaPath, [char(className) '.json'])) || ... + (~isempty(ndi.v2.entityTypesFor(className)) && isfile(fullfile(schemaPath, 'entity.json')) ... + && ~isfile(fullfile(schemaPath, 'subject.json'))); +end diff --git a/src/ndi/+ndi/+setup/+V2/schemaHasField.m b/src/ndi/+ndi/+setup/+V2/schemaHasField.m new file mode 100644 index 0000000000..2c29474fcf --- /dev/null +++ b/src/ndi/+ndi/+setup/+V2/schemaHasField.m @@ -0,0 +1,69 @@ +function [tf, decl] = schemaHasField(className, fieldName) +%SCHEMAHASFIELD Whether the V2 schema on DID_SCHEMA_PATH declares a field. +% +% TF = ndi.setup.V2.schemaHasField(CLASSNAME, FIELDNAME) is true when +% DID_SCHEMA_PATH/.json declares FIELDNAME among its own fields. +% FIELDNAME may be a dotted path to a nested field ('value.type'). +% A field added to did-schema after a schema copy was made reads as absent, +% so a maker can leave it out rather than write a document that schema +% refuses (e.g. subject.name, did-schema #80; subject.type, #84). +% +% [TF, DECL] = ... also returns the field's declaration (its `type`, +% `constraints`, ...), or [] when it is absent, for a field whose TYPE +% changed between schema copies (data_body.format, 2026-10-14). + +arguments + className (1,:) char + fieldName (1,:) char +end +tf = false; +decl = []; +p = getenv('DID_SCHEMA_PATH'); +% a class renamed 2026-10-08 is looked up under whichever name the schema +% in use has (ndi.v2.vetaAliases: 'statement' or 'subject_statement') +names = ndi.v2.vetaAliases(className); +f = ''; +for k = 1:numel(names) + if ~isempty(p) && isfile(fullfile(p, [names{k} '.json'])) + f = fullfile(p, [names{k} '.json']); + break; + end +end +if isempty(f) && any(strcmp(className, {'subject', 'strain', 'product', 'software', ... + 'person', 'organization', 'funding', 'web_resource', 'dataset', 'study', ... + 'publication', 'session', 'epoch'})) && isfile(fullfile(p, 'entity.json')) + % merged into one `entity` class on 2026-10-08 (ndi.setup.V2.mergedEntities): + % the identity fields (name, type, local_identifier, ...) live there + f = fullfile(p, 'entity.json'); +end +if isempty(f) + return; +end +d = jsondecode(fileread(f)); +if ~isfield(d, 'fields') + return; +end +fields = d.fields; +parts = strsplit(fieldName, '.'); +for k = 1:numel(parts) + hit = []; + for m = 1:numel(fields) + if iscell(fields), x = fields{m}; else, x = fields(m); end + if strcmp(x.name, parts{k}) + hit = x; + break; + end + end + if isempty(hit) + return; + end + if k < numel(parts) + if ~isfield(hit, 'fields') || isempty(hit.fields) + return; + end + fields = hit.fields; % jsondecode gives a cell when the fields differ in shape + end +end +tf = true; +decl = hit; +end diff --git a/src/ndi/+ndi/+setup/+V2/schemaHasLeaf.m b/src/ndi/+ndi/+setup/+V2/schemaHasLeaf.m new file mode 100644 index 0000000000..6363a28270 --- /dev/null +++ b/src/ndi/+ndi/+setup/+V2/schemaHasLeaf.m @@ -0,0 +1,23 @@ +function tf = schemaHasLeaf(leaf) +%SCHEMAHASLEAF True when the V2 schema in use can build statement leaf LEAF. +% +% TF = ndi.setup.V2.schemaHasLeaf('time_calculation') is true when the schema +% has that class, or -- on a schema built on or after 2026-10-08, where the +% join leaves are gone and a statement's value kind is a mixin (did-schema +% V_eta_entity_composition_plan.md sec. 1) -- when it has the direction +% (`calculation`) taking a value kind and the kind (`time`). did2.build reads +% the leaf name either way. +% +% See also ndi.setup.V2.schemaHasField, ndi.setup.V2.mergedEntities. + +c = did2.schema.cache.shared(); +tf = c.hasClass(leaf); +if tf || ~ismethod(c, 'valueKindRule') + return; +end +tok = regexp(leaf, '^(.+)_(assertion|observation|manipulation|calculation)$', 'tokens', 'once'); +if isempty(tok) + return; +end +tf = c.hasClass(tok{2}) && ~isempty(c.valueKindRule(tok{2})) && c.hasClass(tok{1}); +end diff --git a/src/ndi/+ndi/+setup/+V2/schemaHasRelation.m b/src/ndi/+ndi/+setup/+V2/schemaHasRelation.m new file mode 100644 index 0000000000..242c2f5e83 --- /dev/null +++ b/src/ndi/+ndi/+setup/+V2/schemaHasRelation.m @@ -0,0 +1,35 @@ +function tf = schemaHasRelation(relationName, schemaPath) +%SCHEMAHASRELATION True when the V2 schema in SCHEMAPATH binds a relation of this name. +% +% TF = ndi.setup.V2.schemaHasRelation(RELATIONNAME, SCHEMAPATH) (SCHEMAPATH +% defaults to DID_SCHEMA_PATH) is true when binding_registry_meta.json lists +% RELATIONNAME among its relation_bindings. A relation added to did-schema +% after a schema copy was made reads as absent, so a maker can report the fact +% rather than write a relation that schema does not know (e.g. `developed_by` +% and the dataset's `hosted_by`, did-schema 9bedceb, 2026-10-15). Asked by +% file, so it works before the schema cache is loaded. +% +% See also ndi.setup.V2.schemaHasClass, ndi.setup.V2.schemaHasField. + +if nargin < 2 + schemaPath = getenv('DID_SCHEMA_PATH'); +end +tf = false; +f = fullfile(schemaPath, 'binding_registry_meta.json'); +if isempty(schemaPath) || ~isfile(f) + return; +end +reg = jsondecode(fileread(f)); +if ~isfield(reg, 'relation_bindings') + return; +end +rows = reg.relation_bindings; +for k = 1:numel(rows) + if iscell(rows), r = rows{k}; else, r = rows(k); end + if isfield(r, 'relation') && isfield(r.relation, 'name') ... + && strcmp(r.relation.name, relationName) + tf = true; + return; + end +end +end diff --git a/src/ndi/+ndi/+setup/+V2/writeDocuments.m b/src/ndi/+ndi/+setup/+V2/writeDocuments.m new file mode 100644 index 0000000000..3fbbff1a44 --- /dev/null +++ b/src/ndi/+ndi/+setup/+V2/writeDocuments.m @@ -0,0 +1,123 @@ +function stats = writeDocuments(dbFile, docs, options) +%WRITEDOCUMENTS Write V2 document structs into a did2 database in batches. +% +% STATS = ndi.setup.V2.writeDocuments(DBFILE, DOCS) adds DOCS (a cell of +% document structs, as built by did2.build) to the did2 sqlite database +% DBFILE, BatchSize documents at a time. Each batch is one did2 +% sqlitedb.add: its ingested files are copied, then its documents are +% inserted in one transaction and committed, so the database and the files +% folder grow as the write goes and a progress line is printed per batch +% (documents, files copied, elapsed time, estimated time left). A +% ndi.gui.component.ProgressBarWindow bar follows it when MATLAB has a +% display (the window is silent when headless). +% +% A failed batch is rolled back by sqlitedb.add, but the batches before it +% stay written: the error says how many documents are already in DBFILE. +% +% Options: +% 'BatchSize' documents per batch (default 2000) +% 'Validate' default true: validate each document against the schema +% as it is inserted. Documents built by did2.build were +% validated when they were built (did2.build.document, +% 'Validate' true, the default), with the same +% did2.schema.cache.validateDocument, so a caller writing +% only such documents can pass false and skip the repeat. +% 'Label' text for the progress lines and bar (default 'documents') +% 'Progress' default true: print per-batch lines and show the bar +% +% STATS fields: documents, batches, files (ingested files in DOCS), +% convertSeconds (struct -> did2.document), addSeconds (copy + insert + +% commit), seconds (total). +% +% See also ndi.setup.V2.createDataset, ndi.setup.V2.createSession. + +arguments + dbFile (1,:) char + docs cell + options.BatchSize (1,1) double {mustBePositive, mustBeInteger} = 2000 + options.Validate (1,1) logical = true + options.Label (1,:) char = 'documents' + options.Progress (1,1) logical = true +end + +n = numel(docs); +nFiles = cellfun(@ingestedCount, docs); +stats = struct('documents', n, 'batches', ceil(n / options.BatchSize), ... + 'files', sum(nFiles), 'convertSeconds', 0, 'addSeconds', 0, 'seconds', 0); +if n == 0 + return; +end + +bar = []; +tag = ''; +if options.Progress + fprintf('writing %d %s (%d ingested file(s)) in %d batch(es) of up to %d\n', ... + n, options.Label, stats.files, stats.batches, options.BatchSize); + try + bar = ndi.gui.component.ProgressBarWindow('NDI V2 write', 'GrabMostRecent', true); + bar.setTimeout(hours(12)); + tag = ['write ' options.Label ' ' char(ndi.ido.unique_id())]; + bar.addBar('Label', sprintf('Writing %d %s', n, options.Label), 'Tag', tag, 'Auto', true); + catch + bar = []; + end +end + +db = did2.database.sqlitedb(dbFile); +closer = onCleanup(@() db.close()); +t0 = tic; +done = 0; +filesDone = 0; +for b = 1:stats.batches + idx = (b-1)*options.BatchSize + 1 : min(b*options.BatchSize, n); + tc = tic; + batch = cellfun(@(d) did2.document(d), docs(idx), 'UniformOutput', false); + stats.convertSeconds = stats.convertSeconds + toc(tc); + ta = tic; + try + db.add(batch, 'Validate', options.Validate); + catch err + error('ndi:setup:V2:writeFailed', ... + ['Writing %s batch %d of %d (documents %d-%d) failed; %d document(s) of %d ' ... + 'are already in %s, so it is INCOMPLETE (write it again with ''Overwrite'', ' ... + 'true). Cause: %s'], options.Label, b, stats.batches, idx(1), idx(end), ... + done, n, dbFile, err.message); + end + stats.addSeconds = stats.addSeconds + toc(ta); + done = idx(end); + filesDone = filesDone + sum(nFiles(idx)); + if options.Progress + el = toc(t0); + left = el / done * (n - done); + fprintf(' %s: %d / %d documents (%.0f%%), %d / %d files, %s elapsed, ~%s left\n', ... + options.Label, done, n, 100 * done / n, filesDone, stats.files, ... + hms(el), hms(left)); + if ~isempty(bar) + try bar.updateBar(tag, done / n); catch, end + end + end +end +stats.seconds = toc(t0); +if options.Progress + fprintf(' %s written in %s (converting %s, copying + inserting %s)\n', options.Label, ... + hms(stats.seconds), hms(stats.convertSeconds), hms(stats.addSeconds)); +end +end + +function k = ingestedCount(d) +k = 0; +if ~isfield(d, 'files') || ~isfield(d.files, 'file_info'), return; end +fi = d.files.file_info; +for a = 1:numel(fi) + if ~isfield(fi(a), 'locations'), continue; end + for c = 1:numel(fi(a).locations) + if isfield(fi(a).locations(c), 'ingest') && fi(a).locations(c).ingest + k = k + 1; + end + end +end +end + +function s = hms(sec) +s = char(duration(0, 0, round(sec), 'Format', 'hh:mm:ss')); +end diff --git a/src/ndi/+ndi/+setup/+conv/+haley/assertionDocuments.m b/src/ndi/+ndi/+setup/+conv/+haley/assertionDocuments.m new file mode 100644 index 0000000000..08fe37be2a --- /dev/null +++ b/src/ndi/+ndi/+setup/+conv/+haley/assertionDocuments.m @@ -0,0 +1,198 @@ +function out = assertionDocuments(session, S, subjectIds, options) +%ASSERTIONDOCUMENTS Stage 7 (Haley): what one session's subjects are. +% +% OUT = ndi.setup.conv.haley.assertionDocuments(SESSION, S, SUBJECTIDS) +% builds the term_assertion documents about the subjects of the one-row +% session table SESSION (from sessionList, with `session_id` assigned). S +% is the subject table (subjectList); SUBJECTIDS maps each subject's +% local_identifier to its document id (sessionDocuments returns it). +% Nothing is written here. Decision log #2, #55. +% +% Assertions: +% cohort species, strain the worms' species and strain (from the spec's +% strain entry), stated once on the cohort with +% `distributive` true: they hold of each worm. +% patch species, strain a patch with bacteria (a culture): OP50 on the +% C. elegans plates, OP50-GFP on the E. coli +% plates. A patch of LB alone gets none. +% assay plate inclusion in analysis = excluded, where the source excludes +% it (decision #2: tagged, not dropped). Per plate: +% no plate has some videos excluded and others not. +% Its reason, where one was written, is the tag's +% notes (or, with a schema whose statements have no +% notes, a text statement `reason for inclusion in +% analysis` beside it): +% the day's tableOfContents `notes` when that day +% is marked `include = no` (decision #73). The +% day's flag itself tags nothing: the lab's +% analysis excludes by plate. +% assay plate behavior recording = Not Performed (NCIT:C141477), on a +% plate never filmed (subjectList `filmed` false), +% noted "Prepared but not filmed: no video." (as +% above, a `reason for behavior recording` text +% statement with an older schema) +% (decision #73). Its worms are kept. +% cohort biological sex hermaphrodite (PATO:0001340), from the spec's +% strain entry (`biological_sex`), distributive +% like species and strain. Only strains that +% state a sex get one, so a patch gets none. +% A strain assertion carries `strain_id`, the dataset-level strain document +% (stage 2), and the strain's term -- its WBStrain node where the spec has +% one (`node`), else its name alone -- so it reads on its own. +% +% The kind of each subject is NOT an assertion: it is subject.type +% (sessionDocuments). Terms with no ontology node yet are plain names +% (decision #55: an ontology lookup of all of them is to come). +% +% Options: +% 'Strains' the spec's `strains` entries (struct array or cell): +% key, name, species {node, name}, and optionally +% node, biological_sex {node, name} +% 'StrainIds' containers.Map, strain key -> document id (stage 2) +% +% OUT fields: documents (cell of structs), counts (struct), skipped +% (cellstr: an assertion not made, and why). + +arguments + session table + S table + subjectIds + options.Strains = struct([]) + options.StrainIds = containers.Map() +end +if height(session) ~= 1 + error('ndi:setup:conv:haley:oneSession', 'Give exactly one session row.'); +end +sid = char(session.session_id{1}); +ref = char(session.local_identifier{1}); +S = S(strcmp(S.session, ref), :); + +strains = containers.Map(); +list = options.Strains; +if isstruct(list), list = num2cell(list); end +for k = 1:numel(list) + strains(list{k}.key) = list{k}; +end +eachMember = struct(); +if ndi.setup.V2.schemaHasField('statement', 'distributive') + eachMember.distributive = true; +end + +out = struct('documents', {{}}, 'skipped', {{}}, 'counts', struct( ... + 'cohort_species', 0, 'cohort_strain', 0, 'cohort_sex', 0, 'patch_species', 0, ... + 'patch_strain', 0, 'patch_sex', 0, 'plate_excluded', 0, 'plate_not_filmed', 0)); +% the reason an excluded plate carries: the day's own note, where the day is +% marked `include = no` (decision #73) +dayReason = ''; +if ismember('include', session.Properties.VariableNames) && ~session.include(1) ... + && ismember('notes', session.Properties.VariableNames) + dayReason = char(session.notes{1}); +end +hasFilmed = ismember('filmed', S.Properties.VariableNames); +% a statement's notes (did-schema 2026-10-14: moved up from interaction) +statementHasNotes = ndi.setup.V2.schemaHasField('statement', 'notes'); +schemaHasText = ndi.setup.V2.schemaHasClass('text'); +hasBacteria = ismember('bacteria', S.Properties.VariableNames); +for k = 1:height(S) + id = S.local_identifier{k}; + switch S.kind{k} + case 'cohort' + out = organismAssertions(out, id, S.strain{k}, 'cohort', eachMember); + case 'patch' + if hasBacteria && ~isempty(S.bacteria{k}) + out = organismAssertions(out, id, S.bacteria{k}, 'patch', struct()); + end + case 'assay_plate' + if S.exclude(k) + out = assert1(out, id, 'inclusion in analysis', ... + did2.build.term('', 'excluded'), struct(), struct(), 'plate_excluded', dayReason); + end + if hasFilmed && ~S.filmed(k) + out = assert1(out, id, 'behavior recording', ... + did2.build.term('NCIT:C141477', 'Not Performed'), struct(), struct(), ... + 'plate_not_filmed', 'Prepared but not filmed: no video.'); + end + end +end + + function out = organismAssertions(out, id, strainKey, who, fields) + if ~isKey(strains, strainKey) + out.skipped{end+1} = sprintf('%s: strain %s is not in the spec; no species or strain', ... + id, strainKey); + return; + end + st = strains(strainKey); + out = assert1(out, id, 'species', did2.build.term(st.species.node, st.species.name), ... + struct(), fields, [who '_species']); + if ndi.setup.V2.mergedEntities() + % one entity class (2026-10-08): the subject is `instance_of` its strain + % (V_eta_entity_composition_plan.md sec. 5); there is no strain assertion + if ~isKey(options.StrainIds, strainKey) + out.skipped{end+1} = sprintf('%s: no strain entity for %s; no strain recorded', ... + id, strainKey); + elseif ~isKey(subjectIds, id) + out.skipped{end+1} = sprintf('%s strain: not a subject of this session', id); + else + out.documents{end+1} = did2.build.directedRelation(subjectIds(id), ... + options.StrainIds(strainKey), did2.build.term('', 'instance_of'), ... + 'Fields', fields, 'SessionId', sid); + out.counts.([who '_strain']) = out.counts.([who '_strain']) + 1; + end + sexOf(st); + return; + end + edges = struct(); + if isKey(options.StrainIds, strainKey) + edges.strain_id = options.StrainIds(strainKey); + else + out.skipped{end+1} = sprintf('%s: no strain document for %s; strain asserted by name only', ... + id, strainKey); + end + node = ''; + if isfield(st, 'node') && ~isempty(st.node), node = char(st.node); end + out = assert1(out, id, 'strain', did2.build.term(node, st.name), edges, fields, ... + [who '_strain']); + sexOf(st); + + function sexOf(st) + if isfield(st, 'biological_sex') && ~isempty(st.biological_sex) + sx = st.biological_sex; + out = assert1(out, id, did2.build.term('PATO:0000047', 'biological sex'), ... + did2.build.term(char(sx.node), char(sx.name)), struct(), fields, [who '_sex']); + end + end + end + + function out = assert1(out, id, variable, value, edges, fields, counter, notes) + if nargin < 8 + notes = ''; + end + if ~isKey(subjectIds, id) + label = variable; + if isstruct(label), label = label.name; end + out.skipped{end+1} = sprintf('%s %s: not a subject of this session', id, label); + return; + end + noteArgs = {}; + if ~isempty(notes) && statementHasNotes + noteArgs = {'Notes', char(notes)}; + end + out.documents{end+1} = did2.build.statement('term_assertion', subjectIds(id), ... + variable, value, 'Edges', edges, 'Fields', fields, 'SessionId', sid, noteArgs{:}); + out.counts.(counter) = out.counts.(counter) + 1; + if ~isempty(notes) && ~statementHasNotes && ~schemaHasText + % neither statement notes nor a text value kind: nowhere to put it + if isstruct(variable), variable = variable.name; end + out.skipped{end+1} = sprintf(['%s %s: the reason ("%s") is not recorded -- the ' ... + 'schema has neither statement notes nor a text value'], id, variable, char(notes)); + elseif ~isempty(notes) && ~statementHasNotes + % a schema whose statements have no `notes` (did-schema before + % 2026-10-14, when notes was on interaction only): the reason is + % its own text statement beside it (decision #73) + if isstruct(variable), variable = variable.name; end + out.documents{end+1} = did2.build.statement('text_assertion', subjectIds(id), ... + did2.build.term('', ['reason for ' variable]), ... + did2.build.valueCell('text', {char(notes)}), 'SessionId', sid); + end + end +end diff --git a/src/ndi/+ndi/+setup/+conv/+haley/densityDocuments.m b/src/ndi/+ndi/+setup/+conv/+haley/densityDocuments.m new file mode 100644 index 0000000000..eac201b1d4 --- /dev/null +++ b/src/ndi/+ndi/+setup/+conv/+haley/densityDocuments.m @@ -0,0 +1,454 @@ +function out = densityDocuments(dataParentDir, sessions, built, options) +%DENSITYDOCUMENTS Stage 12 (Haley): the border amplitude fits, across studies. +% +% OUT = ndi.setup.conv.haley.densityDocuments(DATAPARENTDIR, SESSIONS, BUILT, +% ...) builds, from the sessions built in this run (SESSIONS, the session +% table with session_id; BUILT, each session's stage outputs), the documents +% that carry the E. coli fits over to the C. elegans patches. Decision #65. +% +% THE FITS are the lab's (Haley et al. 2024, analysis/plotForagingGFP.m): +% per E. coli condition (peptone, lawn volume, growth-time condition, +% OD600), a straight line through border amplitude / exposure time +% (a.u. per ms) against growth time (minutes), over the included patches; +% OD600 0.05 and 0.1, which were not imaged, from ONE joint fit per lawn +% volume, f = a*OD600 + b*t, over OD600 0.5-2. The numbers are the published +% ones, ecoli/borderAmplitude.csv (option 'FitsFile'); each is refitted +% here from the per-patch values under the same rules, and a fit that does +% not reproduce is reported (OUT.checks.refitDisagrees). With no CSV the +% refits are stored, and OUT.skipped says so. +% +% Included patches, as plotForagingGFP excludes: image +% 20231230_101739.czi; plate 64; 0.5 uL patches with circularity < 0.75; a +% border / centre ratio < 1 or > 35; then, per image, the lowest xPeak until +% the image's xPeak range is at most 0.1 mm. +% +% Dataset-level documents (OUT.datasetDocuments; written with stage 2's at +% stage 13): +% subject per fit, a GROUP of E. coli patches (type `group`): +% the patches of that condition (joint fits: of OD600 +% 0.5-2), each `member_of` it +% directed_relation patch member_of group +% model_fit_calculation the group's `border amplitude growth`: the line, +% its coefficients and goodness (r2, sse, sse per +% point); conditions the seeding condition; inputs the +% INCLUDED patches' `patch border amplitude` statements +% (stage 10 part C), so the exclusions are the values +% left out; method `linear least squares`, method +% parameters the exclusion rules; time the span of the +% included images +% absolute_time_reference that span +% Per C. elegans worm with encounters (OUT.sessionDocuments{k}, keyed by the +% worm's encounter list like stage 11's values): +% intensity_calculation `estimated patch border amplitude`: encounter.mat +% borderAmplitude (a.u. per ms of exposure), from the +% fit for the patch's condition at its growth time; +% 0 for a patch of LB alone; NaN where no fit matches +% intensity_calculation `estimated cultivation plate border amplitude`: +% borderAmplitudeGrowth, from the 200 uL fit at the +% cultivation plate's growth time +% Each value is checked against the fit it names (OUT.checks +% .encounterDisagrees). Relative density (modelExploit's 10 x / the OD 10 +% mean, floored at 0.01, log10) is not stored: it is computed in the model +% script, not in any source file. +% +% Options: 'DatasetSessionId', 'SoftwareId', 'InterpreterId', +% 'OperatingSystemId' (the run environment, as stage 10), 'FitsFile'. + +arguments + dataParentDir (1,:) char {mustBeFolder} + sessions table + built cell + options.DatasetSessionId (1,:) char = '' + options.SoftwareId (1,:) char = '' + options.InterpreterId (1,:) char = '' + options.OperatingSystemId (1,:) char = '' + options.FitsFile (1,:) char = '' +end +dsid = options.DatasetSessionId; +out = struct('datasetDocuments', {{}}, 'sessionDocuments', {repmat({{}}, height(sessions), 1)}, ... + 'skipped', {{}}, 'checks', struct('refitDisagrees', {{}}, 'encounterDisagrees', {{}}), ... + 'counts', struct('fits', 0, 'groups', 0, 'members', 0, 'inputs', 0, 'refit_checked', 0, ... + 'worms', 0, 'estimates', 0, 'encounters_checked', 0), 'fits', table()); +if isempty(options.OperatingSystemId) + out.skipped{end+1} = 'no operating system document (built by the metadata stage); no fits'; + return; +end +env = {}; +if ~isempty(options.SoftwareId), env = [env, {'SoftwareId', options.SoftwareId}]; end +if ~isempty(options.InterpreterId), env = [env, {'InterpreterId', options.InterpreterId}]; end +env = [env, {'OperatingSystemId', options.OperatingSystemId}]; +tz = 'America/Los_Angeles'; +fmt = 'yyyy-MM-dd''T''HH:mm:ss'; +root = fullfile(dataParentDir, 'haley', 'ecoli'); + +% ---- the E. coli values this run built --------------------------------------- +values = struct('imageNum', {}, 'row', {}, 'patchId', {}, 'statementId', {}); +for k = 1:numel(built) + if ~isempty(built{k}) && isfield(built{k}, 'ecoli') + values = [values, built{k}.ecoli.borderValues]; %#ok + end +end +if isempty(values) + out.skipped{end+1} = ['no E. coli patch values in this run (the E. coli session with ' ... + 'the calculations stage); no fits']; + return; +end +byRow = containers.Map(arrayfun(@(v) sprintf('%d|%d', v.imageNum, v.row), values, ... + 'UniformOutput', false), num2cell(1:numel(values))); + +B = load(fullfile(root, 'bacteria.mat'), 'lawnAnalysis'); +L = B.lawnAnalysis; +need = {'imageNum', 'plateNum', 'fileName', 'exposureTime', 'peptone', 'OD600', 'lawnVolume', ... + 'growthTimeCondition', 'growthTimeTotal', 'borderAmplitude', 'borderCenterRatio', ... + 'circularity', 'xPeak', 'acquisitionTime'}; +missing = setdiff(need, L.Properties.VariableNames); +if ~isempty(missing) + out.skipped{end+1} = sprintf('bacteria.mat lawnAnalysis has no %s; no fits', strjoin(missing, ', ')); + return; +end +L.peptone = cellstr(L.peptone); +L.fileName = cellstr(L.fileName); +% each row's stored statement: its position among its image's rows +L.row = zeros(height(L), 1); +L.statementId = repmat({''}, height(L), 1); +L.patchId = repmat({''}, height(L), 1); +for n = unique(L.imageNum)' + idx = find(L.imageNum == n); + for j = 1:numel(idx) + L.row(idx(j)) = j; + key = sprintf('%d|%d', n, j); + if isKey(byRow, key) + v = values(byRow(key)); + L.statementId{idx(j)} = v.statementId; + L.patchId{idx(j)} = v.patchId; + end + end +end +t = growthMinutes(L.growthTimeTotal); +f = double(L.borderAmplitude) ./ double(L.exposureTime); +L.include = included(L); + +% ---- the published fits -------------------------------------------------------- +file = options.FitsFile; +if isempty(file), file = fullfile(root, 'borderAmplitude.csv'); end +F = table(); +if isfile(file) + F = readtable(file, 'TextType', 'char', 'Delimiter', ','); +else + out.skipped{end+1} = sprintf('no %s: the refitted lines are stored instead', file); +end +joint = @(T) strcmp(T.peptone, 'with') & T.growthTimeCondition == 0 & ismember(T.OD600, [0.05 0.1]); + +% the fits to make: one per CSV row, the joint rows as one fit per volume +fits = struct('kind', {}, 'peptone', {}, 'volume', {}, 'growth', {}, 'od', {}, 'sel', {}, ... + 'members', {}, 'coef', {}, 'goodness', {}, 'n', {}); +if height(F) > 0 + single = F(~joint(F), :); + for r = 1:height(single) + c = single(r, :); + m = strcmp(L.peptone, c.peptone{1}) & L.lawnVolume == c.lawnVolume & ... + L.growthTimeCondition == c.growthTimeCondition & L.OD600 == c.OD600; + fits(end+1) = struct('kind', 'single', 'peptone', c.peptone{1}, 'volume', c.lawnVolume, ... + 'growth', c.growthTimeCondition, 'od', c.OD600, 'sel', m & L.include, 'members', m, ... + 'coef', [c.slope, c.intercept], 'goodness', [c.rsquare, c.sse], 'n', c.dfe + 2); %#ok + end + J = F(joint(F), :); + for vol = unique(J.lawnVolume)' + c = J(J.lawnVolume == vol, :); + m = strcmp(L.peptone, 'with') & L.lawnVolume == vol & L.growthTimeCondition == 0 & ... + L.OD600 >= 0.5 & L.OD600 <= 2; + a = c.intercept ./ c.OD600; + fits(end+1) = struct('kind', 'joint', 'peptone', 'with', 'volume', vol, 'growth', 0, ... + 'od', [0.5 2], 'sel', m & L.include, 'members', m, 'coef', [a(1), c.slope(1)], ... + 'goodness', [c.rsquare(1), c.sse(1)], 'n', c.dfe(1) + 2); %#ok + end +else + G = findgroups(L.peptone, L.lawnVolume, L.growthTimeCondition, L.OD600); + for g = 1:max(G) + m = G == g; + r = find(m, 1); + fits(end+1) = struct('kind', 'single', 'peptone', L.peptone{r}, 'volume', L.lawnVolume(r), ... + 'growth', L.growthTimeCondition(r), 'od', L.OD600(r), 'sel', m & L.include, ... + 'members', m, 'coef', [NaN NaN], 'goodness', [NaN NaN], 'n', NaN); %#ok + end +end + +% ---- refit each, and build its group and its fit --------------------------------- +fitIds = cell(1, numel(fits)); +for i = 1:numel(fits) + q = fits(i); + sel = q.sel & ~isnan(f) & ~isnan(t); + re = [NaN NaN]; + if strcmp(q.kind, 'single') && sum(sel) >= 2 + re = ([t(sel), ones(sum(sel), 1)] \ f(sel))'; + elseif strcmp(q.kind, 'joint') && sum(sel) >= 2 + re = ([double(L.OD600(sel)), t(sel)] \ f(sel))'; + end + label = conditionText(q); + if height(F) == 0 + q.coef = re; + res = f(sel) - designOf(q, L, t, sel) * re'; + q.goodness = [1 - sum(res.^2) / sum((f(sel) - mean(f(sel))).^2), sum(res.^2)]; + q.n = sum(sel); + elseif any(isnan(re)) + out.checks.refitDisagrees{end+1} = sprintf('%s: %d included value(s), too few to refit', ... + label, sum(sel)); + else + out.counts.refit_checked = out.counts.refit_checked + 1; + if any(abs(re - q.coef) > 1e-6 * max(1, abs(q.coef))) + out.checks.refitDisagrees{end+1} = sprintf(['%s: published [%g %g], refitted ' ... + '[%g %g] over %d included value(s)'], label, q.coef, re, sum(sel)); + end + end + if sum(sel) ~= q.n && ~isnan(q.n) && height(F) > 0 + out.checks.refitDisagrees{end+1} = sprintf('%s: %d value(s) by the published dfe, %d included here', ... + label, q.n, sum(sel)); + end + + % its time: the included images' span (none: no fit, and no group) + times = L.acquisitionTime(q.sel); + times = times(~isnat(times)); + if isempty(times) + out.skipped{end+1} = sprintf('%s: no included image with an acquisition time; no fit', label); + fits(i) = q; + continue; + end + w = utcReference(wallClock(min(times), tz), [], fmt, wallClock(max(times), tz), [], fmt, ... + true, tz, dsid); + out.datasetDocuments{end+1} = w; + timeIds = {w.base.id}; + + % the group of patches + grp = ndi.setup.V2.entityDocuments('subject', groupFields(q, label), struct(), ... + 'SessionId', dsid, 'Type', 'group'); + grp = grp{1}; + out.datasetDocuments{end+1} = grp; + out.counts.groups = out.counts.groups + 1; + pats = unique(L.patchId(q.members & ~cellfun(@isempty, L.patchId))); + for p = 1:numel(pats) + out.datasetDocuments{end+1} = did2.build.directedRelation(pats{p}, grp.base.id, ... + 'member_of', 'SessionId', dsid); + out.counts.members = out.counts.members + 1; + end + lost = sum(q.sel & cellfun(@isempty, L.statementId)); + if lost > 0 + out.skipped{end+1} = sprintf(['%s: %d included value(s) are not stored (their image ' ... + 'was not matched to its patches), so not among the fit''s inputs'], label, lost); + end + + inputs = L.statementId(q.sel & ~cellfun(@isempty, L.statementId))'; + fit = did2.build.statement('model_fit_calculation', grp.base.id, ... + did2.build.term('', 'border amplitude growth'), fitValue(q), ... + 'Conditions', conditionsOf(q), 'Method', did2.build.term('', 'linear least squares'), ... + 'MethodParameters', rules(), 'InputIds', inputs, 'TimeReferenceIds', timeIds, ... + 'SessionId', dsid, env{:}, 'Notes', ['Border amplitude divided by exposure time ' ... + '(a.u. per ms) against growth time (min), as plotForagingGFP fits it.']); + out.datasetDocuments{end+1} = fit; + fitIds{i} = fit.base.id; + fits(i) = q; + out.counts.fits = out.counts.fits + 1; + out.counts.inputs = out.counts.inputs + numel(inputs); +end +out.fits = struct2table(rmfield(fits, {'sel', 'members'}), 'AsArray', true); +out.fits.id = fitIds(:); + +% ---- the C. elegans estimates --------------------------------------------------- +for k = 1:numel(built) + if isempty(built{k}) || ~isfield(built{k}, 'encounters') + continue; + end + sid = char(sessions.session_id{k}); + worms = built{k}.encounters.byWorm; + names = keys(worms); + docs = {}; + for w = 1:numel(names) + x = worms(names{w}); + E = x.rows; + need = {'lawnOD600', 'lawnVolume', 'growthCondition', 'peptone', 'lawnGrowth', ... + 'borderAmplitude'}; + if ~all(ismember(need, E.Properties.VariableNames)) || height(E) == 0 + continue; + end + [ids, pred] = patchFits(E, fits, fitIds); + check(names{w}, 'patch', double(E.borderAmplitude), pred); + in = reshape(unique(ids(~cellfun(@isempty, ids)), 'stable'), 1, []); + docs{end+1} = did2.build.statement('intensity_calculation', x.wormId, ... + did2.build.term('', 'estimated patch border amplitude'), ... + did2.build.valueCell('intensity', reshape(double(E.borderAmplitude), 1, [])), ... + 'Method', did2.build.term('', 'calibration from the E. coli border amplitude fits'), ... + x.keyed{:}, 'InputIds', [{x.listId}, in], x.calc{:}, 'Notes', ... + ['Per ms of exposure, from the fit for the patch''s seeding condition at its growth ' ... + 'time (encounter.mat lawnGrowth, min); 0 for a patch of LB alone; NaN where no fit ' ... + 'matches (labelEncounters).']); %#ok + out.counts.estimates = out.counts.estimates + 1; + if all(ismember({'growthLawnGrowth', 'borderAmplitudeGrowth'}, E.Properties.VariableNames)) + g = find(arrayfun(@(q) strcmp(q.kind, 'single') && q.volume == 200, fits), 1); + gIn = {}; + gPred = NaN(height(E), 1); + if ~isempty(g) + if ~isempty(fitIds{g}), gIn = fitIds(g); end + gPred = fits(g).coef(2) + fits(g).coef(1) * growthMinutes(E.growthLawnGrowth); + end + check(names{w}, 'cultivation plate', double(E.borderAmplitudeGrowth), gPred); + docs{end+1} = did2.build.statement('intensity_calculation', x.wormId, ... + did2.build.term('', 'estimated cultivation plate border amplitude'), ... + did2.build.valueCell('intensity', reshape(double(E.borderAmplitudeGrowth), 1, [])), ... + 'Method', did2.build.term('', 'calibration from the E. coli border amplitude fits'), ... + x.keyed{:}, 'InputIds', [{x.listId}, gIn], x.calc{:}, 'Notes', ... + ['Per ms of exposure, from the 200 uL fit at the cultivation plate''s growth time ' ... + '(encounter.mat growthLawnGrowth).']); %#ok + out.counts.estimates = out.counts.estimates + 1; + end + out.counts.worms = out.counts.worms + 1; + end + out.sessionDocuments{k} = docs; +end + + function check(worm, what, got, want) + ok = isnan(got) & isnan(want) | abs(got - want) <= 1e-6 * max(1, abs(want)); + out.counts.encounters_checked = out.counts.encounters_checked + numel(got); + if ~all(ok) + out.checks.encounterDisagrees{end+1} = sprintf(['%s: %d of %d %s value(s) differ ' ... + 'from the fit (first: %g, fit %g)'], worm, sum(~ok), numel(ok), what, ... + got(find(~ok, 1)), want(find(~ok, 1))); + end + end +end + +% ----------------------------------------------------------------------------- +function inc = included(L) +% plotForagingGFP's exclusions, in its order +inc = ~strcmp(L.fileName, '20231230_101739.czi') & L.plateNum ~= 64; +inc = inc & ~((L.circularity < 0.75 & L.lawnVolume == 0.5) | ... + L.borderCenterRatio < 1 | L.borderCenterRatio > 35); +for n = unique(L.imageNum)' + while true + idx = find(L.imageNum == n & inc); + if isempty(idx) || max(L.xPeak(idx)) - min(L.xPeak(idx)) <= 0.1 + break; + end + [~, worst] = min(L.xPeak(idx)); + inc(idx(worst)) = false; + end +end +end + +function m = growthMinutes(x) +if isduration(x) + m = minutes(x); +else + m = double(x); +end +m = m(:); +end + +function X = designOf(q, L, t, sel) +if strcmp(q.kind, 'joint') + X = [double(L.OD600(sel)), t(sel)]; +else + X = [t(sel), ones(sum(sel), 1)]; +end +end + +function [ids, pred] = patchFits(E, fits, fitIds) +% the fit each encounter's patch takes (labelEncounters' match) and its value +n = height(E); +ids = repmat({''}, n, 1); +pred = NaN(n, 1); +pep = cellstr(E.peptone); +growth = double(E.growthCondition); +growth(growth == 60) = 48; % labelEncounters: foragingMini's 60 is 48 +lg = growthMinutes(E.lawnGrowth); +for i = 1:n + od = double(E.lawnOD600(i)); + if isnan(od) + continue; + end + if od <= 1e-10 + pred(i) = 0; % a patch of LB alone + continue; + end + for j = 1:numel(fits) + q = fits(j); + if ~strcmp(q.peptone, pep{i}) || q.volume ~= double(E.lawnVolume(i)) || q.growth ~= growth(i) + continue; + end + if strcmp(q.kind, 'single') && q.od == od + ids{i} = fitIds{j}; + pred(i) = q.coef(2) + q.coef(1) * lg(i); + elseif strcmp(q.kind, 'joint') && ismember(od, [0.05 0.1]) + ids{i} = fitIds{j}; + pred(i) = q.coef(1) * od + q.coef(2) * lg(i); + else + continue; + end + break; + end +end +end + +function s = conditionText(q) +pep = 'with peptone'; +if ~strcmp(q.peptone, 'with'), pep = 'without peptone'; end +if strcmp(q.kind, 'joint') + s = sprintf('E. coli patches, %s, %g uL, %g h, OD600 0.5-2', pep, q.volume, q.growth); +else + s = sprintf('E. coli patches, %s, %g uL, %g h, OD600 %g', pep, q.volume, q.growth, q.od); +end +end + +function f = groupFields(q, label) +id = regexprep(lower(label), '[^a-z0-9.]+', '_'); +f = struct('local_identifier', ['ecoli_fit_group_' regexprep(id, '^e._coli_patches_', '')], ... + 'name', label, 'description', ['The E. coli patches seeded this way, whose border ' ... + 'amplitude the lab fitted as one line (decision #65).']); +if ndi.setup.V2.schemaHasField('subject', 'type') + f.type = did2.build.term('', 'group'); +end +end + +function c = conditionsOf(q) +pep = 'with peptone'; +if ~strcmp(q.peptone, 'with'), pep = 'without peptone'; end +parts = {did2.build.condition('peptone', 'Term', did2.build.term('', pep)), ... + did2.build.condition('seeding volume', 'Quantity', q.volume * 1e-6, 'Unit', 'liter', ... + 'SourceValue', q.volume, 'SourceUnit', 'uL'), ... + did2.build.condition('growth time condition', 'Quantity', q.growth * 3600, 'Unit', 'second', ... + 'SourceValue', q.growth, 'SourceUnit', 'h')}; +if strcmp(q.kind, 'single') + parts{end+1} = did2.build.condition('OD600', 'Quantity', q.od); +end +c = did2.build.list(parts{:}); +end + +function v = fitValue(q) +t = struct('variable', did2.build.term('', 'growth time'), 'unit', did2.build.term('', 'minute')); +y = struct('variable', did2.build.term('', 'border amplitude per exposure time'), ... + 'unit', did2.build.term('', 'arbitrary unit per millisecond')); +if strcmp(q.kind, 'joint') + eq = 'f = a*OD600 + b*t'; + x = did2.build.list(struct('variable', did2.build.term('', 'OD600')), t); + co = did2.build.list(struct('variable', did2.build.term('', 'a'), 'value', q.coef(1)), ... + struct('variable', did2.build.term('', 'b'), 'value', q.coef(2))); +else + eq = 'f = slope*t + intercept'; + x = did2.build.list(t); + co = did2.build.list(struct('variable', did2.build.term('', 'slope'), 'value', q.coef(1)), ... + struct('variable', did2.build.term('', 'intercept'), 'value', q.coef(2))); +end +v = struct('model', did2.build.term('', 'linear'), 'equation', eq, ... + 'independent_variables', x, 'dependent_variable', y, 'coefficients', co, ... + 'goodness', struct('r2', q.goodness(1), 'sse', q.goodness(2), ... + 'sse_per_point', q.goodness(2) / q.n)); +end + +function p = rules() +p = did2.build.list( ... + did2.build.parameter('excluded image', 'Text', '20231230_101739.czi'), ... + did2.build.parameter('excluded plate', 'Text', 'E. coli plate 64'), ... + did2.build.parameter('minimum circularity of a 0.5 uL patch', 'Value', 0.75), ... + did2.build.parameter('minimum border to centre ratio', 'Value', 1), ... + did2.build.parameter('maximum border to centre ratio', 'Value', 35), ... + did2.build.parameter('maximum spread of the intensity peak distance within an image', ... + 'Value', 1e-4, 'Unit', 'meter', 'SourceValue', '0.1', 'SourceUnit', 'mm')); +end diff --git a/src/ndi/+ndi/+setup/+conv/+haley/ecoliDocuments.m b/src/ndi/+ndi/+setup/+conv/+haley/ecoliDocuments.m new file mode 100644 index 0000000000..6b94b0a672 --- /dev/null +++ b/src/ndi/+ndi/+setup/+conv/+haley/ecoliDocuments.m @@ -0,0 +1,470 @@ +function out = ecoliDocuments(dataParentDir, session, R, subjectIds, options) +%ECOLIDOCUMENTS Stage 10 part C (Haley): the E. coli patch profiles. +% +% OUT = ndi.setup.conv.haley.ecoliDocuments(DATAPARENTDIR, SESSION, R, +% SUBJECTIDS, ...) builds, for the E. coli session SESSION, the calculations +% the lab's analysis package (analyzeGFP / analyzeLawnProfiles, Haley et al. +% 2024) made from each analysed fluorescence image: ecoli/bacteria.mat +% `lawnAnalysis` (one row per patch detected in an image) and the image's +% patch mask, ecoli/mask/.png. Decision log #62; the images and +% profile curves, #64. +% +% WHAT THE TIFFS ARE (decision #64). ecoli/images/.tiff is NOT the image +% as acquired: it is analyzeLawnProfiles' `imageNormalized` rounded to +% uint16 -- the raw image (saturated pixels filled by linear +% interpolation) times max(B) divided by B, B the empty-plate background +% image smoothed by getFluorescenceBackground (outliers against a 400 px +% moving median replaced, then a 20 px average filter). Checked exactly +% on three images (Jess, 2026-10-05). The raw images, and the background +% images, are ecoli/raw/.tiff (the recordings, recordingList); every +% calculation below takes the NORMALISED image as its input, the pixels +% the analysis actually used. With no raw/ folder the recordings are the +% normalised TIFFs themselves and no normalised-image calculation is made. +% +% WHICH PATCH IS WHICH. An image's lawnAnalysis rows are its detected +% patches in the order bwlabel numbers its mask (checked over all 1,521 +% analysed images: region count = row count in every one; Jess, +% 2026-10-05; numbered here the same way without the Image Processing +% Toolbox, private/labelRegions). They are matched to the plate's patch +% SUBJECTS by position: +% `rectangle` plate (12 patches, 3 staggered rows of 4) the 12 detected +% patches are numbered as bwlabel numbers an ideal grid -- left to +% right, patches sharing a column top to bottom -- and detected patch +% number k is patch subject k +% one-patch plate its one detected patch is the one patch subject +% fewer than 12 detected on a `rectangle` plate each goes to the +% nearest of the 12 patch positions of the acetate template +% registered on the image's brightfield image (metaData +% brightfieldImageNum -> the extract's `template`; the positions are +% the spec's `ecoli_patch_layout`; ndi.setup.conv.haley. +% matchTemplatePatches, decision #80), when every patch is within +% its max_distance_mm and no two claim one position +% An image matched no way is reported (OUT.skipped, with why) and its +% per-patch values are left out; its mask is still stored. +% +% Per image with an ecoli/images TIFF (when raw/ exists): +% intensity_calculation the plate's `background-normalised fluorescence +% image`: the TIFF, by location (opaque_body, not +% ingested), from the raw image's recording and its +% background image's recording (inputs); method +% parameters the 400 px outlier window and 20 px +% smoothing window +% Per background image used (once): its plate's `background fit r-squared` +% (score_calculation) and `background outlier pixels` (count_calculation), +% from analyzeGFP's backgroundInfo. +% +% Per analysed image, all from the normalised image (input): +% coordinate_system the image's pixel grid, as part A's (only when +% bacteria.mat metaData gives the image's `scale`) +% label_calculation the plate's `bacterial lawn region`: the mask, a +% bool per pixel (ingested body), method parameter the +% 0.5 mm margin the analysis adds around each patch +% item_calculation the plate's `nearest patch` (ecoli/closest/ +% .png), a 0-based index into the patch subjects -- +% only for a matched image, and only when the schema +% has `item` (did-schema PR #87) +% and per matched detected patch, about its patch subject, from the image +% and the mask: +% intensity_calculation `patch border amplitude` (borderAmplitude, the +% basis of the paper's relative density), `patch +% centre amplitude` (centerAmplitude), `patch peak +% intensity` (yPeak), `patch edge intensity` +% (yOuterEdge); background-normalised intensity +% length_calculation `patch edge to intensity peak` (xPeak, mm -> m) +% score_calculation `patch circularity` (0-1) +% intensity_calculation `intensity by distance from patch edge`: the +% patch's profile curve (analyzeGFP lawnProfiles +% .pixelValuesNormalized, its row for this patch) +% in an ingested float64 body keyed by `distance +% from patch edge` (regular, 1 um steps, negative +% outside the patch; NaN where the patch has no +% pixels at that distance) +% Not stored: borderCenterRatio (border / centre amplitude), meanAmplitude, +% FWHM, xHalfMax*, yHalfMax, lawnRadius (decision #60's keep list): each +% is computed from the stored curve. The fitted background images are not +% stored (decision #64): each is reproducible from its raw image. +% +% Options: 'RecordingStatements', 'RecordingRefs', 'SoftwareId', +% 'InterpreterId', 'OperatingSystemId' as part A. 'ProfilesFile': the +% analyzeGFP workspace holding lawnProfiles and backgroundInfo, default +% ecoli/analyzeGFP_24-04-04.mat; read without loading its images +% (private/lawnProfileReader); its `template` (extractTemplates), when +% present, places partly detected images. 'PatchLayout': the spec's +% `ecoli_patch_layout` (decision #80). Without it there are no curves and no +% background fit values, and OUT.skipped says so. +% +% OUT fields: documents, counts (struct), skipped (cellstr), borderValues +% (struct array, one per stored `patch border amplitude`: imageNum, row -- +% its position among the image's lawnAnalysis rows --, patchId, +% statementId; for stage 12's fits, decision #65). + +arguments + dataParentDir (1,:) char {mustBeFolder} + session table + R table + subjectIds + options.RecordingStatements = containers.Map() + options.RecordingRefs = containers.Map() + options.SoftwareId (1,:) char = '' + options.InterpreterId (1,:) char = '' + options.OperatingSystemId (1,:) char = '' + options.ProfilesFile (1,:) char = '' + options.PatchLayout = struct([]) +end +if height(session) ~= 1 + error('ndi:setup:conv:haley:oneSession', 'Give exactly one session row.'); +end +sid = char(session.session_id{1}); +ref = char(session.local_identifier{1}); +out = struct('documents', {{}}, 'skipped', {{}}, 'borderValues', struct('imageNum', {}, 'row', {}, ... + 'patchId', {}, 'statementId', {}), 'counts', struct('coordinate_system', 0, ... + 'mask', 0, 'nearest_patch', 0, 'matched_images', 0, 'template_matched_images', 0, 'unmatched_images', 0, ... + 'patch_values', 0, 'normalised_image', 0, 'profiles', 0, 'background_fit', 0)); +if ~strcmp(char(session.folder{1}), 'ecoli') + return; +end +if isempty(options.OperatingSystemId) + out.skipped{end+1} = sprintf(['%s: no operating system document (the spec''s `macos`, ' ... + 'built by the metadata stage); no calculations'], ref); + return; +end +env = {}; +if ~isempty(options.SoftwareId), env = [env, {'SoftwareId', options.SoftwareId}]; end +if ~isempty(options.InterpreterId), env = [env, {'InterpreterId', options.InterpreterId}]; end +env = [env, {'OperatingSystemId', options.OperatingSystemId}]; +hasItem = ndi.setup.V2.schemaHasField('item', 'value'); + +root = fullfile(dataParentDir, 'haley', 'ecoli'); +B = load(fullfile(root, 'bacteria.mat')); +if ~isfield(B, 'lawnAnalysis') + out.skipped{end+1} = sprintf('%s: bacteria.mat has no lawnAnalysis', ref); + return; +end +L = B.lawnAnalysis; +M = table(); +if isfield(B, 'metaData'), M = B.metaData; end +R = R(strcmp(R.session, ref) & strcmp(R.kind, 'image'), :); +docs = {}; +rawMode = isfolder(fullfile(root, 'raw')); +P = []; +BI = table(); +TPL = struct([]); +pf = options.ProfilesFile; +if isempty(pf), pf = fullfile(root, 'analyzeGFP_24-04-04.mat'); end +if isfile(pf) + P = lawnProfileReader(pf); + w = whos('-file', pf); + if any(strcmp({w.name}, 'backgroundInfo')) + S = load(pf, 'backgroundInfo'); + BI = S.backgroundInfo; + end + if any(strcmp({w.name}, 'template')) + S = load(pf, 'template'); + TPL = S.template; % acetate registrations (extractTemplates) + end +else + out.skipped{end+1} = sprintf('%s: no %s; no profile curves and no background fit values', ref, pf); +end +bgDone = containers.Map(); +bgParams = did2.build.list( ... + did2.build.parameter('background outlier window', 'Value', 400, 'Unit', 'pixel'), ... + did2.build.parameter('background smoothing window', 'Value', 20, 'Unit', 'pixel')); + +for k = 1:height(R) + epoch = R.epoch{k}; + n = str2double(regexp(epoch, '(\d+)$', 'tokens', 'once')); + if ~isKey(options.RecordingStatements, epoch) + continue; % not written + end + plate = R.plate{k}; + rows = L(L.imageNum == n, :); + normFile = fullfile(root, 'images', sprintf('%04d.tiff', n)); + if ~isKey(subjectIds, plate) + if height(rows) > 0 || (rawMode && isfile(normFile)) + out.skipped{end+1} = sprintf('%s: plate %s is not a subject of this session', epoch, plate); + end + continue; + end + imageId = options.RecordingStatements(epoch); + timeIds = {}; + if isKey(options.RecordingRefs, epoch), timeIds = {options.RecordingRefs(epoch)}; end + calc = [{'SessionId', sid, 'TimeReferenceIds', timeIds}, env]; + + % the image's background image, when it is a recording of this session + bgEpoch = ''; + if height(M) > 0 && ismember('backgroundImageNum', M.Properties.VariableNames) + bg = M.backgroundImageNum(M.imageNum == n); + if ~isempty(bg) && ~isnan(bg(1)) && isKey(options.RecordingStatements, sprintf('ecoli_image%04d', bg(1))) + bgEpoch = sprintf('ecoli_image%04d', bg(1)); + end + end + + % the background-normalised image (decision #64): the TIFF in images/ + srcId = imageId; + if rawMode && isfile(normFile) + inputs = {imageId}; + if ~isempty(bgEpoch), inputs{end+1} = options.RecordingStatements(bgEpoch); end %#ok + nm = did2.build.statement('intensity_calculation', subjectIds(plate), ... + did2.build.term('', 'background-normalised fluorescence image'), [], ... + 'DataBody', true, 'DatumType', 'uint16', ... + 'Method', did2.build.term('', 'normalisation to the background illumination'), ... + 'MethodParameters', bgParams, 'InputIds', inputs, calc{:}, 'Notes', ... + ['The raw image (saturated pixels filled by linear interpolation) times the ' ... + 'maximum of its smoothed background image, divided by that smoothed image, ' ... + 'rounded to uint16 (analyzeLawnProfiles imageNormalized).']); + info = dir(normFile); + nb = did2.build.document('opaque_body', struct('format', fileFormat('EDAM:format_3591', 'TIFF', 'image/tiff'), ... + 'filename', sprintf('%04d.tiff', n), 'size_bytes', info.bytes, ... + 'description', 'The background-normalised image, from ecoli/images.'), ... + 'SessionId', sid, 'Edges', struct('owner_id', nm.base.id), 'Files', {'body_data_0'}); + nb.files.file_info = struct('name', 'body_data_0', 'locations', struct( ... + 'delete_original', 0, 'uid', ndi.ido.unique_id(), 'location', normFile, ... + 'parameters', '', 'location_type', 'file', 'ingest', 0)); + docs = [docs, {nm, nb}]; %#ok + srcId = nm.base.id; + out.counts.normalised_image = out.counts.normalised_image + 1; + end + + % the background fit's quality, once per background image + if rawMode && ~isempty(bgEpoch) && ~isKey(bgDone, bgEpoch) && height(BI) > 0 + bgDone(bgEpoch) = true; + i = find(BI.imageNum == bg(1), 1); + bp = R.plate(strcmp(R.epoch, bgEpoch)); + if ~isempty(i) && ~isempty(bp) && isKey(subjectIds, bp{1}) + bgTimes = {}; + if isKey(options.RecordingRefs, bgEpoch), bgTimes = {options.RecordingRefs(bgEpoch)}; end + bc = [{'SessionId', sid, 'TimeReferenceIds', bgTimes}, env, ... + {'InputIds', {options.RecordingStatements(bgEpoch)}, ... + 'Method', did2.build.term('', 'background illumination smoothing'), ... + 'MethodParameters', bgParams}]; + docs{end+1} = did2.build.statement('score_calculation', subjectIds(bp{1}), ... + did2.build.term('', 'background fit r-squared'), ... + did2.build.valueCell('score', double(BI.rsquare(i)), 'Fields', struct( ... + 'scale', did2.build.term('', 'r-squared'), 'scale_min', 0, 'scale_max', 1)), ... + bc{:}); %#ok + docs{end+1} = did2.build.statement('count_calculation', subjectIds(bp{1}), ... + did2.build.term('', 'background outlier pixels'), ... + did2.build.valueCell('count', double(BI.numOutliers(i))), bc{:}); %#ok + out.counts.background_fit = out.counts.background_fit + 1; + end + end + + if height(rows) == 0 + continue; % not analysed + end + maskFile = fullfile(root, 'mask', sprintf('%04d.png', n)); + if ~isfile(maskFile) + out.skipped{end+1} = sprintf('%s: no mask/%04d.png', epoch, n); + continue; + end + + % the image's scale, when the source gives it + scale = NaN; + if height(M) > 0 && ismember('scale', M.Properties.VariableNames) + s = M.scale(M.imageNum == n); + if ~isempty(s), scale = double(s(1)); end + end + pixel = NaN; + if scale > 0 + pixel = 1e-3 / scale; + step = did2.build.valueCell('length', pixel, 'SourceValue', 1 / scale, ... + 'SourceUnit', 'mm', 'Approximate', true); + cs = did2.build.document('coordinate_system', struct( ... + 'origin', did2.build.term('', 'image upper-left corner'), ... + 'dimensions', did2.build.list( ... + struct('axis', did2.build.term('', 'image horizontal axis'), ... + 'positive_direction', did2.build.term('', 'image right'), 'spacing', step), ... + struct('axis', did2.build.term('', 'image vertical axis'), ... + 'positive_direction', did2.build.term('', 'image down'), 'spacing', step))), ... + 'SessionId', sid, 'Edges', struct('referent_id', imageId)); + docs{end+1} = cs; %#ok + out.counts.coordinate_system = out.counts.coordinate_system + 1; + end + + % the mask + mask = logical(imread(maskFile)); + keys = pixelKeys(mask, pixel); + inputs = {srcId}; + if ~rawMode && ~isempty(bgEpoch) + inputs{end+1} = options.RecordingStatements(bgEpoch); %#ok + end + ms = did2.build.statement('label_calculation', subjectIds(plate), ... + did2.build.term('', 'bacterial lawn region'), [], 'DataBody', true, 'DatumType', 'bool', ... + 'Keys', keys, 'Method', did2.build.term('', 'patch detection by threshold'), ... + 'MethodParameters', did2.build.list(did2.build.parameter('margin around each patch', ... + 'Value', 0.5e-3, 'Unit', 'meter', 'SourceValue', '0.5', 'SourceUnit', 'mm')), ... + 'InputIds', inputs, calc{:}); + docs = [docs, {ms, ingestedBody(ms, mask, 'bool', keys, sid, ... + sprintf('The patch mask (1 = within 0.5 mm of a detected patch), from mask/%04d.png.', n))}]; %#ok + out.counts.mask = out.counts.mask + 1; + + % which detected patch is which patch subject + [~, nDetected, centroids] = labelRegions(mask); + if nDetected ~= height(rows) + out.skipped{end+1} = sprintf('%s: %d regions in the mask, %d lawnAnalysis rows; not matched', ... + epoch, nDetected, height(rows)); + out.counts.unmatched_images = out.counts.unmatched_images + 1; + continue; + end + pIds = patchIds(subjectIds, plate); + slot = matchFullImage(centroids, nDetected, numel(pIds)); + match = {'Notes', 'Detected patch matched to its grid patch by position (decision #62).'}; + why = ''; + if isempty(slot) && nDetected < 12 && numel(pIds) == 12 + % some patches not detected: place them by the acetate template + % registered on the image's brightfield image (decision #80) + [slot, why] = templateMatch(n, centroids); + if ~isempty(slot) + out.counts.template_matched_images = out.counts.template_matched_images + 1; + match = {'Notes', sprintf(['Detected patch placed at the nearest patch position of ' ... + 'the acetate template registered on this image''s brightfield image; %d of 12 ' ... + 'patches detected in this image (decision #80).'], nDetected)}; + end + end + if isempty(slot) + if isempty(why), why = sprintf('the plate has %d patch subject(s)', numel(pIds)); end + out.skipped{end+1} = sprintf('%s: %d detected patch(es) on %s; not matched: %s', ... + epoch, nDetected, plate, why); + out.counts.unmatched_images = out.counts.unmatched_images + 1; + continue; + end + out.counts.matched_images = out.counts.matched_images + 1; + + % the profile curves: one row per detected patch, in the same order + curves = []; + if ~isempty(P) + [x, Y] = lawnProfileRead(P, n); + if isempty(Y) + if isKey(P.byImage, n) + out.skipped{end+1} = sprintf('%s: lawnProfiles holds no curves for image %d', epoch, n); + end + elseif size(Y, 1) ~= nDetected || size(Y, 2) ~= numel(x) + out.skipped{end+1} = sprintf('%s: lawnProfiles has %d x %d curves for %d patches and %d distances; no curves', ... + epoch, size(Y, 1), size(Y, 2), nDetected, numel(x)); + elseif numel(x) < 2 || any(abs(diff(x) - 1e-3) > 1e-9) + out.skipped{end+1} = sprintf('%s: lawnProfiles distances are not in 1 um steps; no curves', epoch); + else + curves = struct('Y', Y, 'keys', did2.build.list(did2.build.key('distance from patch edge', ... + numel(x), 'Unit', 'meter', 'Origin', x(1) * 1e-3, 'Spacing', 1e-6, ... + 'SourceUnit', 'mm', 'SourceOrigin', x(1), 'SourceSpacing', 1e-3))); + end + end + + % the nearest-patch map + closestFile = fullfile(root, 'closest', sprintf('%04d.png', n)); + if hasItem && isfile(closestFile) + c = double(imread(closestFile)); + if all(c(:) >= 1 & c(:) <= nDetected) + v = slot(c) - 1; + dt = 'uint8'; + if numel(pIds) > 255, dt = 'uint16'; end + np = did2.build.statement('item_calculation', subjectIds(plate), ... + did2.build.term('', 'nearest patch'), [], 'DataBody', true, 'DatumType', dt, ... + 'Keys', keys, 'InputIds', {ms.base.id}, ... + 'Edges', struct('item_id', {pIds}), calc{:}, match{:}); + docs = [docs, {np, ingestedBody(np, v, dt, keys, sid, ... + 'Each pixel''s nearest patch: a 0-based index into item_id.')}]; %#ok + out.counts.nearest_patch = out.counts.nearest_patch + 1; + else + out.skipped{end+1} = sprintf('%s: closest/%04d.png holds values outside 1..%d; no map', ... + epoch, n, nDetected); + end + end + + % the per-patch values + in = {'InputIds', {srcId, ms.base.id}}; + for j = 1:nDetected + patch = pIds{slot(j)}; + r = rows(j, :); + for q = {{'borderAmplitude', 'patch border amplitude'}, ... + {'centerAmplitude', 'patch centre amplitude'}, ... + {'yPeak', 'patch peak intensity'}, ... + {'yOuterEdge', 'patch edge intensity'}} + col = q{1}{1}; + if ~ismember(col, r.Properties.VariableNames) || isnan(r.(col)), continue; end + v = did2.build.valueCell('intensity', double(r.(col))); + docs{end+1} = did2.build.statement('intensity_calculation', patch, ... + did2.build.term('', q{1}{2}), v, ... + 'Method', did2.build.term('', 'fluorescence profile from the patch edge'), ... + in{:}, calc{:}, match{:}); %#ok + out.counts.patch_values = out.counts.patch_values + 1; + if strcmp(col, 'borderAmplitude') + out.borderValues(end+1) = struct('imageNum', n, 'row', j, 'patchId', patch, ... + 'statementId', docs{end}.base.id); + end + end + if ismember('xPeak', r.Properties.VariableNames) && ~isnan(r.xPeak) + v = did2.build.valueCell('length', r.xPeak * 1e-3, 'SourceValue', r.xPeak, 'SourceUnit', 'mm'); + docs{end+1} = did2.build.statement('length_calculation', patch, ... + did2.build.term('', 'patch edge to intensity peak'), v, ... + 'Method', did2.build.term('', 'fluorescence profile from the patch edge'), ... + in{:}, calc{:}, match{:}); %#ok + out.counts.patch_values = out.counts.patch_values + 1; + end + if ismember('circularity', r.Properties.VariableNames) && ~isnan(r.circularity) + v = did2.build.valueCell('score', r.circularity, 'Fields', struct( ... + 'scale', did2.build.term('', 'circularity'), 'scale_min', 0, 'scale_max', 1)); + docs{end+1} = did2.build.statement('score_calculation', patch, ... + did2.build.term('', 'patch circularity'), v, in{:}, calc{:}, match{:}); %#ok + out.counts.patch_values = out.counts.patch_values + 1; + end + if ~isempty(curves) + pc = did2.build.statement('intensity_calculation', patch, ... + did2.build.term('', 'intensity by distance from patch edge'), [], ... + 'DataBody', true, 'DatumType', 'float64', 'Keys', curves.keys, ... + 'Method', did2.build.term('', 'fluorescence profile from the patch edge'), ... + 'MethodParameters', did2.build.list(did2.build.parameter('distance bin', ... + 'Value', 1e-6, 'Unit', 'meter', 'SourceValue', '0.001', 'SourceUnit', 'mm')), ... + in{:}, calc{:}, match{:}); + docs = [docs, {pc, ingestedBody(pc, curves.Y(j, :), 'float64', curves.keys, sid, ... + ['Mean background-normalised intensity at each distance from the patch edge ' ... + '(lawnProfiles pixelValuesNormalized); NaN where the patch has no pixels.'], 'NaN')}]; %#ok + out.counts.profiles = out.counts.profiles + 1; + end + end +end +out.documents = docs; + + function [slot, why] = templateMatch(imageNum, centroids) + slot = []; + if isempty(options.PatchLayout) + why = 'no ecoli_patch_layout in the spec'; + return; + end + if isempty(TPL) + why = 'the profiles file holds no template (run extractTemplates)'; + return; + end + r = find(M.imageNum == imageNum, 1); + if isempty(r) || ~ismember('brightfieldImageNum', M.Properties.VariableNames) + why = 'metaData names no brightfield image'; + return; + end + t = find(TPL.imageNum == M.brightfieldImageNum(r), 1); + if isempty(t) + why = sprintf('no template registered on brightfield image %g', M.brightfieldImageNum(r)); + return; + end + [slot, why] = ndi.setup.conv.haley.matchTemplatePatches(centroids, TPL.center(t, :), ... + TPL.angle(t), TPL.scale(t), options.PatchLayout); + end +end + +% ----------------------------------------------------------------------------- +function keys = pixelKeys(a, pixel) +% as part A: pixel centres from the upper-left corner, in metres when the +% scale is known, else in pixels +if isnan(pixel) + keys = did2.build.list( ... + did2.build.key('image vertical position', size(a, 1), 'Unit', 'pixel', ... + 'Origin', 0.5, 'Spacing', 1), ... + did2.build.key('image horizontal position', size(a, 2), 'Unit', 'pixel', ... + 'Origin', 0.5, 'Spacing', 1)); + return; +end +keys = did2.build.list( ... + did2.build.key('image vertical position', size(a, 1), 'Unit', 'meter', ... + 'Origin', pixel / 2, 'Spacing', pixel, 'SourceUnit', 'pixel', 'SourceOrigin', 0.5, 'SourceSpacing', 1), ... + did2.build.key('image horizontal position', size(a, 2), 'Unit', 'meter', ... + 'Origin', pixel / 2, 'Spacing', pixel, 'SourceUnit', 'pixel', 'SourceOrigin', 0.5, 'SourceSpacing', 1)); +end diff --git a/src/ndi/+ndi/+setup/+conv/+haley/encounterDocuments.m b/src/ndi/+ndi/+setup/+conv/+haley/encounterDocuments.m new file mode 100644 index 0000000000..93b1341246 --- /dev/null +++ b/src/ndi/+ndi/+setup/+conv/+haley/encounterDocuments.m @@ -0,0 +1,313 @@ +function out = encounterDocuments(dataParentDir, session, R, subjectIds, options) +%ENCOUNTERDOCUMENTS Stage 11 (Haley): each worm's patch encounters. +% +% OUT = ndi.setup.conv.haley.encounterDocuments(DATAPARENTDIR, SESSION, R, +% SUBJECTIDS, ...) builds, for the one-row C. elegans session SESSION, the +% encounters celegans/encounter.mat records for its worms: the visits each +% worm made to a patch, found and measured by the lab's analysis package +% (analyzeEncounters, labelEncounters; Haley et al. 2024). Decision log #63. +% +% ONE WORM ON ONE PLATE. analyzeEncounters finds a worm's encounters over its +% whole track on the plate, its videos joined in order, so `enter`/`exit` are +% row numbers in that joined track (an encounter can start in one video and +% end in the next). The documents are timed by the plate's recording window +% (stage 9's, else the video's own reference) and each time is in seconds +% from that window's start: the video's start plus (frame - 1) / frame rate. +% +% Per worm with encounters (V_eta_study_plan.md, "Repeated events", T2): +% time_calculation `encounter onset`: THE ENCOUNTER LIST, one entry per +% encounter (id >= 1). Every other per-encounter document +% takes its `encounter` key's positions from it +% (`key_id` / `positions_from`, T14) +% item_calculation `patch in contact`: each encounter's patch subject (the +% source's lawnID = its lawnCenters row, #38) and, as +% `offset`, when it ended (`exit`); when the schema has +% `item` +% velocity_calculation `median speed on patch` (velocityOn), `minimum +% speed on patch` (velocityOnMin), `peak speed before +% entry` (velocityBeforeEnter), `minimum speed after +% entry` (velocityAfterEnter), `median speed off patch +% after the encounter` (velocityOff), um/s -> m/s +% acceleration_calculation `deceleration on entry` (decelerate, +% um/s^2 -> m/s^2; a linear fit from -1.5 to +6.5 s) +% time_calculation `time to slow down` (timeSlowDown, s) +% length_calculation `furthest distance off patch during the encounter` +% (distanceOnMax), `furthest distance from the patch edge +% after the encounter` (distanceOffMax), mm -> m +% score_calculation `probability of exploitation` (exploitPosterior), +% `probability of sensing` (sensePosterior), 0-1 +% label_calculation `encounter type` (label: exploit, sample, searchOn, +% searchOff; `unlabelled` where the source left it blank) +% and, for the gap before the first encounter (the source's id 0 row), one +% `median speed off patch before the first encounter` and one `furthest +% distance from the patch edge before the first encounter`. +% Inputs: the worm's midpoint speed and patch-edge distance tracks (stage 10 +% part B) for each of the plate's videos; the per-encounter values also take +% the encounter list. +% Not stored: duration, timeEnter/timeExit, censorEnter/censorExit (from the +% times and the videos' lengths), the plate/strain/growth columns (stages +% 4-8), borderAmplitude* (stage 12). +% +% Options: +% 'Tracks' trackDocuments' OUT.byWorm +% 'PlateWindows' observationDocuments' OUT.windows +% 'RecordingRefs' containers.Map, epoch -> its UTC reference id +% 'SoftwareId', 'InterpreterId', 'OperatingSystemId' as stage 10 +% +% OUT fields: documents, counts (struct), skipped (cellstr), byWorm +% (containers.Map, worm local_identifier -> struct of wormId, listId, keyed +% and calc -- the options its per-encounter documents are built with -- and +% rows, its encounter.mat rows in list order; for stage 12, decision #65). + +arguments + dataParentDir (1,:) char {mustBeFolder} + session table + R table + subjectIds + options.Tracks = containers.Map() + options.PlateWindows = containers.Map() + options.RecordingRefs = containers.Map() + options.SoftwareId (1,:) char = '' + options.InterpreterId (1,:) char = '' + options.OperatingSystemId (1,:) char = '' +end +if height(session) ~= 1 + error('ndi:setup:conv:haley:oneSession', 'Give exactly one session row.'); +end +sid = char(session.session_id{1}); +ref = char(session.local_identifier{1}); +folder = char(session.folder{1}); +out = struct('documents', {{}}, 'skipped', {{}}, 'counts', struct('worms', 0, ... + 'encounters', 0, 'documents', 0), 'byWorm', containers.Map()); +file = fullfile(dataParentDir, 'haley', 'celegans', 'encounter.mat'); +if strcmp(folder, 'ecoli') || ~isfile(file) + return; +end +if ~ndi.setup.V2.schemaHasField('time', 'value') || ~ndi.setup.V2.schemaHasLeaf('time_calculation') + out.skipped{end+1} = sprintf(['%s: the schema in use has no time_calculation ' ... + '(did-schema PR #87); no encounters'], ref); + return; +end +if isempty(options.OperatingSystemId) + out.skipped{end+1} = sprintf(['%s: no operating system document (the spec''s `macos`, ' ... + 'built by the metadata stage); no encounters'], ref); + return; +end +env = {}; +if ~isempty(options.SoftwareId), env = [env, {'SoftwareId', options.SoftwareId}]; end +if ~isempty(options.InterpreterId), env = [env, {'InterpreterId', options.InterpreterId}]; end +env = [env, {'OperatingSystemId', options.OperatingSystemId}]; +hasItem = ndi.setup.V2.schemaHasField('item', 'value'); +byRef = keyByReference(); +tz = 'America/Los_Angeles'; +secondFmt = 'yyyy-MM-dd''T''HH:mm:ss'; + +pre = ndi.setup.conv.haley.idPrefix(folder); +root = fullfile(dataParentDir, 'haley', 'celegans', folder); +I = load(fullfile(root, 'experimentInfo.mat'), 'info'); +D = I.info; +D = D(D.expNum == session.experiment(1), :); +E = load(file, 'encounter'); +E = E.encounter; +E = E(strcmp(E.expName, folder) & E.expNum == session.experiment(1), :); +if height(E) == 0 + return; +end +if ~isfile(fullfile(root, 'midpoint.mat')) + out.skipped{end+1} = sprintf('%s: no midpoint.mat to place the encounters on; none', ref); + return; +end +T = trackTable(fullfile(root, 'midpoint.mat')); +T = T(T.expNum == session.experiment(1), :); +R = R(strcmp(R.session, ref) & strcmp(R.kind, 'behaviour'), :); +docs = {}; + +[G, plates, worms] = findgroups(E.plateNum, E.wormNum); +for g = 1:max(G) + rows = E(G == g, :); + rows = sortrows(rows, 'id'); + plate = sprintf('%s_assayPlate%04d', pre, plates(g)); + worm = sprintf('%s_worm%04d', pre, worms(g)); + if ~isKey(subjectIds, worm) || ~isKey(subjectIds, plate) + continue; + end + % the worm's joined track: its rows of the track table, in table order + W = T(T.plateNum == plates(g) & T.wormNum == worms(g), :); + % the plate's videos: epoch, start, frame rate, by videoNum + vids = D(D.plateNum == plates(g), :); + epochs = arrayfun(@(r) epochOf(pre, vids.videoFileName{r}), (1:height(vids))', 'UniformOutput', false); + [onR, where] = ismember(epochs, R.epoch); + % A video that is not a recording (a failed start with no file, which + % stage 5 reports as rowWithoutFile) holds none of the worm's track; the + % worm is skipped only when a row of its track is on such a video. + if ~any(onR) || ~all(ismember(unique(W.videoNum), vids.videoNum(onR))) + out.skipped{end+1} = sprintf('%s: its track is on a video of %s that is not a recording of this session', ... + worm, plate); + continue; + end + vids = vids(onR, :); + epochs = epochs(onR); + where = where(onR); + start = R.local_start(where); + rate = R.frame_rate(where); + t0 = min(start); + % the plate's window: stage 9's, else the one video's own reference + timeId = ''; + if isKey(options.PlateWindows, plate) + timeId = options.PlateWindows(plate); + elseif height(vids) == 1 && isKey(options.RecordingRefs, epochs{1}) + timeId = options.RecordingRefs(epochs{1}); + else + dur = R.duration(where); + [t1, k1] = max(start + seconds(dur)); + w = utcReference(wallClock(t0, tz), [], secondFmt, wallClock(t1, tz), [], secondFmt, ... + ~isnan(dur(k1)), tz, sid); + docs{end+1} = w; %#ok + timeId = w.base.id; + end + secondsAt = @(idx) arrayfun(@(i) rowSeconds(W, i, vids, start, rate, t0), idx); + + % inputs: the worm's midpoint speed and patch-edge distance in each video + inputs = {}; + for v = 1:numel(epochs) + key = [epochs{v} '|' worm]; + if isKey(options.Tracks, key) + t = options.Tracks(key); + inputs = [inputs, {t.speed, t.patch_edge_distance}]; %#ok + end + end + inputs = inputs(~cellfun(@isempty, inputs)); + calc = [{'SessionId', sid, 'TimeReferenceIds', {timeId}}, env]; + wormId = subjectIds(worm); + method = did2.build.term('', 'patch encounter detection'); + + enc = rows(rows.id >= 1 & ~isnan(rows.enter), :); + gap0 = rows(rows.id == 0, :); + n = height(enc); + out.counts.worms = out.counts.worms + 1; + + % the gap before the first encounter + if height(gap0) == 1 + if ~isnan(gap0.velocityOff) + docs{end+1} = did2.build.statement('velocity_calculation', wormId, ... + did2.build.term('', 'median speed off patch before the first encounter'), ... + speed(gap0.velocityOff), 'Method', method, 'InputIds', inputs, calc{:}); %#ok + end + if ~isnan(gap0.distanceOffMax) + docs{end+1} = did2.build.statement('length_calculation', wormId, ... + did2.build.term('', 'furthest distance from the patch edge before the first encounter'), ... + mm(gap0.distanceOffMax), 'Method', method, 'InputIds', inputs, calc{:}); %#ok + end + end + if n == 0 + continue; + end + out.counts.encounters = out.counts.encounters + n; + + % the encounter list + onset = secondsAt(enc.enter); + list = did2.build.statement('time_calculation', wormId, ... + did2.build.term('', 'encounter onset'), did2.build.valueCell('time', onset), ... + 'Keys', did2.build.list(did2.build.key('encounter', n, 'Origin', 0, 'Spacing', 1)), ... + 'Method', method, 'InputIds', inputs, ... + 'Notes', ['Seconds from the start of the plate''s first recording; the source''s ' ... + '`enter`, a row of the worm''s track over the plate''s videos joined in order.'], calc{:}); + docs{end+1} = list; %#ok + if byRef + keyed = {'Keys', did2.build.list(did2.build.key('encounter', n, 'PositionsFrom', 0)), ... + 'KeyIds', {list.base.id}}; + else % the builder or schema predates key_id + keyed = {'Keys', did2.build.list(did2.build.key('encounter', n, 'Origin', 0, 'Spacing', 1))}; + end + per = [keyed, {'InputIds', [{list.base.id}, inputs]}, calc]; + out.byWorm(worm) = struct('wormId', wormId, 'listId', list.base.id, 'keyed', {keyed}, ... + 'calc', {calc}, 'rows', enc); + + % which patch + pIds = patchIds(subjectIds, plate); + if hasItem && all(enc.lawnID >= 1 & enc.lawnID <= numel(pIds)) + v = struct('item', enc.lawnID(:)' - 1, 'offset', secondsAt(enc.exit)'); + docs{end+1} = did2.build.statement('item_calculation', wormId, ... + did2.build.term('', 'patch in contact'), v, 'Edges', struct('item_id', {pIds}), ... + 'Method', method, per{:}); %#ok + elseif hasItem + out.skipped{end+1} = sprintf('%s: an encounter names a patch %s does not have; no patch list', ... + worm, plate); + end + + % the per-encounter values + q = { + 'velocity_calculation', 'median speed on patch', 'velocityOn', @speed + 'velocity_calculation', 'minimum speed on patch', 'velocityOnMin', @speed + 'velocity_calculation', 'peak speed before entry', 'velocityBeforeEnter', @speed + 'velocity_calculation', 'minimum speed after entry', 'velocityAfterEnter', @speed + 'velocity_calculation', 'median speed off patch after the encounter', 'velocityOff', @speed + 'acceleration_calculation', 'deceleration on entry', 'decelerate', @accel + 'time_calculation', 'time to slow down', 'timeSlowDown', @secs + 'length_calculation', 'furthest distance off patch during the encounter', 'distanceOnMax', @mm + 'length_calculation', 'furthest distance from the patch edge after the encounter', 'distanceOffMax', @mm + 'score_calculation', 'probability of exploitation', 'exploitPosterior', @prob + 'score_calculation', 'probability of sensing', 'sensePosterior', @prob}; + for j = 1:size(q, 1) + if ~ismember(q{j, 3}, enc.Properties.VariableNames), continue; end + x = double(enc.(q{j, 3})); + docs{end+1} = did2.build.statement(q{j, 1}, wormId, did2.build.term('', q{j, 2}), ... + q{j, 4}(x), 'Method', method, per{:}); %#ok + end + if ismember('label', enc.Properties.VariableNames) + names = cellstr(enc.label); + names(cellfun(@isempty, names)) = {'unlabelled'}; + docs{end+1} = did2.build.statement('label_calculation', wormId, ... + did2.build.term('', 'encounter type'), struct('name', reshape(names, 1, [])), ... + 'Method', did2.build.term('', 'thresholds on the encounter probabilities'), ... + 'MethodParameters', did2.build.list( ... + did2.build.parameter('exploit at or above probability of exploitation', 'Value', 0.5), ... + did2.build.parameter('sample at or above probability of sensing', 'Value', 0.5), ... + did2.build.parameter('searchOff below probability of sensing', 'Value', 0.05)), ... + per{:}); %#ok + end +end +out.documents = docs; +out.counts.documents = numel(docs); +end + +% ----------------------------------------------------------------------------- +function s = rowSeconds(W, i, vids, start, rate, t0) +% seconds from T0 of row I of the worm's joined track +if isnan(i) || i < 1 || i > height(W) + s = NaN; + return; +end +v = find(vids.videoNum == W.videoNum(i), 1); +s = seconds(start(v) - t0) + (double(W.frameNum(i)) - 1) / rate(v); +end + +function tf = keyByReference() +% key_id needs both the schema (did-schema PR #87) and the builder (DID-matlab +% PR #217); before both, the per-encounter documents list the same positions +tf = false; +f = fullfile(getenv('DID_SCHEMA_PATH'), 'data.json'); +if ~isfile(f), return; end +d = jsondecode(fileread(f)); +if ~isfield(d, 'depends_on') || ~any(strcmp({d.depends_on.name}, 'key_id')), return; end +try + did2.build.key('encounter', 1, 'PositionsFrom', 0); + tf = true; +catch +end +end + +function v = speed(x), v = did2.build.valueCell('velocity', x(:)' * 1e-6, 'SourceValue', x(:)', 'SourceUnit', 'um/s'); end +function v = accel(x), v = did2.build.valueCell('acceleration', x(:)' * 1e-6, 'SourceValue', x(:)', 'SourceUnit', 'um/s^2'); end +function v = secs(x), v = did2.build.valueCell('time', x(:)'); end +function v = mm(x), v = did2.build.valueCell('length', x(:)' * 1e-3, 'SourceValue', x(:)', 'SourceUnit', 'mm'); end +function v = prob(x), v = did2.build.valueCell('score', x(:)', 'Fields', struct( ... + 'scale', did2.build.term('', 'probability'), 'scale_min', 0, 'scale_max', 1)); end + +function e = epochOf(pre, name) +e = ''; +name = strtrim(char(name)); +if isempty(name), return; end +[~, stem] = fileparts(name); +e = [pre '_' stem]; +end diff --git a/src/ndi/+ndi/+setup/+conv/+haley/extractLawnProfiles.m b/src/ndi/+ndi/+setup/+conv/+haley/extractLawnProfiles.m new file mode 100644 index 0000000000..528a9bf50e --- /dev/null +++ b/src/ndi/+ndi/+setup/+conv/+haley/extractLawnProfiles.m @@ -0,0 +1,62 @@ +function report = extractLawnProfiles(source, destination) +%EXTRACTLAWNPROFILES Copy the E. coli profile curves and background fits into a small file. +% +% REPORT = ndi.setup.conv.haley.extractLawnProfiles(SOURCE, DESTINATION) +% reads analyzeGFP's workspace SOURCE (e.g. analyzeGFP_24-04-04.mat, 119 GB) +% one image at a time and writes to DESTINATION only what the import uses +% (ecoliDocuments, decisions #60 and #64): +% +% lawnProfiles.imageNum 1 x nImages +% lawnProfiles.distances 1 x nImages cell, each image's +% distance from the patch edge (mm) +% lawnProfiles.pixelValuesNormalized 1 x nImages cell, each image's +% nPatches x nBins curves +% backgroundInfo the background fits table +% +% The images inside lawnProfiles (~170 GB) are never read. Point the +% import at the result with import_V2(..., 'EcoliProfilesFile', +% DESTINATION), or save it as ecoli/analyzeGFP_24-04-04.mat beside +% bacteria.mat, where the import looks by default. +% +% REPORT: images (denominator), withCurves, curves (patches in all), +% backgroundRows, destination, bytes. + +arguments + source (1,:) char {mustBeFile} + destination (1,:) char +end + +P = lawnProfileReader(source); +n = numel(P.imageNums); +lawnProfiles = struct('imageNum', P.imageNums, 'distances', {cell(1, n)}, ... + 'pixelValuesNormalized', {cell(1, n)}); +withCurves = 0; +curves = 0; +fprintf('DENOMINATOR: %d image(s) in %s\n', n, source); +for k = 1:n + [x, Y] = lawnProfileRead(P, P.imageNums(k)); + lawnProfiles.distances{k} = x; + lawnProfiles.pixelValuesNormalized{k} = Y; + if ~isempty(Y) + withCurves = withCurves + 1; + curves = curves + size(Y, 1); + end + if mod(k, 250) == 0 + fprintf(' %d / %d images read\n', k, n); + end +end + +backgroundInfo = table(); +w = whos('-file', source); +if any(strcmp({w.name}, 'backgroundInfo')) + S = load(source, 'backgroundInfo'); + backgroundInfo = S.backgroundInfo; +end +save(destination, 'lawnProfiles', 'backgroundInfo', '-v7'); + +d = dir(destination); +report = struct('images', n, 'withCurves', withCurves, 'curves', curves, ... + 'backgroundRows', height(backgroundInfo), 'destination', destination, 'bytes', d.bytes); +fprintf(['%d image(s) read, %d with curves, %d curve(s) in all; %d background row(s); ' ... + 'wrote %s (%.1f MB)\n'], n, withCurves, curves, height(backgroundInfo), destination, d.bytes / 1e6); +end diff --git a/src/ndi/+ndi/+setup/+conv/+haley/extractTemplates.m b/src/ndi/+ndi/+setup/+conv/+haley/extractTemplates.m new file mode 100644 index 0000000000..e68468a0da --- /dev/null +++ b/src/ndi/+ndi/+setup/+conv/+haley/extractTemplates.m @@ -0,0 +1,86 @@ +function report = extractTemplates(source, destination) +%EXTRACTTEMPLATES Copy analyzeGFP's acetate template registrations into the small extract. +% +% REPORT = ndi.setup.conv.haley.extractTemplates(SOURCE, DESTINATION) +% reads `template` from analyzeGFP's workspace SOURCE (e.g. +% analyzeGFP_24-04-04.mat) WITHOUT its masks (3.5 GB) and appends to +% DESTINATION (the file extractLawnProfiles wrote): +% +% template.imageNum, .plateNum, .expNum 1 x N: the brightfield image +% each registration is from +% template.center N x 2, [y x] pixels, AS STORED +% (getAcetateArena: center = +% flip(Centroid)) +% template.offset N x 2, as stored +% template.angle 1 x N, degrees (regionprops +% Orientation) +% template.scale 1 x N, pixels per mm +% +% Each is the acetate frame (16 x 25 mm) found in one brightfield image +% (analyzeGFP.m:128-155); a fluorescence image uses its +% metaData.brightfieldImageNum's. Decision #80. +% +% REPORT: registrations (denominator), destination. + +arguments + source (1,:) char {mustBeFile} + destination (1,:) char {mustBeFile} +end + +fid = H5F.open(source, 'H5F_ACC_RDONLY', 'H5P_DEFAULT'); +c = onCleanup(@() H5F.close(fid)); +template = struct(); +for f = ["imageNum", "plateNum", "expNum", "angle", "scale", "center", "offset"] + v = readField(fid, "/template/" + f); + if iscell(v) + v = cellfun(@(x) reshape(double(x), 1, []), v, 'UniformOutput', false); + v = vertcat(v{:}); + else + v = reshape(double(v), 1, []); + end + template.(f) = v; +end +n = numel(template.imageNum); +fprintf('DENOMINATOR: %d template registration(s) in %s\n', n, source); +for f = ["plateNum", "expNum", "angle", "scale"] + if numel(template.(f)) ~= n + error('ndi:setup:conv:haley:templateShape', 'template.%s has %d entries, imageNum %d.', ... + f, numel(template.(f)), n); + end +end +for f = ["center", "offset"] + if size(template.(f), 1) ~= n || size(template.(f), 2) ~= 2 + error('ndi:setup:conv:haley:templateShape', 'template.%s is %s, not %d x 2.', ... + f, mat2str(size(template.(f))), n); + end +end +save(destination, 'template', '-append'); +fprintf('appended template (%d registrations, %d plate(s)) to %s\n', n, ... + numel(unique(template.plateNum)), destination); +report = struct('registrations', n, 'destination', destination); +end + +function v = readField(fid, path) +% a struct field of a -v7.3 MAT file: numeric data, or object references +% to one dataset per element (a cell) +d = H5D.open(fid, path); +cd = onCleanup(@() H5D.close(d)); +t = H5D.get_type(d); +ct = onCleanup(@() H5T.close(t)); +if H5T.get_class(t) ~= H5ML.get_constant_value('H5T_REFERENCE') + v = H5D.read(d); + return; +end +refs = H5D.read(d, 'H5T_STD_REF_OBJ', 'H5S_ALL', 'H5S_ALL', 'H5P_DEFAULT'); +refs = reshape(refs, size(refs, 1), []); +v = cell(1, size(refs, 2)); +for k = 1:size(refs, 2) + try + o = H5R.dereference(d, 'H5R_OBJECT', refs(:, k)); + catch + o = H5R.dereference(d, 'H5P_DEFAULT', 'H5R_OBJECT', refs(:, k)); + end + v{k} = H5D.read(o); + H5O.close(o); +end +end diff --git a/src/ndi/+ndi/+setup/+conv/+haley/geometryDocuments.m b/src/ndi/+ndi/+setup/+conv/+haley/geometryDocuments.m new file mode 100644 index 0000000000..8b9fd8cb89 --- /dev/null +++ b/src/ndi/+ndi/+setup/+conv/+haley/geometryDocuments.m @@ -0,0 +1,339 @@ +function out = geometryDocuments(dataParentDir, session, R, subjectIds, options) +%GEOMETRYDOCUMENTS Stage 10 part A (Haley): each behaviour video's geometry. +% +% OUT = ndi.setup.conv.haley.geometryDocuments(DATAPARENTDIR, SESSION, R, +% SUBJECTIDS, ...) builds, for the one-row session table SESSION, the +% calculations the lab's analysis package (getForagingInfo, Haley et al. +% 2024) made from each behaviour video's first frame, read from the +% session's experimentInfo.mat rows. Decision log #60. +% +% Per behaviour video (R rows of kind 'behaviour' with a recording +% statement in 'RecordingStatements'), all about the video's assay plate, +% each a calculation whose input is the recording: +% coordinate_system the video's pixel grid: origin the image's upper-left +% corner, x along the image horizontal axis (positive +% right), y along the vertical axis (positive down), one +% pixel = 1/`scale` mm (the lab's scale: the arena's +% known diameter over its mask's size in pixels, so it +% is marked approximate) +% label_calculation one per mask: `arenaMask` (variable `arena region`), +% `refMask` (`reference mark region`; foragingMini's +% arenas have no mark and its masks are empty, so none) +% and `lawnMask` (`bacterial lawn region`). A bool per +% pixel, in a sampled_body keyed [image vertical +% position, image horizontal position], metres with +% pixels as the source unit +% item_calculation `lawnClosest`: which patch is nearest each pixel, as a +% 0-based index into the plate's patch subjects (item_id, +% in lawnCenters order, decision #38). Only when the +% schema in use has `item` (did-schema PR #87) +% score_calculation `lawnRegistration`: the fraction of the arena that +% overlaps once the lawn clip is registered onto the +% video (0-1); its inputs are both recordings +% And per patch (the plate's patch subjects, in lawnCenters order, #38), each +% about the PATCH, its input the recording: +% position_calculation `patch centre` (`lawnCenters`), in the video's +% coordinate system: MATLAB's pixel centres are at 1, +% 2, ..., the system's origin is the image's corner, so +% a centre at column x is x - 0.5 pixels from it +% length_calculation `patch radius` (`lawnRadii`, pixels -> metres) +% score_calculation `patch circularity` (`lawnCircularity`, 0-1) +% (doImport's patch table had these; decision #70) +% The arena's nominal diameter and the patches' nominal diameter and spacing +% are method parameters of the masks that used them; the lawn mask's method +% is the patches' `lawnMethod` when they all share one (else none, and its +% notes list them). `lawnClosestOD600` (the closest map with each patch's +% OD600) and `firstFrame` (frame 0 of the video) are not stored: both are +% already in the dataset. +% +% Each mask and map is written to a temporary gzip file of its raw bytes +% (column-major, little-endian) that the database INGESTS when the session +% is written (files.file_info `ingest` 1, `delete_original` 1), so it is +% held in /.ndi/files with the session. +% +% Options: +% 'RecordingStatements' containers.Map, epoch -> recording statement id +% (sessionDocuments' OUT.recordingStatements) +% 'RecordingRefs' containers.Map, epoch -> its UTC reference id +% 'SoftwareId' the analysis package (spec `haley_analysis`) +% 'InterpreterId' MATLAB (spec `matlab`) +% 'OperatingSystemId' the analysis computer's OS (spec `macos`) +% +% OUT fields: documents (cell of structs), counts (struct), skipped (cellstr), +% byEpoch (containers.Map, epoch -> struct of the ids part B needs: +% coordinate_system, lawn_mask, nearest_patch, pixel (metres per pixel)). + +arguments + dataParentDir (1,:) char {mustBeFolder} + session table + R table + subjectIds + options.RecordingStatements = containers.Map() + options.RecordingRefs = containers.Map() + options.SoftwareId (1,:) char = '' + options.InterpreterId (1,:) char = '' + options.OperatingSystemId (1,:) char = '' +end +if height(session) ~= 1 + error('ndi:setup:conv:haley:oneSession', 'Give exactly one session row.'); +end +sid = char(session.session_id{1}); +ref = char(session.local_identifier{1}); +folder = char(session.folder{1}); +out = struct('documents', {{}}, 'skipped', {{}}, 'counts', struct('coordinate_system', 0, ... + 'arena', 0, 'reference_mark', 0, 'lawn', 0, 'nearest_patch', 0, 'registration', 0, ... + 'patch_centre', 0, 'patch_radius', 0, 'patch_circularity', 0), ... + 'byEpoch', containers.Map()); +if strcmp(folder, 'ecoli') + return; +end +if isempty(options.OperatingSystemId) + % every calculation names the OS it ran on (subject_calculation requires it) + out.skipped{end+1} = sprintf(['%s: no operating system document (the spec''s `macos`, ' ... + 'built by the metadata stage); no calculations'], ref); + return; +end +hasItem = ndi.setup.V2.schemaHasField('item', 'value'); +if ~hasItem + out.skipped{end+1} = sprintf(['%s: the schema in use has no `item` (did-schema PR #87); ' ... + 'no nearest-patch maps'], ref); +end +pre = ndi.setup.conv.haley.idPrefix(folder); +I = load(fullfile(dataParentDir, 'haley', 'celegans', folder, 'experimentInfo.mat'), 'info'); +D = I.info; +D = D(D.expNum == session.experiment(1), :); +R = R(strcmp(R.session, ref), :); +lawnClips = R(strcmp(R.kind, 'lawn'), :); +R = R(strcmp(R.kind, 'behaviour'), :); +docs = {}; +env = runEnvironment(options); + +for k = 1:height(R) + epoch = R.epoch{k}; + if ~isKey(options.RecordingStatements, epoch) + out.skipped{end+1} = sprintf('%s: no recording statement (not written)', epoch); + continue; + end + row = find(arrayfun(@(r) strcmp(epochOf(pre, D.videoFileName{r}), epoch), 1:height(D)), 1); + if isempty(row) + out.skipped{end+1} = sprintf('%s: no experimentInfo row', epoch); + continue; + end + d = D(row, :); + plate = R.plate{k}; + if ~isKey(subjectIds, plate) + out.skipped{end+1} = sprintf('%s: plate %s is not a subject of this session', epoch, plate); + continue; + end + scale = colOr(d, 'scale', NaN); + if ~(scale > 0) + out.skipped{end+1} = sprintf('%s: no pixel scale; no geometry', epoch); + continue; + end + videoId = options.RecordingStatements(epoch); + timeIds = {}; + if isKey(options.RecordingRefs, epoch) + timeIds = {options.RecordingRefs(epoch)}; + end + pixel = 1e-3 / scale; % metres per pixel + + % ---- the video's coordinate system -------------------------------------- + step = did2.build.valueCell('length', pixel, 'SourceValue', 1 / scale, ... + 'SourceUnit', 'mm', 'Approximate', true); + cs = did2.build.document('coordinate_system', struct( ... + 'origin', did2.build.term('', 'image upper-left corner'), ... + 'dimensions', did2.build.list( ... + struct('axis', did2.build.term('', 'image horizontal axis'), ... + 'positive_direction', did2.build.term('', 'image right'), 'spacing', step), ... + struct('axis', did2.build.term('', 'image vertical axis'), ... + 'positive_direction', did2.build.term('', 'image down'), 'spacing', step))), ... + 'SessionId', sid, 'Edges', struct('referent_id', videoId)); + docs{end+1} = cs; %#ok + out.counts.coordinate_system = out.counts.coordinate_system + 1; + + calc = {'SessionId', sid, 'TimeReferenceIds', timeIds, 'InputIds', {videoId}}; + calc = [calc, env]; %#ok + + % ---- masks --------------------------------------------------------------- + arenaParams = did2.build.list(did2.build.parameter('arena diameter', ... + 'Value', colOr(d, 'arenaDiameter', NaN) * 1e-3, 'Unit', 'meter', ... + 'SourceValue', num2str(colOr(d, 'arenaDiameter', NaN)), 'SourceUnit', 'mm')); + [docs, n] = mask(docs, d, 'arenaMask', 'arena region', {'MethodParameters', arenaParams}); + out.counts.arena = out.counts.arena + n; + [docs, n] = mask(docs, d, 'refMask', 'reference mark region', {}); + out.counts.reference_mark = out.counts.reference_mark + n; + lawnArgs = {'MethodParameters', did2.build.list( ... + did2.build.parameter('patch diameter', 'Value', colOr(d, 'lawnDiameter', NaN) * 1e-3, ... + 'Unit', 'meter', 'SourceValue', num2str(colOr(d, 'lawnDiameter', NaN)), 'SourceUnit', 'mm'), ... + did2.build.parameter('patch spacing', 'Value', colOr(d, 'lawnSpacing', NaN) * 1e-3, ... + 'Unit', 'meter', 'SourceValue', num2str(colOr(d, 'lawnSpacing', NaN)), 'SourceUnit', 'mm'))}; + found = cellOr(d, 'lawnMethod'); + if ~iscell(found), found = {}; end + found = found(~cellfun(@isempty, found)); + if ~isempty(found) && numel(unique(found)) == 1 + lawnArgs = [lawnArgs, {'Method', did2.build.term('', ['patch detection by ' found{1}])}]; %#ok + elseif ~isempty(found) + lawnArgs = [lawnArgs, {'Notes', sprintf('Each patch''s detection method, in patch order: %s.', ... + strjoin(found, ', '))}]; %#ok + end + [docs, n, lawnMaskId] = mask(docs, d, 'lawnMask', 'bacterial lawn region', lawnArgs); + out.counts.lawn = out.counts.lawn + n; + ids = struct('coordinate_system', cs.base.id, 'lawn_mask', lawnMaskId, ... + 'nearest_patch', '', 'pixel', pixel); + + % ---- nearest patch --------------------------------------------------------- + closest = cellOr(d, 'lawnClosest'); + if hasItem && ~isempty(closest) + pIds = patchIds(subjectIds, plate); + nPatch = max(closest(:)); + if isempty(pIds) || nPatch > numel(pIds) || any(closest(:) < 1) + out.skipped{end+1} = sprintf(['%s: lawnClosest names patches 1..%d; plate %s has ' ... + '%d patch subject(s); no nearest-patch map'], epoch, nPatch, plate, numel(pIds)); + else + dt = 'uint8'; + if numel(pIds) > 255, dt = 'uint16'; end + inputs = {videoId}; + if ~isempty(lawnMaskId), inputs{end+1} = lawnMaskId; end %#ok + st = did2.build.statement('item_calculation', subjectIds(plate), ... + did2.build.term('', 'nearest patch'), [], 'DataBody', true, 'DatumType', dt, ... + 'Keys', pixelKeys(closest, pixel), 'SessionId', sid, ... + 'TimeReferenceIds', timeIds, 'InputIds', inputs, ... + 'Edges', struct('item_id', {pIds}), env{:}); + docs = [docs, {st, ingestedBody(st, closest - 1, dt, pixelKeys(closest, pixel), sid, ... + 'Each pixel''s nearest patch: a 0-based index into item_id.')}]; %#ok + out.counts.nearest_patch = out.counts.nearest_patch + 1; + ids.nearest_patch = st.base.id; + end + end + + % ---- each patch's centre, radius and circularity --------------------------- + docs = patchGeometry(docs, d, plate, cs.base.id, pixel, calc); + + % ---- lawn clip registration ------------------------------------------------ + fit = colOr(d, 'lawnRegistration', NaN); + lawnEpoch = epochOf(pre, cellOrChar(d, 'lawnFileName')); + if ~isnan(fit) && ~isempty(lawnEpoch) && any(strcmp(lawnClips.epoch, lawnEpoch)) ... + && isKey(options.RecordingStatements, lawnEpoch) + v = did2.build.valueCell('score', fit, 'Fields', struct( ... + 'scale', did2.build.term('', 'fraction of the arena overlapping'), ... + 'scale_min', 0, 'scale_max', 1)); + st = did2.build.statement('score_calculation', subjectIds(plate), ... + did2.build.term('', 'lawn clip registration fit'), v, 'SessionId', sid, ... + 'TimeReferenceIds', timeIds, ... + 'InputIds', {videoId, options.RecordingStatements(lawnEpoch)}, env{:}); + docs{end+1} = st; %#ok + out.counts.registration = out.counts.registration + 1; + end + out.byEpoch(epoch) = ids; +end +out.documents = docs; + +% ============================================================================= + function docs = patchGeometry(docs, d, plate, csId, pixel, calc) + centres = cellOr(d, 'lawnCenters'); + if isempty(centres) + return; + end + pIds = patchIds(subjectIds, plate); + n = size(centres, 1); + if numel(pIds) ~= n + out.skipped{end+1} = sprintf(['%s: %d lawnCenters but %d patch subject(s); ' ... + 'no patch geometry'], epoch, n, numel(pIds)); + return; + end + radii = cellOr(d, 'lawnRadii'); + circ = cellOr(d, 'lawnCircularity'); + for j = 1:n + xy = double(centres(j, 1:2)) - 0.5; + if all(isfinite(xy)) + docs{end+1} = did2.build.statement('position_calculation', pIds{j}, ... + did2.build.term('', 'patch centre'), struct('coordinates', xy), ... + 'Edges', struct('coordinate_system_id', csId), calc{:}); %#ok + out.counts.patch_centre = out.counts.patch_centre + 1; + end + if numel(radii) == n && isfinite(radii(j)) + v = did2.build.valueCell('length', double(radii(j)) * pixel, ... + 'SourceValue', double(radii(j)), 'SourceUnit', 'pixel', 'Approximate', true); + docs{end+1} = did2.build.statement('length_calculation', pIds{j}, ... + did2.build.term('', 'patch radius'), v, calc{:}); %#ok + out.counts.patch_radius = out.counts.patch_radius + 1; + end + if numel(circ) == n && isfinite(circ(j)) + v = did2.build.valueCell('score', double(circ(j)), 'Fields', struct( ... + 'scale', did2.build.term('', 'circularity'), 'scale_min', 0, 'scale_max', 1)); + docs{end+1} = did2.build.statement('score_calculation', pIds{j}, ... + did2.build.term('', 'patch circularity'), v, calc{:}); %#ok + out.counts.patch_circularity = out.counts.patch_circularity + 1; + end + end + end + + function [docs, n, id] = mask(docs, d, column, variable, extra) + % one label_calculation + its body; none when the mask is absent or empty + n = 0; id = ''; + m = cellOr(d, column); + if isempty(m) || ~any(m(:)) + return; + end + st = did2.build.statement('label_calculation', subjectIds(plate), ... + did2.build.term('', variable), [], 'DataBody', true, 'DatumType', 'bool', ... + 'Keys', pixelKeys(m, pixel), calc{:}, extra{:}); + docs = [docs, {st, ingestedBody(st, logical(m), 'bool', pixelKeys(m, pixel), sid, ... + sprintf('The %s mask (1 = inside), from the source''s `%s`.', variable, column))}]; + n = 1; id = st.base.id; + end +end + +% ----------------------------------------------------------------------------- +function e = runEnvironment(options) +% the calculation's software, interpreter and operating system edges +e = {}; +if ~isempty(options.SoftwareId), e = [e, {'SoftwareId', options.SoftwareId}]; end +if ~isempty(options.InterpreterId), e = [e, {'InterpreterId', options.InterpreterId}]; end +if ~isempty(options.OperatingSystemId), e = [e, {'OperatingSystemId', options.OperatingSystemId}]; end +end + +function keys = pixelKeys(a, pixel) +% [image vertical position, image horizontal position]: MATLAB's [row, column]. +% Each position is a pixel's CENTRE, measured from the image's upper-left +% corner (the coordinate system's origin): the first is half a pixel in. +keys = did2.build.list( ... + did2.build.key('image vertical position', size(a, 1), 'Unit', 'meter', ... + 'Origin', pixel / 2, 'Spacing', pixel, 'SourceUnit', 'pixel', 'SourceOrigin', 0.5, 'SourceSpacing', 1), ... + did2.build.key('image horizontal position', size(a, 2), 'Unit', 'meter', ... + 'Origin', pixel / 2, 'Spacing', pixel, 'SourceUnit', 'pixel', 'SourceOrigin', 0.5, 'SourceSpacing', 1)); +end + + +function e = epochOf(pre, name) +% a recording's epoch id from the file name the table records +e = ''; +name = strtrim(char(name)); +if isempty(name), return; end +[~, stem] = fileparts(name); +e = [pre '_' stem]; +end + +function v = colOr(d, name, default) +v = default; +if ismember(name, d.Properties.VariableNames) + x = d.(name); + if iscell(x), x = x{1}; end + if isnumeric(x) && isscalar(x), v = double(x); end +end +end + +function v = cellOr(d, name) +% d.(NAME){1}, or [] when the table has no such column +v = []; +if ismember(name, d.Properties.VariableNames) + v = d.(name); + if iscell(v), v = v{1}; end +end +end + +function v = cellOrChar(d, name) +v = cellOr(d, name); +if isempty(v) || ~(ischar(v) || isstring(v)), v = ''; end +v = char(v); +end diff --git a/src/ndi/+ndi/+setup/+conv/+haley/idPrefix.m b/src/ndi/+ndi/+setup/+conv/+haley/idPrefix.m new file mode 100644 index 0000000000..b5c75bb712 --- /dev/null +++ b/src/ndi/+ndi/+setup/+conv/+haley/idPrefix.m @@ -0,0 +1,21 @@ +function p = idPrefix(folder) +%IDPREFIX The local_identifier prefix for a Haley source folder. +% +% P = ndi.setup.conv.haley.idPrefix(FOLDER) returns the prefix every +% local_identifier from FOLDER starts with (decision log #37): the folder +% name without `foraging`, lower-cased -- foragingConcentration -> +% concentration, foragingMini -> mini, ecoli -> ecoli. +% +% The FOLDER, not the study, is the prefix: plate and worm numbers restart +% in each folder, and foragingConcentration holds two studies numbered +% together, so the folder is what makes a number unique in the dataset. + +arguments + folder (1,:) char +end +p = regexprep(folder, '^foraging', ''); +p = lower(p); +if isempty(p) + error('ndi:setup:conv:haley:badFolder', 'No identifier prefix for folder "%s".', folder); +end +end diff --git a/src/ndi/+ndi/+setup/+conv/+haley/import_V2.m b/src/ndi/+ndi/+setup/+conv/+haley/import_V2.m new file mode 100644 index 0000000000..89c6f6c7b6 --- /dev/null +++ b/src/ndi/+ndi/+setup/+conv/+haley/import_V2.m @@ -0,0 +1,851 @@ +function result = import_V2(dataParentDir, options) +%IMPORT_V2 Import the Haley C. elegans / E. coli foraging dataset as V2 (V_eta). +% +% RESULT = ndi.setup.conv.haley.import_V2(DATAPARENTDIR) runs the import's +% stages over the raw data in DATAPARENTDIR/haley and returns what each +% stage produced. It is being built one stage at a time with the dataset's +% owner, and is meant to be the canonical example of a V2 import (and the +% basis of a GUI importer): every choice it makes is recorded, with its +% reason, in import_V2_decisions.md next to this file. +% +% The stages, in order (decision log, "Stage order"): +% A. dataset, once +% 0 discover list the source files; skip earlier import output +% 1 profile describe tables (ndi.setup.V2.profileTable, on demand) +% 2 metadata dataset, people, organizations, funding, publication, +% software, products, strains, chemicals, recipes, +% instruments -- from import_V2_spec.json (sources: +% the eLife paper, WormBook, the lab's Benchling +% protocols) -- and the seeding suspensions, one set +% per seeding day, from the source tables +% (ndi.setup.conv.haley.seedingSuspensions) +% B. per study (E. coli first); per day: +% 3 sessions the studies (from the spec; decision #50), then one +% session per experiment day, part_of its study +% 4 subjects plates, patches, worm cohorts, worms, acclimation +% and food deprivation plates; each with its type +% 5 acquisition camera/microscope, one epoch per recording +% 6 relations patch part_of plate; worm member_of its cohort; +% cohort contained_in its acclimation, food +% deprivation and assay plates (with when) +% 7 assertions species and strain of each cohort and seeded +% patch; exclusion tags on plates +% 8 manipulations pouring and seeding each plate, its moves +% between cold room, room temperature and +% incubator; each cohort's transfers and food +% deprivation +% 9 observations each filmed assay plate's ambient temperature +% and relative humidity, read by the temperature +% probe over its recordings +% 10 calculations part A: each behaviour video's coordinate system, +% arena / reference mark / lawn masks, nearest-patch +% map and lawn clip registration fit, as the lab's +% analysis package computed them (decision #60); +% part B: each worm's per-frame position, speed, +% distance to the nearest patch edge and nearest +% patch, per body part and video (decision #61); +% part C: each analysed E. coli image's patch mask +% and each detected patch's profile values, on its +% patch subject (decision #62) +% 11 encounters each worm's patch encounters (celegans/encounter.mat): +% an encounter-onset list and the per-encounter +% values keyed by it (decision #63) +% C. dataset-wide, once +% 12 density the E. coli border amplitude fits, each about a +% group of patches, and each C. elegans worm's +% per-encounter estimates from them (decision #65; +% needs the E. coli session and the C. elegans +% sessions in the same run) +% 13 dataset check & write the dataset: ONE V2 database holding +% the dataset-level documents (stage 2's, the +% studies, stage 12's fits) and every selected +% session, ingested (ndi.setup.V2.createDataset; +% decision #66); opens as an ndi.dataset.dir +% +% Nothing is written unless 'Write' is true: by default RESULT holds what +% each stage WOULD create, for inspection. With 'Write', each selected +% session is created under 'OutputRoot' with its V2 database holding the +% session and, when stages 4 and 5 ran, its subjects, acquisition systems, +% epochs and recordings (ndi.setup.conv.haley.sessionDocuments). The +% session folder holds only .ndi/: each recording's document records where +% its raw file is, and NDI finds it there (ndi.file.navigator.bodies); +% nothing is linked, copied or written next to the raw data. The +% dataset-level documents of +% stage 2 (instruments included) are not written yet (stage 13), so a +% recording's instrument_id names a document outside the session. +% +% Options: +% 'Spec' path to the spec (default: import_V2_spec.json here) +% 'Stages' which stages to run (default: all implemented) +% 'OutputRoot' where session directories go (default: +% /haley_V2; the raw data is never +% written to, decision 21) +% 'Sessions' local_identifiers of the sessions to import (default: +% all), e.g. "concentration_0002" +% 'Write' default false; true creates the sessions +% 'Checksums' default true: MD5 of every recording written +% 'Overwrite' default false; true replaces existing sessions +% 'ReadVideos' default true: opens each lawn clip (VideoReader) to +% read its length (stage 5), and on 'Write' each +% video for its pixel format; false skips both +% (videos are then assumed 8-bit) +% 'DatasetSessionId' session id for dataset-level documents (default: a +% new id) +% 'DatasetFolder' where stage 13 writes the dataset (default: +% /dataset) +% 'DatasetReference' the dataset's local_identifier (default 'haley') +% 'EcoliProfilesFile' analyzeGFP's workspace (profile curves, background +% fits; decision #64), wherever it is kept (default: +% /haley/ecoli/analyzeGFP_24-04-04.mat) +% 'SessionFolders' default: false when the dataset stage runs, true +% when it does not (decision #71). false INGESTS every +% session into the dataset and writes no folders: the +% self-contained form to publish, and the fastest. true +% writes each session ONCE, to its own folder, and the +% dataset LINKS it (a `linked_session` document and the +% session's `part_of` relation in the dataset's +% database). Either converts to the other later +% (ndi.dataset/convertLinkedSessionToIngested, +% convertIngestedSessionToLinked). With a schema that has +% no `linked_session` class, true writes each session to +% its folder AND into the dataset, as before. +% 'WriteBatchSize' documents per write batch (default 2000); progress +% is printed per batch, and shown in a +% ProgressBarWindow when MATLAB has a display +% 'ValidateOnWrite' default false: every document was validated when +% did2.build built it, so the database does not +% validate it again on insert; true repeats it +% +% 'RunFolder' with 'Write': where this run's record is kept +% (default: /run_); +% 'none' keeps none. It holds import_log.txt (the +% whole console output, a diary), import_report.mat +% (`report`: the dataset check, the sessions, the +% checks, the density fits, the timing; not the +% documents) and skipped.txt (every reported skip +% and check line). Written even when the run fails, +% with the error at the end of the log. +% +% With 'Write', every folder to be written (the dataset's and, with +% 'SessionFolders', each session's) is checked before anything is built +% or written: one that already holds an NDI database is an error unless +% 'Overwrite' is true. +% +% doImport.m, the original V1 import, is unchanged. +% +% See also ndi.setup.V2.discover, ndi.setup.V2.datasetMetadata. + +arguments + dataParentDir (1,:) char {mustBeFolder} = fullfile(userpath, 'data') + options.Spec (1,:) char = fullfile(fileparts(mfilename('fullpath')), 'import_V2_spec.json') + options.Stages (1,:) string = ["discover", "metadata", "sessions", "subjects", "acquisition", "relations", "assertions", "manipulations", "observations", "calculations", "encounters", "density", "dataset"] + options.OutputRoot (1,:) char = '' + options.Sessions (1,:) string = string.empty(1, 0) + options.Write (1,1) logical = false + options.Checksums (1,1) logical = true + options.Overwrite (1,1) logical = false + options.DatasetSessionId (1,:) char = '' + options.ReadVideos (1,1) logical = true + options.DatasetFolder (1,:) char = '' + options.DatasetReference (1,:) char = 'haley' + options.SessionFolders logical {mustBeScalarOrEmpty} = logical.empty + options.EcoliProfilesFile (1,:) char = '' + options.WriteBatchSize (1,1) double {mustBePositive, mustBeInteger} = 2000 + options.ValidateOnWrite (1,1) logical = false + options.RunFolder (1,:) char = '' +end +if isempty(options.SessionFolders) + % decision #71: the import is for publishing, so a run that writes the + % dataset writes only the dataset (every session ingested, the form NDI + % Cloud holds, and the fastest); a run without the dataset stage writes + % session folders, which are then the only output + options.SessionFolders = ~any(options.Stages == "dataset"); +end + +% With 'Write', the run keeps its own record (log, report, skipped list) in a +% run folder beside the dataset: this outer call sets it up and runs the +% import itself as an inner call with RunFolder 'none'. +if options.Write && ~strcmp(options.RunFolder, 'none') + result = recordedRun(dataParentDir, options); + return; +end + +% Check the requirements up front, so a missing one is reported with its fix +% rather than as "Unable to resolve the name 'did2.build.label'" mid-stage. +if any(ismember(options.Stages, ["metadata", "sessions"])) + ndi.setup.V2.preflight(); +end + +result = struct(); +sid = options.DatasetSessionId; +if isempty(sid) + sid = did.ido.unique_id(); +end +result.datasetSessionId = sid; +% One dataset document id for the whole run: stage 2 builds the dataset with +% it and stage 3's studies are part_of it, whichever stages run. +result.datasetId = ndi.ido.unique_id(); + +if any(options.Stages == "discover") + fprintf('\n== stage 0: discover ==\n'); + result.files = ndi.setup.V2.discover(fullfile(dataParentDir, 'haley')); +end + +if any(options.Stages == "metadata") + fprintf('\n== stage 2: dataset metadata ==\n'); + spec = jsondecode(fileread(options.Spec)); + % The seeding suspensions (stage 8 part B, decision #57) come from the + % source tables, not the spec, but are dataset-level formulations like the + % spec's: built here with them, after the strains and diluents they name. + if isfield(spec, 'seeding') + [result.suspensions, result.suspensionChecks] = ... + ndi.setup.conv.haley.seedingSuspensions(dataParentDir, spec); + f = spec.formulations; + if isstruct(f), f = num2cell(f); end + spec.formulations = [reshape(f, 1, []), result.suspensions.entries]; + end + result.metadata = ndi.setup.V2.datasetMetadata(spec, sid, ... + 'DatasetId', result.datasetId, 'Studies', "exclude"); + fprintf('DENOMINATOR: %d document(s) built from %s\n', ... + numel(result.metadata.documents), options.Spec); + disp(result.metadata.census); + u = result.metadata.unrepresented; + fprintf('spec content with NO place in V2 (recorded here, not in the documents): %d\n', numel(u)); + for k = 1:numel(u) + fprintf(' %-12s %-20s %-16s %s\n', u(k).kind, u(k).key, u(k).field, u(k).why); + end +end + +if any(options.Stages == "sessions") + fprintf('\n== stage 3: sessions ==\n'); + spec = jsondecode(fileread(options.Spec)); + % the studies first: they group the sessions (each spec study's + % source_folder / source_condition picks its days), so they are minted + % here, beside them (decision #50). Dataset-level documents, like stage 2's. + result.studies = ndi.setup.V2.datasetMetadata(spec, sid, ... + 'DatasetId', result.datasetId, 'Studies', "only"); + fprintf('DENOMINATOR: %d study document(s) and %d relation(s) built from %s\n', ... + sum(strcmp(result.studies.census.class, 'study') .* result.studies.census.count), ... + sum(strcmp(result.studies.census.class, 'directed_relation') .* result.studies.census.count), ... + options.Spec); + [result.sessions, result.sessionChecks] = ndi.setup.conv.haley.sessionList(dataParentDir, spec, ... + 'StudyIds', result.studies.ids, 'OutputRoot', options.OutputRoot); + allSessions = result.sessions; + if ~isempty(options.Sessions) + unknown = setdiff(options.Sessions, string(result.sessions.local_identifier)); + if ~isempty(unknown) + error('ndi:setup:conv:haley:unknownSession', 'No session %s.', strjoin(unknown, ', ')); + end + result.sessions = result.sessions(ismember(result.sessions.local_identifier, ... + cellstr(options.Sessions)), :); + fprintf('%d session(s) selected by ''Sessions''\n', height(result.sessions)); + end + disp(result.sessions(:, {'local_identifier', 'study_key', 'date', 'include'})); +end + +if any(options.Stages == "subjects") + fprintf('\n== stage 4: subjects ==\n'); + if ~isfield(result, 'sessions') + error('ndi:setup:conv:haley:needSessions', ... + 'The subjects stage needs the sessions stage: include "sessions" in ''Stages''.'); + end + % Listed against EVERY session, then narrowed: with 'Sessions', a plate + % of an unselected day would otherwise read as having no session. + spec = jsondecode(fileread(options.Spec)); + corrections = {}; + if isfield(spec, 'corrections'), corrections = spec.corrections; end + [result.subjects, result.subjectChecks] = ndi.setup.conv.haley.subjectList( ... + dataParentDir, allSessions, 'Corrections', corrections); + result.subjects = result.subjects(ismember(result.subjects.session, ... + result.sessions.local_identifier), :); + if height(result.subjects) > 0 + disp(groupsummary(result.subjects, {'folder', 'kind'})); + end + if ~options.Write + fprintf('(not written: pass ''Write'', true)\n'); + end +end + +if any(options.Stages == "acquisition") + fprintf('\n== stage 5: acquisition ==\n'); + if ~isfield(result, 'sessions') + error('ndi:setup:conv:haley:needSessions', ... + 'The acquisition stage needs the sessions stage: include "sessions" in ''Stages''.'); + end + [result.recordings, result.recordingChecks] = ndi.setup.conv.haley.recordingList( ... + dataParentDir, result.sessions, 'ReadVideos', options.ReadVideos); + if height(result.recordings) > 0 + disp(groupsummary(result.recordings, {'kind', 'system'})); + end + if ~options.Write + fprintf('(not written: pass ''Write'', true)\n'); + end +end + +if any(options.Stages == "relations") + fprintf('\n== stage 6: relations ==\n'); + if ~isfield(result, 'subjects') || ~isfield(result, 'recordings') + error('ndi:setup:conv:haley:needSubjects', ['The relations stage needs the ' ... + 'subjects and acquisition stages: include "subjects" and "acquisition" in ''Stages''.']); + end + if ~options.Write + fprintf('(built with the session documents: pass ''Write'', true)\n'); + end +end + +if any(options.Stages == "assertions") + fprintf('\n== stage 7: assertions ==\n'); + if ~isfield(result, 'subjects') + error('ndi:setup:conv:haley:needSubjects', ['The assertions stage needs the ' ... + 'subjects stage: include "subjects" in ''Stages''.']); + end + if ~options.Write + fprintf('(built with the session documents: pass ''Write'', true)\n'); + end +end + +if options.Write && isfield(result, 'sessions') + fprintf('\n== write ==\n'); + result = writeSessions(result, dataParentDir, options); +end +end + +function result = recordedRun(dataParentDir, options) +% Run the import with its console output, report and skipped list kept in +% the run folder; the diary that was on before (if any) is restored. +runDir = options.RunFolder; +if isempty(runDir) + outRoot = options.OutputRoot; + if isempty(outRoot), outRoot = fullfile(dataParentDir, 'haley_V2'); end + runDir = fullfile(outRoot, ['run_' char(datetime('now'), 'yyyy-MM-dd_HHmmss')]); +end +if ~isfolder(runDir), mkdir(runDir); end +logFile = fullfile(runDir, 'import_log.txt'); +wasOn = strcmp(get(0, 'Diary'), 'on'); +wasFile = get(0, 'DiaryFile'); +diary off +diary(logFile); +restore = onCleanup(@() restoreDiary(wasOn, wasFile)); +fprintf('run folder: %s\nstarted: %s\n', runDir, char(datetime('now'))); +t0 = tic; +inner = options; +inner.RunFolder = 'none'; +args = namedargs2cell(inner); +failure = []; +try + result = ndi.setup.conv.haley.import_V2(dataParentDir, args{:}); +catch failure + fprintf('\n== the run FAILED ==\n%s\n', getReport(failure, 'extended', 'hyperlinks', 'off')); + result = struct(); +end +elapsed = toc(t0); +fprintf('\nimport took %s\nrun folder: %s\n', char(duration(0, 0, round(elapsed))), runDir); +diary off +report = runReport(result, options, elapsed, failure); %#ok (saved) +save(fullfile(runDir, 'import_report.mat'), 'report'); +writeSkipped(logFile, fullfile(runDir, 'skipped.txt')); +result.runFolder = runDir; +if ~isempty(failure) + rethrow(failure); +end +end + +function restoreDiary(wasOn, wasFile) +diary off +if wasOn + diary(wasFile); +end +end + +function report = runReport(result, options, elapsed, failure) +% what the run decided and checked, without its documents (small to save) +report = struct('finished', datetime('now'), 'seconds', elapsed, 'failed', ~isempty(failure), ... + 'error', '', 'options', options); +if ~isempty(failure), report.error = failure.message; end +for f = {'datasetSessionId', 'datasetId', 'dataset', 'sessions', 'subjectChecks', ... + 'suspensionChecks', 'recordingChecks'} + if isfield(result, f{1}), report.(f{1}) = result.(f{1}); end +end +if isfield(result, 'density') + d = result.density; + report.density = struct(); + for f = {'fits', 'counts', 'checks', 'skipped'} + if isfield(d, f{1}), report.density.(f{1}) = d.(f{1}); end + end +end +end + +function writeSkipped(logFile, outFile) +% every reported skip and check line of the log, under the stage or session +% header it came after +lines = splitlines(string(fileread(logFile))); +keep = strings(0, 1); +inChecks = false; +for k = 1:numel(lines) + t = strtrim(lines(k)); + if startsWith(t, "CHECKS") + inChecks = true; + keep(end+1, 1) = lines(k); %#ok + elseif inChecks && startsWith(lines(k), " ") && t ~= "" + keep(end+1, 1) = lines(k); %#ok + else + inChecks = false; + if startsWith(t, "== ") || startsWith(t, "-- session ") || startsWith(t, "skipped:") ... + || contains(t, "disagrees:") + keep(end+1, 1) = lines(k); %#ok + end + end +end +writelines(keep, outFile); +end + +function result = writeSessions(result, dataParentDir, options) +% One V2 session per selected row: its documents first, then its files. +% Subjects carry a display name (decision #42), which needs did-schema #80. +schema = getenv('DID_SCHEMA_PATH'); +subjectFile = fullfile(schema, 'subject.json'); +if isfield(result, 'subjects') && isfile(subjectFile) + if ~ndi.setup.V2.schemaHasField('subject', 'name') + error('ndi:setup:conv:haley:oldSchema', ... + ['The V2 schema in %s has no subject.name (did-schema PR #80, merged ' ... + '2026-10-01). Update did-schema to main and copy its V_eta schemas ' ... + 'there again; nothing was written.'], schema); + end +end +T = result.sessions; +n = height(T); +checkFoldersFree(T, dataParentDir, options); +T.session_id = arrayfun(@(~) ndi.ido.unique_id(), (1:n)', 'UniformOutput', false); +T.session_doc_id = arrayfun(@(~) ndi.ido.unique_id(), (1:n)', 'UniformOutput', false); +T.time_reference_id = repmat({''}, n, 1); +T.documents = repmat({{}}, n, 1); +built = cell(n, 1); +instrumentIds = containers.Map(); +if isfield(result, 'metadata') + instrumentIds = result.metadata.ids; +end +if isfield(result, 'subjects') && isfield(result, 'recordings') + % building every session's documents (stages 6-11) is the longest part of + % a run before the write: a bar and a line per session + buildBar = []; tag = 'build sessions'; + try + buildBar = ndi.gui.component.ProgressBarWindow('NDI V2 import', 'GrabMostRecent', true); + buildBar.setTimeout(hours(12)); + buildBar.addBar('Label', sprintf('Building %d session(s)', n), 'Tag', tag, 'Auto', true); + catch + buildBar = []; + end + tBuild = tic; + for k = 1:n + fprintf('\n-- session %d of %d: %s --\n', k, n, T.local_identifier{k}); + built{k} = ndi.setup.conv.haley.sessionDocuments(dataParentDir, T(k, :), ... + result.subjects, result.recordings, 'InstrumentIds', instrumentIds, ... + 'Checksums', options.Checksums, 'ReadVideos', options.ReadVideos); + if any(options.Stages == "relations") + spec = jsondecode(fileread(options.Spec)); + protocol = struct(); + if isfield(spec, 'transfer_protocol'), protocol = spec.transfer_protocol; end + rel = ndi.setup.conv.haley.relationDocuments(T(k, :), result.subjects, ... + result.recordings, built{k}.subjectIds, 'Protocol', protocol); + built{k}.documents = [built{k}.documents, rel.documents]; + built{k}.relations = rel; + c = rel.counts; + fprintf(['%s relations: %d patch part_of plate; %d worm member_of cohort; ' ... + 'cohort contained_in: %d assay plate, %d acclimation plate, ' ... + '%d food deprivation plate\n'], T.local_identifier{k}, ... + c.patch_part_of_plate, c.worm_member_of_cohort, c.cohort_in_assay_plate, ... + c.cohort_in_acclimation_plate, c.cohort_in_food_deprivation_plate); + for j = 1:numel(rel.skipped) + fprintf(' skipped: %s\n', rel.skipped{j}); + end + end + if any(options.Stages == "assertions") + spec = jsondecode(fileread(options.Spec)); + strains = struct([]); + if isfield(spec, 'strains'), strains = spec.strains; end + as = ndi.setup.conv.haley.assertionDocuments(T(k, :), result.subjects, ... + built{k}.subjectIds, 'Strains', strains, 'StrainIds', instrumentIds); + built{k}.documents = [built{k}.documents, as.documents]; + built{k}.assertions = as; + c = as.counts; + fprintf(['%s assertions: cohort %d species, %d strain, %d sex; patch %d species, ' ... + '%d strain; %d plate(s) excluded\n'], T.local_identifier{k}, c.cohort_species, ... + c.cohort_strain, c.cohort_sex, c.patch_species, c.patch_strain, c.plate_excluded); + for j = 1:numel(as.skipped) + fprintf(' skipped: %s\n', as.skipped{j}); + end + end + if any(options.Stages == "manipulations") && isfield(built{k}, 'relations') + spec = jsondecode(fileread(options.Spec)); + index = table(); + if isfield(result, 'suspensions'), index = result.suspensions.index; end + ma = ndi.setup.conv.haley.manipulationDocuments(T(k, :), result.subjects, ... + built{k}.subjectIds, 'Preparation', spec.preparation, ... + 'Seeding', spec.seeding, 'Ids', instrumentIds, 'Names', namesOf(spec), ... + 'Suspensions', index, 'Relations', built{k}.relations, ... + 'HandTolerance', reshape(double(spec.transfer_protocol.hand_written_tolerance_seconds), 1, 2)); + built{k}.documents = [built{k}.documents, ma.documents]; + built{k}.manipulations = ma; + c = ma.counts; + fprintf(['%s manipulations: %d pour, %d patch seeding, %d acclimation plate ' ... + 'seeding, %d temperature, %d transfer, %d food deprivation; ' ... + '%d developmental stage observation(s)\n'], ... + T.local_identifier{k}, c.pour, c.seed_patch, c.seed_acclimation_plate, ... + c.temperature, c.transfer, c.food_deprivation, c.developmental_stage); + for j = 1:numel(ma.skipped) + fprintf(' skipped: %s\n', ma.skipped{j}); + end + elseif any(options.Stages == "manipulations") + fprintf('%s: the manipulations stage needs the relations stage (it shares their times)\n', ... + T.local_identifier{k}); + end + if any(options.Stages == "observations") + spec = jsondecode(fileread(options.Spec)); + probe = ''; + if isKey(instrumentIds, spec.environment.instrument) + probe = instrumentIds(spec.environment.instrument); + end + ob = ndi.setup.conv.haley.observationDocuments(T(k, :), result.recordings, ... + built{k}.subjectIds, 'Environment', spec.environment, 'InstrumentId', probe, ... + 'RecordingRefs', built{k}.recordingRefs); + built{k}.documents = [built{k}.documents, ob.documents]; + built{k}.observations = ob; + fprintf('%s observations: %d temperature, %d humidity\n', T.local_identifier{k}, ... + ob.counts.temperature, ob.counts.humidity); + for j = 1:numel(ob.skipped) + fprintf(' skipped: %s\n', ob.skipped{j}); + end + end + if any(options.Stages == "calculations") + run = struct('SoftwareId', '', 'InterpreterId', '', 'OperatingSystemId', ''); + keys = {'haley_analysis', 'matlab', 'macos'}; + names = fieldnames(run); + for j = 1:numel(keys) + if isKey(instrumentIds, keys{j}), run.(names{j}) = instrumentIds(keys{j}); end + end + ge = ndi.setup.conv.haley.geometryDocuments(dataParentDir, T(k, :), result.recordings, ... + built{k}.subjectIds, 'RecordingStatements', built{k}.recordingStatements, ... + 'RecordingRefs', built{k}.recordingRefs, 'SoftwareId', run.SoftwareId, ... + 'InterpreterId', run.InterpreterId, 'OperatingSystemId', run.OperatingSystemId); + built{k}.documents = [built{k}.documents, ge.documents]; + built{k}.geometry = ge; + c = ge.counts; + fprintf(['%s calculations: %d coordinate system, %d arena, %d reference mark, ' ... + '%d lawn mask, %d nearest patch, %d registration; patch %d centre, %d radius, ' ... + '%d circularity\n'], T.local_identifier{k}, ... + c.coordinate_system, c.arena, c.reference_mark, c.lawn, c.nearest_patch, c.registration, ... + c.patch_centre, c.patch_radius, c.patch_circularity); + for j = 1:numel(ge.skipped) + fprintf(' skipped: %s\n', ge.skipped{j}); + end + tr = ndi.setup.conv.haley.trackDocuments(dataParentDir, T(k, :), result.recordings, ... + built{k}.subjectIds, 'Geometry', ge.byEpoch, ... + 'RecordingStatements', built{k}.recordingStatements, ... + 'RecordingRefs', built{k}.recordingRefs, 'SoftwareId', run.SoftwareId, ... + 'InterpreterId', run.InterpreterId, 'OperatingSystemId', run.OperatingSystemId); + built{k}.documents = [built{k}.documents, tr.documents]; + built{k}.tracks = tr; + c = tr.counts; + fprintf(['%s tracks: %d position, %d speed, %d distance to patch edge, ' ... + '%d nearest patch\n'], T.local_identifier{k}, c.position, c.speed, ... + c.patch_edge_distance, c.nearest_patch); + for j = 1:numel(tr.skipped) + fprintf(' skipped: %s\n', tr.skipped{j}); + end + patchLayout = struct([]); + sp = jsondecode(fileread(options.Spec)); + if isfield(sp, 'ecoli_patch_layout'), patchLayout = sp.ecoli_patch_layout; end + ec = ndi.setup.conv.haley.ecoliDocuments(dataParentDir, T(k, :), result.recordings, ... + built{k}.subjectIds, 'RecordingStatements', built{k}.recordingStatements, ... + 'RecordingRefs', built{k}.recordingRefs, 'SoftwareId', run.SoftwareId, ... + 'InterpreterId', run.InterpreterId, 'OperatingSystemId', run.OperatingSystemId, ... + 'ProfilesFile', options.EcoliProfilesFile, 'PatchLayout', patchLayout); + built{k}.documents = [built{k}.documents, ec.documents]; + built{k}.ecoli = ec; + c = ec.counts; + if c.mask > 0 || c.normalised_image > 0 + fprintf(['%s E. coli: %d normalised image(s), %d background fit(s), %d mask, ' ... + '%d image(s) matched to patch subjects, %d not; %d per-patch value(s), ' ... + '%d profile curve(s), %d nearest patch\n'], T.local_identifier{k}, ... + c.normalised_image, c.background_fit, c.mask, c.matched_images, ... + c.unmatched_images, c.patch_values, c.profiles, c.nearest_patch); + end + for j = 1:numel(ec.skipped) + fprintf(' skipped: %s\n', ec.skipped{j}); + end + end + if any(options.Stages == "encounters") && isfield(built{k}, 'tracks') + windows = containers.Map(); + if isfield(built{k}, 'observations'), windows = built{k}.observations.windows; end + en = ndi.setup.conv.haley.encounterDocuments(dataParentDir, T(k, :), result.recordings, ... + built{k}.subjectIds, 'Tracks', built{k}.tracks.byWorm, 'PlateWindows', windows, ... + 'RecordingRefs', built{k}.recordingRefs, 'SoftwareId', run.SoftwareId, ... + 'InterpreterId', run.InterpreterId, 'OperatingSystemId', run.OperatingSystemId); + built{k}.documents = [built{k}.documents, en.documents]; + built{k}.encounters = en; + c = en.counts; + fprintf('%s encounters: %d worm(s), %d encounter(s), %d document(s)\n', ... + T.local_identifier{k}, c.worms, c.encounters, c.documents); + for j = 1:numel(en.skipped) + fprintf(' skipped: %s\n', en.skipped{j}); + end + elseif any(options.Stages == "encounters") + fprintf('%s: the encounters stage needs the calculations stage (its tracks)\n', ... + T.local_identifier{k}); + end + T.documents{k} = built{k}.documents; + T.time_reference_id{k} = built{k}.timeReferenceId; + classes = cellfun(@(d) d.document_class.class_name, built{k}.documents, ... + 'UniformOutput', false); + fprintf('%s: %d document(s) built\n', T.local_identifier{k}, numel(classes)); + disp(groupsummary(table(classes(:), 'VariableNames', {'class'}), 'class')); + for j = 1:numel(built{k}.skipped) + fprintf(' skipped: %s\n', built{k}.skipped{j}); + end + el = toc(tBuild); + fprintf('-- %d of %d session(s) built, %s elapsed, ~%s left --\n', k, n, ... + char(duration(0, 0, round(el))), char(duration(0, 0, round(el / k * (n - k))))); + if ~isempty(buildBar) + try buildBar.updateBar(tag, k / n); catch, end + end + end + if any(options.Stages == "density") + fprintf('\n== stage 12: density ==\n'); + run = struct('SoftwareId', '', 'InterpreterId', '', 'OperatingSystemId', ''); + keys = {'haley_analysis', 'matlab', 'macos'}; + names = fieldnames(run); + for j = 1:numel(keys) + if isKey(instrumentIds, keys{j}), run.(names{j}) = instrumentIds(keys{j}); end + end + de = ndi.setup.conv.haley.densityDocuments(dataParentDir, T, built, ... + 'DatasetSessionId', result.datasetSessionId, 'SoftwareId', run.SoftwareId, ... + 'InterpreterId', run.InterpreterId, 'OperatingSystemId', run.OperatingSystemId); + for k = 1:n + T.documents{k} = [T.documents{k}, de.sessionDocuments{k}]; + end + result.density = de; + c = de.counts; + fprintf(['density: %d fit(s) about %d group(s) of %d patch membership(s), %d input ' ... + 'value(s); %d refit(s) checked; %d worm(s), %d estimate document(s), %d encounter ' ... + 'value(s) checked (dataset-level documents: written at stage 13)\n'], c.fits, ... + c.groups, c.members, c.inputs, c.refit_checked, c.worms, c.estimates, c.encounters_checked); + for j = 1:numel(de.checks.refitDisagrees) + fprintf(' refit disagrees: %s\n', de.checks.refitDisagrees{j}); + end + for j = 1:numel(de.checks.encounterDisagrees) + fprintf(' encounter disagrees: %s\n', de.checks.encounterDisagrees{j}); + end + for j = 1:numel(de.skipped) + fprintf(' skipped: %s\n', de.skipped{j}); + end + end +else + fprintf('(stages 4 and 5 did not run: writing the sessions alone)\n'); +end +% Decision #71: with 'SessionFolders', each session is written ONCE, to its own +% folder, and the dataset LINKS it (a `linked_session` document and the session's +% `part_of` relation in the dataset's database). It needs a schema with the +% `linked_session` class; without one, the sessions are also ingested into the +% dataset, as before. +linked = options.SessionFolders && any(options.Stages == "dataset") && ... + ndi.setup.V2.schemaHasClass('linked_session'); +if options.SessionFolders && any(options.Stages == "dataset") && ~linked + fprintf(['(the schema in use has no linked_session class: each session is written to ' ... + 'its folder AND ingested into the dataset)\n']); +end +if any(options.Stages == "dataset") + fprintf('\n== stage 13: check & write the dataset ==\n'); + result.dataset = writeDataset(result, T, dataParentDir, options, linked); +end +if options.SessionFolders + fprintf('\n== session folders ==\n'); + Tw = T; + if linked && ismember('study_ids', Tw.Properties.VariableNames) + % membership is the dataset's: the part_of relations were written there + Tw = removevars(Tw, 'study_ids'); + end + [Tw, result.sessionObjects] = ndi.setup.V2.makeSessions(Tw, 'Overwrite', options.Overwrite, ... + 'Validate', options.ValidateOnWrite, 'Progress', true, 'BatchSize', options.WriteBatchSize); + T.session_id = Tw.session_id; +end +if linked + result.dataset = listDataset(result.dataset); +end +result.sessions = removevars(T, 'documents'); +result.written = built; +end + +function path = datasetPath(dataParentDir, options) +outRoot = options.OutputRoot; +if isempty(outRoot), outRoot = fullfile(dataParentDir, 'haley_V2'); end +path = options.DatasetFolder; +if isempty(path), path = fullfile(outRoot, 'dataset'); end +end + +function checkFoldersFree(T, dataParentDir, options) +% Every folder this run will write, checked before anything is built, so an +% occupied one stops the run at its start rather than after a long write. +if options.Overwrite, return; end +folders = {}; +if any(options.Stages == "dataset") + folders{end+1} = datasetPath(dataParentDir, options); +end +if options.SessionFolders + folders = [folders, reshape(cellstr(T.path), 1, [])]; +end +taken = {}; +for k = 1:numel(folders) + ndiDir = fullfile(folders{k}, '.ndi'); + if isfolder(ndiDir) && ~isempty([dir(fullfile(ndiDir, '*.sqlite')); dir(fullfile(ndiDir, '*.json'))]) + taken{end+1} = ndiDir; %#ok + end +end +if ~isempty(taken) + error('ndi:setup:conv:haley:foldersTaken', ... + ['%d of %d folder(s) to be written already hold an NDI database; nothing was ' ... + 'built or written. Pass ''Overwrite'', true to replace them, or move them:\n %s'], ... + numel(taken), numel(folders), strjoin(taken, '\n ')); +end +end + +function out = writeDataset(result, T, dataParentDir, options, linked) +% Stage 13 (decision #66): every document of this run checked as one set, and +% written into ONE V2 database -- or, LINKED (decision #71), the dataset-level +% documents, each session's part_of relations and a linked_session per session, +% with the sessions' own documents written to their folders afterwards. +path = datasetPath(dataParentDir, options); +dsDocs = {}; +for f = {'metadata', 'studies'} + if isfield(result, f{1}), dsDocs = [dsDocs, reshape(result.(f{1}).documents, 1, [])]; end %#ok +end +if isfield(result, 'density'), dsDocs = [dsDocs, result.density.datasetDocuments]; end +name = ''; +isDataset = cellfun(@(d) strcmp(ndi.setup.V2.kindOf(d), 'dataset'), dsDocs); +if any(isDataset) + ds = dsDocs{find(isDataset, 1)}; + [~, blk] = ndi.setup.V2.kindOf(ds); % `entity` since 2026-10-08 + if isfield(ds.(blk), 'name'), name = char(ds.(blk).name); end +end +if ~any(isDataset) + fprintf(' no dataset document (the metadata stage did not run): sessions cannot be part of it\n'); +end +sessDocs = cell(1, height(T)); +external = {}; +folders = cellstr(T.path); +for k = 1:height(T) + own = sessionOwn(T(k, :)); + if linked + % in the dataset: the session's part_of relations and where it lives; + % its own documents go to its folder (checked here as one set) + link = did2.build.document('linked_session', ... + struct('path', ndi.v2.linkPath(folders{k}, path)), ... + 'SessionId', result.datasetSessionId, 'Edges', struct('entity_id', own{1}.base.id)); + sessDocs{k} = [own(2:end), {link}]; + external = [external, own(1), reshape(T.documents{k}, 1, [])]; %#ok + continue; + end + d = [own, reshape(T.documents{k}, 1, [])]; + if options.SessionFolders + d = keepOriginals(d); % the session folders, written next, ingest the same files + end + sessDocs{k} = d; +end +[ds, report] = ndi.setup.V2.createDataset(path, options.DatasetReference, ... + result.datasetSessionId, dsDocs, sessDocs, 'Name', name, 'Overwrite', options.Overwrite, ... + 'Validate', options.ValidateOnWrite, 'BatchSize', options.WriteBatchSize, 'External', external); +refs = {}; ids = {}; +if linked + fprintf('dataset %s at %s: %d session(s) to link, listed once their folders are written\n', ... + ds.id(), path, height(T)); +else + [refs, ids] = ds.session_list(); + fprintf('dataset %s at %s: %d session(s) listed by ndi.dataset\n', ds.id(), path, numel(ids)); +end +disp(report.documents.byClass); +if height(report.edges.dangling) > 0 + fprintf(' DANGLING EDGES (an edge naming no document in the dataset):\n'); + disp(report.edges.dangling); +end +for j = 1:numel(report.duplicateIds) + fprintf(' id repeated: %s\n', report.duplicateIds{j}); +end +for j = 1:numel(report.sessions.notInDataset) + fprintf(' session not part_of a study of the dataset: %s\n', report.sessions.notInDataset{j}); +end +out = struct('path', path, 'id', ds.id(), 'report', report, 'sessionReferences', {refs}, ... + 'sessionIds', {ids}); +end + +function out = listDataset(out) +% the sessions of a linked dataset, listed once their folders are written +ds = ndi.dataset.dir(out.path); +[refs, ids] = ds.session_list(); +notes = ds.session_notes(); +fprintf('dataset %s at %s: %d session(s) listed by ndi.dataset (%s)\n', ds.id(), out.path, ... + numel(ids), notes{1}); +for k = 2:numel(notes) + fprintf(' %s\n', notes{k}); +end +out.sessionReferences = refs; +out.sessionIds = ids; +end + +function docs = sessionOwn(row) +% the session document and its study relations, exactly as createSession +% writes them into a session folder (same ids where createSession is given them) +f = struct('local_identifier', char(row.local_identifier{1})); +vars = row.Properties.VariableNames; +if ismember('name', vars) && ~isempty(row.name{1}), f.name = char(row.name{1}); end +if ismember('description', vars) && ~isempty(row.description{1}), f.description = char(row.description{1}); end +sid = char(row.session_id{1}); +edges = struct(); +if ~isempty(row.time_reference_id{1}) + edges.time_reference_id = row.time_reference_id{1}; +end +docs = ndi.setup.V2.entityDocuments('session', f, edges, 'SessionId', sid, ... + 'Id', char(row.session_doc_id{1})); +if ismember('study_ids', vars) + studies = row.study_ids{1}; + if ischar(studies) || isstring(studies), studies = cellstr(studies); end + for k = 1:numel(studies) + docs{end+1} = did2.build.directedRelation(docs{1}.base.id, studies{k}, ... + 'part_of', 'SessionId', sid); %#ok + end +end +end + +function docs = keepOriginals(docs) +% a copy of DOCS whose ingested files are not deleted once copied +for i = 1:numel(docs) + d = docs{i}; + if ~isfield(d, 'files') || ~isfield(d.files, 'file_info') || isempty(d.files.file_info) + continue; + end + fi = d.files.file_info; + for a = 1:numel(fi) + for b = 1:numel(fi(a).locations) + fi(a).locations(b).delete_original = 0; + end + end + d.files.file_info = fi; + docs{i} = d; +end +end + +function m = namesOf(spec) +% formulation and strain keys -> the names a dose's variable reads as +m = containers.Map(); +for f = {'formulations', 'strains'} + if ~isfield(spec, f{1}), continue; end + e = spec.(f{1}); + if isstruct(e), e = num2cell(e); end + for k = 1:numel(e) + x = e{k}; + if isfield(x, 'type') && ~isempty(x.type) + m(x.key) = char(x.type); + elseif isfield(x, 'name') && ~isempty(x.name) + m(x.key) = char(x.name); + end + end +end +end diff --git a/src/ndi/+ndi/+setup/+conv/+haley/import_V2_decisions.md b/src/ndi/+ndi/+setup/+conv/+haley/import_V2_decisions.md new file mode 100644 index 0000000000..0c949ac6d2 --- /dev/null +++ b/src/ndi/+ndi/+setup/+conv/+haley/import_V2_decisions.md @@ -0,0 +1,196 @@ +# `ndi.setup.conv.haley.import_V2` — decision log + +The running record of the choices made while building `import_V2`, the V2 +(V_eta) importer for the Haley (JH) C. elegans / E. coli foraging dataset. +`import_V2` is meant to be the canonical example of importing a dataset +(and the basis of a GUI importer), so the reasoning is kept, not just the result. + +Entries marked **DECIDED** were made by the dataset owner (jess) in +conversation; **OPEN** entries are questions still to answer. The existing +`doImport.m` is left unchanged (backwards compatibility). + +## Principles (DECIDED 2026-09-30) + +- Changes to NDI-matlab and DID-matlab are backwards compatible: new + functions/classes beside the old ones, existing behaviour unchanged. +- Documents are built with `did2.build` (DID-matlab V2), so no importer or + NDIMaker carries a field list of its own. +- New NDI object classes stay thin; everything added must be portable to + NDI-python. +- The import is worked through one section at a time, updating the relevant + NDIMakers and object classes as it goes. +- The spec is an OUTPUT, not an input. A user cannot describe a dataset before + reading its files, so the importer first DISCOVERS and PROFILES the source + tables, then each stage asks its questions about them; the answers + accumulate into a spec that is saved for re-imports and similar datasets. + +## Stage order (DECIDED 2026-09-30, revised after discovery) + +The first outline (discover -> dataset metadata -> sessions -> entities -> ...) +was flat and had no acquisition stage. Discovery changed three things: +every interaction statement needs a time reference (`subject_interaction` +declares `time_reference_id` with `min_count 1`), so EPOCHS must exist before +any statement; the source data comes per study (1-4 GB of tracks per folder) +and is recorded per day, so the loop is NESTED; and some content spans days or +studies (`encounter.mat` covers all five C. elegans studies; relative patch +density comes from the E. coli study), so it runs in a final dataset pass. + + A. dataset, once + 0 discover list source files; skip earlier import output + 1 profile describe tables (on demand) + 2 metadata people, organizations, funding, publication, + strains, products, instruments (BUILT) + B. per study (E. coli FIRST), tables loaded once + named corrections; + per day: + 3 session the studies (#50), then one session per day, + `part_of` its study + 4 subjects cultivation (L4) plates, behaviour plates, patches, worms + 5 acquisition camera / microscope system, one epoch per video or + image, clocks + syncgraph + 6 relations patch part_of plate; worm contained_in its assay + plate and its acclimation plate, each with when (#52) + 7 assertions strain, species, exclusion tag on plates + 8 manipulations plate preparation timeline (absolute UTC times), + food deprivation, transfer + 9 observations temperature/humidity per filmed assay plate (#59); + tracks, geometry and images are calculations (stage 10) + 10 calculations per day: masks, closest-patch maps from the first frame + write the session + C. dataset-wide, once + 11 encounters the cross-study encounter table + 12 calculations cross-study: relative density (E. coli -> C. elegans) + 13 check & write validate, census, write the dataset (#66) + +Each stage returns an id map (source key -> document id) that later stages use. + +REVISED 2026-10-01 (jess): subjects now come BEFORE acquisition. The epochs-first +rule is about STATEMENTS (every statement needs a time reference); subjects are +entities and need none, and creating them first lets each recording name its plate +and each probe map name a real subject document. + +## Decisions + +| # | Date | Decision | Status | +|---|---|---|---| +| 1 | 2026-09-30 | **Plates and patches are subjects** (not `ontologyTableRow` rows), so worms can be related to them and observations can be about them. | DECIDED | +| 2 | 2026-09-30 | **Excluded data is tagged, not dropped**: a `term_assertion` on the plate (e.g. the `exclude` column / tableOfContents `include = no`), carrying the reason where one is recorded (tableOfContents `notes`). | DECIDED | +| 3 | 2026-09-30 | `foragingConcentration/tableOfContents.xlsx` row `0014`: `wormNumber` `0449-0546` is a typo (overlaps row `0013`, `0451-0498`); the intended range is `0499-0546`. The importer must not trust the column blindly: it cross-checks each day's worm range against the worms present in `experimentInfo` and reports mismatches. | DECIDED (typo); correction mechanism OPEN | +| 4 | 2026-09-30 | Images are typed by what a pixel holds: fluorescence = intensity, masks = logical, closest-patch = which patch. Exact V2 classes to be checked against the #73 schema at the observations stage (the V2 `image` data_type was retired in #73 item 51). | DECIDED for the masks and closest-patch map (#60); OPEN for the E. coli images (stage 10 part C) | +| 5 | 2026-09-30 | **One session per experimental day.** An `ndi.session` is a unit of acquisition (one directory of raw files, its DAQ systems, one `syncgraph` of related clocks; `session.m`), so a day's recordings are a session. "All subjects in the dataset" is one `ndi.dataset.database_search` (it spans the dataset's database and every linked session, `dataset.m:748`); "all C. elegans" is a species assertion on each subject, not a level. (`doImport` made 2 sessions: all C. elegans, all E. coli.) | DECIDED | +| 8 | 2026-09-30 | **A grouping of sessions above the day is needed** for foragingConcentration / Matching / Mini / Mutants / Sensory (and the E. coli imaging). V2 has nothing between session and dataset: `part_of` allows only session -> dataset (`build_v_eta.py` `_rel("part_of"...)`), a term assertion must be about a `subject` (not a session), and `follows_protocol` runs dataset -> `web_resource`. Needs a new entity + relation (schema change, team decision). Name: NOT `protocol` (reads as a recipe; the five share one foraging recipe and differ in design). Proposed: `study`, as in ISA's investigation / study / assay. | DECIDED: `study` (built, see #11) | +| 9 | 2026-09-30 | **Videos are imported through a DAQ reader as image-series epochs**, so each recording is an epoch with a clock and the tracks are observations timed against it (`doImport` attached mp4s as `imageStack` files and made no epochs). NDI already has `ndi.daq.system.image` / `ndi.daq.reader.image` / `ndi.daq.reader.image.ndr` (movie clock `dev_local_time` + per-frame times); NDR-matlab's `imagestack` reader (NANSEN `ImageStack`) lists `.mp4`/`.avi`/`.mov` (`+ndr/+reader/imagestack.m:120`), so no new reader is needed; per-frame times still to confirm against `frameRate` / the tracks' `time`. | DECIDED (direction) | +| 10 | 2026-09-30 | **E. coli: one epoch per image**, timed by `acquisitionTime`. The plates were moved between images and the images are not spatially registered, so a time-lapse epoch would claim a registration that does not exist; the plate SUBJECT carries the continuity across images. | DECIDED | +| 11 | 2026-09-30 | `study` entity (proposal, for #8): fields `name` (req), `short_name`, `description`, `factors` (terms: the variables deliberately varied, same vocabulary as statement `variable`), `design` (terms, optional); `global_identifier` inherited. People / publications / protocols as relations. `part_of` widened to session -> study, study -> dataset, study -> study (nesting allowed; transitive lookup in one NDI helper). JH: six flat studies under the dataset. | DECIDED 2026-09-30 (jess approved the fields); BUILT, SIGNED (jess, `schemas/V_eta_study_plan.md`) and MERGED (did-schema PR #78) | +| 12 | 2026-09-30 | **Studies (from the eLife paper, 10.7554/eLife.103191, Materials and methods):** foragingConcentration is TWO studies -- single-density multi-patch (tableOfContents `grid` days) and large single-patch (`single` days); foragingMini = small single-patch (9 mm arena, 1 worm, 8 fps, head/tail tracked); foragingMatching = multi-density multi-patch (no peptone, 2 h); ecoli = bacterial patch density estimation (OP50-GFP, Zeiss Axio Zoom.V16). Settled from the data (2026-09-30): **foragingSensory** = the sensory-mutant study (N2 well-fed 13, PR811 osm-6 13, TU253 mec-4 12 plates; no food deprivation). **foragingMutants** = a broader mutant screen, 93 plates: cat-2, gcy-35, eat-2, npr-1, tax-4, tdc-1, tph-1, tbh-1, osm-6, mec-4, N2 well-fed, and N2 food-deprived (7 plates, the only ones with `starvedDuration > 0`); the paper's well-fed vs 3-h food-deprived analysis is its N2 subset, and most of its strains are not in the paper. | DECIDED | +| 13 | 2026-09-30 | **Organizations carry ROR identifiers**, looked up rather than left blank (this container cannot reach api.ror.org or ror.org pages; ids come from web search and must be confirmed before import). **The dataset has its own licence** (NDI Cloud page), not the paper's. | DECIDED | +| 14 | 2026-09-30 | **Instruments record their product through an `instance_of` relation** (directed_relation, instrument subject -> product), NOT a `subject.product_id` field: the relation lets the product be added after the instrument without rewriting either document. Built in did-schema PR #78 (`9febf40`). The unit's serial number is not built. | DECIDED; BUILT, SIGNED, MERGED (#78) | +| 15 | 2026-09-30 | **Labs and departments are organizations** (openMINDS `Organization`), linked upward by `suborganization_of` (e.g. Salk Molecular Neurobiology Laboratory -> Salk; UCSD Neurosciences Graduate Program -> UCSD), so people can be affiliated with the unit they belong to. | DECIDED | +| 16 | 2026-09-30 | **Dataset licence: CC-BY-4.0** (the NDI Cloud dataset's own licence, confirmed by jess; the paper's is separate). | DECIDED | +| 17 | 2026-09-30 | **`strain.product_id` should become a relation** (strain -> product, a new term such as `distributed_as`; a strain is not a *unit* of a product, so not `instance_of`), for the same late-addition reason as instruments. `chemical.product_id` and `formulation.product_id` stay fields: there the product is part of what the document is. NOT BUILT: it changes an existing field (#73 audit 2 D9) that migrators and `did2.build` see, and JH does not need it (its strains carry WormBase ids). A did-schema proposal to raise when convenient. **BUILT 2026-10-15** (Jess): did-schema 033a22d adds `distributed_as` (strain -> product); on the merged-entity schema `ndi.setup.V2.entityDocuments` writes a strain's stock as `distributed_as` (it had been writing `derived_from`, which says the strain was made from its listing); `derived_from` stays only on a schema copy without the relation. | DECIDED; BUILT 2026-10-15 | +| 18 | 2026-09-30 | **Author roles and award recipients are recorded.** `directed_relation.roles` (14 CRediT roles + `corresponding author`) on each `has_author` edge; `awarded_to` (funding -> person/org) for each grant's recipients. Built in did-schema PR #78. | DECIDED; BUILT, SIGNED, MERGED (#78) | +| 19 | 2026-09-30 | **Dataset required fields:** `accessibility` = free access; `ethics_assessment` = not required (C. elegans and E. coli). | DECIDED | +| 20 | 2026-09-30 | **Shared entities live at the dataset level** (the dataset's own database): people, organizations, funding, publication, studies, software, products, strains and instruments are created once and referred to by id from each day's session. | DECIDED | +| 21 | 2026-09-30 | **Session directories are a separate output tree** (e.g. `haley_V2///`), for C. elegans and E. coli alike: the raw data is never written to, and sessions refer back to the raw files (the did2 backend has no file store yet, so the files stay where they are either way). E. coli could not do otherwise: all its images share one folder. | DECIDED | +| 22 | 2026-09-30 | **`session.description`** holds a day's free text: the notebook entry, the conditions, and notes (tableOfContents). Built, signed and merged: did-schema PR #79. | DECIDED; BUILT, SIGNED, MERGED (#79) | +| 23 | 2026-09-30 | **E. coli: one session per experiment** (`expNum`, 5 in `bacteria.mat`), not per imaging day. | DECIDED | +| 24 | 2026-09-30 | **Session naming.** `local_identifier` is the stable, spaceless handle (e.g. `foragingConcentration_0001`); `session.name` is the display name (e.g. `Foraging concentration, experiment 1 (1 Feb 2022)`). Spaceless is a convention `import_V2` follows, not schema-enforced: nothing checks `pattern` yet, and migrated subjects already carry spaces in `local_identifier`. Built, signed and merged: did-schema PR #79. Follow-up (not done): the session migrator passes a v1 `base.name` through untouched; it could move a non-empty one into `session.name`. | DECIDED; BUILT, SIGNED, MERGED (#79) | +| 25 | 2026-09-30 | **Creating a V2 session**: `ndi.setup.V2.createSession` writes `.ndi/V_eta.sqlite` with the V2 `session` document (+ `part_of` study relations) and the `reference.txt` / `unique_reference.txt` files, then opens it with the unchanged `ndi.session.dir` (which can open but not create a did2 session). | BUILT | +| 26 | 2026-09-30 | **Stage 3 builds the session list from the source's own day index.** C. elegans: one session per `tableOfContents.xlsx` row (local_identifier `_`, e.g. `foragingConcentration_0001`); foragingConcentration rows go to the grid or single study by the first word of `conditions`. E. coli: one per `expNum`, dated by its earliest `metaData.acquisitionTime`. `description` carries the notebook, conditions, worm range, inclusion and notes; the worm range is carried as written and checked against the worms at stage 5. Sessions are only written with `'Write', true`, under `OutputRoot` (default `/haley_V2//`). | BUILT | +| 27 | 2026-10-01 | **Recordings.** A plate has up to two: a short LAWN clip (`lawnFileName`, before the worms go in) and the BEHAVIOUR video (`videoFileName`). The table names `.avi` files; disk holds same-stem `.mp4` conversions, so files are matched by stem and the original name is recorded. WormLab `.csv` files (`wormLabFileName`) are not on disk. Mutants plate 1 (3 Nov 2023 07:35:52) has 0 frames and no file; Mini `camera` 0 means not recorded (6 rows). Both get no epoch and are reported. | DECIDED (from the data) | +| 28 | 2026-10-01 | **One acquisition system per camera** (`camera1`, `camera2`; one microscope for E. coli): each camera has its own clock. | DECIDED | +| 29 | 2026-10-01 | **The `software` documents naming NDI classes** (`ndi.daq.system.image`, the navigator, `ndi.daq.reader.image.ndr`) **live in each session**: `daqsystem_load` rebuilds objects from the session's own database (`+vintage/objectClass.m` searches the session; `session.m` `database_search` is scoped to the session id), and a session opened alone has no link to its dataset. Decision 20's shared software (StreamPix, WormLab, MATLAB) stays at the dataset level. | DECIDED | +| 30 | 2026-10-01 | **Lawn clips get a full extent**: the import opens each clip to read its frame count and rate (option B), with MATLAB's native `VideoReader` (no NANSEN). Behaviour videos take theirs from `experimentInfo`. | DECIDED | +| 31 | 2026-10-01 | **Probe maps** (`.epochprobemap.ndi`, one per epoch: the camera as probe, the plate's `local_identifier` as subject) are written into the session folder, never the raw data. | DECIDED | +| 33 | 2026-10-01 | **Acquisition systems are per session.** NDI rebuilds them only from the session's own database. The physical camera is the dataset-level instrument subject (stage 2, `instance_of` its product); each recording links the two: `instrument_id` = the camera subject, `acquisition_channels_id` -> that session's `acquisition_system`. #73 removed `epoch.instrument_id` on purpose ("the rig is recorded where it is always right -- on each recording statement"), so an epoch needs no edge to its system. | DECIDED | +| 34 | 2026-10-01 | **Each recording's file is an `opaque_body`** (`format` `video/mp4` or `image/tiff`: bytes laid out by their own format), owned by the recording statement, with `filename`, `size_bytes`, `content_hash` + `hash_algorithm`. At import it is NOT held: its file member `body_data_0` is recorded BY LOCATION in `files.file_info.locations` (the absolute path, `location_type` 'file', `ingest` 0) -- the record `ndi.document/add_file` already writes, which V2 allows unchanged (no schema change). Ingestion later takes the bytes in AS THEY ARE (never decoded: NDI's image ingester writes every frame uncompressed, ~11 GB for one 2 MB video); only the location changes. | DECIDED; BUILT | +| 35 | 2026-10-01 | **An epoch may carry its UTC extent** as an `absolute_time_reference` (today `epoch.time_reference_id` names only `relative_time_reference`, a side effect of keying the #52 family on `value.clock`). For the #52 rule the absolute member is the `utc` one. Built in did-schema PR #80 (with `session.time_reference_id`), signed and merged. | DECIDED; BUILT, MERGED (#80) | +| 36 | 2026-10-01 | **A native NDR video reader** (MATLAB `VideoReader`, no NANSEN) is built before stage 4 writes anything, so `acquisition_reader.reader_string` names it from the start. | DECIDED; IN PROGRESS | +| 37 | 2026-10-01 | **`local_identifier` prefixes.** Unique within the DATASET (the schema's rule for `subject` and `session`; `ndi.dataset` search spans every session), and plate/worm numbers restart per source folder, so every identifier starts with the folder name without `foraging`, lower-cased (`ndi.setup.conv.haley.idPrefix`): `concentration`, `matching`, `mini`, `mutants`, `sensory`, `ecoli`. Sessions follow (`concentration_0001`; was `foragingConcentration_0001`). Numbers are four digits everywhere. | DECIDED | +| 38 | 2026-10-01 | **Subjects, per session:** behaviour plates (`concentration_plate0011`, one per distinct `plateNum`; a plate filmed twice -- an hourly continuation or a restart after a glitch -- is ONE subject with two epochs), patches (`concentration_plate0011_patch0007`, one per row of `lawnCenters`, in `closestLawnID` order), worms (`concentration_worm0451`; `wormNum` is unique within a folder and the same across a plate's videos). | DECIDED | +| 39 | 2026-10-01 | **Growth (cultivation) plates are subjects**, one per (strain, pick time) in a session, numbered within the session by pick time then strain: `mutants_0001_growth0001`. The source has no id for them; the strain and pick time are DATA (assertion, manipulation time), not built into the name, so correcting either never renames a subject. | DECIDED | +| 40 | 2026-10-01 | **E. coli plates and patches.** `rectangle` template: 12 patches in a 3 x 4 grid, numbered row by row. `none` template: one large patch when seeded (`OD600` 1, 200 uL, 20 mm; plate 103's 20 uL at `OD600` 0 is a patch of LB alone, relative density 0 -- the eLife paper, Methods: "A '0' density solution was prepared with just LB"), or, with `lawnVolume` 0, a blank plate with no patch. Which detected patch in which image is which grid slot is a later calculation (registering the images); until then `lawnAnalysis` rows are measurements of the plate. | DECIDED; numbering and the plate-level rows superseded by #62 | +| 41 | 2026-10-01 | **Mutants plate 1, 3 Nov 2023:** video 1 (07:35:52) is a failed start (0 frames, no file); 07:42:14 is its restart. | DECIDED (from the data) | +| 42 | 2026-10-01 | **Names and identifiers follow the eLife paper**, for a dataset that stands with it. The filmed plate is an ASSAY plate, the plate L4s are picked onto the day before is an ACCLIMATION plate (Methods: "Acclimation plates contained one large 200 µl patch [...]"). Identifiers are camelCase within a part, `_` between parts: `concentration_assayPlate0011`, `concentration_assayPlate0011_patch0007`, `mutants_0001_acclimationPlate0001`, `concentration_worm0451`, `ecoli_plate0042`. Subjects gain a display `name` (did-schema PR #80, `subject.name`): "Assay Plate 0011", "Patch 0007 on Assay Plate 0011", "Acclimation Plate 0001", "Worm 0451", "Plate 0042"; instruments `camera1` / `camera2` / `axiozoom1` / `spectramax1` are named "Camera 1", "Camera 2", "Axio Zoom.V16", "SpectraMax Plus 384". Names need not be unique and carry no study. | DECIDED | +| 43 | 2026-10-01 | **E. coli scope.** All 126 plates are subjects, analysed or not: the 33 unanalysed seeded plates are the low-density conditions (OD600 0, 0.05, 0.1) whose signal the paper says "could not be detected", and the blank plates (25, 26, 61, 90) and plate 103 (LB alone) are the "matched controls" the images were normalised against. All 1,575 image files are recordings: the 1,521 in `lawnAnalysis` plus 54 shared but unanalysed (`doImport` took only `lawnAnalysis`). The 2,150 `metaData` images with no file (brightfield, backgrounds, the 3,000 ms set of experiments 1-4, experiment 5's exposure sweep) are not recordings; each kept image notes its exposure and its background and brightfield `imageNum`. | DECIDED | +| 44 | 2026-10-01 | **Two camera-2 files on 6 May 2022** (`2022-05-06_15-09-24_2.mp4`, `2022-05-06_15-14-47_2.mp4`, concentration_0016) contain no plate: no epoch; the listing keeps reporting them. | DECIDED | +| 45 | 2026-10-01 | **Writing a session** (`'Write', true`, one or more sessions picked with `'Sessions'`): the session, its subjects, acquisition systems, epochs and recordings go into its V2 database in one transaction (`ndi.setup.conv.haley.sessionDocuments` builds them, `ndi.setup.V2.makeSessions` writes them). The session folder holds only `.ndi/`. The session's own `time_reference_id` is its UTC extent: the first recording's start to the last recording's end. | BUILT | +| 46 | 2026-10-01 | ~~One folder per epoch holding a hard link and a probe map~~ (dropped with the hard links, #32). The `epoch` document's `local_identifier` IS NDI's epoch id: the navigator reads it from there (#51). Each camera's `epoch_file_pattern` names its recordings' file names (`.*_1\.mp4`, `.*_2\.mp4`; the microscope `.*\.tiff`), and the navigator keeps only the bodies whose `filename` matches. | REVISED by #51 | +| 47 | 2026-10-01 | **The recording's `opaque_body` at import:** `format`, `filename`, `size_bytes`, `file_modified` (UTC), `content_hash` + `hash_algorithm` MD5 (`'Checksums'`, default true), and the file's location (#34). `description` names the file as the source table does (the `.avi`). The statement's `datum_type` (required when the value is in a body): a video's from VideoReader's `VideoFormat` (RGB24 / Grayscale -> `uint8`; RGB48 / Mono16 -> `uint16`; with `ReadVideos` false, `uint8` is assumed without opening the file), a TIFF's from its header; an unreadable file is skipped and reported. | BUILT | +| 48 | 2026-10-01 | **Times.** An epoch carries its UTC extent (`absolute_time_reference`, source time zone America/Los_Angeles) and, for a video, its `dev_local_time` extent from 0 (`relative_time_reference`, referent the session document). The recording statement points at a reference to the EPOCH (video: `dev_local_time` 0 to its duration; image: `during`) and at the epoch's UTC reference. A lawn clip's duration is known only when `ReadVideos` is true. | BUILT | +| 49 | 2026-10-01 | **Provisional, to check:** the recording statement's `variable` is the bare name `image intensity` (no ontology term yet); probe types (carried as the statement `method`, #51) are `brightfield-imaging` (cameras) and `wide-field-imaging` (microscope; the E. coli images are fluorescence); `acquisition_channels` carry no channels (the schema binds only ai/ao/di/do). **Reviewed with the user 2026-10-06, still provisional (the names wait for the ontology pass over every free-text term):** (1) `image intensity` is not the right name; replaced in that pass. (2) The method names the CONTRAST METHOD (OME's ContrastMethod/IlluminationType), from the paper's methods (eLife 103191): behaviour videos = brightfield (*"placed face down on a piece of glass suspended above an edge-lit backlight"*, PixeLink PL-B741F cameras); the patch "contrast" videos = oblique (*"a piece of dark cardstock was passed between the light source and the condition plate"*); E. coli images = fluorescence (*"a Zeiss Axio Zoom.V16"*, OP50-GFP), not bare "wide-field". Terms to be looked up in the ontology pass; NOT BUILT -- the code still writes `brightfield-imaging` / `wide-field-imaging`, and does not yet give the contrast videos their own method. (3) `acquisition_channels` stays EMPTY for cameras: it models acquisition wiring (ai/ao/di/do groups), not optics; the optical description lives on the recording, as OME and NWB (OpticalChannel) keep it per channel. A filter set, if one is recorded, goes in a shared `method_parameters` document (`method_parameters_id` from every image statement), or on the instrument if the cube was fixed hardware; excitation/emission are not in the paper, so nothing is recorded. GFP needs no field: it is a property of the bacteria, recorded by the OP50-GFP strain. | PROVISIONAL (reviewed 2026-10-06) | +| 32 | 2026-10-01 | **How NDI finds the raw recordings** when the session folder is separate from the raw data. ~~Hard links in the session folder~~ (dropped 2026-10-01: macOS refused them -- `ln: Operation not permitted` from MATLAB, even within one disk -- and they made every Mac user change a security setting). Now NDI reads a recording through its documents: decision #51. | DECIDED; REVISED by #51 | +| 50 | 2026-10-01 | **The study documents are minted in stage 3**, beside the sessions they group (each spec study's `source_folder` / `source_condition` already assigns days to it), instead of in stage 2. Still dataset-level documents built from the spec, `part_of` the dataset: `ndi.setup.V2.datasetMetadata` takes `'Studies'` "exclude" (stage 2) / "only" (stage 3, with the run's one `DatasetId`). | DECIDED (user, 2026-10-01); BUILT | +| 51 | 2026-10-01 | **NDI finds a recording through its documents** (the long-term design of #34, built instead of hard links). `ndi.file.navigator.bodies`, named by each acquisition system's `epoch_file_pattern` (software `ndi.file.navigator.bodies`), builds the system's epochs from the database: acquisition_system <- acquisition_channels <- recording statement <- opaque_body. Epoch files = each body's file at the location it records (the V2 database backend now opens a file recorded by location and not ingested, `ndi.database.fun.externalFileLocation`; anything else still errors); epoch id = the `epoch` document's `local_identifier`; probe map = the system's name, the statement's `method` as the probe type, and the statement's subject (the plate's document id). It scans no folder and writes nothing: no links, no probe-map files, no hidden epoch-id files. A recording whose file is missing on this computer is left out with a warning. The location is an absolute path: moving the raw data means updating the locations (as for any NDI document whose file is not ingested); uploading to NDI Cloud rewrites them. | BUILT | +| 52 | 2026-10-01 | **Stage 6, relations** (`ndi.setup.conv.haley.relationDocuments`), all `directed_relation`, child -> parent. A **patch `part_of` its plate** (assay or E. coli plate; BFO:0000050; no time -- it holds for the plate's whole life). A **worm `contained_in` each plate it was on, in turn** (RO:0001018 "contained in": located in the space another material entity encloses -- a worm on the agar of a closed dish; RO's "located in" is RO:0001025): its acclimation plate from the pick (`growthTimePicked`), its food deprivation plate (#53) from the move to it (`starvedTime`), its assay plate from the transfer T. Each window ends when the next one starts; the assay window ends when filming ends. **T is not recorded.** The eLife paper (Methods, pp. 22-23): single-density assays moved the worms by agar plug through "an empty NGM plate" to the condition plate, after the condition plate's 1 h at room temperature; multi-density assays moved them by eyelash pick "immediately prior to the assay plate being placed on the imaging setup", recording starting "once the droplet evaporated". It measures "the duration of time since transfer (i.e., the time elapsed in the experiment)" (p. 29), and the source's own `wormGrowth` / `lawnGrowth` run to `timeRecord`. So T = the plate's first behaviour video start, marked approximate (the assay window's start and the end of the window before it). **Where the worms went on before the 'contrast' (lawn) video, T is the lawn clip's start instead** (if within an hour before recording). Which plates: from the gap between each plate's lawn clip and its first behaviour video, per study and day (lawn first: hours in early Concentration, 6-10 min later; worms first: 0.1-1.5 min), confirmed by eye in the videos 2026-10-02 -- Concentration lawn first throughout; Mini worms first throughout; Matching days 1-3 (to 2023-04-04) lawn first except `2023-04-04_16-10-14_2`, days 4-5 worms first; Mutants and Sensory worms first. (`lawnRegistration` is a registration-fit score and `arenaOffset` the arena's position in the frame -- neither records the order; the analysis code, shreklab/Haley-et-al-2024.) Transfer methods: agar plug through an empty NGM plate (Concentration, Mini); eyelash pick into an S-Complete droplet (Matching, Mutants, Sensory). All of it is data in the spec's `transfer_protocol` (rules + named exceptions), applied by `ndi.setup.conv.haley.transferTime`. **Precision** (user, 2026-10-02): video times are exact; the pick time and T are approximate (the start flag); a window's duration is approximate whenever either of its ends is -- an `absolute_time_reference` is start + duration, so an approximate start cannot carry an exact end, and the exact end of filming is stated by the epochs' own references. `starvedTime`: exact for now (to confirm). One time reference per (plate, assay plate), shared by its worms (all moved together). No time where the source has none (an unfilmed plate, a missing pick time). Plates are not related to each other. The brief empty-NGM transfer plate is not a subject. | DECIDED (user, 2026-10-01: relation terms); T: DECIDED (user, 2026-10-02); BUILT **Ends (2026-10-02, needs did-schema CHANGE 6):** each window also records its `end`, with that end's own precision: the end of filming is exact, and an end at the next window's start (a transfer T) is approximate. So the assay window reads start ≈ T, end exact. **Order corrections (2026-10-02, from the lawn-clip-to-recording gap over 541 plates, checked by Jess in the videos):** Matching day 1 (2023-02-24) was worms first (days 2-3 lawn first); Concentration 2022-09-02 08:48:41 (both cameras) was worms first -- the contrast video was taken late, 2.6 s before recording. Both are in the spec's `transfer_protocol` (a dated rule and two exceptions). **Bounds (2026-10-02, CHANGE 7 `tolerance {minus, plus}`):** T is only ever early of the video it is read off, so its bound is [minus 0] with minus = the spec's `max_before_seconds` (300 s; Jess: 5 min is reasonable, 10 big), tightened on a lawn-first plate to the time from the lawn clip's end to the first behaviour video when that is shorter (measured over 541 plates: lawn-first days 5-10 min, early Concentration days up to 5.4 h where all lawn clips were filmed first). The pick time is read off a clock, possibly analog (e.g. an Apple Watch face), and written to the minute: `hand_written_tolerance_seconds` [60 60], with source_value at minute resolution. A window's duration bound is the sum of its ends' (minus = start.plus + end.minus). Video times are exact. | +| 53 | 2026-10-01 | **Food Deprivation Plates** (the paper's "3 hr of food deprivation"; Mutants and Sensory, `starvedTime` / `starvedDuration`): food-deprived worms were moved from their acclimation plate to an unseeded plate before the assay, so that plate is a subject, one per (session, strain, `starvedTime`), numbered within the session by that time then strain: `mutants_0001_foodDeprivationPlate0001`, "Food Deprivation Plate 0001" (name chosen to match the paper's "food-deprived"; it does not name the plate, and its "empty NGM plate" is the brief transfer plate). `subjectList` gains columns `deprivation` and `worms_placed` (was `pick`). | DECIDED (user, 2026-10-01); BUILT **starvedTime is approximate (2026-10-02):** read off a clock and written to the minute like the pick time (all seven have seconds :00, odd minutes, durations 2:58:48-3:21:26 -- recorded, not derived from timeRecord): [60 60] s, minute-resolution source_value. **Two corrections (2026-10-02, from the source profile):** (1) **Sensory was not food-deprived**: all 38 Sensory rows have `starvedTime` NaT and `starvedDuration` 00:00:00, so food deprivation is 7 Mutants plates only (`plateNum` 8, 20, 36, 43, 55, 64, 83) -- `subjectList` already makes a plate only where `starvedTime` is set, so no code changes. (2) **`starvedDuration` is DERIVED, not recorded** (the opposite of what this row said): on all 7 rows it equals `timeRecord - starvedTime` exactly (e.g. 07:53:00 + 3:06:21 = 10:59:21), so it is not read. | +| 54 | 2026-10-02 | **Stage 8, manipulations** (plate preparation, food deprivation). **Transfer = a `term_manipulation` per move** (Jess, 2026-10-02), on the worm cohort (#55): growth -> acclimation plate (the L4 pick), [acclimation -> food deprivation plate,] -> assay plate, each sharing its `contained_in` relation's time reference, with `method` the transfer method where known (the assay transfer from the spec's `transfer_protocol`: agar plug through an empty NGM plate, or eyelash pick into an S-Complete droplet). The relation says where the worm was; the manipulation says how it was moved. Stated once per cohort with `distributive` true (#55), not per worm. Every hand-written time is to the minute -- profiled over the five C. elegans studies, 0 rows with non-zero seconds in any `*Time*` column except the 10 estimated `timeRoomTemp` below -- so each is approximate, [60 60] s, with a minute-resolution `source_value`; `timeRecord` keeps its seconds and stays exact. **(a) Pouring = a `dose_manipulation` of the plate** with the agar `formulation` (NGM, or NGM without peptone; the recipe varies by experiment), the poured volume when known (else no amount). Timed by `timePoured` where it exists (E. coli only); for the C. elegans plates, which have no pour time, by a `relative_time_reference` with relation `intervalBefore` the plate's seeding time (no clock value is invented). **(b) Seeding = a `dose_manipulation` of the plate**, one per patch: `variable` the patch's bacterial strain: OP50 on every C. elegans plate, OP50-GFP on every E. coli plate (as `doImport` had it; confirmed by Jess 2026-10-02), `method` seeding, the patch's volume (`lawnVolume`), formulation = that day's seeding suspension. One suspension stock was made per day of seeding and reused across that day's plates: a `formulation` of the strain (`strain`; did-schema PR #84 lets an ingredient be a strain) and Luria broth (made in the institute: a `formulation`, recipe to come); an OP50-GFP stock (E. coli study only) also holds ampicillin (a `chemical`; the plasmid pFPV25.1 is ampicillin resistant, per the spec's strain entry; confirmed by Jess). Amounts and concentrations are left BLANK until the recipes are found (Jess, 2026-10-02). When filled, concentration: CFU as `particles_per_liter`, OD600 kept as the source unit and value. Whether each OD600 was a dilution of the day's stock (then a formulation of its own) is to confirm. No `derived_from` (plate -> agar, patch -> suspension): the doses carry it. **(c) Temperature = a `temperature_manipulation` of the plate at each move**: into the cold room ~4 C, out to room temperature ~25 C (values from the paper, per Jess; to quote the Methods sentence). The value is the temperature the plate was moved INTO (`variable` ambient temperature), not heat applied; approximate, no tolerance (no bound is known). Each is a window: start = the move, end = the next move or the pick/recording. Seeding to cold room is at room temperature with no move, so it gets none. Growth plates: `growthTimeColdRoom` (4 C), `growthTimeRoomTemp` (25 C); assay plates: `timeColdRoom`, `timeRoomTemp`. No time, no manipulation: Concentration's growth plates lack `growthTimeSeed`/`growthTimeColdRoom` on 144 of 193 rows (before 28 Apr 2022). **(d) The 10 estimated `timeRoomTemp`** (Concentration; exactly the rows where `timeRoomTemp == timeRecord - 1 h`, and exactly the 10 with non-zero seconds) are listed by name in the spec's corrections, marked approximate with a description saying they were estimated, and no tolerance. **(e) Food deprivation = a `term_manipulation` of the worm cohort** (#55; `distributive`; redundant with the `contained_in` relation, kept for lookup), sharing that relation's time reference: `starvedTime` (+-60 s) to the transfer T. | DECIDED (user, 2026-10-02); recipes and antibiotic to come; NOT BUILT | +| 55 | 2026-10-02 | **Stage 7, assertions; subject types; worm cohorts** (`ndi.setup.conv.haley.assertionDocuments`; did-schema PR #84: `subject.type`, `distributive`, both signed by Jess 2026-10-02). **Subject types** (`subject.type`, the coarse kind, a field rather than an assertion -- species cannot tell an organism from a culture): worm organism; cohort group; every plate (assay, acclimation, food deprivation, E. coli) material; a patch with bacteria culture (a lawn: a population whose members are never subjects); a patch of LB alone (E. coli OD600 0) material. **Worm cohorts, superseding #52's per-worm relations**: the worms of one assay plate were moved together, so they are one `group` subject, `_worms`, "Worms on Assay Plate 0011"; each worm is `member_of` it (no time), and the plates they were on (`contained_in`, with #52's windows) and how they got there (stage 8's transfers) are stated once, on the cohort, with `distributive` true (they hold of each worm). Per worm: 1 relation instead of 2-3, and stage 8 needs none. Not nested: a cohort is `contained_in` each plate in turn rather than being a member of a cohort per holding plate, because `subjectList` gives each assay plate exactly one acclimation and at most one food deprivation plate. Queries for one worm follow `member_of` to its cohort (the same walk the exclusion propagation needs; not built yet). **Assertions** (`term_assertion`): each cohort's `species` (from the spec's strain entry, e.g. NCBITaxon:6239) and `strain` (the strain's name as the term, `strain_id` -> the stage-2 strain document), both `distributive`; each seeded patch's `species` (NCBITaxon:562) and `strain` (OP50 on the C. elegans plates, OP50-GFP on the E. coli plates, #54); an excluded assay plate's `inclusion in analysis` = `excluded` (#2). Exclusion is per plate: profiled 2026-10-02 over all five C. elegans studies, no plate has some videos excluded and others not (Concentration 4 of 193 plates excluded, Mini 4 of 234, the others 0); only `doImport`'s tags count (the source `exclude`), so a tableOfContents `include = no` day gets no tag (its session description says it). E. coli plates are not excluded: one with no signal to analyse is not an exclusion (#43). **Terms with no ontology node yet**, all to be looked up together: `inclusion in analysis` / `excluded`, `ambient temperature`, the subject types, the methods (`seeding`, the transfer methods). **Schema guard**: `subject.type` and `distributive` are written only when the schema on DID_SCHEMA_PATH declares them (`ndi.setup.V2.schemaHasField`), so the import still runs on a did-schema without #84. | DECIDED (user, 2026-10-02); BUILT | +| 56 | 2026-10-02 | **Stage 8 details, from Jess's Benchling protocols** (230510_seeding_S-complete, 231109_OP50_cultures, 231229_OP50_OP50-GFP_cultures, 240102_seeding_concentration, 240102B_NGM_plates, 230523_foraging_matching) and the source tables. Supersedes the open items of #54 where it says so. **Agar (the pour dose)**: NGM, 3% agar, per 730 mL Milli-Q water: 22.5 g agar (EMD 101614), 2.25 g NaCl (Fisher S271-1), 1.875 g peptone (BD 211677; left out where the source `peptone` says `without`), autoclaved, cooled to 60-65 C, then 18.8 mL 1 M KPO4 pH 6.0, 750 uL 1 M MgSO4, 750 uL 1 M CaCl2, 750 uL 5 mg/mL cholesterol in ethanol. **25 mL per 100 mm dish, every plate** (assay, acclimation, food deprivation, E. coli, and Mini's; Jess). Peptone is per plate from the `peptone` column (Matching and the E. coli study have both). **E. coli pour times are real** (`timePoured`, 4 batches): 23 Mar 2022 (26 plates, expNum 5; NOT a typo -- its cold-room time 24 Mar 2022 and 27:02 drying agree, and the plates were seeded 28 Apr 2022), 3 Oct 2023 (10, without), 8 Nov 2023 (56, with), 16 Nov 2023 (34). **Seeding suspensions**: one culture per batch (a single colony in 150 mL LB, overnight at room temperature, then 2-6 weeks at 4 C); OP50-GFP grown in LB with **carbenicillin 100 ug/mL** (not ampicillin: the plasmid gives ampicillin resistance; the paper agrees), its LB blank and diluent too. Each seeding day: master (pelleted, resuspended) -> a stock at OD600 ~10 -> one dilution per OD600 (stock + diluent; OD600 0 = diluent alone), each its own `formulation`, the next made from the one before. The stock's measured OD600 is in the source (`OD600Real`, per row) and so is `CFU`: the stock's concentration is `OD600Real`, each dilution's is nominal x `OD600Real` / 10 (nominal kept as the source value). No Benchling transcription is needed. **Transfer**: every move to an assay plate is two steps -- an agar plug onto an empty NGM plate (about a minute, to shed bacteria), then onto the assay plate by agar plug or by eyelash pick into an **S-Complete** drop (worms die in water). One `term_manipulation` per move, on the cohort; `method` names the second step; `method_parameters` carry the cleaning step and the drop. **Room temperature** is 20 C (the paper: "stored at 20C until the experiment"; recordings averaged 21.99 C, from `temp`, a stage-9 observation), approximate. **Answered 2026-10-02 (Jess)**: (a) the diluent is **S-Complete for every Matching seeding** (the master washed and diluted in it, as 230510), LB for the other studies; a Matching OD600 0 patch is S-Complete alone. (b) **Worms were always kept in the 20 C incubator**: an acclimation or food deprivation plate goes to 20 C (`temperature_manipulation`, approximate) when worms are placed on it (`growthTimePicked`, `starvedTime`), in every study. (d) **plate 90's peptone label is wrong**: a named correction in the spec, `without` (its 16 Nov 2023 batch). (e) **no correction**: Mini's 6 rows with `arenaDiameter` 0 are plates never filmed, so nothing was measured; stage 9 writes arena geometry only where it was measured. (c) **the E. coli plates' agar is the plain NGM recipe**: Jess expects carbenicillin was added, but the NGM entries for the four E. coli pours do not record it, so it is not listed as an ingredient; each E. coli pour dose's `notes` says it was probably added (the protocol requires an antibiotic for plasmid-carrying bacteria) and is unrecorded. Was open: whether the agar of the E. coli plates had carbenicillin (the NGM protocol: "If bacteria has a plasmid (e.g. OP50-GFP), it requires that an antibiotic be added to the plates"); to read in the NGM entries for the four E. coli pours, 23 Mar 2022, 3 Oct 2023, 8 Nov 2023 and 16 Nov 2023. **Where the recipes live, and their source (Jess, 2026-10-02)**: every chemical and recipe is a dataset-level document (#20; written at stage 13 like the instruments), and a session's doses point at them: the chemicals; NGM with and without peptone (240102B, itself adapted from WormBook); LB and S-Complete, made by the Salk media kitchen to the standard recipes, **written out from WormBook** (Stiernagle 2006, "Maintenance of C. elegans", doi:10.1895/wormbook.1.101.1, taken as the source of truth): S-Complete = 1 L S Basal (5.85 g NaCl, 1 g K2HPO4, 6 g KH2PO4, 1 mL 5 mg/mL cholesterol in ethanol, water to 1 L) + 10 mL 1 M potassium citrate pH 6 (20 g citric acid monohydrate, 293.5 g tri-potassium citrate monohydrate per L) + 10 mL trace metals (1.86 g disodium EDTA, 0.69 g FeSO4.7H2O, 0.2 g MnCl2.4H2O, 0.29 g ZnSO4.7H2O, 0.025 g CuSO4.5H2O per L) + 3 mL 1 M CaCl2 + 3 mL 1 M MgSO4; LB = 10 g tryptone, 5 g yeast extract, 5 g NaCl per L (WormBook's, the Lennox form). Each is `documented_by` a `web_resource` for that chapter (did-schema PR #84 lets a formulation be the child); the media kitchen is an `organization` in the spec, `suborganization_of` Salk. The cultures, per-seeding-day stocks and dilutions are dataset-level too: one seeding day's suspensions serve plates filmed on later days and in other sessions. **Part A BUILT (2026-10-03)**: the spec's `chemicals` (20) and `formulations` (9: 1 M potassium phosphate pH 6, NGM with and without peptone, S Basal, trace metals, 1 M potassium citrate pH 6, S-Complete, LB, LB with carbenicillin 100 ug/mL), built by `ndi.setup.V2.datasetMetadata` with the other dataset-level documents; LB and S-Complete also point at a `product` from the media kitchen (a formulation may list its ingredients AND its product). The 1 M potassium phosphate recipe (35.6 g K2HPO4 + 108.3 g KH2PO4 per litre, pH 6.0) is WormBook's as quoted by a web search (the page itself is blocked from the build container), so it is to check against the chapter. | DECIDED (user, 2026-10-02); NOT BUILT | +| 57 | 2026-10-03 | **Stage 8 part B, the seeding suspensions: what the source's `OD600Real` and `CFU` are** (refines #56; from Jess's seeding-day Benchling entries 230327_seeding_concentration, 230327_seeding_S-complete, 230510_seeding_S-complete and 240102_seeding_concentration, and the culture entries 231109 and 231229). **Both are measured on the seeding day, on that day's OD600 10 solution**, not on the culture: each seeding-day entry makes a master from the culture, measures it, makes the OD600 10 solution, measures that (`OD600Real`: 10.4 on the 230327 LB entry = Mini 2023-03-27, 9.9375 on the 230327 S-Complete entry = Matching 2023-03-27, 10.4875 on 230510 = Matching 2023-05-10, 9.8875 on 240102 = the E. coli seeding of 2024-01-02), and on some days plates it to count CFU. The culture entries record only dates (inoculated, checked next day), never an OD600 or a CFU. **Identical values on the same day mean the same solution** (Jess): Mini and Matching on 2023-02-17 (OD600Real 10.1000, CFU 1117) used one solution. **The CFU conversion is the lab code's** (`reportMetaData.m:56-57`, `doImport.m:156-157`): raw `CFU` x 2e7 = CFU per mL of the OD600 10 solution, checked against two entries: 230327 LB counts 846 colonies = 8.46e6 CFU in 0.5 uL (= 1.692e10 per mL = 846 x 2e7), and 240102 gives 1e7 in 0.5 uL (= 2e10 per mL = its `CFU` 1000 x 2e7). So `particles_per_liter` = `CFU` x 2e10, the raw value kept as the source value. **Measured per source table, 2026-10-03** (Jess, MATLAB): 33 (study, seeding day) pairs; 0 with more than one `OD600Real` or `CFU`; 0 rows with no `timeSeed` or `OD600Real`; 6 with no `CFU` (Mini 2022-12-27, 2023-01-06, 2023-02-03, 2023-08-16; Matching 2023-05-10 -- 230510 has no CFU section; and Matching 2023-03-27 after the correction below). Growth (acclimation plate) seeding: 23 days, 21 of them also an assay seeding day of the same study; Concentration has only 2 (144 of its 193 rows have no `growthTimeSeed`). **Two named corrections** (spec `corrections`; Jess): Matching 2023-03-27 `CFU` 846 -> none (the S-Complete entry measured no CFU; 846 is Mini's, copied); E. coli 2022-04-28 `CFU` 457 -> none (Concentration's OP50 count; an OP50-GFP suspension cannot share it). **The cultures are not identifiable from the source**: no column names a bottle, and the CFU belongs to the day's solution. The entries do name it (e.g. "OP50, 3/25/2023" for both 230327 seedings; the 240102 master came from an OP50-GFP culture inoculated 30 Dec 2023, three days before) but they are not in the data. **No master document (option B, Jess 2026-10-03)**: the OD600 10 solution lists the strain directly; the master's own reading is only in the Benchling entries, and a master document would carry nothing from the data. Formulations have no notes field, so a suspension is identified by its values and by the dated seeding doses that point at it (stage 8 part C). **Acclimation plates** (the paper: "one large patch of 200 µl of OD600 = 1"; the LB seeding entries have a 200 uL "growth condition lawn" step, the S-Complete ones do not): all seeded alike, in LB (Jess: Matching's too), so a growth seeding uses that day's LB solution, its own study's if it has one, else another study's (Matching's days have Mini's). **When studies sharing one solution have different diluents** (2023-02-17, Mini LB and Matching S-Complete; which it was is not recorded) the solution lists no diluent. **Built** (`ndi.setup.conv.haley.seedingSuspensions`, run in stage 2 so the suspensions are built and validated with the spec's formulations; the spec's `seeding` section names each study's strain and diluent): per (strain, day, OD600Real, CFU) one OD600 10 solution -- the strain at OD600Real (source unit OD600; `particles_per_liter` = CFU x 2e10 when measured), then the diluent -- and one dilution per nominal OD600 the day used (assay patches and acclimation plates; not 0, the diluent itself, nor 10, the solution itself) -- the solution as `volume_fraction` nominal/10 (source value the nominal OD600), then the diluent; no `type` (corrected 2026-10-03, Jess: the signed `formulation.value.type` rule leaves a one-off mixture such as a day's dilution untyped -- a type names a standard recipe, and "bacterial suspension" only repeated the strain ingredient). Keys `suspension__[_a|_b]_od`. It also returns an index (study, day, nominal OD600, assay or growth -> formulation key) for part C's seeding doses, and reports a growth seeding day with no solution (on the profile: one Mutants day). | DECIDED (user, 2026-10-03); BUILT (stage 8 part B) | +| 58 | 2026-10-03 | **Stage 8 part C BUILT: the manipulations** (`ndi.setup.conv.haley.manipulationDocuments`, decisions #54, #56, #57; settings in the spec's `preparation` section, corrections in `corrections`). `subjectList` now carries a `prep` struct per plate and patch (seeding, cold room, room temperature, pour times; peptone; OD600; volume), with the spec's per-plate corrections applied (plate 90's peptone -> without). Per session: **pour** `dose_manipulation` of every plate, 25 mL of NGM / NGM without peptone (`formulation_id` the stage 2 recipe), at E. coli `timePoured`, else a `relative_time_reference` `intervalBefore` the plate's seeding (a food deprivation plate, never seeded: before the worms went on); an E. coli pour's `notes` say carbenicillin was probably added but is unrecorded. **seeding** `dose_manipulation` per patch (its `lawnVolume`) and per acclimation plate (200 uL), `formulation_id` that day's suspension via part B's index, at the seeding time. **temperature** `temperature_manipulation` per move, a window to the next move: assay plate cold room (`timeColdRoom` -> `timeRoomTemp`, 4 C), room temperature (`timeRoomTemp` -> end of filming, 20 C); acclimation plate cold room (`growthTimeColdRoom` -> `growthTimeRoomTemp`), room temperature (-> the pick), incubator (the pick -> the last cohort's move off it, 20 C); food deprivation plate incubator (`starvedTime` -> the last move off it); E. coli plate cold room after pouring (`timePouredColdRoom` -> seeding), cold room after seeding (`timeSeedColdRoom` -> `timeRoomTemp`), room temperature (from `timeRoomTemp`, no end). **transfers** `term_manipulation` per cohort move at its `contained_in` relation's own time reference (stage 6 now also returns each assay plate's T, end of filming and protocol method): value the plate kind moved to (`acclimation plate` / `food deprivation plate` / `assay plate`), method `picking` onto the acclimation plate, none onto the food deprivation plate (not recorded), and from the transfer protocol onto the assay plate (`transfer by agar plug` / `transfer by eyelash pick`, with method parameters `cleaning step` and, for the eyelash pick, `droplet` S-Complete). **food deprivation** `term_manipulation` on the cohort at the food deprivation window. Cohort statements are `distributive`. No time, no manipulation. **Reviewed with Jess 2026-10-03**: (1) the pour's variable names what was poured, `NGM agar` or `NGM agar (no peptone)` by the plate's peptone (a term with no node until the ontology lookup); (2) acclimation and food deprivation plates are NGM with peptone -- peptone was left out only to stop bacterial growth, in Matching (the paper: "poured without Bacto peptone (BD 211677) to prevent bacterial growth") and the E. coli plates that parallel it; (3) an E. coli plate left the cold room just before seeding -- the paper seeds "onto cold, low-moisture nematode growth medium (NGM) plates" -- so the cold room window after pouring ends at the seeding, approximate, up to 5 min earlier (Jess) plus the seeding time's own +/- 60 s: bound [360 60] s; (4) **variable / value / method, settled 2026-10-03 (Jess)**, by one test -- the statement reads "the subject's VARIABLE was set to VALUE, by METHOD": the variable names a property (never the act), the value its state, the method how it was done, and only when that adds something. So: a transfer is `location` = the kind of plate moved to (`acclimation plate` / `food deprivation plate` / `assay plate`; the plate itself is the contained_in relation's parent), method `worm picking` onto the acclimation plate, `agar plug transfer` onto the food deprivation plate (Jess: most likely), `agar plug transfer` or `eyelash picking` onto the assay plate; food deprivation is `food availability` = `none` (no method: the move that caused it is its own transfer); a temperature manipulation's method is how it was imposed -- `refrigeration` (cold room), `ambient exposure` (the bench), `incubation` (the 20 C incubator) -- which is what tells room temperature from the incubator. Proposed as a tenet (the variable/value/method rule), not yet written to the tenets. The food deprivation plate's pour is anchored before the worms went on. **The 10 estimated `timeRoomTemp`** are Concentration's first day (expNum 1, 1 Feb 2022), plates 1-10, one row each (Jess, 2026-10-03, from the data: exactly the rows where `timeRoomTemp == timeRecord - 1 h`); the lab's `getExperimentInfo.m:270` fills `timeRoomTemp` in that way when none was recorded. All 10 are listed in the spec's `corrections`. `subjectList` still reports every assay plate whose `timeRoomTemp` is exactly a recording's start minus 1 h and that no correction lists (`roomTempLooksEstimated`); a listed one (a `corrections` entry with folder, plate, column `timeRoomTemp`, reason) makes that time approximate with no bound (`utcReference` 'estimated') and puts the reason in the room-temperature manipulation's `notes`. | BUILT 2026-10-03 | +| 59 | 2026-10-03 | **Stage 9, observations: ambient temperature and relative humidity** (`ndi.setup.conv.haley.observationDocuments`; did-schema PR #86 for `humidity`). The source's `temp` and `humidity` (experimentInfo, per behaviour video) were read by the lab's **temperature probe** at the rigs, and the value noted is its **mean over the hour of recording** (Jess, 2026-10-03: "I think"); the paper reports their means over all experiments (21.99 C, 51.39 %). Profiled 2026-10-03 (Jess) over the five C. elegans studies: no zeros; NaN only on Mini's 6 never-filmed plates; ranges temp 19.35-23.90 C, humidity 19.50-68.44 %; **0 of 597 plates** have videos that differ (Concentration 193, Mini 234, Matching 50 over 60 rows, Mutants 84 over 93, Sensory 36 over 38). So: **one reading per filmed assay plate**, a `temperature_observation` (variable `ambient temperature`, the manipulations' term; celsius, source unit C) and a `humidity_observation` (variable `relative humidity`; `percent_relative_humidity`, no source unit: it is the stored unit), subject the assay plate, `instrument_id` the probe (a new spec instrument, `temperature_probe_1`, make and model not recorded), timed by the plate's recordings: the video's own UTC reference when filmed once, else an exact window from the first video's start to the last one's end. `notes`: the probe's mean over the recording, noted by hand. No method (T16: the instrument says how). A plate whose videos disagree gets none (reported); a plate whose video is not on disk gets none (its time is the recording's). The E. coli study has no such columns. Humidity is written only when the schema in use has `humidity`. **Fixed with it (stage 5):** a recording's `instrument_id` was always empty -- `sessionDocuments` looked the camera up as `camera1` / `axiozoom1`, but stage 2 keys its ids by spec key (`camera_1`, `axio_zoom_1`); now tested. | DECIDED (user, 2026-10-03); BUILT | +| 60 | 2026-10-04 | **Stage 10 part A, each behaviour video's geometry** (`ndi.setup.conv.haley.geometryDocuments`), from experimentInfo's per-video columns, which the lab's analysis package computed from the video's first frame (`getForagingInfo`, github.com/shreklab/Haley-et-al-2024). Per behaviour video on disk, all about its assay plate, each a calculation whose input is the recording: **a `coordinate_system`** on the recording (origin the image's upper-left corner; x along the image horizontal axis, positive right; y along the vertical axis, positive down; one pixel = 1/`scale` mm, approximate, since `scale` is the arena's nominal diameter over its mask's size in pixels -- Jess); **a `label_calculation` per mask**, a bool per pixel in an ingested `sampled_body` keyed [image vertical position, image horizontal position] in metres with pixels as the source unit: `arenaMask` (`arena region`, method parameter the nominal arena diameter), `refMask` (`reference mark region`; foragingMini's arenas have no mark and its masks are empty, so none), `lawnMask` (`bacterial lawn region`, method parameters the nominal patch diameter and spacing, method `patch detection by ` when every patch shares one, else none and the notes list them); **an `item_calculation`** for `lawnClosest` (each pixel's nearest patch, a 0-based index into the plate's patch subjects in lawnCenters order, #38; did-schema PR #87, whose build had `item.value` required, contradicting its signed text -- fixed there); **a `score_calculation`** for `lawnRegistration` (the fraction of the arena that overlaps once the lawn clip is registered onto the video, 0-1), its inputs both recordings. Not stored: `lawnClosestOD600` (the closest map with each patch's OD600, which the seeding documents give) and `firstFrame` (frame 0 of the video). **Run environment** (Jess, 2026-10-04): software = the analysis package (spec `haley_analysis`, its commit as the version), interpreter = MATLAB 2024a, operating system = macOS (spec `macos`, version not recorded). **Bytes are ingested** (DID-matlab #215, merged 2026-10-04): each mask and map is a temporary gzip of its raw bytes (column-major, little-endian; format application/octet-stream, compression gzip, MD5) that the database copies into /.ndi/files on write and then deletes. **Terms are names only** (no ontology lookup from here, Jess): `image upper-left corner`, `image horizontal axis` / `image right`, `image vertical axis` / `image down`. Settles #4 for the masks and closest map; the E. coli fluorescence images are part C. | DECIDED (user, 2026-10-04); BUILT | +| 61 | 2026-10-04 | **Stage 10 part B, the worm tracks** (`ndi.setup.conv.haley.trackDocuments`), from each folder's `midpoint.mat` (and `head.mat` / `tail.mat` in foragingMini), the tables the lab's analysis package made from the WormLab exports (`analyzeWormLabTracks`). The keep list (Jess, 2026-10-03): per worm, body part and behaviour video, all about the worm, each keyed by the **0-based video frame** (the source's 1-based `frameNum`), each in an ingested `sampled_body` (#60): **positions** (`position_calculation`, ` position`, in the video's coordinate system from part A, pixels from the image's upper-left corner = the source's 1-based pixel coordinates less half a pixel, the same pixel-centre convention as the masks' keys; method `tracking by WormLab`; input the recording). **The track is kept as the analysis package left it and `noTrack` is not stored** (Jess, 2026-10-03: filling gaps of at most 10 s by linear interpolation is a curation step like WormLab's own hand-tuned ones): the gap fill is stated as method parameters (`gap filling` linear interpolation, `longest gap filled` 10 s); longer gaps and positions off the image stay NaN. **Speed** (`velocity_calculation`, `velocitySmooth`, um/s -> m/s; method parameters the 4 s window and the 600 um/s step cut-off; input the positions). **Distance to the nearest patch edge** (`length_calculation`, `distanceLawnEdge`, pixels -> m via the video's pixel size, positive on a patch, negative off it; inputs the positions and the lawn mask). **Nearest patch** (`item_calculation`, `closestLawnID`, a 0-based index into the plate's patch subjects; inputs the positions and the nearest-patch map; only when the schema has `item`, did-schema PR #87). On the frames the source marks `noTrack`, `analyzeWormLabTracks` reads `closestLawnID` at pixel (1, 1), so those frames hold the body's fill value (255) rather than a meaningless patch -- the one use made of `noTrack`. Not stored (computed from the positions by a stated rule): `time`, `timeOffset`, `distance`, `velocityEuclidean`, `pathAngle`, `turn`, `distanceNeighbor1-3`, `distanceArenaEdge`, `closestOD600`, `nearArena`, `outOfBounds`. Run environment as #60 (the analysis package, MATLAB, macOS). | DECIDED (user, 2026-10-03); BUILT | +| 62 | 2026-10-05 | **Stage 10 part C, the E. coli patch profiles** (`ndi.setup.conv.haley.ecoliDocuments`), from `ecoli/bacteria.mat` `lawnAnalysis` (one row per patch detected in an analysed fluorescence image) and each image's `ecoli/mask/.png`. **Which detected patch is which patch subject** (supersedes #40's "measurements of the plate" and its "numbered row by row"; Jess, 2026-10-05): an image's rows are its detected patches in `bwlabel` order of its mask -- checked by Jess over all 1,521 analysed images, region count = row count in every one (0 disagree); the mask is a logical image, 2208 x 2752. A `rectangle` plate's 12 patches sit in 3 staggered rows of 4 (image 2's centres: rows at y 600 / 1100 / 1600 px, the middle row half a step right). Jess: the numbering is "probably the auto output of bwlabel"; `bwlabel` numbers by each region's first pixel, column by column, so two patches sharing a column can swap between images (on image 2 the bottom-left comes first). So the patch SUBJECTS are numbered as `bwlabel` numbers an IDEAL grid: left to right, patches sharing a column top to bottom; each image's 12 detected patches are sorted into 3 rows (y) and 8 staggered columns (x, a quarter of the in-row step apart) and take those numbers. A one-patch plate's one detected patch is its one patch. Any other image (another count, rows not cleanly apart, not 8 columns) is not matched: its per-patch values are left out and reported, its mask still stored. The labelling is done without the Image Processing Toolbox (`private/labelRegions`, `graph`/`conncomp`, the same order as `bwlabel`). **Documents**, per analysed image, from its recording: the image's `coordinate_system` when `metaData` gives its `scale` (as #60; the mask keys are then metres, else pixels); the plate's `bacterial lawn region` mask (`label_calculation`, bool, ingested; method `patch detection by threshold`, method parameter the 0.5 mm margin `analyzeLawnProfiles` adds; inputs the image and its background image when recorded); for a matched image, the plate's `nearest patch` map from `ecoli/closest/.png` (`item_calculation` over the patch subjects, when the schema has `item`). Per matched detected patch, about its patch subject (inputs the image and the mask; notes say it was matched by position): `intensity_calculation` `patch border amplitude` (borderAmplitude), `patch centre amplitude` (centerAmplitude), `patch peak intensity` (yPeak), `patch edge intensity` (yOuterEdge), background-normalised arbitrary units; `length_calculation` `patch edge to intensity peak` (xPeak, mm -> m); `score_calculation` `patch circularity` (0-1). Not stored: borderCenterRatio, meanAmplitude, FWHM, xHalfMax*, yHalfMax, lawnRadius (#60's keep list). **Deferred to a later pass:** the profile curves (`lawnProfiles.distances` / `pixelValuesNormalized`) and the fitted background images (`background`) are not in `bacteria.mat`; they are in `analyzeGFP_24-04-04.mat` (119 GB, on the lab's shared drive Jess/foragingGFP), which waits for a download (Jess, 2026-10-05). **Revised by #64:** the image these documents are calculated from is the background-normalised image, not the recording; the curves are added. | DECIDED (user, 2026-10-05); BUILT; REVISED by #64 | +| 63 | 2026-10-05 | **Stage 11, the encounters** (`ndi.setup.conv.haley.encounterDocuments`), from `celegans/encounter.mat` (21,543 rows; one per visit of one worm to one patch, plus an `id` 0 row per worm for the gap before its first encounter), made by the lab's `analyzeEncounters` / `labelEncounters`. **Model** (Jess, 2026-10-05; did-schema PR #87, signed: "Repeated events" in T2, `key_id` in T14): per worm, a LIST statement -- `time_calculation` `encounter onset`, one entry per encounter -- and one statement per measured quantity, each with one value per encounter, whose `encounter` key takes its positions from the list (`key_id` / `positions_from`; until both did-schema #87 and DID-matlab #217 are in, the same positions are listed inline). Rejected: one set of documents per encounter (about 20x the documents), an `encounter` data type bundling the values (T12), a repeating time reference. **One worm on one plate, not one video:** `analyzeEncounters` joins a worm's videos in order, so `enter`/`exit` are rows of that joined track and an encounter can cross videos; every time is in seconds from the start of the plate's first recording (the video's start + (frameNum - 1) / frame rate, the row located in `midpoint.mat`), under the plate's recording window (stage 9's, else the video's own reference). **Kept** (Jess's rule: hard values in a paper figure; `label` added back by Jess): the patch (`item_calculation` `patch in contact`, lawnID -> the plate's patch subject, #38, with the exit times as `offset`); `velocityOn`, `velocityOnMin`, `velocityBeforeEnter`, `velocityAfterEnter`, `velocityOff` (um/s -> m/s); `decelerate` (um/s^2 -> m/s^2, `acceleration_calculation`); `timeSlowDown` (s, `time_calculation`); `distanceOnMax`, `distanceOffMax` (mm -> m); `exploitPosterior`, `sensePosterior` (`score_calculation`, probability 0-1); `label` (`label_calculation` `encounter type`, method parameters the thresholds 0.5 / 0.5 / 0.05; `unlabelled` where the source left it blank). The off-patch values describe the gap AFTER each encounter (the source's row k); the `id` 0 row's are their own two statements (`... before the first encounter`). Inputs: the worm's midpoint speed and patch-edge distance tracks (#61) for each of the plate's videos, plus the list. **Not stored:** `duration`, `timeEnter`/`timeExit` (time since transfer), `censorEnter`/`censorExit` (from the frames and the videos' lengths), the plate/strain/growth columns (stages 4-8); `borderAmplitude`/`borderAmplitudeGrowth` are stage 12. | DECIDED (user, 2026-10-05); BUILT | +| 64 | 2026-10-05 | **The E. coli images, and the profile curves** (`recordingList`, `ecoliDocuments`; reader `private/lawnProfileReader` / `lawnProfileRead`). **What the TIFFs are**, found with Jess from `analyzeGFP_24-04-04.mat` (`whos`: `data` 3725 cells = the images `getMetaDataCZI` read from the `.czi` files with Bio-Formats, series 1 plane 1; `lawnProfiles` 1 x 1575; `background` 114 smoothed images; `backgroundInfo` 114 x 3) and the analysis package at its cited commit: `ecoli/images/.tiff` is NOT the image as acquired. It is `analyzeLawnProfiles`' `imageNormalized` rounded to uint16: the raw image, saturated pixels filled by linear interpolation, times max(B) divided by B, where B is the empty-plate background image smoothed by `getFluorescenceBackground(..., 'smooth')` (outliers against a 400 px moving median in both directions replaced, then a 20 px average filter). Checked exactly: `round(imageNormalized) == tiff` on images 2, 3, 5, and recomputing it from `data{n}` and `background{j}` reproduces the TIFF (max difference 0) on the same three. Nothing in the analysis package writes the TIFFs, `bacteria.mat`, `mask/` or `closest/`; `bacteria.mat`'s `metaData` numbers the 3,725 images exactly as `analyzeGFP`'s (all file names, all `imageNum`). The `.czi` originals are not available (Jess). **So (Jess, 2026-10-05, option 2):** the raw images are extracted from `data` -- the 1,575 images with a TIFF plus the 89 empty-plate images they were normalised by, 1,664 in all -- to `ecoli/raw/.tiff` (uint16, deflate, each read back identical), and **those are the recordings** (`recordingList` reads `raw/`; with no `raw/` it falls back to `images/` and a check says the recordings are normalised). Each TIFF in `images/` becomes its plate's `intensity_calculation` `background-normalised fluorescence image` (uint16, the file by location, not ingested), inputs the raw image's recording and its background's; method `normalisation to the background illumination`, method parameters the 400 px outlier window and 20 px smoothing window. **The fitted backgrounds are not stored** (reproducible from the raw empty-plate image and the code); each used background's fit quality is kept on its plate: `score_calculation` `background fit r-squared` and `count_calculation` `background outlier pixels`, from `backgroundInfo`. The masks, nearest-patch maps and per-patch values (#62) now take the NORMALISED image as input -- the pixels `analyzeLawnProfiles` used. **The profile curves** (Jess: "curves sound reasonable"): per matched patch per image, an `intensity_calculation` `intensity by distance from patch edge` about the patch subject, its row of `lawnProfiles.pixelValuesNormalized` (mean background-normalised intensity in each 1 um bin of distance from the patch edge, edge-aligned by the code's curvature rule; NaN where the patch has no pixels) in an ingested float64 body keyed by `distance from patch edge` (regular, origin `distances(1)` mm -> m, step 1e-6 m; about -0.6 to +1.15 mm, positive inside the patch); method `fluorescence profile from the patch edge`, method parameter the 1 um bin; inputs the normalised image and the mask. Rows are in `bwlabel` order like the image's `lawnAnalysis` rows, so #62's match applies; an image whose curve count differs from its detected patches, or whose distances are not 1 um steps, gets none and is reported. **Read without loading:** the workspace is -v7.3 (HDF5); only `lawnProfiles.imageNum` and the object references of `distances` / `pixelValuesNormalized` are read, then each image's two arrays (0.06 s for three images, Jess); the image fields (~170 GB) are never touched. Default location `ecoli/analyzeGFP_24-04-04.mat` (option `ProfilesFile`). Not stored from `lawnProfiles`: `image` (the raw image, now a recording), `imageNormalized` (the TIFF), `labeled` (the dilated, labelled mask; the mask is stored), `pixelValues` (never filled). | DECIDED (user, 2026-10-05); BUILT | +| 65 | 2026-10-05 | **Stage 12, the border amplitude fits and their estimates** (`ndi.setup.conv.haley.densityDocuments`, a dataset-wide stage run after the sessions are built, before they are written). **Where it comes from** (the analysis package at `44a89de`): `plotForagingGFP.m` fits, per E. coli condition (peptone, lawn volume, growth-time condition, OD600), a straight line through border amplitude / exposure time (a.u. per ms) against growth time (min) over the included patches, and writes `borderAmplitude.csv`; OD600 0.05 and 0.1, which were not imaged, come from ONE joint fit per lawn volume, `f = a*OD600 + b*t` over OD600 0.5-2, so their CSV rows share its goodness. `labelEncounters.m` gives each encounter `borderAmplitude = intercept + slope * lawnGrowth` from its patch's condition (0 for a patch of LB alone, NaN where none matches) and `borderAmplitudeGrowth` from the 200 uL fit at the cultivation plate's `growthLawnGrowth`; checked by hand on encounter.mat's row 2 (0.166054 + 0.000493137 x 73.751 = 0.20242) and its cultivation value (55.7163 - 0.0068074 x 3106.4 = 34.57). The Drive's `Jess/foragingGFP/borderAmplitude.csv` is byte-identical to the package's `data/borderAmplitude.csv` (3,839 bytes, MD5 159a9d27...). **Which patches are in a fit** (plotForagingGFP's rules): not image `20231230_101739.czi`, not plate 64, not a 0.5 uL patch with circularity < 0.75, not a border / centre ratio < 1 or > 35; then, per image, the lowest `xPeak` until the image's `xPeak` range is at most 0.1 mm. **What a fit is about** (Jess, 2026-10-05, after asking whether a calculation could take statements as its subject, whether relations could take several subjects, and whether 1,500 relations is too many): a statement's `subject_id` must be a `subject`, and the fit's sources already go in `input_id` (T2's provenance rule), so the subject is what the fit describes -- a population of patches. T4: a many-way group is a reified subject plus binary `member_of` edges, never a list; T4 is kept (binary memberships can each carry their own facts and change one at a time; about 1,500 small relations are ~1% of the E. coli session's documents). The strain was considered and is not possible: a `strain` is an entity, not a subject. So, at the DATASET level (decision #20; the instruments are subjects there already): **per fit, a group subject** (`group`; `ecoli_fit_group_`, name the condition) **whose members are the condition's E. coli patch subjects** (`member_of`; the joint fits' groups: the OD600 0.5-2 patches), and **its `model_fit_calculation`** `border amplitude growth`: model `linear`, equation `f = slope*t + intercept` (joint: `f = a*OD600 + b*t`), in the source's units (growth time in minutes, border amplitude per exposure time in a.u. per ms, so the coefficients are the CSV's own), goodness r2, sse and sse per point (n = dfe + 2; adjusted r2 and rmse follow); conditions peptone, seeding volume, growth-time condition and (single fits) OD600; method `linear least squares`, method parameters the exclusion rules; **inputs exactly the INCLUDED patches' `patch border amplitude` statements** (#62), so the exclusions are the values left out; time the span of the included images. **The numbers are the published CSV's** (Jess: `ecoli/borderAmplitude.csv`, next to `bacteria.mat`); each fit is ALSO refitted from the per-patch values under the rules above and a disagreement is reported, as is a CSV `dfe` that disagrees with the included count; with no CSV the refits are stored and the run says so. A fit with no included image (no time) is reported and neither it nor its group is made. **Per C. elegans worm with encounters** (Jess: save the borderAmplitudes), keyed by its encounter list as stage 11's values: `intensity_calculation` `estimated patch border amplitude` (encounter.mat `borderAmplitude`) and `estimated cultivation plate border amplitude` (`borderAmplitudeGrowth`), method `calibration from the E. coli border amplitude fits`, inputs the list and the fit(s) used; every value is checked against the fit it names (labelEncounters' match, foragingMini's growth condition 60 read as 48) and a disagreement is reported. **Not stored:** relative density (`modelExploit.m`: 10 x the value / the foragingConcentration OD 10 mean, floored at 0.01, log10) -- computed in the model script, in no source file. Dataset-level documents are written with stage 2's at stage 13. | DECIDED (user, 2026-10-05); BUILT | +| 66 | 2026-10-05 | **Stage 13, check & write the dataset** (`ndi.setup.V2.createDataset`, called by `import_V2`'s stage `dataset`). Until now the sessions were written each to its own folder and the dataset-level documents (stage 2's dataset, people, organizations, funding, publication, software, products, strains, formulations and instruments; stage 3's studies and their relations; stage 12's fit groups, memberships and fits) were built and written nowhere, so every edge from a session to them dangled. **What NDI could do** (read 2026-10-05): `ndi.dataset.dir` sits on an `ndi.session.dir`, which already opens a V2 database; but `ndi.dataset` lists its sessions ONLY from `session_in_a_dataset` documents, a v1 class V_eta does not have (the migrator turns it into a `dataset` and a session `part_of` relation); an INGESTED session is one whose documents are in the dataset's own database, opened as `ndi.session.dir(reference, datasetPath, sessionId)`. **Decided (Jess, 2026-10-05, option A):** ONE V2 database (`/dataset`, option `DatasetFolder`) holding the dataset's own `session` document, every dataset-level document and every selected session's documents -- each session's `session` document (the same id its folder's has) and its `part_of` study relations included -- so every edge can be checked in one place and the dataset is in the form NDI Cloud holds. Rejected: B, linked session folders (the dataset would have to record where each session lives, and V_eta has no field or class for it; and cross-database edges cannot be checked together). **The check** (before writing; `Strict` refuses to write on a failure, off by default so a report is always produced): the denominator first (documents, dataset-level, sessions, edge values checked); every `depends_on` value names a document in the set (dangling edges, by class and edge); no id repeats; every session is `part_of` a study that is `part_of` the dataset; the census by class and by session. **NDI changes** (backwards compatible: both run only when no `session_in_a_dataset` document exists): `ndi.dataset.dir` finds a V2 dataset's own session from the V2 `dataset` document's session_id (the folder's `unique_reference.txt` names whichever session was opened LAST, since `ndi.session.dir` rewrites it on every open), and rebuilds its session list after reopening; `ndi.dataset/build_session_info` lists every other `session` document as an ingested session, opened by its `local_identifier`. **Files:** ingested bodies are copied into the dataset's `.ndi/files`; raw recordings stay where they are, by location. **Session folders** (option `SessionFolders`, default true; recommended to keep, Jess agreed with the recommendations): each session is still written to its own folder as a working copy; the dataset is written FIRST and keeps each ingested body's temporary file, which the session folder then ingests and deletes. | DECIDED (user, 2026-10-05); BUILT | +| 67 | 2026-10-06 | **A grid-template plate's patches each take their own OD600** (found by the first all-sessions run, which stopped at matching_0001: every seeding dose lacked its suspension). On a Matching plate `experimentInfo.OD600` is the TEMPLATE, one value per position (19, the empty centre OD600 0), while `lawnCenters` has the 18 patches, so no patch got an OD600 and none matched that day's solutions (2023-02-17: OD600 0, 1, 5, 10 recorded). `subjectList` now reads each patch's OD600 from the lab's `lawnClosestOD600` map (each pixel's nearest patch's OD600) at the patch's centre, cross-checks it against the template's non-zero positions in order, and reports a disagreement or a missing map (`patchOD600` check); the template's order is used only when the map is unreadable. Plates whose OD600 is one value, or one per patch, are unchanged. | BUILT | +| 68 | 2026-10-07 | **Strain terms and biological sex, from the old import** (`doImport` and its helpers). Each strain in the spec carries its WormBase node (`node`), and the strain assertion's term is that node plus the name, not the name alone: OP50 WBStrain:00041969, OP50-GFP 00041972, N2 00000001, PR811 00030796, TU253 00035037, DA465 00005463, DA609 00005493, and from `doImport` CB1112 00004246, CX6448 00005280, MT15434 00027519. FK103 00007498, MT13113 00027424 and MT9455 00027363 were looked up by hand on wormbase.org (`ndi.ontology.lookup` fails for every WBStrain query, by name and by id); a blank node would fall back to the name. One node per term: OP50's NCBITaxon:637912 is a second identifier for the same strain, kept on the strain entity's `identifiers`, not on the term. Every C. elegans strain also carries `biological_sex` hermaphrodite (PATO:0001340), asserted on the cohort as `biological sex` (PATO:0000047), distributive like species and strain; bacteria get none. Jess, 2026-10-07. | +| 69 | 2026-10-08 | **More of the old import (`doImport`), decided with Jess.** (a) **OP50** carries `NCBITaxon:637912` (the old import's id, *E. coli* OP50) as a second identifier beside its URL. (b) **Developmental stage**: one distributive `term_observation` per cohort, `developmental stage` = `L4 larva`, method `visual inspection` (the worms were chosen as L4s by eye at the pick), at the pick's hand-written time; with no pick time, a relative time before the cohort's next move. No stage at the assay: it follows from the pick and assay times (the old `age` label was calendar days, coarser than the stored times). Names alone until the terms are looked up (they show in `termInventory`). Spec `preparation.developmental_stage`. (c) **Background strain**: every *C. elegans* strain but N2 has N2 as its background, as doImport asserted. (d) **Assay type**: carried by the study each session is `part_of`; doImport set it per plate from the folder and the plate's `condition`, so `subjectList` compares the two on every run and reports the plates compared and any that disagree (`assayTypeDisagrees`). (e) **Plate role** (cultivation / behaviour): not recorded as its own assertion; a plate's role is in its name and identifier, filmed plates are those with recordings, and each cohort's moves name the plate kinds. (f) **Old worm identifiers**: not kept. | DECIDED (Jess, 2026-10-08); BUILT | +| 70 | 2026-10-08 | **The rest of doImport's columns, checked one by one against V2.** (a) **Patch geometry** (`lawnCenters`, `lawnRadii`, `lawnCircularity`, doImport's patch table): per behaviour video, each patch's centre (`position_calculation` `patch centre`, in the video's coordinate system: MATLAB pixel centres minus 0.5), radius (`length_calculation`, pixels to metres, approximate) and circularity (`score_calculation`, 0-1), each a calculation from the recording. (b) **Recording dimensions** (doImport's `imageStack_parameters`): a recording's `keys` are [image vertical, image horizontal] position in pixels and, for a video, `time` at 1 / frame rate; written only when the file is opened (`ReadVideos`, or a TIFF), so they are never partial. (c) **doImport's extra exclusion** (`identifyValidRows` on `growthCondition`): measured by Jess 2026-10-08, the only NaN rows are 6 foragingMini plates (experiment 11 plates 77-80, experiment 24 plates 233-234), none excluded by the source, and all 6 are the never-filmed plates (`arenaDiameter` 0, no video; #54e). No data, so nothing to exclude: not applied; `subjectList` lists such plates (`noGrowthCondition`). (d) **foragingMutants row 1** (doImport set `arenaDiameter` 30): plate 1's failed first video (scale 0, no patches); V2 writes no geometry for it, and its second video has 30 mm. No correction. (e) **foragingMini experiment 24**: the 12 plates share one seeding, cold room and pick (10:41), so one acclimation plate, but `growthTimeRoomTemp` steps down a minute per plate (09:03 to 08:52), a fill-down; doImport used the last row, V2 kept the first and reported it (`growthDisagrees`). Measured by Jess: it is the only one of the 24 Mini experiments with more than one acclimation room temperature time, and every day has plates/2 + 1 distinct assay room temperature times, one pair of plates prepared on the acclimation plate's schedule. On experiment 24 that pair is plates 225 and 230 (cold room 24 Aug 14:25, the acclimation plate's; room temperature 28 Aug 09:02); on experiment 23, plates 213 and 218 give exactly that day's single time. So **09:02**, a named correction (`growth_room_temp_mini_experiment_24`), inferred from the data: Jess does not recall the day. Relative density and the CFU scale are #65 and #57. | DECIDED (Jess, 2026-10-08); BUILT | +| 71 | 2026-10-09 | **Each session is written once (stage 13 with `SessionFolders`).** Until now a session was written twice: ingested into the dataset (#66) and again into its own folder as a working copy. did-schema's `V_eta_linked_session_plan.md` (signed 2026-10-09) gives a session two forms in a dataset, ingested or LINKED (a `linked_session` document naming its folder), and NDI now reads both (`ndi.dataset`). **Built** at Jess's request ("Fix the Haley import"): with `SessionFolders` (default true) each session's documents are written ONCE, to its folder, and the dataset holds the dataset-level documents, each session's `part_of` study relation (membership is the dataset's, so the folder no longer carries it) and a `linked_session` per session. The whole set is still checked as one before anything is written (`ndi.setup.V2.createDataset`'s new `External`: the sessions' documents are checked, not written into the dataset). `SessionFolders` false ingests every session and writes no folders: the self-contained form to share. Either converts to the other later (`convertLinkedSessionToIngested`, `convertIngestedSessionToLinked`). A linked session opened through its dataset reaches the dataset's documents (`database_search_with_dataset`); opened on its own, its folder holds only its own. `verifyDataset` and `termInventory` read the dataset's database and every linked session's (`ndi.setup.V2.datasetDatabaseFiles`). With a schema that has no `linked_session` class, `SessionFolders` true writes both, as before. **Default (Jess, 2026-10-14):** the import is purely for publishing, so a run that writes the dataset INGESTS every session and writes no folders (`SessionFolders` false: the form NDI Cloud holds, and the fastest); a run without the dataset stage writes session folders, its only output. Linked is `'SessionFolders', true`. | DECIDED (Jess, 2026-10-14); BUILT | +| 72 | 2026-10-14 | **foragingConcentration experiment 14 is a single-patch day.** Its `tableOfContents.xlsx` row `0014` says `grid [OD600 = 0, 0.05, 0.1, 0.5, 1, 2, 3, 4, 5, 10, 1 (48H)]`, but all 12 of its plates (133-144) say `single` in `experimentInfo` and each has 1 patch (`lawnCenters`), like experiment 3's single plates, while the grid days either side (13, 15) have 19 patches per plate (measured by Jess, 2026-10-14). The row's first word is a copy of its neighbours' (the same row had the worm-range typo, #3). A named correction (`conditions_concentration_experiment_14`, `table: tableOfContents`) sets it to `single`, so `concentration_0014` is in `large_single_patch` (experiments 3, 8, 9, 14). `sessionList` applies a tableOfContents correction only while the cell still reads `was`, and reports each one applied (`corrected`) or not (`correctionNotApplied`). | DECIDED (Jess, 2026-10-14); BUILT | +| 73 | 2026-10-14 | **Exclusion reasons, the day flag, and plates never filmed.** (a) **The lab's analysis excludes by plate, not by day** (checked in shreklab/Haley-et-al-2024 at `44a89de`: `getExperimentInfo.m:173-178` reads only the notebook's per-plate `exclude`; `getForagingInfo.m:76` analyses every experiment; the figures select worms by condition and `~encounter.exclude`, e.g. `fig2.m:50`; nothing reads the tableOfContents `include`). `concentration_0001`, the only day marked `include = no` ("no contrast video, so patches are poorly aligned"), has 10 plates, 3 excluded (1, 3, 6) and 7 with 318 unexcluded encounters in `encounter.mat` (Jess, 2026-10-14): 7 of its plates are in the analysis. So the day's flag tags nothing, and is NOT written into the session's description any more (it read "Included in analysis: no", which was untrue). This corrects #2's "tableOfContents `include = no`": only the plate's `exclude` excludes. (b) **An excluded plate's tag has the reason beside it where one was written** (the tag's `notes`: did-schema moved `notes` from interaction up to statement for this, Jess 2026-10-14; with an older schema, whose assertions have no notes, a `text_assertion` `reason for inclusion in analysis` beside it): its day's tableOfContents `notes` when the day is marked `include = no`; otherwise no reason (the notebook's `exclude` has none). (c) **A plate never filmed** (`arenaDiameter` 0 on every row: Mini plates 77-80 and 233-234, #54e, #70c) gets `behavior recording` = Not Performed (`NCIT:C141477`, "An indicator that specifies that an activity did not occur."), noted "Prepared but not filmed: no video." (likewise); it is not excluded and its worms are kept, so the numbering is whole and nothing is hidden. `subjectList` reports each (`notFilmed`) and describes the plate as not filmed. | DECIDED (Jess, 2026-10-14); BUILT | +| 74 | 2026-10-14 | **foragingMini plates 55 and 189: room temperature time is an estimate.** Each plate's `timeRoomTemp` is exactly its recording's start minus 1 h, which is what the analysis package fills in when the notebook's `conditionPlateRoomTempTime` is blank (`getExperimentInfo.m:269-270`, shreklab/Haley-et-al-2024 at `44a89de`). The notebook spreadsheet is not with the data (0 spreadsheets under `foragingMini` besides `tableOfContents.xlsx`, Jess 2026-10-14), so the blank cannot be confirmed; marking a measured time approximate costs less than presenting an estimate as measured. Two named corrections (`room_temp_estimated_mini_plate_55`, `_189`) mark them approximate, as #54d does for Concentration plates 1-10. | DECIDED (Jess, 2026-10-14); BUILT | +| 75 | 2026-10-14 | **An acclimation plate's seeding time known only to a span.** The 12 assay plates of `concentration_0015` (145-156) share one acclimation plate and agree on its cold room (29 Apr 2022 15:20), room temperature (2 May 15:10) and pick (2 May 17:08), but give its seeding as 28 Apr 2022 12:39 (146, 147, 150, 152, 154, 155) or 12:41 (145, 148, 149, 151, 153, 156), alternating with no fill-down pattern (measured by Jess, 2026-10-14). Two readings: two acclimation plates seeded two minutes apart, or one whose seeding time is known to two minutes; Jess chose the second (option B), which adds nothing that may not exist. So when the assay plates disagree ONLY on the seeding time, the acclimation plate is seeded at the span's midpoint (12:40), its tolerance widened by half the span (60 s) on top of the hand-written tolerance, reported once (`growthSeedSpread`) rather than as `growthDisagrees`. Any other disagreement keeps the first plate's times and is reported as before. | DECIDED (Jess, 2026-10-14); BUILT | +| 76 | 2026-10-14 | **A Matching patch's OD600 comes from the lab's map, and the cross-check compares densities, not order.** The import reads each patch's OD600 from `lawnClosestOD600` at its centre, as the analysis does (shreklab/Haley-et-al-2024 at `44a89de`: `analyzeEncounters.m:254` takes every encounter's `lawnOD600` from that map; `analyzeWormLabTracks.m:302`, `figS2.m`, `plotTracks.m` and `createVideo.m` use it too; the `OD600` list's order labels no patch). The check against the template's `OD600` list compared the two in order and flagged 14 plates (5, 6, 8, 10, 11, 21, 24, 26, 27, 31, 33, 34, 43, 47; measured by Jess, 2026-10-14), but on every one the map and the list hold exactly the same densities: `lawnCenters` lists patches in detection order, which swaps neighbours lying nearly level. The check now compares the sorted densities, so it reports only a plate whose map and template hold different densities. | DECIDED (Jess, 2026-10-14); BUILT | +| 77 | 2026-10-14 | **foragingSensory experiment 2: the acclimation plates' room temperature date is a typo.** All 18 plates record `growthTimeRoomTemp` 5 Jan 2024 09:49, after their worms were picked onto the acclimation plates on 4 Jan (11:08, 11:16, 11:24); seeding (29 Dec 11:16), cold room (30 Dec 11:16) and recording (5 Jan from 08:11) are in order, and `starvedTime` is empty (no food deprivation step) (measured by Jess, 2026-10-14). 4 Jan 09:49 puts the plates on the bench 1 h 19 min to 1 h 35 min before the picks, as on other days. A named correction (`growth_room_temp_sensory_experiment_2`, as #70e) sets it, so the room temperature period no longer ends before it starts. | DECIDED (Jess, 2026-10-14); BUILT | +| 78 | 2026-10-14 | **The E. coli profile curves come from a 112 MB extract of analyzeGFP's workspace.** `analyzeGFP_24-04-04.mat` (119.2 GB) is on the Google shared drive `Jess/foragingGFP`, not with the data. `ndi.setup.conv.haley.extractLawnProfiles` reads it one image at a time through `lawnProfileReader` (never the ~170 GB of images in `lawnProfiles`) and writes `lawnProfiles.{imageNum, distances, pixelValuesNormalized}` and `backgroundInfo` to `ecoli/analyzeGFP_24-04-04.mat`, where the import looks. Run by Jess, 2026-10-14: 1575 images read, 1521 with curves, **7204 curves** (= `lawnAnalysis`'s 7204 rows), 114 background rows, 112.0 MB. | BUILT; RUN (Jess, 2026-10-14) | +| 79 | 2026-10-14 | **The 2,061 E. coli images with no file are acquisitions never kept as TIFFs, and stay out.** `bacteria.mat` `metaData` lists 3,725 images; `ecoli/raw/` holds 1,664 TIFFs; the 2,061 without one (654 fluorescence) include none of the 1,575 analysed images and none of the 89 background images those were normalised against (measured by Jess, 2026-10-14). They exist only as the microscope's `.czi` originals (e.g. `20231230_093910.czi`). They are not recorded (a recording with no bytes says only that a picture was taken), and the import's report now says how many missing images were analysed or used as a background, so a real gap would read as one. | DECIDED (Jess, 2026-10-14); BUILT | +| 80 | 2026-10-14 | **E. coli images with patches missing are placed by the acetate template.** 34 analysed images detected neither 12 patches nor 1 (27 with 11; one each with 3, 4, 5, 7, 10; two with 6), so decision #62's numbering by position could not match them and their per-patch values (267) were left out. On these plates every patch has the same OD600 (plates 36, 77, 97, 116: 0.5; 117, 124: 1), so density cannot tell patches apart. A first build fitted the grid to the detected centres alone; Jess's check of image 13 showed it mislabelled an image with only two rows present (it split a row in two), so it was replaced. **The anchor is the analysis's own registration**: `analyzeGFP.m:128-155` registers the 16 x 25 mm acetate on every brightfield image of a `rectangle` plate (`getAcetateArena`: centre stored [y x], angle, scale px/mm; 574 registrations), and each fluorescence image uses its `brightfieldImageNum`'s (`analyzeGFP.m:291`). `ndi.setup.conv.haley.extractTemplates` copies those (not their 3.5 GB of masks) into the extract. `ndi.setup.conv.haley.patchLayout` learned the 12 positions in the acetate's frame from every full image (Jess, 2026-10-14): columns ~3.0 mm apart, rows at y -5.2 / 0 / +5.25 mm, row 2 half a column right; median deviation 0.26-0.33 mm per patch, maximum 1.54 mm, closest two positions 5.97 mm apart; the angle's sign is immaterial at these angles (0.303 vs 0.301 mm). 221 images deviate > 0.5 mm, in runs of consecutive images at nearly one value (a plate sitting off-centre in its acetate), none beyond 1.54 mm. The positions are the spec's `ecoli_patch_layout`. `ndi.setup.conv.haley.matchTemplatePatches` puts each detected patch of a partly detected image in the acetate's frame and gives it the nearest position, when within `max_distance_mm` (2.0, a third of the spacing) and no two patches claim one; otherwise the image is reported with why. `ndi.setup.conv.haley.showPatchMatch` draws an image's mask with the numbers given and the 12 template positions, for checking by eye. | DECIDED (Jess, 2026-10-14); BUILT; VERIFIED BY EYE (Jess, 2026-10-14: image 13 and the 34 partly detected images, every patch inside its template position) | +| 81 | 2026-10-14 | **mutants_0001 plate 1's first video is not on disk and stays unrecorded.** Its row names `2023-11-03_07-35-52_1.avi` (scale 0, arena 0: the failed start of #70d); no such file exists. Worms 1-4 are listed on both of the plate's videos, and video 2 (`07-42-14`) is on disk, so they keep their video. An unnamed file 3 min earlier on the same camera, `2023-11-03_07-33-06_1.mp4`, may be the same failed start, but Jess cannot confirm it (2026-10-14) and it would carry nothing measured, so it is not linked. The report line now says a missing video of scale 0 on a plate with another video is a failed start whose worms are on the other video. | DECIDED (Jess, 2026-10-14); BUILT | +| 82 | 2026-10-14 | **An acclimation seeding day with no measured suspension is recorded at its nominal strength, unmeasured.** The 12 acclimation plates of `mutants_0003` were seeded on 9 Nov 2023, a day on which no assay plate was seeded, so no `OD600Real` or `CFU` was read and no suspension existed for their dose; the dose was left out, as if the plates were unseeded. Every acclimation plate was seeded alike (200 uL at `growthOD600`, OD600 1, in LB: #56, the paper's Methods), so the seeding is known and only its measurement is missing. Jess does not know whether `231109_OP50_cultures` records an OD600 for it (2026-10-14). So `seedingSuspensions` makes an unmeasured suspension for such a day: one formulation per nominal OD600 (`suspension___unmeasured_od`: the strain at that nominal OD600 in the growth diluent, no stock, no CFU), and the dose's notes say the suspension was not measured and its strength is nominal. If a reading turns up, a named correction replaces it. | DECIDED (Jess, 2026-10-14); BUILT | +| 83 | 2026-10-14 | **concentration_0016: two unnamed camera-2 files, and plates 161/162 change camera.** (a) Of the day's 24 video files, the import finds every row's video and lawn clip (`lawnFileName`) except `2022-05-06_15-09-24_2` and `15-14-47_2`: a camera-2 lawn clip and video in the last hour, the pattern of a plate, but with no row. The day's 44 worms (0595-0638) are all on its 11 listed plates (Jess, 2026-10-14), so it is not a missing plate (an empty run, or a plate given no worms); the files stay out, and the report now says when a day's worms are all on listed plates. (b) Plates 161 and 162 are the only plates that day whose lawn clip and video are on different cameras (161: lawn `11-36-33_1`, video `11-44-44_2`; 162 the reverse). Jess does not recall whether the plates moved or the names were swapped, and a check from the images would not be reliable (both are grid plates differing only in density), so the source is kept as written and each of their recordings carries a note saying so (option A); `recordingList` reports any such plate (`cameraSwap`), and a recording's note now reaches its statement. | DECIDED (Jess, 2026-10-14); BUILT | +| 84 | 2026-10-14 | **A data body's `format` is an EDAM file-format term**, not a media type (did-schema 877bbde, Jess -- lightsheet point 4): recordings `EDAM:format_3997` MPEG-4 or `EDAM:format_3591` TIFF, the E. coli normalised images TIFF, ingested masks and maps `EDAM:format_2333` binary (compression still gzip). On a schema copy whose `data_body.format` is still a string (the CI "current" leg), `fileFormat` writes the media type #34 and #60 name. Amends #34 and #60. | DECIDED (user, 2026-10-14); BUILT | +| 85 | 2026-10-15 | **Each program names its maker, and the dataset its host** (did-schema 9bedceb, Jess, from stage 2's report of six spec facts with no place): each software entry's `vendor` becomes a `developed_by` relation to that organization (StreamPix -> NorPix, WormLab -> MBF, MATLAB -> MathWorks, Photoshop and Illustrator -> Adobe), not `sold_by` -- free and open-source software was never sold; and the NDI Cloud copy's `host` becomes the DATASET's `hosted_by` relation to `ndi_cloud` (the copy itself is one of the dataset's identifiers). On a schema without either relation (`ndi.setup.V2.schemaHasRelation`), both are reported in the stage-2 list as before. | DECIDED (user, 2026-10-15); BUILT | +| 6 | — | Distance-to-patch model (was `distance_metadata`). **Settled by #61** (2026-10-04): each worm's distance to the nearest patch edge is a `length_calculation` from `midpoint.mat` `distanceLawnEdge`, per frame, positive on a patch, inputs the positions and the lawn mask; nothing is built from `distance_metadata`. | CLOSED by #61 | +| 7 | — | Encounters: one statement per encounter, or a keyed series per worm. **Settled by #63** (2026-10-05): a keyed series per worm -- an `encounter onset` list and one statement per measured quantity keyed by it (`key_id`, did-schema #87); one set of documents per encounter was rejected (about 20x the documents). | CLOSED by #63 | + +## Source inventory (from the file list, 2026-09-30) + +Raw data (everything under `.ndi/`, `.ndi_/`, `haley_2025/` and `haley/*.zip` +is output of earlier `doImport` runs and must be skipped by discovery): + +- `celegans//experimentInfo.mat` — one row per plate-video, 67–70 + columns: plate prep timeline, bacteria/OD600, cultivation ("growth*", the L4 + plate), environment (temp, humidity), video/camera, arena and lawn geometry, + and per-plate 1024x1024 images (`firstFrame`, `arenaMask`, `refMask`, + `lawnMask`, `lawnClosest`, `lawnClosestOD600`). Experiments: + foragingConcentration (193 rows, 18 days), foragingMatching (60 rows / 50 + plates, 5 days), foragingMini (234, 24), foragingMutants (93 / 84, 4), + foragingSensory (38 / 36, 2). Extra columns: `configuration` (Matching, + Mutants, Sensory), `starvedTime`/`starvedDuration` (Mutants, Sensory). +- `celegans//{midpoint,head,tail}.mat` — per frame per worm tracks + (foragingSensory midpoint: 1,550,152 rows x 25 columns: position, velocity, + path angle, turn, neighbour distances, distance to lawn/arena edge, closest + lawn, flags). head/tail only in foragingMini. +- `celegans/encounter.mat` — 21,543 encounters x 39 columns, all experiments. +- `celegans//tableOfContents.xlsx` — one row per experiment day: + number, directory, lab notebook, worm range, conditions, include, notes. + NOT read by `doImport`. +- `celegans//videos//*.mp4` — one per plate-video. +- `ecoli/bacteria.mat` — `info` (126 plates, 5 experiments), `metaData` + (3,725 images incl. brightfield/background), `lawnAnalysis` (7,204 patches + over 1,521 fluorescence images). +- `ecoli/{images/*.tiff, mask/*.png, closest/*.png}` — 1,575 each. The + TIFFs are background-normalised, not raw (#64); the raw images are + extracted from `analyzeGFP_24-04-04.mat` to `ecoli/raw/` (1,664). + +Identity: `expNum`, `plateNum` and `wormNum` restart per experiment folder +(e.g. `encounter.mat` has 232 distinct `plateNum` for 618 plate rows), so every +key is qualified by the experiment name. + +To reconcile: 1,575 image files on disk vs 1,521 analysed images in +`lawnAnalysis` vs 3,725 images in `metaData`. diff --git a/src/ndi/+ndi/+setup/+conv/+haley/import_V2_spec.json b/src/ndi/+ndi/+setup/+conv/+haley/import_V2_spec.json new file mode 100644 index 0000000000..6c039eb092 --- /dev/null +++ b/src/ndi/+ndi/+setup/+conv/+haley/import_V2_spec.json @@ -0,0 +1,1872 @@ +{ + "_about": "Dataset-level metadata for the Haley V2 import (stage 2). Sources: the eLife paper (Haley, Chen, Aoi, Chalasani, eLife 13:RP103191, doi:10.7554/eLife.103191; Key resources table, Materials and methods, Funding, Author contributions, Author ORCIDs, Data availability) and the NDI Cloud dataset page (licence). ROR ids came from web search and are marked '_verify' where the search result was not titled with the organization's name. Every entry has a local `key`; references between entries use keys, which the maker resolves to document ids. See import_V2_decisions.md.", + "organizations": [ + { + "key": "salk", + "name": "Salk Institute for Biological Studies", + "identifiers": [ + { + "scheme": "ROR", + "value": "https://ror.org/03xez1567" + } + ] + }, + { + "key": "salk_mnl", + "name": "Molecular Neurobiology Laboratory", + "parent": "salk" + }, + { + "key": "ucsd", + "name": "University of California, San Diego", + "short_name": "UC San Diego", + "identifiers": [ + { + "scheme": "ROR", + "value": "https://ror.org/0168r3w48" + } + ], + "_verify": "ROR from a search summary, record page not read" + }, + { + "key": "ucsd_ngp", + "name": "Neurosciences Graduate Program", + "parent": "ucsd" + }, + { + "key": "ucsd_hdsi", + "name": "Halıcıoğlu Data Science Institute", + "parent": "ucsd" + }, + { + "key": "ucsd_neurobiology", + "name": "Department of Neurobiology", + "parent": "ucsd" + }, + { + "key": "nih", + "name": "National Institutes of Health", + "short_name": "NIH", + "identifiers": [ + { + "scheme": "ROR", + "value": "https://ror.org/01cwqze88" + } + ], + "_verify": "top ROR search result, untitled" + }, + { + "key": "nimh", + "name": "National Institute of Mental Health", + "short_name": "NIMH", + "identifiers": [ + { + "scheme": "ROR", + "value": "https://ror.org/05xj56w78" + } + ], + "parent": "nih" + }, + { + "key": "nsf", + "name": "U.S. National Science Foundation", + "short_name": "NSF", + "identifiers": [ + { + "scheme": "ROR", + "value": "https://ror.org/021nxhr62" + } + ], + "_verify": "top ROR search result, untitled" + }, + { + "key": "dorsett_brown", + "name": "Joe W. and Dorothy Dorsett Brown Foundation", + "identifiers": [ + { + "scheme": "URL", + "value": "https://www.thebrownfoundation.org/" + } + ] + }, + { + "key": "cgc", + "name": "Caenorhabditis Genetics Center", + "short_name": "CGC", + "identifiers": [ + { + "scheme": "URL", + "value": "https://cgc.umn.edu/" + } + ] + }, + { + "key": "ndi_cloud", + "name": "NDI Cloud", + "identifiers": [ + { + "scheme": "URL", + "value": "https://ndi-cloud.com/" + } + ] + }, + { + "key": "norpix", + "name": "NorPix" + }, + { + "key": "mbf", + "name": "MBF Bioscience" + }, + { + "key": "mathworks", + "name": "MathWorks" + }, + { + "key": "adobe", + "name": "Adobe" + }, + { + "key": "pixelink", + "name": "PixeLink" + }, + { + "key": "navitar", + "name": "Navitar" + }, + { + "key": "advanced_illumination", + "name": "Advanced Illumination" + }, + { + "key": "zeiss", + "name": "Carl Zeiss" + }, + { + "key": "molecular_devices", + "name": "Molecular Devices" + }, + { + "key": "cricut", + "name": "Cricut" + }, + { + "key": "bd", + "name": "Becton, Dickinson and Company", + "short_name": "BD" + }, + { + "key": "salk_media_kitchen", + "name": "Media Kitchen", + "parent": "salk", + "_note": "Made the LB and S-Complete (Jess, 2026-10-02; decision #56)." + }, + { + "key": "emd_millipore", + "name": "EMD Millipore" + }, + { + "key": "fisher", + "name": "Fisher Scientific" + } + ], + "people": [ + { + "key": "haley", + "given_name": "Jessica A", + "family_name": "Haley", + "identifiers": [ + { + "scheme": "ORCID", + "value": "https://orcid.org/0000-0001-6282-7124" + } + ], + "affiliations": [ + "ucsd_ngp", + "salk_mnl" + ] + }, + { + "key": "chen", + "given_name": "Tianyi", + "family_name": "Chen", + "identifiers": [ + { + "scheme": "ORCID", + "value": "https://orcid.org/0009-0003-7572-2155" + } + ], + "affiliations": [ + "salk_mnl" + ] + }, + { + "key": "aoi", + "given_name": "Mikio", + "family_name": "Aoi", + "email": "maoi@ucsd.edu", + "identifiers": [ + { + "scheme": "ORCID", + "value": "https://orcid.org/0000-0002-7052-880X" + } + ], + "affiliations": [ + "ucsd_hdsi", + "ucsd_neurobiology" + ] + }, + { + "key": "chalasani", + "given_name": "Sreekanth H", + "family_name": "Chalasani", + "email": "schalasani@salk.edu", + "identifiers": [ + { + "scheme": "ORCID", + "value": "https://orcid.org/0000-0003-2522-8338" + } + ], + "affiliations": [ + "salk_mnl" + ] + } + ], + "funding": [ + { + "key": "nih_r01", + "name": "RO1 MH096881", + "identifiers": [ + { + "scheme": "AwardNumber", + "value": "R01 MH096881" + } + ], + "funder": "nimh", + "recipients": [ + "chalasani" + ] + }, + { + "key": "dorsett_brown_award", + "name": "Dorsett Brown Foundation award", + "funder": "dorsett_brown", + "recipients": [ + "haley", + "chalasani" + ] + }, + { + "key": "salk_innovation", + "name": "Salk Innovation Grant", + "funder": "salk", + "recipients": [ + "haley", + "chalasani" + ] + }, + { + "key": "nsf_grfp", + "name": "Graduate Research Fellowship Program", + "identifiers": [ + { + "scheme": "AwardNumber", + "value": "DGE-1650112" + } + ], + "funder": "nsf", + "recipients": [ + "haley" + ] + }, + { + "key": "ucsd_ssra", + "name": "Undergraduate Summer Research Award", + "funder": "ucsd", + "recipients": [ + "chen" + ] + } + ], + "publications": [ + { + "key": "elife_2025", + "name": "Accept–reject decision-making revealed via a quantitative and ethological study of C. elegans foraging", + "publication_date": "2025-09-10", + "authors": "Haley JA, Chen T, Aoi M, Chalasani SH", + "identifiers": [ + { + "scheme": "DOI", + "value": "https://doi.org/10.7554/eLife.103191" + } + ] + } + ], + "web_resources": [ + { + "key": "code", "type": "software", + "name": "Analysis code (GitHub)", + "identifiers": [ + { + "scheme": "URL", + "value": "https://github.com/shreklab/Haley-et-al-2024" + } + ] + }, + { + "key": "ndi_cloud_page", + "name": "NDI Cloud dataset page", + "identifiers": [ + { + "scheme": "URL", + "value": "https://ndi-cloud.com/datasets/682e7772cdf3f24938176fac" + } + ], + "host": "ndi_cloud" + }, + { + "key": "wormbook_maintenance", "type": "publication", + "name": "Stiernagle T (2006) Maintenance of C. elegans. WormBook", + "identifiers": [ + { + "scheme": "DOI", + "value": "https://doi.org/10.1895/wormbook.1.101.1" + } + ], + "_note": "Source of truth for the standard recipes (S Basal, S-Complete, trace metals, 1 M potassium citrate, 1 M potassium phosphate pH 6, LB); decision #56." + } + ], + "software": [ + { + "key": "streampix", + "name": "StreamPix", + "version": "8", + "identifiers": [ + { + "scheme": "RRID", + "value": "RRID:SCR_015773" + } + ], + "vendor": "norpix" + }, + { + "key": "wormlab", + "name": "WormLab", + "identifiers": [ + { + "scheme": "RRID", + "value": "RRID:SCR_017669" + } + ], + "vendor": "mbf" + }, + { + "key": "matlab", + "name": "MATLAB", + "version": "2024a", + "identifiers": [ + { + "scheme": "RRID", + "value": "RRID:SCR_001622" + } + ], + "vendor": "mathworks" + }, + { + "key": "photoshop", + "name": "Photoshop", + "version": "2024", + "identifiers": [ + { + "scheme": "RRID", + "value": "RRID:SCR_014199" + } + ], + "vendor": "adobe" + }, + { + "key": "illustrator", + "name": "Illustrator", + "version": "2024", + "identifiers": [ + { + "scheme": "RRID", + "value": "RRID:SCR_010279" + } + ], + "vendor": "adobe" + }, + { + "key": "haley_analysis", + "name": "Haley et al. 2024 analysis package", + "version": "44a89de21430cea3312d312e17054647a074f90c", + "identifiers": [ + { + "scheme": "URL", + "value": "https://github.com/shreklab/Haley-et-al-2024" + } + ] + }, + { + "key": "macos", + "name": "macOS" + } + ], + "products": [ + { + "key": "camera", + "name": "PL-B741F camera", + "vendor": "pixelink" + }, + { + "key": "lens_60135", + "name": "Navitar lens", + "catalog_number": "1-60135", + "vendor": "navitar" + }, + { + "key": "lens_6044", + "name": "Navitar 0.25x lens", + "catalog_number": "1-6044", + "vendor": "navitar" + }, + { + "key": "backlight", + "name": "Edge-lit backlight", + "vendor": "advanced_illumination" + }, + { + "key": "axio_zoom", + "name": "Axio Zoom.V16", + "vendor": "zeiss" + }, + { + "key": "spectrophotometer", + "name": "SpectraMax Plus 384", + "vendor": "molecular_devices" + }, + { + "key": "cutter", + "name": "Cricut Maker 3", + "vendor": "cricut" + }, + { + "key": "peptone", + "name": "Bacto peptone", + "catalog_number": "211677", + "vendor": "bd" + }, + { + "key": "agar", + "name": "Agar", + "catalog_number": "101614", + "vendor": "emd_millipore", + "_note": "240102B_NGM_plates (Benchling)." + }, + { + "key": "nacl_s271", + "name": "Sodium chloride", + "catalog_number": "S271-1", + "vendor": "fisher", + "_note": "240102B_NGM_plates (Benchling)." + }, + { + "key": "lb_kitchen", + "name": "LB", + "vendor": "salk_media_kitchen" + }, + { + "key": "s_complete_kitchen", + "name": "S-Complete", + "vendor": "salk_media_kitchen" + } + ], + "strains": [ + { + "key": "OP50", + "name": "OP50", + "node": "WBStrain:00041969", + "_node_synonym": "Also NCBITaxon:637912 (the V1 import's identifier, Escherichia coli OP50): a term carries one node, so it is a second identifier below (Jess, 2026-10-08), beside its URL.", + "species": { + "node": "NCBITaxon:562", + "name": "Escherichia coli" + }, + "genetic_strain_type": "wild type", + "identifiers": [ + { + "scheme": "RRID", + "value": "RRID:WB-STRAIN:WBStrain00041969" + }, + { + "scheme": "URL", + "value": "https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?id=637912" + }, + { + "scheme": "NCBITaxon", + "value": "NCBITaxon:637912" + } + ], + "source": "cgc" + }, + { + "key": "OP50-GFP", + "name": "OP50-GFP", + "node": "WBStrain:00041972", + "species": { + "node": "NCBITaxon:562", + "name": "Escherichia coli" + }, + "genetic_strain_type": "transgenic", + "background": [ + "OP50" + ], + "description": "OP50 carrying a GFP plasmid (pFPV25.1); ampicillin resistant.", + "source": "cgc", + "identifiers": [ + { + "scheme": "RRID", + "value": "RRID:WB-STRAIN:WBStrain00041972" + } + ] + }, + { + "key": "N2", + "name": "N2", + "node": "WBStrain:00000001", + "species": { + "node": "NCBITaxon:6239", + "name": "Caenorhabditis elegans" + }, + "biological_sex": { + "node": "PATO:0001340", + "name": "hermaphrodite" + }, + "genetic_strain_type": "wild type", + "description": "N2 Bristol, wild isolate.", + "source": "cgc", + "identifiers": [ + { + "scheme": "RRID", + "value": "RRID:WB-STRAIN:WBStrain00000001" + } + ] + }, + { + "key": "PR811", + "name": "PR811", + "background": [ + "N2" + ], + "node": "WBStrain:00030796", + "genotype": "osm-6(p811) V", + "species": { + "node": "NCBITaxon:6239", + "name": "Caenorhabditis elegans" + }, + "biological_sex": { + "node": "PATO:0001340", + "name": "hermaphrodite" + }, + "genetic_strain_type": "mutant", + "source": "cgc", + "identifiers": [ + { + "scheme": "RRID", + "value": "RRID:WB-STRAIN:WBStrain00030796" + } + ] + }, + { + "key": "TU253", + "name": "TU253", + "background": [ + "N2" + ], + "node": "WBStrain:00035037", + "genotype": "mec-4(u253) X", + "species": { + "node": "NCBITaxon:6239", + "name": "Caenorhabditis elegans" + }, + "biological_sex": { + "node": "PATO:0001340", + "name": "hermaphrodite" + }, + "genetic_strain_type": "mutant", + "source": "cgc", + "identifiers": [ + { + "scheme": "RRID", + "value": "RRID:WB-STRAIN:WBStrain00035037" + } + ] + }, + { + "key": "CB1112", + "name": "CB1112", + "background": [ + "N2" + ], + "node": "WBStrain:00004246", + "_node": "WormBase id from the V1 import (doImport.m, strainNames for strains its lookup could not resolve); the V2 search found none (2026-10-07).", + "genotype": "cat-2(e1112) II", + "species": { + "node": "NCBITaxon:6239", + "name": "Caenorhabditis elegans" + }, + "biological_sex": { + "node": "PATO:0001340", + "name": "hermaphrodite" + }, + "genetic_strain_type": "mutant", + "source": "cgc", + "_verify": "WormBase WBStrain id not found by search; RRID by strain name", + "identifiers": [ + { + "scheme": "RRID", + "value": "RRID:WB-STRAIN:CB1112" + } + ] + }, + { + "key": "CX6448", + "name": "CX6448", + "background": [ + "N2" + ], + "node": "WBStrain:00005280", + "_node": "WormBase id from the V1 import (doImport.m, strainNames for strains its lookup could not resolve); the V2 search found none (2026-10-07).", + "genotype": "gcy-35(ok769) I", + "species": { + "node": "NCBITaxon:6239", + "name": "Caenorhabditis elegans" + }, + "biological_sex": { + "node": "PATO:0001340", + "name": "hermaphrodite" + }, + "genetic_strain_type": "mutant", + "source": "cgc", + "_verify": "WormBase WBStrain id not found by search; RRID by strain name", + "identifiers": [ + { + "scheme": "RRID", + "value": "RRID:WB-STRAIN:CX6448" + } + ] + }, + { + "key": "DA465", + "name": "DA465", + "background": [ + "N2" + ], + "node": "WBStrain:00005463", + "genotype": "eat-2(ad465) II", + "species": { + "node": "NCBITaxon:6239", + "name": "Caenorhabditis elegans" + }, + "biological_sex": { + "node": "PATO:0001340", + "name": "hermaphrodite" + }, + "genetic_strain_type": "mutant", + "source": "cgc", + "identifiers": [ + { + "scheme": "RRID", + "value": "RRID:WB-STRAIN:WBStrain00005463" + } + ] + }, + { + "key": "DA609", + "name": "DA609", + "background": [ + "N2" + ], + "node": "WBStrain:00005493", + "genotype": "npr-1(ad609) X", + "species": { + "node": "NCBITaxon:6239", + "name": "Caenorhabditis elegans" + }, + "biological_sex": { + "node": "PATO:0001340", + "name": "hermaphrodite" + }, + "genetic_strain_type": "mutant", + "source": "cgc", + "identifiers": [ + { + "scheme": "RRID", + "value": "RRID:WB-STRAIN:WBStrain00005493" + } + ] + }, + { + "key": "FK103", + "name": "FK103", + "background": [ + "N2" + ], + "node": "WBStrain:00007498", + "_node": "Looked up by hand on wormbase.org by Jess, 2026-10-07 (ndi.ontology.lookup WBStrain: fails: the WormBase scrape and API both error).", + "genotype": "tax-4(ks28) III", + "species": { + "node": "NCBITaxon:6239", + "name": "Caenorhabditis elegans" + }, + "biological_sex": { + "node": "PATO:0001340", + "name": "hermaphrodite" + }, + "genetic_strain_type": "mutant", + "source": "cgc", + "_verify": "WormBase WBStrain id not found by search; RRID by strain name", + "identifiers": [ + { + "scheme": "RRID", + "value": "RRID:WB-STRAIN:FK103" + } + ] + }, + { + "key": "MT13113", + "name": "MT13113", + "background": [ + "N2" + ], + "node": "WBStrain:00027424", + "_node": "Looked up by hand on wormbase.org by Jess, 2026-10-07 (ndi.ontology.lookup WBStrain: fails: the WormBase scrape and API both error).", + "genotype": "tdc-1(n3419) II", + "species": { + "node": "NCBITaxon:6239", + "name": "Caenorhabditis elegans" + }, + "biological_sex": { + "node": "PATO:0001340", + "name": "hermaphrodite" + }, + "genetic_strain_type": "mutant", + "source": "cgc", + "_verify": "WormBase WBStrain id not found by search; RRID by strain name", + "identifiers": [ + { + "scheme": "RRID", + "value": "RRID:WB-STRAIN:MT13113" + } + ] + }, + { + "key": "MT15434", + "name": "MT15434", + "background": [ + "N2" + ], + "node": "WBStrain:00027519", + "_node": "WormBase id from the V1 import (doImport.m, strainNames for strains its lookup could not resolve); the V2 search found none (2026-10-07).", + "genotype": "tph-1(mg280) II", + "species": { + "node": "NCBITaxon:6239", + "name": "Caenorhabditis elegans" + }, + "biological_sex": { + "node": "PATO:0001340", + "name": "hermaphrodite" + }, + "genetic_strain_type": "mutant", + "source": "cgc", + "_verify": "WormBase WBStrain id not found by search; RRID by strain name", + "identifiers": [ + { + "scheme": "RRID", + "value": "RRID:WB-STRAIN:MT15434" + } + ] + }, + { + "key": "MT9455", + "name": "MT9455", + "background": [ + "N2" + ], + "node": "WBStrain:00027363", + "_node": "Looked up by hand on wormbase.org by Jess, 2026-10-07 (ndi.ontology.lookup WBStrain: fails: the WormBase scrape and API both error).", + "genotype": "tbh-1(n3247) X", + "species": { + "node": "NCBITaxon:6239", + "name": "Caenorhabditis elegans" + }, + "biological_sex": { + "node": "PATO:0001340", + "name": "hermaphrodite" + }, + "genetic_strain_type": "mutant", + "source": "cgc", + "_verify": "WormBase WBStrain id not found by search; RRID by strain name", + "identifiers": [ + { + "scheme": "RRID", + "value": "RRID:WB-STRAIN:MT9455" + } + ] + } + ], + "chemicals": [ + { + "key": "water", + "substance": "water" + }, + { + "key": "agar_emd", + "substance": "agar", + "product": "agar" + }, + { + "key": "nacl_fisher", + "substance": "sodium chloride", + "product": "nacl_s271" + }, + { + "key": "nacl", + "substance": "sodium chloride" + }, + { + "key": "peptone_bd", + "substance": "peptone", + "product": "peptone" + }, + { + "key": "kh2po4", + "substance": "potassium dihydrogen phosphate" + }, + { + "key": "k2hpo4", + "substance": "dipotassium hydrogen phosphate" + }, + { + "key": "mgso4_1m", + "substance": "magnesium sulfate", + "concentration": { + "molar": 1, + "source_value": 1, + "source_unit": "M" + } + }, + { + "key": "cacl2_1m", + "substance": "calcium chloride", + "concentration": { + "molar": 1, + "source_value": 1, + "source_unit": "M" + } + }, + { + "key": "cholesterol_5", + "substance": "cholesterol", + "concentration": { + "grams_per_liter": 5, + "source_value": 5, + "source_unit": "mg/mL" + }, + "_note": "5 mg/mL in ethanol." + }, + { + "key": "tryptone", + "substance": "tryptone" + }, + { + "key": "yeast_extract", + "substance": "yeast extract" + }, + { + "key": "citric_acid_monohydrate", + "substance": "citric acid monohydrate" + }, + { + "key": "tripotassium_citrate_monohydrate", + "substance": "tripotassium citrate monohydrate" + }, + { + "key": "edta_disodium", + "substance": "disodium EDTA" + }, + { + "key": "feso4", + "substance": "iron(II) sulfate heptahydrate" + }, + { + "key": "mncl2", + "substance": "manganese(II) chloride tetrahydrate" + }, + { + "key": "znso4", + "substance": "zinc sulfate heptahydrate" + }, + { + "key": "cuso4", + "substance": "copper(II) sulfate pentahydrate" + }, + { + "key": "carbenicillin_50", + "substance": "carbenicillin", + "concentration": { + "grams_per_liter": 50, + "source_value": 50, + "source_unit": "mg/mL" + }, + "_note": "Frozen stock, 50 mg/mL (231229_OP50_OP50-GFP_cultures)." + } + ], + "formulations": [ + { + "key": "kpo4_1m_ph6", + "type": "1 M potassium phosphate buffer, pH 6.0", + "ingredients": [ + { + "ingredient": "kh2po4", + "concentration": { + "grams_per_liter": 108.3, + "source_value": 108.3, + "source_unit": "g/L" + } + }, + { + "ingredient": "k2hpo4", + "concentration": { + "grams_per_liter": 35.6, + "source_value": 35.6, + "source_unit": "g/L" + } + }, + { + "ingredient": "water" + } + ], + "ph": 6.0, + "documented_by": [ + "wormbook_maintenance" + ] + }, + { + "key": "ngm", + "type": "NGM, 3% agar", + "ingredients": [ + { + "ingredient": "water", + "volume": { + "liters": 0.73, + "source_value": 730, + "source_unit": "mL" + } + }, + { + "ingredient": "agar_emd", + "mass": { + "grams": 22.5, + "source_value": 22.5, + "source_unit": "g" + } + }, + { + "ingredient": "nacl_fisher", + "mass": { + "grams": 2.25, + "source_value": 2.25, + "source_unit": "g" + } + }, + { + "ingredient": "peptone_bd", + "mass": { + "grams": 1.875, + "source_value": 1.875, + "source_unit": "g" + } + }, + { + "ingredient": "kpo4_1m_ph6", + "volume": { + "liters": 0.0188, + "source_value": 18.8, + "source_unit": "mL" + } + }, + { + "ingredient": "mgso4_1m", + "volume": { + "liters": 0.00075, + "source_value": 750, + "source_unit": "uL" + } + }, + { + "ingredient": "cacl2_1m", + "volume": { + "liters": 0.00075, + "source_value": 750, + "source_unit": "uL" + } + }, + { + "ingredient": "cholesterol_5", + "volume": { + "liters": 0.00075, + "source_value": 750, + "source_unit": "uL" + } + } + ], + "_note": "240102B_NGM_plates (Benchling; Kathleen Quach, adapted from WormBook): one batch, 25 mL per 100 mm plate." + }, + { + "key": "ngm_no_peptone", + "type": "NGM, 3% agar, no peptone", + "ingredients": [ + { + "ingredient": "water", + "volume": { + "liters": 0.73, + "source_value": 730, + "source_unit": "mL" + } + }, + { + "ingredient": "agar_emd", + "mass": { + "grams": 22.5, + "source_value": 22.5, + "source_unit": "g" + } + }, + { + "ingredient": "nacl_fisher", + "mass": { + "grams": 2.25, + "source_value": 2.25, + "source_unit": "g" + } + }, + { + "ingredient": "kpo4_1m_ph6", + "volume": { + "liters": 0.0188, + "source_value": 18.8, + "source_unit": "mL" + } + }, + { + "ingredient": "mgso4_1m", + "volume": { + "liters": 0.00075, + "source_value": 750, + "source_unit": "uL" + } + }, + { + "ingredient": "cacl2_1m", + "volume": { + "liters": 0.00075, + "source_value": 750, + "source_unit": "uL" + } + }, + { + "ingredient": "cholesterol_5", + "volume": { + "liters": 0.00075, + "source_value": 750, + "source_unit": "uL" + } + } + ], + "_note": "240102B_NGM_plates without the peptone (the source `peptone` = without)." + }, + { + "key": "s_basal", + "type": "S Basal", + "ingredients": [ + { + "ingredient": "nacl", + "concentration": { + "grams_per_liter": 5.85, + "source_value": 5.85, + "source_unit": "g/L" + } + }, + { + "ingredient": "k2hpo4", + "concentration": { + "grams_per_liter": 1, + "source_value": 1, + "source_unit": "g/L" + } + }, + { + "ingredient": "kh2po4", + "concentration": { + "grams_per_liter": 6, + "source_value": 6, + "source_unit": "g/L" + } + }, + { + "ingredient": "cholesterol_5", + "concentration": { + "volume_fraction": 0.001, + "source_value": 1, + "source_unit": "mL/L" + } + }, + { + "ingredient": "water" + } + ], + "documented_by": [ + "wormbook_maintenance" + ] + }, + { + "key": "trace_metals", + "type": "trace metals solution", + "ingredients": [ + { + "ingredient": "edta_disodium", + "concentration": { + "grams_per_liter": 1.86, + "source_value": 1.86, + "source_unit": "g/L" + } + }, + { + "ingredient": "feso4", + "concentration": { + "grams_per_liter": 0.69, + "source_value": 0.69, + "source_unit": "g/L" + } + }, + { + "ingredient": "mncl2", + "concentration": { + "grams_per_liter": 0.2, + "source_value": 0.2, + "source_unit": "g/L" + } + }, + { + "ingredient": "znso4", + "concentration": { + "grams_per_liter": 0.29, + "source_value": 0.29, + "source_unit": "g/L" + } + }, + { + "ingredient": "cuso4", + "concentration": { + "grams_per_liter": 0.025, + "source_value": 0.025, + "source_unit": "g/L" + } + }, + { + "ingredient": "water" + } + ], + "documented_by": [ + "wormbook_maintenance" + ] + }, + { + "key": "k_citrate_1m_ph6", + "type": "1 M potassium citrate, pH 6.0", + "ingredients": [ + { + "ingredient": "citric_acid_monohydrate", + "concentration": { + "grams_per_liter": 20, + "source_value": 20, + "source_unit": "g/L" + } + }, + { + "ingredient": "tripotassium_citrate_monohydrate", + "concentration": { + "grams_per_liter": 293.5, + "source_value": 293.5, + "source_unit": "g/L" + } + }, + { + "ingredient": "water" + } + ], + "ph": 6.0, + "documented_by": [ + "wormbook_maintenance" + ] + }, + { + "key": "s_complete", + "type": "S-Complete", + "ingredients": [ + { + "ingredient": "s_basal", + "volume": { + "liters": 1, + "source_value": 1, + "source_unit": "L" + } + }, + { + "ingredient": "k_citrate_1m_ph6", + "volume": { + "liters": 0.01, + "source_value": 10, + "source_unit": "mL" + } + }, + { + "ingredient": "trace_metals", + "volume": { + "liters": 0.01, + "source_value": 10, + "source_unit": "mL" + } + }, + { + "ingredient": "cacl2_1m", + "volume": { + "liters": 0.003, + "source_value": 3, + "source_unit": "mL" + } + }, + { + "ingredient": "mgso4_1m", + "volume": { + "liters": 0.003, + "source_value": 3, + "source_unit": "mL" + } + } + ], + "product": "s_complete_kitchen", + "documented_by": [ + "wormbook_maintenance" + ], + "_note": "S Medium in WormBook; made by the Salk media kitchen. The diluent for every Matching seeding (decision #56)." + }, + { + "key": "lb", + "type": "LB", + "ingredients": [ + { + "ingredient": "tryptone", + "concentration": { + "grams_per_liter": 10, + "source_value": 10, + "source_unit": "g/L" + } + }, + { + "ingredient": "yeast_extract", + "concentration": { + "grams_per_liter": 5, + "source_value": 5, + "source_unit": "g/L" + } + }, + { + "ingredient": "nacl", + "concentration": { + "grams_per_liter": 5, + "source_value": 5, + "source_unit": "g/L" + } + }, + { + "ingredient": "water" + } + ], + "product": "lb_kitchen", + "documented_by": [ + "wormbook_maintenance" + ], + "_note": "WormBook's LB (5 g NaCl per litre, the Lennox form); made by the Salk media kitchen." + }, + { + "key": "lb_carb", + "type": "LB with carbenicillin", + "ingredients": [ + { + "ingredient": "lb" + }, + { + "ingredient": "carbenicillin_50", + "concentration": { + "grams_per_liter": 0.1, + "source_value": 100, + "source_unit": "ug/mL" + } + } + ], + "_note": "OP50-GFP is grown and diluted in it (231229_OP50_OP50-GFP_cultures: 300 uL of 50 mg/mL per 150 mL)." + } + ], + "instruments": [ + { + "key": "camera_1", + "local_identifier": "camera1", + "name": "Camera 1", + "product": "camera", + "description": "Behaviour camera (1 of 2; experimentInfo `camera`)." + }, + { + "key": "camera_2", + "local_identifier": "camera2", + "name": "Camera 2", + "product": "camera", + "description": "Behaviour camera (2 of 2; experimentInfo `camera`)." + }, + { + "key": "axio_zoom_1", + "local_identifier": "axiozoom1", + "name": "Axio Zoom.V16", + "product": "axio_zoom", + "description": "Fluorescence stereo microscope used for the E. coli lawn imaging." + }, + { + "key": "spectrophotometer_1", + "local_identifier": "spectramax1", + "name": "SpectraMax Plus 384", + "product": "spectrophotometer" + }, + { + "key": "temperature_probe_1", + "local_identifier": "temperatureprobe1", + "name": "Temperature probe", + "description": "The probe at the behaviour rigs that read the temperature and humidity of each recording (experimentInfo `temp`, `humidity`); make and model not recorded." + } + ], + "environment": { + "instrument": "temperature_probe_1", + "temperature": { + "variable": "ambient temperature" + }, + "humidity": { + "variable": "relative humidity" + }, + "notes": "The probe's mean over the recording, noted by hand." + }, + "studies": [ + { + "key": "single_density_multi_patch", + "name": "Single-density, multi-patch assay", + "short_name": "foragingConcentration (grid)", + "source_folder": "foragingConcentration", + "source_condition": "grid", + "description": "Four young adult hermaphrodites forage in a 30 mm arena on an isometric grid of 19 patches (0.5 µl, 6 mm apart), all of one OP50 density (OD600 0-10; also 1 (12H) and 1 (48H) grown patches); 1 h at 3 fps.", + "factors": [ + { + "node": "", + "name": "OD600" + }, + { + "node": "", + "name": "bacterial growth time" + } + ] + }, + { + "key": "large_single_patch", + "name": "Large, single-patch assay", + "short_name": "foragingConcentration (single)", + "source_folder": "foragingConcentration", + "source_condition": "single", + "description": "As the single-density multi-patch assay, but one large 20 µl patch in the centre of the 30 mm arena.", + "factors": [ + { + "node": "", + "name": "OD600" + } + ] + }, + { + "key": "small_single_patch", + "name": "Small, single-patch assay", + "short_name": "foragingMini", + "source_folder": "foragingMini", + "description": "One young adult in a 9 mm arena with one 0.5 µl patch (OD600 0, 0.5, 1, 5, 10); 1 h at 8 fps and ~105 px/mm, head and tail tracked.", + "factors": [ + { + "node": "", + "name": "OD600" + } + ] + }, + { + "key": "multi_density_multi_patch", + "name": "Multi-density, multi-patch assay", + "short_name": "foragingMatching", + "source_folder": "foragingMatching", + "description": "Grids of 18 patches combining OD600 1, 5 and 10 on peptone-free NGM; animals released from an S-Complete droplet at the arena centre; 2 h recordings.", + "factors": [ + { + "node": "", + "name": "OD600" + }, + { + "node": "", + "name": "patch configuration" + } + ] + }, + { + "key": "mutant_screen", + "name": "Mutant screen and food deprivation (single-density, multi-patch)", + "short_name": "foragingMutants", + "source_folder": "foragingMutants", + "description": "Single-density, multi-patch environments with ten mutant strains and N2, well-fed and 3-h food-deprived. The paper's food-deprivation analysis is the N2 subset; most strains are not reported in the paper.", + "factors": [ + { + "node": "", + "name": "strain" + }, + { + "node": "", + "name": "food deprivation" + } + ] + }, + { + "key": "sensory_mutants", + "name": "Sensory mutants (single-density, multi-patch)", + "short_name": "foragingSensory", + "source_folder": "foragingSensory", + "description": "Wild type (N2) and the sensory mutants osm-6 (PR811) and mec-4 (TU253) in single-density, multi-patch environments.", + "factors": [ + { + "node": "", + "name": "strain" + } + ] + }, + { + "key": "patch_density_estimation", + "name": "Bacterial patch density estimation", + "short_name": "ecoli", + "source_folder": "ecoli", + "description": "OP50-GFP patches seeded as in the behaviour assays and imaged for several hours on a Zeiss Axio Zoom.V16 to estimate relative bacterial density from the patch-edge fluorescence.", + "factors": [ + { + "node": "", + "name": "OD600" + }, + { + "node": "", + "name": "bacterial growth time" + } + ] + } + ], + "dataset": { + "name": "Accept-reject decision-making revealed via a quantitative and ethological study of C. elegans foraging", + "short_name": "haley_2025", + "description": "All raw (behaviour videos, fluorescence microscopy images) and processed (animal and bacterial patch locations over time) data for Haley et al., eLife (2025).", + "keyword": [ + "C. elegans", + "foraging", + "decision-making", + "behavior", + "E. coli" + ], + "license": "CC-BY-4.0", + "accessibility": { + "node": "", + "name": "free access" + }, + "ethics_assessment": { + "node": "", + "name": "not required" + }, + "release_date": "2025-08-01", + "identifiers": [ + { + "scheme": "DOI", + "value": "https://doi.org/10.63884/ndic.2025.pb77mj2s" + } + ], + "authors": [ + { + "person": "haley", + "roles": [ + "conceptualization", + "resources", + "data curation", + "software", + "formal analysis", + "supervision", + "funding acquisition", + "validation", + "investigation", + "visualization", + "methodology", + "writing - original draft", + "project administration", + "writing - review & editing" + ] + }, + { + "person": "chen", + "roles": [ + "data curation", + "investigation", + "writing - review & editing" + ] + }, + { + "person": "aoi", + "roles": [ + "formal analysis", + "supervision", + "visualization", + "methodology", + "writing - review & editing", + "corresponding author" + ] + }, + { + "person": "chalasani", + "roles": [ + "conceptualization", + "resources", + "supervision", + "funding acquisition", + "writing - review & editing", + "corresponding author" + ] + } + ], + "funding": [ + "nih_r01", + "dorsett_brown_award", + "salk_innovation", + "nsf_grfp", + "ucsd_ssra" + ], + "cites": [ + "elife_2025" + ], + "stored_at": [ + "ndi_cloud_page" + ], + "documented_by": [ + "code" + ] + }, + "transfer_protocol": { + "_about": "How and when the worms were moved onto each assay plate -- an import directive (decision #52), not dataset metadata. The transfer time is not recorded: on a plate whose 'contrast' (lawn) video was filmed AFTER the worms went on, the transfer is taken as that video's start; otherwise as the first behaviour video's start; approximate either way. Methods: the eLife paper, Assay preparation and recording (agar plugs; eyelash pick into an S-Complete droplet for multi-density). Contrast-video order: from the gap between each plate's lawn clip and its recording, checked by eye in the videos, 2026-10-02 (Matching day 1, 2023-02-24, is worms first; days 2-3 lawn first).", + "max_before_seconds": 300, + "_max_before": "T is read off the first video after the transfer, so the transfer was at most this long before it (Jess Haley, 2026-10-02: 5 min; 10 would be big). On a lawn-first plate the lawn clip's end bounds it more tightly when closer.", + "hand_written_tolerance_seconds": [60, 60], + "_hand_written": "Pick (growthTimePicked) and food-deprivation (starvedTime) times were read off a clock -- possibly analog (e.g. an Apple Watch face) -- and written to the minute: +/- 60 s covers a truncated digital reading and a by-eye analog one (Jess Haley, 2026-10-02).", + "rules": [ + { + "source_folder": "foragingConcentration", + "method": "agar plug, through an empty NGM plate", + "contrast_video": "before_worms" + }, + { + "source_folder": "foragingMini", + "method": "agar plug, through an empty NGM plate", + "contrast_video": "after_worms" + }, + { + "source_folder": "foragingMatching", + "to_date": "2023-02-24", + "method": "eyelash pick into an S-Complete droplet", + "contrast_video": "after_worms" + }, + { + "source_folder": "foragingMatching", + "from_date": "2023-03-01", + "to_date": "2023-04-04", + "method": "eyelash pick into an S-Complete droplet", + "contrast_video": "before_worms" + }, + { + "source_folder": "foragingMatching", + "from_date": "2023-05-18", + "method": "eyelash pick into an S-Complete droplet", + "contrast_video": "after_worms" + }, + { + "source_folder": "foragingMutants", + "method": "eyelash pick into an S-Complete droplet", + "contrast_video": "after_worms" + }, + { + "source_folder": "foragingSensory", + "method": "eyelash pick into an S-Complete droplet", + "contrast_video": "after_worms" + } + ], + "exceptions": [ + { + "video": "2023-04-04_16-10-14_2", + "contrast_video": "after_worms", + "why": "worms visible in the contrast video (checked by eye, 2026-10-02); the rest of that day was lawn first" + }, + { + "video": "2022-09-02_08-48-41_1", + "contrast_video": "after_worms", + "why": "lawn clip 2.6 s before recording; the contrast video was taken late, once the worms were on (Jess Haley, 2026-10-02); the rest of that day was lawn first" + }, + { + "video": "2022-09-02_08-48-41_2", + "contrast_video": "after_worms", + "why": "as 2022-09-02_08-48-41_1, the other camera of the same pair" + } + ] + }, + "seeding": { + "_about": "Stage 8 part B (decisions #56, #57): which strain and diluent each study's seeding suspensions used. Read by ndi.setup.conv.haley.seedingSuspensions. Acclimation (growth) plates were all seeded the same way, 200 uL at OD600 1 in LB (the paper; Jess, 2026-10-03: Matching's too), so a growth seeding uses that day's LB solution: its own study's, else another study's (Matching's days have Mini's).", + "strain": { + "default": "OP50", + "ecoli": "OP50-GFP" + }, + "diluent": { + "default": "lb", + "foragingMatching": "s_complete", + "ecoli": "lb_carb" + }, + "growth_diluent": "lb" + }, + "preparation": { + "_about": "Stage 8 part C (decisions #54, #56, #57): how the plates were prepared and the worms moved, read by ndi.setup.conv.haley.manipulationDocuments. Every value is from the eLife paper's Methods, the lab's Benchling protocols or Jess Haley; terms are names until the ontology lookup.", + "pour": { + "_about": "Every plate is 25 mL of NGM in a 100 mm dish (240102B_NGM_plates; Jess: Mini's too). The agar is NGM, or NGM without peptone where the plate's `peptone` says so (Matching, E. coli); acclimation and food deprivation plates have no `peptone` column and are the plain recipe.", + "variable": { + "with": "NGM agar", + "without": "NGM agar (no peptone)" + }, + "method": "pouring", + "volume_ml": 25, + "agar": { + "with": "ngm", + "without": "ngm_no_peptone" + }, + "default_peptone": "with", + "notes": { + "ecoli": "The NGM entry for this pour records no antibiotic. Carbenicillin was probably added (the NGM protocol requires an antibiotic for plasmid-carrying bacteria such as OP50-GFP) but is unrecorded (Jess Haley, 2026-10-02)." + } + }, + "seed": { + "_about": "One dose per patch at the patch's volume (`lawnVolume`) and OD600; one per acclimation plate, 200 uL (the paper: 'one large patch of 200 µl of OD600 = 1'). The suspension is the day's (seedingSuspensions).", + "method": "seeding", + "growth_volume_ul": 200 + }, + "temperature": { + "_about": "The temperature a plate was moved INTO, not heat applied; approximate, no tolerance (no bound is known). Cold room ~4 C, room temperature 20 C (the paper: 'stored at 20C until the experiment'); acclimation and food deprivation plates go into the 20 C incubator when the worms are put on them.", + "variable": "ambient temperature", + "methods": { + "cold_room": "refrigeration", + "room": "ambient exposure", + "incubator": "incubation" + }, + "_methods": "How the temperature was imposed (Jess, 2026-10-03): the method is what tells the uncontrolled bench (ambient exposure) from the controlled 20 C incubator (incubation).", + "cold_room_celsius": 4, + "room_celsius": 20, + "incubator_celsius": 20, + "out_before_seeding_seconds": 300, + "_out_before_seeding": "An E. coli plate left the cold room shortly before it was seeded: the paper seeds onto cold, low-moisture NGM plates (Methods, Bacterial cultures), and Jess puts it at about 5 min. So the cold-room window after pouring ends at the seeding time, approximate, up to this much earlier (plus the seeding time's own hand-written bound)." + }, + "transfer": { + "_about": "One term_manipulation per move, on the cohort, at its contained_in relation's time. The variable is the cohort's location and the value the kind of plate moved to (the plate itself is the parent of the contained_in relation at the same time); the method the move's last step (onto a food deprivation plate: agar plug, per Jess); every move onto an assay plate first went through an empty NGM plate for about a minute to shed bacteria (Jess, 2026-10-02).", + "variable": "location", + "pick_method": "worm picking", + "deprivation_method": "agar plug transfer", + "assay_methods": [ + { + "contains": "eyelash", + "method": "eyelash picking", + "droplet": "S-Complete" + }, + { + "contains": "agar plug", + "method": "agar plug transfer" + } + ], + "cleaning": "agar plug onto an empty NGM plate for about 1 min, to shed bacteria" + }, + "food_deprivation": { + "variable": "food availability", + "value": "none" + }, + "developmental_stage": { + "_about": "One term_observation per cohort (decision #69, Jess 2026-10-08): the worms were chosen as L4 larvae by eye when picked onto the acclimation plate. At the pick's time (hand-written, with its tolerance), or, with no pick time, a relative time before the cohort's next move. Distributive: it holds of each worm. No stage at the assay: it follows from the pick and assay times. Names alone until the terms are looked up (termInventory lists them).", + "variable": "developmental stage", + "value": "L4 larva", + "method": "visual inspection" + } + }, + "ecoli_patch_layout": {"positions_mm": [[-10.46, -5.199], [-10.425, 5.262], [-7.491, 0.06], [-4.439, -5.217], [-4.374, 5.249], [-1.375, -0.008], [1.649, -5.192], [1.619, 5.185], [4.698, -0.059], [7.719, -5.29], [7.734, 5.231], [10.686, -0.068]], "angle_sign": -1, "max_distance_mm": 2.0, "source": "ndi.setup.conv.haley.patchLayout, run by Jess Haley 2026-10-14: the median position of each patch, in the acetate template's frame (mm), over every analysed full 12-patch image of a rectangle plate; median deviation 0.26-0.33 mm per patch, maximum 1.54 mm; closest two positions 5.97 mm apart. The angle sign is immaterial at these angles (median deviation 0.303 mm with +, 0.301 mm with -). max_distance_mm is a third of the 5.97 mm spacing, above the largest deviation seen (decision #80)."}, + "corrections": [ + { + "key": "conditions_concentration_experiment_14", + "folder": "foragingConcentration", + "experiment": 14, + "table": "tableOfContents", + "column": "conditions", + "value": "single [OD600 = 0, 0.05, 0.1, 0.5, 1, 2, 3, 4, 5, 10, 1 (48H)]", + "was": "grid [OD600 = 0, 0.05, 0.1, 0.5, 1, 2, 3, 4, 5, 10, 1 (48H)]", + "reason": "Row 0014 says grid, but all 12 of its plates (133-144) say single in experimentInfo and each has one patch, as experiment 3's single plates do, while the grid days either side (13, 15) have 19 patches per plate. The row's first word is a copy of its neighbours'; the session is in large_single_patch (Jess Haley, 2026-10-14; decision #72)." + }, + { + "key": "cfu_matching_2023_03_27", + "folder": "foragingMatching", + "seed_day": "2023-03-27", + "column": "CFU", + "value": null, + "was": 846, + "reason": "The S-Complete seeding of 27 Mar 2023 measured no CFU (230327_seeding_S-complete has no CFU section); 846 is the LB seeding's (Mini) count from the same day, copied (Jess, 2026-10-03)." + }, + { + "key": "cfu_ecoli_2022_04_28", + "folder": "ecoli", + "seed_day": "2022-04-28", + "column": "CFU", + "value": null, + "was": 457, + "reason": "457 is the OP50 count of Concentration's seeding the same day; an OP50-GFP suspension cannot share it, so it was copied and the real value is unknown (Jess, 2026-10-03)." + }, + { + "key": "peptone_ecoli_plate_90", + "folder": "ecoli", + "plate": 90, + "column": "peptone", + "value": "without", + "was": "with", + "reason": "Plate 90 is labelled with peptone, but its label is wrong: it is from the 16 Nov 2023 pour, without peptone (Jess Haley, 2026-10-02; decision #56)." + }, + { + "key": "growth_room_temp_mini_experiment_24", + "folder": "foragingMini", + "experiment": 24, + "column": "growthTimeRoomTemp", + "value": "2023-08-28T09:02", + "was": "28 Aug 2023 09:03 (plate 223) down to 08:52 (plate 234), one minute earlier per plate", + "reason": "The 12 assay plates of foragingMini experiment 24 share one acclimation plate (one seeding, cold room and pick), but growthTimeRoomTemp steps down a minute per plate, a spreadsheet fill-down; it is the only one of the 24 Mini experiments with more than one. Plates 225 and 230 were prepared on the acclimation plate's schedule (cold room 24 Aug 14:25, the acclimation plate's) and record its room temperature time: 28 Aug 09:02. On experiment 23 the same pairing (plates 213, 218) gives exactly that day's single time. Inferred from the data, 2026-10-08: Jess does not recall the day, and her notes were not checked (decision #70e)." + }, + { + "key": "room_temp_estimated_mini_plate_55", + "folder": "foragingMini", + "plate": 55, + "column": "timeRoomTemp", + "approximate": true, + "reason": "timeRoomTemp is exactly the recording's start minus 1 h, the value getExperimentInfo.m fills in when the notebook cell is blank (shreklab/Haley-et-al-2024 at 44a89de, getExperimentInfo.m:269-270). The notebook is not with the data, so the blank cannot be confirmed; it is marked an estimate with no known bound (decision #74, Jess Haley 2026-10-14)." + }, + { + "key": "room_temp_estimated_mini_plate_189", + "folder": "foragingMini", + "plate": 189, + "column": "timeRoomTemp", + "approximate": true, + "reason": "timeRoomTemp is exactly the recording's start minus 1 h, the value getExperimentInfo.m fills in when the notebook cell is blank (shreklab/Haley-et-al-2024 at 44a89de, getExperimentInfo.m:269-270). The notebook is not with the data, so the blank cannot be confirmed; it is marked an estimate with no known bound (decision #74, Jess Haley 2026-10-14)." + }, + { + "key": "growth_room_temp_sensory_experiment_2", + "folder": "foragingSensory", + "experiment": 2, + "column": "growthTimeRoomTemp", + "value": "2024-01-04T09:49", + "was": "5 Jan 2024 09:49 (every plate of experiment 2)", + "reason": "Experiment 2's acclimation plates are recorded out of the cold room on 5 Jan 2024 09:49, after their worms were picked onto them on 4 Jan (11:08, 11:16, 11:24); every other step is in order (seeded 29 Dec 11:16, cold room 30 Dec 11:16, recorded 5 Jan from 08:11). The day is a typo, likely copied from the recording date: 4 Jan 09:49 is 1 h 19 min to 1 h 35 min before the picks, in line with other days (Jess Haley, 2026-10-14; decision #77)." + }, + { + "key": "room_temp_estimated_concentration_plate_01", + "folder": "foragingConcentration", + "plate": 1, + "column": "timeRoomTemp", + "approximate": true, + "reason": "timeRoomTemp was not recorded for this plate; it was filled in as the recording's start minus 1 h (getExperimentInfo.m), so it is an estimate with no known bound (decision #54d; listed from the data, Jess Haley 2026-10-03)." + }, + { + "key": "room_temp_estimated_concentration_plate_02", + "folder": "foragingConcentration", + "plate": 2, + "column": "timeRoomTemp", + "approximate": true, + "reason": "timeRoomTemp was not recorded for this plate; it was filled in as the recording's start minus 1 h (getExperimentInfo.m), so it is an estimate with no known bound (decision #54d; listed from the data, Jess Haley 2026-10-03)." + }, + { + "key": "room_temp_estimated_concentration_plate_03", + "folder": "foragingConcentration", + "plate": 3, + "column": "timeRoomTemp", + "approximate": true, + "reason": "timeRoomTemp was not recorded for this plate; it was filled in as the recording's start minus 1 h (getExperimentInfo.m), so it is an estimate with no known bound (decision #54d; listed from the data, Jess Haley 2026-10-03)." + }, + { + "key": "room_temp_estimated_concentration_plate_04", + "folder": "foragingConcentration", + "plate": 4, + "column": "timeRoomTemp", + "approximate": true, + "reason": "timeRoomTemp was not recorded for this plate; it was filled in as the recording's start minus 1 h (getExperimentInfo.m), so it is an estimate with no known bound (decision #54d; listed from the data, Jess Haley 2026-10-03)." + }, + { + "key": "room_temp_estimated_concentration_plate_05", + "folder": "foragingConcentration", + "plate": 5, + "column": "timeRoomTemp", + "approximate": true, + "reason": "timeRoomTemp was not recorded for this plate; it was filled in as the recording's start minus 1 h (getExperimentInfo.m), so it is an estimate with no known bound (decision #54d; listed from the data, Jess Haley 2026-10-03)." + }, + { + "key": "room_temp_estimated_concentration_plate_06", + "folder": "foragingConcentration", + "plate": 6, + "column": "timeRoomTemp", + "approximate": true, + "reason": "timeRoomTemp was not recorded for this plate; it was filled in as the recording's start minus 1 h (getExperimentInfo.m), so it is an estimate with no known bound (decision #54d; listed from the data, Jess Haley 2026-10-03)." + }, + { + "key": "room_temp_estimated_concentration_plate_07", + "folder": "foragingConcentration", + "plate": 7, + "column": "timeRoomTemp", + "approximate": true, + "reason": "timeRoomTemp was not recorded for this plate; it was filled in as the recording's start minus 1 h (getExperimentInfo.m), so it is an estimate with no known bound (decision #54d; listed from the data, Jess Haley 2026-10-03)." + }, + { + "key": "room_temp_estimated_concentration_plate_08", + "folder": "foragingConcentration", + "plate": 8, + "column": "timeRoomTemp", + "approximate": true, + "reason": "timeRoomTemp was not recorded for this plate; it was filled in as the recording's start minus 1 h (getExperimentInfo.m), so it is an estimate with no known bound (decision #54d; listed from the data, Jess Haley 2026-10-03)." + }, + { + "key": "room_temp_estimated_concentration_plate_09", + "folder": "foragingConcentration", + "plate": 9, + "column": "timeRoomTemp", + "approximate": true, + "reason": "timeRoomTemp was not recorded for this plate; it was filled in as the recording's start minus 1 h (getExperimentInfo.m), so it is an estimate with no known bound (decision #54d; listed from the data, Jess Haley 2026-10-03)." + }, + { + "key": "room_temp_estimated_concentration_plate_10", + "folder": "foragingConcentration", + "plate": 10, + "column": "timeRoomTemp", + "approximate": true, + "reason": "timeRoomTemp was not recorded for this plate; it was filled in as the recording's start minus 1 h (getExperimentInfo.m), so it is an estimate with no known bound (decision #54d; listed from the data, Jess Haley 2026-10-03)." + } + ] +} diff --git a/src/ndi/+ndi/+setup/+conv/+haley/manipulationDocuments.m b/src/ndi/+ndi/+setup/+conv/+haley/manipulationDocuments.m new file mode 100644 index 0000000000..13fc207e8f --- /dev/null +++ b/src/ndi/+ndi/+setup/+conv/+haley/manipulationDocuments.m @@ -0,0 +1,465 @@ +function out = manipulationDocuments(session, S, subjectIds, options) +%MANIPULATIONDOCUMENTS Stage 8 (Haley): how one session's plates were made and its worms moved. +% +% OUT = ndi.setup.conv.haley.manipulationDocuments(SESSION, S, SUBJECTIDS, ...) +% builds the manipulation documents of the one-row session table SESSION +% (from sessionList, with `session_id` assigned). S is the subject table +% (subjectList, whose `prep` column holds each plate's and patch's times, +% peptone, OD600 and volume); SUBJECTIDS maps each subject's +% local_identifier to its document id. Nothing is written here. Decision +% log #54, #56, #57; every term is a name until the ontology lookup. +% +% Plates (assay, acclimation, food deprivation, E. coli): +% pour dose_manipulation: 25 mL of NGM agar (or NGM agar without +% peptone, from the plate's `peptone`; only Matching and the +% E. coli study used it), formulation_id the recipe. +% When: E. coli `timePoured`; a C. elegans plate, which has no +% pour time, a relative_time_reference `intervalBefore` its +% seeding (an unseeded food deprivation plate: before the +% worms went on). No time, no pour. +% seeding dose_manipulation per patch (assay and E. coli plates) at +% the patch's volume, and per acclimation plate (200 uL); +% variable the strain (the diluent's name for a patch of +% diluent alone), formulation_id that day's suspension +% (seedingSuspensions' index). When: the seeding time. +% temperature temperature_manipulation per move: the temperature moved +% INTO, approximate. Assay plate: cold room (`timeColdRoom` +% to `timeRoomTemp`), room temperature (`timeRoomTemp` to the +% end of filming). Acclimation plate: cold room +% (`growthTimeColdRoom` to `growthTimeRoomTemp`), room +% temperature (to the pick), incubator (the pick to the last +% move off it). Food deprivation plate: incubator +% (`starvedTime` to the last move off it). E. coli plate: cold +% room after pouring (to seeding, which was onto cold plates: +% the end is up to 5 min early), cold room after seeding (to +% `timeRoomTemp`), room temperature (from `timeRoomTemp`). +% Seeding to cold room is at room temperature with no move: +% none. A `timeRoomTemp` the spec's corrections mark as +% estimated is approximate with no bound, and says so. +% Worm cohorts (`distributive` true when the schema has it): +% transfer term_manipulation per move (acclimation, food +% deprivation, assay plate) at the time of the cohort's +% contained_in relation (stage 6): variable `location`, +% value the kind of plate moved to (the plate itself is +% the relation's parent), method the move's technique +% (worm picking; agar plug transfer; onto the assay +% plate agar plug transfer or eyelash picking, from the +% transfer protocol, with the cleaning step through an +% empty NGM plate and any S-Complete droplet as method +% parameters). +% food deprivation term_manipulation `food availability` = `none` at +% the food deprivation window. +% Each temperature manipulation's method says how it was imposed: +% refrigeration (cold room), ambient exposure (the bench), incubation (the +% 20 C incubator) -- the method is what tells room temperature from the +% incubator. +% Hand-written times are approximate, +/- the hand-written tolerance (to the +% minute); video times (the end of filming) are exact. +% +% Options: +% 'Preparation' the spec's `preparation` section (required) +% 'Seeding' the spec's `seeding` section (strain per folder) +% 'Ids' containers.Map: dataset-level key -> document id (stage 2: +% recipes, strains, suspensions) +% 'Names' containers.Map: formulation or strain key -> its name +% 'Suspensions' seedingSuspensions' index table +% 'Relations' relationDocuments' output (timeReferenceIds, assaySpans) +% 'HandTolerance' [minus plus] s for a hand-written time (default [60 60]) +% +% OUT fields: documents (cell of structs: manipulations and their time +% references), counts (struct), skipped (cellstr: a manipulation not made, +% and why). + +arguments + session table + S table + subjectIds + options.Preparation struct + options.Seeding struct = struct() + options.Ids = containers.Map() + options.Names = containers.Map() + options.Suspensions = table() + options.Relations = struct() + options.HandTolerance (1,2) double = [60 60] +end +if height(session) ~= 1 + error('ndi:setup:conv:haley:oneSession', 'Give exactly one session row.'); +end +sid = char(session.session_id{1}); +ref = char(session.local_identifier{1}); +tz = 'America/Los_Angeles'; +S = S(strcmp(S.session, ref), :); +cfg = options.Preparation; +hand = options.HandTolerance; +minuteFmt = 'yyyy-MM-dd''T''HH:mm'; +secondFmt = 'yyyy-MM-dd''T''HH:mm:ss'; +refs = getOr(options.Relations, 'timeReferenceIds', containers.Map()); +spans = getOr(options.Relations, 'assaySpans', containers.Map()); +eachMember = struct(); +if ndi.setup.V2.schemaHasField('statement', 'distributive') + eachMember.distributive = true; +end + +docs = {}; +out = struct('documents', {{}}, 'skipped', {{}}, 'counts', struct('pour', 0, ... + 'seed_patch', 0, 'seed_acclimation_plate', 0, 'temperature', 0, 'transfer', 0, ... + 'food_deprivation', 0, 'developmental_stage', 0)); + +% ---- plates ------------------------------------------------------------------ +plateKinds = {'assay_plate', 'acclimation_plate', 'food_deprivation_plate', 'plate'}; +for k = find(ismember(S.kind, plateKinds))' + id = S.local_identifier{k}; + kind = S.kind{k}; + folder = S.folder{k}; + p = S.prep(k); + roomTol = hand; + if ~isempty(p.room_temp_note), roomTol = 'estimated'; end + + % seeding + seedRef = ''; + if ~isnat(p.seeded) && ismember(kind, {'assay_plate', 'plate', 'acclimation_plate'}) + seedTol = hand; + if isfield(p, 'seeded_spread_s') + % a seeding time known only to a span (decision #75) + seedTol = hand + p.seeded_spread_s; + end + seedRef = instant(p.seeded, seedTol); + if strcmp(kind, 'acclimation_plate') + seedAcclimation(id, folder, p, seedRef); + else + pk = find(strcmp(S.kind, 'patch') & startsWith(S.local_identifier, [id '_patch']))'; + for j = pk + seedPatch(S(j, :), folder, p.seeded, seedRef); + end + end + end + + % temperature + incubatorRef = ''; + switch kind + case 'assay_plate' + temperature(id, p.cold_room, hand, p.room_temp, roomTol, minuteFmt, ... + cfg.temperature.cold_room_celsius, '', cfg.temperature.methods.cold_room); + stop = NaT; + if isKey(spans, id) + span = spans(id); % (a Map takes one level of indexing) + stop = span.stop; + end + temperature(id, p.room_temp, roomTol, stop, [], secondFmt, ... + cfg.temperature.room_celsius, p.room_temp_note, cfg.temperature.methods.room); + case 'acclimation_plate' + placed = S.worms_placed(k); + temperature(id, p.cold_room, hand, p.room_temp, hand, minuteFmt, ... + cfg.temperature.cold_room_celsius, '', cfg.temperature.methods.cold_room); + % a named correction of the time (decision #70e) is its note + temperature(id, p.room_temp, hand, placed, hand, minuteFmt, ... + cfg.temperature.room_celsius, p.room_temp_note, cfg.temperature.methods.room); + [stop, stopTol, stopFmt] = lastMoveOff(id); + incubatorRef = temperature(id, placed, hand, stop, stopTol, stopFmt, ... + cfg.temperature.incubator_celsius, '', cfg.temperature.methods.incubator); + case 'food_deprivation_plate' + [stop, stopTol, stopFmt] = lastMoveOff(id); + incubatorRef = temperature(id, S.worms_placed(k), hand, stop, stopTol, stopFmt, ... + cfg.temperature.incubator_celsius, '', cfg.temperature.methods.incubator); + case 'plate' + % out of the cold room shortly before seeding (seeded cold): the + % window ends at the seeding, up to out_before_seeding_seconds earlier + outTol = [getOr(cfg.temperature, 'out_before_seeding_seconds', 0) + hand(1), hand(2)]; + temperature(id, p.poured_cold_room, hand, p.seeded, outTol, minuteFmt, ... + cfg.temperature.cold_room_celsius, '', cfg.temperature.methods.cold_room); + temperature(id, p.cold_room, hand, p.room_temp, hand, minuteFmt, ... + cfg.temperature.cold_room_celsius, '', cfg.temperature.methods.cold_room); + temperature(id, p.room_temp, hand, NaT, [], minuteFmt, ... + cfg.temperature.room_celsius, '', cfg.temperature.methods.room); + end + + % pouring: E. coli plates have its time; the others are before seeding + % (an unseeded plate: before the worms went on) + pourRef = ''; + if ~isnat(p.poured) + pourRef = instant(p.poured, hand); + else + anchor = seedRef; + if isempty(anchor), anchor = incubatorRef; end + if ~isempty(anchor) + d = did2.build.relativeTimeReference(anchor, 'Relation', 'intervalBefore', ... + 'SessionId', sid); + docs{end+1} = d; %#ok + pourRef = d.base.id; + end + end + if isempty(pourRef) + out.skipped{end+1} = sprintf('%s: no pour (no pour, seeding or worm time to place it)', id); + else + peptone = p.peptone; + if ~isfield(cfg.pour.agar, peptone), peptone = cfg.pour.default_peptone; end + notes = ''; + if isfield(cfg.pour, 'notes') && isfield(cfg.pour.notes, folder) + notes = cfg.pour.notes.(folder); + end + dose(id, cfg.pour.variable.(peptone), cfg.pour.volume_ml, 'mL', cfg.pour.agar.(peptone), ... + pourRef, cfg.pour.method, notes, 'pour'); + end +end + +% ---- worm cohorts: the moves, and food deprivation ----------------------------- +cohorts = find(strcmp(S.kind, 'cohort'))'; +for k = cohorts + cid = S.local_identifier{k}; + assay = regexprep(cid, '_worms$', ''); + if ~isempty(S.acclimation{k}) + move(cid, S.acclimation{k}, 'acclimation plate', cfg.transfer.pick_method, []); + end + if isfield(cfg, 'developmental_stage') + stageAtPick(cid, k, assay); + end + if ~isempty(S.deprivation{k}) + key = [S.deprivation{k} '|' cid]; + move(cid, S.deprivation{k}, 'food deprivation plate', cfg.transfer.deprivation_method, []); + if isKey(refs, key) + statement('term_manipulation', cid, cfg.food_deprivation.variable, ... + did2.build.term('', cfg.food_deprivation.value), refs(key), ... + struct('Fields', eachMember), 'food_deprivation'); + end + end + method = ''; + params = []; + if isKey(spans, assay) + span = spans(assay); + [method, params] = assayMethod(span.method); + end + move(cid, assay, 'assay plate', method, params); +end + +out.documents = docs; + +% ============================================================================= + function id = instant(t, tol) + % a hand-written instant: when something was done + d = utcReference(wallClock(t, tz), tol, fmtOf(tol), NaT, [], minuteFmt, false, tz, sid); + docs{end+1} = d; + id = d.base.id; + end + + function refId = temperature(subject, t0, tol0, t1, tol1, fmt1, celsius, note, method) + % moved INTO CELSIUS at T0, until T1 (when known); returns the window + refId = ''; + if isnat(t0) + return; + end + t0 = wallClock(t0, tz); + hasEnd = ~isnat(t1); + if hasEnd + t1 = wallClock(t1, tz); + if t1 < t0 + out.skipped{end+1} = sprintf(['%s at %g C: ends (%s) before it starts (%s); ' ... + 'recorded with its start only'], subject, celsius, char(t1), char(t0)); + hasEnd = false; + end + end + d = utcReference(t0, tol0, fmtOf(tol0), t1, tol1, fmt1, hasEnd, tz, sid); + docs{end+1} = d; + refId = d.base.id; + v = did2.build.composite('temperature_manipulation', 'value', struct('celsius', celsius, ... + 'source_value', celsius, 'source_unit', 'C', 'approximate', true)); + statement('temperature_manipulation', subject, cfg.temperature.variable, v, refId, ... + struct('Notes', note, 'Method', method), 'temperature'); + end + + function seedPatch(row, folder, t, seedRef) + pid = row.local_identifier{1}; + pp = row.prep(1); + key = suspension(folder, t, pp.od600, 'assay'); + if ~isempty(row.bacteria{1}) + variable = nameOf(row.bacteria{1}); + elseif ~isempty(key) + variable = nameOf(key); % a patch of the diluent alone + else + variable = 'diluent'; + end + if isnan(pp.volume_ul) + out.skipped{end+1} = sprintf('%s: seeding volume not recorded; no seeding dose', pid); + return; + end + dose(pid, variable, pp.volume_ul, 'uL', key, seedRef, cfg.seed.method, '', 'seed_patch'); + end + + function seedAcclimation(id, folder, p, seedRef) + strain = getOr(getOr(options.Seeding, 'strain', struct()), 'default', 'OP50'); + [key, note] = suspension(folder, p.seeded, p.od600, 'growth'); + dose(id, nameOf(strain), cfg.seed.growth_volume_ul, 'uL', key, seedRef, ... + cfg.seed.method, note, 'seed_acclimation_plate'); + end + + function dose(subject, variable, amount, unit, formulationKey, timeId, method, notes, counter) + scale = struct('uL', 1e-6, 'mL', 1e-3); + v = did2.build.composite('dose_manipulation', 'value', struct('volume', ... + struct('liters', amount * scale.(unit), 'source_value', amount, 'source_unit', unit))); + x = struct('Method', method, 'Notes', notes, 'Edges', struct()); + % a dose names what was given: did-schema requires dose.formulation_id + % (#73 item 59), so a dose whose suspension is not known is not built + % and is reported (a day with no measured solution gets an + % unmeasured one, decision #82, so this is now a defect if it fires) + if ~isempty(formulationKey) && isKey(options.Ids, formulationKey) + x.Edges.formulation_id = options.Ids(formulationKey); + elseif ~isempty(formulationKey) + out.skipped{end+1} = sprintf('%s: formulation %s was not built; no %s dose', ... + subject, formulationKey, counter); + return; + else + out.skipped{end+1} = sprintf('%s: no formulation known (no solution recorded); no %s dose', ... + subject, counter); + return; + end + statement('dose_manipulation', subject, variable, v, timeId, x, counter); + end + + function move(cohort, plate, value, method, params) + moveKey = [plate '|' cohort]; % (nested: a name of its own, not the caller's `key`) + if ~isKey(refs, moveKey) + out.skipped{end+1} = sprintf('%s to %s: no time for the move; no transfer', cohort, plate); + return; + end + x = struct('Fields', eachMember, 'Method', method, 'MethodParameters', params); + statement('term_manipulation', cohort, cfg.transfer.variable, ... + did2.build.term('', value), refs(moveKey), x, 'transfer'); + end + + function [method, params] = assayMethod(protocolMethod) + % the transfer protocol's method -> the last step, plus the cleaning + % step every assay transfer went through + method = ''; + params = did2.build.parameter('cleaning step', 'Text', cfg.transfer.cleaning); + rules = cfg.transfer.assay_methods; + if isstruct(rules), rules = num2cell(rules); end + for r = 1:numel(rules) + if contains(protocolMethod, rules{r}.contains) + method = rules{r}.method; + if isfield(rules{r}, 'droplet') + params = did2.build.list(params, ... + did2.build.parameter('droplet', 'Term', rules{r}.droplet)); + end + return; + end + end + end + + function statement(leaf, subject, variable, value, timeId, x, counter) + if ~isKey(subjectIds, subject) + out.skipped{end+1} = sprintf('%s %s: not a subject of this session', subject, counter); + return; + end + args = {'SessionId', sid, 'TimeReferenceIds', {timeId}}; + if isfield(x, 'Method') && ~isempty(x.Method), args = [args, {'Method', x.Method}]; end + if isfield(x, 'MethodParameters') && ~isempty(x.MethodParameters) + args = [args, {'MethodParameters', x.MethodParameters}]; + end + if isfield(x, 'Notes') && ~isempty(x.Notes), args = [args, {'Notes', x.Notes}]; end + if isfield(x, 'Fields') && ~isempty(fieldnames(x.Fields)) + args = [args, {'Fields', x.Fields}]; + end + if isfield(x, 'Edges') && ~isempty(fieldnames(x.Edges)) + args = [args, {'Edges', x.Edges}]; + end + docs{end+1} = did2.build.statement(leaf, subjectIds(subject), variable, value, args{:}); + out.counts.(counter) = out.counts.(counter) + 1; + end + + function stageAtPick(cid, k, assay) + % the worms were picked as L4 larvae (decision #69): an observation at + % the pick, or, with no pick time, before the cohort's next move + st = cfg.developmental_stage; + when = ''; + pick = NaT; + h = strcmp(S.local_identifier, S.acclimation{k}); + if any(h), pick = S.worms_placed(find(h, 1)); end + if ~isnat(pick) + when = instant(pick, hand); + else + next = {}; + if ~isempty(S.deprivation{k}), next{end+1} = [S.deprivation{k} '|' cid]; end + next{end+1} = [assay '|' cid]; + next = next(cellfun(@(x) isKey(refs, x), next)); + if ~isempty(next) + d = did2.build.relativeTimeReference(refs(next{1}), 'Relation', 'intervalBefore', ... + 'SessionId', sid); + docs{end+1} = d; + when = d.base.id; + end + end + if isempty(when) + out.skipped{end+1} = sprintf('%s: no pick time and no later move; no developmental stage', cid); + return; + end + statement('term_observation', cid, st.variable, did2.build.term('', st.value), when, ... + struct('Fields', eachMember, 'Method', did2.build.term('', st.method)), 'developmental_stage'); + end + + function [t, tol, fmt] = lastMoveOff(holding) + % the latest time a cohort left HOLDING: its next plate's start (a + % food deprivation plate's starvedTime, or the assay transfer T) + t = NaT; tol = []; fmt = minuteFmt; + on = find(strcmp(S.kind, 'cohort') & (strcmp(S.acclimation, holding) | ... + strcmp(S.deprivation, holding)))'; + for c = on + if strcmp(S.acclimation{c}, holding) && ~isempty(S.deprivation{c}) + q = S.worms_placed(strcmp(S.local_identifier, S.deprivation{c})); + if isempty(q), continue; end + q = wallClock(q(1), tz); qTol = hand; qFmt = minuteFmt; + else + a = regexprep(S.local_identifier{c}, '_worms$', ''); + if ~isKey(spans, a), continue; end + sa = spans(a); + q = sa.T; qTol = sa.tol; qFmt = secondFmt; + end + if ~isnat(q) && (isnat(t) || q > t) + t = q; tol = qTol; fmt = qFmt; + end + end + end + + function [key, note] = suspension(folder, t, od, kind) + key = ''; + note = ''; + I = options.Suspensions; + if isempty(I) || height(I) == 0 || isnat(t) || isnan(od) + return; + end + day = char(t, 'yyyy-MM-dd'); + hit = strcmp(I.folder, folder) & strcmp(cellstr(char(I.day, 'yyyy-MM-dd')), day) & ... + abs(I.od600 - od) < 1e-9 & strcmp(I.kind, kind); + hit = find(hit, 1); + if ~isempty(hit) + key = I.key{hit}; + if ismember('note', I.Properties.VariableNames) + note = I.note{hit}; + end + end + end + + function n = nameOf(key) + if isKey(options.Names, key) + n = options.Names(key); + else + n = key; + end + end + + function f = fmtOf(tol) + % a time filled in by rule keeps its seconds; a hand-written one is to + % the minute + if ischar(tol) || isstring(tol) + f = secondFmt; + else + f = minuteFmt; + end + end +end + +function v = getOr(s, name, default) +if isstruct(s) && isfield(s, name) + v = s.(name); +else + v = default; +end +end diff --git a/src/ndi/+ndi/+setup/+conv/+haley/matchTemplatePatches.m b/src/ndi/+ndi/+setup/+conv/+haley/matchTemplatePatches.m new file mode 100644 index 0000000000..8f19e8d942 --- /dev/null +++ b/src/ndi/+ndi/+setup/+conv/+haley/matchTemplatePatches.m @@ -0,0 +1,51 @@ +function [slot, why] = matchTemplatePatches(c, centerYX, angleDeg, scale, layout) +%MATCHTEMPLATEPATCHES Place detected E. coli patches by the acetate template's frame. +% +% [SLOT, WHY] = ndi.setup.conv.haley.matchTemplatePatches(C, CENTERYX, +% ANGLEDEG, SCALE, LAYOUT) takes the centroids C (N x 2, [x y] pixels) of +% the patches detected in one image and the acetate template registered on +% that image's brightfield image (analyzeGFP's `template`: CENTERYX the +% centre as stored, [y x] pixels; ANGLEDEG; SCALE pixels per mm), puts each +% centroid in the acetate's frame (mm) and gives it the nearest of the 12 +% patch positions in LAYOUT. SLOT(j) is that patch's number (decision #62's +% numbering). SLOT is [] and WHY says why when a patch is farther than +% LAYOUT.max_distance_mm from every position, or two patches claim one. +% Decision #80. +% +% LAYOUT (the spec's `ecoli_patch_layout`): positions_mm (12 x 2), +% angle_sign (+1/-1: how the template angle turns the frame), +% max_distance_mm. + +slot = []; +why = ''; +pos = reshape(double(layout.positions_mm), [], 2); +n = size(c, 1); +if n == 0 + why = 'no patch detected'; + return; +end +if any(~isfinite([centerYX(:); angleDeg; scale])) || scale <= 0 + why = 'the template registration is incomplete'; + return; +end +th = double(layout.angle_sign) * angleDeg * pi / 180; +dx = c(:, 1) - centerYX(2); +dy = c(:, 2) - centerYX(1); +u = ( cos(th) * dx + sin(th) * dy) / scale; +v = (-sin(th) * dx + cos(th) * dy) / scale; +slot = zeros(n, 1); +far = 0; +for j = 1:n + [d, k] = min(vecnorm(pos - [u(j) v(j)], 2, 2)); + slot(j) = k; + far = max(far, d); +end +if far > layout.max_distance_mm + why = sprintf('a patch is %.2f mm from the nearest template position (limit %.2f mm)', ... + far, layout.max_distance_mm); + slot = []; +elseif numel(unique(slot)) ~= n + why = 'two detected patches are nearest the same template position'; + slot = []; +end +end diff --git a/src/ndi/+ndi/+setup/+conv/+haley/observationDocuments.m b/src/ndi/+ndi/+setup/+conv/+haley/observationDocuments.m new file mode 100644 index 0000000000..3258fbc803 --- /dev/null +++ b/src/ndi/+ndi/+setup/+conv/+haley/observationDocuments.m @@ -0,0 +1,137 @@ +function out = observationDocuments(session, R, subjectIds, options) +%OBSERVATIONDOCUMENTS Stage 9 (Haley): the temperature and humidity readings. +% +% OUT = ndi.setup.conv.haley.observationDocuments(SESSION, R, SUBJECTIDS, ...) +% builds, for the one-row session table SESSION, an observation of each +% filmed assay plate's ambient temperature and relative humidity, from the +% behaviour videos' `temp` and `humidity` (R rows, from recordingList). +% Decision log #59. +% +% The lab's temperature probe read both; the value noted is its mean over +% the hour of recording (Jess, 2026-10-03). Every video of a plate carries +% the same reading (profiled over the five C. elegans studies, 0 of 597 +% plates differ), so there is ONE reading per plate, over the plate's +% recordings: the video's own UTC reference when it was filmed once, else a +% window from the first video's start to the last one's end. A plate whose +% videos disagree gets none (reported in OUT.skipped). No value, no +% observation. +% +% temperature_observation variable `ambient temperature` (the term the +% manipulations move a plate into), celsius +% humidity_observation variable `relative humidity`, +% percent_relative_humidity (did-schema PR #86; +% only when the schema in use has `humidity`) +% +% Both: subject = the assay plate, instrument = the probe (stage 2), notes +% = the spec's `environment.notes`. +% +% Options: +% 'Environment' the spec's `environment` section (required) +% 'InstrumentId' the probe's document id ('' leaves instrument_id empty) +% 'RecordingRefs' containers.Map, epoch -> its UTC reference id +% (sessionDocuments' OUT.recordingRefs) +% +% OUT fields: documents (cell of structs), counts (struct: temperature, +% humidity), skipped (cellstr), windows (containers.Map, plate -> the id of +% the time reference spanning its recordings, for stage 11). + +arguments + session table + R table + subjectIds + options.Environment struct + options.InstrumentId (1,:) char = '' + options.RecordingRefs = containers.Map() +end +if height(session) ~= 1 + error('ndi:setup:conv:haley:oneSession', 'Give exactly one session row.'); +end +sid = char(session.session_id{1}); +ref = char(session.local_identifier{1}); +tz = 'America/Los_Angeles'; +cfg = options.Environment; +secondFmt = 'yyyy-MM-dd''T''HH:mm:ss'; +hasHumidity = ndi.setup.V2.schemaHasField('humidity', 'value'); + +docs = {}; +out = struct('documents', {{}}, 'skipped', {{}}, 'counts', struct('temperature', 0, 'humidity', 0), ... + 'windows', containers.Map()); +if ~all(ismember({'temp', 'humidity'}, R.Properties.VariableNames)) + out.documents = docs; + return; +end +R = R(strcmp(R.session, ref) & strcmp(R.kind, 'behaviour'), :); +R = R(~isnan(R.temp) | ~isnan(R.humidity), :); +if ~hasHumidity && any(~isnan(R.humidity)) + out.skipped{end+1} = sprintf(['%s: the schema in use has no `humidity` (did-schema PR #86); ' ... + 'no humidity observations'], ref); +end + +plates = unique(R.plate, 'stable'); +for k = 1:numel(plates) + plate = plates{k}; + P = sortrows(R(strcmp(R.plate, plate), :), 'local_start'); + if ~isKey(subjectIds, plate) + out.skipped{end+1} = sprintf('%s: not a subject of this session', plate); + continue; + end + temp = oneValue(P.temp); + humidity = oneValue(P.humidity); + if isempty(temp) || isempty(humidity) + out.skipped{end+1} = sprintf('%s: its %d videos disagree on temp or humidity; no reading', ... + plate, height(P)); + continue; + end + timeId = window(P); + out.windows(plate) = timeId; + if ~isnan(temp) + v = did2.build.composite('temperature_observation', 'value', struct('celsius', temp, ... + 'source_value', temp, 'source_unit', 'C')); + statement('temperature_observation', plate, cfg.temperature.variable, v, timeId, 'temperature'); + end + if ~isnan(humidity) && hasHumidity + v = did2.build.composite('humidity_observation', 'value', ... + struct('percent_relative_humidity', humidity)); + statement('humidity_observation', plate, cfg.humidity.variable, v, timeId, 'humidity'); + end +end +out.documents = docs; + +% ============================================================================= + function id = window(P) + % the plate's recordings: one video's own reference, else first start + % to last end (the video times are exact) + if height(P) == 1 && isKey(options.RecordingRefs, P.epoch{1}) + id = options.RecordingRefs(P.epoch{1}); + return; + end + t0 = wallClock(P.local_start(1), tz); + t1 = wallClock(P.local_start(end), tz) + seconds(P.duration(end)); + hasEnd = ~isnan(P.duration(end)); + if ~hasEnd, t1 = NaT; end + d = utcReference(t0, [], secondFmt, t1, [], secondFmt, hasEnd, tz, sid); + docs{end+1} = d; + id = d.base.id; + end + + function statement(leaf, plate, variable, value, timeId, counter) + args = {'SessionId', sid, 'TimeReferenceIds', {timeId}}; + if ~isempty(options.InstrumentId), args = [args, {'InstrumentId', options.InstrumentId}]; end + if isfield(cfg, 'notes') && ~isempty(cfg.notes), args = [args, {'Notes', cfg.notes}]; end + docs{end+1} = did2.build.statement(leaf, subjectIds(plate), ... + did2.build.term('', variable), value, args{:}); + out.counts.(counter) = out.counts.(counter) + 1; + end +end + +function v = oneValue(x) +% the one value X holds (NaN when none); [] when its values differ +u = unique(x(~isnan(x))); +if isempty(u) + v = NaN; +elseif numel(u) == 1 + v = u; +else + v = []; +end +end diff --git a/src/ndi/+ndi/+setup/+conv/+haley/patchLayout.m b/src/ndi/+ndi/+setup/+conv/+haley/patchLayout.m new file mode 100644 index 0000000000..dd187fb894 --- /dev/null +++ b/src/ndi/+ndi/+setup/+conv/+haley/patchLayout.m @@ -0,0 +1,122 @@ +function L = patchLayout(dataParentDir, options) +%PATCHLAYOUT Learn where the 12 E. coli patches sit in the acetate template's frame. +% +% L = ndi.setup.conv.haley.patchLayout(DATAPARENTDIR) reads, under +% DATAPARENTDIR/haley/ecoli, bacteria.mat (metaData, lawnAnalysis), the +% extract (analyzeGFP_24-04-04.mat, with `template` from +% ndi.setup.conv.haley.extractTemplates) and the patch masks (mask/.png). +% For every analysed fluorescence image of a `rectangle` plate in which all +% 12 patches were detected, it numbers the patches as the import does +% (decision #62) and converts each centre into the acetate's own frame: +% minus the template centre (stored [y x]), rotated by minus the template +% angle, divided by the template scale (px/mm). The layout is each +% patch's median position over those images. Nothing is written. +% +% The angle's sign convention (regionprops Orientation, y down) is not +% assumed: both signs are tried and the one whose positions agree best +% across images is kept, and both spreads are printed. +% +% It prints its denominator first (images considered, used, and why the +% rest were not), then the 12 positions in mm with their spread, the +% images with any patch more than 'OutlierMm' from its position, and a +% JSON line for the spec. Decision #80. +% +% L fields: positions (12 x 2 mm), spread (12 x 2: median and max +% distance, mm), angleSign, imagesUsed, outliers (table), minSpacing (mm). +% +% Options: 'OutlierMm' (default 0.5), 'ProfilesFile' (the extract; +% default ecoli/analyzeGFP_24-04-04.mat). + +arguments + dataParentDir (1,:) char {mustBeFolder} + options.OutlierMm (1,1) double = 0.5 + options.ProfilesFile (1,:) char = '' +end +e = fullfile(dataParentDir, 'haley', 'ecoli'); +B = load(fullfile(e, 'bacteria.mat'), 'metaData', 'lawnAnalysis'); +M = B.metaData; +A = B.lawnAnalysis; +pf = options.ProfilesFile; +if isempty(pf), pf = fullfile(e, 'analyzeGFP_24-04-04.mat'); end +X = load(pf, 'template'); +T = X.template; + +imgs = unique(A.imageNum); +nRows = arrayfun(@(i) sum(A.imageNum == i), imgs); +why = containers.Map({'not 12 lawnAnalysis rows', 'not a rectangle plate', 'no template', ... + 'no mask file', 'mask regions not 12', 'not numbered as a full image'}, {0, 0, 0, 0, 0, 0}); +P = nan(12, 2, 0, 2); % slot x [u v] x image x sign +used = []; +for i = 1:numel(imgs) + n = imgs(i); + if nRows(i) ~= 12, why('not 12 lawnAnalysis rows') = why('not 12 lawnAnalysis rows') + 1; continue; end + r = find(M.imageNum == n, 1); + if isempty(r) || ~strcmp(M.template{r}, 'rectangle') + why('not a rectangle plate') = why('not a rectangle plate') + 1; continue; + end + t = find(T.imageNum == M.brightfieldImageNum(r), 1); + if isempty(t), why('no template') = why('no template') + 1; continue; end + f = fullfile(e, 'mask', sprintf('%04d.png', n)); + if ~isfile(f), why('no mask file') = why('no mask file') + 1; continue; end + [~, k, c] = labelRegions(imread(f) > 0); + if k ~= 12, why('mask regions not 12') = why('mask regions not 12') + 1; continue; end + slot = matchFullImage(c, 12, 12); + if isempty(slot), why('not numbered as a full image') = why('not numbered as a full image') + 1; continue; end + dx = c(:, 1) - T.center(t, 2); + dy = c(:, 2) - T.center(t, 1); + for s = 1:2 + th = (3 - 2 * s) * T.angle(t) * pi / 180; % +angle, then -angle + u = ( cos(th) * dx + sin(th) * dy) / T.scale(t); + v = (-sin(th) * dx + cos(th) * dy) / T.scale(t); + P(slot, :, numel(used) + 1, s) = [u v]; + end + used(end + 1) = n; %#ok +end +fprintf('DENOMINATOR: %d analysed image(s); %d used (12 patches, a rectangle plate, a template)\n', ... + numel(imgs), numel(used)); +k = keys(why); +for i = 1:numel(k) + if why(k{i}) > 0, fprintf(' not used, %s: %d\n', k{i}, why(k{i})); end +end +if isempty(used) + error('ndi:setup:conv:haley:noLayout', 'No image to learn the layout from.'); +end + +spreadOf = @(Q) squeeze(vecnorm(Q - median(Q, 3), 2, 2)); % 12 x images +d1 = spreadOf(P(:, :, :, 1)); +d2 = spreadOf(P(:, :, :, 2)); +fprintf('median distance from the layout: %.3f mm with +angle, %.3f mm with -angle\n', ... + median(d1(:)), median(d2(:))); +s = 1 + (median(d2(:)) < median(d1(:))); +Q = P(:, :, :, s); +D = spreadOf(Q); +pos = median(Q, 3); +spread = [median(D, 2), max(D, [], 2)]; +gaps = squareform_(pos); +L = struct('positions', pos, 'spread', spread, 'angleSign', 3 - 2 * s, ... + 'imagesUsed', used, 'minSpacing', min(gaps(gaps > 0))); + +fprintf('angle sign kept: %+d; closest two patches %.2f mm apart\n', L.angleSign, L.minSpacing); +fprintf(' patch x mm y mm median dev max dev\n'); +for j = 1:12 + fprintf(' %4d %8.2f %8.2f %9.3f %8.3f\n', j, pos(j, 1), pos(j, 2), spread(j, 1), spread(j, 2)); +end +bad = find(any(D > options.OutlierMm, 1)); +L.outliers = table(reshape(used(bad), [], 1), reshape(max(D(:, bad), [], 1), [], 1), ... + 'VariableNames', {'imageNum', 'maxDeviationMm'}); +fprintf('%d image(s) with a patch more than %.2f mm from its position\n', numel(bad), options.OutlierMm); +if ~isempty(bad), disp(L.outliers); end +fprintf('spec: {"angle_sign": %d, "positions_mm": %s}\n', L.angleSign, ... + jsonencode(round(pos, 3))); +end + +function g = squareform_(p) +% pairwise distances between the rows of P +n = size(p, 1); +g = zeros(n); +for a = 1:n + for b = 1:n + g(a, b) = norm(p(a, :) - p(b, :)); + end +end +end diff --git a/src/ndi/+ndi/+setup/+conv/+haley/private/fileFormat.m b/src/ndi/+ndi/+setup/+conv/+haley/private/fileFormat.m new file mode 100644 index 0000000000..3837d8c8c3 --- /dev/null +++ b/src/ndi/+ndi/+setup/+conv/+haley/private/fileFormat.m @@ -0,0 +1,15 @@ +function f = fileFormat(node, name, mediaType) +%FILEFORMAT A data body's `format`, as the schema in use declares it. +% +% F = fileFormat(NODE, NAME, MEDIATYPE) is the file-format term {NODE, NAME} +% when data_body.format is an ontology_term (did-schema 877bbde, Jess +% 2026-10-14: EDAM format terms, EMPTY where EDAM has none), and the IANA +% MEDIATYPE string on a schema copy made before that. + +[tf, decl] = ndi.setup.V2.schemaHasField('data_body', 'format'); +if tf && strcmp(decl.type, 'ontology_term') + f = struct('node', node, 'name', name); +else + f = mediaType; +end +end diff --git a/src/ndi/+ndi/+setup/+conv/+haley/private/ingestedBody.m b/src/ndi/+ndi/+setup/+conv/+haley/private/ingestedBody.m new file mode 100644 index 0000000000..aa02f84e4c --- /dev/null +++ b/src/ndi/+ndi/+setup/+conv/+haley/private/ingestedBody.m @@ -0,0 +1,44 @@ +function b = ingestedBody(st, a, datumType, keys, sid, description, fillValue) +%INGESTEDBODY A sampled_body holding array A for statement ST, INGESTED. +% +% B = ingestedBody(ST, A, DATUMTYPE, KEYS, SID, DESCRIPTION) writes A's raw +% bytes (column-major, little-endian, as DATUMTYPE: 'bool', 'uint8', +% 'uint16', 'float64', ...) to a temporary file, gzips it, and returns the +% body document, keyed by KEYS, whose file the database COPIES into the +% session's .ndi/files when the session is written and then deletes +% (files.file_info `ingest` 1, `delete_original` 1). Format +% binary (EDAM:format_2333; the media type application/octet-stream on an +% older schema), compression gzip, MD5 of the stored file. +% FILLVALUE (optional, char) is what a position with no value holds. +% Stage 10, decision #60. + +if nargin < 7 + fillValue = ''; +end +raw = [tempname '.bin']; +fid = fopen(raw, 'w', 'ieee-le'); +switch datumType + case 'bool', fwrite(fid, uint8(a), 'uint8'); + case 'float64', fwrite(fid, double(a), 'double'); + case 'float32', fwrite(fid, single(a), 'single'); + otherwise, fwrite(fid, cast(a, datumType), datumType); +end +fclose(fid); +gz = gzip(raw); +gz = gz{1}; +delete(raw); +info = dir(gz); +byteOrder = ''; +if ~any(strcmp(datumType, {'bool', 'uint8', 'int8'})), byteOrder = 'little'; end +datumOrder = ''; +if numel(keys) > 1, datumOrder = 'F'; end +b = did2.build.sampledBody(st.base.id, keys, 'DatumOrder', datumOrder, ... + 'ByteOrder', byteOrder, 'FillValue', fillValue, 'Complete', true, ... + 'Format', fileFormat('EDAM:format_2333', 'Binary format', 'application/octet-stream'), ... + 'Compression', 'gzip', ... + 'ContentHash', ndi.fun.file.MD5(gz), 'HashAlgorithm', 'MD5', ... + 'Description', description, 'Fields', struct('size_bytes', info.bytes), 'SessionId', sid); +b.files.file_info = struct('name', 'body_data_0', 'locations', struct( ... + 'delete_original', 1, 'uid', ndi.ido.unique_id(), 'location', gz, ... + 'parameters', '', 'location_type', 'file', 'ingest', 1)); +end diff --git a/src/ndi/+ndi/+setup/+conv/+haley/private/labelRegions.m b/src/ndi/+ndi/+setup/+conv/+haley/private/labelRegions.m new file mode 100644 index 0000000000..ca0d028591 --- /dev/null +++ b/src/ndi/+ndi/+setup/+conv/+haley/private/labelRegions.m @@ -0,0 +1,41 @@ +function [labeled, n, centroids] = labelRegions(mask) +%LABELREGIONS The connected regions of a logical MASK, numbered as bwlabel. +% +% [LABELED, N, CENTROIDS] = labelRegions(MASK) numbers MASK's 8-connected +% regions 1..N in the order bwlabel does (by each region's first pixel in +% column-major order) without the Image Processing Toolbox. CENTROIDS is +% N x 2, [x y] in pixels (column, row), as regionprops' Centroid. +% Stage 10 part C, decision #62. + +mask = logical(mask); +[h, w] = size(mask); +idx = find(mask); +n = 0; +labeled = zeros(h, w); +centroids = zeros(0, 2); +if isempty(idx) + return; +end +node = zeros(h, w); +node(idx) = 1:numel(idx); +[r, c] = ind2sub([h w], idx); +s = []; t = []; +for d = [1 0; 0 1; 1 1; -1 1]' % down, right, down-right, up-right + rr = r + d(1); cc = c + d(2); + ok = rr >= 1 & rr <= h & cc >= 1 & cc <= w; + nb = zeros(size(idx)); + nb(ok) = node(sub2ind([h w], rr(ok), cc(ok))); + keep = nb > 0; + s = [s; node(idx(keep))]; %#ok + t = [t; nb(keep)]; %#ok +end +comp = conncomp(graph(s, t, [], numel(idx)))'; +n = max(comp); +first = accumarray(comp, idx, [n 1], @min); +[~, order] = sort(first); +rank = zeros(n, 1); +rank(order) = 1:n; +lab = rank(comp); +labeled(idx) = lab; +centroids = [accumarray(lab, c, [n 1], @mean), accumarray(lab, r, [n 1], @mean)]; +end diff --git a/src/ndi/+ndi/+setup/+conv/+haley/private/lawnProfileRead.m b/src/ndi/+ndi/+setup/+conv/+haley/private/lawnProfileRead.m new file mode 100644 index 0000000000..e6f76b7e19 --- /dev/null +++ b/src/ndi/+ndi/+setup/+conv/+haley/private/lawnProfileRead.m @@ -0,0 +1,50 @@ +function [x, Y] = lawnProfileRead(P, imageNum) +%LAWNPROFILEREAD One image's patch intensity profiles from analyzeGFP. +% +% [X, Y] = lawnProfileRead(P, IMAGENUM), P from lawnProfileReader. X is the +% 1 x nBins distance from the patch edge (mm; negative outside the patch, +% 1 um steps); Y is nPatches x nBins, the background-normalised intensity +% in each bin, NaN where a patch has no pixels at that distance. Rows are +% in the order analyzeLawnProfiles numbered the patches (bwlabel), the +% order of that image's lawnAnalysis rows. Both are empty when the image +% is not in the file or no patch was found in it. Decision #64. + +x = []; Y = []; +if ~isKey(P.byImage, imageNum) + return; +end +k = P.byImage(imageNum); +if ~P.hdf5 + x = P.loaded.distances{k}; + Y = P.loaded.pixelValuesNormalized{k}; + return; +end +fid = H5F.open(P.file, 'H5F_ACC_RDONLY', 'H5P_DEFAULT'); +c = onCleanup(@() H5F.close(fid)); +x = deref(fid, '/lawnProfiles/distances', P.refs.distances(:, k)); +Y = deref(fid, '/lawnProfiles/pixelValuesNormalized', P.refs.pixelValuesNormalized(:, k)); +x = reshape(double(x), 1, []); +Y = double(Y); +if ~isempty(Y) && size(Y, 2) ~= numel(x) && size(Y, 1) == numel(x) + Y = Y.'; +end +end + +function v = deref(fid, path, ref) +d = H5D.open(fid, path); +cd = onCleanup(@() H5D.close(d)); +try + o = H5R.dereference(d, 'H5R_OBJECT', ref); +catch + o = H5R.dereference(d, 'H5P_DEFAULT', 'H5R_OBJECT', ref); +end +co = onCleanup(@() H5O.close(o)); +v = []; +try + a = H5A.open(o, 'MATLAB_empty'); % an empty array: MATLAB stores only its size + H5A.close(a); + return; +catch +end +v = H5D.read(o); +end diff --git a/src/ndi/+ndi/+setup/+conv/+haley/private/lawnProfileReader.m b/src/ndi/+ndi/+setup/+conv/+haley/private/lawnProfileReader.m new file mode 100644 index 0000000000..2af3fabcb2 --- /dev/null +++ b/src/ndi/+ndi/+setup/+conv/+haley/private/lawnProfileReader.m @@ -0,0 +1,39 @@ +function P = lawnProfileReader(file) +%LAWNPROFILEREADER Open analyzeGFP's lawnProfiles for reading, one image at a time. +% +% P = lawnProfileReader(FILE) reads the image numbers of `lawnProfiles` in +% FILE (analyzeGFP's output, e.g. analyzeGFP_24-04-04.mat; Haley et al. +% 2024, analysis/analyzeGFP.m) and the references to each image's +% `distances` and `pixelValuesNormalized`, without loading the struct: +% its image fields (image, imageNormalized, labeled) are ~170 GB. Read one +% image's curves with lawnProfileRead(P, imageNum). Decision #64. +% +% A -v7.3 file is HDF5 and is read through its object references. Any +% other MAT file is small enough to load, and is. +% +% P fields: file, imageNums (row), byImage (containers.Map imageNum -> +% position), hdf5 (logical), refs (struct of the two reference arrays, +% HDF5 only), loaded (the struct, non-HDF5 only). + +P = struct('file', file, 'imageNums', [], 'byImage', containers.Map('KeyType', 'double', ... + 'ValueType', 'double'), 'hdf5', false, 'refs', struct(), 'loaded', struct()); +if H5F.is_hdf5(file) + P.hdf5 = true; + P.imageNums = double(reshape(h5read(file, '/lawnProfiles/imageNum'), 1, [])); + fid = H5F.open(file, 'H5F_ACC_RDONLY', 'H5P_DEFAULT'); + c = onCleanup(@() H5F.close(fid)); + for f = ["distances", "pixelValuesNormalized"] + d = H5D.open(fid, "/lawnProfiles/" + f); + r = H5D.read(d, 'H5T_STD_REF_OBJ', 'H5S_ALL', 'H5S_ALL', 'H5P_DEFAULT'); + H5D.close(d); + P.refs.(f) = reshape(r, size(r, 1), []); + end +else + S = load(file, 'lawnProfiles'); + P.loaded = S.lawnProfiles; + P.imageNums = double(reshape(P.loaded.imageNum, 1, [])); +end +for k = 1:numel(P.imageNums) + P.byImage(P.imageNums(k)) = k; +end +end diff --git a/src/ndi/+ndi/+setup/+conv/+haley/private/matchFullImage.m b/src/ndi/+ndi/+setup/+conv/+haley/private/matchFullImage.m new file mode 100644 index 0000000000..d119e15d75 --- /dev/null +++ b/src/ndi/+ndi/+setup/+conv/+haley/private/matchFullImage.m @@ -0,0 +1,41 @@ +function slot = matchFullImage(c, nDetected, nSubjects) +%MATCHFULLIMAGE SLOT(j) = the patch subject number of detected patch j, or [] when the +% image cannot be matched. 12: number as bwlabel numbers an ideal staggered +% grid (left to right; patches sharing a column, top to bottom). 1: itself. +slot = []; +if nDetected == 1 && nSubjects == 1 + slot = 1; + return; +end +if nDetected ~= 12 || nSubjects ~= 12 + return; +end +% C: the detected patches' centroids, x and y in label order +% three rows of four, by y +[~, byY] = sort(c(:, 2)); +rowOf = zeros(12, 1); +rowOf(byY) = repelem(1:3, 4); +gapY = diff(c(byY, 2)); +if min(gapY([4 8])) <= max(gapY([1:3 5:7 9:11])) + return; % the rows are not cleanly apart +end +% within each row, the step between neighbours +steps = []; +for r = 1:3 + steps = [steps; diff(sort(c(rowOf == r, 1)))]; %#ok +end +tol = median(steps) / 4; +% columns: patches whose x differ by less than a quarter step share one +[~, byX] = sort(c(:, 1)); +col = zeros(12, 1); +col(byX(1)) = 1; +for i = 2:12 + col(byX(i)) = col(byX(i - 1)) + (c(byX(i), 1) - c(byX(i - 1), 1) >= tol); +end +if max(col) ~= 8 + return; % not the staggered 8-column layout +end +[~, order] = sortrows([col, c(:, 2)]); % left to right, then top to bottom +slot = zeros(12, 1); +slot(order) = 1:12; +end diff --git a/src/ndi/+ndi/+setup/+conv/+haley/private/patchIds.m b/src/ndi/+ndi/+setup/+conv/+haley/private/patchIds.m new file mode 100644 index 0000000000..974288f3c1 --- /dev/null +++ b/src/ndi/+ndi/+setup/+conv/+haley/private/patchIds.m @@ -0,0 +1,10 @@ +function ids = patchIds(subjectIds, plate) +%PATCHIDS The plate's patch subject ids, patch0001, patch0002, ... in order +% (decision #38: one per row of lawnCenters, numbered in that order). +ids = {}; +k = 1; +while isKey(subjectIds, sprintf('%s_patch%04d', plate, k)) + ids{end+1} = subjectIds(sprintf('%s_patch%04d', plate, k)); %#ok + k = k + 1; +end +end diff --git a/src/ndi/+ndi/+setup/+conv/+haley/private/trackTable.m b/src/ndi/+ndi/+setup/+conv/+haley/private/trackTable.m new file mode 100644 index 0000000000..2ca5624efd --- /dev/null +++ b/src/ndi/+ndi/+setup/+conv/+haley/private/trackTable.m @@ -0,0 +1,15 @@ +function T = trackTable(file) +%TRACKTABLE The track table in FILE (.mat), loaded once per file: +% the tables are up to GBs and every session of a folder, and stages 10 +% and 11, read the same one. +persistent cacheFile cacheStamp cacheTable +info = dir(file); +if ~isempty(cacheFile) && strcmp(cacheFile, file) && isequal(cacheStamp, info.datenum) + T = cacheTable; + return; +end +S = load(file); +f = fieldnames(S); +T = S.(f{1}); +cacheFile = file; cacheStamp = info.datenum; cacheTable = T; +end diff --git a/src/ndi/+ndi/+setup/+conv/+haley/private/utcReference.m b/src/ndi/+ndi/+setup/+conv/+haley/private/utcReference.m new file mode 100644 index 0000000000..d9e4e7d30a --- /dev/null +++ b/src/ndi/+ndi/+setup/+conv/+haley/private/utcReference.m @@ -0,0 +1,43 @@ +function d = utcReference(t0, tol0, fmt0, t1, tol1, fmt1, hasEnd, tz, sid) +% A window from T0 (wall clock) to T1 (when HASEND), each with its bound +% [minus plus] seconds ([] = exact) and the precision it was written at (FMT, +% so source_value does not invent seconds a hand-written time never had). The +% duration's bound follows: it is shortest when the start is late and the end +% early (minus = start.plus + end.minus), longest the other way round. +% A bound of 'estimated' is an approximate time with NO known bound (a time +% the source filled in by rule, decision #54d): approximate, no tolerance, and +% a duration it touches is approximate with no tolerance too. +[a0, tol0] = bound(tol0); +args = {'SourceValue', char(t0, fmt0), 'SourceTimezone', tz, ... + 'Approximate', a0, 'Tolerance', tol0}; +if hasEnd + [a1, tol1] = bound(tol1); + s0 = tol0; if isempty(s0), s0 = [0 0]; end + e1 = tol1; if isempty(e1), e1 = [0 0]; end + durTol = [s0(2) + e1(1), s0(1) + e1(2)]; + if all(durTol == 0), durTol = []; end + args = [args, {'Duration', elapsed(t0, t1, tz), ... + 'DurationApproximate', ~isempty(durTol) || a0 || a1, 'DurationTolerance', durTol, ... + 'End', utcText(t1, tz), 'EndSourceValue', char(t1, fmt1), 'EndSourceTimezone', tz, ... + 'EndApproximate', a1, 'EndTolerance', tol1}]; +end +d = did2.build.absoluteTimeReference(utcText(t0, tz), 'SessionId', sid, args{:}); +end + +function [approx, tol] = bound(tol) +if ischar(tol) || isstring(tol) % 'estimated': approximate, bound unknown + approx = true; + tol = []; +else + approx = ~isempty(tol); +end +end + +function s = elapsed(t0, t1, tz) +% seconds from wall-clock T0 to T1, both read in TZ: a window across a +% daylight-saving change is an hour shorter or longer than the clocks say +a = t0; b = t1; +if isempty(a.TimeZone), a.TimeZone = tz; end +if isempty(b.TimeZone), b.TimeZone = tz; end +s = seconds(b - a); +end diff --git a/src/ndi/+ndi/+setup/+conv/+haley/private/utcText.m b/src/ndi/+ndi/+setup/+conv/+haley/private/utcText.m new file mode 100644 index 0000000000..7b6424149c --- /dev/null +++ b/src/ndi/+ndi/+setup/+conv/+haley/private/utcText.m @@ -0,0 +1,5 @@ +function s = utcText(t, tz) +t.TimeZone = tz; +t.TimeZone = 'UTC'; +s = char(t, 'yyyy-MM-dd''T''HH:mm:ss.SSS''Z'''); +end diff --git a/src/ndi/+ndi/+setup/+conv/+haley/private/wallClock.m b/src/ndi/+ndi/+setup/+conv/+haley/private/wallClock.m new file mode 100644 index 0000000000..418f7cbdcf --- /dev/null +++ b/src/ndi/+ndi/+setup/+conv/+haley/private/wallClock.m @@ -0,0 +1,8 @@ +function t = wallClock(t, tz) +% The source's wall-clock time with no zone attached: a value that already +% carries a zone is first expressed in TZ. +if ~isempty(t.TimeZone) + t.TimeZone = tz; + t.TimeZone = ''; +end +end diff --git a/src/ndi/+ndi/+setup/+conv/+haley/recordingList.m b/src/ndi/+ndi/+setup/+conv/+haley/recordingList.m new file mode 100644 index 0000000000..28127fc00c --- /dev/null +++ b/src/ndi/+ndi/+setup/+conv/+haley/recordingList.m @@ -0,0 +1,383 @@ +function [R, checks] = recordingList(dataParentDir, sessions, options) +%RECORDINGLIST Stage 5 (Haley): the recordings, one epoch each, per session. +% +% [R, CHECKS] = ndi.setup.conv.haley.recordingList(DATAPARENTDIR, SESSIONS) +% reads the raw data under DATAPARENTDIR/haley and returns one row per +% recording that has a file on disk, assigned to a session of SESSIONS +% (the table from ndi.setup.conv.haley.sessionList). Decision log #27-#36. +% +% C. elegans, from /experimentInfo.mat `info` and the session's +% videos/ folder: +% behaviour `videoFileName` (an .avi name; disk holds the same-stem +% .mp4). Extent from the table: numFrames / frameRate. +% lawn `lawnFileName`: the short clip of the plate before the worms +% go in, shared by a plate's video rows. Its extent is read +% from the file (VideoReader) only with 'ReadVideos', true. +% E. coli, from ecoli/bacteria.mat `metaData` and ecoli/raw: +% image one per RAW image on disk (raw/.tiff: the +% microscope images, extracted from analyzeGFP's `data` -- +% the 1,575 analysed and shared images and the 89 empty-plate +% background images they were normalised by, decision #64); +% an instant. The TIFFs in images/ are NOT recordings: each is +% its raw image divided by its smoothed background +% (analyzeLawnProfiles' imageNormalized, rounded), and the +% calculations stage builds it as such. With no raw/ folder, +% images/ is used as before and a check says the recordings +% are background-normalised. Images with no file are not +% recordings; each kept image's `note` names its background +% and brightfield image. +% +% Columns: session, kind, epoch (the epoch's local_identifier: the file +% stem with the folder prefix, e.g. concentration_2022-02-04_12-10-51_1, +% ecoli_image0002), system (camera1 / camera2 / microscope), plate (the +% plate subject's local_identifier), file (relative to DATAPARENTDIR/haley), +% source_name (the name the table records, e.g. the .avi), size_bytes, +% local_start (the wall-clock start as recorded, no time zone), +% n_frames, frame_rate, duration (seconds; NaN when not known), note, +% temp and humidity (a behaviour video's `temp` (C) and `humidity` (%RH), +% the probe's mean over the recording; NaN otherwise). +% +% CHECKS (second output, also printed), nothing changed: +% rowWithoutFile a table row names a recording with no file on disk +% fileWithoutRow a video on disk no row names +% cameraDisagrees the `camera` column and the file name's _N disagree +% noFrames a behaviour row with numFrames or frameRate 0 +% imagesWithoutFile E. coli metaData images with no file, by kind +% noRawImages no ecoli/raw/ folder: the recordings fall back to the +% background-normalised TIFFs in images/ +% unreadable 'ReadVideos': a lawn clip VideoReader could not open +% +% Options: +% 'ReadVideos' default false. true opens every lawn clip with MATLAB's +% VideoReader to read its frame count and rate (decision +% #30); behaviour videos are not opened. + +arguments + dataParentDir (1,:) char {mustBeFolder} + sessions table + options.ReadVideos (1,1) logical = false +end + +root = fullfile(dataParentDir, 'haley'); +checks = struct('rowWithoutFile', {{}}, 'fileWithoutRow', {{}}, 'cameraDisagrees', {{}}, ... + 'noFrames', {{}}, 'imagesWithoutFile', {{}}, 'noRawImages', {{}}, 'unreadable', {{}}, ... + 'cameraSwap', {{}}); +rows = {}; +nFiles = 0; + +% ---- C. elegans ------------------------------------------------------------- +folders = unique(sessions.folder(~strcmp(sessions.folder, 'ecoli'))); +for f = 1:numel(folders) + folder = folders{f}; + pre = ndi.setup.conv.haley.idPrefix(folder); + I = load(fullfile(root, 'celegans', folder, 'experimentInfo.mat'), 'info'); + I = I.info; + nFiles = nFiles + 1; + mine = sessions(strcmp(sessions.folder, folder), :); + for s = 1:height(mine) + vdir = mine.video_folder{s}; + onDisk = dir(fullfile(root, vdir, '*.mp4')); + onDisk = {onDisk.name}; + used = false(size(onDisk)); + D = I(I.expNum == mine.experiment(s), :); + seenLawn = {}; + for r = 1:height(D) + plate = sprintf('%s_assayPlate%04d', pre, D.plateNum(r)); + % -- the behaviour video -- + src = strtrim(char(D.videoFileName{r})); + if ~isempty(src) + [~, stem] = fileparts(src); + k = find(strcmp(onDisk, [stem '.mp4']), 1); + if isempty(k) + msg = sprintf('%s: plate %d video %d names %s; no %s.mp4 in %s', ... + mine.local_identifier{s}, D.plateNum(r), D.videoNum(r), src, stem, vdir); + % a failed start (scale 0) whose plate has another video + % carries nothing measured: say so (decision #81) + if ismember('scale', D.Properties.VariableNames) && D.scale(r) == 0 && ... + sum(D.plateNum == D.plateNum(r)) > 1 + msg = [msg '; it is a failed start (scale 0) and the plate''s ' ... + 'other video holds its worms']; + end + checks.rowWithoutFile{end+1} = msg; + else + used(k) = true; + cam = cameraOf(stem); + if ismember('camera', D.Properties.VariableNames) && D.camera(r) ~= cam + checks.cameraDisagrees{end+1} = sprintf('%s: %s is camera %d by name, %d in the table', ... + mine.local_identifier{s}, stem, cam, D.camera(r)); + end + nf = D.numFrames(r); fr = D.frameRate(r); + dur = NaN; + if nf > 0 && fr > 0 + dur = nf / fr; + else + checks.noFrames{end+1} = sprintf('%s: %s has numFrames %g, frameRate %g', ... + mine.local_identifier{s}, stem, nf, fr); + end + row = recRow(mine.local_identifier{s}, 'behaviour', [pre '_' stem], ... + sprintf('camera%d', cam), plate, fullfile(vdir, [stem '.mp4']), src, ... + bytesOf(root, vdir, stem), D.timeRecord(r), nf, fr, dur); + % the probe's readings for this recording (stage 9, decision #59) + row.temp = colValue(D, 'temp', r); + row.humidity = colValue(D, 'humidity', r); + rows{end+1} = row; %#ok + end + end + % -- the lawn clip (one per plate, shared by its video rows) -- + if ismember('lawnFileName', D.Properties.VariableNames) + src = strtrim(char(D.lawnFileName{r})); + if ~isempty(src) && ~any(strcmp(seenLawn, src)) + seenLawn{end+1} = src; %#ok + [~, stem] = fileparts(src); + k = find(strcmp(onDisk, [stem '.mp4']), 1); + if isempty(k) + checks.rowWithoutFile{end+1} = sprintf('%s: plate %d lawn clip %s; no %s.mp4 in %s', ... + mine.local_identifier{s}, D.plateNum(r), src, stem, vdir); + else + used(k) = true; + nf = NaN; fr = NaN; dur = NaN; + if options.ReadVideos + [nf, fr, dur, err] = readVideo(fullfile(root, vdir, [stem '.mp4'])); + if ~isempty(err) + checks.unreadable{end+1} = sprintf('%s: %s.mp4: %s', ... + mine.local_identifier{s}, stem, err); + end + end + rows{end+1} = recRow(mine.local_identifier{s}, 'lawn', [pre '_' stem], ... + sprintf('camera%d', cameraOf(stem)), plate, fullfile(vdir, [stem '.mp4']), ... + src, bytesOf(root, vdir, stem), stampOf(stem), nf, fr, dur); %#ok + end + end + end + end + % a plate whose lawn clip and video are on different cameras, when + % no other plate that day changes camera (decision #83) + here = find(cellfun(@(q) strcmp(q.session, mine.local_identifier{s}), rows)); + camOf = containers.Map(); + for q = here + x = rows{q}; + camOf([x.plate '|' x.kind]) = x.system; + end + swapped = {}; + for q = here + x = rows{q}; + if strcmp(x.kind, 'lawn') && isKey(camOf, [x.plate '|behaviour']) && ... + ~strcmp(camOf([x.plate '|behaviour']), x.system) + swapped{end+1} = x.plate; %#ok + end + end + for q = here + x = rows{q}; + if any(strcmp(swapped, x.plate)) + rows{q}.note = sprintf(['This plate''s lawn clip and behaviour video were filmed on ' ... + 'different cameras (%s, %s), as the source table names them; %d plate(s) ' ... + 'did so that day. Whether the plates moved or the names were swapped is ' ... + 'not known (decision #83).'], camOf([x.plate '|lawn']), ... + camOf([x.plate '|behaviour']), numel(swapped)); + end + end + for p = swapped + checks.cameraSwap{end+1} = sprintf('%s: %s''s lawn clip and video are on different cameras; noted', ... + mine.local_identifier{s}, p{1}); + end + % worms all on listed plates: an unnamed file is not a missing plate + allWorms = ''; + w = []; + if ismember('wormNum', D.Properties.VariableNames) + if iscell(D.wormNum) + w = cell2mat(cellfun(@(v) reshape(double(v), 1, []), D.wormNum(:)', 'UniformOutput', false)); + else + w = reshape(double(D.wormNum), 1, []); + end + end + if ~isnan(mine.worm_first(s)) && ~isempty(w) && ... + all(ismember(mine.worm_first(s):mine.worm_last(s), w)) + allWorms = sprintf(['; the day''s worms (%d-%d) are all on listed plates, so it is ' ... + 'not a missing plate'], mine.worm_first(s), mine.worm_last(s)); + end + for k = find(~used) + checks.fileWithoutRow{end+1} = sprintf('%s: %s is named by no row%s', ... + mine.local_identifier{s}, fullfile(vdir, onDisk{k}), allWorms); + end + end +end + +% ---- E. coli ---------------------------------------------------------------- +mine = sessions(strcmp(sessions.folder, 'ecoli'), :); +if height(mine) > 0 + B = load(fullfile(root, 'ecoli', 'bacteria.mat'), 'metaData'); + M = B.metaData; + % the images the lab analysed (one lawnAnalysis row per detected patch), + % and the background images they were normalised against: a missing + % file among these is a gap, any other is an acquisition never kept as + % a TIFF (decision #79) + w = whos('-file', fullfile(root, 'ecoli', 'bacteria.mat')); + analysed = []; + if any(strcmp({w.name}, 'lawnAnalysis')) + A = load(fullfile(root, 'ecoli', 'bacteria.mat'), 'lawnAnalysis'); + analysed = unique(double(A.lawnAnalysis.imageNum)); + if ismember('backgroundImageNum', M.Properties.VariableNames) + bgUsed = double(M.backgroundImageNum(ismember(M.imageNum, analysed))); + analysed = union(analysed, bgUsed(~isnan(bgUsed))); + end + end + nFiles = nFiles + 1; + idir = fullfile('ecoli', 'raw'); + if ~isfolder(fullfile(root, idir)) + idir = fullfile('ecoli', 'images'); + checks.noRawImages{end+1} = ['ecoli: no raw/ folder; the recordings are the TIFFs in ' ... + 'images/, which are background-normalised, not as acquired (decision #64)']; + end + missing = 0; missingFluor = 0; missingUsed = 0; + for r = 1:height(M) + s = find(mine.experiment == double(M.expNum(r)), 1); + if isempty(s) + continue; + end + name = sprintf('%04d.tiff', M.imageNum(r)); + fi = dir(fullfile(root, idir, name)); + if isempty(fi) + missing = missing + 1; + missingUsed = missingUsed + ismember(double(M.imageNum(r)), analysed); + if ismember('fluorescence', M.Properties.VariableNames) && M.fluorescence(r) + missingFluor = missingFluor + 1; + end + continue; + end + src = ''; + if ismember('fileName', M.Properties.VariableNames) + src = char(M.fileName{r}); + end + note = imageNote(M, r); + rows{end+1} = recRow(mine.local_identifier{s}, 'image', sprintf('ecoli_image%04d', M.imageNum(r)), ... + 'microscope', sprintf('ecoli_plate%04d', M.plateNum(r)), fullfile(idir, name), src, ... + fi.bytes, M.acquisitionTime(r), 1, NaN, NaN, note); %#ok + end + if missing > 0 + if isempty(analysed) + used = 'whether any was analysed is unknown (bacteria.mat has no lawnAnalysis)'; + elseif missingUsed == 0 + used = 'none was analysed or used as a background: acquisitions never kept as a TIFF'; + else + used = sprintf('%d of them analysed or used as a background', missingUsed); + end + checks.imagesWithoutFile{end+1} = sprintf(['ecoli: %d of %d metaData image(s) have no file in ' ... + '%s/ (%d of them fluorescence); %s'], missing, height(M), idir, missingFluor, used); + end +end + +% ---- report ------------------------------------------------------------------- +if isempty(rows) + R = cell2table(cell(0, 15), 'VariableNames', {'session', 'kind', 'epoch', 'system', ... + 'plate', 'file', 'source_name', 'size_bytes', 'local_start', 'n_frames', ... + 'frame_rate', 'duration', 'note', 'temp', 'humidity'}); +else + R = struct2table([rows{:}], 'AsArray', true); + [u, ~, j] = unique(R.epoch); + dup = u(accumarray(j, 1) > 1); + if ~isempty(dup) + error('ndi:setup:conv:haley:duplicateEpoch', ... + 'Two recordings would share epoch %s.', strjoin(dup, ', ')); + end +end +kinds = {'behaviour', 'lawn', 'image'}; +counts = cellfun(@(k) sum(strcmp(R.kind, k)), kinds); +gb = 0; +if height(R) > 0 + gb = sum(R.size_bytes, 'omitnan') / 1e9; +end +fprintf(['DENOMINATOR: %d recording(s) with a file, in %d session(s), from %d source table(s): ' ... + '%d behaviour video(s), %d lawn clip(s), %d E. coli image(s); %.1f GB\n'], ... + height(R), numel(unique(R.session)), nFiles, counts, gb); +names = fieldnames(checks); +fprintf('CHECKS (reported, nothing changed): %d finding(s)\n', ... + sum(cellfun(@(n) numel(checks.(n)), names))); +for k = 1:numel(names) + for m = 1:numel(checks.(names{k})) + fprintf(' %-18s %s\n', names{k}, checks.(names{k}){m}); + end +end +end + +% ============================================================================= + +function s = recRow(session, kind, epoch, system, plate, file, src, bytes, t, nf, fr, dur, note) +if nargin < 13 + note = ''; +end +% One kind of value for the whole column: the sources mix zoned and unzoned +% datetimes (which cannot be concatenated), so a zoned one is expressed in +% the lab's time zone and its zone dropped -- `local_start` is wall-clock time. +if ~isempty(t.TimeZone) + t.TimeZone = 'America/Los_Angeles'; + t.TimeZone = ''; +end +t.Format = 'dd-MMM-yyyy HH:mm:ss'; +s = struct('session', session, 'kind', kind, 'epoch', epoch, 'system', system, ... + 'plate', plate, 'file', file, 'source_name', src, 'size_bytes', double(bytes), ... + 'local_start', t, 'n_frames', double(nf), 'frame_rate', double(fr), 'duration', double(dur), ... + 'note', note, 'temp', NaN, 'humidity', NaN); +end + +function v = colValue(D, name, r) +% D.(NAME)(R) as a double, NaN when the table has no such column +v = NaN; +if ismember(name, D.Properties.VariableNames) + v = double(D.(name)(r)); +end +end + +function note = imageNote(M, r) +% The processing facts an image file cannot carry: its exposure, and the +% background and brightfield images it was paired with (by imageNum; those +% images were not shared as files). +parts = {}; +if ismember('exposureTime', M.Properties.VariableNames) + parts{end+1} = sprintf('exposure %g ms', M.exposureTime(r)); +end +if ismember('backgroundImageNum', M.Properties.VariableNames) && ~isnan(M.backgroundImageNum(r)) + parts{end+1} = sprintf('background image %d', M.backgroundImageNum(r)); +end +if ismember('brightfieldImageNum', M.Properties.VariableNames) && ~isnan(M.brightfieldImageNum(r)) + parts{end+1} = sprintf('brightfield image %d', M.brightfieldImageNum(r)); +end +note = strjoin(parts, '; '); +end + +function c = cameraOf(stem) +% The camera is the file name's last _N (2022-02-04_12-10-51_1 -> 1). +tok = regexp(stem, '_(\d+)$', 'tokens', 'once'); +c = NaN; +if ~isempty(tok) + c = str2double(tok{1}); +end +end + +function t = stampOf(stem) +% The wall-clock start in the name: yyyy-MM-dd_HH-mm-ss_N. +tok = regexp(stem, '^(\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2})', 'tokens', 'once'); +t = NaT; +if ~isempty(tok) + t = datetime(tok{1}, 'InputFormat', 'yyyy-MM-dd_HH-mm-ss'); +end +end + +function b = bytesOf(root, vdir, stem) +d = dir(fullfile(root, vdir, [stem '.mp4'])); +b = NaN; +if ~isempty(d) + b = d.bytes; +end +end + +function [nf, fr, dur, err] = readVideo(file) +nf = NaN; fr = NaN; dur = NaN; err = ''; +try + v = VideoReader(file); + nf = v.NumFrames; + fr = v.FrameRate; + dur = v.Duration; +catch e + err = e.message; +end +end diff --git a/src/ndi/+ndi/+setup/+conv/+haley/relationDocuments.m b/src/ndi/+ndi/+setup/+conv/+haley/relationDocuments.m new file mode 100644 index 0000000000..b6ca8de46d --- /dev/null +++ b/src/ndi/+ndi/+setup/+conv/+haley/relationDocuments.m @@ -0,0 +1,233 @@ +function out = relationDocuments(session, S, R, subjectIds, options) +%RELATIONDOCUMENTS Stage 6 (Haley): how one session's subjects are related. +% +% OUT = ndi.setup.conv.haley.relationDocuments(SESSION, S, R, SUBJECTIDS) +% builds the directed_relation documents among the subjects of the +% one-row session table SESSION (from sessionList, with `session_id` +% assigned). S and R are the subject and recording tables (subjectList, +% recordingList); SUBJECTIDS maps each subject's local_identifier to its +% document id (ndi.setup.conv.haley.sessionDocuments returns it). Nothing +% is written here. Decision log #52, #53. +% +% Relations (child -> parent): +% patch part_of its plate (assay plate, or E. coli plate). No +% time: a patch is part of its plate for the +% plate's whole life. +% worm member_of its cohort (the worms of its assay plate, moved +% together; decision #55). No time. +% cohort contained_in each plate its worms were on, in turn: its acclimation +% plate from the pick (`growthTimePicked`), its +% food deprivation plate if it has one from the move +% to it (`starvedTime`), its assay plate from the +% transfer T; each window ends when the next starts, +% the assay window when filming ends. T is not +% recorded: it is read off the plate's videos with +% the spec's `transfer_protocol` +% (ndi.setup.conv.haley.transferTime) -- the lawn +% clip's start where the contrast video was filmed +% after the worms went on, else the first behaviour +% video's start -- marked approximate (the assay +% window's start, and the end of the window before it). No time where the source +% has none (an unfilmed plate, a missing pick time). +% Each time carries its own bound (`tolerance` +% [minus plus] s, CHANGE 7; decisions #52, #53): +% the pick and starvedTime, read off a clock and +% written to the minute, the spec's +% `hand_written_tolerance_seconds` (+/- 60) and a +% minute-resolution source_value; T [max_before 0] +% or tighter (transferTime) -- it is only ever early; +% a window's duration the sum of its ends' bounds. +% Exact (no tolerance): video times -- the end of +% filming here; each epoch's own reference states +% them exactly. +% Stated once, on the cohort, with `distributive` +% true (it holds of each worm) when the schema has +% the field (did-schema #84). +% +% Wall-clock times are America/Los_Angeles, as everywhere in this import. +% +% Option 'Protocol': the spec's `transfer_protocol` section (default: none, +% so T is the first behaviour video's start everywhere). +% +% OUT fields: documents (cell of structs: relations and their time +% references), counts (struct: one count per relation kind), skipped +% (cellstr: a relation whose parent is not a subject of this session), +% timeReferenceIds (containers.Map: '|' -> the id of that +% window's time reference, for the moves stage 8 states at the same times), +% assaySpans (containers.Map: assay plate -> struct T, tol, stop, method: +% the transfer time with its bound, the end of filming (wall clock) and the +% protocol's transfer method, for the moves and windows of stage 8). + +arguments + session table + S table + R table + subjectIds + options.Protocol = struct() +end +if height(session) ~= 1 + error('ndi:setup:conv:haley:oneSession', 'Give exactly one session row.'); +end +sid = char(session.session_id{1}); +ref = char(session.local_identifier{1}); +tz = 'America/Los_Angeles'; +S = S(strcmp(S.session, ref), :); +R = R(strcmp(R.session, ref), :); + +out = struct('documents', {{}}, 'skipped', {{}}, 'counts', struct( ... + 'patch_part_of_plate', 0, 'worm_member_of_cohort', 0, 'cohort_in_assay_plate', 0, ... + 'cohort_in_acclimation_plate', 0, 'cohort_in_food_deprivation_plate', 0), ... + 'timeReferenceIds', containers.Map(), 'assaySpans', containers.Map()); +docs = {}; +eachMember = struct(); % `distributive`, when the schema has it (#84) +if ndi.setup.V2.schemaHasField('directed_relation', 'distributive') + eachMember.distributive = true; +end + +% ---- patch part_of plate -------------------------------------------------------- +patches = S(strcmp(S.kind, 'patch'), :); +for k = 1:height(patches) + child = patches.local_identifier{k}; + parent = regexprep(child, '_patch\d+$', ''); % _patchNNNN (decision #42) + [docs, out] = relate(docs, out, subjectIds, child, parent, 'part_of', {}, sid, ... + 'patch_part_of_plate', struct()); +end + +% ---- worm member_of its cohort -------------------------------------------------- +worms = S(strcmp(S.kind, 'worm'), :); +for k = 1:height(worms) + if isempty(worms.cohort{k}) + continue; + end + [docs, out] = relate(docs, out, subjectIds, worms.local_identifier{k}, worms.cohort{k}, ... + 'member_of', {}, sid, 'worm_member_of_cohort', struct()); +end + +% ---- cohorts: each plate the worms were on, in turn ---------------------------- +% acclimation plate (from the pick) -> [food deprivation plate (from +% starvedTime)] -> assay plate (from the transfer T). Each window runs from +% its plate's start to the next one's; the assay window ends with filming. +% T is not recorded: ndi.setup.conv.haley.transferTime reads it off the +% plate's videos with the spec's transfer protocol (approximate either way). +behaviour = R(strcmp(R.kind, 'behaviour'), :); +plates = S(strcmp(S.kind, 'assay_plate'), :); +holding = S(ismember(S.kind, {'acclimation_plate', 'food_deprivation_plate'}), :); +cohorts = S(strcmp(S.kind, 'cohort'), :); +lawns = R(strcmp(R.kind, 'lawn'), :); +spanOf = containers.Map(); % assay plate id -> struct T, tol, stop (no zone) +handTol = [60 60]; % a time read off a clock and written to the minute +if isfield(options.Protocol, 'hand_written_tolerance_seconds') + handTol = reshape(double(options.Protocol.hand_written_tolerance_seconds), 1, 2); +end +minuteFmt = 'yyyy-MM-dd''T''HH:mm'; % written to the minute: source_value says so +secondFmt = 'yyyy-MM-dd''T''HH:mm:ss'; % read off a video's timestamp +for k = 1:height(plates) + id = plates.local_identifier{k}; + v = behaviour(strcmp(behaviour.plate, id), :); + if height(v) == 0 + continue; + end + dur = v.duration; + dur(isnan(dur)) = 0; + t = wallClock(v.local_start, tz); + [t0, first] = min(t); + [~, stem] = fileparts(v.file{first}); + l = lawns(strcmp(lawns.plate, id), :); + lawnStart = NaT; lawnEnd = NaT; + stems = {stem}; + if height(l) > 0 + [lawnStart, li] = min(wallClock(l.local_start, tz)); + [~, stems{2}] = fileparts(l.file{li}); % an exception may name the lawn clip + if ~isnan(l.duration(li)) % known with 'ReadVideos' + lawnEnd = lawnStart + seconds(l.duration(li)); + end + end + [T, rule] = ndi.setup.conv.haley.transferTime(options.Protocol, plates.folder{k}, ... + t0, lawnStart, stems, lawnEnd); + spanOf(id) = struct('T', T, 'tol', rule.tolerance, 'stop', max(t + seconds(dur)), ... + 'method', rule.method); +end +refOf = containers.Map(); % plate id|cohort id -> time reference id +for k = 1:height(cohorts) + w = cohorts(k, :); + plate = plates.local_identifier(plates.plate == w.plate & strcmp(plates.folder, w.folder)); + if isempty(plate) + out.skipped{end+1} = sprintf('%s: no assay plate %d in this session', ... + w.local_identifier{1}, w.plate); %#ok + continue; + end + plate = plate{1}; + T = NaT; Ttol = []; filmedEnd = NaT; + if isKey(spanOf, plate) + span = spanOf(plate); + T = span.T; Ttol = span.tol; filmedEnd = span.stop; + end + % each move: when (wall clock), its bound [minus plus] s, how it was written + seq = struct('id', {}, 'start', {}, 'tol', {}, 'fmt', {}, 'counter', {}); + if ~isempty(w.acclimation{1}) % the pick: read off a clock + seq(end+1) = struct('id', w.acclimation{1}, 'start', placed(holding, w.acclimation{1}, tz), ... + 'tol', handTol, 'fmt', minuteFmt, 'counter', 'cohort_in_acclimation_plate'); %#ok + end + if ~isempty(w.deprivation{1}) % starvedTime: read off a clock + seq(end+1) = struct('id', w.deprivation{1}, 'start', placed(holding, w.deprivation{1}, tz), ... + 'tol', handTol, 'fmt', minuteFmt, 'counter', 'cohort_in_food_deprivation_plate'); %#ok + end + seq(end+1) = struct('id', plate, 'start', T, 'tol', Ttol, 'fmt', secondFmt, ... + 'counter', 'cohort_in_assay_plate'); %#ok % T: read off a video, early only + for i = 1:numel(seq) + if i < numel(seq) % ends when the next move starts + stop = seq(i+1).start; stopTol = seq(i+1).tol; stopFmt = seq(i+1).fmt; + else % the end of filming: exact + stop = filmedEnd; stopTol = []; stopFmt = secondFmt; + end + times = {}; + key = [seq(i).id '|' w.local_identifier{1}]; + if isKey(refOf, key) + times = {refOf(key)}; + elseif ~isnat(seq(i).start) + dur = []; + if ~isnat(stop) + dur = seconds(stop - seq(i).start); + if dur < 0 + out.skipped{end+1} = sprintf(['%s on %s: ends (%s) before it starts (%s); ' ... + 'recorded with its start only'], w.local_identifier{1}, seq(i).id, ... + char(stop), char(seq(i).start)); %#ok + dur = []; + end + end + d = utcReference(seq(i).start, seq(i).tol, seq(i).fmt, ... + stop, stopTol, stopFmt, ~isempty(dur), tz, sid); + docs{end+1} = d; %#ok + refOf(key) = d.base.id; + times = {d.base.id}; + end + [docs, out] = relate(docs, out, subjectIds, w.local_identifier{1}, seq(i).id, ... + 'contained_in', times, sid, seq(i).counter, eachMember); + end +end +out.documents = docs; +out.timeReferenceIds = refOf; +out.assaySpans = spanOf; +end + +% ----------------------------------------------------------------------------- +function [docs, out] = relate(docs, out, ids, child, parent, relation, times, sid, counter, fields) +if ~isKey(ids, child) || ~isKey(ids, parent) + out.skipped{end+1} = sprintf('%s %s %s: not both subjects of this session', ... + child, relation, parent); + return; +end +docs{end+1} = did2.build.directedRelation(ids(child), ids(parent), relation, ... + 'TimeReferenceIds', times, 'SessionId', sid, 'Fields', fields); +out.counts.(counter) = out.counts.(counter) + 1; +end + +function t = placed(holding, id, tz) +% when the worms were put onto an acclimation / food deprivation plate +t = holding.worms_placed(strcmp(holding.local_identifier, id)); +if isempty(t) + t = NaT; +else + t = wallClock(t(1), tz); +end +end diff --git a/src/ndi/+ndi/+setup/+conv/+haley/seedingSuspensions.m b/src/ndi/+ndi/+setup/+conv/+haley/seedingSuspensions.m new file mode 100644 index 0000000000..2c987eea5a --- /dev/null +++ b/src/ndi/+ndi/+setup/+conv/+haley/seedingSuspensions.m @@ -0,0 +1,331 @@ +function [S, checks] = seedingSuspensions(dataParentDir, spec) +%SEEDINGSUSPENSIONS Stage 8 part B (Haley): the bacterial suspensions plates were seeded with. +% +% [S, CHECKS] = ndi.setup.conv.haley.seedingSuspensions(DATAPARENTDIR, SPEC) +% reads the seeding columns of every source table under DATAPARENTDIR/haley +% and returns the suspensions as spec-shaped `formulations` entries, for +% ndi.setup.V2.datasetMetadata to build with the spec's own (decisions #56, +% #57). They are dataset-level: one day's suspensions served plates filmed on +% later days and in other sessions. +% +% Per seeding day the lab pelleted a culture, resuspended it (the master), +% made an OD600 10 solution from it, measured that solution (`OD600Real`) +% and, on most days, its CFU (`CFU`), then diluted it to each OD600 the day +% needed. So, per day: +% OD600 10 solution ingredients: the strain at OD600Real (source unit +% OD600; particles_per_liter = CFU x 2e10 when the +% CFU was measured -- decision #57), then the diluent +% dilution, per OD600 ingredients: the OD600 10 solution as a +% volume_fraction of nominal/10 (source value: the +% nominal OD600), then the diluent +% OD600 10 is the solution itself and OD600 0 the diluent alone, so neither +% is a dilution. There is no master or culture entry (decision #57, option +% B): the master's own reading is only in the Benchling entries, and no +% column names a culture bottle. +% +% Studies that share a seeding day AND read identically (same OD600Real and +% CFU) used one solution (Jess, decision #57): it is built once. If those +% studies' diluents differ (SPEC.seeding.diluent), the solution's diluent is +% unknown and it lists none. The spec's `corrections` with column `CFU` or +% `OD600Real` are applied first (by folder and seed_day). +% +% Acclimation (growth) plates were all seeded alike, 200 uL at `growthOD600` +% (OD600 1) in SPEC.seeding.growth_diluent (LB; the paper's Methods, and Jess +% for Matching's): a growth seeding uses that day's solution in that diluent +% (or of unknown diluent), its own study's if it has one, else another +% study's -- Matching's S-Complete days use Mini's LB solution. Its nominal +% OD600 joins that solution's dilutions. A growth seeding day with no such +% solution (no assay plate seeded that day, so no OD600Real or CFU read: +% foragingMutants 2023-11-09) gets an UNMEASURED one (decision #82): one +% formulation per nominal OD600, the strain at that nominal OD600 in the +% growth diluent, with no stock and no measured concentration; its index +% rows carry a `note` saying so, which the dose repeats. +% +% S has fields: +% entries 1xN cell of formulation entries (key, ingredients; no +% `type`: a day's one-off mixture is no standard recipe -- the +% signed formulation.value.type rule, did-schema #84), +% each solution before its dilutions +% index table: folder, day, od600 (nominal), key -- which formulation a +% plate of FOLDER seeded on DAY at OD600 was pipetted from +% ('' for OD600 0 with an unknown diluent); kind is "assay" or +% "growth" +% CHECKS (also printed): missingColumns (a table without the seeding +% columns; nothing built from it), twoReadings (a study-day with more than +% one OD600Real/CFU; the first is used), noReading (rows with no timeSeed or +% OD600Real), growthWithoutSolution (a growth seeding day with no measured +% solution; an unmeasured one is made, decision #82). +% +% See also ndi.setup.V2.datasetMetadata, ndi.setup.conv.haley.import_V2. + +arguments + dataParentDir (1,:) char {mustBeFolder} + spec struct +end + +cfg = spec.seeding; +root = fullfile(dataParentDir, 'haley'); +checks = struct('missingColumns', {{}}, 'twoReadings', {{}}, 'noReading', {{}}, ... + 'growthWithoutSolution', {{}}); +need = {'timeSeed', 'OD600Real', 'CFU', 'OD600'}; + +% ---- every source table: (folder, table) ------------------------------------ +tables = {}; +d = dir(fullfile(root, 'celegans', '*', 'experimentInfo.mat')); +for k = 1:numel(d) + I = load(fullfile(d(k).folder, d(k).name), 'info'); + [~, folder] = fileparts(d(k).folder); + tables(end+1, :) = {folder, I.info}; %#ok +end +f = fullfile(root, 'ecoli', 'bacteria.mat'); +if isfile(f) + B = load(f, 'info'); + tables(end+1, :) = {'ecoli', B.info}; +end + +% ---- one reading per study and seeding day --------------------------------- +R = struct('folder', {}, 'strain', {}, 'diluent', {}, 'day', {}, 'od', {}, 'cfu', {}, ... + 'nominal', {}); +growth = struct('folder', {}, 'day', {}, 'nominal', {}); +nTables = 0; +nRows = 0; +for t = 1:size(tables, 1) + folder = tables{t, 1}; + T = tables{t, 2}; + missing = setdiff(need, T.Properties.VariableNames); + if ~isempty(missing) + checks.missingColumns{end+1} = sprintf('%s: no column(s) %s', folder, strjoin(missing, ', ')); + continue; + end + nTables = nTables + 1; + nRows = nRows + height(T); + T = applyCorrections(T, folder, spec); + day = dateshift(T.timeSeed, 'start', 'day'); + od = double(T.OD600Real); + cfu = double(T.CFU); + bad = isnat(day) | isnan(od); + if any(bad) + checks.noReading{end+1} = sprintf('%s: %d row(s) with no timeSeed or OD600Real', folder, nnz(bad)); + end + days = unique(day(~bad)); + for k = 1:numel(days) + r = find(~bad & day == days(k)); + % a missing CFU is a value here (-1), so two NaN rows are one reading + readings = unique(fillmissing([od(r), cfu(r)], 'constant', -1), 'rows'); + if size(readings, 1) > 1 + checks.twoReadings{end+1} = sprintf('%s %s: %d readings; the first is used', ... + folder, datestr(days(k), 'yyyy-mm-dd'), size(readings, 1)); %#ok + end + nominal = []; + for j = r(:)' + nominal = [nominal, flat(T.OD600(j, :))]; %#ok + end + R(end+1) = struct('folder', folder, 'strain', pick(cfg.strain, folder), ... + 'diluent', pick(cfg.diluent, folder), 'day', days(k), 'od', od(r(1)), ... + 'cfu', cfu(r(1)), 'nominal', uniq(nominal)); %#ok + end + if all(ismember({'growthTimeSeed', 'growthOD600'}, T.Properties.VariableNames)) + g = dateshift(T.growthTimeSeed, 'start', 'day'); + ok = ~isnat(g); + gd = unique(g(ok)); + for k = 1:numel(gd) + growth(end+1) = struct('folder', folder, 'day', gd(k), ... + 'nominal', uniq(double(T.growthOD600(ok & g == gd(k))))); %#ok + end + end +end + +% ---- solutions: identical same-day readings are one solution --------------- +sig = arrayfun(@(x) sprintf('%s|%s|%.6g|%.6g', x.strain, datestr(x.day, 'yyyymmdd'), x.od, x.cfu), ... + R, 'UniformOutput', false); %#ok +[usig, ~, grp] = unique(sig, 'stable'); +sol = struct('key', {}, 'strain', {}, 'day', {}, 'od', {}, 'cfu', {}, 'diluent', {}, ... + 'folders', {}, 'nominal', {}); +for s = 1:numel(usig) + members = R(grp == s); + x = members(1); + dils = unique({members.diluent}); + diluent = ''; % studies with different diluents shared it: unknown + if isscalar(dils) + diluent = dils{1}; + end + sameDay = find(cellfun(@(q) startsWith(q, sprintf('%s|%s|', x.strain, ... + datestr(x.day, 'yyyymmdd'))), usig)); %#ok + key = sprintf('suspension_%s_%s', lower(regexprep(x.strain, '\W', '')), ... + datestr(x.day, 'yyyymmdd')); %#ok + if numel(sameDay) > 1 + key = sprintf('%s_%c', key, 'a' + find(sameDay == s) - 1); + end + sol(s) = struct('key', key, 'strain', x.strain, 'day', x.day, 'od', x.od, 'cfu', x.cfu, ... + 'diluent', diluent, 'folders', {{members.folder}}, 'nominal', uniq([members.nominal])); +end + +% ---- growth seedings: that day's solution in the growth diluent ------------ +% Every acclimation plate was seeded alike (SPEC.seeding.growth_diluent): from +% its own study's solution of the day when that is in the growth diluent (or +% of unknown diluent), else from another study's that is. +growthRows = zeros(0, 3); % [growth k, solution s, nominal] +gdil = getOr(cfg, 'growth_diluent', ''); +for k = 1:numel(growth) + strain = pick(cfg.strain, growth(k).folder); + cand = []; + if ~isempty(sol) % [sol.day] of no solutions is a double, not a datetime + cand = find(strcmp({sol.strain}, strain) & [sol.day] == growth(k).day & ... + (strcmp({sol.diluent}, gdil) | strcmp({sol.diluent}, ''))); + end + own = cand(arrayfun(@(c) any(strcmp(sol(c).folders, growth(k).folder)), cand)); + if ~isempty(own) + s = own(1); + elseif ~isempty(cand) + s = cand(1); + else + % no reading that day: the seeding happened, its strength is the + % nominal one only (decision #82) + checks.growthWithoutSolution{end+1} = sprintf(['%s %s: acclimation plates seeded, ' ... + 'no %s solution measured that day; recorded at its nominal OD600, unmeasured'], ... + growth(k).folder, datestr(growth(k).day, 'yyyy-mm-dd'), gdil); %#ok + key = sprintf('suspension_%s_%s_unmeasured', lower(regexprep(strain, '\W', '')), ... + datestr(growth(k).day, 'yyyymmdd')); %#ok + s = find(strcmp({sol.key}, key), 1); + if isempty(s) + sol(end + 1) = struct('key', key, 'strain', strain, 'day', growth(k).day, 'od', NaN, ... + 'cfu', NaN, 'diluent', gdil, 'folders', {{growth(k).folder}}, 'nominal', []); + s = numel(sol); + end + end + sol(s).nominal = uniq([sol(s).nominal, growth(k).nominal]); + for n = growth(k).nominal + growthRows(end+1, :) = [k, s, n]; %#ok + end +end + +% ---- entries and the index ------------------------------------------------- +entries = {}; +index = table('Size', [0 6], 'VariableTypes', {'cell', 'datetime', 'double', 'cell', 'cell', 'cell'}, ... + 'VariableNames', {'folder', 'day', 'od600', 'key', 'kind', 'note'}); +for s = 1:numel(sol) + x = sol(s); + if isnan(x.od) + % unmeasured (decision #82): no stock; each nominal OD600 is its own + % formulation, its strength the nominal value alone + for n = x.nominal + ing = {struct('ingredient', x.strain, 'concentration', ... + struct('source_value', n, 'source_unit', 'OD600'))}; + if ~isempty(x.diluent) + ing{end+1} = struct('ingredient', x.diluent); %#ok + end + entries{end+1} = struct('key', keyFor(x.key, n, x.diluent), 'ingredients', {ing}); %#ok + end + continue; + end + c = struct(); + if ~isnan(x.cfu) + c.particles_per_liter = x.cfu * 2e10; + end + c.source_value = x.od; + c.source_unit = 'OD600'; + ing = {struct('ingredient', x.strain, 'concentration', c)}; + if ~isempty(x.diluent) + ing{end+1} = struct('ingredient', x.diluent); %#ok + end + entries{end+1} = struct('key', x.key, 'ingredients', {ing}); %#ok + for n = x.nominal(x.nominal > 0 & x.nominal ~= 10) + ing = {struct('ingredient', x.key, 'concentration', struct('volume_fraction', n / 10, ... + 'source_value', n, 'source_unit', 'OD600'))}; + if ~isempty(x.diluent) + ing{end+1} = struct('ingredient', x.diluent); %#ok + end + entries{end+1} = struct('key', keyFor(x.key, n, x.diluent), 'ingredients', {ing}); %#ok + end +end +for r = 1:numel(R) + x = sol(grp(r)); + for n = R(r).nominal + index(end+1, :) = {R(r).folder, R(r).day, n, keyFor(x.key, n, x.diluent), 'assay', ''}; %#ok + end +end +for k = 1:size(growthRows, 1) + g = growth(growthRows(k, 1)); + x = sol(growthRows(k, 2)); + n = growthRows(k, 3); + note = ''; + if isnan(x.od) + note = sprintf(['The suspension was not measured that day (no assay plate was seeded, so ' ... + 'no OD600 or CFU was read); its strength is the nominal OD600 %g (decision #82).'], n); + end + index(end+1, :) = {g.folder, g.day, n, keyFor(x.key, n, x.diluent), 'growth', note}; %#ok +end + +S = struct('entries', {entries}, 'index', index); +fprintf(['DENOMINATOR: %d source table(s) with seeding columns, %d row(s); %d study-day ' ... + 'reading(s) -> %d OD600 10 solution(s), %d formulation(s) in all; %d growth seeding ' ... + 'day(s), %d matched to a solution\n'], ... + nTables, nRows, numel(R), numel(usig), numel(entries), numel(growth), ... + numel(unique(growthRows(:, 1)))); +names = fieldnames(checks); +n = sum(cellfun(@(q) numel(checks.(q)), names)); +if n > 0 + fprintf('CHECKS (reported, nothing changed): %d finding(s)\n', n); + for k = 1:numel(names) + for j = 1:numel(checks.(names{k})) + fprintf(' %-22s %s\n', names{k}, checks.(names{k}){j}); + end + end +end +end + +function T = applyCorrections(T, folder, spec) +% The spec's named corrections to a seeding column (decision #57). +if ~isfield(spec, 'corrections') + return; +end +c = spec.corrections; +if isstruct(c), c = num2cell(c); end +for k = 1:numel(c) + x = c{k}; + if ~strcmp(x.folder, folder) || ~any(strcmp(x.column, {'CFU', 'OD600Real'})) ... + || ~isfield(x, 'seed_day') + continue; + end + r = dateshift(T.timeSeed, 'start', 'day') == datetime(x.seed_day, 'InputFormat', 'yyyy-MM-dd'); + v = x.value; + if isempty(v), v = NaN; end + col = double(T.(x.column)); + col(r) = v; + T.(x.column) = col; +end +end + +function k = keyFor(key, n, diluent) +if n == 10 + k = key; +elseif n == 0 + k = diluent; +else + k = sprintf('%s_od%s', key, strrep(sprintf('%g', n), '.', 'p')); +end +end + +function v = flat(x) +if iscell(x), x = [x{:}]; end +if iscell(x), x = [x{:}]; end +v = double(x(:)'); +v = v(~isnan(v)); +end + +function v = uniq(v) +v = unique(round(double(v(:)'), 9)); +v = v(~isnan(v)); +end + +function v = pick(s, folder) +if isfield(s, folder) + v = s.(folder); +else + v = s.default; +end +end + +function v = getOr(s, name, default) +if isfield(s, name), v = s.(name); else, v = default; end +end diff --git a/src/ndi/+ndi/+setup/+conv/+haley/sessionDocuments.m b/src/ndi/+ndi/+setup/+conv/+haley/sessionDocuments.m new file mode 100644 index 0000000000..9b82789c58 --- /dev/null +++ b/src/ndi/+ndi/+setup/+conv/+haley/sessionDocuments.m @@ -0,0 +1,387 @@ +function out = sessionDocuments(dataParentDir, session, S, R, options) +%SESSIONDOCUMENTS Stages 4-5 (Haley), written: every document for ONE session. +% +% OUT = ndi.setup.conv.haley.sessionDocuments(DATAPARENTDIR, SESSION, S, R) +% builds, for the one-row session table SESSION (from sessionList, with +% `session_id` already assigned), the V2 documents of its subjects (S rows, +% from subjectList) and recordings (R rows, from recordingList). Nothing is +% written here (ndi.setup.V2.makeSessions writes the documents), and +% nothing is ever written into the session folder or the raw data: NDI +% finds each recording through its documents (ndi.file.navigator.bodies, +% decision #51). Decision log #28-#35, #45-#51. +% +% Documents, all with base.session_id = SESSION.session_id: +% subject one per S row (local_identifier, name, description, +% and type when the schema has subject.type) +% absolute_time_reference the session's UTC extent (first recording start +% to last recording end); OUT.timeReferenceId, for +% session.time_reference_id +% software the three NDI classes an acquisition system is +% rebuilt from (decision #29): ndi.daq.system.image, +% ndi.file.navigator.bodies, ndi.daq.reader.image.ndr +% acquisition_reader one per reader string: 'video' (cameras), +% 'tiffstack' (microscope) +% epoch_file_pattern one per system: the file names that are its +% recordings (the navigator filters bodies by it) +% acquisition_system one per system in the session (camera1, camera2, +% microscope; decision #28) +% acquisition_channels one per system (no channel numbers: a camera has +% no ai/ao/di/do channels, the only types the +% schema binds) +% and per recording (R row): +% epoch local_identifier = R.epoch, which is also NDI's +% epoch id (the navigator reads it from here) +% relative_time_reference dev_local_time from 0 for the recording's +% duration, referent the SESSION document (videos) +% absolute_time_reference its UTC start (and duration when known) +% intensity_observation the recording: subject = the plate, instrument = +% the dataset-level camera / microscope subject +% (when 'InstrumentIds' has it), acquisition +% channels = this session's system; times = a +% relative reference to the EPOCH (videos: the +% same extent; images: `during`) + the UTC one; +% method = the imaging method, which is also the +% NDI probe type (brightfield-imaging / +% wide-field-imaging) +% datum_type: a video's from VideoReader's +% VideoFormat (RGB24/Grayscale -> uint8, +% RGB48/Mono16 -> uint16), a TIFF's from its +% header (imfinfo); unreadable -> skipped +% opaque_body the file, NOT held (decision #34): format, +% filename, size, modified time, MD5, and the +% file member body_data_0 recorded BY LOCATION +% (files.file_info.locations: the absolute path, +% location_type 'file', ingest 0) -- the bytes +% stay where they are until ingestion +% +% OUT fields: documents (cell of structs), timeReferenceId, skipped +% (cellstr: recordings not written, and why), subjectIds (containers.Map, +% subject local_identifier -> document id, for stage 6), recordingRefs +% (containers.Map, epoch -> the id of its UTC reference, for stages 9-10), +% recordingStatements (containers.Map, epoch -> the id of its recording +% statement, the input of stage 10's calculations). +% +% Options: +% 'InstrumentIds' containers.Map, spec key -> document id (from stage 2: +% camera_1, camera_2, axio_zoom_1). Missing keys leave +% instrument_id empty. +% 'Checksums' default true: MD5 of every recording (reads every byte) +% 'ReadVideos' default true: open each video (VideoReader) for its +% pixel format; false assumes uint8 (8-bit) without +% looking + +arguments + dataParentDir (1,:) char {mustBeFolder} + session table + S table + R table + options.InstrumentIds = containers.Map() + options.Checksums (1,1) logical = true + options.ReadVideos (1,1) logical = true +end + +if height(session) ~= 1 + error('ndi:setup:conv:haley:oneSession', 'Give exactly one session row.'); +end +sid = char(session.session_id{1}); +if isempty(sid) + error('ndi:setup:conv:haley:noSessionId', 'The session row has no session_id yet.'); +end +sessionDocId = char(session.session_doc_id{1}); +ref = char(session.local_identifier{1}); +root = fullfile(dataParentDir, 'haley'); +if ~startsWith(root, filesep) && isempty(regexp(root, '^[A-Za-z]:', 'once')) + root = fullfile(pwd, root); % a body records an ABSOLUTE location +end +tz = 'America/Los_Angeles'; +S = S(strcmp(S.session, ref), :); +R = R(strcmp(R.session, ref), :); + +docs = {}; +out = struct('documents', {{}}, 'timeReferenceId', '', 'skipped', {{}}, ... + 'subjectIds', containers.Map(), 'recordingRefs', containers.Map(), ... + 'recordingStatements', containers.Map()); + +% ---- subjects --------------------------------------------------------------- +subjectIds = containers.Map(); +hasType = ndi.setup.V2.schemaHasField('subject', 'type') && ismember('type', S.Properties.VariableNames); +for k = 1:height(S) + f = struct('local_identifier', S.local_identifier{k}, 'name', S.name{k}); + if ~isempty(S.description{k}) + f.description = S.description{k}; + end + if hasType % decision #55; did-schema #84 + f.type = did2.build.term('', S.type{k}); + end + built = ndi.setup.V2.entityDocuments('subject', f, struct(), 'SessionId', sid); + subjectIds(S.local_identifier{k}) = built{1}.base.id; + docs = [docs, built]; %#ok +end + +% ---- the session's own UTC extent ----------------------------------------- +if height(R) > 0 + dur = R.duration; + dur(isnan(dur)) = 0; + t0 = min(R.local_start); + t1 = max(R.local_start + seconds(dur)); + z0 = t0; z1 = t1; % elapsed time in the zone (daylight saving) + if isempty(z0.TimeZone), z0.TimeZone = tz; end + if isempty(z1.TimeZone), z1.TimeZone = tz; end + d = utcReference(t0, seconds(z1 - z0), tz, sid); + out.timeReferenceId = d.base.id; + docs{end+1} = d; +end + +% ---- acquisition systems ---------------------------------------------------- +systems = unique(R.system, 'stable'); +if ~isempty(systems) + sw = struct(); + sw.system = softwareDoc('ndi.daq.system.image', sid); + sw.navigator = softwareDoc('ndi.file.navigator.bodies', sid); + sw.reader = softwareDoc('ndi.daq.reader.image.ndr', sid); + docs = [docs, {sw.system, sw.navigator, sw.reader}]; +end +readers = containers.Map(); +channelIds = containers.Map(); +for k = 1:numel(systems) + sys = systems{k}; + [readerString, filePattern] = systemFiles(sys); + if ~isKey(readers, readerString) + d = did2.build.document('acquisition_reader', struct('reader_string', readerString), ... + 'SessionId', sid, 'Edges', struct('software_id', sw.reader.base.id)); + readers(readerString) = d.base.id; + docs{end+1} = d; %#ok + end + pat = did2.build.document('epoch_file_pattern', struct( ... + 'file_pattern', {{filePattern}}, ... + 'epoch_map_format', 'ndi.epoch.epochprobemap_daqsystem'), ... + 'SessionId', sid, 'Edges', struct('software_id', sw.navigator.base.id)); + asys = did2.build.document('acquisition_system', struct('name', sys), 'SessionId', sid, ... + 'Edges', struct('software_id', sw.system.base.id, 'reader_id', readers(readerString), ... + 'epoch_file_pattern_id', pat.base.id)); + ch = did2.build.document('acquisition_channels', struct(), 'SessionId', sid, ... + 'Edges', struct('acquisition_system_id', asys.base.id)); + channelIds(sys) = ch.base.id; + docs = [docs, {pat, asys, ch}]; %#ok +end + +% ---- recordings: epoch, times, statement, body, files ----------------------- +for k = 1:height(R) + r = R(k, :); + plate = r.plate{1}; + if ~isKey(subjectIds, plate) + out.skipped{end+1} = sprintf('%s: plate %s is not a subject of this session', ... + r.epoch{1}, plate); + continue; + end + src = fullfile(root, r.file{1}); + [~, base, ext] = fileparts(src); + [datumType, why, hw] = datumTypeOf(src, ext, options.ReadVideos); + if isempty(datumType) + out.skipped{end+1} = sprintf('%s: %s', r.epoch{1}, why); + continue; + end + + isImage = strcmp(r.kind{1}, 'image'); + dur = r.duration; + if isImage || isnan(dur) + dur = []; + end + + absRef = utcReference(r.local_start, dur, tz, sid); + epochRefs = {absRef.base.id}; + if ~isImage + relSession = did2.build.relativeTimeReference(sessionDocId, 'Clock', 'dev_local_time', ... + 'Start', 0, 'Duration', dur, 'SessionId', sid); + epochRefs{end+1} = relSession.base.id; %#ok + docs{end+1} = relSession; %#ok + end + ep = ndi.setup.V2.entityDocuments('epoch', struct('local_identifier', r.epoch{1}), ... + struct('time_reference_id', {epochRefs}), 'SessionId', sid); + ep = ep{1}; + if isImage + relEpoch = did2.build.relativeTimeReference(ep.base.id, 'Relation', 'intervalDuring', 'SessionId', sid); + else + relEpoch = did2.build.relativeTimeReference(ep.base.id, 'Clock', 'dev_local_time', ... + 'Start', 0, 'Duration', dur, 'SessionId', sid); + end + + instrumentKey = instrumentFor(r.system{1}); + instrumentId = ''; + if isKey(options.InstrumentIds, instrumentKey) + instrumentId = options.InstrumentIds(instrumentKey); + end + keys = recordingKeys(hw, isImage, r); + keyArgs = {}; + if ~isempty(keys), keyArgs = {'Keys', keys}; end + st = did2.build.statement('intensity_observation', subjectIds(plate), ... + did2.build.term('', 'image intensity'), [], 'DataBody', true, 'DatumType', datumType, ... + keyArgs{:}, ... + 'Method', did2.build.term('', probeType(r.system{1})), ... + 'InstrumentId', instrumentId, 'AcquisitionChannelsId', channelIds(r.system{1}), ... + 'TimeReferenceIds', {relEpoch.base.id, absRef.base.id}, 'SessionId', sid, ... + 'Notes', noteOf(r)); + + info = dir(src); + f = struct('format', mediaType(ext), 'filename', [base ext], ... + 'size_bytes', info.bytes, ... + 'file_modified', isoUtc(datetime(info.datenum, 'ConvertFrom', 'datenum', 'TimeZone', 'local')), ... + 'description', sprintf('The recording; the source table names it %s.', r.source_name{1})); + if options.Checksums + f.content_hash = ndi.fun.file.MD5(src); + f.hash_algorithm = 'MD5'; + end + body = did2.build.document('opaque_body', f, 'SessionId', sid, ... + 'Edges', struct('owner_id', st.base.id), 'Files', {'body_data_0'}); + % recorded BY LOCATION, not ingested (as ndi.document/add_file writes it) + body.files.file_info = struct('name', 'body_data_0', 'locations', struct( ... + 'delete_original', 0, 'uid', ndi.ido.unique_id(), 'location', src, ... + 'parameters', '', 'location_type', 'file', 'ingest', 0)); + docs = [docs, {absRef, ep, relEpoch, st, body}]; %#ok + out.recordingRefs(r.epoch{1}) = absRef.base.id; + out.recordingStatements(r.epoch{1}) = st.base.id; + +end +out.documents = docs; +out.subjectIds = subjectIds; +end + +% ----------------------------------------------------------------------------- +function [readerString, filePattern] = systemFiles(sys) +switch sys + case 'camera1' + readerString = 'video'; filePattern = '.*_1\.mp4\>'; + case 'camera2' + readerString = 'video'; filePattern = '.*_2\.mp4\>'; + case 'microscope' + readerString = 'tiffstack'; filePattern = '.*\.tiff\>'; + otherwise + error('ndi:setup:conv:haley:unknownSystem', 'Unknown acquisition system "%s".', sys); +end +end + +function key = instrumentFor(sys) +% acquisition system -> the dataset-level instrument subject's SPEC KEY (the +% key stage 2's id map uses; not its local_identifier, decision #59) +switch sys + case 'microscope', key = 'axio_zoom_1'; + case 'camera1', key = 'camera_1'; + case 'camera2', key = 'camera_2'; + otherwise, key = sys; +end +end + +function t = probeType(sys) +% ndi_common/probe/probetype2object.json; all map to ndi.probe.image +if strcmp(sys, 'microscope') + t = 'wide-field-imaging'; +else + t = 'brightfield-imaging'; +end +end + +function keys = recordingKeys(hw, isImage, r) +% The recording's dimensions (decision #70, doImport's imageStack_parameters): +% [image vertical position, image horizontal position], in pixels from the +% upper-left corner (pixel centres at 0.5, 1.5, ...; the scale is a +% calculation, stage 10), then for a video `time`, one per frame from the +% recording's start at 1 / frame rate. Only when the pixel size is known (the +% file was opened: 'ReadVideos', or a TIFF), so the keys are always complete. +keys = []; +if isempty(hw), return; end +k = {did2.build.key('image vertical position', hw(1), 'Unit', 'pixel', 'Origin', 0.5, 'Spacing', 1), ... + did2.build.key('image horizontal position', hw(2), 'Unit', 'pixel', 'Origin', 0.5, 'Spacing', 1)}; +if ~isImage + nf = r.n_frames; fr = r.frame_rate; + if ~(nf > 0 && fr > 0), return; end + k{end+1} = did2.build.key('time', nf, 'Unit', 'second', 'Origin', 0, 'Spacing', 1 / fr, ... + 'SourceUnit', 'frame', 'SourceOrigin', 0, 'SourceSpacing', 1); +end +keys = did2.build.list(k{:}); +end + +function [t, why, hw] = datumTypeOf(file, ext, readVideos) +% How the recording's pixel values are encoded (data_type.datum_type), and +% its [height width] in pixels when the file is opened ([] otherwise). +t = ''; why = ''; hw = []; +switch lower(ext) + case '.mp4' + if ~readVideos + t = 'uint8'; % assumed ('ReadVideos', false): not opened + return; + end + try + v = VideoReader(file); + fmt = v.VideoFormat; + hw = [v.Height v.Width]; + catch err + why = sprintf('VideoReader cannot open it (%s)', err.message); + return; + end + switch fmt + case {'RGB24', 'Grayscale', 'Indexed', 'RGB24 Signed'} + t = 'uint8'; + case {'RGB48', 'Mono16', 'RGB48 Signed', 'Mono16 Signed'} + t = 'uint16'; + otherwise + why = sprintf('video format %s has no datum_type mapping', fmt); + end + case {'.tif', '.tiff'} + try + info = imfinfo(file); + catch err + why = sprintf('cannot read the TIFF header (%s)', err.message); + return; + end + bits = info(1).BitsPerSample(1); + hw = [info(1).Height info(1).Width]; + if isfield(info, 'SampleFormat') && strcmpi(info(1).SampleFormat, 'IEEE floating point') + t = sprintf('float%d', bits); + elseif any(bits == [8 16 32 64]) + t = sprintf('uint%d', bits); + else + why = sprintf('%d bits per sample has no datum_type', bits); + end +end +end + +function m = mediaType(ext) +% the file format: an EDAM format term (fileFormat writes the media type on +% a schema whose data_body.format is still a string) +switch lower(ext) + case '.mp4', m = fileFormat('EDAM:format_3997', 'MPEG-4', 'video/mp4'); + case {'.tif', '.tiff'}, m = fileFormat('EDAM:format_3591', 'TIFF', 'image/tiff'); + otherwise + error('ndi:setup:conv:haley:unknownFormat', 'No media type for "%s".', ext); +end +end + +function d = utcReference(localStart, dur, tz, sid) +t = localStart; +t.TimeZone = tz; +u = t; +u.TimeZone = 'UTC'; +d = did2.build.absoluteTimeReference(char(u, 'yyyy-MM-dd''T''HH:mm:ss.SSS''Z'''), ... + 'SourceValue', char(t, 'yyyy-MM-dd''T''HH:mm:ss'), 'SourceTimezone', tz, ... + 'Duration', dur, 'SessionId', sid); +end + +function s = isoUtc(t) +% dir() reports the file time in this computer's own time zone +t.TimeZone = 'UTC'; +s = char(t, 'yyyy-MM-dd''T''HH:mm:ss''Z'''); +end + +function d = softwareDoc(name, sid) +% a `software` entity (or, since 2026-10-08, an `entity` of type software) +d = ndi.setup.V2.entityDocuments('software', struct('name', name), struct(), 'SessionId', sid); +d = d{1}; +end + +function n = noteOf(r) +% a recording row's note (recordingList: e.g. decision #83's camera swap) +n = ''; +if ismember('note', r.Properties.VariableNames) && ~isempty(r.note{1}) + n = char(r.note{1}); +end +end diff --git a/src/ndi/+ndi/+setup/+conv/+haley/sessionList.m b/src/ndi/+ndi/+setup/+conv/+haley/sessionList.m new file mode 100644 index 0000000000..22056242aa --- /dev/null +++ b/src/ndi/+ndi/+setup/+conv/+haley/sessionList.m @@ -0,0 +1,277 @@ +function [T, checks] = sessionList(dataParentDir, spec, options) +%SESSIONLIST Stage 3 (Haley): the sessions to create, one per experiment. +% +% T = ndi.setup.conv.haley.sessionList(DATAPARENTDIR, SPEC) reads the raw +% data under DATAPARENTDIR/haley and returns one row per session, ready for +% ndi.setup.V2.makeSessions (decision log entries 5, 21-24): +% +% C. elegans one row per row of /tableOfContents.xlsx, +% i.e. one per experiment day. The row's study is found from +% SPEC.studies by `source_folder`, and for foragingConcentration +% (two studies in one folder) by the FIRST WORD of the row's +% `conditions` (`grid` / `single`) against `source_condition`. +% E. coli one row per `expNum` in ecoli/bacteria.mat (entry 23), dated +% by that experiment's earliest image (`metaData.acquisitionTime`). +% +% Columns: local_identifier (e.g. concentration_0001; the prefix is +% ndi.setup.conv.haley.idPrefix of the folder), name, description +% (notebook, conditions, worm range, inclusion, notes), path +% (OUTPUTROOT//), study_key, study_ids, source, +% folder, video_folder (C. elegans: celegans//videos/, +% relative to DATAPARENTDIR/haley; '' for E. coli), experiment, date, +% include, notes (the row's own `notes`; '' for E. coli), worm_first, +% worm_last. `include` is the row's flag as written: the lab's analysis +% excludes by plate, not by day (decision #73), so a day marked `no` can +% still have plates in the analysis. +% +% CHECKS (second output, also printed) reports what looks wrong in the +% source's own day index, WITHOUT changing anything: +% sharedDay sessions of one study with the same directoryName (a +% copy-down in the spreadsheet: foragingMatching had all +% five rows at 23-02-24, 2026-09-30) +% noVideoFolder sessions whose videos/ folder is missing +% unusedVideoDay video folders no session points at +% ecoliSpread E. coli experiments whose images span more than 14 days +% (a stray timestamp would move the session's date) +% +% The spec's `corrections` naming `"table": "tableOfContents"` replace a +% cell of a folder's row (by `experiment`) BEFORE the study is picked, and +% only while the cell still reads `was`: a source fixed since then is +% reported (correctionNotApplied), not overwritten. Each one applied is +% reported (corrected). Decision #72: row 0014 of foragingConcentration +% says grid, its plates are single. +% +% The tableOfContents worm range is kept (worm_first / worm_last) but NOT +% trusted: the subjects stage cross-checks it against the worms present +% (decision 3; foragingConcentration row 0014 is a known typo). +% +% Options: +% 'StudyIds' containers.Map, study key -> study document id (from stage +% 2); when given, `study_ids` is filled in +% 'OutputRoot' where session directories go (default: /haley_V2) +% +% Errors ndi:setup:conv:haley:* when a tableOfContents lacks a required +% column, or a row matches no study or more than one. + +arguments + dataParentDir (1,:) char {mustBeFolder} + spec (1,1) struct + options.StudyIds = containers.Map() + options.OutputRoot (1,:) char = '' +end + +root = fullfile(dataParentDir, 'haley'); +outRoot = options.OutputRoot; +if isempty(outRoot) + outRoot = fullfile(dataParentDir, 'haley_V2'); +end +studies = asCell(spec.studies); +rows = {}; +checks = struct('sharedDay', {{}}, 'noVideoFolder', {{}}, 'unusedVideoDay', {{}}, ... + 'ecoliSpread', {{}}, 'corrected', {{}}, 'correctionNotApplied', {{}}); +tocFixes = tocCorrections(spec); + +% ---- C. elegans: the tableOfContents of each study folder ------------------ +folders = unique(cellfun(@(s) char(s.source_folder), studies, 'UniformOutput', false)); +folders = folders(~strcmp(folders, 'ecoli')); +need = {'experimentNumber', 'directoryName', 'notebookName', 'wormNumber', ... + 'conditions', 'include', 'notes'}; +for f = 1:numel(folders) + folder = folders{f}; + toc = fullfile(root, 'celegans', folder, 'tableOfContents.xlsx'); + if ~isfile(toc) + error('ndi:setup:conv:haley:noTableOfContents', 'Missing %s.', toc); + end + opts = detectImportOptions(toc); + opts = setvartype(opts, 'char'); + C = readtable(toc, opts); + missing = setdiff(need, C.Properties.VariableNames); + if ~isempty(missing) + error('ndi:setup:conv:haley:badTableOfContents', '%s has no column(s) %s.', ... + toc, strjoin(missing, ', ')); + end + [C, checks] = applyTocCorrections(C, folder, tocFixes, checks); + seenDays = {}; + for r = 1:height(C) + if isempty(strtrim(C.experimentNumber{r})) + continue; % a blank spreadsheet row + end + dayDir = strtrim(C.directoryName{r}); + if any(strcmp(seenDays, dayDir)) + checks.sharedDay{end+1} = sprintf('%s: experiment %s shares directoryName %s with an earlier row', ... + folder, strtrim(C.experimentNumber{r}), dayDir); + end + seenDays{end+1} = dayDir; %#ok + if ~isfolder(fullfile(root, 'celegans', folder, 'videos', dayDir)) + checks.noVideoFolder{end+1} = sprintf('%s: experiment %s -> videos/%s does not exist', ... + folder, strtrim(C.experimentNumber{r}), dayDir); + end + n = str2double(C.experimentNumber{r}); + cond = strtrim(C.conditions{r}); + word = lower(strtok(cond)); + study = pickStudy(studies, folder, word, toc, r); + day = datetime(strtrim(C.directoryName{r}), 'InputFormat', 'yy-MM-dd'); + [w1, w2] = wormRange(C.wormNumber{r}); + include = ~strcmpi(strtrim(C.include{r}), 'no'); + % the row's `include` is NOT written into the description: the lab's + % analysis excludes by plate, and the plates' exclusion tags say which + % (decision #73; a day marked `no` had 7 of 10 plates analysed) + desc = sprintf('Lab notebook: %s. Conditions: %s. Worms: %s.', ... + strtrim(C.notebookName{r}), cond, strtrim(C.wormNumber{r})); + if ~isempty(strtrim(C.notes{r})) + desc = sprintf('%s Notes: %s.', desc, strtrim(C.notes{r})); + end + pre = ndi.setup.conv.haley.idPrefix(folder); + id = sprintf('%s_%04d', pre, n); + rows{end+1} = row(id, study, n, day, desc, include, w1, w2, ... + fullfile(outRoot, pre, id), ['tableOfContents ' folder], folder, ... + fullfile('celegans', folder, 'videos', dayDir), options.StudyIds, ... + strtrim(C.notes{r})); %#ok + end + v = dir(fullfile(root, 'celegans', folder, 'videos')); + v = {v([v.isdir] & ~startsWith({v.name}, '.')).name}; + unused = setdiff(v, seenDays); + for k = 1:numel(unused) % by index: a 0x1 cell would still run a `for u = ...` loop once + checks.unusedVideoDay{end+1} = sprintf('%s: videos/%s is not named by any tableOfContents row', ... + folder, unused{k}); + end +end + +% ---- E. coli: one session per expNum -------------------------------------- +if any(cellfun(@(s) strcmp(char(s.source_folder), 'ecoli'), studies)) + bfile = fullfile(root, 'ecoli', 'bacteria.mat'); + B = load(bfile, 'info', 'metaData'); + study = pickStudy(studies, 'ecoli', '', bfile, 0); + exps = unique(B.info.expNum); + for e = 1:numel(exps) + n = exps(e); + plates = unique(B.info.plateNum(B.info.expNum == n)); + t = B.metaData.acquisitionTime(B.metaData.expNum == n); + if isempty(t) + day = NaT; + desc = sprintf('%d plate(s); no images recorded in metaData.', numel(plates)); + else + day = dateshift(min(t), 'start', 'day'); + if max(t) - min(t) > days(14) + checks.ecoliSpread{end+1} = sprintf(['ecoli experiment %d: images span %s to %s; ' ... + 'the session is dated by the earliest'], n, char(min(t), 'yyyy-MM-dd'), ... + char(max(t), 'yyyy-MM-dd')); + end + desc = sprintf('%d plate(s) imaged; %d image(s) from %s to %s.', numel(plates), ... + numel(t), char(min(t), 'yyyy-MM-dd HH:mm'), char(max(t), 'yyyy-MM-dd HH:mm')); + end + id = sprintf('ecoli_%04d', n); + rows{end+1} = row(id, study, n, day, desc, true, NaN, NaN, ... + fullfile(outRoot, 'ecoli', id), 'bacteria.mat', 'ecoli', '', options.StudyIds); %#ok + end +end + +T = struct2table([rows{:}], 'AsArray', true); +fprintf('DENOMINATOR: %d session(s): %d C. elegans from %d tableOfContents file(s), %d E. coli\n', ... + height(T), sum(~startsWith(T.local_identifier, 'ecoli_')), numel(folders), ... + sum(startsWith(T.local_identifier, 'ecoli_'))); +names = fieldnames(checks); +fprintf('CHECKS (reported, nothing changed): %d finding(s)\n', ... + sum(cellfun(@(n) numel(checks.(n)), names))); +for k = 1:numel(names) + for m = 1:numel(checks.(names{k})) + fprintf(' %-15s %s\n', names{k}, checks.(names{k}){m}); + end +end +end + +% ============================================================================= + +function s = row(id, study, n, day, desc, include, w1, w2, path, source, folder, videoFolder, studyIds, notes) +if nargin < 14 + notes = ''; +end +if isnat(day) + when = 'date unknown'; +else + when = char(day, 'd MMM yyyy'); +end +ids = {}; +if isKey(studyIds, char(study.key)) + ids = {studyIds(char(study.key))}; +end +s = struct('local_identifier', id, ... + 'name', sprintf('%s, experiment %d (%s)', char(study.name), n, when), ... + 'description', desc, 'path', path, 'study_key', char(study.key), ... + 'study_ids', {ids}, 'source', source, 'folder', folder, 'video_folder', videoFolder, ... + 'experiment', n, 'date', day, ... + 'include', include, 'notes', notes, 'worm_first', w1, 'worm_last', w2); +end + +function c = tocCorrections(spec) +% the spec's corrections to a tableOfContents cell +c = {}; +if ~isfield(spec, 'corrections'), return; end +list = spec.corrections; +if isstruct(list), list = num2cell(list); end +for k = 1:numel(list) + x = list{k}; + if isfield(x, 'table') && strcmp(char(x.table), 'tableOfContents') + c{end+1} = x; %#ok + end +end +end + +function [C, checks] = applyTocCorrections(C, folder, fixes, checks) +for k = 1:numel(fixes) + x = fixes{k}; + if ~strcmp(char(x.folder), folder) + continue; + end + col = char(x.column); + r = find(str2double(C.experimentNumber) == double(x.experiment)); + if isempty(r) || ~ismember(col, C.Properties.VariableNames) + checks.correctionNotApplied{end+1} = sprintf('%s: %s has no experiment %d or no column %s', ... + char(x.key), folder, double(x.experiment), col); + continue; + end + have = strtrim(C.(col){r(1)}); + if ~strcmp(have, char(x.was)) + checks.correctionNotApplied{end+1} = sprintf(['%s: %s experiment %d %s reads "%s", ' ... + 'not "%s"; left as it is'], char(x.key), folder, double(x.experiment), col, have, char(x.was)); + continue; + end + C.(col){r(1)} = char(x.value); + checks.corrected{end+1} = sprintf('%s: %s experiment %d %s "%s" -> "%s"', ... + char(x.key), folder, double(x.experiment), col, have, char(x.value)); +end +end + +function study = pickStudy(studies, folder, word, where, r) +hit = {}; +for k = 1:numel(studies) + s = studies{k}; + if ~strcmp(char(s.source_folder), folder) + continue; + end + if isfield(s, 'source_condition') && ~isempty(s.source_condition) ... + && ~strcmpi(char(s.source_condition), word) + continue; + end + hit{end+1} = s; %#ok +end +if numel(hit) ~= 1 + error('ndi:setup:conv:haley:noStudy', ... + '%s row %d (folder %s, condition "%s") matches %d studies in the spec, not 1.', ... + where, r, folder, word, numel(hit)); +end +study = hit{1}; +end + +function [a, b] = wormRange(txt) +a = NaN; b = NaN; +tok = regexp(strtrim(txt), '^(\d+)\s*-\s*(\d+)$', 'tokens', 'once'); +if ~isempty(tok) + a = str2double(tok{1}); + b = str2double(tok{2}); +end +end + +function c = asCell(v) +if iscell(v), c = reshape(v, 1, []); else, c = num2cell(reshape(v, 1, [])); end +end diff --git a/src/ndi/+ndi/+setup/+conv/+haley/showPatchMatch.m b/src/ndi/+ndi/+setup/+conv/+haley/showPatchMatch.m new file mode 100644 index 0000000000..785f30454e --- /dev/null +++ b/src/ndi/+ndi/+setup/+conv/+haley/showPatchMatch.m @@ -0,0 +1,97 @@ +function T = showPatchMatch(dataParentDir, imageNums) +%SHOWPATCHMATCH Draw E. coli images with each detected patch's matched patch number. +% +% T = ndi.setup.conv.haley.showPatchMatch(DATAPARENTDIR, IMAGENUMS) opens, +% for each image number, its patch mask (DATAPARENTDIR/haley/ecoli/mask/ +% .png) in a figure and writes on each detected patch, in red, the +% patch number the import gives it: by position on a full image (decision +% #62), else by the nearest patch position of the acetate template +% registered on the image's brightfield image (decision #80). The 12 +% template positions are drawn as yellow circles with their numbers, so +% an undetected patch's place shows. Nothing is written. +% +% It reads bacteria.mat (metaData), the extract (ecoli/analyzeGFP_24-04-04 +% .mat, with `template` from extractTemplates) and the spec's +% `ecoli_patch_layout`. +% +% T: one row per image -- imageNum, detected, method ('full image', +% 'template' or ''), missing (patch numbers with no detected patch), why +% (when not matched). +% +% Example: +% ndi.setup.conv.haley.showPatchMatch('/Users/me/data', [3626 3627]); + +arguments + dataParentDir (1,:) char {mustBeFolder} + imageNums (1,:) double +end +e = fullfile(dataParentDir, 'haley', 'ecoli'); +B = load(fullfile(e, 'bacteria.mat'), 'metaData'); +M = B.metaData; +X = load(fullfile(e, 'analyzeGFP_24-04-04.mat'), 'template'); +TPL = X.template; +spec = jsondecode(fileread(fullfile(fileparts(mfilename('fullpath')), 'import_V2_spec.json'))); +layout = spec.ecoli_patch_layout; +pos = reshape(layout.positions_mm, [], 2); + +rows = {}; +for n = imageNums + f = fullfile(e, 'mask', sprintf('%04d.png', n)); + if ~isfile(f) + rows(end+1, :) = {n, 0, '', '', 'no mask file'}; %#ok + continue; + end + m = imread(f) > 0; + [~, k, c] = labelRegions(m); + slot = matchFullImage(c, k, 12); + method = 'full image'; + why = ''; + r = find(M.imageNum == n, 1); + t = []; + if ~isempty(r) + t = find(TPL.imageNum == M.brightfieldImageNum(r), 1); + end + if isempty(slot) + method = 'template'; + if isempty(t) + why = 'no template registered on its brightfield image'; + else + [slot, why] = ndi.setup.conv.haley.matchTemplatePatches(c, TPL.center(t, :), ... + TPL.angle(t), TPL.scale(t), layout); + end + if isempty(slot), method = ''; end + end + missing = ''; + if ~isempty(slot), missing = mat2str(setdiff(1:12, slot)); end + rows(end+1, :) = {n, k, method, missing, why}; %#ok + + figure('Name', sprintf('image %d', n)); + imagesc(m); axis image; colormap(gray); hold on; + if ~isempty(t) + % the template positions in this image: the inverse of + % matchTemplatePatches' transform + th = layout.angle_sign * TPL.angle(t) * pi / 180; + u = pos(:, 1) * TPL.scale(t); + v = pos(:, 2) * TPL.scale(t); + px = TPL.center(t, 2) + cos(th) * u - sin(th) * v; + py = TPL.center(t, 1) + sin(th) * u + cos(th) * v; + plot(px, py, 'yo', 'MarkerSize', 22, 'LineWidth', 1.5); + for j = 1:12 + text(px(j) + 40, py(j) - 40, sprintf('%d', j), 'Color', 'y', 'FontSize', 11); + end + end + for j = 1:k + label = '?'; + if ~isempty(slot), label = sprintf('%d', slot(j)); end + text(c(j, 1), c(j, 2), label, 'Color', 'r', 'FontSize', 18, 'FontWeight', 'bold', ... + 'HorizontalAlignment', 'center'); + end + if isempty(slot) + title(sprintf('image %d: %d detected; not matched: %s', n, k, why)); + else + title(sprintf('image %d: %d detected; %s; missing %s', n, k, method, missing)); + end + hold off; +end +T = cell2table(rows, 'VariableNames', {'imageNum', 'detected', 'method', 'missing', 'why'}); +end diff --git a/src/ndi/+ndi/+setup/+conv/+haley/subjectList.m b/src/ndi/+ndi/+setup/+conv/+haley/subjectList.m new file mode 100644 index 0000000000..a16774fc99 --- /dev/null +++ b/src/ndi/+ndi/+setup/+conv/+haley/subjectList.m @@ -0,0 +1,834 @@ +function [S, checks] = subjectList(dataParentDir, sessions, options) +%SUBJECTLIST Stage 4 (Haley): the subjects to create, per session. +% +% [S, CHECKS] = ndi.setup.conv.haley.subjectList(DATAPARENTDIR, SESSIONS) +% reads the raw data under DATAPARENTDIR/haley and returns one row per +% subject, each assigned to a session of SESSIONS (the table from +% ndi.setup.conv.haley.sessionList). Decision log #37-#41. +% +% Names follow the eLife paper (decision #42): the plate an animal is +% filmed on is an ASSAY plate, the plate L4s are picked onto the day +% before is an ACCLIMATION plate. +% +% C. elegans, from /experimentInfo.mat `info` (one row per +% plate-video): +% assay_plate one per distinct plateNum concentration_assayPlate0011 +% "Assay Plate 0011" +% patch one per row of the plate's concentration_assayPlate0011_patch0007 +% lawnCenters (numbered in that "Patch 0007 on Assay Plate 0011" +% order, which is closestLawnID's) +% cohort the worms of one assay plate, concentration_assayPlate0011_worms +% moved together (decision #55): "Worms on Assay Plate 0011" +% a group subject; each worm is +% member_of it, and the plates +% and moves are stated on it +% worm one per distinct wormNum concentration_worm0451 "Worm 0451" +% acclimation_plate one per (strain, pick time) concentration_0001_acclimationPlate0001 +% in a session, numbered by pick "Acclimation Plate 0001" +% time then strain (the source +% has no id for it; decision #39) +% food_deprivation_plate one per (strain, mutants_0001_foodDeprivationPlate0001 +% starvedTime) in a session: the "Food Deprivation Plate 0001" +% unseeded plate food-deprived worms were moved to +% from their acclimation plate (decision #53) +% E. coli, from ecoli/bacteria.mat `info` (one row per plate; all 126 are +% kept, analysed or not -- decision #43): +% plate one per plate ecoli_plate0042 "Plate 0042" +% patch `rectangle` template: 12 (3 ecoli_plate0042_patch0012 +% staggered rows of 4, numbered +% left to right, top to bottom +% within a column: decision +% #62); `none` seeded: 1 (one +% large patch; OD600 0 = LB +% alone); `none` with +% lawnVolume 0: none +% +% Numbers restart in each source folder, so every local_identifier starts +% with ndi.setup.conv.haley.idPrefix(folder) (decision #37). +% +% Columns: session, kind, local_identifier, name, description, folder, +% plate, patch, worm, strain, acclimation (the acclimation plate an assay +% plate's worms came from), exclude (the source's `exclude` flag, for the +% assertions stage), deprivation (the food deprivation plate a +% food-deprived assay plate's worms came through; '' otherwise), +% worms_placed (when worms were put onto this plate: an acclimation +% plate's `growthTimePicked`, a food deprivation plate's `starvedTime`; +% NaT for the other kinds), bacteria (a patch's strain key: OP50 on the +% C. elegans plates, OP50-GFP on the E. coli plates; '' for a patch of LB +% alone, OD600 0), cohort (a worm's cohort), type (subject.type, decision +% #55: worm organism, cohort group, a patch with bacteria culture, every +% plate and a patch of LB alone material), prep (stage 8, decisions #54, +% #56: a struct of how a plate or patch was made -- seeded, cold_room, +% room_temp (an assay plate's `timeSeed`/`timeColdRoom`/`timeRoomTemp`, an +% acclimation plate's `growthTime*`, an E. coli plate's `timeSeed`/ +% `timeSeedColdRoom`/`timeRoomTemp`), poured and poured_cold_room (E. coli), +% peptone (the label, or the spec's correction), od600 and volume_ul (a +% patch's; an acclimation plate's `growthOD600`), room_temp_note (the +% correction's reason when `timeRoomTemp` was estimated), room_temp_by_rule +% (it is exactly a recording's start minus 1 h); NaT/NaN/'' where the +% source has nothing). +% +% Option 'Corrections': the spec's `corrections`; those naming a `plate` +% are applied to its prep (decision #56: plate 90's peptone; #54d: an +% estimated `timeRoomTemp`); those naming an `experiment` and column +% `growthTimeRoomTemp` replace that day's acclimation plate room temperature +% time (decision #70e: foragingMini experiment 24). +% +% CHECKS (second output, also printed) report what looks wrong in the +% source WITHOUT changing anything: +% plateWithoutSession a plate whose expNum has no session (no +% tableOfContents row); its subjects are skipped +% plateOnTwoDays one plateNum under two expNums (kept on the first) +% wormOnTwoPlates one wormNum on two plates (kept on the first) +% wormRange a session whose worms differ from its +% tableOfContents range (decision 3: the range is a +% check, never trusted) +% noLawnCenters a plate with no patch positions (no patches made) +% patchCountDisagrees lawnCenters and lawnRadii list different counts +% noPickTime a plate with no growthTimePicked (its +% acclimation plate is grouped by strain alone) +% ecoliSeeding an E. coli plate whose template and seeding +% disagree, or a plateNum used by two experiments +% noGrowthCondition an assay plate whose `growthCondition` is NaN that +% the source does not exclude. doImport ALSO +% excluded these (identifyValidRows); V2 excludes on +% `exclude` alone, so each is listed for a decision +% assayTypeDisagrees a plate whose original assay type (doImport: folder +% + its `condition`) is not its session's study's +% growthDisagrees an acclimation plate whose assay plates give +% different growth times (the first is kept) +% correctionUnmatched a spec correction naming a plate not listed +% roomTempLooksEstimated an assay plate whose `timeRoomTemp` is exactly a +% recording's start minus 1 h (filled in by rule) +% that no correction lists +% +% Nothing is written; the subjects stage of import_V2 prints S. + +arguments + dataParentDir (1,:) char {mustBeFolder} + sessions table + options.Corrections = {} +end + +root = fullfile(dataParentDir, 'haley'); +checks = struct('plateWithoutSession', {{}}, 'plateOnTwoDays', {{}}, ... + 'wormOnTwoPlates', {{}}, 'wormRange', {{}}, 'noLawnCenters', {{}}, ... + 'patchCountDisagrees', {{}}, 'noPickTime', {{}}, 'ecoliSeeding', {{}}, ... + 'growthDisagrees', {{}}, 'correctionUnmatched', {{}}, 'roomTempLooksEstimated', {{}}, ... + 'patchOD600', {{}}, 'assayTypeDisagrees', {{}}, 'noGrowthCondition', {{}}, 'notFilmed', {{}}, ... + 'growthSeedSpread', {{}}); +rows = {}; +dayHits = containers.Map(); % experiment-level correction key -> plates it applied to +assayChecked = 0; % plates whose original assay type was compared with their study +assayUnchecked = {}; % folders whose experimentInfo has no `condition` column + +% ---- C. elegans ------------------------------------------------------------- +folders = unique(sessions.folder(~strcmp(sessions.folder, 'ecoli'))); +nFiles = 0; +for f = 1:numel(folders) + folder = folders{f}; + pre = ndi.setup.conv.haley.idPrefix(folder); + file = fullfile(root, 'celegans', folder, 'experimentInfo.mat'); + if ~isfile(file) + error('ndi:setup:conv:haley:noExperimentInfo', 'Missing %s.', file); + end + I = load(file, 'info'); + I = I.info; + nFiles = nFiles + 1; + need = {'expNum', 'plateNum', 'wormNum', 'strainID', 'growthTimePicked', 'lawnCenters'}; + missing = setdiff(need, I.Properties.VariableNames); + if ~isempty(missing) + error('ndi:setup:conv:haley:badExperimentInfo', '%s has no column(s) %s.', ... + file, strjoin(missing, ', ')); + end + mine = sessions(strcmp(sessions.folder, folder), :); + + % One record per assay plate, gathered first: acclimation plates are + % numbered per session, which needs every plate of the session. + plates = unique(I.plateNum); + P = struct('plate', {}, 'session', {}, 'worms', {}, 'strain', {}, 'pick', {}, ... + 'nPatch', {}, 'nVideo', {}, 'filmed', {}, 'exclude', {}, 'starved', {}, 'od600', {}, ... + 'prep', {}, 'growth', {}); + wormHome = containers.Map('KeyType', 'double', 'ValueType', 'double'); + for k = 1:numel(plates) + p = double(plates(k)); + R = I(I.plateNum == plates(k), :); + days = unique(R.expNum); + if numel(days) > 1 + checks.plateOnTwoDays{end+1} = sprintf('%s: plate %d appears under expNum %s; kept on %d', ... + folder, p, mat2str(double(days(:)')), days(1)); + end + s = find(mine.experiment == double(days(1)), 1); + if isempty(s) + checks.plateWithoutSession{end+1} = sprintf('%s: plate %d (expNum %d) has no session', ... + folder, p, days(1)); + continue; + end + % the original import's assay type for this plate (doImport, from the + % folder and the plate's own `condition`) against the study its session + % is part_of: the study carries the assay type, so they must agree + if ismember('condition', I.Properties.VariableNames) && ismember('study_key', mine.Properties.VariableNames) + assayChecked = assayChecked + 1; + want = originalAssayType(folder, R.condition(1)); + got = studyAssayType(mine.study_key{s}); + if ~strcmp(want, got) + checks.assayTypeDisagrees{end+1} = sprintf(['%s: plate %d is %s in the original ' ... + 'import (condition "%s") but its session is in study %s (%s)'], ... + folder, p, want, conditionText(R.condition(1)), mine.study_key{s}, got); + end + elseif ~any(strcmp(assayUnchecked, folder)) + assayUnchecked{end+1} = folder; %#ok + end + w = cellfun(@(x) double(x(:)'), R.wormNum, 'UniformOutput', false); + worms = unique([w{:}]); + kept = []; + for m = 1:numel(worms) + if isKey(wormHome, worms(m)) && wormHome(worms(m)) ~= p + checks.wormOnTwoPlates{end+1} = sprintf('%s: worm %d is on plate %d and plate %d; kept on %d', ... + folder, worms(m), wormHome(worms(m)), p, wormHome(worms(m))); + continue; + end + wormHome(worms(m)) = p; + kept(end+1) = worms(m); %#ok + end + % Patch positions from the plate's first video that HAS them: a failed + % start (Mutants plate 1, 3 Nov 2023 07:35:52; decision #41) has none, + % and its restart does. + hasLawn = find(~cellfun(@isempty, R.lawnCenters), 1); + nPatch = 0; + if isempty(hasLawn) + checks.noLawnCenters{end+1} = sprintf('%s: plate %d has no lawnCenters; no patches made', folder, p); + else + nPatch = size(R.lawnCenters{hasLawn}, 1); + if ismember('lawnRadii', I.Properties.VariableNames) && numel(R.lawnRadii{hasLawn}) ~= nPatch + checks.patchCountDisagrees{end+1} = sprintf('%s: plate %d has %d lawnCenters and %d lawnRadii', ... + folder, p, nPatch, numel(R.lawnRadii{hasLawn})); + end + end + pick = R.growthTimePicked(1); + if isnat(pick) + checks.noPickTime{end+1} = sprintf('%s: plate %d has no growthTimePicked', folder, p); + end + ex = false; + if ismember('growthCondition', I.Properties.VariableNames) + gc = double(R.growthCondition); + ex0 = ismember('exclude', I.Properties.VariableNames) && any(R.exclude); + if any(isnan(gc)) && ~ex0 + checks.noGrowthCondition{end+1} = sprintf(['%s: plate %d (expNum %d) has NaN ' ... + 'growthCondition and is not excluded; doImport excluded it'], folder, p, days(1)); + end + end + if ismember('exclude', I.Properties.VariableNames) + ex = any(R.exclude); + end + starved = NaT; + if ismember('starvedTime', I.Properties.VariableNames) && ~isnat(R.starvedTime(1)) + starved = R.starvedTime(1); + end + od = []; % the patches' OD600 when the source has it + if ismember('OD600', I.Properties.VariableNames) + od = R.OD600(1, :); + if iscell(od), od = od{1}; end + od = double(od(:)'); + if numel(od) > 1 && numel(od) ~= nPatch && nPatch > 0 + % a grid template (multi-density): one OD600 per template + % position, more positions than patches (Matching: 19, the + % empty centre is OD600 0). Each patch's own OD600 is read + % from the lab's nearest-patch OD600 map at its centre. + [od, why] = patchDensities(R, hasLawn, od, nPatch); + if ~isempty(why) + checks.patchOD600{end+1} = sprintf('%s: plate %d: %s', folder, p, why); + end + end + end + % a plate never filmed: arena diameter 0 on every row (decisions + % #54e, #70c, #73) -- prepared, worms picked, nothing recorded + filmed = true; + if ismember('arenaDiameter', I.Properties.VariableNames) + filmed = ~all(double(R.arenaDiameter) == 0); + end + if ~filmed + checks.notFilmed{end+1} = sprintf('%s: plate %d (expNum %d) was not filmed', ... + folder, p, days(1)); + end + P(end+1) = struct('plate', p, 'session', mine.local_identifier{s}, 'worms', kept, ... + 'strain', char(R.strainID{1}), 'pick', pick, 'nPatch', nPatch, ... + 'nVideo', height(R) * filmed, 'filmed', filmed, 'exclude', ex, 'starved', starved, 'od600', od, ... + 'prep', prepOf(R, 'timeSeed', 'timeColdRoom', 'timeRoomTemp', NaN), ... + 'growth', growthPrepOf(R)); %#ok + [P(end).growth, dayHits] = growthCorrection(P(end).growth, folder, days(1), ... + options.Corrections, dayHits); + end + + % Acclimation plates: one per (session, strain, pick time), numbered within the + % session by pick time, then strain. + growthOf = cell(1, numel(P)); + for s = 1:height(mine) + idx = find(strcmp({P.session}, mine.local_identifier{s})); + if isempty(idx) + continue; + end + keys = cell(numel(idx), 1); + for m = 1:numel(idx) + keys{m} = growthKey(P(idx(m))); + end + [u, ~, grp] = unique(keys); + picks = NaT(numel(u), 1); + strains = cell(numel(u), 1); + for g = 1:numel(u) + first = idx(find(grp == g, 1)); + picks(g) = P(first).pick; + strains{g} = P(first).strain; + end + G = table(picks, strains, (1:numel(u))', 'VariableNames', {'pick', 'strain', 'k'}); + G = sortrows(G, {'pick', 'strain'}); + for n = 1:height(G) + g = G.k(n); + gid = sprintf('%s_acclimationPlate%04d', mine.local_identifier{s}, n); + members = idx(grp == g); + for m = 1:numel(members) + growthOf{members(m)} = gid; + end + if isnat(G.pick(n)) + when = 'pick time not recorded'; + else + when = ['picked ' char(G.pick(n), 'yyyy-MM-dd HH:mm')]; + end + r = subjRow(mine.local_identifier{s}, 'acclimation_plate', gid, ... + sprintf('Acclimation Plate %04d', n), ... + sprintf('Acclimation plate of %s, %s; the worms of %d assay plate(s) came from it.', ... + G.strain{n}, when, numel(members)), folder, NaN, NaN, NaN, G.strain{n}, '', false, ... + G.pick(n)); + % its preparation: the growth columns of the assay plates it served, + % which should agree; the first is kept and a disagreement reported. + % When they disagree ONLY on the seeding time, the plate was seeded + % once, somewhere in that span: its midpoint, widened by half the + % span (decision #75: concentration_0015, 12:39 / 12:41) + r.prep = P(members(1)).growth; + seeds = repmat(P(members(1)).growth.seeded, 1, numel(members)); + others = true(1, numel(members)); + base = r.prep; base.seeded = NaT; + for m = 1:numel(members) + g = P(members(m)).growth; + seeds(m) = g.seeded; + g.seeded = NaT; + others(m) = samePrep(g, base); + end + spread = all(others) && ~any(isnat(seeds)) && min(seeds) < max(seeds); + if spread + r.prep.seeded = min(seeds) + (max(seeds) - min(seeds)) / 2; + r.prep.seeded_spread_s = seconds(max(seeds) - min(seeds)) / 2; + checks.growthSeedSpread{end+1} = sprintf(['%s: its %d assay plates give seeding ' ... + 'times from %s to %s; recorded as %s +/- %g s plus the hand-written tolerance'], ... + gid, numel(members), char(min(seeds), 'yyyy-MM-dd HH:mm'), ... + char(max(seeds), 'yyyy-MM-dd HH:mm'), char(r.prep.seeded, 'yyyy-MM-dd HH:mm:ss'), ... + r.prep.seeded_spread_s); + end + for m = 2:numel(members) * ~spread % a span is all that disagrees + if ~samePrep(P(members(m)).growth, r.prep) + checks.growthDisagrees{end+1} = sprintf(['%s: assay plates %d and %d give ' ... + 'different growth seeding/cold room/room temperature times; kept %d''s'], ... + gid, P(members(1)).plate, P(members(m)).plate, P(members(1)).plate); + end + end + rows{end+1} = r; %#ok + end + end + + % Food deprivation plates: the unseeded plate a food-deprived plate's + % worms were moved to from their acclimation plate (`starvedTime`, the + % paper's "3 hr of food deprivation"); one per (session, strain, time + % moved), numbered within the session by that time, then strain. + deprivationOf = repmat({''}, 1, numel(P)); + deprived = arrayfun(@(q) ~isnat(q.starved), P); % (an empty P gives logical []) + for s = 1:height(mine) + idx = find(strcmp({P.session}, mine.local_identifier{s}) & deprived); + if isempty(idx) + continue; + end + keys = arrayfun(@(q) [q.strain '|' char(q.starved, 'yyyyMMddHHmmss')], P(idx), ... + 'UniformOutput', false); + [u, ~, grp] = unique(keys); + times = NaT(numel(u), 1); + strains = cell(numel(u), 1); + for g = 1:numel(u) + first = idx(find(grp == g, 1)); + times(g) = P(first).starved; + strains{g} = P(first).strain; + end + G = table(times, strains, (1:numel(u))', 'VariableNames', {'starved', 'strain', 'k'}); + G = sortrows(G, {'starved', 'strain'}); + for n = 1:height(G) + g = G.k(n); + did = sprintf('%s_foodDeprivationPlate%04d', mine.local_identifier{s}, n); + members = idx(grp == g); + for m = 1:numel(members) + deprivationOf{members(m)} = did; + end + rows{end+1} = subjRow(mine.local_identifier{s}, 'food_deprivation_plate', did, ... + sprintf('Food Deprivation Plate %04d', n), ... + sprintf(['Unseeded plate the %s worms of %d assay plate(s) were moved to from ' ... + 'their acclimation plate at %s, for food deprivation before the assay.'], ... + G.strain{n}, numel(members), char(G.starved(n), 'yyyy-MM-dd HH:mm')), ... + folder, NaN, NaN, NaN, G.strain{n}, '', false, G.starved(n)); %#ok + end + end + + % Assay plates, their patches and worms. + for m = 1:numel(P) + q = P(m); + pid = sprintf('%s_assayPlate%04d', pre, q.plate); + pname = sprintf('Assay Plate %04d', q.plate); + if q.filmed + filmedText = sprintf('filmed in %d video(s)', q.nVideo); + else + filmedText = 'not filmed'; + end + r = subjRow(q.session, 'assay_plate', pid, pname, ... + sprintf('Assay plate %d: %d worm(s), %d patch(es), %s.', ... + q.plate, numel(q.worms), q.nPatch, filmedText), folder, q.plate, NaN, NaN, ... + q.strain, growthOf{m}, q.exclude, NaT, deprivationOf{m}); + r.filmed = q.filmed; + r.prep = q.prep; + % a timeRoomTemp of exactly a recording's start minus 1 h was filled + % in by rule (decision #54d): the spec's corrections list each one + if ismember('timeRecord', I.Properties.VariableNames) && ~isnat(q.prep.room_temp) + t = I.timeRecord(I.plateNum == q.plate); + r.prep.room_temp_by_rule = any(q.prep.room_temp == t - hours(1)); + end + rows{end+1} = r; %#ok + for k = 1:q.nPatch + r = subjRow(q.session, 'patch', sprintf('%s_patch%04d', pid, k), ... + sprintf('Patch %04d on %s', k, pname), ... + sprintf('Patch %d of assay plate %d.', k, q.plate), folder, q.plate, k, NaN, ... + '', '', false); + % OP50 on every C. elegans plate (decision #54); a patch the source + % gives OD600 0 is LB alone, with no bacteria. + od = q.od600; + if numel(od) == q.nPatch, od = od(k); end + if ~(~isempty(od) && all(od == 0)) + r.bacteria = 'OP50'; + end + % seeded with the plate, at its own OD600 and the plate's volume + r.prep.seeded = q.prep.seeded; + r.prep.volume_ul = q.prep.volume_ul; + if isscalar(od), r.prep.od600 = od; end + rows{end+1} = r; %#ok + end + % The plate's worms are one cohort (decision #55): moved together, so + % the plates they were on and how they got there are stated once, on + % the cohort, and each worm is a member of it. + cohort = ''; + if ~isempty(q.worms) + cohort = [pid '_worms']; + r = subjRow(q.session, 'cohort', cohort, ['Worms on ' pname], ... + sprintf('The %d %s worm(s) of assay plate %d, moved together.', ... + numel(q.worms), q.strain, q.plate), folder, q.plate, NaN, NaN, ... + q.strain, growthOf{m}, false, NaT, deprivationOf{m}); + rows{end+1} = r; %#ok + end + for k = 1:numel(q.worms) + r = subjRow(q.session, 'worm', sprintf('%s_worm%04d', pre, q.worms(k)), ... + sprintf('Worm %04d', q.worms(k)), ... + sprintf('Worm %d, on assay plate %d.', q.worms(k), q.plate), folder, q.plate, ... + NaN, q.worms(k), q.strain, growthOf{m}, false, NaT, deprivationOf{m}); + r.cohort = cohort; + rows{end+1} = r; %#ok + end + end + + % The tableOfContents worm range, checked against the worms present. + for s = 1:height(mine) + a = mine.worm_first(s); b = mine.worm_last(s); + if isnan(a) || isnan(b) + continue; + end + idx = strcmp({P.session}, mine.local_identifier{s}); + present = sort([P(idx).worms]); + if ~isequal(present, a:b) + if isempty(present) + got = 'no worms'; + else + got = sprintf('%d worm(s), %d-%d', numel(present), present(1), present(end)); + end + checks.wormRange{end+1} = sprintf('%s: tableOfContents says worms %d-%d (%d); the plates have %s', ... + mine.local_identifier{s}, a, b, b - a + 1, got); + end + end +end + +% ---- E. coli ---------------------------------------------------------------- +mine = sessions(strcmp(sessions.folder, 'ecoli'), :); +if height(mine) > 0 + file = fullfile(root, 'ecoli', 'bacteria.mat'); + B = load(file, 'info'); + info = B.info; + nFiles = nFiles + 1; + need = {'expNum', 'plateNum', 'template', 'OD600', 'lawnVolume'}; + missing = setdiff(need, info.Properties.VariableNames); + if ~isempty(missing) + error('ndi:setup:conv:haley:badBacteriaInfo', '%s info has no column(s) %s.', ... + file, strjoin(missing, ', ')); + end + [u, ~, j] = unique(info.plateNum); + for k = find(accumarray(j, 1) > 1)' + checks.ecoliSeeding{end+1} = sprintf('ecoli: plateNum %d is used by more than one plate', u(k)); + end + for r = 1:height(info) + n = double(info.expNum(r)); + p = double(info.plateNum(r)); + s = find(mine.experiment == n, 1); + if isempty(s) + checks.plateWithoutSession{end+1} = sprintf('ecoli: plate %d (expNum %d) has no session', p, n); + continue; + end + template = lower(strtrim(char(info.template{r}))); + seeded = info.lawnVolume(r) > 0; + switch template + case 'rectangle' + nPatch = 12; + if ~seeded + checks.ecoliSeeding{end+1} = sprintf('ecoli: plate %d has the rectangle template but lawnVolume 0', p); + end + what = '12 patches in a 3 x 4 grid'; + case 'none' + % Seeded (lawnVolume > 0) = one large patch. OD600 0 is a patch + % of LB alone, a bacteria-free patch at relative density 0 (the + % eLife paper, Methods: "A '0' density solution was prepared + % with just LB"), so it is still a patch. Unseeded = a blank + % plate. Only an OD600 with no volume is inconsistent. + nPatch = double(seeded); + if ~seeded && info.OD600(r) > 0 + checks.ecoliSeeding{end+1} = sprintf('ecoli: plate %d has OD600 %g but lawnVolume 0', ... + p, info.OD600(r)); + end + if ~seeded + what = 'no bacteria'; + elseif info.OD600(r) == 0 + what = 'one large patch of LB alone (relative density 0)'; + else + what = 'one large lawn'; + end + otherwise + nPatch = 0; + checks.ecoliSeeding{end+1} = sprintf('ecoli: plate %d has unknown template "%s"; no patches made', ... + p, template); + what = ['template ' template]; + end + pid = sprintf('ecoli_plate%04d', p); + pname = sprintf('Plate %04d', p); + pl = subjRow(mine.local_identifier{s}, 'plate', pid, pname, ... + sprintf('E. coli plate %d: %s.', p, what), 'ecoli', p, NaN, NaN, '', '', false); + R1 = info(r, :); + pl.prep = prepOf(R1, 'timeSeed', 'timeSeedColdRoom', 'timeRoomTemp', NaN); + pl.prep.poured = colOr(R1, 'timePoured', NaT); + pl.prep.poured_cold_room = colOr(R1, 'timePouredColdRoom', NaT); + rows{end+1} = pl; %#ok + for k = 1:nPatch + pr = subjRow(mine.local_identifier{s}, 'patch', sprintf('%s_patch%04d', pid, k), ... + sprintf('Patch %04d on %s', k, pname), ... + sprintf('Patch %d of E. coli plate %d.', k, p), 'ecoli', p, k, NaN, '', '', false); + if info.OD600(r) > 0 % OD600 0 is LB alone: no bacteria + pr.bacteria = 'OP50-GFP'; % decision #54 + end + pr.prep.seeded = pl.prep.seeded; + pr.prep.od600 = double(info.OD600(r)); + pr.prep.volume_ul = double(info.lawnVolume(r)); + rows{end+1} = pr; %#ok + end + end +end + +% ---- the spec's named corrections to a plate (decisions #54, #56) ------------ +% By folder and plate: `peptone` replaces the plate's label (plate 90); a +% `timeRoomTemp` with `approximate` marks that time as estimated, keeping it +% and its reason. Corrections by seeding day are seedingSuspensions'. +corr = options.Corrections; +if isstruct(corr), corr = num2cell(corr); end +for c = 1:numel(corr) + x = corr{c}; + if ~isfield(x, 'plate') + continue; + end + hit = false; + for k = 1:numel(rows) + r = rows{k}; + if strcmp(r.folder, x.folder) && isequal(r.plate, double(x.plate)) && ... + any(strcmp(r.kind, {'assay_plate', 'plate'})) + switch x.column + case 'peptone' + r.prep.peptone = char(x.value); + case 'timeRoomTemp' + r.prep.room_temp_note = char(x.reason); + otherwise + continue; + end + rows{k} = r; + hit = true; + end + end + if ~hit + checks.correctionUnmatched{end+1} = sprintf('%s: no %s plate %d in the selected sessions', ... + char(x.key), x.folder, double(x.plate)); + end +end + +for c = 1:numel(corr) + x = corr{c}; + % only the per-experiment corrections this stage applies (a + % tableOfContents correction is sessionList's) + if isfield(x, 'experiment') && strcmp(x.column, 'growthTimeRoomTemp') ... + && ~isKey(dayHits, char(x.key)) && any(strcmp(sessions.folder, x.folder)) + checks.correctionUnmatched{end+1} = sprintf('%s: no %s experiment %d in the selected sessions', ... + char(x.key), x.folder, double(x.experiment)); + end +end + +for k = 1:numel(rows) + r = rows{k}; + if r.prep.room_temp_by_rule && isempty(r.prep.room_temp_note) + checks.roomTempLooksEstimated{end+1} = sprintf(['%s: timeRoomTemp is exactly a ' ... + 'recording''s start minus 1 h but no correction lists it'], r.local_identifier); + end +end + +% ---- report ------------------------------------------------------------------- +if isempty(rows) + S = cell2table(cell(0, 19), 'VariableNames', {'session', 'kind', 'local_identifier', ... + 'name', 'description', 'folder', 'plate', 'patch', 'worm', 'strain', 'acclimation', ... + 'deprivation', 'exclude', 'filmed', 'worms_placed', 'bacteria', 'cohort', 'prep', 'type'}); +else + S = struct2table([rows{:}], 'AsArray', true); + S.type = arrayfun(@(k) typeOf(S.kind{k}, S.bacteria{k}), (1:height(S))', ... + 'UniformOutput', false); +end +dup = {}; +if height(S) > 0 + [u, ~, j] = unique(S.local_identifier); + dup = u(accumarray(j, 1) > 1); +end +if ~isempty(dup) + error('ndi:setup:conv:haley:duplicateSubject', ... + 'Two subjects would share local_identifier %s.', strjoin(dup, ', ')); +end +kinds = {'assay_plate', 'acclimation_plate', 'food_deprivation_plate', 'plate', 'patch', ... + 'cohort', 'worm'}; +counts = cellfun(@(k) sum(strcmp(S.kind, k)), kinds); +fprintf(['DENOMINATOR: %d subject(s) in %d session(s), from %d source file(s): ' ... + '%d assay plate(s), %d acclimation plate(s), %d food deprivation plate(s), %d E. coli plate(s), ' ... + '%d patch(es), %d worm cohort(s), %d worm(s)\n'], ... + height(S), numel(unique(S.session)), nFiles, counts); +notCompared = 'none'; +if ~isempty(assayUnchecked), notCompared = strjoin(assayUnchecked, ', '); end +fprintf(['ASSAY TYPE: %d assay plate(s) compared with their study, %d disagree; ' ... + 'folders not compared (no condition column): %s\n'], assayChecked, ... + numel(checks.assayTypeDisagrees), notCompared); +names = fieldnames(checks); +fprintf('CHECKS (reported, nothing changed): %d finding(s)\n', ... + sum(cellfun(@(n) numel(checks.(n)), names))); +for k = 1:numel(names) + for m = 1:numel(checks.(names{k})) + fprintf(' %-20s %s\n', names{k}, checks.(names{k}){m}); + end +end +end + +% ============================================================================= + +function s = subjRow(session, kind, id, name, desc, folder, plate, patch, worm, strain, acclimation, exclude, placed, deprivation) +if nargin < 13 + placed = NaT; +end +if nargin < 14 + deprivation = ''; +end +s = struct('session', session, 'kind', kind, 'local_identifier', id, 'name', name, ... + 'description', desc, 'folder', folder, 'plate', plate, 'patch', patch, 'worm', worm, ... + 'strain', strain, 'acclimation', acclimation, 'deprivation', deprivation, ... + 'exclude', logical(exclude), 'filmed', true, 'worms_placed', placed, 'bacteria', '', 'cohort', '', ... + 'prep', blankPrep()); +end + +function p = blankPrep() +% What stage 8 needs to know about how a plate or patch was made (decisions +% #54, #56): NaT / NaN / '' where the source has nothing. +p = struct('seeded', NaT, 'cold_room', NaT, 'room_temp', NaT, 'room_temp_note', '', ... + 'poured', NaT, 'poured_cold_room', NaT, 'peptone', '', 'od600', NaN, 'volume_ul', NaN, ... + 'room_temp_by_rule', false, 'seeded_spread_s', 0); +end + +function p = prepOf(R, seeded, coldRoom, roomTemp, od600) +% A plate's preparation from the first of its rows (the columns repeat per row). +p = blankPrep(); +p.seeded = colOr(R, seeded, NaT); +p.cold_room = colOr(R, coldRoom, NaT); +p.room_temp = colOr(R, roomTemp, NaT); +p.od600 = double(od600); +p.volume_ul = double(colOr(R, 'lawnVolume', NaN)); +v = colOr(R, 'peptone', ''); +if iscell(v), v = v{1}; end +if isstring(v) && any(ismissing(v)), v = ''; end +p.peptone = lower(strtrim(char(v))); +end + +function p = growthPrepOf(R) +% The acclimation plate's preparation, from its assay plates' growth +% columns. Its volume (200 uL) and agar are the spec's, not the assay plate's. +p = prepOf(R, 'growthTimeSeed', 'growthTimeColdRoom', 'growthTimeRoomTemp', ... + colOr(R, 'growthOD600', NaN)); +p.volume_ul = NaN; +p.peptone = ''; +end + +function v = colOr(R, name, default) +% R.(name)(1) when the table has that column, else DEFAULT. +if ismember(name, R.Properties.VariableNames) && height(R) > 0 + v = R.(name)(1, :); + if iscell(v), v = v{1}; end + if isnumeric(v) || islogical(v), v = double(v(1)); end + if isdatetime(v), v = v(1); end +else + v = default; +end +end + +function tf = samePrep(a, b) +same = @(x, y) (isnat(x) && isnat(y)) || isequal(x, y); +tf = same(a.seeded, b.seeded) && same(a.cold_room, b.cold_room) && same(a.room_temp, b.room_temp); +end + +function t = typeOf(kind, bacteria) +% subject.type (decision #55): plates are material; a patch with bacteria is +% a culture (a lawn: a population whose members are never subjects), one of +% LB alone is material; a worm is an organism; a cohort is a group. +switch kind + case 'worm' + t = 'organism'; + case 'cohort' + t = 'group'; + case 'patch' + if isempty(bacteria), t = 'material'; else, t = 'culture'; end + otherwise + t = 'material'; +end +end + +function k = growthKey(p) +if isnat(p.pick) + t = 'NaT'; +else + t = char(p.pick, 'yyyyMMddHHmmss'); +end +k = [p.strain '|' t]; +end + +function [od, why] = patchDensities(R, row, template, nPatch) +% Each patch's OD600 on a grid-template plate: the lab's lawnClosestOD600 +% map (each pixel's nearest patch's OD600) read at the patch's centre +% (lawnCenters, [x y] pixels). The template's non-zero positions are the +% cross-check, compared as the set of densities the plate holds and NOT in +% order: lawnCenters lists patches in detection order, which swaps +% neighbours lying nearly level, so an order comparison flagged 14 Matching +% plates holding exactly the same densities (decision #76). Falls back to the +% template when the map is missing or unreadable; NaN (and WHY) when neither +% gives one value per patch. +why = ''; +fromTemplate = template(template ~= 0); +if numel(fromTemplate) ~= nPatch, fromTemplate = []; end +fromMap = []; +if ismember('lawnClosestOD600', R.Properties.VariableNames) + M = R.lawnClosestOD600{row}; + c = R.lawnCenters{row}; + if ~isempty(M) && size(c, 1) == nPatch + for flip = [false true] % [x y], else [y x] + xy = round(c); + if flip, xy = xy(:, [2 1]); end + ok = all(xy >= 1, 2) & xy(:, 1) <= size(M, 2) & xy(:, 2) <= size(M, 1); + if ~all(ok), continue; end + v = M(sub2ind(size(M), xy(:, 2), xy(:, 1)))'; + if all(ismember(v, template(template ~= 0))) + fromMap = double(v); + break; + end + end + end +end +if ~isempty(fromMap) + od = fromMap; + if ~isempty(fromTemplate) && ~isequal(sort(fromMap), sort(fromTemplate)) + why = sprintf(['the OD600 map gives densities %s, the template %s; the map''s ' ... + 'values are used'], mat2str(sort(fromMap)), mat2str(sort(fromTemplate))); + end +elseif ~isempty(fromTemplate) + od = fromTemplate; + why = 'no readable OD600 map; the template''s non-zero positions, in order, are used'; +else + od = nan(1, nPatch); + why = sprintf('%d patch(es), template %s, no readable OD600 map: OD600 not known', ... + nPatch, mat2str(template)); +end +end + +function t = originalAssayType(folder, condition) +% doImport.m's assay type (lines 140-152): grid plates are multi-density in +% foragingMatching and single-density elsewhere; single plates are small in +% foragingMini and large elsewhere +c = lower(conditionText(condition)); +switch c + case 'grid' + if strcmp(folder, 'foragingMatching') + t = 'MultiDensityMultiPatch'; + else + t = 'SingleDensityMultiPatch'; + end + case 'single' + if strcmp(folder, 'foragingMini') + t = 'SmallSinglePatch'; + else + t = 'LargeSinglePatch'; + end + otherwise + t = ['unknown (' c ')']; +end +end + +function t = studyAssayType(key) +% the assay each spec study runs (its name says so) +switch char(key) + case {'single_density_multi_patch', 'mutant_screen', 'sensory_mutants'} + t = 'SingleDensityMultiPatch'; + case 'large_single_patch' + t = 'LargeSinglePatch'; + case 'small_single_patch' + t = 'SmallSinglePatch'; + case 'multi_density_multi_patch' + t = 'MultiDensityMultiPatch'; + otherwise + t = ['not an assay study (' char(key) ')']; +end +end + +function s = conditionText(c) +if iscell(c), c = c{1}; end +if iscell(c), c = c{1}; end +s = strtrim(char(string(c))); +end + +function [g, hits] = growthCorrection(g, folder, experiment, corr, hits) +% A named correction of a day's acclimation plate time (decision #70e): the +% spec's `corrections` entry naming this folder and experiment, column +% growthTimeRoomTemp, value a wall-clock time 'yyyy-MM-ddTHH:mm' +if isstruct(corr), corr = num2cell(corr); end +for c = 1:numel(corr) + x = corr{c}; + if ~isfield(x, 'experiment') || ~strcmp(x.folder, folder) ... + || double(x.experiment) ~= double(experiment) || ~strcmp(x.column, 'growthTimeRoomTemp') + continue; + end + g.room_temp = datetime(x.value, 'InputFormat', 'yyyy-MM-dd''T''HH:mm'); + g.room_temp_by_rule = false; + g.room_temp_note = char(x.reason); + k = char(x.key); + if isKey(hits, k), hits(k) = hits(k) + 1; else, hits(k) = 1; end +end +end diff --git a/src/ndi/+ndi/+setup/+conv/+haley/termInventory.m b/src/ndi/+ndi/+setup/+conv/+haley/termInventory.m new file mode 100644 index 0000000000..fd8c0623e3 --- /dev/null +++ b/src/ndi/+ndi/+setup/+conv/+haley/termInventory.m @@ -0,0 +1,145 @@ +function T = termInventory(path, options) +%TERMINVENTORY Every term a written V2 dataset uses, plain text or ontology. +% +% T = ndi.setup.conv.haley.termInventory(PATH) reads every document in the +% V2 dataset at PATH (the folder holding .ndi; its own database and each +% linked session's, ndi.setup.V2.datasetDatabaseFiles) and lists each term it +% carries: every {node, name} pair (or {name} alone: did2.build stores a +% term with no node without one), wherever it sits (a statement's +% variable or method, a unit, a key's or condition's variable, an axis, a +% scale, a subject type, an asserted value, ...). A term with an empty +% node is PLAIN TEXT; one with a node is an ONTOLOGY term. +% +% One row per distinct (name, node, field): +% kind "plain text" or "ontology" +% name, node the term +% field where it sits, block.field with [] for array elements +% classes the document classes it appears in +% documents how many documents carry it there +% occurrences how many times (a document can carry it more than once) +% example_id one document that carries it +% +% Options: +% 'OutFile' also write T to this .csv, with empty review columns +% (proposed_node, proposed_name, notes) to fill in +% 'BatchSize' documents read per query (default 5000) +% +% Prints its denominators first: documents read, term occurrences found, +% distinct terms (plain text / ontology). Writes nothing else. +% +% See also ndi.setup.conv.haley.verifyDataset, ndi.setup.conv.haley.import_V2. + +arguments + path (1,:) char {mustBeFolder} + options.OutFile (1,:) char = '' + options.BatchSize (1,1) double {mustBePositive, mustBeInteger} = 5000 +end + +% the dataset's own database, then each linked session's (decision #71) +files = ndi.setup.V2.datasetDatabaseFiles(path); +nDocs = 0; + +index = containers.Map(); % key -> row number +rows = struct('name', {}, 'node', {}, 'field', {}, 'classes', {}, 'docs', {}, ... + 'occurrences', {}, 'example_id', {}); +nOcc = 0; +read = 0; +t0 = tic; +for f = 1:numel(files) + db = did2.database.sqlitedb(files{f}); + dbid = db.testHookDbId(); + nHere = mksqlite(dbid, 'SELECT COUNT(*) AS n FROM documents'); + nHere = double(nHere.n); + nDocs = nDocs + nHere; + for offset = 0:options.BatchSize:nHere - 1 + B = mksqlite(dbid, sprintf(['SELECT id, classname, body FROM documents ' ... + 'ORDER BY id LIMIT %d OFFSET %d'], options.BatchSize, offset)); + for k = 1:numel(B) + found = walk(jsondecode(B(k).body), '', {}); + read = read + 1; + seen = containers.Map(); + for j = 1:size(found, 1) + key = [found{j, 1} char(31) found{j, 2} char(31) found{j, 3}]; + nOcc = nOcc + 1; + if ~isKey(index, key) + rows(end+1) = struct('name', found{j, 1}, 'node', found{j, 2}, ... + 'field', found{j, 3}, 'classes', {{B(k).classname}}, 'docs', 0, ... + 'occurrences', 0, 'example_id', B(k).id); %#ok + index(key) = numel(rows); + end + r = index(key); + rows(r).occurrences = rows(r).occurrences + 1; + if ~isKey(seen, key) + seen(key) = true; + rows(r).docs = rows(r).docs + 1; + if ~any(strcmp(rows(r).classes, B(k).classname)) + rows(r).classes{end+1} = B(k).classname; + end + end + end + end + fprintf(' terms: %d documents read (database %d of %d), %s\n', read, f, numel(files), ... + char(duration(0, 0, round(toc(t0)), 'Format', 'hh:mm:ss'))); + end + db.close(); +end + +kind = repmat("plain text", numel(rows), 1); +kind(~cellfun(@isempty, {rows.node})) = "ontology"; +T = table(kind, string({rows.name})', string({rows.node})', string({rows.field})', ... + string(cellfun(@(c) strjoin(sort(c), ', '), {rows.classes}, 'UniformOutput', false))', ... + [rows.docs]', [rows.occurrences]', string({rows.example_id})', ... + 'VariableNames', {'kind', 'name', 'node', 'field', 'classes', 'documents', ... + 'occurrences', 'example_id'}); +T = sortrows(T, {'kind', 'field', 'name'}); + +nPlain = numel(unique(T.name(T.kind == "plain text"))); +nOnt = height(unique(T(T.kind == "ontology", {'name', 'node'}))); +fprintf(['DENOMINATOR: %d document(s) read of %d; %d term occurrence(s); %d distinct ' ... + '(name, node, field) row(s): %d distinct plain-text name(s), %d distinct ontology term(s)\n'], ... + read, nDocs, nOcc, height(T), nPlain, nOnt); + +if ~isempty(options.OutFile) + out = T; + out.proposed_node = strings(height(T), 1); + out.proposed_name = strings(height(T), 1); + out.notes = strings(height(T), 1); + writetable(out, options.OutFile); + fprintf('written: %s\n', options.OutFile); +end +end + +% ----------------------------------------------------------------------------- +function found = walk(x, path, found) +% every {node, name} struct under X, as rows {name, node, path} +if isstruct(x) + f = fieldnames(x); + % a term: node and/or name and nothing else. did2.build leaves an empty + % node out, so a plain-text term is stored as {name} alone. + if ~isempty(f) && all(ismember(f, {'node', 'name'})) + for i = 1:numel(x) + nm = ''; nd = ''; + if isfield(x, 'name'), nm = char(string(x(i).name)); end + if isfield(x, 'node'), nd = char(string(x(i).node)); end + if isempty(nm) && isempty(nd), continue; end % a blank (unset) term + found(end+1, :) = {nm, nd, path}; %#ok + end + return; + end + sub = path; + if numel(x) > 1, sub = [path '[]']; end + for i = 1:numel(x) + for j = 1:numel(f) + found = walk(x(i).(f{j}), join(sub, f{j}), found); + end + end +elseif iscell(x) + for i = 1:numel(x) + found = walk(x{i}, [path '[]'], found); + end +end +end + +function p = join(a, b) +if isempty(a), p = b; else, p = [a '.' b]; end +end diff --git a/src/ndi/+ndi/+setup/+conv/+haley/trackDocuments.m b/src/ndi/+ndi/+setup/+conv/+haley/trackDocuments.m new file mode 100644 index 0000000000..3dceec5dee --- /dev/null +++ b/src/ndi/+ndi/+setup/+conv/+haley/trackDocuments.m @@ -0,0 +1,241 @@ +function out = trackDocuments(dataParentDir, session, R, subjectIds, options) +%TRACKDOCUMENTS Stage 10 part B (Haley): each worm's tracks, per video. +% +% OUT = ndi.setup.conv.haley.trackDocuments(DATAPARENTDIR, SESSION, R, +% SUBJECTIDS, ...) builds, for the one-row session table SESSION, the +% per-frame track calculations of each worm, body part and behaviour video, +% from the folder's .mat tables (midpoint; head and tail in +% foragingMini), which the lab's analysis package made from the WormLab +% exports (analyzeWormLabTracks, Haley et al. 2024). Decision log #61. +% +% Per worm, body part and video, all about the worm, each keyed by the +% 0-based video frame (the source's 1-based `frameNum`) and in an ingested +% sampled_body: +% position_calculation ` position`: x and y in the video's +% coordinate system (part A), in pixels from the +% image's upper-left corner (the source's 1-based +% pixel coordinates less half a pixel). The track +% as the analysis package left it: gaps of at most +% 10 s filled by linear interpolation (method +% parameters), longer gaps and positions off the +% image NaN; `noTrack` is not stored (Jess, +% 2026-10-03: filling a gap is a curation step like +% WormLab's own). Input: the recording. +% velocity_calculation ` speed` (`velocitySmooth`), m/s: the +% 4 s moving mean of the step lengths, steps faster +% than 600 um/s left out (method parameters). +% Input: the positions. +% length_calculation ` distance to the nearest patch edge` +% (`distanceLawnEdge`), m, positive on a patch, +% negative off it. Inputs: the positions and the +% lawn mask. +% item_calculation `nearest patch` (`closestLawnID`): a 0-based index +% into the plate's patch subjects (item_id). The +% source reads it at pixel (1, 1) on the frames it +% marks `noTrack`, so those frames hold the body's +% fill value. Only when the schema has `item` +% (did-schema PR #87). Inputs: the positions and the +% nearest-patch map (or the lawn mask). +% The rest of each table is not stored (decision log #60/#61's keep list): +% identifiers, times (the recording has them), and values computed from +% the positions by a stated rule (step distance, path angle, turns, +% neighbour distances, arena-edge distance, nearArena, outOfBounds). +% +% Options: +% 'Geometry' part A's OUT.byEpoch (epoch -> coordinate system, +% lawn mask, nearest-patch map ids, metres per pixel) +% 'RecordingStatements' containers.Map, epoch -> recording statement id +% 'RecordingRefs' containers.Map, epoch -> its UTC reference id +% 'SoftwareId', 'InterpreterId', 'OperatingSystemId' as part A +% +% OUT fields: documents (cell of structs), counts (struct), skipped (cellstr), +% byWorm (containers.Map, '|' -> struct of the +% ids of that worm's midpoint position, speed and patch-edge distance in that +% video: stage 11's inputs). + +arguments + dataParentDir (1,:) char {mustBeFolder} + session table + R table + subjectIds + options.Geometry = containers.Map() + options.RecordingStatements = containers.Map() + options.RecordingRefs = containers.Map() + options.SoftwareId (1,:) char = '' + options.InterpreterId (1,:) char = '' + options.OperatingSystemId (1,:) char = '' +end +if height(session) ~= 1 + error('ndi:setup:conv:haley:oneSession', 'Give exactly one session row.'); +end +sid = char(session.session_id{1}); +ref = char(session.local_identifier{1}); +folder = char(session.folder{1}); +out = struct('documents', {{}}, 'skipped', {{}}, 'counts', struct('position', 0, ... + 'speed', 0, 'patch_edge_distance', 0, 'nearest_patch', 0), 'byWorm', containers.Map()); +if strcmp(folder, 'ecoli') + return; +end +if isempty(options.OperatingSystemId) + out.skipped{end+1} = sprintf(['%s: no operating system document (the spec''s `macos`, ' ... + 'built by the metadata stage); no track calculations'], ref); + return; +end +hasItem = ndi.setup.V2.schemaHasField('item', 'value'); +env = {}; +if ~isempty(options.SoftwareId), env = [env, {'SoftwareId', options.SoftwareId}]; end +if ~isempty(options.InterpreterId), env = [env, {'InterpreterId', options.InterpreterId}]; end +env = [env, {'OperatingSystemId', options.OperatingSystemId}]; + +pre = ndi.setup.conv.haley.idPrefix(folder); +root = fullfile(dataParentDir, 'haley', 'celegans', folder); +I = load(fullfile(root, 'experimentInfo.mat'), 'info'); +D = I.info; +D = D(D.expNum == session.experiment(1), :); +R = R(strcmp(R.session, ref) & strcmp(R.kind, 'behaviour'), :); +docs = {}; +gapFill = did2.build.list( ... + did2.build.parameter('gap filling', 'Term', did2.build.term('', 'linear interpolation')), ... + did2.build.parameter('longest gap filled', 'Value', 10, 'Unit', 'second')); + +for part = {'midpoint', 'head', 'tail'} + bodyPart = part{1}; + file = fullfile(root, [bodyPart '.mat']); + if ~isfile(file) + continue; + end + T = trackTable(file); + T = T(T.expNum == session.experiment(1), :); + for k = 1:height(R) + epoch = R.epoch{k}; + row = find(arrayfun(@(r) strcmp(epochOf(pre, D.videoFileName{r}), epoch), 1:height(D)), 1); + if isempty(row) || ~isKey(options.RecordingStatements, epoch) || ~isKey(options.Geometry, epoch) + continue; + end + geo = options.Geometry(epoch); + V = T(T.plateNum == D.plateNum(row) & T.videoNum == D.videoNum(row), :); + if height(V) == 0 + continue; + end + videoId = options.RecordingStatements(epoch); + timeIds = {}; + if isKey(options.RecordingRefs, epoch), timeIds = {options.RecordingRefs(epoch)}; end + calc = [{'SessionId', sid, 'TimeReferenceIds', timeIds}, env]; + plate = R.plate{k}; + worms = unique(V.wormNum); + for w = reshape(worms, 1, []) + worm = sprintf('%s_worm%04d', pre, w); + if ~isKey(subjectIds, worm) + out.skipped{end+1} = sprintf('%s: %s %s is not a subject of this session', epoch, worm, bodyPart); + continue; + end + W = sortrows(V(V.wormNum == w, :), 'frameNum'); + n = height(W); + if ~isequal(W.frameNum(:)', 1:n) + out.skipped{end+1} = sprintf('%s: %s %s frames are not 1..%d; no track', ... + epoch, worm, bodyPart, n); + continue; + end + frame = did2.build.key('video frame', n, 'Origin', 0, 'Spacing', 1, ... + 'SourceOrigin', 1, 'SourceSpacing', 1); + frames = did2.build.list(frame); + wormId = subjectIds(worm); + + % positions + xy = [W.xPosition, W.yPosition] - 0.5; + keys = did2.build.list(frame, did2.build.key('image coordinate', 2, ... + 'Labels', {'image horizontal axis', 'image vertical axis'})); + pos = did2.build.statement('position_calculation', wormId, ... + did2.build.term('', [bodyPart ' position']), [], 'DataBody', true, ... + 'DatumType', 'float64', 'Keys', keys, ... + 'Method', did2.build.term('', 'tracking by WormLab'), 'MethodParameters', gapFill, ... + 'InputIds', {videoId}, ... + 'Edges', struct('coordinate_system_id', geo.coordinate_system), calc{:}); + docs = [docs, {pos, ingestedBody(pos, xy, 'float64', keys, sid, ... + sprintf('The %s track: x, y per video frame, pixels from the upper-left corner.', bodyPart), ... + 'NaN')}]; %#ok + out.counts.position = out.counts.position + 1; + ids = struct('position', pos.base.id, 'speed', '', 'patch_edge_distance', ''); + + % speed + if ismember('velocitySmooth', W.Properties.VariableNames) + sp = did2.build.statement('velocity_calculation', wormId, ... + did2.build.term('', [bodyPart ' speed']), [], 'DataBody', true, ... + 'DatumType', 'float64', 'Keys', frames, ... + 'Method', did2.build.term('', 'moving mean of step lengths'), ... + 'MethodParameters', did2.build.list( ... + did2.build.parameter('window', 'Value', 4, 'Unit', 'second'), ... + did2.build.parameter('fastest step kept', 'Value', 600e-6, ... + 'Unit', 'meter per second', 'SourceValue', '600', 'SourceUnit', 'um/s')), ... + 'InputIds', {pos.base.id}, calc{:}); + docs = [docs, {sp, ingestedBody(sp, W.velocitySmooth * 1e-6, 'float64', frames, sid, ... + 'Speed per video frame, m/s (the source''s velocitySmooth, um/s).', 'NaN')}]; %#ok + out.counts.speed = out.counts.speed + 1; + ids.speed = sp.base.id; + end + + % distance to the nearest patch edge + if ismember('distanceLawnEdge', W.Properties.VariableNames) && ~isempty(geo.lawn_mask) + de = did2.build.statement('length_calculation', wormId, ... + did2.build.term('', [bodyPart ' distance to the nearest patch edge']), [], ... + 'DataBody', true, 'DatumType', 'float64', 'Keys', frames, ... + 'Method', did2.build.term('', 'distance transform of the lawn mask'), ... + 'Notes', 'Positive on a patch, negative off it.', ... + 'InputIds', {pos.base.id, geo.lawn_mask}, calc{:}); + docs = [docs, {de, ingestedBody(de, W.distanceLawnEdge * geo.pixel, 'float64', frames, sid, ... + 'Signed distance to the nearest patch edge per video frame, m (the source''s distanceLawnEdge, pixels).', ... + 'NaN')}]; %#ok + out.counts.patch_edge_distance = out.counts.patch_edge_distance + 1; + ids.patch_edge_distance = de.base.id; + end + if strcmp(bodyPart, 'midpoint') + out.byWorm([epoch '|' worm]) = ids; + end + + % nearest patch + if hasItem && ismember('closestLawnID', W.Properties.VariableNames) + pIds = patchIds(subjectIds, plate); + id = double(W.closestLawnID); + if ismember('noTrack', W.Properties.VariableNames) + id(logical(W.noTrack)) = NaN; % read at pixel (1,1): meaningless + end + if isempty(pIds) || any(id > numel(pIds) | id < 1) + out.skipped{end+1} = sprintf('%s: %s names patches the plate %s does not have; no nearest patch', ... + epoch, worm, plate); + else + dt = 'uint8'; fill = 255; + if numel(pIds) >= 255, dt = 'uint16'; fill = 65535; end + v = id - 1; + v(isnan(v)) = fill; + inputs = {pos.base.id}; + if ~isempty(geo.nearest_patch) + inputs{end+1} = geo.nearest_patch; %#ok + elseif ~isempty(geo.lawn_mask) + inputs{end+1} = geo.lawn_mask; %#ok + end + np = did2.build.statement('item_calculation', wormId, ... + did2.build.term('', 'nearest patch'), [], 'DataBody', true, 'DatumType', dt, ... + 'Keys', frames, 'InputIds', inputs, ... + 'Edges', struct('item_id', {pIds}), calc{:}); + docs = [docs, {np, ingestedBody(np, v, dt, frames, sid, ... + sprintf(['The nearest patch per video frame: a 0-based index into item_id; ' ... + '%d where the source did not track the worm.'], fill), num2str(fill))}]; %#ok + out.counts.nearest_patch = out.counts.nearest_patch + 1; + end + end + end + end +end +out.documents = docs; +end + +% ----------------------------------------------------------------------------- + +function e = epochOf(pre, name) +% a recording's epoch id from the file name the table records +e = ''; +name = strtrim(char(name)); +if isempty(name), return; end +[~, stem] = fileparts(name); +e = [pre '_' stem]; +end diff --git a/src/ndi/+ndi/+setup/+conv/+haley/transferTime.m b/src/ndi/+ndi/+setup/+conv/+haley/transferTime.m new file mode 100644 index 0000000000..f9a39cd620 --- /dev/null +++ b/src/ndi/+ndi/+setup/+conv/+haley/transferTime.m @@ -0,0 +1,98 @@ +function [T, rule] = transferTime(protocol, folder, behaviourStart, lawnStart, behaviourVideo, lawnEnd) +%TRANSFERTIME When an assay plate's worms were moved onto it (decision #52). +% +% [T, RULE] = ndi.setup.conv.haley.transferTime(PROTOCOL, FOLDER, +% BEHAVIOURSTART, LAWNSTART, BEHAVIOURVIDEO) returns the transfer time T of +% one assay plate as wall-clock time (no time zone; NaT when the plate was +% never filmed). The source does not record it, so it is read off the +% plate's videos with PROTOCOL, the spec's `transfer_protocol` section: +% +% contrast video AFTER the worms went on (worms first): T = the lawn +% clip's start (LAWNSTART), when it is at most an hour before +% BEHAVIOURSTART; otherwise BEHAVIOURSTART +% contrast video BEFORE the worms (lawn first), or no rule: T = +% BEHAVIOURSTART, the plate's first behaviour video start +% +% T is approximate either way, and only ever EARLY of the video it is read +% off: RULE.tolerance = [minus 0] seconds bounds it (decision #52, CHANGE 7). +% minus is the protocol's `max_before_seconds` (default 300), except on a +% lawn-first plate whose lawn clip ends before BEHAVIOURSTART: the transfer +% came after that clip, so minus = min(max_before_seconds, BEHAVIOURSTART - +% LAWNEND). LAWNEND (optional, default LAWNSTART: a wider but still true +% bound) is the lawn clip's end. FOLDER is the source folder +% (foragingConcentration, ...); BEHAVIOURVIDEO is the plate's video file +% names without extension -- a char, or a cellstr such as {first behaviour +% video, lawn clip} -- which a protocol `exceptions` entry names by any one +% of them (e.g. 2023-04-04_16-10-14_2 is that plate's lawn clip). RULE is a struct: method (how the +% worms were moved, '' with no rule), contrast_video ('before_worms', +% 'after_worms' or ''), source ('rule', 'exception' or 'none'), and used +% ('lawn clip' or 'behaviour video'), and tolerance ([minus plus] seconds, +% [] when there is no T). +% +% Rules match by `source_folder` and, when given, `from_date` / `to_date` +% (inclusive, yyyy-MM-dd) against BEHAVIOURSTART's day; an exception +% overrides the contrast-video order of the rule that matched. + +if nargin < 6 || isnat(lawnEnd) + lawnEnd = lawnStart; +end +rule = struct('method', '', 'contrast_video', '', 'source', 'none', 'used', 'behaviour video', ... + 'tolerance', []); +T = behaviourStart; +if isnat(behaviourStart) + return; +end +maxBefore = double(getOr(protocol, 'max_before_seconds', 300)); +day = dateshift(behaviourStart, 'start', 'day'); +for r = asCell(getOr(protocol, 'rules', {})) + x = r{1}; + if ~strcmp(char(x.source_folder), folder) + continue; + end + if isfield(x, 'from_date') && day < datetime(x.from_date, 'InputFormat', 'yyyy-MM-dd') + continue; + end + if isfield(x, 'to_date') && day > datetime(x.to_date, 'InputFormat', 'yyyy-MM-dd') + continue; + end + rule.method = char(getOr(x, 'method', '')); + rule.contrast_video = char(x.contrast_video); + rule.source = 'rule'; + break; +end +for e = asCell(getOr(protocol, 'exceptions', {})) + x = e{1}; + if any(strcmp(char(x.video), cellstr(behaviourVideo))) + rule.contrast_video = char(x.contrast_video); + rule.source = 'exception'; + end +end +if strcmp(rule.contrast_video, 'after_worms') && ~isnat(lawnStart) ... + && lawnStart <= behaviourStart && behaviourStart - lawnStart <= hours(1) + T = lawnStart; + rule.used = 'lawn clip'; +end +rule.tolerance = [maxBefore 0]; +if strcmp(rule.used, 'behaviour video') && ~isnat(lawnEnd) && lawnEnd <= behaviourStart + % lawn first: the worms went on after the lawn clip ended + rule.tolerance(1) = min(maxBefore, seconds(behaviourStart - lawnEnd)); +end +end + +function v = getOr(s, name, default) +if isstruct(s) && isfield(s, name) + v = s.(name); +else + v = default; +end +end + +function c = asCell(x) +if iscell(x) + c = x(:)'; +elseif isstruct(x) + c = num2cell(x(:)'); +else + c = {}; +end +end diff --git a/src/ndi/+ndi/+setup/+conv/+haley/verifyDataset.m b/src/ndi/+ndi/+setup/+conv/+haley/verifyDataset.m new file mode 100644 index 0000000000..9194ec8768 --- /dev/null +++ b/src/ndi/+ndi/+setup/+conv/+haley/verifyDataset.m @@ -0,0 +1,301 @@ +function report = verifyDataset(path, options) +%VERIFYDATASET Check a written Haley V2 dataset against what the import built. +% +% REPORT = ndi.setup.conv.haley.verifyDataset(PATH) reads the V2 dataset +% at PATH (the folder holding .ndi, e.g. /haley_V2/dataset) -- its +% own database and each LINKED session's (ndi.setup.V2.datasetDatabaseFiles) +% -- and checks it, printing each check's denominator first: +% +% 1 open ndi.dataset.dir opens it; every session it lists opens, +% with the reference it is listed under +% 2 census documents per class and per session, read from the +% database; with 'Expected', compared to what the import +% counted before writing (differences listed) +% 3 edges every depends_on value names a document in the database +% (dangling ones counted by class and edge name) +% 4 files every file the database records exists: an ingested one +% in .ndi/files, one recorded by location where it points +% 5 hashes every ingested body whose document records an MD5 +% content_hash still has that MD5 ('Hashes', false skips) +% +% Options: +% 'Expected' the import's result (r = import_V2(...)); its +% r.dataset.report.documents is the expected census +% 'Hashes' default true +% +% REPORT has one field per check, and `failed`, the names of the checks +% that found a problem. It writes nothing. +% +% See also ndi.setup.conv.haley.import_V2, ndi.setup.V2.createDataset. + +arguments + path (1,:) char {mustBeFolder} + options.Expected = [] + options.Hashes (1,1) logical = true +end + +report = struct('failed', {{}}); +ndiDir = fullfile(path, '.ndi'); +dbFile = fullfile(ndiDir, ndi.database.implementations.database.did2sqlite.DEFAULTFILENAME()); +if ~isfile(dbFile) + error('ndi:setup:conv:haley:noDataset', 'No V2 database at %s.', dbFile); +end + +prog = progressBar(); + +% ---- 1 open ------------------------------------------------------------------- +fprintf('\n== 1 open ==\n'); +ds = ndi.dataset.dir(path); +[refs, ids] = ds.session_list(); +bad = {}; +prog.start('open', sprintf('Opening %d session(s)', numel(ids)), numel(ids)); +for k = 1:numel(ids) + prog.step('open', k); + try + s = ds.open_session(ids{k}); + if ~strcmp(s.reference, refs{k}) + bad{end+1} = sprintf('%s opened with reference %s', refs{k}, s.reference); %#ok + end + catch err + bad{end+1} = sprintf('%s: %s', refs{k}, err.message); %#ok + end +end +fprintf('DENOMINATOR: dataset %s; %d session(s) listed, %d opened\n', ds.id(), numel(ids), ... + numel(ids) - numel(bad)); +printList(bad); +report.open = struct('datasetId', ds.id(), 'sessions', {refs}, 'problems', {bad}); +if ~isempty(bad), report.failed{end+1} = 'open'; end + +% Every database of the dataset: its own, then each linked session's +% (decision #71); an ingested session is in the first. +[files, folders, linkProblems] = ndi.setup.V2.datasetDatabaseFiles(path); +fprintf('DENOMINATOR: %d database(s) read: the dataset''s and %d linked session folder(s)\n', ... + numel(files), numel(files) - 1); +printList(linkProblems); +if ~isempty(linkProblems), report.failed{end+1} = 'open'; end +dbs = cell(1, numel(files)); +for f = 1:numel(files) + dbs{f} = did2.database.sqlitedb(files{f}); +end +closer = onCleanup(@() cellfun(@(x) x.close(), dbs)); +qf = @(f, sql) mksqlite(dbs{f}.testHookDbId(), sql); + +% ---- 2 census ----------------------------------------------------------------- +fprintf('\n== 2 census ==\n'); +cls = {}; nCls = []; sid = {}; nSid = []; +for f = 1:numel(dbs) + a = qf(f, 'SELECT classname AS class, COUNT(*) AS n FROM documents GROUP BY classname'); + b = qf(f, 'SELECT session_id, COUNT(*) AS n FROM documents GROUP BY session_id'); + cls = [cls, {a.class}]; nCls = [nCls, double([a.n])]; %#ok + sid = [sid, {b.session_id}]; nSid = [nSid, double([b.n])]; %#ok +end +byClass = sumBy(cls, nCls, 'class'); +bySession = sumBy(sid, nSid, 'session_id'); +total = sum(byClass.n); +fprintf('DENOMINATOR: %d document(s) in %d class(es) and %d session id(s), over %d database(s)\n', ... + total, height(byClass), height(bySession), numel(dbs)); +report.census = struct('total', total, 'byClass', byClass, 'bySession', bySession, ... + 'differences', {{}}); +if ~isempty(options.Expected) + E = options.Expected.dataset.report.documents; + d = [compareCounts(E.byClass, 'class', byClass, 'class', 'class'), ... + compareCounts(E.bySession, 'session_id', bySession, 'session_id', 'session')]; + fprintf(' compared with the import''s own count: %d difference(s)\n', numel(d)); + printList(d); + report.census.differences = d; + if ~isempty(d), report.failed{end+1} = 'census'; end +else + fprintf(' (no ''Expected'': counts reported, not compared)\n'); +end +disp(byClass); + +% ---- 3 edges ------------------------------------------------------------------ +% an edge dangles when no database of the dataset holds its target: a linked +% session's documents point at the dataset's (a strain, a study), and the +% dataset's part_of relations and linked_session documents at the sessions' +fprintf('\n== 3 edges ==\n'); +checked = 0; +dangRows = cell(0, 2); +for f = 1:numel(dbs) + n = qf(f, 'SELECT COUNT(*) AS n FROM depends_on WHERE document_id <> '''''); + checked = checked + double(n.n); + local = qf(f, ['SELECT d.classname AS class, e.name AS edge, e.document_id AS target ' ... + 'FROM depends_on e JOIN documents d ON d.id = e.doc_id ' ... + 'LEFT JOIN documents t ON t.id = e.document_id ' ... + 'WHERE e.document_id <> '''' AND t.id IS NULL']); + for r = 1:numel(local) + found = false; + for g = setdiff(1:numel(dbs), f) + if dbs{g}.has(local(r).target), found = true; break; end + end + if ~found + dangRows(end+1, :) = {local(r).class, local(r).edge}; %#ok + end + end +end +nDang = size(dangRows, 1); +dang = []; +if nDang > 0 + dang = groupsummary(cell2table(dangRows, 'VariableNames', {'class', 'edge'}), {'class', 'edge'}); +end +fprintf('DENOMINATOR: %d edge value(s) naming a document, over %d database(s); %d dangling\n', ... + checked, numel(dbs), nDang); +if nDang > 0, disp(dang); end +report.edges = struct('checked', checked, 'dangling', nDang, 'byClassEdge', dang); +if nDang > 0, report.failed{end+1} = 'edges'; end + +% ---- 4 files ------------------------------------------------------------------ +fprintf('\n== 4 files ==\n'); +missing = {}; +nIngested = 0; +nRecorded = 0; +for f = 1:numel(dbs) + F = qf(f, 'SELECT doc_id, filename, uid, location, ingested FROM files'); + fileDir = fullfile(folders{f}, '.ndi', 'files'); + nRecorded = nRecorded + numel(F); + prog.start(sprintf('files %d', f), sprintf('Checking %d file(s) of database %d', numel(F), f), numel(F)); + for k = 1:numel(F) + prog.step(sprintf('files %d', f), k); + if F(k).ingested + nIngested = nIngested + 1; + p = fullfile(fileDir, F(k).uid); + else + p = F(k).location; + end + if ~isfile(p) + missing{end+1} = sprintf('%s (%s of document %s)', p, F(k).filename, F(k).doc_id); %#ok + end + end +end +fprintf('DENOMINATOR: %d file(s) recorded: %d ingested, %d by location; %d missing\n', ... + nRecorded, nIngested, nRecorded - nIngested, numel(missing)); +printList(missing, 20); +report.files = struct('recorded', nRecorded, 'ingested', nIngested, 'missing', {missing}); +if ~isempty(missing), report.failed{end+1} = 'files'; end + +% ---- 5 hashes ----------------------------------------------------------------- +if options.Hashes + fprintf('\n== 5 hashes ==\n'); + checked = 0; noHash = 0; wrong = {}; nB = 0; + for f = 1:numel(dbs) + B = qf(f, ['SELECT f.doc_id, f.uid, d.body FROM files f JOIN documents d ON d.id = f.doc_id ' ... + 'WHERE f.ingested = 1']); + fileDir = fullfile(folders{f}, '.ndi', 'files'); + nB = nB + numel(B); + prog.start(sprintf('hashes %d', f), sprintf('Hashing %d file(s) of database %d', numel(B), f), numel(B)); + for k = 1:numel(B) + prog.step(sprintf('hashes %d', f), k); + [h, alg] = recordedHash(jsondecode(B(k).body)); + if isempty(h) || ~strcmpi(alg, 'MD5') + noHash = noHash + 1; + continue; + end + p = fullfile(fileDir, B(k).uid); + if ~isfile(p), continue; end % counted under files + checked = checked + 1; + if ~strcmpi(ndi.fun.file.MD5(p), h) + wrong{end+1} = sprintf('document %s: %s', B(k).doc_id, p); %#ok + end + end + end + fprintf('DENOMINATOR: %d ingested file(s); %d with a recorded MD5 checked, %d with none; %d differ\n', ... + nB, checked, noHash, numel(wrong)); + printList(wrong, 20); + report.hashes = struct('ingested', nB, 'checked', checked, 'noHash', noHash, ... + 'differ', {wrong}); + if ~isempty(wrong), report.failed{end+1} = 'hashes'; end +end + +fprintf('\n== verdict ==\n'); +if isempty(report.failed) + fprintf('all checks passed\n'); +else + fprintf('FAILED: %s\n', strjoin(report.failed, ', ')); +end +end + +% ----------------------------------------------------------------------------- +function prog = progressBar() +% one ProgressBarWindow bar per check (when MATLAB has a display), plus a +% printed line about every 5% with elapsed time and an estimate of the rest +w = []; +try + w = ndi.gui.component.ProgressBarWindow('NDI V2 verify', 'GrabMostRecent', true); + w.setTimeout(hours(12)); +catch + w = []; +end +state = struct('n', 0, 'every', 1, 't0', 0, 'label', ''); +prog.start = @start; +prog.step = @step; + function start(tag, label, n) + state.n = n; + state.every = max(1, round(n / 20)); + state.t0 = tic; + state.label = label; + if ~isempty(w) + try w.addBar('Label', label, 'Tag', tag, 'Auto', true); catch, end + end + end + function step(tag, k) + if mod(k, state.every) ~= 0 && k ~= state.n, return; end + el = toc(state.t0); + fprintf(' %s: %d / %d (%.0f%%), %s elapsed, ~%s left\n', state.label, k, state.n, ... + 100 * k / state.n, hms(el), hms(el / k * (state.n - k))); + if ~isempty(w) + try w.updateBar(tag, k / state.n); catch, end + end + end +end + +function s = hms(sec) +s = char(duration(0, 0, round(sec), 'Format', 'hh:mm:ss')); +end + +function T = sumBy(keys, n, name) +% a table of KEYS (cellstr) and the sum of N over each, sorted by key +if isempty(keys) + T = table(cell(0, 1), zeros(0, 1), 'VariableNames', {name, 'n'}); + return; +end +[u, ~, j] = unique(keys(:)); +T = table(u, accumarray(j, n(:)), 'VariableNames', {name, 'n'}); +end + +function d = compareCounts(E, eKey, A, aKey, what) +% differences between the expected (groupsummary: GroupCount) and actual (n) counts +d = {}; +e = containers.Map(cellstr(string(E.(eKey))), num2cell(E.GroupCount)); +a = containers.Map(cellstr(string(A.(aKey))), num2cell(double(A.n))); +for k = union(keys(e), keys(a)) + x = 0; y = 0; + if isKey(e, k{1}), x = e(k{1}); end + if isKey(a, k{1}), y = a(k{1}); end + if x ~= y + d{end+1} = sprintf('%s %s: expected %d, found %d', what, k{1}, x, y); %#ok + end +end +end + +function [h, alg] = recordedHash(s) +% the content_hash and hash_algorithm of a body document, in whichever block holds them +h = ''; alg = ''; +for f = reshape(fieldnames(s), 1, []) + b = s.(f{1}); + if isstruct(b) && isscalar(b) && isfield(b, 'content_hash') && ~isempty(b.content_hash) + h = char(b.content_hash); + if isfield(b, 'hash_algorithm'), alg = char(b.hash_algorithm); end + return; + end +end +end + +function printList(items, limit) +if nargin < 2, limit = Inf; end +for k = 1:min(numel(items), limit) + fprintf(' %s\n', items{k}); +end +if numel(items) > limit + fprintf(' ... and %d more\n', numel(items) - limit); +end +end diff --git a/src/ndi/+ndi/+test/+cloud/lightsheet_blob_cloud_roundtrip.m b/src/ndi/+ndi/+test/+cloud/lightsheet_blob_cloud_roundtrip.m new file mode 100644 index 0000000000..472b561555 --- /dev/null +++ b/src/ndi/+ndi/+test/+cloud/lightsheet_blob_cloud_roundtrip.m @@ -0,0 +1,373 @@ +function info = lightsheet_blob_cloud_roundtrip(options) +%NDI.TEST.CLOUD.LIGHTSHEET_BLOB_CLOUD_ROUNDTRIP - end-to-end cloud round trip for the blob fixture. +% +% INFO = ndi.test.cloud.LIGHTSHEET_BLOB_CLOUD_ROUNDTRIP(...) +% +% Manual test (not part of the automated suite). Set +% NDI_CLOUD_USERNAME / NDI_CLOUD_PASSWORD in the shell, then call +% this from a MATLAB prompt to smoke-test the whole cloud pipeline. +% +% Full pipeline in one call, staged entirely in the OS temp directory +% so a run leaves no artefacts behind on the caller's project tree: +% +% a) Fresh temp directories for the session, the dataset, and the +% eventual download. +% b) Build a small synthetic OME-Zarr fixture on disk via +% ndi.test.lightsheet.makeBlobFixture. +% c) Open an ndi.session on the session temp dir and ingest the +% fixture into it via ndi.fun.doc.lightsheet.fromOMEZarr (writes +% the lightsheetZarrPyramid + level documents). +% d) Launch napariViewLightsheet on the just-built pyramid. +% e) Create an ndi.dataset.dir on the dataset temp dir and attach the +% session via add_ingested_session (a lightsheet-only session is +% trivially ingested -- it declares no daq systems, so +% isIngested() returns true without a further ingest() call). +% f) ndi.cloud.uploadDataset with 'uploadAsNew', 'skipMetadataEditorMetadata', +% and 'remoteDatasetName' set to "Blob Test Dataset ". +% g) ndi.cloud.downloadDataset the cloud dataset back into a THIRD +% temp dir (SyncFiles defaults to false, which now works end-to-end +% via DID-matlab#201's fetchSeriesManifestBytes contract). +% h) List the sessions on the downloaded dataset and open the first +% one. +% i) Search that session for lightsheetZarrPyramid documents and +% launch napariViewLightsheet on the first hit. +% +% Name-Value Arguments: +% Launcher - char, path to napariViewLightsheet. Default +% '/usr/local/bin/napariViewLightsheet'. +% Shape - fixture spatial shape, default [128 128 128]. +% NumChannels - default 2. +% ChunkShape - default [32 32 32]. +% NumLevels - default 3. +% Codec - 'raw' or 'blosc-zstd'. Default 'blosc-zstd' (exercises +% the encodeChunk MEX fast path). +% Clevel - default 5. +% Reduction - 'mean' or 'max'. Default 'mean'. +% ViewLocal - launch napari on the local session too (step d). +% Default true. Set false to skip the local view and go +% straight to the cloud path. +% ViewCloud - launch napari on the downloaded session (step i). +% Default true. Both viewers block for a moment while +% they open; if you are running headless, set both to +% false. +% WaitViewer - forward to ndi.fun.doc.lightsheet.view. Default false +% (viewer backgrounds; MATLAB stays interactive). +% Verbose - print progress narration. Default true. +% +% Returns a struct with everything the run produced: +% .sessionDir - the session's temp dir +% .zarrPath - the on-disk OME-Zarr store +% .datasetDir - the local dataset's temp dir +% .downloadDir - where downloadDataset copied the cloud state +% .cloudDatasetId - the id returned by uploadDataset +% .cloudDatasetName - the "Blob Test Dataset ..." label sent up +% .localSession - the freshly-built session (post-ingest) +% .localPyramidId - the pyramid document id from step (c) +% .downloadedDataset - the ndi.dataset.dir opened from downloadDir +% .downloadedSession - the first session on the downloaded dataset +% .downloadedPyramidId - the pyramid id we opened in step (i) +% +% Set NDI_CLOUD_USERNAME / NDI_CLOUD_PASSWORD before calling; the upload +% and download steps will fail without them. +% +% See also: ndi.test.lightsheet.makeBlobFixture, +% ndi.fun.doc.lightsheet.fromOMEZarr, +% ndi.fun.doc.lightsheet.view, +% ndi.cloud.uploadDataset, +% ndi.cloud.downloadDataset + + arguments + options.Launcher (1,:) char = '/usr/local/bin/napariViewLightsheet' + options.Shape (1,3) double {mustBePositive, mustBeInteger} = [128 128 128] + options.NumChannels (1,1) double {mustBePositive, mustBeInteger} = 2 + options.ChunkShape (1,3) double {mustBePositive, mustBeInteger} = [32 32 32] + options.NumLevels (1,1) double {mustBePositive, mustBeInteger} = 3 + options.Codec (1,:) char {mustBeMember(options.Codec, {'raw','blosc-zstd'})} = 'blosc-zstd' + options.Clevel (1,1) double {mustBeInteger, mustBeGreaterThanOrEqual(options.Clevel,1), mustBeLessThanOrEqual(options.Clevel,9)} = 5 + options.Reduction (1,:) char {mustBeMember(options.Reduction, {'mean','max'})} = 'mean' + options.ViewLocal (1,1) logical = true + options.ViewCloud (1,1) logical = true + options.WaitViewer (1,1) logical = false + options.Verbose (1,1) logical = true + options.DebugLogFile (1,:) char = fullfile(tempdir, 'NapariLightSheetDebugging.txt') + end + + verbose = options.Verbose; + function narrate(fmt, varargin) + if verbose + fprintf(['[roundtrip] ' fmt '\n'], varargin{:}); + end + end + + % ndi.document.readblankdefinition keeps a persistent containers.Map + % of parsed JSON per document type. A MATLAB session that had the + % pre-fix lightsheetZarrLevel schema loaded (before chunk.bin became + % a DID file_series) hands the stale definition to every later + % construction, and addFileSeries('chunk.bin', ...) then refuses the + % write with "chunk.bin is not declared as a file series". Nuke the + % cache up front so a re-run of this demo cannot inherit a stale + % session state. + try + ndi.document.readblankdefinition('--clear-cache'); + catch + % Older NDI without the sentinel; safe to skip -- the definition + % cache is a persistent inside the same file, so a `clear + % functions` at the caller resolves the same way. + end + + % --- (a) three fresh temp directories ------------------------------- + sessionDir = mustMakeTempDir(); + datasetDir = mustMakeTempDir(); + downloadDir = mustMakeTempDir(); + zarrParent = mustMakeTempDir(); + narrate('Temp session : %s', sessionDir); + narrate('Temp dataset : %s', datasetDir); + narrate('Temp download : %s', downloadDir); + narrate('Temp zarr root : %s', zarrParent); + + % --- (b) build the blob fixture ------------------------------------- + narrate('Building blob fixture ([shape=%dx%dx%d, channels=%d, chunks=%dx%dx%d, levels=%d])...', ... + options.Shape(1), options.Shape(2), options.Shape(3), options.NumChannels, ... + options.ChunkShape(1), options.ChunkShape(2), options.ChunkShape(3), ... + options.NumLevels); + [zarrPath, ~] = ndi.test.lightsheet.makeBlobFixture(zarrParent, ... + 'Shape', options.Shape, ... + 'NumChannels', options.NumChannels, ... + 'ChunkShape', options.ChunkShape, ... + 'NumLevels', options.NumLevels); + narrate('Zarr fixture : %s', zarrPath); + + % --- (c) ingest the fixture into a fresh session -------------------- + narrate('Opening session on %s ...', sessionDir); + S = ndi.session.dir('blob', sessionDir); + + subjectName = 'blob@vhlab'; + sub = ndi.document('subject', ... + 'base.session_id', S.id(), ... + 'subject.local_identifier', subjectName); + S.database_add(sub); + narrate('Added subject document "%s".', subjectName); + + narrate('Running fromOMEZarr (codec=%s, clevel=%d)...', options.Codec, options.Clevel); + [P, ~, ~] = ndi.fun.doc.lightsheet.fromOMEZarr(S, zarrPath, ... + 'subjectID', sub.id(), ... + 'materializeChunks', true, ... + 'codec', options.Codec, ... + 'clevel', options.Clevel); + narrate('Built lightsheetZarrPyramid %s.', P.id()); + + % Set up the shared debug log. Truncate on entry so a run's log is + % just that run's log; both viewer subprocesses append their + % NDI_LIGHTSHEET_DEBUG stderr into it with clear === LOCAL === / + % === CLOUD === separators so the file reads as one narrative. + initDebugLog(options.DebugLogFile); + narrate('Napari debug log: %s', options.DebugLogFile); + + % --- (d) view the local pyramid ------------------------------------- + if options.ViewLocal + narrate('Launching napari viewer on the LOCAL session ...'); + launchDebugView(S, P.id(), options.Launcher, options.Reduction, ... + options.WaitViewer, options.DebugLogFile, 'LOCAL'); + else + narrate('Skipping local view (ViewLocal=false).'); + end + + % --- (e) make a dataset, ingest the session ------------------------- + narrate('Building dataset at %s ...', datasetDir); + D = ndi.dataset.dir('blob_dataset', datasetDir); + if ~S.isIngested() + error('lightsheet_blob_cloud_roundtrip:notIngested', ... + ['Session %s is not fully ingested. A lightsheet-only session with no ' ... + 'daq systems should be trivially ingested; check whether an unexpected ' ... + 'daq system got attached during setup.'], S.id()); + end + D = D.add_ingested_session(S); + narrate('Added session %s to dataset (ingested).', S.id()); + + % --- (f) upload to NDI Cloud ---------------------------------------- + cloudDatasetName = buildRemoteDatasetName(); + narrate('Uploading to NDI Cloud as remote name "%s" ...', cloudDatasetName); + [ok, cloudDatasetId, msg] = ndi.cloud.uploadDataset(D, ... + 'uploadAsNew', true, ... + 'skipMetadataEditorMetadata', true, ... + 'remoteDatasetName', cloudDatasetName); + if ~ok + error('lightsheet_blob_cloud_roundtrip:uploadFailed', ... + 'ndi.cloud.uploadDataset failed: %s', msg); + end + narrate('Cloud dataset id: %s', cloudDatasetId); + + % Server-side zip extraction has to finish before a download can see + % the ingested-session's per-file objects. Without this wait, + % Ddown.session_list() can come back empty on a fast download -- + % the race the Python round-trip test guards against with the same + % call (see tests/test_cloud_file_series_round_trip.py). + narrate('Waiting for server-side bulk-upload extraction to finish ...'); + [waitOk, waitInfo] = ndi.cloud.api.files.waitForAllBulkUploads(cloudDatasetId); + if ~waitOk + error('lightsheet_blob_cloud_roundtrip:extractionIncomplete', ... + 'waitForAllBulkUploads did not confirm completion (state=%s, elapsed=%.1fs).', ... + waitInfo.state, waitInfo.elapsed); + end + narrate('Bulk-upload extraction complete (%.1fs).', waitInfo.elapsed); + + % --- (g) download it back into a fresh temp dir --------------------- + narrate(['Downloading dataset back to %s (SyncFiles defaults false; the ' ... + 'DID-matlab#201 handler-fetch path installs manifests on demand) ...'], downloadDir); + Ddown = ndi.cloud.downloadDataset(cloudDatasetId, downloadDir); + narrate('Downloaded dataset opened at %s.', char(Ddown.path)); + + % --- (h) list and open the first session ---------------------------- + [refs, ids] = Ddown.session_list(); + if isempty(ids) + error('lightsheet_blob_cloud_roundtrip:noSessions', ... + 'Downloaded dataset carries no sessions; upload must have dropped the ingested session.'); + end + narrate('Downloaded dataset lists %d session(s): %s', numel(ids), strjoin(refs, ', ')); + firstId = ids{1}; + narrate('Opening the first session (%s) ...', firstId); + Sdown = Ddown.open_session(firstId); + + % --- (i) find the pyramid doc and open it --------------------------- + q = ndi.query('','isa','lightsheetZarrPyramid'); + pyramidDocs = Sdown.database_search(q); + if isempty(pyramidDocs) + error('lightsheet_blob_cloud_roundtrip:noPyramid', ... + 'No lightsheetZarrPyramid documents on the downloaded session %s.', firstId); + end + downloadedPyramidId = pyramidDocs{1}.id(); + narrate('Found lightsheetZarrPyramid %s on the downloaded session.', downloadedPyramidId); + + if options.ViewCloud + narrate('Launching napari viewer on the DOWNLOADED session ...'); + launchDebugView(Sdown, downloadedPyramidId, options.Launcher, ... + options.Reduction, options.WaitViewer, options.DebugLogFile, 'CLOUD'); + % Give napari + the customFileHandler a moment to try the first + % chunk fetches, then surface the tail so a black canvas comes + % with its cause on the MATLAB console instead of just a + % pointer to the log file. Only runs on non-blocking viewers + % (WaitViewer=false) -- otherwise the launcher hasn't + % returned yet and we'd read an empty log. + if ~options.WaitViewer + narrate('Sleeping 10 s so napari can attempt its first chunk fetches ...'); + pause(10); + dumpLogTail(options.DebugLogFile, 100); + end + else + narrate('Skipping cloud view (ViewCloud=false).'); + end + + info = struct( ... + 'sessionDir', sessionDir, ... + 'zarrPath', zarrPath, ... + 'datasetDir', datasetDir, ... + 'downloadDir', downloadDir, ... + 'cloudDatasetId', cloudDatasetId, ... + 'cloudDatasetName', cloudDatasetName, ... + 'localSession', S, ... + 'localPyramidId', P.id(), ... + 'downloadedDataset', Ddown, ... + 'downloadedSession', Sdown, ... + 'downloadedPyramidId', downloadedPyramidId); + narrate('Round trip complete.'); +end + +% ===================================================================== + +function d = mustMakeTempDir() + d = tempname; + if ~mkdir(d) + error('lightsheet_blob_cloud_roundtrip:tempMkdirFailed', ... + 'Could not create temp directory at %s.', d); + end +end + +function name = buildRemoteDatasetName() + % "Blob Test Dataset yyyymmddTHHMMSS RRR" where RRR is 0-1000 inclusive. + % Compact timestamp avoids embedded spaces / colons the cloud UI is + % happier with; still round-trips as a valid dataset label. + dstr = datestr(datetime('now'), 'yyyymmddTHHMMSS'); %#ok + r = randi([0 1000]); + name = sprintf('Blob Test Dataset %s %d', dstr, r); +end + +function initDebugLog(logfile) +% Truncate the shared debug log so a rerun starts clean, then write a +% header row so the file is never zero bytes (Finder / editors show it +% cleanly). The log lands in an OS-writable dir by default (tempdir), +% but any caller-supplied path works as long as its parent exists. + parent = fileparts(logfile); + if ~isempty(parent) && ~isfolder(parent) + mkdir(parent); + end + fid = fopen(logfile, 'w'); + if fid < 0 + error('lightsheet_blob_cloud_roundtrip:debugLogOpen', ... + 'Cannot open debug log %s for writing.', logfile); + end + fprintf(fid, '=== napariViewLightsheet debug log ===\n'); + fprintf(fid, 'started at %s\n\n', char(datetime('now'))); + fclose(fid); +end + +function dumpLogTail(logfile, nlines) +% Read the last NLINES of LOGFILE and print them to the MATLAB console +% under a clear separator. Runs via `tail -n ` in a +% subprocess so a huge log costs one syscall, not a MATLAB-side file +% read. Falls back silently if the log does not exist yet. + if ~isfile(logfile) + fprintf('[roundtrip] debug log %s not written yet; nothing to dump.\n', logfile); + return; + end + quotedLog = ['''' strrep(logfile, '''', '''\''''') '''']; + tailCmd = sprintf('tail -n %d %s', nlines, quotedLog); + fprintf('\n===== last %d lines of %s =====\n', nlines, logfile); + system(tailCmd); + fprintf('===== end of debug log tail =====\n\n'); +end + +function launchDebugView(session, pyramidId, launcher, reduction, waitForIt, logfile, tag) +% Build the viewer command via ndi.fun.doc.lightsheet.viewCommand, then +% run it via system() with NDI_LIGHTSHEET_DEBUG=1 in the environment +% and stderr+stdout appended to logfile. Bypasses +% ndi.fun.doc.lightsheet.view because that helper does not expose a +% redirect hook -- we want the Python side's [lightsheet] chunk fetch +% failures written where the user can grep them after the run. +% +% TAG is a short label (e.g. 'LOCAL' / 'CLOUD') that lands as a +% separator in the log, so a single log file can hold both viewer +% invocations from one run. + + sessionPath = char(session.path); + cmd = ndi.fun.doc.lightsheet.viewCommand(launcher, sessionPath, pyramidId, ... + 'reduction', reduction); + + % Append a header into the log identifying this invocation. + fid = fopen(logfile, 'a'); + if fid >= 0 + fprintf(fid, '\n=== %s : %s ===\n', tag, char(datetime('now'))); + fprintf(fid, 'session : %s\n', sessionPath); + fprintf(fid, 'pyramid : %s\n', pyramidId); + fprintf(fid, 'command : %s\n\n', cmd); + fclose(fid); + end + + % Route Python-side debug prints (NDI_LIGHTSHEET_DEBUG=1 triggers + % the [lightsheet] chunk-fetch failure lines) into the log file. + % Redirect both stdout and stderr; the launcher wrapper's own + % output lands in the same file so a chunk miss and its cause + % (auth, HTTP status, missing manifest) sit adjacent. + quotedLog = ['''' strrep(logfile, '''', '''\''''') '''']; + if ~waitForIt && ~ispc + fullCmd = ['NDI_LIGHTSHEET_DEBUG=1 ' cmd ' >>' quotedLog ' 2>&1 &']; + else + fullCmd = ['NDI_LIGHTSHEET_DEBUG=1 ' cmd ' >>' quotedLog ' 2>&1']; + end + fprintf('Executing: %s\n', fullCmd); + status = system(fullCmd); + if status ~= 0 && waitForIt + warning('lightsheet_blob_cloud_roundtrip:viewerNonZeroExit', ... + 'Viewer returned status %d; see %s for stderr.', status, logfile); + end +end + diff --git a/src/ndi/+ndi/+test/+lightsheet/ingestBlobFixture.m b/src/ndi/+ndi/+test/+lightsheet/ingestBlobFixture.m new file mode 100644 index 0000000000..f713444570 --- /dev/null +++ b/src/ndi/+ndi/+test/+lightsheet/ingestBlobFixture.m @@ -0,0 +1,117 @@ +function [session, pyramidDoc, info] = ingestBlobFixture(options) +% NDI.TEST.LIGHTSHEET.INGESTBLOBFIXTURE - one-line end-to-end blob demo +% +% [SESSION, PYRAMIDDOC, INFO] = NDI.TEST.LIGHTSHEET.INGESTBLOBFIXTURE() +% [SESSION, PYRAMIDDOC, INFO] = NDI.TEST.LIGHTSHEET.INGESTBLOBFIXTURE( ... +% 'sessionDir', '/path/to/session', 'zarrDir', '/path/to/store', ...) +% +% Writes the 300x300x300 blob fixture (see makeBlobFixture) into a +% session, calls fromOMEZarr to build the pyramid and level +% documents, and returns the ready-to-view result. Meant for a +% copy-paste smoke test of the whole ingest -> document -> viewer +% pathway. +% +% Name-Value: +% sessionDir - char, where to build the session. Empty (default) +% puts it under a fresh tempname. +% zarrDir - char, parent directory for the OME-Zarr store. +% Empty (default) uses tempname; the store lives at +% fullfile(zarrDir, 'blob.ome.zarr'). Pass a stable +% path here to keep the fixture between runs (e.g. +% for repeated cloud-upload experiments). +% subject - char, local_identifier for the subject document +% created inside the session (default 'blob@vhlab'). +% Shape - passed through to makeBlobFixture (default +% [300 300 300]). +% NumChannels - passed through to makeBlobFixture (default 2). +% Set 1 for the old single-channel layout. +% ChannelNames - passed through to makeBlobFixture. Empty +% (default) uses 'Ch1', 'Ch2', ... +% ChunkShape - passed through to makeBlobFixture (default +% [64 64 64], spatial only). +% NumLevels - passed through to makeBlobFixture (default 4). +% VoxelSize - passed through to makeBlobFixture (default +% [1 1 1]). +% +% Returns: +% SESSION - ndi.session.dir opened at sessionDir. +% PYRAMIDDOC - the lightsheetZarrPyramid parent document. +% INFO - the struct fromOMEZarr's third output would return +% (zarrPath, pyramids, reductions, sourceFileID, +% sharedLevel0), plus: +% subjectID - the subject document's id +% fixture - the ground-truth struct that +% makeBlobFixture returned +% +% Example: +% [S, P] = ndi.test.lightsheet.ingestBlobFixture(); +% ndi.gui.app.LightsheetZarrManager(S); +% +% See also: ndi.test.lightsheet.makeBlobFixture, +% ndi.fun.doc.lightsheet.fromOMEZarr, +% ndi.gui.app.LightsheetZarrManager + + arguments + options.sessionDir char = '' + options.zarrDir char = '' + options.subject (1,:) char = 'blob@vhlab' + options.Shape (1,3) double {mustBePositive, mustBeInteger} = [300 300 300] + options.NumChannels (1,1) double {mustBePositive, mustBeInteger} = 2 + options.ChannelNames = {} + options.ChunkShape (1,3) double {mustBePositive, mustBeInteger} = [64 64 64] + options.NumLevels (1,1) double {mustBePositive, mustBeInteger} = 4 + options.VoxelSize (1,3) double {mustBePositive} = [1 1 1] + options.materializeChunks (1,1) logical = true + options.codec (1,:) char {mustBeMember(options.codec, {'raw','blosc-zstd'})} = 'raw' + options.clevel (1,1) double {mustBeInteger, mustBeGreaterThanOrEqual(options.clevel, 1), mustBeLessThanOrEqual(options.clevel, 9)} = 5 + % Match fromOMEZarr's default: 'default' opens the NDI + % ProgressBarWindow. Pass a function handle for a custom + % receiver, or [] to stay silent (which is what CI needs + % without the auto-headless detection kicking in). + options.progressFcn = 'default' + % See fromOMEZarr: -1 = auto (use any open pool, else serial), + % 0/1 = force serial, N>1 = size a new pool. + options.numWorkers (1,1) double {mustBeInteger, mustBeGreaterThanOrEqual(options.numWorkers, -1)} = -1 + end + + sessionDir = options.sessionDir; + if isempty(sessionDir) + sessionDir = tempname; + end + if ~isfolder(sessionDir) + mkdir(sessionDir); + end + + zarrParent = options.zarrDir; + if isempty(zarrParent) + zarrParent = tempname; + mkdir(zarrParent); + elseif ~isfolder(zarrParent) + mkdir(zarrParent); + end + + [zarrPath, gt] = ndi.test.lightsheet.makeBlobFixture(zarrParent, ... + 'Shape', options.Shape, ... + 'NumChannels', options.NumChannels, ... + 'ChannelNames', options.ChannelNames, ... + 'NumLevels', options.NumLevels, ... + 'ChunkShape', options.ChunkShape, ... + 'VoxelSize', options.VoxelSize); + + session = ndi.session.dir('blob', sessionDir); + sub = ndi.document('subject', 'base.session_id', session.id(), ... + 'subject.local_identifier', options.subject); + session.database_add(sub); + + [pyramidDoc, ~, info] = ndi.fun.doc.lightsheet.fromOMEZarr( ... + session, zarrPath, ... + 'subjectID', sub.id(), ... + 'materializeChunks', options.materializeChunks, ... + 'codec', options.codec, ... + 'clevel', options.clevel, ... + 'progressFcn', options.progressFcn, ... + 'numWorkers', options.numWorkers); + + info.subjectID = sub.id(); + info.fixture = gt; +end diff --git a/src/ndi/+ndi/+test/+lightsheet/makeBlobFixture.m b/src/ndi/+ndi/+test/+lightsheet/makeBlobFixture.m new file mode 100644 index 0000000000..b34337817e --- /dev/null +++ b/src/ndi/+ndi/+test/+lightsheet/makeBlobFixture.m @@ -0,0 +1,480 @@ +function [zarrPath, gt] = makeBlobFixture(parentDir, options) +% NDI.TEST.LIGHTSHEET.MAKEBLOBFIXTURE - synthetic lightsheet OME-Zarr with real bytes +% +% [ZARRPATH, GT] = NDI.TEST.LIGHTSHEET.MAKEBLOBFIXTURE() +% [ZARRPATH, GT] = NDI.TEST.LIGHTSHEET.MAKEBLOBFIXTURE(PARENTDIR, ...) +% +% Writes a small OME-Zarr (NGFF v0.4) store on disk carrying a +% deterministic uint16 lightsheet volume, downsampled into a mean + +% max ladder that shares its raw level 0. Unlike the metadata-only +% fixtures the unit tests use, this one writes REAL chunk bytes -- +% uncompressed raw (Zarr v2 with `compressor: null`), which every +% zarr reader accepts -- so an end-to-end run can ingest it with +% ndi.fun.doc.lightsheet.fromOMEZarr, materialize chunks onto +% lightsheetZarrLevel documents, and be viewed by the napari client +% that reads them back. +% +% The store is multi-channel by default: axes 'c,z,y,x' with 2 +% channels labelled 'Ch1' and 'Ch2', chunks 1 x (spatial) so a +% reader can address one channel per chunk. Napari can be told to +% colour each channel independently via the OMERO metadata this +% writer emits (Ch1 -> green, Ch2 -> magenta by default, extending +% to the palette the client keeps in sync with). +% +% Every channel carries the same three features at three spatial +% scales, with per-channel offsets so they read differently in each +% colour: +% - a centred RING pattern: Gaussian envelope (sigma Shape/8) +% modulated by a radial cosine of wavelength Shape/5, so +% brightness oscillates in concentric shells around Shape/2. +% Same for every channel -- makes a natural registration mark. +% - a medium Gaussian at (2/3, 1/3, 1/2)*Shape in Ch1 and the +% mirrored (1/3, 2/3, 1/2)*Shape in Ch2, sigma Shape/20, +% peak ~30k -- a compact companion that stays a small ball at +% every level. Positions rotate around the volume when more +% channels are requested. +% - a bright single-voxel spike of value 60k, placed at Shape/6 +% (Ch1) and Shape*5/6 (Ch2). Positions distribute along the +% diagonal when more channels are requested. +% The ring envelope is 3*sigma ~ 3/8 of a side, so the feature +% stays well inside the box. The spike is much brighter than the +% local Gaussian background and fills one voxel out of the whole +% volume, so max downsampling holds it up while mean spreads it +% over its block -- the mean-vs-max choice is visible at every +% pyramid level. +% +% PARENTDIR - char, directory to write into. Empty (default) puts +% the store under a fresh tempname; the caller can pass +% a stable path (e.g. under an S3-synced dir) to keep +% the fixture between runs. The store lives at +% fullfile(PARENTDIR, 'blob.ome.zarr'). +% +% Name-Value: +% Shape (1,3) double - [Z Y X] spatial shape, default +% [300 300 300]. Channels are added on top +% (see NumChannels). +% NumChannels (1,1) - number of channels (default 2). Set 1 to +% fall back to the old 3-D 'z,y,x' layout. +% ChannelNames - cellstr / string array, one label per +% channel. Empty (default) produces 'Ch1', +% 'Ch2', ... Written into the OMERO block +% of .zattrs so napari picks them up. +% NumLevels (1,1) - number of pyramid levels (default 4) +% ChunkShape (1,3) - spatial chunk shape (default [64 64 64]). +% For a multi-channel store the on-disk +% chunk shape is [1, ChunkShape(:)'], so +% one chunk holds one channel. +% VoxelSize (1,3) - level 0 voxel size (default [1 1 1] um). +% Every subsequent level scales by 2x on +% the spatial axes. +% Seed (1,1) - reserved for future noisy fixtures; the +% default fixture is deterministic without +% it (default 42). +% +% ZARRPATH - char, the absolute path to /blob.ome.zarr +% GT - struct with fields: +% shape - level-0 [Z Y X] +% numChannels - number of channels written +% channelNames - cellstr of channel labels +% chunkShape - spatial chunk shape +% voxelSize - level-0 voxel size +% numLevels - the ladder depth +% levels - 1xN struct(path, shape) for each level +% across mean and max (shape is the +% on-disk shape, i.e. includes the c dim) +% volumes - struct with fields: +% level0 - the raw uint16 volume, +% shape (C, Z, Y, X) when +% NumChannels > 1 +% mean - 1xN cell of mean downsamples +% max - 1xN cell of max downsamples +% +% Example -- end to end round-trip on the local filesystem: +% [zp, gt] = ndi.test.lightsheet.makeBlobFixture(pwd); +% S = ndi.session.dir('demo', tempname); +% sub = ndi.document('subject', 'base.session_id', S.id(), ... +% 'subject.local_identifier', 'blob@vhlab'); +% S.database_add(sub); +% [pyr, lds] = ndi.fun.doc.lightsheet.fromOMEZarr(S, zp, ... +% 'subjectID', sub.id()); +% ndi.fun.doc.lightsheet.view(S, pyr.id()); +% +% See also: ndi.fun.doc.lightsheet.fromOMEZarr, +% ndi.fun.doc.lightsheet.makePyramid, +% ndr.format.omezarr.listPyramids + + arguments + parentDir char = '' + options.Shape (1,3) double {mustBePositive, mustBeInteger} = [300 300 300] + options.NumChannels (1,1) double {mustBePositive, mustBeInteger} = 2 + options.ChannelNames = {} + options.NumLevels (1,1) double {mustBePositive, mustBeInteger} = 4 + options.ChunkShape (1,3) double {mustBePositive, mustBeInteger} = [64 64 64] + options.VoxelSize (1,3) double {mustBePositive} = [1 1 1] + options.Seed (1,1) double = 42 + end + + if isempty(parentDir) + parentDir = tempname; + mkdir(parentDir); + elseif ~isfolder(parentDir) + mkdir(parentDir); + end + zarrPath = fullfile(parentDir, 'blob.ome.zarr'); + if isfolder(zarrPath) + rmdir(zarrPath, 's'); + end + mkdir(zarrPath); + + channelNames = resolveChannelNames(options.ChannelNames, options.NumChannels); + + hasChannels = options.NumChannels > 1 || ~isempty(options.ChannelNames); + if hasChannels + onDiskChunkShape = [1, options.ChunkShape]; + else + onDiskChunkShape = options.ChunkShape; + end + + % Build level 0 and the two reduction ladders. Multi-channel path + % keeps channels as a leading axis; single-channel path stays 3-D + % so it stays compatible with anything reading the old layout. + lvl0 = buildBlobVolume(options.Shape, options.NumChannels, hasChannels); + meanLadder = cell(1, options.NumLevels); + maxLadder = cell(1, options.NumLevels); + meanLadder{1} = lvl0; + maxLadder{1} = lvl0; + for L = 2:options.NumLevels + meanLadder{L} = downsampleSpatial(meanLadder{L-1}, 2, @meanReduce, hasChannels); + maxLadder{L} = downsampleSpatial(maxLadder{L-1}, 2, @maxReduce, hasChannels); + end + + % Every Zarr v2 group node needs a .zgroup marker, or readers + % refuse the store with "No group found". The root is one, and so + % are the mean/ and max/ subdirectories that hold the coarser + % levels for each reduction. Written before the array levels so + % the reader sees a valid group tree. + writeZGroup(zarrPath); + if options.NumLevels > 1 + mkdir(fullfile(zarrPath, 'mean')); + mkdir(fullfile(zarrPath, 'max')); + writeZGroup(fullfile(zarrPath, 'mean')); + writeZGroup(fullfile(zarrPath, 'max')); + end + + % Write level 0 (shared) and each reduction's coarser levels. + writeZarrLevel(fullfile(zarrPath, '0'), lvl0, onDiskChunkShape); + for L = 2:options.NumLevels + writeZarrLevel(fullfile(zarrPath, 'mean', sprintf('%d', L-1)), ... + meanLadder{L}, onDiskChunkShape); + writeZarrLevel(fullfile(zarrPath, 'max', sprintf('%d', L-1)), ... + maxLadder{L}, onDiskChunkShape); + end + + % .zattrs axes. The spatial 2x downsampling only applies to + % spatial axes; the channel axis carries no scale. + if hasChannels + ax = {struct('name','c','type','channel'), ... + struct('name','z','type','space','unit','micrometer'), ... + struct('name','y','type','space','unit','micrometer'), ... + struct('name','x','type','space','unit','micrometer')}; + scaleForLevel = @(L) [1, options.VoxelSize * 2^(L-1)]; + else + ax = {struct('name','z','type','space','unit','micrometer'), ... + struct('name','y','type','space','unit','micrometer'), ... + struct('name','x','type','space','unit','micrometer')}; + scaleForLevel = @(L) options.VoxelSize * 2^(L-1); + end + + meanDatasets = cell(1, options.NumLevels); + maxDatasets = cell(1, options.NumLevels); + for L = 1:options.NumLevels + scale = scaleForLevel(L); + if L == 1 + meanDatasets{L} = zarrDataset('0', scale); + maxDatasets{L} = zarrDataset('0', scale); + else + meanDatasets{L} = zarrDataset(sprintf('mean/%d', L-1), scale); + maxDatasets{L} = zarrDataset(sprintf('max/%d', L-1), scale); + end + end + multiscales = { ... + struct('name','mean','type','box', ... + 'axes', {ax}, 'datasets', {meanDatasets}), ... + struct('name','max','type','max', ... + 'axes', {ax}, 'datasets', {maxDatasets}) }; + + zattrs = struct('multiscales', {multiscales}); + if hasChannels + zattrs.omero = struct('channels', {buildOmeroChannels(channelNames)}); + end + writeJSON(fullfile(zarrPath, '.zattrs'), zattrs); + + % Level manifest for the ground truth. + levels = struct('path', {}, 'shape', {}); + levels(end+1) = struct('path', '0', 'shape', size(lvl0)); + for L = 2:options.NumLevels + levels(end+1) = struct('path', sprintf('mean/%d', L-1), ... + 'shape', size(meanLadder{L})); %#ok + levels(end+1) = struct('path', sprintf('max/%d', L-1), ... + 'shape', size(maxLadder{L})); %#ok + end + + gt = struct( ... + 'shape', options.Shape, ... + 'numChannels', options.NumChannels, ... + 'channelNames', {channelNames}, ... + 'chunkShape', options.ChunkShape, ... + 'voxelSize', options.VoxelSize, ... + 'numLevels', options.NumLevels, ... + 'levels', levels, ... + 'volumes', struct('level0', lvl0, ... + 'mean', {meanLadder}, ... + 'max', {maxLadder})); +end + +% ===================================================================== + +function names = resolveChannelNames(userNames, numChannels) + if isempty(userNames) + names = arrayfun(@(k) sprintf('Ch%d', k), 1:numChannels, ... + 'UniformOutput', false); + return; + end + if isstring(userNames) + userNames = cellstr(userNames); + end + if ~iscellstr(userNames) + error('NDI:lightsheet:makeBlobFixture:badChannelNames', ... + 'ChannelNames must be a cellstr or string array.'); + end + if numel(userNames) ~= numChannels + error('NDI:lightsheet:makeBlobFixture:channelCountMismatch', ... + 'ChannelNames has %d entries but NumChannels is %d.', ... + numel(userNames), numChannels); + end + names = reshape(userNames, 1, []); +end + +function vol = buildBlobVolume(shape, numChannels, hasChannels) +% Deterministic uint16 volume with three features at three spatial +% scales. Multi-channel path returns (C, Z, Y, X) with per-channel +% variations in the medium blob and spike positions so channels read +% differently in each napari colour. Single-channel path returns +% (Z, Y, X) that matches the pre-channel fixture byte-for-byte. + if ~hasChannels + vol = oneChannelVolume(shape, 1, 1); + return; + end + vol = zeros([numChannels, shape], 'uint16'); + for ch = 1:numChannels + one = oneChannelVolume(shape, ch, numChannels); + vol(ch, :, :, :) = reshape(one, [1, shape]); + end +end + +function vol = oneChannelVolume(shape, ch, numChannels) + Z = shape(1); Y = shape(2); X = shape(3); + [zz, yy, xx] = ndgrid(1:Z, 1:Y, 1:X); + sigmaWide = mean(shape) / 8; + sigmaMedium = mean(shape) / 20; + lambdaWide = mean(shape) / 5; + + % Wide ring pattern -- IDENTICAL across channels, so a viewer sees + % it as a coincidence mark. + c1 = shape / 2; + r1 = sqrt((zz - c1(1)).^2 + (yy - c1(2)).^2 + (xx - c1(3)).^2); + envelope = exp(-r1.^2 / (2 * sigmaWide^2)); + rings = 0.5 + 0.5 * cos(2 * pi * r1 / lambdaWide); + wide = envelope .* (25000 + 25000 * rings); + + % Medium blob position rotates around the volume so each channel + % sits somewhere different. Ch1 lands at the pre-channel default + % (2/3, 1/3, 1/2) for backward-compat, Ch2 mirrors it, N>2 fills + % in between. + if numChannels <= 1 + alpha = 1; + else + alpha = 1 - (ch - 1) / (numChannels - 1); % 1, ..., 0 + end + c2 = [shape(1) * lerp(1/3, 2/3, alpha), ... + shape(2) * lerp(2/3, 1/3, alpha), ... + shape(3) / 2]; + r2sq = (zz - c2(1)).^2 + (yy - c2(2)).^2 + (xx - c2(3)).^2; + medium = 30000 * exp(-r2sq / (2 * sigmaMedium^2)); + + % Spike positions spaced along the volume diagonal: Ch1 at Shape/6 + % (matches the pre-channel default), Ch2 at Shape*5/6, N>2 + % interpolates between them so no two spikes overlap. + if numChannels <= 1 + spikeFrac = 1/6; + else + spikeFrac = lerp(1/6, 5/6, (ch - 1) / (numChannels - 1)); + end + spikePos = max(1, min(shape, round(shape * spikeFrac))); + spike = zeros(shape); + spike(spikePos(1), spikePos(2), spikePos(3)) = 60000; + + vol = uint16(min(65535, wide + medium + spike)); +end + +function y = lerp(a, b, t) + y = a + (b - a) * t; +end + +function out = downsampleSpatial(vol, factor, reducer, hasChannels) +% Downsample by FACTOR on spatial axes only. When HASCHANNELS is +% true, the leading axis is the channel dim and is preserved +% verbatim; otherwise we fall back to the pre-channel 3-D reducer. + if ~hasChannels + out = downsampleBlockUint16(vol, factor, reducer); + return; + end + C = size(vol, 1); + sz3 = size(vol); + sz3 = sz3(2:end); + outShape3 = ceil(sz3 / factor); + out = zeros([C, outShape3], 'like', vol); + for c = 1:C + one = reshape(vol(c, :, :, :), sz3); + down = downsampleBlockUint16(one, factor, reducer); + out(c, :, :, :) = reshape(down, [1, size(down)]); + end +end + +function out = downsampleBlockUint16(vol, factor, reducer) +% Reduce each factor x factor x factor block into one voxel using +% REDUCER (a function handle). Output shape is ceil(size / factor); +% trailing edge blocks are smaller. Mirrors what OME-Zarr writers +% commonly do (bioformats2raw, make_zarr_levels.py). + sz = size(vol); + out_shape = ceil(sz / factor); + out = zeros(out_shape, 'like', vol); + for zi = 1:out_shape(1) + z0 = (zi - 1) * factor + 1; + z1 = min(z0 + factor - 1, sz(1)); + for yi = 1:out_shape(2) + y0 = (yi - 1) * factor + 1; + y1 = min(y0 + factor - 1, sz(2)); + for xi = 1:out_shape(3) + x0 = (xi - 1) * factor + 1; + x1 = min(x0 + factor - 1, sz(3)); + block = vol(z0:z1, y0:y1, x0:x1); + out(zi, yi, xi) = reducer(block); + end + end + end +end + +function v = meanReduce(block) + v = uint16(round(mean(double(block(:))))); +end + +function v = maxReduce(block) + v = max(block(:)); +end + +function writeZarrLevel(arrayDir, vol, chunkShape) +% Write one Zarr v2 array: a .zarray metadata file plus one raw chunk +% file per non-empty tile. Handles both 3-D (z,y,x) and 4-D (c,z,y,x) +% layouts; chunkShape must match the array's dimensionality. + if ~isfolder(arrayDir), mkdir(arrayDir); end + shape = size(vol); + if numel(shape) ~= numel(chunkShape) + error('NDI:lightsheet:makeBlobFixture:chunkArity', ... + 'chunkShape length %d does not match array ndims %d.', ... + numel(chunkShape), numel(shape)); + end + writeJSON(fullfile(arrayDir, '.zarray'), struct( ... + 'zarr_format', 2, ... + 'shape', shape, ... + 'chunks', chunkShape, ... + 'dtype', ' numel(shape) + lo = zeros(1, numel(shape)); + hi = zeros(1, numel(shape)); + for a = 1:numel(shape) + lo(a) = (key(a) - 1) * chunkShape(a) + 1; + hi(a) = min(lo(a) + chunkShape(a) - 1, shape(a)); + subs{a} = lo(a):hi(a); + end + % Pad edge tiles with zeros so on-disk chunks are all the same + % size (Zarr v2 requires this). + chunk = zeros(chunkShape, 'like', vol); + localSubs = arrayfun(@(a) 1:(hi(a) - lo(a) + 1), 1:numel(shape), ... + 'UniformOutput', false); + chunk(localSubs{:}) = vol(subs{:}); + % Zarr expects C-order bytes. MATLAB is column-major, so + % permute the axes into reversed order before linearising. + permuted = permute(chunk, numel(shape):-1:1); + raw = typecast(permuted(:), 'uint8'); + + keyStr = strjoin(arrayfun(@(k) num2str(k - 1), key, ... + 'UniformOutput', false), '.'); + fid = fopen(fullfile(arrayDir, keyStr), 'w'); + fwrite(fid, raw); + fclose(fid); + return; + end + for c = 1:nChunks(dim) + writeChunks(arrayDir, vol, shape, chunkShape, nChunks, subs, ... + [key c], dim + 1); + end +end + +function d = zarrDataset(pathStr, scale) + d = struct('path', pathStr, ... + 'coordinateTransformations', {{ ... + struct('type', 'scale', 'scale', scale) }}); +end + +function channels = buildOmeroChannels(channelNames) +% OMERO 'channels' block that napari-ome-zarr reads to pick up +% per-channel names + display colours. Palette matches the client's +% default (green, magenta, ...). Windows are wide-open so napari's +% auto-contrast still runs. + palette = {'00FF00', 'FF00FF', '00FFFF', 'FFFF00', ... + '0080FF', 'FF8000', '808080'}; + n = numel(channelNames); + channels = cell(1, n); + for i = 1:n + color = palette{mod(i - 1, numel(palette)) + 1}; + channels{i} = struct( ... + 'label', channelNames{i}, ... + 'color', color, ... + 'active', true, ... + 'window', struct('start', 0, 'end', 65535, ... + 'min', 0, 'max', 65535)); + end +end + +function writeZGroup(dir) + writeJSON(fullfile(dir, '.zgroup'), struct('zarr_format', 2)); +end + +function writeJSON(filePath, s) + % MATLAB's jsonencode emits empty [] for MATLAB `[]`, but Zarr v2 + % expects null for `compressor` and `filters` -- a `[]` there is + % refused by the zarr-python 3 reader with "Expected None, a + % numcodecs.abc.Codec, or a dict ... Got instead." + % Post-process the string so those two fields become JSON null. + txt = jsonencode(s); + txt = regexprep(txt, '"compressor":\[\]', '"compressor":null'); + txt = regexprep(txt, '"filters":\[\]', '"filters":null'); + fid = fopen(filePath, 'w'); + fwrite(fid, txt); + fclose(fid); +end diff --git a/src/ndi/+ndi/+v2/anyOf.m b/src/ndi/+ndi/+v2/anyOf.m new file mode 100644 index 0000000000..f9fd703848 --- /dev/null +++ b/src/ndi/+ndi/+v2/anyOf.m @@ -0,0 +1,16 @@ +function q = anyOf(qs) +%ANYOF The OR of the ndi.query objects in cell array QS, as a balanced tree. +% +% Q = ndi.v2.anyOf(QS) nests log2(N) deep rather than N, which keeps the +% SQL and the in-memory recheck shallow for a few hundred ids. Callers +% keep N to about 200 per search. + +while numel(qs) > 1 + n = floor(numel(qs) / 2); + nx = cell(1, ceil(numel(qs) / 2)); + for i = 1:n, nx{i} = qs{2*i - 1} | qs{2*i}; end + if mod(numel(qs), 2), nx{end} = qs{end}; end + qs = nx; +end +q = qs{1}; +end diff --git a/src/ndi/+ndi/+v2/axesTable.m b/src/ndi/+ndi/+v2/axesTable.m new file mode 100644 index 0000000000..03ed5ab29f --- /dev/null +++ b/src/ndi/+ndi/+v2/axesTable.m @@ -0,0 +1,42 @@ +function T = axesTable(keys) +%AXESTABLE A value's keys (its axes) as a table. +% +% T = ndi.v2.axesTable(KEYS), KEYS a data.keys struct array: one row per +% key with variable, unit, n and coordinates (a cell: the numeric +% positions, origin + spacing * (0:n-1) for a regular key, or the labels' +% names). + +keys = ndi.v2.entries(keys); +variable = strings(0, 1); unit = strings(0, 1); n = zeros(0, 1); coordinates = cell(0, 1); +for k = 1:numel(keys) + key = keys{k}; + variable(end+1, 1) = string(ndi.v2.termName(field(key, 'variable', ''))); %#ok + unit(end+1, 1) = string(ndi.v2.termName(field(key, 'unit', ''))); %#ok + nk = double(field(key, 'n', NaN)); + if isempty(nk), nk = NaN; end + n(end+1, 1) = nk; %#ok + c = []; + if logical(field(key, 'regular', false)) && ~isnan(nk) + o = field(field(key, 'origin', struct()), 'value', 0); + s = field(field(key, 'spacing', struct()), 'value', 1); + c = double(o) + double(s) * (0:nk-1)'; + elseif ~isempty(field(key, 'values', [])) + c = field(field(key, 'values', struct()), 'values', []); + c = c(:); + elseif ~isempty(field(key, 'labels', [])) + l = field(key, 'labels', []); + l = ndi.v2.entries(l); + c = strings(numel(l), 1); + for j = 1:numel(l), c(j) = string(ndi.v2.termName(l{j})); end + end + coordinates{end+1, 1} = c; %#ok +end +T = table(variable, unit, n, coordinates); +end + +function v = field(s, name, default) +v = default; +if isstruct(s) && isfield(s, name) && ~isempty(s(1).(name)) + v = s(1).(name); +end +end diff --git a/src/ndi/+ndi/+v2/blockOf.m b/src/ndi/+ndi/+v2/blockOf.m new file mode 100644 index 0000000000..36bd404c94 --- /dev/null +++ b/src/ndi/+ndi/+v2/blockOf.m @@ -0,0 +1,17 @@ +function v = blockOf(p, block, field, default) +%BLOCKOF P.(BLOCK).(FIELD), or DEFAULT when either is absent. + +if nargin < 4, default = []; end +v = default; +% a block renamed 2026-10-08 is read under either name (ndi.v2.vetaAliases) +names = ndi.v2.vetaAliases(block); +if ~isempty(ndi.v2.entityTypesFor(block)) + names{end+1} = 'entity'; % merged into `entity` on 2026-10-08 +end +for k = 1:numel(names) + if isfield(p, names{k}) && isstruct(p.(names{k})) && isfield(p.(names{k}), field) + v = p.(names{k}).(field); + return; + end +end +end diff --git a/src/ndi/+ndi/+v2/blockQuery.m b/src/ndi/+ndi/+v2/blockQuery.m new file mode 100644 index 0000000000..2a49ec4000 --- /dev/null +++ b/src/ndi/+ndi/+v2/blockQuery.m @@ -0,0 +1,24 @@ +function q = blockQuery(path, op, param1, param2) +%BLOCKQUERY An ndi.query on a block field under every name the block has had. +% +% Q = ndi.v2.blockQuery(PATH, OP, PARAM1, PARAM2) is ndi.query(PATH, OP, +% PARAM1, PARAM2) ORed with the same query on each other name of PATH's +% first part: a block renamed 2026-10-08 (ndi.v2.vetaAliases), and for an +% entity class merged into `entity` that day (ndi.v2.entityTypesFor) the +% `entity` block, so 'subject.local_identifier' also matches +% 'entity.local_identifier'. +% +% See also ndi.v2.blockOf, ndi.v2.isaQuery. + +if nargin < 4, param2 = ''; end +parts = strsplit(char(path), '.'); +names = ndi.v2.vetaAliases(parts{1}); +if ~isempty(ndi.v2.entityTypesFor(parts{1})) + names{end+1} = 'entity'; +end +q = []; +for k = 1:numel(names) + one = ndi.query(strjoin([names(k), parts(2:end)], '.'), op, param1, param2); + if isempty(q), q = one; else, q = q | one; end +end +end diff --git a/src/ndi/+ndi/+v2/classChain.m b/src/ndi/+ndi/+v2/classChain.m new file mode 100644 index 0000000000..b8f9171008 --- /dev/null +++ b/src/ndi/+ndi/+v2/classChain.m @@ -0,0 +1,31 @@ +function chain = classChain(p) +%CLASSCHAIN Every class a document IS: its own and all its superclasses. +% +% CHAIN = ndi.v2.classChain(P) returns a cellstr, the document's class +% first. P is document properties (ndi.v2.props). The chain comes from the +% V2 schema (did2.schema.cache); a class the schema does not know (a v1 +% document) falls back to the superclasses the document itself lists. +% +% See also ndi.v2.directParents, ndi.statement.fromDocument. + +name = char(p.document_class.class_name); +listed = {}; +if isfield(p.document_class, 'superclasses') + sc = p.document_class.superclasses; + if isstruct(sc), sc = num2cell(sc); end + for k = 1:numel(sc) + if isstruct(sc{k}) && isfield(sc{k}, 'class_name') + listed{end+1} = char(sc{k}.class_name); %#ok + end + end +end +try + chain = [{name}, reshape(did2.schema.cache.shared().superclasses(name), 1, [])]; + % a statement's value kind is not in its class's chain: the document + % lists it after (did-schema V_eta_entity_composition_plan.md sec. 1) + chain = ndi.v2.vetaName([chain, listed(~ismember(listed, chain))]); + return; +catch +end +chain = ndi.v2.vetaName([{name}, listed]); % names as of 2026-10-08 (ndi.v2.vetaName) +end diff --git a/src/ndi/+ndi/+v2/closest.m b/src/ndi/+ndi/+v2/closest.m new file mode 100644 index 0000000000..c93ff3c28f --- /dev/null +++ b/src/ndi/+ndi/+v2/closest.m @@ -0,0 +1,40 @@ +function c = closest(word, candidates) +%CLOSEST The candidate nearest to WORD, for a "did you mean", or '' when none is near. +% +% C = ndi.v2.closest(WORD, CANDIDATES) compares WORD with each candidate +% ignoring case, by edit distance (the single-character insertions, +% deletions and substitutions that turn one into the other), and returns +% the nearest one when it is within max(1, min(3, floor(length/4))) edits +% of WORD -- 'stran' -> 'strain', 'Caenorhabdiits elegans' -> +% 'Caenorhabditis elegans' -- and '' otherwise (or when WORD is itself one +% of them). + +c = ''; +candidates = cellstr(candidates); +if isempty(candidates) || isempty(word), return; end +w = lower(char(word)); +if any(strcmpi(candidates, w)), return; end +limit = max(1, min(3, floor(numel(w) / 4))); +best = Inf; +for k = 1:numel(candidates) + d = editDistance(w, lower(candidates{k})); + if d < best + best = d; + c = candidates{k}; + end +end +if best > limit, c = ''; end +end + +function d = editDistance(a, b) +m = numel(a); n = numel(b); +prev = 0:n; +for i = 1:m + cur = [i, zeros(1, n)]; + for j = 1:n + cur(j + 1) = min([prev(j + 1) + 1, cur(j) + 1, prev(j) + (a(i) ~= b(j))]); + end + prev = cur; +end +d = prev(end); +end diff --git a/src/ndi/+ndi/+v2/copyDocuments.m b/src/ndi/+ndi/+v2/copyDocuments.m new file mode 100644 index 0000000000..6b0fd7e1da --- /dev/null +++ b/src/ndi/+ndi/+v2/copyDocuments.m @@ -0,0 +1,84 @@ +function report = copyDocuments(sourceDb, targetDb, docs, options) +%COPYDOCUMENTS Copy V2 documents, and the files they ingested, to another database. +% +% REPORT = ndi.v2.copyDocuments(SOURCEDB, TARGETDB, DOCS) adds DOCS (a cell +% of document property structs, read from SOURCEDB) to TARGETDB. Both are +% did2.database.sqlitedb objects (an ndi session's is its +% database.db). Each file a document INGESTED into SOURCEDB is ingested +% into TARGETDB from SOURCEDB's copy (/files/), which is +% left in place; a file recorded BY LOCATION (`ingest` 0) stays where it is +% and is recorded the same way. Ids, edges and session ids are unchanged. +% +% Nothing is written when a file a document ingested is missing from +% SOURCEDB: the copy would lose it. +% +% REPORT: documents (how many copied), ingested (files copied into +% TARGETDB's store) and byLocation (files still recorded by location). +% +% Options: +% 'BatchSize' documents per write (default 2000) +% 'Validate' default false: the documents were validated when written +% to SOURCEDB +% +% See also ndi.dataset, did2.database.sqlitedb. + +arguments + sourceDb + targetDb + docs cell + options.BatchSize (1,1) double {mustBePositive, mustBeInteger} = 2000 + options.Validate (1,1) logical = false +end + +report = struct('documents', numel(docs), 'ingested', 0, 'byLocation', 0); +missing = {}; +for i = 1:numel(docs) + [docs{i}, nIn, nBy, miss] = fromStore(docs{i}, sourceDb.fileDir); + report.ingested = report.ingested + nIn; + report.byLocation = report.byLocation + nBy; + missing = [missing, miss]; %#ok +end +if ~isempty(missing) + error('ndi:v2:copyDocuments:fileMissing', ... + ['%d ingested file(s) are missing from %s, so nothing was copied (the copy ' ... + 'would lose them):\n %s'], numel(missing), sourceDb.fileDir, ... + strjoin(missing(1:min(end, 20)), '\n ')); +end +for b = 1:options.BatchSize:numel(docs) + part = docs(b:min(b + options.BatchSize - 1, numel(docs))); + targetDb.add(cellfun(@(d) did2.document(d), part, 'UniformOutput', false), ... + 'Validate', options.Validate); +end +end + +% ----------------------------------------------------------------------------- +function [d, nIn, nBy, missing] = fromStore(d, fileDir) +% D with every ingested location pointed at FILEDIR's copy, kept there +nIn = 0; nBy = 0; missing = {}; +if ~isfield(d, 'files') || ~isstruct(d.files) || ~isfield(d.files, 'file_info') ... + || isempty(d.files.file_info) + return; +end +fi = d.files.file_info; +if iscell(fi), fi = [fi{:}]; end +for a = 1:numel(fi) + if ~isfield(fi(a), 'locations'), continue; end + if iscell(fi(a).locations), fi(a).locations = [fi(a).locations{:}]; end + for c = 1:numel(fi(a).locations) + L = fi(a).locations(c); + if isfield(L, 'ingest') && logical(L.ingest) + src = fullfile(fileDir, char(L.uid)); + if ~isfile(src) + missing{end+1} = sprintf('%s of document %s (%s)', char(fi(a).name), ... + d.base.id, src); %#ok + end + fi(a).locations(c).location = src; + fi(a).locations(c).delete_original = 0; + nIn = nIn + 1; + else + nBy = nBy + 1; + end + end +end +d.files.file_info = fi; +end diff --git a/src/ndi/+ndi/+v2/datasetSessions.m b/src/ndi/+ndi/+v2/datasetSessions.m new file mode 100644 index 0000000000..16d9849d9b --- /dev/null +++ b/src/ndi/+ndi/+v2/datasetSessions.m @@ -0,0 +1,193 @@ +function [sessions, notes] = datasetSessions(database, datasetSessionId, datasetPath, options) +%DATASETSESSIONS The sessions of a V2 dataset: ingested and linked. +% +% [SESSIONS, NOTES] = ndi.v2.datasetSessions(DATABASE, DATASETSESSIONID, +% DATASETPATH) reads a V2 dataset's own database (DATABASE, an ndi.database +% searched without a session scope) and returns its sessions, one row of a +% struct array each: +% +% session_id the session's id (its documents' base.session_id) +% reference its local_identifier +% entity_id the id of its `session` entity document +% is_linked false: ingested (its documents are in DATABASE) +% true: linked (they are in the folder a `linked_session` +% document names) +% path a linked session's folder, absolute ('' when ingested) +% link_id the `linked_session` document's id ('' when ingested) +% session the linked session, opened (an ndi.session.dir; [] when +% ingested) +% +% The rule (V_eta_linked_session_plan.md, signed 2026-10-09): a session is +% in the dataset when its entity document is `part_of` the dataset, or +% `part_of` a study that is `part_of` the dataset; the relation is in the +% dataset's database for both kinds. A session is LINKED when a +% `linked_session` document names it: its `path` is the session's folder +% (relative to DATASETPATH when inside it) and its `entity_id` the +% session's entity document, which must be in that folder. +% +% NOTES is a cellstr of what was left out and why, the denominator first: +% - a `session` document in DATABASE that is not part_of the dataset; +% - a linked folder that is missing, cannot be opened, or holds another +% session; +% - a linked session that is not part_of the dataset. +% When DATABASE holds no `dataset` document (an import whose metadata stage +% did not run), membership cannot be read, so every session is listed and +% NOTES says so. +% +% Options: +% 'OpenLinked' default true. false lists linked sessions without opening +% them (session_id and reference are then '' and +% `session` is []), e.g. to find a link whose folder moved. +% +% See also ndi.dataset, ndi.dataset.dir. + +arguments + database + datasetSessionId (1,:) char + datasetPath (1,:) char + options.OpenLinked (1,1) logical = true +end + +sessions = struct('session_id', {}, 'reference', {}, 'entity_id', {}, ... + 'is_linked', {}, 'path', {}, 'link_id', {}, 'session', {}); +notes = {}; + +% --- membership: part_of the dataset, directly or through a study --- +datasetDocs = database.search(ndi.v2.isaQuery('dataset')); +datasetDocs = datasetDocs(cellfun(@(d) strcmp(ndi.v2.kindOf(d.document_properties), ... + 'dataset'), datasetDocs)); +knowMembership = ~isempty(datasetDocs); +% level 1: part_of the dataset (studies, or sessions directly); level 2: +% part_of one of those (the sessions of a study) +level1 = {}; +for k = 1:numel(datasetDocs) + level1 = [level1, partOfChildren(database, datasetDocs{k}.id())]; %#ok +end +level1 = unique(level1); +members = level1; +for k = 1:numel(level1) + members = [members, partOfChildren(database, level1{k})]; %#ok +end +members = unique(members); + +% --- ingested: `session` documents in the dataset's own database --- +sessionDocs = database.search(ndi.v2.isaQuery('session')); +nIngested = 0; +left = {}; +for i = 1:numel(sessionDocs) + p = sessionDocs{i}.document_properties; + if ~strcmp(ndi.v2.kindOf(p), 'session') || strcmp(p.base.session_id, datasetSessionId) + continue; + end + ref = char(ndi.v2.blockOf(p, 'session', 'local_identifier', '')); + if knowMembership && ~ismember(p.base.id, members) + left{end+1} = ref; %#ok + continue; + end + nIngested = nIngested + 1; + sessions(end+1) = struct('session_id', p.base.session_id, 'reference', ref, ... + 'entity_id', p.base.id, 'is_linked', false, 'path', '', 'link_id', '', ... + 'session', []); %#ok +end + +% --- linked: `linked_session` documents --- +links = {}; +try + links = database.search(ndi.query('', 'isa', 'linked_session', '')); +catch + % a schema without the class: no linked sessions +end +nLinked = 0; +problems = {}; +for i = 1:numel(links) + p = links{i}.document_properties; + rel = char(ndi.v2.blockOf(p, 'linked_session', 'path', '')); + e = ndi.v2.edgeIds(p, 'entity_id'); + if isempty(e) + problems{end+1} = sprintf('linked_session %s names no session entity', p.base.id); %#ok + continue; + end + entityId = e{1}; + folder = ndi.v2.resolveLinkPath(rel, datasetPath); + if knowMembership && ~ismember(entityId, members) + problems{end+1} = sprintf('%s (linked) is not part_of the dataset', folder); %#ok + continue; + end + row = struct('session_id', '', 'reference', '', 'entity_id', entityId, ... + 'is_linked', true, 'path', folder, 'link_id', p.base.id, 'session', []); + if options.OpenLinked + [S, sp, why] = openLinked(folder, entityId); + if isempty(S) + problems{end+1} = why; %#ok + continue; + end + row.session = S; + row.session_id = S.id(); + row.reference = char(ndi.v2.blockOf(sp, 'session', 'local_identifier', '')); + end + nLinked = nLinked + 1; + sessions(end+1) = row; %#ok +end + +notes{end+1} = sprintf(['DENOMINATOR: %d session document(s) and %d linked_session ' ... + 'document(s) read; %d ingested and %d linked session(s) listed'], ... + sum(cellfun(@(d) strcmp(ndi.v2.kindOf(d.document_properties), 'session') && ... + ~strcmp(d.document_properties.base.session_id, datasetSessionId), sessionDocs)), ... + numel(links), nIngested, nLinked); +if ~knowMembership + notes{end+1} = ['no dataset document: membership (part_of) cannot be read, so every ' ... + 'session is listed']; +end +if ~isempty(left) + notes{end+1} = sprintf('%d session document(s) not part_of the dataset, not listed: %s', ... + numel(left), strjoin(left, ', ')); +end +for k = 1:numel(problems) + notes{end+1} = ['not listed: ' problems{k}]; %#ok +end +end + +% ----------------------------------------------------------------------------- +function children = partOfChildren(database, parentId) +% the ids of the documents `part_of` PARENTID, read from relations in DATABASE +children = {}; +q = ndi.query('', 'isa', 'directed_relation', '') & ... + ndi.query('', 'depends_on', 'parent_id', parentId); +rels = database.search(q); +for k = 1:numel(rels) + p = rels{k}.document_properties; + if ~strcmp(ndi.v2.termName(ndi.v2.blockOf(p, 'directed_relation', 'relation', '')), 'part_of') + continue; + end + c = ndi.v2.edgeIds(p, 'child_id'); + children = [children, c(:)']; %#ok +end +end + +function [S, sp, why] = openLinked(folder, entityId) +% the session in FOLDER, checked to hold the session entity ENTITYID +S = []; sp = []; why = ''; +if ~isfolder(folder) + why = sprintf('%s (linked): no such folder', folder); + return; +end +try + S0 = ndi.session.dir(folder); +catch err + why = sprintf('%s (linked) cannot be opened: %s', folder, err.message); + return; +end +d = S0.database_search(ndi.query('base.id', 'exact_string', entityId, '')); +if isempty(d) + why = sprintf(['%s (linked) holds another session (%s): its session document %s ' ... + 'is not there'], folder, S0.id(), entityId); + return; +end +sp = d{1}.document_properties; +if ~strcmp(ndi.v2.kindOf(sp), 'session') + why = sprintf('%s (linked): document %s is a %s, not a session', folder, entityId, ... + ndi.v2.kindOf(sp)); + return; +end +S = S0; +end diff --git a/src/ndi/+ndi/+v2/directParents.m b/src/ndi/+ndi/+v2/directParents.m new file mode 100644 index 0000000000..d2e855d194 --- /dev/null +++ b/src/ndi/+ndi/+v2/directParents.m @@ -0,0 +1,35 @@ +function parents = directParents(className) +%DIRECTPARENTS The classes CLASSNAME names as its own superclasses (V2 schema). +% +% PARENTS = ndi.v2.directParents(CLASSNAME) returns a cellstr; empty when +% the schema does not know the class. + +parents = {}; +try + s = did2.schema.cache.shared().getClass(className); +catch + s = []; +end +if isempty(s) + % a class renamed 2026-10-08, asked for by the name the schema in use lacks + alias = setdiff(ndi.v2.vetaAliases(className), {char(className)}); + try + if ~isempty(alias), s = did2.schema.cache.shared().getClass(alias{1}); end + catch + end +end +if isempty(s) + return; +end +if ~isfield(s, 'document_class') || ~isfield(s.document_class, 'superclasses') + return; +end +sc = s.document_class.superclasses; +if isstruct(sc), sc = num2cell(sc); end +for k = 1:numel(sc) + if isstruct(sc{k}) && isfield(sc{k}, 'class_name') + parents{end+1} = char(sc{k}.class_name); %#ok + end +end +parents = ndi.v2.vetaName(parents); % names as of 2026-10-08 +end diff --git a/src/ndi/+ndi/+v2/edgeIds.m b/src/ndi/+ndi/+v2/edgeIds.m new file mode 100644 index 0000000000..d2a9bfbf1f --- /dev/null +++ b/src/ndi/+ndi/+v2/edgeIds.m @@ -0,0 +1,31 @@ +function ids = edgeIds(p, name) +%EDGEIDS The document ids a document's NAME edge points at. +% +% IDS = ndi.v2.edgeIds(P, NAME) returns a cellstr (row), in order: every +% depends_on entry called NAME (a V2 edge may repeat its name) or NAME_ +% (the v1 numbered form). Empty values are left out. A V2 edge holds the +% id in `document_id`, a v1 edge in `value`. + +ids = {}; +if ~isfield(p, 'depends_on') || isempty(p.depends_on) + return; +end +dep = p.depends_on; +if isstruct(dep), dep = num2cell(dep); end +for k = 1:numel(dep) + e = dep{k}; + if ~isfield(e, 'name'), continue; end + n = char(e.name); + if ~(strcmp(n, name) || ~isempty(regexp(n, ['^' regexptranslate('escape', name) '_\d+$'], 'once'))) + continue; + end + v = ''; + if isfield(e, 'document_id'), v = e.document_id; elseif isfield(e, 'value'), v = e.value; end + v = char(v); + if ~isempty(v), ids{end+1} = v; end %#ok +end +if isempty(ids) && strcmp(name, 'entity_id') + % a statement written before 2026-10-08 names what it is about `subject_id` + ids = ndi.v2.edgeIds(p, 'subject_id'); +end +end diff --git a/src/ndi/+ndi/+v2/entityQuery.m b/src/ndi/+ndi/+v2/entityQuery.m new file mode 100644 index 0000000000..2b918eb7ab --- /dev/null +++ b/src/ndi/+ndi/+v2/entityQuery.m @@ -0,0 +1,10 @@ +function q = entityQuery(id) +%ENTITYQUERY Statements about the entity ID: its `entity_id` (or, written before 2026-10-08, `subject_id`) edge. +% +% Q = ndi.v2.entityQuery(ID) is ndi.query('', 'depends_on', 'entity_id', ID) +% OR the same for `subject_id`, the edge's name until 2026-10-08. +% +% See also ndi.v2.isaQuery, ndi.v2.vetaAliases. + +q = ndi.query('', 'depends_on', 'entity_id', id) | ndi.query('', 'depends_on', 'subject_id', id); +end diff --git a/src/ndi/+ndi/+v2/entityTypesFor.m b/src/ndi/+ndi/+v2/entityTypesFor.m new file mode 100644 index 0000000000..683f29df00 --- /dev/null +++ b/src/ndi/+ndi/+v2/entityTypesFor.m @@ -0,0 +1,22 @@ +function types = entityTypesFor(className) +%ENTITYTYPESFOR The `entity.type` values a pre-2026-10-08 entity class became. +% +% TYPES = ndi.v2.entityTypesFor(CLASSNAME). On 2026-10-08 subject, strain, +% product, software, person, organization, funding, web_resource, dataset, +% study, publication, session and epoch merged into one `entity` class with a +% bound `type` (did-schema V_eta_entity_composition_plan.md sec. 4). A +% `subject` became one of the seven physical kinds, each other class the type +% of its own name. TYPES is {} for any other name. +% +% See also ndi.v2.isaQuery, ndi.v2.blockOf. + +switch char(className) + case 'subject' + types = {'organism', 'culture', 'tissue', 'cell', 'group', 'device', 'material'}; + case {'strain', 'product', 'software', 'person', 'organization', 'funding', ... + 'web_resource', 'dataset', 'study', 'publication', 'session', 'epoch', 'protocol'} + types = {char(className)}; + otherwise + types = {}; +end +end diff --git a/src/ndi/+ndi/+v2/entries.m b/src/ndi/+ndi/+v2/entries.m new file mode 100644 index 0000000000..f60ead7f35 --- /dev/null +++ b/src/ndi/+ndi/+v2/entries.m @@ -0,0 +1,16 @@ +function c = entries(x) +%ENTRIES A decoded JSON list as a cell array (row), one element per entry. +% +% C = ndi.v2.entries(X). jsondecode gives a struct array when a list's +% entries share their fields and a cell array when they do not (a term +% parameter beside a numeric one), so a list is walked as cells and never +% concatenated with [x{:}]. Empty gives {}. + +if isempty(x) + c = {}; +elseif iscell(x) + c = reshape(x, 1, []); +else + c = num2cell(reshape(x, 1, [])); +end +end diff --git a/src/ndi/+ndi/+v2/fieldQuery.m b/src/ndi/+ndi/+v2/fieldQuery.m new file mode 100644 index 0000000000..df5374981e --- /dev/null +++ b/src/ndi/+ndi/+v2/fieldQuery.m @@ -0,0 +1,18 @@ +function q = fieldQuery(path, op, value) +%FIELDQUERY A field query on a V_eta block under its current or its old name. +% +% Q = ndi.v2.fieldQuery('statement.variable.name', 'exact_string', V) is the +% ndi.query on that path OR on 'subject_statement.variable.name', the +% block's name until 2026-10-08 (ndi.v2.vetaAliases on the first segment). +% +% See also ndi.v2.termQuery, ndi.v2.isaQuery. + +parts = strsplit(char(path), '.'); +names = ndi.v2.vetaAliases(parts{1}); +q = []; +for k = 1:numel(names) + p = strjoin([names(k), parts(2:end)], '.'); + one = ndi.query(p, op, value, ''); + if isempty(q), q = one; else, q = q | one; end +end +end diff --git a/src/ndi/+ndi/+v2/getDocument.m b/src/ndi/+ndi/+v2/getDocument.m new file mode 100644 index 0000000000..e62e25b11a --- /dev/null +++ b/src/ndi/+ndi/+v2/getDocument.m @@ -0,0 +1,14 @@ +function doc = getDocument(container, id) +%GETDOCUMENT The ndi.document with base.id ID in CONTAINER, or [] when absent. +% +% DOC = ndi.v2.getDocument(CONTAINER, ID); CONTAINER is an ndi.session or +% ndi.dataset. A session searches only its own documents: a dataset-level +% document (a person, a study) is found from the dataset. + +doc = []; +if isempty(id), return; end +d = container.database_search(ndi.query('base.id', 'exact_string', char(id), '')); +if ~isempty(d) + doc = d{1}; +end +end diff --git a/src/ndi/+ndi/+v2/getDocuments.m b/src/ndi/+ndi/+v2/getDocuments.m new file mode 100644 index 0000000000..0dbe6eab07 --- /dev/null +++ b/src/ndi/+ndi/+v2/getDocuments.m @@ -0,0 +1,41 @@ +function docs = getDocuments(container, ids, cache) +%GETDOCUMENTS The documents with these ids, a few searches in all. +% +% DOCS = ndi.v2.getDocuments(CONTAINER, IDS) returns a containers.Map, +% id -> ndi.document, for the ids found in CONTAINER (an ndi.session or +% ndi.dataset); 200 ids per search. With a CACHE (a containers.Map), the +% documents found are also stored in it under 'doc:', and ids already +% there are not searched again. +% +% See also ndi.v2.getDocument, ndi.v2.timesOf. + +if nargin < 3, cache = []; end +docs = containers.Map('KeyType', 'char', 'ValueType', 'any'); +ids = unique(cellstr(ids), 'stable'); +todo = {}; +for k = 1:numel(ids) + if isa(cache, 'containers.Map') && isKey(cache, ['doc:' ids{k}]) + docs(ids{k}) = cache(['doc:' ids{k}]); + elseif ~isempty(ids{k}) + todo{end+1} = ids{k}; %#ok + end +end +for c = 1:200:numel(todo) + part = todo(c:min(c + 199, numel(todo))); + qs = cellfun(@(i) ndi.query('base.id', 'exact_string', i, ''), part, 'UniformOutput', false); + while numel(qs) > 1 % a balanced OR: nested log2(N) deep + n = floor(numel(qs) / 2); + nx = cell(1, ceil(numel(qs) / 2)); + for i = 1:n, nx{i} = qs{2*i - 1} | qs{2*i}; end + if mod(numel(qs), 2), nx{end} = qs{end}; end + qs = nx; + end + found = container.database_search(qs{1}); + for i = 1:numel(found) + p = ndi.v2.props(found{i}); + id = char(p.base.id); + docs(id) = found{i}; + if isa(cache, 'containers.Map'), cache(['doc:' id]) = found{i}; end + end +end +end diff --git a/src/ndi/+ndi/+v2/hasWildcard.m b/src/ndi/+ndi/+v2/hasWildcard.m new file mode 100644 index 0000000000..afc003b8f6 --- /dev/null +++ b/src/ndi/+ndi/+v2/hasWildcard.m @@ -0,0 +1,7 @@ +function tf = hasWildcard(pattern) +%HASWILDCARD Does PATTERN carry a '*' that is not escaped as '\*'. +% +% See also ndi.v2.matchTerm. + +tf = ischar(pattern) && ~isempty(regexp(pattern, '(?_' +% ('temperature_observation'), BLOCK the direction; +% - an entity: 'subject' for the physical types, else its type ('session', +% 'epoch', 'strain'), BLOCK 'entity'; +% - anything else: its own class, BLOCK the same. +% +% See also ndi.v2.kindOf, ndi.v2.isaQuery. + +name = char(p.document_class.class_name); +block = name; +directions = {'assertion', 'observation', 'manipulation', 'calculation'}; +if any(strcmp(name, directions)) + supers = p.document_class.superclasses; + if iscell(supers), supers = [supers{:}]; end + names = arrayfun(@(s) char(s.class_name), supers, 'UniformOutput', false); + k = find(strcmp(names, 'base'), 1); + if ~isempty(k) && k < numel(names) && ~strcmp(names{k + 1}, 'value') + name = [names{k + 1} '_' name]; + end +elseif strcmp(name, 'entity') + kind = ndi.v2.kindOf(p); + if any(strcmp(kind, ndi.v2.entityTypesFor('subject'))) + name = 'subject'; + else + name = kind; + end +end +end diff --git a/src/ndi/+ndi/+v2/linkPath.m b/src/ndi/+ndi/+v2/linkPath.m new file mode 100644 index 0000000000..2c7d4655ce --- /dev/null +++ b/src/ndi/+ndi/+v2/linkPath.m @@ -0,0 +1,23 @@ +function path = linkPath(folder, datasetPath) +%LINKPATH The `path` a linked_session document records for FOLDER. +% +% PATH = ndi.v2.linkPath(FOLDER, DATASETPATH): FOLDER relative to the +% dataset's folder DATASETPATH when FOLDER is inside it (written with '/', +% so it reads the same on any system), else FOLDER absolute +% (V_eta_linked_session_plan.md). A dataset moved with its sessions inside +% it then still finds them. +% +% See also ndi.v2.resolveLinkPath. + +arguments + folder (1,:) char + datasetPath (1,:) char +end +f = char(java.io.File(folder).getCanonicalPath()); +d = char(java.io.File(datasetPath).getCanonicalPath()); +if startsWith(f, [d filesep]) + path = strrep(f(numel(d)+2:end), filesep, '/'); +else + path = f; +end +end diff --git a/src/ndi/+ndi/+v2/matchTerm.m b/src/ndi/+ndi/+v2/matchTerm.m new file mode 100644 index 0000000000..b16034d491 --- /dev/null +++ b/src/ndi/+ndi/+v2/matchTerm.m @@ -0,0 +1,51 @@ +function tf = matchTerm(t, pattern) +%MATCHTERM Does the term T match PATTERN. +% +% TF = ndi.v2.matchTerm(T, PATTERN): T is a term ({name, node}) or text +% (a name). PATTERN matches the name ignoring case or the node exactly; +% '*' in it is a wildcard (any run of characters), '\*' a literal star. +% A cell array PATTERN matches when any of its patterns does. A node's +% CURIE prefix ignores case ('NCBITaxon:6239' is 'ncbitaxon:6239'): +% CURIE_lookups_meta.json, "Prefixes are matched case-insensitively". +% The local part stays exact. +% +% See also ndi.v2.hasWildcard, ndi.entity.search. + +if iscell(pattern) + tf = any(cellfun(@(x) ndi.v2.matchTerm(t, x), pattern)); + return; +end +name = ''; node = ''; +if iscell(t) && ~isempty(t), t = t{1}; end +if isstruct(t) && ~isempty(t) + t = t(1); + if isfield(t, 'name'), name = char(t.name); end + if isfield(t, 'node'), node = char(t.node); end +elseif ischar(t) || isstring(t) + name = char(t); +end +pattern = char(pattern); +node = lowerPrefix(node); +if ~ndi.v2.hasWildcard(pattern) + pattern = strrep(pattern, '\*', '*'); + tf = strcmpi(name, pattern) || (~isempty(node) && strcmp(node, lowerPrefix(pattern))); + return; +end +parts = regexp(pattern, '(?= wS + tf = true; + return; + end +end +end + +function t = zeroNaN(t) +% [minus plus] seconds, 0 where unknown +if isempty(t), t = [0 0]; end +if isscalar(t), t = [t t]; end +t(isnan(t)) = 0; +end diff --git a/src/ndi/+ndi/+v2/parseUtc.m b/src/ndi/+ndi/+v2/parseUtc.m new file mode 100644 index 0000000000..036eeef49a --- /dev/null +++ b/src/ndi/+ndi/+v2/parseUtc.m @@ -0,0 +1,21 @@ +function t = parseUtc(s) +%PARSEUTC A V2 `utc` timestamp string as a datetime in UTC (NaT when empty or unreadable). + +t = NaT('TimeZone', 'UTC'); +if isempty(s), return; end +s = char(s); +fmts = {'yyyy-MM-dd''T''HH:mm:ss.SSSXXX', 'yyyy-MM-dd''T''HH:mm:ssXXX', ... + 'yyyy-MM-dd''T''HH:mm:ss.SSS''Z''', 'yyyy-MM-dd''T''HH:mm:ss''Z''', ... + 'yyyy-MM-dd''T''HH:mm:ss.SSS', 'yyyy-MM-dd''T''HH:mm:ss'}; +for k = 1:numel(fmts) + try + t = datetime(s, 'InputFormat', fmts{k}, 'TimeZone', 'UTC'); + return; + catch + end +end +try + t = datetime(s, 'TimeZone', 'UTC'); +catch +end +end diff --git a/src/ndi/+ndi/+v2/props.m b/src/ndi/+ndi/+v2/props.m new file mode 100644 index 0000000000..3520daa769 --- /dev/null +++ b/src/ndi/+ndi/+v2/props.m @@ -0,0 +1,17 @@ +function p = props(doc) +%PROPS The document properties of DOC (an ndi.document, a did2.document or a struct). +% +% P = ndi.v2.props(DOC) +% +% See also ndi.entity, ndi.statement. + +if isa(doc, 'ndi.document') + p = doc.document_properties; +elseif isa(doc, 'did2.document') + p = doc.documentProperties; +elseif isstruct(doc) + p = doc; +else + error('ndi:v2:badDocument', 'Expected an ndi.document, did2.document or struct, got %s.', class(doc)); +end +end diff --git a/src/ndi/+ndi/+v2/readBody.m b/src/ndi/+ndi/+v2/readBody.m new file mode 100644 index 0000000000..f75e26454f --- /dev/null +++ b/src/ndi/+ndi/+v2/readBody.m @@ -0,0 +1,97 @@ +function [values, info] = readBody(container, bodyDoc, datumType, keys) +%READBODY Decode a sampled_body's bytes into a MATLAB array. +% +% [VALUES, INFO] = ndi.v2.readBody(CONTAINER, BODYDOC, DATUMTYPE) reads the +% file of the sampled_body BODYDOC (through CONTAINER's database, so an +% ingested file is read from the file store and one recorded by location +% where it is), gunzips it when `compression` is gzip, and decodes it as +% DATUMTYPE (the OWNER statement's data_type.datum_type: the one place the +% type lives; a body never repeats it) in the body's `byte_order`, shaped +% by its keys (`n` per key) in its `datum_order` (F column-major, C +% row-major). INFO has the file path and the shape. KEYS (optional) are +% the keys to shape by, normally the owner statement's data.keys; when +% empty the body's own are used. +% +% One file per body (one chunk); a chunked body is an error for now. +% +% See also ndi.statement/value, ndi.value. + +if nargin < 4, keys = []; end +p = ndi.v2.props(bodyDoc); +fi = []; +if isfield(p, 'files') && isfield(p.files, 'file_info'), fi = p.files.file_info; end +if isempty(fi) + error('ndi:v2:noBodyFile', 'Body %s records no file.', char(p.base.id)); +end +if numel(fi) > 1 + error('ndi:v2:chunkedBody', ['Body %s is chunked (%d files); reading chunked ' ... + 'bodies is not built yet.'], char(p.base.id), numel(fi)); +end +name = char(fi(1).name); +[tf, file] = container.database_existbinarydoc(char(p.base.id), name); +if ~tf + error('ndi:v2:bodyFileMissing', 'The file "%s" of body %s is not on this computer.', ... + name, char(p.base.id)); +end +info = struct('file', file, 'shape', []); + +raw = file; +cleanup = []; +if strcmpi(char(ndi.v2.blockOf(p, 'data_body', 'compression', '')), 'gzip') + tmp = [tempname '.gz']; + copyfile(file, tmp); + out = gunzip(tmp); + raw = out{1}; + cleanup = onCleanup(@() deleteQuietly({tmp, raw})); +end + +order = 'ieee-le'; +if strcmpi(char(ndi.v2.blockOf(p, 'sampled_body', 'byte_order', '')), 'big'), order = 'ieee-be'; end +[precision, post] = precisionOf(datumType); +fid = fopen(raw, 'r', order); +if fid < 0 + error('ndi:v2:bodyOpen', 'Could not open %s.', raw); +end +values = fread(fid, Inf, precision); +fclose(fid); +values = post(values); + +% shape: one size per key (KEYS when given: the owner statement's) +if isempty(keys), keys = ndi.v2.blockOf(p, 'data', 'keys', []); end +keys = ndi.v2.entries(keys); +sz = []; +for k = 1:numel(keys) + if isfield(keys{k}, 'n') && ~isempty(keys{k}.n), sz(end+1) = double(keys{k}.n); end %#ok +end +if numel(sz) == numel(keys) && ~isempty(sz) && prod(sz) == numel(values) + if numel(sz) == 1 + values = reshape(values, sz, 1); + elseif strcmpi(char(ndi.v2.blockOf(p, 'sampled_body', 'datum_order', 'F')), 'C') + values = permute(reshape(values, fliplr(sz)), numel(sz):-1:1); + else + values = reshape(values, sz); + end + info.shape = sz; +end +clear cleanup +end + +function [precision, post] = precisionOf(t) +post = @(x) x; +switch char(t) + case 'float64', precision = 'double=>double'; + case 'float32', precision = 'single=>single'; + case 'bool', precision = 'uint8=>uint8'; post = @(x) logical(x); + case 'utf8', precision = 'uint8=>char'; + case {'uint8', 'uint16', 'uint32', 'uint64', 'int8', 'int16', 'int32', 'int64'} + precision = [char(t) '=>' char(t)]; + otherwise + error('ndi:v2:datumType', 'Reading datum_type "%s" is not built yet.', char(t)); +end +end + +function deleteQuietly(files) +for k = 1:numel(files) + if isfile(files{k}), delete(files{k}); end +end +end diff --git a/src/ndi/+ndi/+v2/relationNames.m b/src/ndi/+ndi/+v2/relationNames.m new file mode 100644 index 0000000000..6696067236 --- /dev/null +++ b/src/ndi/+ndi/+v2/relationNames.m @@ -0,0 +1,33 @@ +function [directed, undirected] = relationNames() +%RELATIONNAMES The relation names the V2 schema binds, directed and undirected. +% +% [DIRECTED, UNDIRECTED] = ndi.v2.relationNames() reads the bound values of +% directed_relation.relation and undirected_relation.relation from the +% schema (did2.schema.cache; did-schema's binding registry). Cellstr rows; +% empty when the schema has none. + +directed = namesOf('directed_relation'); +undirected = namesOf('undirected_relation'); +end + +function n = namesOf(className) +n = {}; +try + s = did2.schema.cache.shared().getClass(className); +catch + return; +end +fs = s.fields; +if isstruct(fs), fs = num2cell(fs); end +for k = 1:numel(fs) + f = fs{k}; + if ~isfield(f, 'name') || ~strcmp(f.name, 'relation'), continue; end + if isfield(f, 'constraints') && isstruct(f.constraints) && isfield(f.constraints, 'binding') ... + && isfield(f.constraints.binding, 'values') + v = f.constraints.binding.values; + if isstruct(v), v = num2cell(v); end + n = cellfun(@(x) char(x.name), v, 'UniformOutput', false); + n = reshape(n, 1, []); + end +end +end diff --git a/src/ndi/+ndi/+v2/resolveLinkPath.m b/src/ndi/+ndi/+v2/resolveLinkPath.m new file mode 100644 index 0000000000..9d1b4dd359 --- /dev/null +++ b/src/ndi/+ndi/+v2/resolveLinkPath.m @@ -0,0 +1,26 @@ +function folder = resolveLinkPath(path, datasetPath) +%RESOLVELINKPATH A linked session's folder, as an absolute path. +% +% FOLDER = ndi.v2.resolveLinkPath(PATH, DATASETPATH): a `linked_session` +% PATH is relative to the dataset's folder DATASETPATH when the session is +% inside it, else absolute. Either way FOLDER is absolute. '/' and '\' are +% both read as separators, so a dataset written on one system opens on +% another. +% +% See also ndi.v2.linkPath, ndi.v2.datasetSessions. + +arguments + path (1,:) char + datasetPath (1,:) char +end +p = strrep(strrep(path, '\', filesep), '/', filesep); +if isAbsolute(p) + folder = p; +else + folder = fullfile(datasetPath, p); +end +end + +function tf = isAbsolute(p) +tf = startsWith(p, filesep) || ~isempty(regexp(p, '^[A-Za-z]:', 'once')); +end diff --git a/src/ndi/+ndi/+v2/searchStatements.m b/src/ndi/+ndi/+v2/searchStatements.m new file mode 100644 index 0000000000..d58a3374b5 --- /dev/null +++ b/src/ndi/+ndi/+v2/searchStatements.m @@ -0,0 +1,325 @@ +function [docs, info] = searchStatements(container, kind, filt) +%SEARCHSTATEMENTS The statement documents of a kind that match a filter. +% +% [DOCS, INFO] = ndi.v2.searchStatements(CONTAINER, KIND, FILT) searches the +% ndi.session or ndi.dataset CONTAINER for statements of KIND -- 'statement' +% (any), 'assertion', 'interaction', 'observation', 'manipulation', +% 'calculation', or a document class -- matching FILT, a struct with any of: +% +% variable the statement's variable +% method an interaction's method (an assertion has none: an error) +% value the statement's value +% formulation a dose's formulation (its name or its type) +% subject document ids: only statements about these subjects +% (searched 200 at a time) +% synonyms true: a VALUE pattern that matches no term by its name or +% node is also compared with the names of each term's node +% (its label and synonyms, ndi.v2.synonyms: looked up once +% and kept on disk); default false +% withDataset true: a session opened from its dataset also searches +% the dataset's documents (a strain's statements live +% there); default false +% at, during, before, after, duration +% when it held: see ndi.v2.timeFilter; 'tolerant' (true or +% false) widens each time by its tolerance +% +% Each is a pattern or a cell array of patterns (any of them). A pattern +% matches a term's name ignoring case or its node exactly, '*' being a +% wildcard and '\*' a literal star (ndi.v2.matchTerm). A VALUE is matched by +% the kind of value the statement has: +% +% a term ({name, node}) as above +% a date (an ISO instant) as text ('2023-11-*'), or compared: '>2023-11-16', +% '>=', '<', '<=', at the coarser of the two +% precisions ('2023-11' is neither before nor +% after '2023-11-16'; '>=' and '<=' hold) +% a single number equal to a number (20 or '20'), or compared: +% '>20', '>=0.02', in the value's canonical unit +% (a dose: its one amount -- volume in liters, +% mass in grams, ...) +% A value kept as an array or in a data body has no single value: a VALUE +% pattern does not match it, and if every statement that matched the rest +% of FILT is one, that is an error (ndi:v2:searchStatements:noSingleValue). +% +% The class, variable and method narrow the database search (wildcards +% too, when did2 has the `wildcard` operator: DID-matlab #218); a plain +% text value narrows it for terms; everything is then rechecked here. +% DOCS is a cell array. INFO has fields: +% structural the statements matching class, variable and method +% values the text of each one's value (for "values here: ...") + +arguments + container + kind (1,:) char + filt (1,1) struct +end +filt = normalise(filt); +if ~isempty(filt.method) && strcmp(kind, 'assertion') + error('ndi:v2:searchStatements:assertionMethod', 'An assertion has no method.'); +end + +q = ndi.v2.isaQuery(classOf(kind)); +q = andTerm(q, 'statement.variable', filt.variable); +q = andTerm(q, 'interaction.method', filt.method); +narrowed = false; +if ~isempty(filt.value) && all(cellfun(@isPlainText, filt.value)) + % a term value can be narrowed in the database; a statement whose value + % is not a term passes to the recheck. When that finds nothing, search + % again without it: "the variable is there, the value is not" (a + % warning naming the values) is not "nothing has this variable" + cand = bySubject(container, q & ... + (ndi.v2.termQuery('term.value', filt.value) | ndi.query('', '~isa', 'term', '')), filt.subject, filt.withDataset); + narrowed = ~isempty(cand); + if isempty(cand) + cand = bySubject(container, q, filt.subject, filt.withDataset); + end +else + cand = bySubject(container, q, filt.subject, filt.withDataset); +end + +structural = keepStructural(cand, filt); +info = struct('structural', numel(structural), 'values', {{}}, 'beforeTime', 0); +docs = {}; +single = 0; +forms = formulationsOf(container, structural, filt.formulation); +matched = false(1, numel(filt.value)); % a pattern some value matched by name or node +byValue = {}; +for i = 1:numel(structural) + p = ndi.v2.props(structural{i}); + [kindOfValue, v, txt] = ndi.v2.statementValue(p); + info.values{end+1} = txt; + if ~strcmp(kindOfValue, 'none'), single = single + 1; end + if ~isempty(filt.value) + hit = cellfun(@(pat) valueMatches(kindOfValue, v, pat), filt.value); + matched = matched | hit; + if ~any(hit), continue; end + end + byValue{end+1} = structural{i}; %#ok +end +% a pattern no term matched by its name or node: the names its nodes go by +synonymPats = filt.value(~matched & cellfun(@isTextPattern, filt.value)); +if filt.synonyms && ~isempty(synonymPats) + pool = structural; + if narrowed % the database kept only the direct matches: look at the rest + pool = keepStructural(bySubject(container, q, filt.subject, filt.withDataset), filt); + end + have = cellfun(@docIdOf, byValue, 'UniformOutput', false); + for i = 1:numel(pool) + if any(strcmp(docIdOf(pool{i}), have)), continue; end + [kindOfValue, v] = ndi.v2.statementValue(ndi.v2.props(pool{i})); + if ~strcmp(kindOfValue, 'term') || ~synonymMatches(v, synonymPats), continue; end + byValue{end+1} = pool{i}; %#ok + have{end+1} = docIdOf(pool{i}); %#ok + end +end +for i = 1:numel(byValue) + if ~isempty(filt.formulation) + f = ndi.v2.edgeIds(ndi.v2.props(byValue{i}), 'formulation_id'); + if ~any(cellfun(@(x) isKey(forms, x) && forms(x), f)), continue; end + end + docs{end+1} = byValue{i}; %#ok +end +tf = intersect({'at', 'during', 'before', 'after', 'duration'}, fieldnames(filt)); +info.time = struct('unresolved', 0, 'nozone', 0, 'bound', 0, 'docs', 0); +info.timed = ~isempty(tf); +if info.timed && ~isempty(docs) + tol = isfield(filt, 'tolerant') && ~isempty(filt.tolerant) && logical(filt.tolerant); + [keepT, info.time] = ndi.v2.timeFilter(container, docs, filt, tol); + info.beforeTime = numel(docs); + docs = docs(keepT); +end +if ~isempty(filt.value) && ~isempty(structural) && single == 0 + error('ndi:v2:searchStatements:noSingleValue', ... + ['Every %s matching this filter keeps its value as an array or in a data body, ' ... + 'so it has no single value to compare.'], kind); +end +info.values = unique(info.values(~cellfun(@isempty, info.values))); +end + +% ------------------------------------------------------------------------- + +function docs = bySubject(container, q, ids, withDataset) +% search Q, restricted to statements about IDS ({} for any), 200 ids a search +search = @(x) container.database_search(x); +if withDataset && isa(container, 'ndi.session') + search = @(x) container.database_search_with_dataset(x); +end +if isempty(ids) + docs = search(q); + return; +end +docs = {}; +for c = 1:200:numel(ids) + part = ids(c:min(c + 199, numel(ids))); + qs = cellfun(@(i) ndi.v2.entityQuery(i), part, 'UniformOutput', false); + docs = [docs, reshape(search(q & ndi.v2.anyOf(qs)), 1, [])]; %#ok +end +end + +function s = keepStructural(cand, filt) +% the candidates whose variable and method match FILT's patterns +s = {}; +for i = 1:numel(cand) + p = ndi.v2.props(cand{i}); + if ~patternsMatch(ndi.v2.blockOf(p, 'statement', 'variable', []), filt.variable), continue; end + if ~patternsMatch(ndi.v2.blockOf(p, 'interaction', 'method', []), filt.method), continue; end + s{end+1} = cand{i}; %#ok +end +end + +function tf = synonymMatches(v, pats) +% does any name a term's node goes by (ndi.v2.synonyms) match a pattern? +tf = false; +if ~isstruct(v) || ~isfield(v, 'node'), return; end +for k = 1:numel(v) + node = char(v(k).node); + if isempty(node), continue; end + names = ndi.v2.synonyms(node); + for j = 1:numel(names) + if any(cellfun(@(pat) ndi.v2.matchTerm(struct('name', names{j}, 'node', ''), pat), pats)) + tf = true; + return; + end + end +end +end + +function id = docIdOf(d) +p = ndi.v2.props(d); +id = char(p.base.id); +end + +function filt = normalise(filt) +if ~isfield(filt, 'subject') || isempty(filt.subject) + filt.subject = {}; +else + filt.subject = unique(cellstr(filt.subject), 'stable'); +end +if ~isfield(filt, 'withDataset') || isempty(filt.withDataset) + filt.withDataset = false; +end +filt.withDataset = logical(filt.withDataset); +if ~isfield(filt, 'synonyms') || isempty(filt.synonyms) + filt.synonyms = false; +end +filt.synonyms = logical(filt.synonyms); +for f = {'variable', 'method', 'value', 'formulation'} + if ~isfield(filt, f{1}) || isempty(filt.(f{1})) + filt.(f{1}) = {}; + elseif ~iscell(filt.(f{1})) + filt.(f{1}) = {filt.(f{1})}; + end + filt.(f{1}) = cellfun(@asPattern, filt.(f{1}), 'UniformOutput', false); +end +end + +function x = asPattern(x) +if isstring(x), x = char(x); end +end + +function c = classOf(kind) +% the six kinds are the class names themselves since 2026-10-08 (isaQuery +% also finds documents written as `subject_`) +c = kind; +end + +function tf = isTextPattern(p) +% a name pattern, wildcards allowed, not a comparison ('>20') +tf = ischar(p) && isempty(comparison(p)); +end + +function tf = isPlainText(p) +tf = ischar(p) && ~ndi.v2.hasWildcard(p) && isempty(comparison(p)); +end + +function q = andTerm(q, path, patterns) +% narrow by a term field when the database can (ndi.v2.termQuery); the +% recheck does the rest +patterns = patterns(cellfun(@ischar, patterns)); +if isempty(patterns), return; end +t = ndi.v2.termQuery(path, patterns); +if ~isempty(t), q = q & t; end +end + +function tf = patternsMatch(term, patterns) +tf = isempty(patterns) || any(cellfun(@(p) ischar(p) && ndi.v2.matchTerm(term, p), patterns)); +end + +function forms = formulationsOf(container, docs, patterns) +% formulation id -> does it match one of PATTERNS (by its name or its type) +forms = containers.Map('KeyType', 'char', 'ValueType', 'logical'); +if isempty(patterns), return; end +ids = {}; +for i = 1:numel(docs) + ids = [ids, ndi.v2.edgeIds(ndi.v2.props(docs{i}), 'formulation_id')]; %#ok +end +ids = unique(ids); +if isempty(ids), return; end +ents = ndi.entity.fetchMany(container, ids); +for i = 1:numel(ents) + e = ents{i}; + t = ndi.v2.blockOf(e.document_properties(), 'formulation', 'value', struct()); + ty = []; + if isstruct(t) && isfield(t, 'type'), ty = t.type; end + forms(e.document_id) = any(cellfun(@(p) ischar(p) && ... + (ndi.v2.matchTerm(struct('name', e.name, 'node', ''), p) || ndi.v2.matchTerm(ty, p)), patterns)); +end +end + +function tf = valueMatches(k, v, pat) +tf = false; +[op, rhs] = comparison(pat); +switch k + case 'term' + tf = isempty(op) && ischar(pat) && ndi.v2.matchTerm(v, pat); + case 'text' + tf = isempty(op) && ischar(pat) && ndi.v2.matchTerm(struct('name', v, 'node', ''), pat); + case 'date' + if ~isempty(op) + n = min(numel(v), numel(rhs)); + tf = compareText(v(1:n), rhs(1:n), op); + elseif ischar(pat) + tf = ndi.v2.matchTerm(struct('name', v, 'node', ''), pat); + end + case 'number' + if ~isempty(op) + r = str2double(rhs); + tf = ~isnan(r) && compareNumber(v, r, op); + elseif isnumeric(pat) && isscalar(pat) + tf = v == pat; + elseif ischar(pat) && ~ndi.v2.hasWildcard(pat) + tf = v == str2double(pat); + end +end +end + +function [op, rhs] = comparison(pat) +% '>=x', '<=x', '>x', '=|<=|>|<)\s*(.+)$', 'tokens', 'once'); +if ~isempty(t), op = t{1}; rhs = strtrim(t{2}); end +end + +function tf = compareText(a, b, op) +c = sign(double(strcmp(a, b) == 0) * (2 * double(issorted({b, a})) - 1)); +tf = applyOp(c, op); +end + +function tf = compareNumber(a, b, op) +tf = applyOp(sign(a - b), op); +end + +function tf = applyOp(c, op) +switch op + case '>', tf = c > 0; + case '>=', tf = c >= 0; + case '<', tf = c < 0; + case '<=', tf = c <= 0; + otherwise, tf = false; +end +end + +function v = firstOr(x, default) +if isempty(x), v = default; else, v = x(1); end +end diff --git a/src/ndi/+ndi/+v2/sessionDocuments.m b/src/ndi/+ndi/+v2/sessionDocuments.m new file mode 100644 index 0000000000..831ce1862e --- /dev/null +++ b/src/ndi/+ndi/+v2/sessionDocuments.m @@ -0,0 +1,22 @@ +function docs = sessionDocuments(database, sessionId) +%SESSIONDOCUMENTS Every document of one session, as stored. +% +% DOCS = ndi.v2.sessionDocuments(DATABASE, SESSIONID) returns the documents +% of a V2 database (DATABASE, an ndi did2sqlite database) whose +% base.session_id is SESSIONID: a cell array of property structs, read from +% the did2 database itself, so each is exactly what is stored (the ndi read +% path adds class blocks a copy must not carry). What +% ndi.v2.copyDocuments copies when a session is ingested into a dataset or +% moved out of one. +% +% See also ndi.v2.copyDocuments, ndi.dataset. + +arguments + database (1,1) ndi.database.implementations.database.did2sqlite + sessionId (1,:) char +end +q = ndi.database.implementations.database.did2sqlite.toDid2Query( ... + ndi.query('base.session_id', 'exact_string', sessionId, '')); +found = database.db.search(q); +docs = cellfun(@(d) d.toStruct(), found, 'UniformOutput', false); +end diff --git a/src/ndi/+ndi/+v2/statementValue.m b/src/ndi/+ndi/+v2/statementValue.m new file mode 100644 index 0000000000..793cb43917 --- /dev/null +++ b/src/ndi/+ndi/+v2/statementValue.m @@ -0,0 +1,81 @@ +function [k, v, t, u] = statementValue(p) +%STATEMENTVALUE A statement's value as one comparable thing, and as text. +% +% [K, V, T] = ndi.v2.statementValue(P): P is a statement document's +% properties (ndi.v2.props). K is the kind of value: 'term' (V a +% {name, node} struct), 'date' (V the ISO instant text), 'number' (V a +% scalar in the canonical unit; a dose's one amount), 'text' (a string, +% or the `text` kind's text), or 'none' +% (an array, a value in a data body, a structure with no single amount, +% or no value at all). T is V as text ('' for 'none'); U the unit of a +% number (its canonical field, e.g. 'liters', 'celsius'). Nothing is read +% but P: a value in a data body is 'none' without opening the body. +% +% See also ndi.v2.searchStatements, ndi.statement/summary. + +[k, v, u] = valueOf(p); +switch k + case 'term', t = ndi.v2.termName(v); + case {'date', 'text'}, t = v; + case 'number', t = num2str(v, 6); + otherwise, t = ''; +end +end + +function [k, v, u] = valueOf(p) +k = 'none'; v = []; u = ''; +if logical(firstOr(ndi.v2.blockOf(p, 'value', 'data_body', false), false)) + return; +end +chain = ndi.v2.classChain(p); +c = ''; +for i = 1:numel(chain) + if any(strcmp(ndi.v2.directParents(chain{i}), 'value')), c = chain{i}; break; end +end +if isempty(c), return; end +raw = ndi.v2.blockOf(p, c, 'value', []); +if iscell(raw) + if numel(raw) ~= 1, return; end + raw = raw{1}; +end +if isstruct(raw) && numel(raw) ~= 1, return; end +if strcmp(c, 'date') && isstruct(raw) && isfield(raw, 'instant') + k = 'date'; v = char(raw.instant); return; +end +if isstruct(raw) && all(ismember(fieldnames(raw), {'name', 'node'})) + k = 'term'; v = raw; return; +end +if isstruct(raw) && isfield(raw, 'text') + % the `text` value kind {text, language} (2026-10-08): compared as text + k = 'text'; v = char(raw.text); return; +end +if ischar(raw) || isstring(raw) + k = 'text'; v = char(raw); return; +end +if isnumeric(raw) && isscalar(raw) + k = 'number'; v = double(raw); u = ndi.value.canonicalField(c); return; +end +if ~isstruct(raw), return; end +cf = ndi.value.canonicalField(c); +if ~isempty(cf) && isfield(raw, cf) && isnumeric(raw.(cf)) && isscalar(raw.(cf)) + k = 'number'; v = double(raw.(cf)); u = cf; return; +end +% a structured value (a dose): its one amount, if it has exactly one +amounts = []; units = {}; +f = fieldnames(raw); +for i = 1:numel(f) + s = raw.(f{i}); + sf = ndi.value.canonicalField(f{i}); + if isstruct(s) && isscalar(s) && ~isempty(sf) && isfield(s, sf) && isnumeric(s.(sf)) && isscalar(s.(sf)) + amounts(end+1) = double(s.(sf)); %#ok + units{end+1} = sf; %#ok + end +end +if isscalar(amounts) + k = 'number'; v = amounts; u = units{1}; +end +end + +function v = firstOr(x, default) +if isempty(x), v = default; else, v = x(1); end +end diff --git a/src/ndi/+ndi/+v2/stays.m b/src/ndi/+ndi/+v2/stays.m new file mode 100644 index 0000000000..755babe261 --- /dev/null +++ b/src/ndi/+ndi/+v2/stays.m @@ -0,0 +1,76 @@ +function S = stays(container, ids, side, cache) +%STAYS The contained_in relations of some subjects, with when each held. +% +% S = ndi.v2.stays(CONTAINER, IDS, SIDE, CACHE) reads the directed +% relations named `contained_in` whose SIDE end ('child' or 'parent') is +% one of the ids IDS, and resolves their time references in batches +% (ndi.v2.timesOf, CACHE shared). S is a struct array, one entry per +% relation: +% id the relation document's id +% child, parent the subject that was contained, and its container +% distributive the relation holds of each member of a group child +% start, end when the stay began and ended (UTC datetimes) +% tol, end_tol [minus plus] seconds of the start and of the end +% decided true when both ends are known: a stay with no time, or +% with no end, or known only as a bound on its anchor +% ('before the seeding') cannot be compared with anything +% +% Used by ndi.entity for context (V_eta tenet T17): a container's +% interactions reach what it contained only while it was contained. +% +% See also ndi.v2.overlaps, ndi.v2.timesOf. + +if nargin < 4 || ~isa(cache, 'containers.Map') + cache = containers.Map('KeyType', 'char', 'ValueType', 'any'); +end +ids = unique(cellstr(ids), 'stable'); +S = struct('id', {}, 'child', {}, 'parent', {}, 'distributive', {}, 'start', {}, ... + 'end', {}, 'tol', {}, 'end_tol', {}, 'decided', {}); +if isempty(ids), return; end +edge = [char(side) '_id']; +docs = {}; +for c = 1:200:numel(ids) + part = ids(c:min(c + 199, numel(ids))); + q = ndi.v2.anyOf(cellfun(@(i) ndi.query('', 'depends_on', edge, i), part, 'UniformOutput', false)); + docs = [docs, reshape(container.database_search(ndi.query('', 'isa', 'directed_relation', '') & q), 1, [])]; %#ok +end +refs = {}; +keep = false(1, numel(docs)); +for i = 1:numel(docs) + p = ndi.v2.props(docs{i}); + keep(i) = strcmp(ndi.v2.termName(ndi.v2.blockOf(p, 'directed_relation', 'relation', '')), 'contained_in'); + if keep(i), refs = [refs, ndi.v2.edgeIds(p, 'time_reference_id')]; end %#ok +end +docs = docs(keep); +times = containers.Map('KeyType', 'char', 'ValueType', 'any'); +if ~isempty(refs), times = ndi.v2.timesOf(container, unique(refs), cache); end +for i = 1:numel(docs) + p = ndi.v2.props(docs{i}); + child = ndi.v2.edgeIds(p, 'child_id'); + parent = ndi.v2.edgeIds(p, 'parent_id'); + if isempty(child) || isempty(parent), continue; end + d = ndi.v2.blockOf(p, 'directed_relation', 'distributive', false); + e = struct('id', char(p.base.id), 'child', child{1}, 'parent', parent{1}, ... + 'distributive', ~isempty(d) && (islogical(d) || isnumeric(d)) && logical(d(1)), ... + 'start', NaT('TimeZone', 'UTC'), 'end', NaT('TimeZone', 'UTC'), ... + 'tol', [0 0], 'end_tol', [0 0], 'decided', false); + r = ndi.v2.edgeIds(p, 'time_reference_id'); + for k = 1:numel(r) + if ~isKey(times, r{k}), continue; end + t = times(r{k}); + if isnat(t.start) || isnat(t.end) || strcmp(t.resolved_by, 'relation'), continue; end + e.start = t.start; e.end = t.end; + e.tol = zeroNaN(t.tolerance); e.end_tol = zeroNaN(t.end_tolerance); + e.decided = true; + break; + end + S(end+1) = e; %#ok +end +end + +function t = zeroNaN(t) +% [minus plus] seconds, 0 where unknown +if isempty(t), t = [0 0]; end +if isscalar(t), t = [t t]; end +t(isnan(t)) = 0; +end diff --git a/src/ndi/+ndi/+v2/subjectTypes.m b/src/ndi/+ndi/+v2/subjectTypes.m new file mode 100644 index 0000000000..86e2291e78 --- /dev/null +++ b/src/ndi/+ndi/+v2/subjectTypes.m @@ -0,0 +1,14 @@ +function t = subjectTypes() +%SUBJECTTYPES The entity types that are subjects: what `isa subject` finds. +% +% T = ndi.v2.subjectTypes() is {'organism', 'culture', 'tissue', 'cell', +% 'group'}: the biological types, which a v1 `subject` document always +% was. A device (a probe, a camera) and a material (a plate) are entities +% of the experiment too, but not subjects, so ndi.query('', 'isa', +% 'subject') leaves them out (Jess Haley, +% 2026-10-08). A subject document with no type (v1) is a subject. +% +% See also ndi.query, ndi.v2.entityTypesFor. + +t = {'organism', 'culture', 'tissue', 'cell', 'group'}; +end diff --git a/src/ndi/+ndi/+v2/synonymFolder.m b/src/ndi/+ndi/+v2/synonymFolder.m new file mode 100644 index 0000000000..0db3586898 --- /dev/null +++ b/src/ndi/+ndi/+v2/synonymFolder.m @@ -0,0 +1,18 @@ +function f = synonymFolder() +%SYNONYMFOLDER Where ndi.v2.synonyms keeps the names it has looked up. +% +% F = ndi.v2.synonymFolder() is the folder named by the environment +% variable NDI_SYNONYM_CACHE when it is set, else 'ontology-synonyms' +% beside NDI's file cache (ndi.common.PathConstants.FileCacheFolder), or +% in tempdir when MATLAB has no userpath. +% +% See also ndi.v2.synonyms. + +f = getenv('NDI_SYNONYM_CACHE'); +if ~isempty(f), return; end +if ~isempty(userpath) + f = fullfile(userpath, 'Documents', 'NDI', 'ontology-synonyms'); +else + f = fullfile(tempdir, 'NDI', 'ontology-synonyms'); +end +end diff --git a/src/ndi/+ndi/+v2/synonyms.m b/src/ndi/+ndi/+v2/synonyms.m new file mode 100644 index 0000000000..5568554e26 --- /dev/null +++ b/src/ndi/+ndi/+v2/synonyms.m @@ -0,0 +1,71 @@ +function names = synonyms(node, options) +%SYNONYMS The names an ontology term goes by: its label and its synonyms. +% +% NAMES = ndi.v2.synonyms(NODE) is a cell array of the names of the term +% NODE ('NCBITaxon:6239' gives 'Caenorhabditis elegans', 'C. elegans', +% ...), from ndi.ontology.lookup (VH-Lab ndi-ontology-matlab). Each node +% is looked up once and kept on disk, one file per node, in FOLDER (by +% default ndi.v2.synonymFolder()), so a later search, in this or any MATLAB +% session, reads the file. {} when the node is not a CURIE, when +% ndi.ontology is not installed, or when the lookup fails (offline, an +% unknown prefix): a failure is not kept on disk, so the node is asked +% for again in a new session, but not again in this one. +% +% Options: +% 'Refresh' true: look the node up again and rewrite its file +% 'Folder' where the files are +% +% See also ndi.v2.synonymFolder, ndi.v2.searchStatements, ndi.entity.search. + +arguments + node (1,:) char + options.Refresh (1,1) logical = false + options.Folder (1,:) char = ndi.v2.synonymFolder() +end +persistent failed +if isempty(failed), failed = containers.Map('KeyType', 'char', 'ValueType', 'logical'); end +names = {}; +if isempty(regexp(node, '^[A-Za-z][\w.-]*:\S+$', 'once')), return; end +file = fullfile(options.Folder, [regexprep(lower(node), '[^a-z0-9_.-]', '_') '.json']); +if ~options.Refresh && isfile(file) + try + c = jsondecode(fileread(file)); + names = asNames(c.names); + return; + catch + % an unreadable file is looked up again and rewritten + end +end +if ~options.Refresh && isKey(failed, lower(node)), return; end +if isempty(which('ndi.ontology.lookup')) + failed(lower(node)) = true; + return; +end +try + [~, name, ~, ~, syn] = ndi.ontology.lookup(node); +catch + failed(lower(node)) = true; + return; +end +names = asNames([{name}, reshape(cellstr(syn), 1, [])]); +try + if ~isfolder(options.Folder), mkdir(options.Folder); end + c = struct('node', node, 'names', {names}, ... + 'looked_up', char(datetime('now', 'TimeZone', 'UTC', 'Format', 'yyyy-MM-dd''T''HH:mm:ss''Z'''))); + fid = fopen(file, 'w'); + if fid > 0 + fwrite(fid, jsonencode(c), 'char'); + fclose(fid); + end +catch + % a folder that cannot be written: the names are still returned +end +end + +function n = asNames(x) +% a row cell array of the non-empty, distinct names in X +if isempty(x), n = {}; return; end +n = cellstr(x); +n = reshape(n(~cellfun(@isempty, n)), 1, []); +n = unique(n, 'stable'); +end diff --git a/src/ndi/+ndi/+v2/termName.m b/src/ndi/+ndi/+v2/termName.m new file mode 100644 index 0000000000..aec09f5766 --- /dev/null +++ b/src/ndi/+ndi/+v2/termName.m @@ -0,0 +1,17 @@ +function n = termName(t) +%TERMNAME The name of a term ({node, name} or {name}); its node when it has no name. + +n = ''; +if isempty(t), return; end +if iscell(t), t = t{1}; end +if isstruct(t) + t = t(1); + if isfield(t, 'name') && ~isempty(t.name) + n = char(t.name); + elseif isfield(t, 'node') + n = char(t.node); + end +elseif ischar(t) || isstring(t) + n = char(t); +end +end diff --git a/src/ndi/+ndi/+v2/termQuery.m b/src/ndi/+ndi/+v2/termQuery.m new file mode 100644 index 0000000000..e78bbe84d7 --- /dev/null +++ b/src/ndi/+ndi/+v2/termQuery.m @@ -0,0 +1,40 @@ +function q = termQuery(path, patterns) +%TERMQUERY An ndi.query for a term field matching any of PATTERNS. +% +% Q = ndi.v2.termQuery(PATH, PATTERNS): PATH is a term field (e.g. +% 'subject_statement.variable'); each pattern matches PATH.name ignoring +% case or PATH.node exactly ('*' a wildcard, '\*' a literal star). Q is [] +% when nothing can be sent to the database: no text patterns, or a +% wildcard and no `wildcard` operator in did2 (ndi.v2.hasWildcardOperator). +% The database answer is a superset to recheck with ndi.v2.matchTerm (a +% wildcard ignores case on the node too). + +q = []; +patterns = cellstr(patterns); +if any(cellfun(@ndi.v2.hasWildcard, patterns)) && ~ndi.v2.hasWildcardOperator() + return; +end +% a block renamed 2026-10-08 is searched under both names (ndi.v2.vetaAliases) +parts = strsplit(char(path), '.'); +paths = cellfun(@(b) strjoin([{b}, parts(2:end)], '.'), ndi.v2.vetaAliases(parts{1}), ... + 'UniformOutput', false); +for k = 1:numel(patterns) + for j = 1:numel(paths) + one = termOne(paths{j}, patterns{k}); + if isempty(q), q = one; else, q = q | one; end + end +end +end + +function one = termOne(path, p) +% one pattern on one path: by name (ignoring case) or by node +if ndi.v2.hasWildcard(p) + one = ndi.query([path '.name'], 'wildcard', p, '') | ndi.query([path '.node'], 'wildcard', p, ''); +else + p = strrep(p, '\*', '*'); + % a node's CURIE prefix ignores case (CURIE_lookups_meta.json); the + % local part is rechecked exactly by ndi.v2.matchTerm + one = ndi.query([path '.name'], 'exact_string_anycase', p, '') | ... + ndi.query([path '.node'], 'exact_string_anycase', p, ''); +end +end diff --git a/src/ndi/+ndi/+v2/timeFilter.m b/src/ndi/+ndi/+v2/timeFilter.m new file mode 100644 index 0000000000..bffdb7ba03 --- /dev/null +++ b/src/ndi/+ndi/+v2/timeFilter.m @@ -0,0 +1,249 @@ +function [keep, info] = timeFilter(container, docs, filt, tolerant, cache) +%TIMEFILTER Which documents' times meet the time filters in FILT. +% +% [KEEP, INFO] = ndi.v2.timeFilter(CONTAINER, DOCS, FILT, TOLERANT, CACHE) +% reads the time references (time_reference_id) of the statement or +% relation documents DOCS, in batches (ndi.v2.timesOf, CACHE shared), and +% keeps a document when any of its references meets every time filter in +% FILT -- a struct with any of: +% +% at t the time covers instant t +% during [t1 t2] the time overlaps the window; a date ('2023-11-16') +% or a month ('2023-11') is the whole of it +% before t it ends before t +% after t it starts after t +% duration '>=3h' how long it lasted: > >= < <= and a number with +% s, min, h or d (seconds when no unit) +% +% A time t is a datetime (with a zone, used as is; without one, read in +% the reference's own zone) or text: '2023-11-16T14:00', '2023-11-16 +% 14:00', '2023-11-16' (MATLAB's 'now', 'today', 'yesterday' and +% 'tomorrow' work too); text ending in Z or an offset (+08:00) is that +% zone, other text is read in the zone the lab wrote the reference in +% (its source_timezone). A reference whose zone is not recorded cannot be +% compared with such text, and is counted, not matched. +% +% TOLERANT (default false): a time also matches when its tolerance makes +% a match possible (a transfer stated at 14:03 that may have been up to 5 +% min earlier is 'at' 14:00). +% +% A reference that only orders the time against its anchor ("before the +% seeding", no offset) is a bound: 'before' and 'after' use it when it +% decides them; 'at', 'during' and 'duration' cannot, and do not match. +% A reference that cannot be resolved (its anchor not reachable, or with +% no time of its own) does not match. +% +% INFO counts what could not be compared: unresolved (references with no +% time), nozone (text with no zone to read it in), bound (bounds that +% could not decide), and, for messages, docs (DOCS with no time at all). + +if nargin < 4 || isempty(tolerant), tolerant = false; end +if nargin < 5 || ~isa(cache, 'containers.Map'), cache = containers.Map('KeyType', 'char', 'ValueType', 'any'); end +names = intersect({'at', 'during', 'before', 'after', 'duration'}, fieldnames(filt), 'stable'); +names = names(cellfun(@(n) ~isempty(filt.(n)), names)); +keep = true(1, numel(docs)); +info = struct('unresolved', 0, 'nozone', 0, 'bound', 0, 'docs', 0); +if isempty(names), return; end +refs = cell(1, numel(docs)); +for i = 1:numel(docs) + refs{i} = ndi.v2.edgeIds(ndi.v2.props(docs{i}), 'time_reference_id'); +end +all_ = unique([refs{:}]); +times = containers.Map('KeyType', 'char', 'ValueType', 'any'); +if ~isempty(all_), times = ndi.v2.timesOf(container, all_, cache); end +for i = 1:numel(docs) + if isempty(refs{i}) + info.docs = info.docs + 1; + keep(i) = false; + continue; + end + ok = false; + for r = 1:numel(refs{i}) + if ~isKey(times, refs{i}{r}) + info.unresolved = info.unresolved + 1; + continue; + end + [m, why] = meets(times(refs{i}{r}), filt, names, tolerant); + if ~isempty(why), info.(why) = info.(why) + 1; end + if m, ok = true; break; end + end + keep(i) = ok; +end +end + +% ------------------------------------------------------------------------- + +function [m, why] = meets(t, filt, names, tolerant) +% does the one time T meet every filter +m = false; why = ''; +if isnat(t.start) + why = 'unresolved'; + return; +end +isBound = strcmp(t.resolved_by, 'relation'); +s = t.start; e = t.end; +if isnat(e), e = s; end +tolS = zeroNaN(t.tolerance); tolE = zeroNaN(t.end_tolerance); +if all(tolE == 0), tolE = tolS; end +if ~tolerant, tolS = [0 0]; tolE = [0 0]; end +for k = 1:numel(names) + n = names{k}; + switch n + case 'duration' + if isBound, why = 'bound'; return; end + [op, secs] = parseDuration(filt.duration); + lo = seconds((e - tolE(1)) - (s + tolS(2))); + hi = seconds((e + tolE(2)) - (s - tolS(1))); + if ~tolerant, lo = seconds(e - s); hi = lo; end + if ~any(applyOp([lo hi], secs, op)), return; end + otherwise + [q, ok] = queryTime(filt.(n), n, t.timezone); + if ~ok, why = 'nozone'; return; end + if isBound + [decided, yes] = boundDecides(t, n, q); + if ~decided, why = 'bound'; return; end + if ~yes, return; end + continue; + end + sLo = s - seconds(tolS(1)); eHi = e + seconds(tolE(2)); + sHi = s + seconds(tolS(2)); eLo = e - seconds(tolE(1)); + switch n + case 'at', hit = sLo <= q(1) && q(1) <= eHi; + case 'during', hit = sLo <= q(2) && eHi >= q(1); + case 'before', hit = eLo < q(1); + case 'after', hit = sHi > q(1); + end + if ~hit, return; end + end +end +m = true; +end + +function [decided, yes] = boundDecides(t, n, q) +% a time known only as before / after / within its anchor (t.start, t.end +% are the anchor's): decide 'before' and 'after' when the bound does +decided = false; yes = false; +rel = lower(t.relation); +aS = t.start; aE = t.end; +if isnat(aE), aE = aS; end +switch n + case 'before' + if any(strcmp(rel, {'before', 'intervalbefore', 'intervalmeets', 'meets'})) + if aS <= q(1), decided = true; yes = true; end % it ended before the anchor began + elseif any(strcmp(rel, {'during', 'intervalduring', 'intervalin', 'inside', 'intervalstarts', 'intervalfinishes'})) + if aE < q(1), decided = true; yes = true; elseif aS >= q(1), decided = true; end + elseif any(strcmp(rel, {'after', 'intervalafter', 'intervalmetby', 'metby'})) + if aE >= q(1), decided = true; end % it began after the anchor ended + end + case 'after' + if any(strcmp(rel, {'after', 'intervalafter', 'intervalmetby', 'metby'})) + if aE >= q(1), decided = true; yes = true; end + elseif any(strcmp(rel, {'during', 'intervalduring', 'intervalin', 'inside', 'intervalstarts', 'intervalfinishes'})) + if aS > q(1), decided = true; yes = true; elseif aE <= q(1), decided = true; end + elseif any(strcmp(rel, {'before', 'intervalbefore', 'intervalmeets', 'meets'})) + if aS <= q(1), decided = true; end + end +end +end + +function [q, ok] = queryTime(v, n, tz) +% a filter's time(s) as UTC datetimes: q(1) (and q(2) for 'during') +ok = true; +if strcmp(n, 'during') + if iscell(v) && numel(v) == 2 + [a, ok1] = oneTime(v{1}, tz, 'start'); [b, ok2] = oneTime(v{2}, tz, 'end'); + elseif isa(v, 'datetime') && numel(v) == 2 + [a, ok1] = oneTime(v(1), tz, 'start'); [b, ok2] = oneTime(v(2), tz, 'end'); + else + [a, ok1] = oneTime(v, tz, 'start'); [b, ok2] = oneTime(v, tz, 'end'); + end + q = [a b]; ok = ok1 && ok2; +else + [q, ok] = oneTime(v, tz, 'start'); +end +end + +function [d, ok] = oneTime(v, tz, edge) +% one time as UTC. A date or a month of text is its first (EDGE 'start') +% or last (EDGE 'end') instant. +ok = true; +if isa(v, 'datetime') + d = v; + if isempty(d.TimeZone) + if isempty(tz), ok = false; d = NaT('TimeZone', 'UTC'); return; end + d.TimeZone = tz; + end + d.TimeZone = 'UTC'; + return; +end +v = strtrim(char(v)); +zoned = ~isempty(regexp(v, '(Z|[+-]\d\d:?\d\d)$', 'once')); +if ~zoned && isempty(tz) + ok = false; d = NaT('TimeZone', 'UTC'); return; +end +whole = ''; +if ~isempty(regexp(v, '^\d{4}-\d{2}$', 'once')), whole = 'month'; v = [v '-01']; end +if ~isempty(regexp(v, '^\d{4}-\d{2}-\d{2}$', 'once')) && isempty(whole), whole = 'day'; end +if zoned + z = regexp(v, '(Z|[+-]\d\d:?\d\d)$', 'match', 'once'); + v = strtrim(v(1:end - numel(z))); + if strcmp(z, 'Z'), zone = 'UTC'; else, zone = z; end +else + zone = tz; +end +fmts = {'yyyy-MM-dd''T''HH:mm:ss.SSS', 'yyyy-MM-dd''T''HH:mm:ss', 'yyyy-MM-dd''T''HH:mm', ... + 'yyyy-MM-dd HH:mm:ss.SSS', 'yyyy-MM-dd HH:mm:ss', 'yyyy-MM-dd HH:mm', 'yyyy-MM-dd'}; +d = NaT; +for k = 1:numel(fmts) + try + d = datetime(v, 'InputFormat', fmts{k}, 'TimeZone', zone); + break; + catch + end +end +if isnat(d) + error('ndi:v2:timeFilter:badTime', ... + 'Cannot read the time ''%s'': use e.g. ''2023-11-16T14:00'', ''2023-11-16'' or a datetime.', v); +end +if strcmp(edge, 'end') + if strcmp(whole, 'day'), d = d + days(1) - seconds(1e-3); end + if strcmp(whole, 'month'), d = d + calmonths(1) - seconds(1e-3); end +end +d.TimeZone = 'UTC'; +end + +function [op, secs] = parseDuration(v) +% '>=3h' -> '>=', 10800 +t = regexp(strtrim(char(v)), '^(>=|<=|>|<|=)?\s*([0-9.eE+-]+)\s*([a-zA-Z]*)$', 'tokens', 'once'); +if isempty(t) || isnan(str2double(t{2})) + error('ndi:v2:timeFilter:badDuration', ... + 'Cannot read the duration ''%s'': use e.g. ''>=3h'', ''<30min'', ''>10s''.', char(v)); +end +op = t{1}; +if isempty(op), op = '='; end +x = str2double(t{2}); +switch lower(t{3}) + case {'', 's', 'sec', 'secs', 'second', 'seconds'}, secs = x; + case {'m', 'min', 'mins', 'minute', 'minutes'}, secs = 60 * x; + case {'h', 'hr', 'hrs', 'hour', 'hours'}, secs = 3600 * x; + case {'d', 'day', 'days'}, secs = 86400 * x; + otherwise + error('ndi:v2:timeFilter:badDuration', 'Unknown unit ''%s'' in ''%s'': s, min, h or d.', t{3}, char(v)); +end +end + +function tf = applyOp(vals, ref, op) +% is any of VALS (the possible range, ends inclusive) OP REF +lo = min(vals); hi = max(vals); +switch op + case '>', tf = hi > ref; + case '>=', tf = hi >= ref; + case '<', tf = lo < ref; + case '<=', tf = lo <= ref; + case '=', tf = lo <= ref && ref <= hi; +end +end + +function t = zeroNaN(t) +t(isnan(t)) = 0; +end diff --git a/src/ndi/+ndi/+v2/timeOf.m b/src/ndi/+ndi/+v2/timeOf.m new file mode 100644 index 0000000000..d558031efe --- /dev/null +++ b/src/ndi/+ndi/+v2/timeOf.m @@ -0,0 +1,155 @@ +function r = timeOf(container, refDoc, depth, cache) +%TIMEOF A time reference, read and resolved to wall-clock time where it can be. +% +% R = ndi.v2.timeOf(CONTAINER, REFDOC) returns a struct: +% kind 'absolute' or 'relative' +% start, end datetime (UTC): the instant or interval, resolved through +% the referent chain when the reference is relative (NaT +% when it cannot be) +% start_offset, end_offset seconds from the referent (relative only) +% relation the OWL-Time relation to the referent, when given +% clock the clock the offsets are on +% referent_id the document the reference is relative to +% approximate true when the start or end is marked approximate +% tolerance [minus plus] seconds of the start (NaN when none) +% end_tolerance [minus plus] seconds of the end (NaN when none) +% timezone the zone the source wrote the time in (source_timezone: +% an absolute reference's own, a relative one's anchor's) +% resolved_by 'value', 'offset', 'relation' (the referent's own extent) +% or '' (not resolved) +% +% A relative reference resolves through its referent's own time +% references (a session's, an epoch's, a statement's), up to 8 levels. +% +% R = ndi.v2.timeOf(CONTAINER, REFDOC, DEPTH, CACHE): CACHE, a +% containers.Map, holds documents under 'doc:' (ndi.v2.getDocuments +% fills it) and each referent's resolved anchor under 'anchor:', so a +% referent shared by many references is read and resolved once +% (ndi.v2.timesOf). + +if nargin < 3, depth = 0; end +if nargin < 4, cache = []; end +p = ndi.v2.props(refDoc); +k = char(p.document_class.class_name); +r = struct('kind', '', 'start', NaT('TimeZone', 'UTC'), 'end', NaT('TimeZone', 'UTC'), ... + 'start_offset', NaN, 'end_offset', NaN, 'relation', '', 'clock', '', 'referent_id', '', ... + 'approximate', false, 'tolerance', [NaN NaN], 'end_tolerance', [NaN NaN], ... + 'timezone', '', 'resolved_by', ''); +v = ndi.v2.blockOf(p, k, 'value', struct()); +if strcmp(k, 'absolute_time_reference') + r.kind = 'absolute'; + st = getOr(v, 'start', struct()); + r.start = ndi.v2.parseUtc(getOr(st, 'utc', '')); + r.approximate = logical(getOr(st, 'approximate', false)); + r.tolerance = tol(getOr(st, 'tolerance', [])); + r.timezone = char(getOr(st, 'source_timezone', '')); + en = endOf(v); + r.end = ndi.v2.parseUtc(getOr(en, 'utc', '')); + r.end_tolerance = tol(getOr(en, 'tolerance', [])); + if isempty(r.timezone), r.timezone = char(getOr(en, 'source_timezone', '')); end + du = getOr(v, 'duration', struct()); + if isnat(r.end) && ~isempty(getOr(du, 'seconds', [])) + r.end = r.start + seconds(double(du.seconds)); + end + r.approximate = r.approximate || logical(getOr(en, 'approximate', false)); + if ~isnat(r.start), r.resolved_by = 'value'; end + return; +end +r.kind = 'relative'; +r.relation = ndi.v2.termName(getOr(v, 'relation', '')); +r.clock = ndi.v2.termName(getOr(v, 'clock', '')); +ids = ndi.v2.edgeIds(p, 'referent_id'); +if ~isempty(ids), r.referent_id = ids{1}; end +st = getOr(v, 'start', struct()); +r.start_offset = double(getOr(st, 'seconds', NaN)); +r.tolerance = tol(getOr(st, 'tolerance', [])); +r.approximate = logical(getOr(st, 'approximate', false)); +en = endOf(v); +r.end_offset = double(getOr(en, 'seconds', NaN)); +r.end_tolerance = tol(getOr(en, 'tolerance', [])); +du = getOr(v, 'duration', struct()); +if isnan(r.end_offset) && ~isnan(r.start_offset) && ~isempty(getOr(du, 'seconds', [])) + r.end_offset = r.start_offset + double(du.seconds); +end +if isempty(r.referent_id) || depth >= 8 + return; +end +if isa(cache, 'containers.Map') && isKey(cache, ['anchor:' r.referent_id]) + anchor = cache(['anchor:' r.referent_id]); +else + ref = docOf(container, r.referent_id, cache); + if isempty(ref), return; end + anchor = anchorOf(container, ref, depth, cache); + if isa(cache, 'containers.Map'), cache(['anchor:' r.referent_id]) = anchor; end +end +if isempty(anchor) || isnat(anchor.start) + return; +end +r.timezone = anchor.timezone; +if ~isnan(r.start_offset) + r.start = anchor.start + seconds(r.start_offset); + if ~isnan(r.end_offset), r.end = anchor.start + seconds(r.end_offset); end + r.resolved_by = 'offset'; +else + r.start = anchor.start; % a relation alone: the referent's own extent + r.end = anchor.end; + r.resolved_by = 'relation'; + r.approximate = true; +end +end + +function a = anchorOf(container, refDoc, depth, cache) +% the referent's own time: its first time reference that resolves +a = []; +p = ndi.v2.props(refDoc); +if any(strcmp(ndi.v2.classChain(p), 'time_reference')) + a = ndi.v2.timeOf(container, refDoc, depth + 1, cache); + return; +end +ids = ndi.v2.edgeIds(p, 'time_reference_id'); +for i = 1:numel(ids) + d = docOf(container, ids{i}, cache); + if isempty(d), continue; end + t = ndi.v2.timeOf(container, d, depth + 1, cache); + if ~isnat(t.start) + a = t; + return; + end +end +end + +function d = docOf(container, id, cache) +% a document by id: from CACHE when it holds it (or held its absence) +if isa(cache, 'containers.Map') && isKey(cache, ['doc:' id]) + d = cache(['doc:' id]); + return; +end +d = ndi.v2.getDocument(container, id); +if isa(cache, 'containers.Map'), cache(['doc:' id]) = d; end +end + +function v = getOr(s, name, default) +v = default; +if isstruct(s) && isfield(s, name) && ~isempty(s(1).(name)) + v = s(1).(name); +end +end + +function t = tol(x) +t = [NaN NaN]; +if isstruct(x) && ~isempty(x) + if isfield(x, 'minus') && ~isempty(x.minus), t(1) = double(x.minus); end + if isfield(x, 'plus') && ~isempty(x.plus), t(2) = double(x.plus); end +end +end + +function e = endOf(v) +% the value's `end` cell (a reserved word: a JSON reader may rename it) +e = struct(); +for n = {'end', 'xEnd', 'x_end', 'end_'} + if isstruct(v) && isfield(v, n{1}) && ~isempty(v(1).(n{1})) + e = v(1).(n{1}); + return; + end +end +end diff --git a/src/ndi/+ndi/+v2/timesOf.m b/src/ndi/+ndi/+v2/timesOf.m new file mode 100644 index 0000000000..1b500b8b00 --- /dev/null +++ b/src/ndi/+ndi/+v2/timesOf.m @@ -0,0 +1,46 @@ +function T = timesOf(container, refIds, cache) +%TIMESOF Many time references, read and resolved together. +% +% T = ndi.v2.timesOf(CONTAINER, REFIDS) returns a containers.Map, time +% reference id -> the struct ndi.v2.timeOf gives, for each id found. The +% references, what they are relative to, and those documents' own time +% references are fetched level by level in batches (ndi.v2.getDocuments) +% before anything is resolved, and each anchor is resolved once however +% many references point at it: a thousand references relative to one +% session cost one lookup of the session. CACHE (a containers.Map, made +% when not given) keeps documents and results for later calls. +% +% See also ndi.v2.timeOf. + +if nargin < 3 || ~isa(cache, 'containers.Map') + cache = containers.Map('KeyType', 'char', 'ValueType', 'any'); +end +refIds = unique(cellstr(refIds), 'stable'); +T = containers.Map('KeyType', 'char', 'ValueType', 'any'); +% prefetch: the references, then what they point at, then those documents' +% time references, up to timeOf's 8 levels +todo = refIds; +for level = 1:8 + if isempty(todo), break; end + got = ndi.v2.getDocuments(container, todo, cache); + next = {}; + for k = 1:numel(todo) + if ~isKey(got, todo{k}), continue; end + p = ndi.v2.props(got(todo{k})); + next = [next, ndi.v2.edgeIds(p, 'referent_id'), ndi.v2.edgeIds(p, 'time_reference_id')]; %#ok + end + next = unique(next, 'stable'); + todo = next(~cellfun(@(i) isKey(cache, ['doc:' i]), next)); +end +for k = 1:numel(refIds) + key = ['doc:' refIds{k}]; + if ~isKey(cache, key), continue; end + if isKey(cache, ['time:' refIds{k}]) + T(refIds{k}) = cache(['time:' refIds{k}]); + else + r = ndi.v2.timeOf(container, cache(key), 0, cache); + cache(['time:' refIds{k}]) = r; + T(refIds{k}) = r; + end +end +end diff --git a/src/ndi/+ndi/+v2/vetaAliases.m b/src/ndi/+ndi/+v2/vetaAliases.m new file mode 100644 index 0000000000..6ba27f2635 --- /dev/null +++ b/src/ndi/+ndi/+v2/vetaAliases.m @@ -0,0 +1,24 @@ +function names = vetaAliases(name) +%VETAALIASES A V_eta class, block or edge name and the spelling it had before 2026-10-08. +% +% NAMES = ndi.v2.vetaAliases(NAME) is {NAME} for a name that was not +% renamed, and {NAME, OLD} for one that was: 'statement' gives +% {'statement', 'subject_statement'}, 'value' gives {'value', 'data_type'}, +% 'entity_id' gives {'entity_id', 'subject_id'}. Readers search both, so a +% dataset written before the rename still reads (ndi.v2.vetaName). +% +% See also ndi.v2.vetaName, ndi.v2.isaQuery. + +name = char(ndi.v2.vetaName(name)); +switch name + case 'statement', names = {name, 'subject_statement'}; + case 'assertion', names = {name, 'subject_assertion'}; + case 'interaction', names = {name, 'subject_interaction'}; + case 'observation', names = {name, 'subject_observation'}; + case 'manipulation', names = {name, 'subject_manipulation'}; + case 'calculation', names = {name, 'subject_calculation'}; + case 'value', names = {name, 'data_type'}; + case 'entity_id', names = {name, 'subject_id'}; + otherwise, names = {name}; +end +end diff --git a/src/ndi/+ndi/+v2/vetaName.m b/src/ndi/+ndi/+v2/vetaName.m new file mode 100644 index 0000000000..c447bb8d37 --- /dev/null +++ b/src/ndi/+ndi/+v2/vetaName.m @@ -0,0 +1,41 @@ +function out = vetaName(name) +%VETANAME A V_eta class or block name in its current spelling. +% +% OUT = ndi.v2.vetaName(NAME) maps the names V_eta used until 2026-10-08 to +% the ones it uses since (did-schema V_eta_tenets.md, T2 amendment): the +% statement family drops its `subject_` prefix and `data_type` becomes +% `value`. Any other name comes back unchanged. NAME may be a cellstr. +% +% subject_statement -> statement +% subject_assertion -> assertion +% subject_interaction -> interaction +% subject_observation -> observation +% subject_manipulation -> manipulation +% subject_calculation -> calculation +% data_type -> value +% +% Documents written before the rename keep the old names (and the edge +% `subject_id` for `entity_id`), so the readers here accept both. +% +% See also ndi.v2.vetaAliases. + +if iscell(name) + out = cellfun(@ndi.v2.vetaName, name, 'UniformOutput', false); + return; +end +name = char(name); +[old, new] = pairs(); +k = find(strcmp(old, name), 1); +if isempty(k) + out = name; +else + out = new{k}; +end +end + +function [old, new] = pairs() +old = {'subject_statement', 'subject_assertion', 'subject_interaction', ... + 'subject_observation', 'subject_manipulation', 'subject_calculation', 'data_type'}; +new = {'statement', 'assertion', 'interaction', ... + 'observation', 'manipulation', 'calculation', 'value'}; +end diff --git a/src/ndi/+ndi/+vintage/edgeName.m b/src/ndi/+ndi/+vintage/edgeName.m index 0caa7fc8c9..df5cca382d 100644 --- a/src/ndi/+ndi/+vintage/edgeName.m +++ b/src/ndi/+ndi/+vintage/edgeName.m @@ -5,7 +5,9 @@ % % Returns V1_EDGE_NAME unchanged for a v1 document, for a document of a % class V_eta did not rename, and for an edge the map has no row for. -% Only a V_eta document with a mapped edge gets a different answer. +% Only a V_eta document with a mapped edge gets a different answer: the +% first of the row's candidate names (ndi.vintage.map) that the document +% carries, or the current name when it carries none. % % Split out from ndi.vintage.edge so the numbered-family reader can share % it, and so a test can assert the translation without needing a document @@ -28,8 +30,30 @@ rows = entry.edges; for i = 1:size(rows, 1) if strcmp(v1_edge_name, rows{i,1}) - name = rows{i,2}; + candidates = ndi.vintage.names(rows{i,2}); + name = candidates{1}; + % The first candidate the document actually carries, under its own + % name or as the first member of a numbered family (`name_1`). + % A document carrying none of them gets the CURRENT name, so a + % caller's not-found message names the edge today's schema uses. + present = documentEdgeNames(ndi_document_obj); + for j = 1:numel(candidates) + if any(strcmp(candidates{j}, present)) ... + || any(strcmp([candidates{j} '_1'], present)) + name = candidates{j}; + break; + end + end return; end end end + +function present = documentEdgeNames(ndi_document_obj) +present = {}; +props = ndi_document_obj.document_properties; +if isfield(props, 'depends_on') && isstruct(props.depends_on) ... + && isfield(props.depends_on, 'name') + present = {props.depends_on.name}; +end +end diff --git a/src/ndi/+ndi/+vintage/edge_n.m b/src/ndi/+ndi/+vintage/edge_n.m index 1dc9c246bb..2a77d4274e 100644 --- a/src/ndi/+ndi/+vintage/edge_n.m +++ b/src/ndi/+ndi/+vintage/edge_n.m @@ -9,10 +9,11 @@ % `acquisition_metadata_reader_1..N`. So the map stores base names and % `dependency_value_n` does the expansion, exactly as it did before. % -% The map's entries for these two families are the base names WITHOUT the -% `_#` the schema writes (schemas/V_eta/stable/acquisition_system.json -% declares `acquisition_metadata_reader_#`); `#` is the schema's notation -% for the family, not part of any stored key. +% The map's entries for the numbered families are the base names WITHOUT +% the `_#` the schema used to write (`acquisition_metadata_reader_#`); `#` +% was the schema's notation for the family, not part of any stored key. +% did-schema T15 later replaced numbered families with ONE REPEATED name +% (`epoch_parameter_reader_id` x N); both forms are read here. % % See also: ndi.vintage.edge, ndi.vintage.edgeName, ndi.vintage.map. @@ -23,6 +24,31 @@ end name = ndi.vintage.edgeName(ndi_document_obj, v1_edge_name); + +% A REPEATED NAME IS A FAMILY TOO. did-schema T15 stores a family as one +% name repeated (`clock_alignment_configuration_id` N times) rather than +% numbered (`..._1`, `..._2`). dependency_value_n reads only the first +% entry of an unnumbered name, so the repeated form is collected here, in +% stored order. +props = ndi_document_obj.document_properties; +if isfield(props, 'depends_on') && isstruct(props.depends_on) ... + && isfield(props.depends_on, 'name') + hits = find(strcmp(name, {props.depends_on.name})); + if numel(hits) > 1 + values = {}; + for i = 1:numel(hits) + v = ndi.document.i_readDependencyTarget(props.depends_on(hits(i))); + if ~isempty(v) + values{end+1} = v; %#ok + end + end + if isempty(values) && options.ErrorIfNotFound + error(['Dependency name ' name ' not found.']); + end + return; + end +end + values = ndi_document_obj.dependency_value_n(name, ... 'ErrorIfNotFound', options.ErrorIfNotFound); end diff --git a/src/ndi/+ndi/+vintage/elementFields.m b/src/ndi/+ndi/+vintage/elementFields.m index 61be1261b1..777dde1cb6 100644 --- a/src/ndi/+ndi/+vintage/elementFields.m +++ b/src/ndi/+ndi/+vintage/elementFields.m @@ -140,8 +140,8 @@ % v1 read would have produced from an empty field. v = ''; q = ndi.query('', 'isa', 'term_assertion', '') & ... - ndi.query('subject_statement.variable.name', 'exact_string', label, '') & ... - ndi.query('', 'depends_on', 'subject_id', docId); + ndi.v2.fieldQuery('statement.variable.name', 'exact_string', label) & ... + ndi.v2.entityQuery(docId); docs = ndi_session_obj.database_search(q); if isempty(docs) return; @@ -175,14 +175,15 @@ % repository. specimen = ''; found = false; -q = ndi.query('', 'isa', 'subject_observation', '') & ... +q = ndi.v2.isaQuery('observation') & ... ndi.query('', 'depends_on', 'instrument_id', elementId); docs = ndi_session_obj.database_search(q); if isempty(docs) return; end found = true; -specimen = docs{1}.dependency_value('subject_id', 'ErrorIfNotFound', 0); +specimen = ndi.v2.edgeIds(ndi.v2.props(docs{1}), 'entity_id'); +if isempty(specimen), specimen = ''; else, specimen = specimen{1}; end end function target = relationTarget(ndi_session_obj, childId, relationName) diff --git a/src/ndi/+ndi/+vintage/elementSubjectDocs.m b/src/ndi/+ndi/+vintage/elementSubjectDocs.m index 19eebe5368..f1a2386552 100644 --- a/src/ndi/+ndi/+vintage/elementSubjectDocs.m +++ b/src/ndi/+ndi/+vintage/elementSubjectDocs.m @@ -52,7 +52,7 @@ label = ndi.vintage.elementLabel('class'); q = ndi.query('', 'isa', 'term_assertion', '') & ... - ndi.query('subject_statement.variable.name', 'exact_string', label, ''); + ndi.v2.fieldQuery('statement.variable.name', 'exact_string', label); if ~isempty(classFilter) q = q & ndi.query('term.value.name', 'contains_string', classFilter, ''); end @@ -67,7 +67,8 @@ % rather than on anything meaningful. ids = {}; for i = 1:numel(assertions) - sid = assertions{i}.dependency_value('subject_id', 'ErrorIfNotFound', 0); + sid = ndi.v2.edgeIds(ndi.v2.props(assertions{i}), 'entity_id'); + if isempty(sid), sid = ''; else, sid = sid{1}; end if isempty(sid) || any(strcmp(sid, ids)) continue; end diff --git a/src/ndi/+ndi/+vintage/entryFor.m b/src/ndi/+ndi/+vintage/entryFor.m index 18ee25b333..ed70794385 100644 --- a/src/ndi/+ndi/+vintage/entryFor.m +++ b/src/ndi/+ndi/+vintage/entryFor.m @@ -40,7 +40,7 @@ vintage = 'v1'; return; end - if strcmp(name, m(i).eta_class) + if any(strcmp(name, ndi.vintage.names(m(i).eta_class))) entry = m(i); vintage = 'V_eta'; return; diff --git a/src/ndi/+ndi/+vintage/field.m b/src/ndi/+ndi/+vintage/field.m index a7369a0efc..b5fd5aa45a 100644 --- a/src/ndi/+ndi/+vintage/field.m +++ b/src/ndi/+ndi/+vintage/field.m @@ -5,8 +5,9 @@ % % Both the BLOCK name and the FIELD name move: a v1 `filenavigator` % document carries `filenavigator.fileparameters`, and its V_eta -% successor carries `epoch_file_pattern.data_file_pattern`. Callers name -% the v1 field and get the right one either way. +% successor carries `epoch_file_pattern.file_pattern` (or +% `data_file_pattern`, its name before did-schema's #73 review). Callers +% name the v1 field and get the right one either way. % % FOUND IS RETURNED RATHER THAN ERRORING, and that is deliberate: several % of these fields are legitimately absent (v1 leaves `fileparameters` @@ -16,7 +17,7 @@ % % ONE SHAPE CHANGE IS NOT A RENAME AND IS NOT HANDLED HERE. v1 stores % `fileparameters` as a MATLAB expression the reader `eval`s -% (+ndi/+file/navigator.m:44-45); V_eta declares `data_file_pattern` as a +% (+ndi/+file/navigator.m:44-45); V_eta declares `file_pattern` as a % string list, already the value the eval produced. So the caller must % eval the v1 form and NOT eval the V_eta form. `ndi.vintage.entryFor` % tells it which it has; doing it here would hide an eval inside a @@ -42,15 +43,30 @@ blockName = entry.v1_class; fieldName = v1_field_name; else - blockName = entry.eta_class; - fieldName = v1_field_name; + % The block is named by the class the document DECLARES: a map entry + % may list several V_eta class names (ndi.vintage.map), and the + % document carries exactly one of them. + blockName = props.document_class.class_name; + candidates = {v1_field_name}; rows = entry.fields; for i = 1:size(rows, 1) if strcmp(v1_field_name, rows{i,1}) - fieldName = rows{i,2}; + candidates = ndi.vintage.names(rows{i,2}); break; end end + fieldName = candidates{1}; + if isfield(props, blockName) && isstruct(props.(blockName)) && isscalar(props.(blockName)) + % The first candidate the document actually carries: the current + % name for a document built to today's schema, an older one for a + % document written before the rename. + for i = 1:numel(candidates) + if isfield(props.(blockName), candidates{i}) + fieldName = candidates{i}; + break; + end + end + end end if ~isfield(props, blockName) diff --git a/src/ndi/+ndi/+vintage/isaQuery.m b/src/ndi/+ndi/+vintage/isaQuery.m index ec8182d43a..4c6e6e02bc 100644 --- a/src/ndi/+ndi/+vintage/isaQuery.m +++ b/src/ndi/+ndi/+vintage/isaQuery.m @@ -44,7 +44,10 @@ end % Both names, so neither vintage is privileged. `|` is did.query/or. -if ~strcmp(entry.v1_class, entry.eta_class) - q = q | ndi.query('', 'isa', entry.eta_class, ''); +etaNames = ndi.vintage.names(entry.eta_class); +for i = 1:numel(etaNames) + if ~strcmp(entry.v1_class, etaNames{i}) + q = q | ndi.query('', 'isa', etaNames{i}, ''); + end end end diff --git a/src/ndi/+ndi/+vintage/lightsheetChunkName.m b/src/ndi/+ndi/+vintage/lightsheetChunkName.m new file mode 100644 index 0000000000..f98ab241a0 --- /dev/null +++ b/src/ndi/+ndi/+vintage/lightsheetChunkName.m @@ -0,0 +1,33 @@ +function name = lightsheetChunkName(levelDoc, k) +%LIGHTSHEETCHUNKNAME The file name of a lightsheet level's K-th stored chunk. +% +% NAME = ndi.vintage.lightsheetChunkName(LEVELDOC, K) +% +% K is the chunk's ONE-BASED linear index in C order (the index +% ndi.fun.doc.lightsheet.makePyramid computes and has always put in the +% name). LEVELDOC is a level in either vintage: +% +% v1 lightsheetZarrLevel, series `chunk.bin`: chunk.bin_K +% V_eta sampled_body, series `body_data`: body_data_ +% +% The V_eta name is not a different chunk. DID-matlab +% +migrators_j/lightsheet_zarr_level.m renames the series chunk.bin -> +% body_data and every member chunk.bin_k -> body_data_{k-1}, because V_eta +% numbers series members from 0 and v1 from 1 (ndi.vintage.seriesMemberBase); +% both name the same manifest slot and the same bytes. +% +% See also: ndi.vintage.seriesMemberBase, ndi.vintage.lightsheetLevel. + +arguments + levelDoc + k (1,1) double {mustBeInteger, mustBePositive} +end + +props = ndi.v2.props(levelDoc); +if isfield(props, 'lightsheetZarrLevel') + stem = 'chunk.bin'; +else + stem = 'body_data'; +end +name = sprintf('%s_%d', stem, k - 1 + ndi.vintage.seriesMemberBase(levelDoc)); +end diff --git a/src/ndi/+ndi/+vintage/lightsheetKeys.m b/src/ndi/+ndi/+vintage/lightsheetKeys.m new file mode 100644 index 0000000000..9bdb0fb5ab --- /dev/null +++ b/src/ndi/+ndi/+vintage/lightsheetKeys.m @@ -0,0 +1,151 @@ +function f = lightsheetKeys(keys) +%LIGHTSHEETKEYS A V_eta lightsheet array's `keys`, read back into v1's terms. +% +% F = ndi.vintage.lightsheetKeys(KEYS) +% +% KEYS is the `keys` list of a migrated lightsheet body (a level's +% sampled_body, or the OME-Zarr store's opaque_body): one entry per array +% dimension, in dimension order, as DID-matlab +% +migrators_j/private/jLightsheetKeys.m writes them. It may arrive as a +% struct array or a cell array (a JSON round trip makes either). +% +% F has the v1 fields NDI's lightsheet readers use, per axis and in the +% keys' order: +% axes_order 'tczyx'-style: time -> t, channel -> c, z/y/x as named +% shape each key's `n` +% chunks each key's `chunk` ([] when no key has one) +% voxel_size the axis spacing IN THE SOURCE'S OWN UNIT +% translation the axis origin, likewise +% units cellstr, the source's unit per axis ('' if none) +% voxel_size_units the unit of the first space axis (v1 hard-coded +% 'micrometer' there; the migrator converts it) +% channel_names the `c` key's labels, comma-joined ('' if none) +% +% SOURCE UNITS, NOT CANONICAL ONES. The migrator converts a space or time +% axis to V_eta's canonical unit (meter, second) and keeps what the source +% said in spacing.source_value / origin.source_value / source_unit. v1 readers and their +% callers (chooseLevel's target voxel size, the napari viewer) work in the +% source's micrometres, so the source values are read where they exist and +% the canonical ones only where the source's were already canonical. +% +% AN AXIS WITH NO SPACING READS AS 1, ORIGIN AS 0. The `c` key is +% categorical (names or indices, no spacing). v1 stored the NGFF `scale` +% there, which for a channel axis is 1. This is a reader's convention so +% chooseLevel can compare whole vectors; it is not a value the document +% states. A space/time axis the source gave no scale is written by the +% migrator as REGULAR with origin 0, spacing 1 and NO unit (its indices +% 0..n-1; Jess 2026-10-15), so it reads as voxel size 1, translation 0 and +% unit '' from the key itself. (Before that it was written as `values` +% 0..n-1 and read as 1 and 0 by the convention above; both forms read the +% same.) +% +% See also: ndi.vintage.lightsheetLevel, ndi.vintage.lightsheetPyramid. + +list = asList(keys); +n = numel(list); +f = struct('axes_order', '', 'shape', zeros(1, n), 'chunks', [], ... + 'voxel_size', ones(1, n), 'translation', zeros(1, n), ... + 'units', {repmat({''}, 1, n)}, 'voxel_size_units', '', 'channel_names', ''); +chunks = nan(1, n); +axes = blanks(n); +for k = 1:n + key = list{k}; + v = lower(termName(get(key, 'variable'))); + switch v + case 'time', axes(k) = 't'; + case 'channel', axes(k) = 'c'; + case {'z', 'y', 'x'}, axes(k) = v; + otherwise + if ~isempty(v), axes(k) = v(1); else, axes(k) = '?'; end + end + f.shape(k) = double(firstOr(get(key, 'n'), 0)); + c = get(key, 'chunk'); + if ~isempty(c), chunks(k) = double(c(1)); end + + % did2.build.key writes a regular axis as origin = {value, + % source_value} and spacing = {value, source_value}: the canonical + % number and, when the unit was converted, the source's own. + unit = termName(get(key, 'source_unit')); + [spacing, sourceSpacing] = pairOf(get(key, 'spacing'), get(key, 'source_spacing')); + [origin, sourceOrigin] = pairOf(get(key, 'origin'), get(key, 'source_origin')); + if ~isempty(sourceSpacing) + spacing = sourceSpacing; + if ~isempty(sourceOrigin), origin = sourceOrigin; end + elseif isempty(unit) + unit = termName(get(key, 'unit')); + end + if ~isempty(spacing), f.voxel_size(k) = double(spacing(1)); end + if ~isempty(origin), f.translation(k) = double(origin(1)); end + f.units{k} = unit; + if axes(k) == 'c' + labels = get(key, 'labels'); + names = cellfun(@termName, asList(labels), 'UniformOutput', false); + f.channel_names = strjoin(names, ','); + end +end +f.axes_order = axes; +if any(~isnan(chunks)) + chunks(isnan(chunks)) = f.shape(isnan(chunks)); + f.chunks = chunks; +end +space = find(ismember(axes, 'zyx'), 1); +if ~isempty(space) + f.voxel_size_units = f.units{space}; +end +end + +function v = get(s, name) +v = []; +if isstruct(s) && isfield(s, name) + v = s.(name); +end +end + +function [v, source] = pairOf(p, flatSource) +% A key's origin or spacing: the {value, source_value} pair did2.build.key +% writes, or a bare number with the source in a sibling field (flatSource). +v = []; +source = flatSource; +if isstruct(p) + if isfield(p, 'value'), v = p(1).value; end + if isfield(p, 'source_value') && ~isempty(p(1).source_value) + source = p(1).source_value; + end +elseif isnumeric(p) + v = p; +end +end + +function v = firstOr(v, default) +if isempty(v) + v = default; +elseif iscell(v) + v = v{1}; +elseif ~ischar(v) + v = v(1); +end +end + +function n = termName(t) +% A term or a name: {node, name} struct, a bare name, or a one-element cell. +n = ''; +if iscell(t) + if isempty(t), return; end + t = t{1}; +end +if isstruct(t) && ~isempty(t) && isfield(t, 'name') + n = char(t(1).name); +elseif ischar(t) || isstring(t) + n = char(t); +end +end + +function c = asList(x) +if iscell(x) + c = reshape(x, 1, []); +elseif isstruct(x) + c = reshape(num2cell(x), 1, []); +else + c = {}; +end +end diff --git a/src/ndi/+ndi/+vintage/lightsheetLevel.m b/src/ndi/+ndi/+vintage/lightsheetLevel.m new file mode 100644 index 0000000000..3c271f9e1d --- /dev/null +++ b/src/ndi/+ndi/+vintage/lightsheetLevel.m @@ -0,0 +1,213 @@ +function [p, vintage] = lightsheetLevel(levelDoc, pyramidDoc) +%LIGHTSHEETLEVEL A lightsheet pyramid level's fields, by their v1 names, in either vintage. +% +% [P, VINTAGE] = ndi.vintage.lightsheetLevel(LEVELDOC) +% [P, VINTAGE] = ndi.vintage.lightsheetLevel(LEVELDOC, PYRAMIDDOC) +% +% v1 LEVELDOC (lightsheetZarrLevel, what NDI writes today): P IS the +% document's `lightsheetZarrLevel` block, untouched, and VINTAGE is 'v1'. +% Nothing is recomputed, so every v1 reader reads exactly what it read +% before. +% +% V_eta LEVELDOC (the `sampled_body` DID-matlab +% +migrators_j/lightsheet_zarr_level.m makes of a level that held bytes, +% its id kept, owner_id = the pyramid's observation): P is a struct with +% the v1 block's field names, read back from where the migrator put each +% value, and VINTAGE is 'V_eta': +% +% axes_order, shape, chunks, voxel_size, translation, voxel_size_units +% from the body's `keys` (ndi.vintage.lightsheetKeys; +% spacing and origin in the SOURCE's unit, as v1) +% chunk_grid ceil(shape ./ chunks), as makePyramid computes it +% n_chunks_stored the `body_data` series' slot count (series_info), +% else the number of body_data_ members it names +% chunk_index_origin 0: the first member is body_data_0 +% (ndi.vintage.seriesMemberBase). v1 says 1. +% chunk_order sampled_body.datum_order +% byte_order, fill_value sampled_body's (fill_value parsed as a number) +% codec 'raw' when the body has no `compression`, else the +% compression ('blosc' for v1's 'blosc-zstd': a Blosc +% chunk's header carries its own settings, #73 L4) +% reduction_function the `summary statistic` condition: NCIT:C25564 -> +% 'max', NCIT:C53319 -> 'mean', else the term's name; +% 'none' for a body with no such condition +% redundant data_body.redundant (false on level 0) +% dtype the PYRAMID's: its observation's +% value.source_data_type (the v1 spelling, ' + end +elseif isstruct(conditions) + list = num2cell(reshape(conditions, 1, [])); +end +for i = 1:numel(list) + c = list{i}; + if ~isfield(c, 'variable') || ~strcmpi(termOf(c.variable, 'name'), 'summary statistic') + continue; + end + t = []; + if isfield(c, 'term') && isstruct(c.term) && isfield(c.term, 'value') + t = c.term.value; + end + switch termOf(t, 'node') + case 'NCIT:C25564', r = 'max'; + case 'NCIT:C53319', r = 'mean'; + otherwise + name = lower(termOf(t, 'name')); + switch name + case 'maximum', r = 'max'; + case 'arithmetic mean', r = 'mean'; + case 'minimum', r = 'min'; + case '', r = 'none'; + otherwise, r = name; + end + end + return; +end +end + +function s = termOf(t, part) +s = ''; +if iscell(t) + if isempty(t), return; end + t = t{1}; +end +if isstruct(t) && ~isempty(t) && isfield(t, part) + s = char(t(1).(part)); +elseif strcmp(part, 'name') && (ischar(t) || isstring(t)) + s = char(t); +end +end + +function n = storedCount(props) +n = 0; +if ~isfield(props, 'files') || ~isstruct(props.files) + return; +end +files = props.files; +if isfield(files, 'series_info') && ~isempty(files.series_info) + si = files.series_info; + if iscell(si), si = [si{:}]; end + for i = 1:numel(si) + if isfield(si(i), 'name') && strcmp(char(si(i).name), 'body_data') ... + && isfield(si(i), 'count') && ~isempty(si(i).count) + n = double(si(i).count); + return; + end + end +end +if isfield(files, 'file_info') && ~isempty(files.file_info) + fi = files.file_info; + if iscell(fi), fi = [fi{:}]; end + names = arrayfun(@(x) char(x.name), fi, 'UniformOutput', false); + n = sum(~cellfun(@isempty, regexp(names, '^body_data_\d+$', 'once'))); +end +end + +function d = pyramidDtype(pp) +d = charOr(ndi.v2.blockOf(pp, 'value', 'source_data_type'), ''); +if isempty(d) + d = charOr(ndi.v2.blockOf(pp, 'value', 'data_type'), ''); +end +if isempty(d) && isfield(pp, 'lightsheetZarrPyramid') + d = charOr(pp.lightsheetZarrPyramid.dtype, ''); +end +end + +function v = fillOf(x) +v = 0; +if isempty(x), return; end +if ischar(x) || isstring(x) + v = str2double(x); +else + v = double(x(1)); +end +end + +function s = charOr(x, default) +if isempty(x) + s = default; +elseif iscell(x) + s = char(x{1}); +else + s = char(x); +end +end diff --git a/src/ndi/+ndi/+vintage/lightsheetLevelDocs.m b/src/ndi/+ndi/+vintage/lightsheetLevelDocs.m new file mode 100644 index 0000000000..faf7328155 --- /dev/null +++ b/src/ndi/+ndi/+vintage/lightsheetLevelDocs.m @@ -0,0 +1,36 @@ +function docs = lightsheetLevelDocs(session, pyramidDoc) +%LIGHTSHEETLEVELDOCS The level documents of one lightsheet pyramid, in its vintage. +% +% DOCS = ndi.vintage.lightsheetLevelDocs(SESSION, PYRAMIDDOC) +% +% PYRAMIDDOC is an ndi.document (either vintage). DOCS is the cell +% SESSION.database_search returns for: +% +% v1 isa lightsheetZarrLevel, depends_on lightsheetZarrPyramid_id = +% the pyramid's id -- EXACTLY the query the lightsheet readers +% have always run, so a v1 session is searched as before. +% V_eta isa sampled_body, depends_on owner_id = the pyramid's id. The +% migration keeps the pyramid's id on its observation and each +% level's id on its sampled_body, whose owner_id names the +% observation (DID-matlab +migrators_j/lightsheet_zarr_level.m). +% Only the levels are sampled_bodies: the store record the +% observation also owns is an opaque_body. +% +% `isa sampled_body` does not bridge onto `isa lightsheetZarrLevel` the way +% ndi.vintage.isaQuery bridges a renamed class: sampled_body is every V_eta +% sampled array, so only the owner edge makes one a level. That is also why +% lightsheet has no ndi.vintage.map row (see ndi.vintage.lightsheetPyramidDocs). +% +% See also: ndi.vintage.lightsheetLevels, ndi.vintage.lightsheetPyramidDocs. + +props = ndi.v2.props(pyramidDoc); +id = props.base.id; +if isfield(props, 'lightsheetZarrPyramid') + q1 = ndi.query('', 'isa', 'lightsheetZarrLevel'); + q2 = ndi.query('depends_on', 'depends_on', 'lightsheetZarrPyramid_id', id); +else + q1 = ndi.query('', 'isa', 'sampled_body', ''); + q2 = ndi.query('', 'depends_on', 'owner_id', id); +end +docs = session.database_search(q1 & q2); +end diff --git a/src/ndi/+ndi/+vintage/lightsheetLevels.m b/src/ndi/+ndi/+vintage/lightsheetLevels.m new file mode 100644 index 0000000000..aff7446ad1 --- /dev/null +++ b/src/ndi/+ndi/+vintage/lightsheetLevels.m @@ -0,0 +1,57 @@ +function [recs, vintages] = lightsheetLevels(levelDocs, pyramidDoc) +%LIGHTSHEETLEVELS Every level of one pyramid, by v1 field names, with level indices. +% +% [RECS, VINTAGES] = ndi.vintage.lightsheetLevels(LEVELDOCS, PYRAMIDDOC) +% +% LEVELDOCS is a cell of level documents of ONE pyramid (as +% ndi.vintage.lightsheetLevelDocs returns them). RECS is a 1-by-N cell of +% ndi.vintage.lightsheetLevel structs in the same order, VINTAGES a cellstr. +% +% A v1 level carries its own `level` and is returned as it is. +% +% A V_eta level does not carry its index: DID-matlab +% +migrators_j/lightsheet_zarr_level.m drops `level` as derivable. It is +% derived here from the set, by the two facts the migrator does keep: +% * level 0 is the one body NOT `redundant` (the measurement, #73 L1); +% * every other level is `redundant`, and within one reduction its index +% is its rank by resolution, finest first: 1, 2, ... in order of +% decreasing voxel count (prod(shape)). +% KNOWN LIMIT, carried rather than guessed around: a v1 level that held no +% bytes folded to NOTHING in the migration (#73 L3(c)), so if an +% intermediate level of a ladder was never materialized, the levels above +% it rank one lower than their v1 index. The order and the shapes are +% right; only the integer can differ. +% +% See also: ndi.vintage.lightsheetLevel, ndi.fun.doc.lightsheet.levelTable. + +n = numel(levelDocs); +recs = cell(1, n); +vintages = cell(1, n); +for k = 1:n + [recs{k}, vintages{k}] = ndi.vintage.lightsheetLevel(levelDocs{k}, pyramidDoc); +end + +eta = find(strcmp(vintages, 'V_eta')); +if isempty(eta) + return; +end +redundant = cellfun(@(r) r.redundant, recs(eta)); +for k = reshape(eta(~redundant), 1, []) + recs{k}.level = 0; +end +red = eta(redundant); +if isempty(red) + return; +end +reductions = cellfun(@(r) r.reduction_function, recs(red), 'UniformOutput', false); +groups = unique(reductions); +for gi = 1:numel(groups) + members = red(strcmp(reductions, groups{gi})); + voxels = cellfun(@(r) prod(double(r.shape)), recs(members)); + ladder = unique(voxels); % ascending + ladder = ladder(end:-1:1); % finest (most voxels) first + for k = reshape(members, 1, []) + recs{k}.level = find(ladder == prod(double(recs{k}.shape)), 1); + end +end +end diff --git a/src/ndi/+ndi/+vintage/lightsheetMethod.m b/src/ndi/+ndi/+vintage/lightsheetMethod.m new file mode 100644 index 0000000000..e6f0d2e74f --- /dev/null +++ b/src/ndi/+ndi/+vintage/lightsheetMethod.m @@ -0,0 +1,25 @@ +function m = lightsheetMethod() +%LIGHTSHEETMETHOD The method term that marks a migrated lightsheet pyramid. +% +% M = ndi.vintage.lightsheetMethod() +% +% M is struct('node', 'FBbi:00000369', 'name', 'selective plane +% illumination fluorescence microscopy') -- light-sheet microscopy (checked +% on OLS). DID-matlab +migrators_j/lightsheet_zarr_pyramid.m sets it as +% `interaction.method` on the intensity observation it makes of EVERY v1 +% lightsheetZarrPyramid, OME-Zarr and SmartSPIM alike (Jess 2026-10-15). It +% is what says "lightsheet pyramid" rather than "some fluorescence +% measurement", so ndi.vintage.lightsheetPyramidDocs finds migrated pyramids +% by its node. It must stay the migrator's, character for character; +% ndi.unittest.vintage.TestLightsheetMigrated pins it against the migrator's +% output when DID-matlab's migrators are on the path. +% +% (Until 2026-10-15 the marker was the name of the builder `software` +% entity the observation's software_id names. That entity is still +% written, as the record of what built the pyramid; it no longer marks it.) +% +% See also: ndi.vintage.lightsheetPyramidDocs. + +m = struct('node', 'FBbi:00000369', ... + 'name', 'selective plane illumination fluorescence microscopy'); +end diff --git a/src/ndi/+ndi/+vintage/lightsheetPyramid.m b/src/ndi/+ndi/+vintage/lightsheetPyramid.m new file mode 100644 index 0000000000..57bd197a87 --- /dev/null +++ b/src/ndi/+ndi/+vintage/lightsheetPyramid.m @@ -0,0 +1,166 @@ +function [p, vintage] = lightsheetPyramid(pyramidDoc, session, levelRecs) +%LIGHTSHEETPYRAMID A lightsheet pyramid's fields, by their v1 names, in either vintage. +% +% [P, VINTAGE] = ndi.vintage.lightsheetPyramid(PYRAMIDDOC) +% [P, VINTAGE] = ndi.vintage.lightsheetPyramid(PYRAMIDDOC, SESSION) +% [P, VINTAGE] = ndi.vintage.lightsheetPyramid(PYRAMIDDOC, SESSION, LEVELRECS) +% +% v1 PYRAMIDDOC (lightsheetZarrPyramid, what NDI writes today): P IS the +% document's `lightsheetZarrPyramid` block, untouched; VINTAGE 'v1'. SESSION +% and LEVELRECS are not used. +% +% V_eta PYRAMIDDOC (the intensity `observation` DID-matlab +% +migrators_j/lightsheet_zarr_pyramid.m makes of it, its id kept): P has +% the v1 block's field names, read from where the migration put each one: +% +% dtype value.source_data_type (the v1 spelling), else +% value.data_type +% axes_order, shape_level0, voxel_size_level0, translation_level0, +% voxel_size_units, axes_units, channel_names, n_channels +% the full-resolution array's keys. The migration moves +% them onto the OME-Zarr STORE's unheld opaque_body (owned +% by the observation; did2.convert.foldGenericFiles), so +% with SESSION that body is read first. A SmartSPIM +% store's body (a TIFF folder) has no keys, and without +% SESSION there is no body to read: then level 0's keys +% (LEVELRECS, ndi.vintage.lightsheetLevels), which the +% same pass completed from the pyramid's units and +% channel names. +% label the observation's `statement.notes` (Jess +% 2026-10-15: the label stays there, permanently). When +% the batch pass did not run, the notes also carry, after +% a newline and '-- store body: ', the text the pass +% would have moved to the store body; only the part +% before the LAST such separator is the label. The +% separator is DID-matlab's lightsheet_zarr_pyramid.m / +% foldGenericFiles.m NOTES_SEPARATOR, character for +% character. +% pyramid_name parsed from the store body's `description` +% ('NGFF multiscales name "..."'); '' when there is no +% store body (or no SESSION). +% pyramid_type '' always: the migration DROPS it (Jess 2026-10-15; it +% repeats the levels' summary statistic, and the v1 +% docs say not to trust it). +% pipeline_version ''. It became a `version` text_assertion about the +% software entity, not read here. +% byte_order level 0's, if given. +% +% See also: ndi.vintage.lightsheetLevels, ndi.vintage.lightsheetPyramidDocs, +% ndi.fun.doc.lightsheet.levelTable. + +if nargin < 2, session = []; end +if nargin < 3, levelRecs = {}; end +props = ndi.v2.props(pyramidDoc); +if isfield(props, 'lightsheetZarrPyramid') + p = props.lightsheetZarrPyramid; + vintage = 'v1'; + return; +end +vintage = 'V_eta'; + +p = struct('label', '', 'pyramid_name', '', 'pyramid_type', '', ... + 'axes_order', '', 'axes_units', '', 'channel_names', '', 'n_channels', 0, ... + 'shape_level0', [], 'voxel_size_level0', [], 'voxel_size_units', '', ... + 'translation_level0', [], 'dtype', '', 'pipeline_version', '', 'byte_order', ''); + +p.label = labelOf(charOr(ndi.v2.blockOf(props, 'statement', 'notes', ''))); + +p.dtype = charOr(ndi.v2.blockOf(props, 'value', 'source_data_type')); +if isempty(p.dtype) + p.dtype = charOr(ndi.v2.blockOf(props, 'value', 'data_type')); +end + +frame = []; +if ~isempty(session) + stores = session.database_search(ndi.query('', 'isa', 'opaque_body', '') & ... + ndi.query('', 'depends_on', 'owner_id', props.base.id)); + for k = 1:numel(stores) + sp = stores{k}.document_properties; + d = blockValue(sp, 'description'); + if ~isempty(d) + p.pyramid_name = quoted(charOr(d), 'NGFF multiscales name'); + end + keys = blockValue(sp, 'keys'); + if ~isempty(keys) && isempty(frame) + frame = ndi.vintage.lightsheetKeys(keys); + end + end +end + +level0 = []; +for k = 1:numel(levelRecs) + r = levelRecs{k}; + if isfield(r, 'level') && isequal(r.level, 0) + level0 = r; + break; + end +end +if isempty(frame) && ~isempty(level0) + frame = struct('axes_order', level0.axes_order, 'shape', level0.shape, ... + 'voxel_size', level0.voxel_size, 'translation', level0.translation, ... + 'voxel_size_units', level0.voxel_size_units, 'units', {{}}, 'channel_names', ''); +end +if ~isempty(level0) && isfield(level0, 'byte_order') + p.byte_order = level0.byte_order; +end +if isempty(frame) + return; +end +p.axes_order = frame.axes_order; +p.shape_level0 = frame.shape; +p.voxel_size_level0 = frame.voxel_size; +p.translation_level0 = frame.translation; +p.voxel_size_units = frame.voxel_size_units; +if ~isempty(frame.units) + p.axes_units = strjoin(frame.units, ','); +end +p.channel_names = frame.channel_names; +c = find(frame.axes_order == 'c', 1); +if ~isempty(c) + p.n_channels = frame.shape(c); +end +end + +function label = labelOf(notes) +% The label: NOTES up to the LAST separator DID-matlab's migrator writes +% before the text its store-record pass moves out (present only when that pass +% did not run), else all of NOTES. +SEPARATOR = sprintf('\n-- store body: '); +label = notes; +at = strfind(notes, SEPARATOR); +if ~isempty(at) + label = notes(1:at(end) - 1); +end +end + +function s = quoted(text, label) +s = ''; +tok = regexp(text, [regexptranslate('escape', label) ' "([^"]*)"'], 'tokens', 'once'); +if ~isempty(tok) + s = tok{1}; +end +end + +function v = blockValue(props, name) +v = []; +skip = {'base', 'document_class', 'depends_on', 'files'}; +names = fieldnames(props); +for i = 1:numel(names) + if any(strcmp(names{i}, skip)), continue; end + b = props.(names{i}); + if isstruct(b) && isscalar(b) && isfield(b, name) && ~isempty(b.(name)) + v = b.(name); + return; + end +end +end + +function s = charOr(x) +if isempty(x) + s = ''; +elseif iscell(x) + s = char(x{1}); +else + s = char(x); +end +end diff --git a/src/ndi/+ndi/+vintage/lightsheetPyramidDocs.m b/src/ndi/+ndi/+vintage/lightsheetPyramidDocs.m new file mode 100644 index 0000000000..1332a463b4 --- /dev/null +++ b/src/ndi/+ndi/+vintage/lightsheetPyramidDocs.m @@ -0,0 +1,48 @@ +function docs = lightsheetPyramidDocs(session) +%LIGHTSHEETPYRAMIDDOCS Every lightsheet pyramid in a session, in both vintages. +% +% DOCS = ndi.vintage.lightsheetPyramidDocs(SESSION) +% +% v1 pyramids first, found exactly as before (isa lightsheetZarrPyramid), +% then migrated ones. +% +% HOW A MIGRATED PYRAMID IS RECOGNISED. DID-matlab +% +migrators_j/lightsheet_zarr_pyramid.m makes each v1 pyramid an intensity +% `observation` (its id kept) whose `interaction.method` is +% ndi.vintage.lightsheetMethod() -- FBbi:00000369 'selective plane +% illumination fluorescence microscopy' (Jess 2026-10-15). Its class +% (`observation`), value kind (`intensity`) and variable (NCIT:C184706 +% 'Fluorescence Intensity') are shared by any fluorescence measurement; the +% method is what says light-sheet. So a migrated pyramid is an observation +% whose method node is that term, found in ONE query on the field path +% `interaction.method.node` (`interaction` declares `method`, an +% ontology_term {node, name}; ndi.v2.fieldQuery also asks the block's name +% before 2026-10-08, `subject_interaction`). +% +% (Until 2026-10-15 it was found through the builder `software` entity's +% name. The entity is still written; it no longer marks the pyramid.) +% +% This is why lightsheet has no ndi.vintage.map row. ndi.vintage.entryFor +% keys on the class name alone, and `observation` / `sampled_body` are every +% V_eta statement and array: a row would reroute the edge and field reads +% ndi.vintage.edge / field do for EVERY observation in a session (e.g. the +% time-reference reads in ndi.file.navigator.bodies), not just pyramids. +% +% See also: ndi.vintage.lightsheetMethod, ndi.vintage.lightsheetLevelDocs, +% ndi.vintage.lightsheetPyramid. + +M = ndi.vintage.lightsheetMethod(); + +docs = session.database_search(ndi.query('', 'isa', 'lightsheetZarrPyramid')); +docs = reshape(docs, 1, []); + +obs = session.database_search(ndi.v2.isaQuery('observation') & ... + ndi.v2.fieldQuery('interaction.method.node', 'exact_string', M.node)); +seen = cellfun(@(d) d.id(), docs, 'UniformOutput', false); +for j = 1:numel(obs) + id = obs{j}.id(); + if any(strcmp(seen, id)), continue; end + seen{end+1} = id; %#ok + docs{end+1} = obs{j}; %#ok +end +end diff --git a/src/ndi/+ndi/+vintage/map.m b/src/ndi/+ndi/+vintage/map.m index 5d71a4b2dc..8e76611325 100644 --- a/src/ndi/+ndi/+vintage/map.m +++ b/src/ndi/+ndi/+vintage/map.m @@ -28,7 +28,10 @@ % ------------ % concept the v1 class name, and the key callers use % v1_class the class a v1 document declares -% eta_class the class a V_eta document declares +% eta_class the class a V_eta document declares: a char, or a +% cellstr when V_eta renamed the class more than once +% (CURRENT name first, then older ones still found in +% documents written before the rename) % object_field v1: the block field holding the MATLAB class to % construct (the "object-reconstruction key") % object_edge V_eta: the edge to a `software` entity whose `name` @@ -36,12 +39,23 @@ % the class string into an entity -- see % DID-matlab +migrators_j/private/jSoftware.m, called % with the implementation class as NAME. -% edges N-by-2 cellstr, {v1_edge, eta_edge}. Base names: a +% edges N-by-2 cell, {v1_edge, eta_edge}. Base names: a % numbered family (`daqmetadatareader_id_1`, ...) is % stored by its base and expanded by ndi.vintage.edge_n. -% fields N-by-2 cellstr, {v1_field, eta_field}, within the +% fields N-by-2 cell, {v1_field, eta_field}, within the % document's own block. % +% ONE V_eta NAME OR SEVERAL. An eta_edge or eta_field may be a cellstr of +% candidates, current name first. V_eta kept changing after documents +% were written: did-schema's #73 review renamed `data_file_pattern` and +% `metadata_file_pattern` to `file_pattern` / `file_regex`, +% `acquisition_metadata_reader` to `epoch_parameter_reader`, and (T15) +% numbered edge families such as `clock_alignment_configuration_#` to one +% repeated name `clock_alignment_configuration_id`. The DID-matlab +% migrators still emit the older names, while documents built with +% did2.build carry the current ones, so both are live. Readers take the +% first candidate the document actually carries. +% % WHAT IS ABSENT, AND A CORRECTION TO WHY % --------------------------------------- % THIS SECTION USED TO SAY `element` "is NOT here, and cannot be added by @@ -124,7 +138,8 @@ e(k).isa_bridges = true; e(k).edges = {'daqreader_id', 'reader_id'; ... 'filenavigator_id', 'epoch_file_pattern_id'; ... - 'daqmetadatareader_id', 'acquisition_metadata_reader'}; + 'daqmetadatareader_id', {'epoch_parameter_reader_id', ... + 'acquisition_metadata_reader'}}; e(k).fields = cell(0,2); % ---- filenavigator -> epoch_file_pattern ----------------------------- @@ -138,7 +153,7 @@ e(k).object_assertion = ''; e(k).isa_bridges = true; e(k).edges = cell(0,2); -e(k).fields = {'fileparameters', 'data_file_pattern'; ... +e(k).fields = {'fileparameters', {'file_pattern', 'data_file_pattern'}; ... 'epochprobemap_fileparameters', 'epoch_map_pattern'; ... 'epochprobemap_class', 'epoch_map_format'}; @@ -146,13 +161,13 @@ k = k + 1; e(k).concept = 'daqmetadatareader'; e(k).v1_class = 'daqmetadatareader'; -e(k).eta_class = 'acquisition_metadata_reader'; +e(k).eta_class = {'epoch_parameter_reader', 'acquisition_metadata_reader'}; e(k).object_field = 'ndi_daqmetadatareader_class'; e(k).object_edge = 'software_id'; e(k).object_assertion = ''; e(k).isa_bridges = true; e(k).edges = cell(0,2); -e(k).fields = {'tab_separated_file_parameter', 'metadata_file_pattern'}; +e(k).fields = {'tab_separated_file_parameter', {'file_regex', 'metadata_file_pattern'}}; % ---- daqreader -> acquisition_reader --------------------------------- % `daqreader_ndr` folds into this too (the reader subtype was encoded in @@ -188,7 +203,8 @@ e(k).object_edge = 'software_id'; e(k).object_assertion = ''; e(k).isa_bridges = true; -e(k).edges = {'syncrule_id', 'clock_alignment_configuration'}; +e(k).edges = {'syncrule_id', {'clock_alignment_configuration_id', ... + 'clock_alignment_configuration'}}; e(k).fields = cell(0,2); % ---- syncrule -> clock_alignment_configuration ----------------------- @@ -204,7 +220,8 @@ e(k).object_edge = 'software_id'; e(k).object_assertion = ''; e(k).isa_bridges = true; -e(k).edges = {'acquisition_channels', 'acquisition_channels'}; +e(k).edges = {'acquisition_channels', {'acquisition_channels_id', ... + 'acquisition_channels'}}; e(k).fields = cell(0,2); % ---- element -> subject (+ satellites) ------------------------------- diff --git a/src/ndi/+ndi/+vintage/names.m b/src/ndi/+ndi/+vintage/names.m new file mode 100644 index 0000000000..3463ce0746 --- /dev/null +++ b/src/ndi/+ndi/+vintage/names.m @@ -0,0 +1,19 @@ +function c = names(x) +%NAMES The candidate V_eta names held in one ndi.vintage.map cell, as a list. +% +% C = ndi.vintage.names(X) returns X as a row cellstr: {X} for a char, +% X itself for a cellstr. A map entry's `eta_class`, and the V_eta side of +% an `edges` or `fields` row, is either one name or a list of candidates +% (current name first, then older ones -- see ndi.vintage.map), and every +% reader iterates the list the same way. +% +% See also: ndi.vintage.map, ndi.vintage.entryFor. + +if ischar(x) || (isstring(x) && isscalar(x)) + c = {char(x)}; +elseif iscell(x) + c = reshape(cellfun(@char, x, 'UniformOutput', false), 1, []); +else + c = {}; +end +end diff --git a/src/ndi/+ndi/+vintage/objectClass.m b/src/ndi/+ndi/+vintage/objectClass.m index 5d3a1e7eff..e82ed85a51 100644 --- a/src/ndi/+ndi/+vintage/objectClass.m +++ b/src/ndi/+ndi/+vintage/objectClass.m @@ -76,7 +76,7 @@ ['"%s" (v1 "%s") is not an NDI object type -- ndi.vintage.map ' ... 'carries it for its class rename only, and v1 stored no ' ... 'implementation class on it. There is nothing to construct.'], ... - entry.eta_class, entry.v1_class); + className, entry.v1_class); end % V_eta: one hop to the software entity. @@ -88,7 +88,7 @@ 'The migrator leaves this edge empty only when the source named ' ... 'no class (+migrators_j/daqsystem.m: jSoftware returns [] for an ' ... 'empty name).'], ... - entry.eta_class, props.base.id, entry.object_edge, entry.object_field); + className, props.base.id, entry.object_edge, entry.object_field); end swDocs = ndi_session_obj.database_search( ... @@ -97,17 +97,18 @@ error('NDI:vintage:softwareNotFound', ... ['"%s".%s names software document %s and %d document(s) match; ' ... 'the object class lives on that document'], ... - entry.eta_class, entry.object_edge, swId, numel(swDocs)); + className, entry.object_edge, swId, numel(swDocs)); end swProps = swDocs{1}.document_properties; -if ~isfield(swProps, 'software') || ~isstruct(swProps.software) ... - || ~isfield(swProps.software, 'name') || isempty(swProps.software.name) +% the `software` block, or from 2026-10-08 an `entity` of type software +objClass = ndi.v2.blockOf(swProps, 'software', 'name'); +if isempty(objClass) error('NDI:vintage:softwareHasNoName', ... ['software document %s carries no `name`, and `name` is where ' ... 'jSoftware puts the implementation class string'], swId); end -objClass = swProps.software.name; +objClass = char(objClass); end function v = assertionValue(ndi_document_obj, ndi_session_obj, label) @@ -118,8 +119,8 @@ % (DID-matlab +migrators_j/element.m, kindAssertion). docId = ndi_document_obj.document_properties.base.id; q = ndi.query('', 'isa', 'term_assertion', '') & ... - ndi.query('subject_statement.variable.name', 'exact_string', label, '') & ... - ndi.query('', 'depends_on', 'subject_id', docId); + ndi.v2.fieldQuery('statement.variable.name', 'exact_string', label) & ... + ndi.v2.entityQuery(docId); docs = ndi_session_obj.database_search(q); if numel(docs) ~= 1 error('NDI:vintage:assertionNotFound', ... diff --git a/src/ndi/+ndi/+vintage/objectName.m b/src/ndi/+ndi/+vintage/objectName.m new file mode 100644 index 0000000000..f26af51489 --- /dev/null +++ b/src/ndi/+ndi/+vintage/objectName.m @@ -0,0 +1,27 @@ +function name = objectName(ndi_document_obj) +%OBJECTNAME The name an NDI object document carries, either vintage. +% +% NAME = ndi.vintage.objectName(NDI_DOCUMENT_OBJ) +% +% v1 keeps an object's name in `base.name` (a daq system's 'intan1'). +% did-schema #73 item 54 moved names onto the classes that have one, so a +% document built to the current V_eta schema carries it in its own block +% (`acquisition_system.name`), and `base.name` is did_v1-only. Documents +% migrated before that change still carry `base.name`. This returns the +% block's `name` when the document has a non-empty one, else `base.name`, +% else ''. +% +% See also: ndi.vintage.field, ndi.vintage.map. + +name = ''; +props = ndi_document_obj.document_properties; +blockName = props.document_class.class_name; +if isfield(props, blockName) && isstruct(props.(blockName)) && isscalar(props.(blockName)) ... + && isfield(props.(blockName), 'name') && ~isempty(props.(blockName).name) + name = props.(blockName).name; + return; +end +if isfield(props, 'base') && isfield(props.base, 'name') && ~isempty(props.base.name) + name = props.base.name; +end +end diff --git a/src/ndi/+ndi/+vintage/seriesMemberBase.m b/src/ndi/+ndi/+vintage/seriesMemberBase.m new file mode 100644 index 0000000000..4322dd0aa6 --- /dev/null +++ b/src/ndi/+ndi/+vintage/seriesMemberBase.m @@ -0,0 +1,69 @@ +function base = seriesMemberBase(doc) +%SERIESMEMBERBASE The number of a file series' FIRST member: 1 (v1) or 0 (V_eta). +% +% BASE = ndi.vintage.seriesMemberBase(DOC) +% +% DOC is an ndi.document, a did.document or a document_properties struct. +% A file series names its members NAME_. The numbering depends on +% the DOCUMENT'S VINTAGE, never on a file name: +% 1 a v1 document (no document_class.schema_version, or one before +% V_eta): NAME_1 is manifest slot 0. This is what NDI writes today +% (makePyramid's chunk.bin_1.., the ingested epoch series). +% 0 a V_eta document: NAME_0 is slot 0 (DID-schema #73 item 17, kept by +% Jess 2026-10-14), so a migrated lightsheet level's chunk.bin_k is +% body_data_{k-1}. +% The manifest is the same either way; only the names differ. +% +% THE SAME RULE DID USES. When the DID-matlab on the path has +% did.document/seriesMemberBase (DID-matlab 1a6bd9f), this asks it, so NDI +% and DID can never disagree. An older DID-matlab (the V2 line NDI's +% requirements.txt pins, which predates it) gets the same rule restated +% here: rank document_class.schema_version with +% did2.convert.schemaVersionRank when that exists, else compare it to +% 'V_eta'. A version neither recognises is v1 (base 1), which is what every +% document written before V_eta is. +% +% Callers that walk a series' slots (ndi.database.fun.extract_docs_files, +% ndi.database.internal.list_binary_files) take member m (1-based slot) as +% NAME_. +% +% See also: ndi.vintage.lightsheetChunkName, did.document/seriesMemberOf. + +base = 1; +if isa(doc, 'did.document') + if ismethod(doc, 'seriesMemberBase') + base = double(doc.seriesMemberBase()); + return; + end + props = doc.document_properties; +elseif isstruct(doc) + props = doc; +else + return; +end + +if ~isfield(props, 'document_class') || ~isstruct(props.document_class) ... + || ~isfield(props.document_class, 'schema_version') + return; +end +sv = props.document_class.schema_version; +if isstring(sv) && isscalar(sv), sv = char(sv); end +if ~ischar(sv) || isempty(sv) + return; +end +if ~isempty(which('did2.convert.schemaVersionRank')) + try + [rank, known] = did2.convert.schemaVersionRank(sv); + etaRank = did2.convert.schemaVersionRank('V_eta'); + if known && rank >= etaRank + base = 0; + end + return; + catch + % fall through to the literal comparison + end +end +if strcmp(sv, 'V_eta') + base = 0; +end +end diff --git a/src/ndi/+ndi/+vintage/syncruleParameters.m b/src/ndi/+ndi/+vintage/syncruleParameters.m index 3b2ffcef0b..0911710ced 100644 --- a/src/ndi/+ndi/+vintage/syncruleParameters.m +++ b/src/ndi/+ndi/+vintage/syncruleParameters.m @@ -16,7 +16,9 @@ % This is that reader change. V_eta does three different things to one v1 % `parameters` bag: % -% RENAMED into typed fields on `clock_alignment_configuration` +% RENAMED into typed fields on `clock_alignment_configuration` (since +% did-schema #73 item 22, `method_parameters` entries named by +% `variable`; both forms are read) % epochclocktype -> clock (an ontology_term; the name half) % number_fullpath_matches -> minimum_matching_file_paths % syncfilename -> sync_file_name @@ -24,6 +26,8 @@ % % MOVED OUT into two `acquisition_channels` documents, one per device. % daqsystem1_name / daqsystem1 <- acquisition_channels_1.base.name +% (current schema: its +% acquisition_system's `name`) % daqsystem_ch1 <- its `channels` array, re-serialised % (and _2 likewise). The pair is UNORDERED by construction -- the % migrator's own note: the rule is symmetric and both consumers @@ -71,13 +75,19 @@ props = ruleDoc.document_properties; cfg = struct(); -if isfield(props, entry.eta_class) && isstruct(props.(entry.eta_class)) - cfg = props.(entry.eta_class); +blockName = props.document_class.class_name; +if isfield(props, blockName) && isstruct(props.(blockName)) + cfg = props.(blockName); end % ---- the two device documents ---------------------------------------- -[name1, ch1] = deviceHalf(ruleDoc, ndi_session_obj, 1); -[name2, ch2] = deviceHalf(ruleDoc, ndi_session_obj, 2); +% A numbered pair (`acquisition_channels_1`/`_2`, as the migrators write it) +% or one repeated name (`acquisition_channels_id` twice, did-schema T15). +% Unordered either way, so 1 and 2 are positions. +channelIds = ndi.vintage.edge_n(ruleDoc, 'acquisition_channels', ... + 'ErrorIfNotFound', 0); +[name1, ch1] = deviceHalf(channelIds, ndi_session_obj, 1); +[name2, ch2] = deviceHalf(channelIds, ndi_session_obj, 2); % ---- the typed fields ------------------------------------------------- clockName = ''; @@ -128,15 +138,15 @@ % ===================== helpers ============================================= -function [deviceName, channelSpec] = deviceHalf(ruleDoc, ndi_session_obj, n) -%DEVICEHALF Resolve `acquisition_channels_` to a device name + channel spec. +function [deviceName, channelSpec] = deviceHalf(channelIds, ndi_session_obj, n) +%DEVICEHALF Resolve the n-th channel group to a device name + channel spec. deviceName = ''; channelSpec = ''; -edgeName = sprintf('acquisition_channels_%d', n); -id = ruleDoc.dependency_value(edgeName, 'ErrorIfNotFound', 0); -if isempty(id) +edgeName = sprintf('acquisition_channels (%d)', n); +if numel(channelIds) < n || isempty(channelIds{n}) return; end +id = channelIds{n}; docs = ndi_session_obj.database_search( ... ndi.query('base.id', 'exact_string', id, '')); if numel(docs) ~= 1 @@ -154,6 +164,16 @@ if isfield(p, 'base') && isfield(p.base, 'name') deviceName = p.base.name; end +% ...EXCEPT ON A DOCUMENT BUILT TO THE CURRENT SCHEMA. did-schema #73 item 54 +% keeps a device's name on its acquisition_system (`acquisition_system.name`) +% and `base.name` is did_v1-only, so such a channel group names no device +% itself and reaches it through `acquisition_system_id`. +if isempty(deviceName) + sysDoc = linkedDoc(docs{1}, ndi_session_obj, 'acquisition_system_id'); + if ~isempty(sysDoc) + deviceName = ndi.vintage.objectName(sysDoc); + end +end if isfield(p, 'acquisition_channels') && isstruct(p.acquisition_channels) ... && isfield(p.acquisition_channels, 'channels') channelSpec = channelsToSpec(p.acquisition_channels.channels); @@ -204,6 +224,56 @@ v = ''; if isstruct(s) && isfield(s, name) v = s.(name); + return; +end +e = methodParameter(s, name); +if ~isempty(e) + if isfield(e, 'text') && ~isempty(e.text) + v = e.text; + elseif isfield(e, 'value') && isstruct(e.value) && isfield(e.value, 'source_value') + v = e.value.source_value; + end +end +end + +function d = linkedDoc(doc, ndi_session_obj, edgeName) +%LINKEDDOC The one document an edge names, or [] if it names none. +d = []; +id = doc.dependency_value(edgeName, 'ErrorIfNotFound', 0); +if isempty(id) + return; +end +hits = ndi_session_obj.database_search(ndi.query('base.id', 'exact_string', id, '')); +if numel(hits) == 1 + d = hits{1}; +end +end + +function e = methodParameter(cfg, name) +%METHODPARAMETER A knob held in `method_parameters`, or [] if absent. +% did-schema #73 item 22 turned the typed fields into `parameter[]` +% entries named by `variable`. Until that variable is bound it is a LABEL +% ({name}, no node), so an entry is matched by the V_eta field name or by +% the v1 parameter name it came from. +e = []; +if ~isstruct(cfg) || ~isfield(cfg, 'method_parameters') || isempty(cfg.method_parameters) + return; +end +v1Names = struct('minimum_matching_file_paths', 'number_fullpath_matches', ... + 'sync_file_name', 'syncfilename', ... + 'minimum_embedded_file_overlap', 'minEmbeddedFileOverlap'); +wanted = {name}; +if isfield(v1Names, name) + wanted{end+1} = v1Names.(name); +end +mp = cfg.method_parameters; +for k = 1:numel(mp) + if iscell(mp), c = mp{k}; else, c = mp(k); end + if isfield(c, 'variable') && isstruct(c.variable) && isfield(c.variable, 'name') ... + && any(strcmp(c.variable.name, wanted)) + e = c; + return; + end end end @@ -220,10 +290,18 @@ end function v = requireNumber(cfg, etaName, whatV1Called, objClass) -if ~isstruct(cfg) || ~isfield(cfg, etaName) || isempty(cfg.(etaName)) - error('NDI:vintage:syncruleMissingField', ... - ['cannot rebuild `%s` for a %s: the configuration carries no ' ... - '`%s`.'], whatV1Called, objClass, etaName); +if isstruct(cfg) && isfield(cfg, etaName) && ~isempty(cfg.(etaName)) + v = cfg.(etaName); + return; +end +e = methodParameter(cfg, etaName); +if ~isempty(e) && isfield(e, 'value') && isstruct(e.value) ... + && isfield(e.value, 'value') && ~isempty(e.value.value) + v = e.value.value; + return; end -v = cfg.(etaName); +error('NDI:vintage:syncruleMissingField', ... + ['cannot rebuild `%s` for a %s: the configuration carries no ' ... + '`%s`, as a field or as a method_parameters entry.'], ... + whatV1Called, objClass, etaName); end diff --git a/src/ndi/+ndi/assertion.m b/src/ndi/+ndi/assertion.m new file mode 100644 index 0000000000..63ad5e7116 --- /dev/null +++ b/src/ndi/+ndi/assertion.m @@ -0,0 +1,20 @@ +classdef assertion < ndi.statement + % ndi.assertion - a statement that simply holds of a subject + % + % A subject's species, strain, sex, or its inclusion in an analysis: a + % variable and a value, with no time, method or instrument. Everything + % it does comes from ndi.statement. + % + % See also ndi.statement, ndi.entity/assertions. + + methods + function obj = assertion(container, doc) + % ASSERTION - use ndi.statement.fromDocument + arguments + container = [] + doc = [] + end + obj = obj@ndi.statement(container, doc); + end + end +end diff --git a/src/ndi/+ndi/calculation.m b/src/ndi/+ndi/calculation.m new file mode 100644 index 0000000000..0ed2b7262b --- /dev/null +++ b/src/ndi/+ndi/calculation.m @@ -0,0 +1,48 @@ +classdef calculation < ndi.interaction + % ndi.calculation - a statement computed from other documents + % + % ndi.calculation Methods (beyond ndi.interaction): + % inputs - what it was computed from: statements as ndi.statement + % objects, anything else as its ndi.document (a cell array) + % interpreter, operating_system - the software it ran under (ndi.entity) + % + % software() (from ndi.interaction) is the program that computed it. + % + % See also ndi.interaction, ndi.statement. + + methods + function obj = calculation(container, doc) + % CALCULATION - use ndi.statement.fromDocument + arguments + container = [] + doc = [] + end + obj = obj@ndi.interaction(container, doc); + end + + function x = inputs(obj) + % INPUTS - what the calculation was computed from (a cell array) + x = {}; + ids = ndi.v2.edgeIds(obj.document_properties(), 'input_id'); + for i = 1:numel(ids) + d = ndi.v2.getDocument(obj.container_, ids{i}); + if isempty(d), continue; end + if any(strcmp(ndi.v2.classChain(ndi.v2.props(d)), 'statement')) + x{end+1} = ndi.statement.fromDocument(obj.container_, d); %#ok + else + x{end+1} = d; %#ok + end + end + end + + function s = interpreter(obj) + % INTERPRETER - the interpreter entity(s) (e.g. MATLAB), a cell array + s = obj.entitiesAt('interpreter_id'); + end + + function s = operating_system(obj) + % OPERATING_SYSTEM - the operating-system entity(s), a cell array + s = obj.entitiesAt('operating_system_id'); + end + end +end diff --git a/src/ndi/+ndi/dataset.m b/src/ndi/+ndi/dataset.m index cc95b3e1b0..51fceea575 100644 --- a/src/ndi/+ndi/dataset.m +++ b/src/ndi/+ndi/dataset.m @@ -26,6 +26,11 @@ % SAME map -- Code Analyzer flags this and it is a real bug % when two datasets are open in the same MATLAB session. BinaryDocSessions = [] + + % SessionNotes - a V2 dataset's session listing: its denominator and + % every session left out and why (ndi.v2.datasetSessions). Read with + % session_notes(). + SessionNotes = {} end methods @@ -65,15 +70,35 @@ ref = ndi_dataset_obj.session.reference; end % unique_reference_string() - function ndi_dataset_obj = add_linked_session(ndi_dataset_obj, ndi_session_obj) + function ndi_dataset_obj = add_linked_session(ndi_dataset_obj, ndi_session_obj, options) % ADD_LINKED_SESSION - link an ndi.session to an ndi.dataset % - % NDI_DATASET_OBJ = ADD_LINKED_SESSION(NDI_DATASET_OBJ, NDI_SESSION_OBJ) + % NDI_DATASET_OBJ = ADD_LINKED_SESSION(NDI_DATASET_OBJ, NDI_SESSION_OBJ, ...) % % Add an ndi.session object to an ndi.dataset, without ingesting the session % into the dataset. Instead, the ndi.session is linked to the dataset, but % the session remains where it is. % + % In a V2 dataset (V_eta_linked_session_plan.md), this writes two + % documents into the dataset's database: a `part_of` relation from + % the session's `session` entity to the dataset (or to the study + % 'PartOf' names), and a `linked_session` document recording the + % session's folder (relative to the dataset's folder when inside it). + % The session must be a V2 session. + % + % Options: + % 'PartOf' V2 only: the id of the dataset's study (an entity + % document in the dataset) the session is part_of; + % default: the dataset itself + arguments + ndi_dataset_obj (1,1) ndi.dataset + ndi_session_obj (1,1) ndi.session + options.PartOf (1,:) char = '' + end + if ndi_dataset_obj.v2Listing() + ndi_dataset_obj.addSessionV2(ndi_session_obj, true, options.PartOf); + return; + end if isempty(ndi_dataset_obj.session_array) ndi_dataset_obj.build_session_info; end @@ -139,11 +164,27 @@ % | to false to force the old staged % | copy. See % | ndi.dataset.copySessionToDataset. + % PartOf ('') | V2 only: the id of the dataset's + % | study the session is part_of + % | (default: the dataset itself) + % + % In a V2 dataset (V_eta_linked_session_plan.md), every document of + % the session is copied into the dataset's database, ids unchanged, + % with the files it ingested; files recorded by location stay where + % they are. Unless the copied documents already make the session + % part_of the dataset or one of its studies, a `part_of` relation + % is added. The session must be a V2 session. % arguments ndi_dataset_obj (1,1) ndi.dataset ndi_session_obj (1,1) ndi.session options.ReferenceInPlace (1,1) logical = true + options.PartOf (1,:) char = '' + end + + if ndi_dataset_obj.v2Listing() + ndi_dataset_obj.addSessionV2(ndi_session_obj, false, options.PartOf); + return; end if isempty(ndi_dataset_obj.session_array) @@ -386,6 +427,25 @@ function unlink_session(ndi_dataset_obj, ndi_session_id, options) error('Operation not confirmed. Set areYouSure to true or confirm via dialog.'); end + if ndi_dataset_obj.v2Listing() + % V2: the session leaves the dataset -- its linked_session + % document and the part_of relations naming it go; its + % folder is untouched + session_obj = []; + if options.AlsoDeleteSessionAfterUnlinking + session_obj = ndi_dataset_obj.open_session(ndi_session_id); + end + info = ndi_dataset_obj.session_info(match_idx); + entityId = ndi_dataset_obj.v2EntityIdOf(match_idx); + ids = [{info.session_doc_in_dataset_id}, ndi_dataset_obj.v2MembershipIds(entityId)]; + ndi_dataset_obj.v2Remove(ids); + ndi_dataset_obj.build_session_info(); + if ~isempty(session_obj) + session_obj.deleteSessionDataStructures(options.areYouSure, options.DeleteSessionAskToConfirm); + end + return; + end + % If we need to delete the session later, we need the object. session_obj = []; if options.AlsoDeleteSessionAfterUnlinking @@ -476,6 +536,23 @@ function convertLinkedSessionToIngested(ndi_dataset_obj, session_id, options) ndi_session_obj = ndi_dataset_obj.open_session(session_id); + if ndi_dataset_obj.v2Listing() + % V2 (V_eta_linked_session_plan.md): copy the session's + % documents in, with the files they ingested, and delete the + % linked_session document. The part_of relation stays. + if ~ndi_dataset_obj.confirm(options, ['Are you sure you want to convert ' ... + 'linked session ' session_id ' to an ingested session? This will copy ' ... + 'all of its documents into the dataset.'], 'Confirm Conversion') + error('Operation not confirmed. Set areYouSure to true or confirm via dialog.'); + end + linkId = ndi_dataset_obj.session_info(match_idx).session_doc_in_dataset_id; + docs = ndi.v2.sessionDocuments(ndi_session_obj.database, session_id); + ndi.v2.copyDocuments(ndi_session_obj.database.db, ndi_dataset_obj.v2Db(), docs); + ndi_dataset_obj.v2Remove({linkId}); + ndi_dataset_obj.build_session_info(); + return; + end + if ~ndi_session_obj.isIngested() error(['Session with ID ' session_id ' and reference ' ... ndi_session_obj.reference ' is not yet fully ingested. ' ... @@ -546,6 +623,161 @@ function convertLinkedSessionToIngested(ndi_dataset_obj, session_id, options) end % convertLinkedSessionToIngested() + function convertIngestedSessionToLinked(ndi_dataset_obj, session_id, folder, options) + % CONVERTINGESTEDSESSIONTOLINKED - move an ingested session out to its own folder + % + % CONVERTINGESTEDSESSIONTOLINKED(NDI_DATASET_OBJ, SESSION_ID, FOLDER, ...) + % + % V2 datasets only (V_eta_linked_session_plan.md). Writes every + % document of the ingested session SESSION_ID into a new V2 session + % in FOLDER, with the files they ingested (files recorded by + % location stay where they are), deletes them from the dataset's + % database, and adds a `linked_session` document naming FOLDER. + % The `part_of` relations that make the session a member of the + % dataset stay in the dataset's database. FOLDER must not already + % hold an NDI database. + % + % Options: + % areYouSure (false) - must be true to proceed, unless confirmed by user + % askUserToConfirm (true) - if true, will ask user for confirmation via dialog + % + % See also: ndi.dataset/convertLinkedSessionToIngested, + % ndi.dataset/add_linked_session + arguments + ndi_dataset_obj (1,1) ndi.dataset + session_id (1,:) char + folder (1,:) char + options.areYouSure (1,1) logical = false + options.askUserToConfirm (1,1) logical = true + end + if ~ndi_dataset_obj.v2Listing() + error('ndi:dataset:notV2', ... + 'convertIngestedSessionToLinked works on a V2 dataset; this one is v1.'); + end + if isempty(ndi_dataset_obj.session_info) + ndi_dataset_obj.build_session_info(); + end + match_idx = find(strcmp(session_id, {ndi_dataset_obj.session_info.session_id}), 1); + if isempty(match_idx) + error(['Session with ID ' session_id ' not found in dataset ' ndi_dataset_obj.id() '.']); + end + info = ndi_dataset_obj.session_info(match_idx); + if info.is_linked + error(['Session with ID ' session_id ' is already a linked session.']); + end + ndiDir = fullfile(folder, '.ndi'); + if isfolder(ndiDir) && ~isempty([dir(fullfile(ndiDir, '*.sqlite')); dir(fullfile(ndiDir, '*.json'))]) + error('ndi:dataset:folderTaken', '%s already holds an NDI database.', ndiDir); + end + if ~ndi_dataset_obj.confirm(options, ['Are you sure you want to move ingested session ' ... + session_id ' out of the dataset into ' folder '?'], 'Confirm Conversion') + error('Operation not confirmed. Set areYouSure to true or confirm via dialog.'); + end + + entityId = info.session_doc_in_dataset_id; + docs = ndi.v2.sessionDocuments(ndi_dataset_obj.session.database, session_id); + stay = ndi_dataset_obj.v2MembershipIds(entityId); + move = docs(~ismember(cellfun(@(d) d.base.id, docs, 'UniformOutput', false), stay)); + + if ~isfolder(ndiDir), mkdir(ndiDir); end + target = did2.database.sqlitedb(fullfile(ndiDir, ... + ndi.database.implementations.database.did2sqlite.DEFAULTFILENAME())); + ndi.v2.copyDocuments(ndi_dataset_obj.v2Db(), target, move); + target.close(); + % the two files ndi.session.dir reads its id and reference from + % (as ndi.setup.V2.createSession writes them) + vlt.file.str2text(fullfile(ndiDir, 'unique_reference.txt'), session_id); + vlt.file.str2text(fullfile(ndiDir, 'reference.txt'), info.session_reference); + S = ndi.session.dir(folder); + if ~strcmp(S.id(), session_id) + error('ndi:dataset:moveMismatch', ... + 'The session written to %s opened with id %s, not %s; the dataset is unchanged.', ... + folder, S.id(), session_id); + end + + ndi_dataset_obj.v2Remove(cellfun(@(d) d.base.id, move, 'UniformOutput', false)); + link = did2.build.document('linked_session', ... + struct('path', ndi.v2.linkPath(folder, ndi_dataset_obj.getpath())), ... + 'SessionId', ndi_dataset_obj.id(), 'Edges', struct('entity_id', entityId)); + ndi_dataset_obj.v2Db().add({did2.document(link)}); + ndi_dataset_obj.build_session_info(); + end % convertIngestedSessionToLinked() + + function report = makeSelfContained(ndi_dataset_obj, options) + % MAKESELFCONTAINED - copy every file the dataset records by location into it + % + % REPORT = MAKESELFCONTAINED(NDI_DATASET_OBJ, ...) + % + % V2 datasets only (V_eta_linked_session_plan.md). A file is either + % INGESTED (copied into the dataset's file store) or recorded BY + % LOCATION (read where it is: raw recordings too large to copy). + % This copies every by-location file of every document in the + % dataset's database into the store and records it as ingested, so + % the dataset no longer depends on those folders -- the step before + % a dataset is shared. Linked sessions are not touched (their + % documents are not in the dataset): convert them to ingested first. + % + % REPORT (printed first, the denominator first): documents inspected, + % documents with a by-location file, files ingested, files whose + % location holds nothing on this computer (left by location, each + % named), locations that are not files (left), and linked sessions + % not included. + % + % Options: + % 'DryRun' default false: true reports what would be copied and + % changes nothing + arguments + ndi_dataset_obj (1,1) ndi.dataset + options.DryRun (1,1) logical = false + end + if ~ndi_dataset_obj.v2Listing() + error('ndi:dataset:notV2', 'makeSelfContained works on a V2 dataset; this one is v1.'); + end + if isempty(ndi_dataset_obj.session_info) + ndi_dataset_obj.build_session_info(); + end + db = ndi_dataset_obj.v2Db(); + ids = db.allIds(); + report = struct('documents', numel(ids), 'withByLocation', 0, 'ingested', 0, ... + 'missing', {{}}, 'notFiles', 0, ... + 'linkedSessions', sum([ndi_dataset_obj.session_info.is_linked]), ... + 'dryRun', options.DryRun); + stage = tempname; + for i = 1:numel(ids) + d = db.get(ids{i}).toStruct(); + [d2, nIn, missing, notFiles, keep] = ingestByLocation(d, db.fileDir, stage); + report.missing = [report.missing, missing]; + report.notFiles = report.notFiles + notFiles; + if nIn == 0 + continue; + end + report.withByLocation = report.withByLocation + 1; + report.ingested = report.ingested + nIn; + if options.DryRun + continue; + end + % a document's body cannot be changed in place: it is + % removed and added again, its already-ingested files staged + % first so removing it cannot lose them + for k = 1:size(keep, 1) + if ~isfolder(stage), mkdir(stage); end + copyfile(keep{k, 1}, keep{k, 2}); + end + db.remove(ids{i}); + db.add({did2.document(d2)}, 'Validate', false); + end + if isfolder(stage), rmdir(stage, 's'); end + fprintf(['DENOMINATOR: %d document(s) inspected; %d with a by-location file; ' ... + '%d file(s) %s; %d location(s) holding nothing here (left by location); %d ' ... + 'location(s) not a file (left); %d linked session(s) not included\n'], ... + report.documents, report.withByLocation, report.ingested, ... + ifelse(options.DryRun, 'would be ingested', 'ingested'), numel(report.missing), ... + report.notFiles, report.linkedSessions); + for k = 1:min(numel(report.missing), 20) + fprintf(' nothing at: %s\n', report.missing{k}); + end + end % makeSelfContained() + function ndi_session_obj = open_session(ndi_dataset_obj, session_id) % OPEN_SESSION - open an ndi.session object from an ndi.dataset % @@ -583,7 +815,17 @@ function convertLinkedSessionToIngested(ndi_dataset_obj, session_id, options) % ndi_dataset_obj.session_info(match_).session_creator_input5, ... % ndi_dataset_obj.session_info(match_).session_creator_input6); ndi_session_obj = ndi_dataset_obj.session_array(match).session; - mksqlite('close'); % TODO: update ndi.session with a close database files method + if ndi_dataset_obj.session_info(match_).is_linked && ndi_dataset_obj.isV2() + % it reaches the dataset's documents through the dataset + ndi_session_obj.datasetDatabase = ndi_dataset_obj.session.database; + end + if ~ndi_dataset_obj.isV2() + % closes EVERY mksqlite connection: harmless for the legacy + % backend, which reopens its file per call, but a V2 + % database (did2.database.sqlitedb) keeps its connection + % open, and this would close the session just opened + mksqlite('close'); % TODO: update ndi.session with a close database files method + end end end end % open_session() @@ -614,7 +856,7 @@ function convertLinkedSessionToIngested(ndi_dataset_obj, session_id, options) session_doc_ids = {ndi_dataset_obj.session_info.session_doc_in_dataset_id}; dataset_session_doc_id = ''; - q_dataset_session_doc = ndi.query('','isa','session') & ndi.query('base.session_id','exact_string',ndi_dataset_obj.id()); + q_dataset_session_doc = ndi.v2.isaQuery('session') & ndi.query('base.session_id','exact_string',ndi_dataset_obj.id()); doc = ndi_dataset_obj.session.database_search(q_dataset_session_doc); if isscalar(doc) dataset_session_doc_id = doc{1}.id(); @@ -624,6 +866,24 @@ function convertLinkedSessionToIngested(ndi_dataset_obj, session_id, options) end % session_list() + function notes = session_notes(ndi_dataset_obj) + % SESSION_NOTES - how a V2 dataset's sessions were listed + % + % NOTES = SESSION_NOTES(NDI_DATASET_OBJ) + % + % For a V2 dataset, a cellstr: the denominator of the listing + % (session and linked_session documents read, sessions listed), + % then every session left out and why -- a session document not + % part_of the dataset, a linked folder that is missing or holds + % another session. {} for a v1 dataset. + % + % See also: ndi.dataset/session_list, ndi.v2.datasetSessions + if isempty(ndi_dataset_obj.session_info) + ndi_dataset_obj.build_session_info(); + end + notes = ndi_dataset_obj.SessionNotes; + end % session_notes() + function p = getpath(ndi_dataset_obj) % GETPATH - Return the path of the dataset % @@ -1106,16 +1366,80 @@ function build_session_info(ndi_dataset_obj) ndi_dataset_obj.session_info(end+1) = info_here; end + opened = {}; + if isempty(session_info_doc) && ndi_dataset_obj.isV2() + % A V2 dataset records no session_in_a_dataset documents + % (V_eta_linked_session_plan.md, signed 2026-10-09): a session + % is in it when its `session` entity is `part_of` the dataset + % or a study of it. INGESTED: its documents are in this + % database, opened from the dataset's folder. LINKED: a + % `linked_session` document names its folder, opened there. + [v2, notes] = ndi.v2.datasetSessions(ndi_dataset_obj.session.database, ... + ndi_dataset_obj.id(), ndi_dataset_obj.getpath()); + ndi_dataset_obj.SessionNotes = notes; + left = notes(startsWith(notes, 'not listed') | contains(notes, 'not part_of')); + if ~isempty(left) + warning('ndi:dataset:sessionsLeftOut', '%s\n %s', ... + 'Some sessions of this dataset could not be listed:', strjoin(left, '\n ')); + end + for i = 1:numel(v2) + patharg = ''; + docId = v2(i).entity_id; + if v2(i).is_linked + patharg = v2(i).path; + docId = v2(i).link_id; + end + info_here = struct('session_id', v2(i).session_id, ... + 'session_reference', v2(i).reference, 'is_linked', double(v2(i).is_linked), ... + 'session_creator', 'ndi.session.dir', ... + 'session_creator_input1', v2(i).reference, ... + 'session_creator_input2', patharg, 'session_creator_input3', '', ... + 'session_creator_input4', '', 'session_creator_input5', '', ... + 'session_creator_input6', '', 'session_doc_in_dataset_id', docId); + ndi_dataset_obj.session_info(end+1) = info_here; + opened{end+1} = v2(i).session; %#ok + end + elseif isempty(session_info_doc) + % not a V2 database, and no session_in_a_dataset documents: + % each other session document is an ingested session + v2 = ndi_dataset_obj.session.database.search(ndi.v2.isaQuery('session')); + for i=1:numel(v2) + p = v2{i}.document_properties; + [~, blk] = ndi.v2.kindOf(p); % `entity` since 2026-10-08 + if strcmp(p.base.session_id, ndi_dataset_obj.id()) || ~isfield(p.(blk),'local_identifier') + continue; + end + info_here = struct('session_id', p.base.session_id, ... + 'session_reference', p.(blk).local_identifier, 'is_linked', 0, ... + 'session_creator', 'ndi.session.dir', ... + 'session_creator_input1', p.(blk).local_identifier, ... + 'session_creator_input2', '', 'session_creator_input3', '', ... + 'session_creator_input4', '', 'session_creator_input5', '', ... + 'session_creator_input6', '', 'session_doc_in_dataset_id', v2{i}.id()); + ndi_dataset_obj.session_info(end+1) = info_here; + end + end + % now we have session_info structure, build the initial session_array ndi_dataset_obj.session_array = did.datastructures.emptystruct('session_id','session'); for i=1:numel(ndi_dataset_obj.session_info) session_array_here.session_id = ndi_dataset_obj.session_info(i).session_id; session_array_here.session = []; % initially don't open it + if i <= numel(opened) && ~isempty(opened{i}) + session_array_here.session = opened{i}; % a linked session, opened to list it + opened{i}.datasetDatabase = ndi_dataset_obj.session.database; + end ndi_dataset_obj.session_array(i) = session_array_here; % entries will match end end % build_session_info() + function tf = isV2(ndi_dataset_obj) + % ISV2 - is this dataset's database a V2 (did2) one? + tf = isa(ndi_dataset_obj.session.database, ... + 'ndi.database.implementations.database.did2sqlite'); + end % isV2() + function open_linked_sessions(ndi_dataset_obj) % OPEN_LINKED_SESSIONS - ensure that all linked sessions are open % @@ -1187,5 +1511,227 @@ function ensureBinaryDocSessions(ndi_dataset_obj) end end % ensureBinaryDocSessions + % --- V2 datasets (V_eta_linked_session_plan.md) --- + + function tf = v2Listing(ndi_dataset_obj) + % V2LISTING - are this dataset's sessions listed the V2 way? + % True for a did2 database holding no session_in_a_dataset + % document: membership is `part_of`, a linked session a + % `linked_session` document. + tf = false; + if ~ndi_dataset_obj.isV2() + return; + end + q = ndi.query('', 'isa', 'session_in_a_dataset', ''); + tf = isempty(ndi_dataset_obj.session.database.search(q)); + end % v2Listing() + + function db = v2Db(ndi_dataset_obj) + % V2DB - the dataset's did2.database.sqlitedb + db = ndi_dataset_obj.session.database.db; + end % v2Db() + + function v2Remove(ndi_dataset_obj, ids) + % V2REMOVE - delete documents (by id) from the dataset's database + db = ndi_dataset_obj.v2Db(); + for k = 1:numel(ids) + if ~isempty(ids{k}) && db.has(ids{k}) + db.remove(ids{k}); + end + end + end % v2Remove() + + function ids = v2MembershipIds(ndi_dataset_obj, entityId) + % V2MEMBERSHIPIDS - the part_of relations in the dataset's database + % whose child is ENTITYID (the relations that make a session a + % member), as a cellstr of document ids + ids = {}; + q = ndi.query('', 'isa', 'directed_relation', '') & ... + ndi.query('', 'depends_on', 'child_id', entityId); + rels = ndi_dataset_obj.session.database.search(q); + for k = 1:numel(rels) + p = rels{k}.document_properties; + if strcmp(ndi.v2.termName(ndi.v2.blockOf(p, 'directed_relation', 'relation', '')), 'part_of') + ids{end+1} = p.base.id; %#ok + end + end + end % v2MembershipIds() + + function tf = v2HasPartOf(ndi_dataset_obj, childId, parentId) + % V2HASPARTOF - does the dataset's database hold CHILDID part_of PARENTID? + tf = false; + ids = ndi_dataset_obj.v2MembershipIds(childId); + for k = 1:numel(ids) + d = ndi_dataset_obj.session.database.search(ndi.query('base.id', 'exact_string', ids{k}, '')); + if ~isempty(d) && any(strcmp(ndi.v2.edgeIds(d{1}.document_properties, 'parent_id'), parentId)) + tf = true; + return; + end + end + end % v2HasPartOf() + + function entityId = v2EntityIdOf(ndi_dataset_obj, idx) + % V2ENTITYIDOF - the session entity id of session_info(IDX): the + % entity document itself for an ingested session, the + % linked_session's entity_id edge for a linked one + info = ndi_dataset_obj.session_info(idx); + entityId = info.session_doc_in_dataset_id; + if info.is_linked + d = ndi_dataset_obj.session.database.search( ... + ndi.query('base.id', 'exact_string', entityId, '')); + e = ndi.v2.edgeIds(d{1}.document_properties, 'entity_id'); + entityId = e{1}; + end + end % v2EntityIdOf() + + function addSessionV2(ndi_dataset_obj, ndi_session_obj, linked, partOf) + % ADDSESSIONV2 - add a V2 session to a V2 dataset, linked or ingested + if ~isa(ndi_session_obj.database, 'ndi.database.implementations.database.did2sqlite') + error('ndi:dataset:notV2Session', ['Session %s is not a V2 session; only a ' ... + 'V2 session can be added to a V2 dataset.'], ndi_session_obj.reference); + end + if linked && ~ndi.setup.V2.schemaHasClass('linked_session') + error('ndi:dataset:noLinkedSessionClass', ['The schema in use has no ' ... + 'linked_session class (V_eta_linked_session_plan.md), so a session cannot ' ... + 'be linked to a V2 dataset with it.']); + end + if isempty(ndi_dataset_obj.session_info) + ndi_dataset_obj.build_session_info(); + end + if any(strcmp(ndi_session_obj.id(), {ndi_dataset_obj.session_info.session_id})) + error(['ndi.session object with id ' ndi_session_obj.id() ... + ' is already part of dataset ' ndi_dataset_obj.id() '.']); + end + % the session's own `session` entity document + own = ndi_session_obj.database_search(ndi.v2.isaQuery('session')); + own = own(cellfun(@(d) strcmp(ndi.v2.kindOf(d.document_properties), 'session') && ... + strcmp(d.document_properties.base.session_id, ndi_session_obj.id()), own)); + if ~isscalar(own) + error('ndi:dataset:sessionDocument', ['Session %s holds %d session ' ... + 'documents of its own; it must hold exactly one.'], ndi_session_obj.reference, ... + numel(own)); + end + entityId = own{1}.id(); + % the parent it is part_of: the dataset, or a study of it + dsid = ndi_dataset_obj.id(); + explicit = ~isempty(partOf); + if ~explicit + d = ndi_dataset_obj.session.database.search(ndi.v2.isaQuery('dataset')); + d = d(cellfun(@(x) strcmp(ndi.v2.kindOf(x.document_properties), 'dataset'), d)); + if ~isscalar(d) + error('ndi:dataset:noDatasetDocument', ['The dataset holds %d dataset ' ... + 'document(s), so ''PartOf'' must name the study the session is part_of.'], ... + numel(d)); + end + partOf = d{1}.id(); + elseif isempty(ndi_dataset_obj.session.database.search(ndi.query('base.id', ... + 'exact_string', partOf, ''))) + error('ndi:dataset:noSuchParent', ['''PartOf'' names %s, which is not a ' ... + 'document of this dataset.'], partOf); + end + + db = ndi_dataset_obj.v2Db(); + if ~linked + docs = ndi.v2.sessionDocuments(ndi_session_obj.database, ndi_session_obj.id()); + here = cellfun(@(x) db.has(x.base.id), docs); + if any(here) + error('ndi:dataset:documentsPresent', ['%d of the %d documents of session %s ' ... + 'are already in the dataset.'], sum(here), numel(docs), ndi_session_obj.reference); + end + % a membership the session's own documents already state is + % not stated again (each fact once, T17): with no 'PartOf', + % its own part_of a study of the dataset (or the dataset) is + % enough; with one, its own part_of that parent is + parents = {}; + for k = 1:numel(docs) + p = docs{k}; + if strcmp(char(p.document_class.class_name), 'directed_relation') && ... + any(strcmp(ndi.v2.edgeIds(p, 'child_id'), entityId)) && ... + strcmp(ndi.v2.termName(ndi.v2.blockOf(p, 'directed_relation', 'relation', '')), 'part_of') + parents = [parents, ndi.v2.edgeIds(p, 'parent_id')]; %#ok + end + end + if explicit + stated = ismember(partOf, parents); + else + stated = ismember(partOf, parents) || ... + any(cellfun(@(x) ndi_dataset_obj.v2HasPartOf(x, partOf), parents)); + end + ndi.v2.copyDocuments(ndi_session_obj.database.db, db, docs); + if stated + ndi_dataset_obj.build_session_info(); + return; + end + end + add = {did2.build.directedRelation(entityId, partOf, 'part_of', 'SessionId', dsid)}; + if linked + add{end+1} = did2.build.document('linked_session', ... + struct('path', ndi.v2.linkPath(ndi_session_obj.getpath(), ndi_dataset_obj.getpath())), ... + 'SessionId', dsid, 'Edges', struct('entity_id', entityId)); + end + db.add(cellfun(@(x) did2.document(x), add, 'UniformOutput', false)); + ndi_dataset_obj.build_session_info(); + end % addSessionV2() + end % methods protected + + methods (Static, Access = protected) + function tf = confirm(options, question, title) + % CONFIRM - options.areYouSure, or the user's answer to a dialog + tf = options.areYouSure; + if ~tf && options.askUserToConfirm + tf = strcmp(questdlg(question, title, 'Yes', 'No', 'No'), 'Yes'); + end + end % confirm() + end % methods (Static, Access = protected) end % class + +% ----------------------------------------------------------------------------- +function [d, nIn, missing, notFiles, keep] = ingestByLocation(d, fileDir, stage) +% D with every by-location file location that holds a file here marked +% ingested; KEEP (N x 2: stored copy, staging path) the files D already +% ingested, pointed at their staging path so D can be removed and added back. +nIn = 0; missing = {}; notFiles = 0; keep = cell(0, 2); +if ~isfield(d, 'files') || ~isstruct(d.files) || ~isfield(d.files, 'file_info') ... + || isempty(d.files.file_info) + return; +end +fi = d.files.file_info; +if iscell(fi), fi = [fi{:}]; end +for a = 1:numel(fi) + if ~isfield(fi(a), 'locations'), continue; end + if iscell(fi(a).locations), fi(a).locations = [fi(a).locations{:}]; end + for c = 1:numel(fi(a).locations) + L = fi(a).locations(c); + if isfield(L, 'ingest') && logical(L.ingest) + staged = fullfile(stage, char(L.uid)); + keep(end+1, :) = {fullfile(fileDir, char(L.uid)), staged}; %#ok + fi(a).locations(c).location = staged; + fi(a).locations(c).delete_original = 1; + continue; + end + type = 'file'; + if isfield(L, 'location_type') && ~isempty(L.location_type), type = lower(char(L.location_type)); end + if ~strcmp(type, 'file') + notFiles = notFiles + 1; + continue; + end + if ~isfile(char(L.location)) + missing{end+1} = sprintf('%s of document %s (%s)', char(fi(a).name), d.base.id, ... + char(L.location)); %#ok + continue; + end + if ~isfield(L, 'uid') || isempty(L.uid) || ~did.file.isSafeUid(char(L.uid)) + fi(a).locations(c).uid = char(ndi.ido.unique_id()); + end + fi(a).locations(c).ingest = 1; + fi(a).locations(c).delete_original = 0; + nIn = nIn + 1; + end +end +d.files.file_info = fi; +end + +function out = ifelse(tf, a, b) +if tf, out = a; else, out = b; end +end diff --git a/src/ndi/+ndi/entity.m b/src/ndi/+ndi/entity.m new file mode 100644 index 0000000000..8ed3239ccb --- /dev/null +++ b/src/ndi/+ndi/entity.m @@ -0,0 +1,2272 @@ +classdef entity + % ndi.entity - something a V2 dataset names: a subject, a person, an organization, ... + % + % An ndi.entity is a read-only view of one V2 `entity` document (a subject, + % strain, study, person, organization, product, software, funding, + % publication, ...), with the session or dataset it was read from. Nothing + % about it changes after it is made: its statements and relations are + % other documents, found by query each time they are asked for, so a new + % statement or relation added to the database is seen at once. + % + % Every entity is read as an ndi.entity: a worm, a plate, a probe, a + % person, a strain. ndi.subject is v1's class and is not used here; + % ndi.session and ndi.dataset are not entities (design: + % src/ndi/docs/NDI-matlab/manual/developer/V2_Object_Layer.md). + % + % Make one with ndi.entity.fromDocument or ndi.entity.search: + % + % p = ndi.entity.search(ds, 'person'); % a cell array + % p{1}.name % 'Jess Haley' + % p{1}.parents('affiliated_with') % her organizations + % + % READ THROUGH THE DATASET. A search through an ndi.session sees only + % that session's documents, and the shared entities (people, + % organizations, studies, strains, software, instruments) are stored + % with the dataset. An entity read through the ndi.dataset reaches + % everything; one read through a session does not reach those. + % + % ndi.entity Properties (read from the document when asked; '' without one): + % name - a display name + % kind - the document class ('person', 'subject', ...) + % document_id - the document's base.id + % type, local_identifier, description + % + % ndi.entity Methods: + % document - the ndi.document + % document_properties - its properties (a struct) + % global_identifiers - table: scheme, value (ORCID, ROR, RRID, DOI, ...) + % summary - a table, one row per entity (ndi.summary for a cell) + % relations - table of the relations it (or an array of + % entities) takes part in + % parents, children - the entities across a relation, or a path of + % them, from one entity or an array of them + % ancestors, descendants - the nearest entities of a type or kind, + % following any relation: you need not know which + % statements - the statements about it (or an array of + % entities), inherited by default; assertions, + % observations, manipulations, calculations, + % interactions: statements of one kind + % members, memberOf, parts, partOf + % fromDocument, search - (static) make entities; search finds them by + % what is true of them + % + % See also ndi.statement, ndi.v2.subjectTypes. + + properties (Dependent, SetAccess = private) + % Read from the entity's document each time they are asked for: + % nothing is stored. Empty for an entity with no document. + name % a display name (see get.name) + kind % the document class, e.g. 'person', 'subject' + document_id % the document's base.id + type % the entity's type: a subject's coarse kind ('organism', + % 'group', ...); with one entity class (2026-10-08) every + % entity's ('person', 'strain', ...); '' when none + local_identifier % its handle within the dataset ('' when none) + description % free text ('' when none) + end + + properties (SetAccess = protected, GetAccess = public, Hidden) + entity_document_ = [] % the ndi.document (or document struct) this entity reads + container_ = [] % the ndi.session or ndi.dataset it was read from + end + + methods + function obj = entity(container, doc) + % ENTITY - an ndi.entity for DOC, read from CONTAINER + % + % OBJ = ndi.entity(CONTAINER, DOC); CONTAINER is an ndi.session or + % ndi.dataset, DOC an entity document (ndi.entity.fromDocument + % also takes a document id). + arguments + container = [] + doc = [] + end + if nargin == 0 + return; + end + obj.container_ = container; + obj.entity_document_ = doc; + end + + function d = document(obj) + % DOCUMENT - the entity's ndi.document + d = obj.entity_document_; + end + + function p = document_properties(obj) + % DOCUMENT_PROPERTIES - the entity document's properties (a struct) + p = ndi.v2.props(obj.entity_document_); + end + + function id = get.document_id(obj) + % DOCUMENT_ID - the entity document's base.id ('' without a document) + id = ''; + if isempty(obj.entity_document_), return; end + p = obj.document_properties(); + id = char(p.base.id); + end + + function k = get.kind(obj) + % KIND - the entity's document class, e.g. 'person', 'subject'; + % for an `entity` (one entity class, 2026-10-08) its `type`, so a + % person is 'person' under either schema. A subject's type is a + % physical kind ('organism', 'material', ...), so its kind is + % 'subject', the block its fields are in under the earlier schema. + k = ''; + if isempty(obj.entity_document_), return; end + p = obj.document_properties(); + k = ndi.v2.kindOf(p); + if any(strcmp(k, ndi.v2.entityTypesFor('subject'))) + k = 'subject'; + end + end + + function n = get.name(obj) + % NAME - a display name + % + % The class's own `name` when it has one; a person's given and + % family names; else a subject's local_identifier, else base.name. + n = ''; + if isempty(obj.entity_document_), return; end + p = obj.document_properties(); + k = obj.kind; + n = char(ndi.v2.blockOf(p, k, 'name', '')); + if isempty(n) && strcmp(k, 'person') + n = strtrim([char(ndi.v2.blockOf(p, k, 'given_name', '')) ' ' ... + char(ndi.v2.blockOf(p, k, 'family_name', ''))]); + end + if isempty(n) + n = char(ndi.v2.blockOf(p, k, 'local_identifier', '')); + end + if isempty(n) + n = char(ndi.v2.blockOf(p, 'base', 'name', '')); + end + end + + function t = get.type(obj) + % TYPE - the entity's type: a subject's coarse kind, or (one entity + % class) any entity's ('' for an entity with no document) + t = ''; + if isempty(obj.entity_document_), return; end + t = ndi.v2.termName(ndi.v2.blockOf(obj.document_properties(), obj.kind, 'type', '')); + end + + function v = get.local_identifier(obj) + % LOCAL_IDENTIFIER - its handle within the dataset, from the document + v = ''; + if isempty(obj.entity_document_), return; end + v = char(ndi.v2.blockOf(obj.document_properties(), obj.kind, 'local_identifier', '')); + end + + function v = get.description(obj) + % DESCRIPTION - free text, from the document + v = ''; + if isempty(obj.entity_document_), return; end + v = char(ndi.v2.blockOf(obj.document_properties(), obj.kind, 'description', '')); + end + + function s = statements(obj, kind, filters, options) + % STATEMENTS - the statements about this entity (or these entities) + % + % S = STATEMENTS(E, KIND, FILTERS, ...) returns a cell array of + % ndi.statement objects (each the right child: ndi.observation, + % ndi.manipulation, ndi.calculation, ndi.assertion). E may be an + % array of entities; from a cell array C call statements([C{:}]). + % The entities are searched together, a few searches in all; a + % statement that holds of several of them comes back once for each, + % and each knows which entity it was asked for (about()) and how + % it holds of it (via()). + % + % KIND (default 'statement', any): 'assertion', 'interaction', + % 'observation', 'manipulation', 'calculation', or a document + % class ('velocity_calculation'). FILTERS (default {}): a cell of + % filters on the statement, the words ndi.entity.search uses -- + % 'variable', 'method', 'value', 'formulation', and when: 'at', + % 'during', 'before', 'after', 'duration' (ndi.v2.timeFilter). + % 'during' also takes statements, {'observation', {'variable', + % 'ambient temperature', 'value', '>22'}}: kept when it overlaps + % one matching that which holds of the same entity by the same + % rules. + % s = w.statements('manipulation') + % s = w.statements('manipulation', {'method', 'refrigeration'}) + % s = w.statements('statement', {'variable', 'midpoint speed'}) + % + % Options: + % 'inherited' (default true) -- the same rules as + % ndi.entity.search: statements about a group it belongs to + % (member_of, groups of groups too) that are marked + % distributive; assertions about a whole it is part or a + % sample of (part_of, sample_of, aliquot_of, passage_of); + % assertions about its types (instance_of: its strain, its + % product). false: its own statements only. + % 'context' (default false) -- true adds what the entity was IN + % (V_eta tenet T17): the observations, manipulations and + % calculations of each container it was contained_in (its own + % stays, a distributive stay of a group it is in, and the + % containers of those), whose time overlaps the stay. They are + % not about it: via() says 'contained_in' (after any + % member_of), and summary's stated_on names the container. A + % stay or statement whose times cannot be compared is left out. + % 'tolerant' (default false) -- true widens each time by its + % tolerance (and each stay, for 'context'). + % 'synonyms' (default true) -- a 'value' that matches no term + % by its name or node also matches a term it is a synonym of + % (ndi.v2.synonyms: ndi.ontology.lookup, kept on disk). + arguments + obj + kind (1,:) char = 'statement' + filters cell = {} + options.inherited (1,1) logical = true + options.context (1,1) logical = false + options.tolerant (1,1) logical = false + options.synonyms (1,1) logical = true + end + f = ndi.entity.statementFilter(lower(kind), filters); + f.tolerant = options.tolerant; + f.synonyms = options.synonyms; + kinds = struct('kind', lower(kind), 'filt', f); + s = {}; + subjects = obj(arrayfun(@(x) ~isempty(x.container_), obj)); + if isempty(subjects), return; end + container = subjects(1).container_; + ids = arrayfun(@(x) x.document_id, subjects, 'UniformOutput', false); + L = ndi.entity.statementLinks(container, ids, kinds, options.inherited, options.context); + s = cell(1, numel(L.doc)); + for i = 1:numel(L.doc) + s{i} = ndi.statement.fromDocument(container, L.doc{i}).withContext(L.about{i}, L.via{i}); + end + end % statements() + + function s = assertions(obj, filters, options) + % ASSERTIONS - what is asserted about the entity (or entities): species, ... + % + % S = ASSERTIONS(E, FILTERS, ...) is STATEMENTS(E, 'assertion', + % FILTERS, ...): a cell array of ndi.assertion objects, inherited + % by default; options as STATEMENTS. + % a = w.assertions({'variable', 'species'}) + % a = w.assertions('inherited', false) + % ndi.summary(a) makes the table. + arguments + obj + filters cell = {} + options.inherited (1,1) logical = true + options.context (1,1) logical = false + options.tolerant (1,1) logical = false + options.synonyms (1,1) logical = true + end + o = namedArgs(options); + s = obj.statements('assertion', filters, o{:}); + end % assertions() + + function s = observations(obj, filters, options) + % OBSERVATIONS - what was observed of the entity (or entities) + % + % S = OBSERVATIONS(E, FILTERS, ...) is STATEMENTS(E, 'observation', + % FILTERS, ...), a cell array of ndi.observation objects: + % o = w.observations({'variable', 'temperature'}) + arguments + obj + filters cell = {} + options.inherited (1,1) logical = true + options.context (1,1) logical = false + options.tolerant (1,1) logical = false + options.synonyms (1,1) logical = true + end + o = namedArgs(options); + s = obj.statements('observation', filters, o{:}); + end % observations() + + function s = manipulations(obj, filters, options) + % MANIPULATIONS - what was done to the entity (or entities) + % + % S = MANIPULATIONS(E, FILTERS, ...) is STATEMENTS(E, 'manipulation', + % FILTERS, ...), a cell array of ndi.manipulation objects: + % m = w.manipulations({'method', 'refrigeration'}) + arguments + obj + filters cell = {} + options.inherited (1,1) logical = true + options.context (1,1) logical = false + options.tolerant (1,1) logical = false + options.synonyms (1,1) logical = true + end + o = namedArgs(options); + s = obj.statements('manipulation', filters, o{:}); + end % manipulations() + + function s = calculations(obj, filters, options) + % CALCULATIONS - what was calculated of the entity (or entities) + % + % S = CALCULATIONS(E, FILTERS, ...) is STATEMENTS(E, 'calculation', + % FILTERS, ...), a cell array of ndi.calculation objects: + % c = calculations([w{:}], {'variable', 'midpoint speed'}) + arguments + obj + filters cell = {} + options.inherited (1,1) logical = true + options.context (1,1) logical = false + options.tolerant (1,1) logical = false + options.synonyms (1,1) logical = true + end + o = namedArgs(options); + s = obj.statements('calculation', filters, o{:}); + end % calculations() + + function s = interactions(obj, filters, options) + % INTERACTIONS - observations, manipulations and calculations of the entity + % + % S = INTERACTIONS(E, FILTERS, ...) is STATEMENTS(E, 'interaction', + % FILTERS, ...): every statement with a time and a method. + arguments + obj + filters cell = {} + options.inherited (1,1) logical = true + options.context (1,1) logical = false + options.tolerant (1,1) logical = false + options.synonyms (1,1) logical = true + end + o = namedArgs(options); + s = obj.statements('interaction', filters, o{:}); + end % interactions() + + function m = members(obj) + % MEMBERS - a group's members (the subjects that are member_of it), a cell array + m = obj.children('member_of'); + end + + function g = memberOf(obj) + % MEMBEROF - the groups this subject is member_of, a cell array + g = obj.parents('member_of'); + end + + function p = parts(obj) + % PARTS - the subjects that are part_of this one (a plate's patches), a cell array + p = obj.children('part_of'); + end + + function w = partOf(obj) + % PARTOF - what this subject is part_of (a patch's plate), a cell array + w = obj.parents('part_of'); + end + + function T = global_identifiers(obj) + % GLOBAL_IDENTIFIERS - table with columns scheme and value + p = obj.document_properties(); + g = ndi.v2.blockOf(p, 'entity', 'global_identifier', []); + scheme = strings(0, 1); value = strings(0, 1); + if ischar(g) || isstring(g) + g = cellstr(g); + end + if ~iscell(g) || ~all(cellfun(@ischar, g)) + g = ndi.v2.entries(g); + end + for i = 1:numel(g) + sc = ''; v = ''; + if ischar(g{i}) + % a CURIE or an IRI since 2026-10-08: the prefix is the scheme + tok = regexp(g{i}, '^([A-Za-z][A-Za-z0-9_.]*):(?!//)(.*)$', 'tokens', 'once'); + if isempty(tok) + sc = 'IRI'; v = g{i}; + else + sc = tok{1}; v = tok{2}; + end + end + if isstruct(g{i}) && isfield(g{i}, 'scheme'), sc = ndi.v2.termName(g{i}.scheme); end + if isstruct(g{i}) && isfield(g{i}, 'value'), v = char(g{i}.value); end + scheme(end+1, 1) = string(sc); %#ok + value(end+1, 1) = string(v); %#ok + end + T = table(scheme, value); + end + + function T = summary(obj, options) + % SUMMARY - one table row per entity + % + % T = SUMMARY(E) for an entity or an array of one class; for a cell + % array (what search returns) use ndi.summary. Columns: name, + % kind, id; for subjects also type, local_identifier, and one + % column per asserted variable (species, strain, ...), inherited + % as statements inherits (several values joined with + % '; '). 'nodes', true adds beside each a _node column, + % the values' ontology nodes. + arguments + obj + options.nodes (1,1) logical = false + end + T = ndi.entity.summaryOf(num2cell(obj), 'nodes', options.nodes); + end + + function T = relations(obj, relationName, options) + % RELATIONS - the relations this entity (or these entities) take part in + % + % T = RELATIONS(OBJ) returns a table, one row per relation document + % and entity of OBJ at one end of it: + % start_id the entity of OBJ the row is about + % start_name its name + % relation the relation's name (e.g. 'member_of', 'part_of') + % direction "out" when this entity is the child (the subject of + % the sentence: worm member_of cohort), "in" when it + % is the parent + % other_id the document id at the other end + % roles the relation's roles, joined with ', ' + % relation_id the relation document's id + % T = RELATIONS(OBJ, RELATIONNAME) keeps one relation (or any of + % a cell array of names). 'Direction' 'out', 'in' or 'both' + % (default). + % + % OBJ may be an array of entities (of one class); from a cell + % array C call it as a function, RELATIONS([C{:}]) -- one search + % per direction for every 200 entities, not one per entity. A + % relation whose two ends are both in OBJ gives two rows, one + % for each end. + arguments + obj + relationName {mustBeText} = '' + options.Direction (1,:) char {mustBeMember(options.Direction, {'out', 'in', 'both'})} = 'both' + end + start_id = strings(0, 1); start_name = strings(0, 1); + relation = strings(0, 1); direction = strings(0, 1); other_id = strings(0, 1); + roles = strings(0, 1); relation_id = strings(0, 1); + T = table(start_id, start_name, relation, direction, other_id, roles, relation_id); + if isempty(obj) + return; + end + ids = arrayfun(@(x) x.document_id, obj, 'UniformOutput', false); + [ids, iu] = unique(ids(:), 'stable'); + names = arrayfun(@(x) string(x.name), obj(iu), 'UniformOutput', false); + nameOf = containers.Map(ids, names); + container = obj(1).container_; + if isempty(relationName), wanted = {}; else, wanted = cellstr(relationName); end + sides = {'out', 'child_id', 'parent_id'; 'in', 'parent_id', 'child_id'}; + chunk = 200; + for s = 1:2 + if ~strcmp(options.Direction, 'both') && ~strcmp(options.Direction, sides{s, 1}) + continue; + end + for c = 1:chunk:numel(ids) + part = ids(c:min(c + chunk - 1, numel(ids))); + q = ndi.entity.anyOf(cellfun(@(i) ndi.query('', 'depends_on', sides{s, 2}, i), ... + part, 'UniformOutput', false)); + docs = container.database_search(ndi.query('', 'isa', 'directed_relation', '') & q); + for i = 1:numel(docs) + p = ndi.v2.props(docs{i}); + r = ndi.v2.termName(ndi.v2.blockOf(p, 'directed_relation', 'relation', '')); + if ~isempty(wanted) && ~any(strcmp(r, wanted)) + continue; + end + mine = ndi.v2.edgeIds(p, sides{s, 2}); + mine = unique(mine(ismember(mine, part)), 'stable'); + other = ndi.v2.edgeIds(p, sides{s, 3}); + rl = ndi.v2.entries(ndi.v2.blockOf(p, 'directed_relation', 'roles', [])); + rn = {}; + for j = 1:numel(rl), rn{end+1} = ndi.v2.termName(rl{j}); end %#ok + for m = 1:numel(mine) + start_id(end+1, 1) = string(mine{m}); %#ok + start_name(end+1, 1) = nameOf(mine{m}); %#ok + relation(end+1, 1) = string(r); %#ok + direction(end+1, 1) = string(sides{s, 1}); %#ok + other_id(end+1, 1) = string(strjoin(other, ', ')); %#ok + roles(end+1, 1) = string(strjoin(rn, ', ')); %#ok + relation_id(end+1, 1) = string(p.base.id); %#ok + end + end + end + end + T = table(start_id, start_name, relation, direction, other_id, roles, relation_id); + end + + function e = parents(obj, path, options) + % PARENTS - the entities this one points to, across one relation or a path of them + % + % E = PARENTS(OBJ, RELATIONNAME), a cell array: e.g. a worm's + % parents('member_of') is its cohort. An end that is not found + % from this container (see the class help) is left out. + % + % RELATIONNAME may be a PATH, a cell array of relation names + % followed in turn, and OBJ may be an array of entities (of one + % class). From a cell array C, call it as a function -- + % PARENTS([C{:}], ...) -- since [C{:}].parents(...) is not + % valid MATLAB: + % + % w.parents({'member_of', 'contained_in'}) % a worm's plates + % parents([worms{:}], {'member_of', 'contained_in'}) + % + % Each step is one search over every entity reached so far, not + % one per entity. Each entity is returned once. + % 'Table', true returns a table instead, one row per (start, end) + % pair: start_id, start_name, end_id, end_name, end (the entity). + arguments + obj + path {mustBeText} + options.Table (1,1) logical = false + end + e = obj.walk(path, 'out', options.Table); + end + + function e = children(obj, path, options) + % CHILDREN - the entities that point to this one, across one relation or a path of them + % + % E = CHILDREN(OBJ, RELATIONNAME), a cell array: e.g. a cohort's + % children('member_of') are its worms. As with PARENTS, + % RELATIONNAME may be a path and OBJ an array: + % + % plate.children({'contained_in', 'member_of'}) % the worms that were on a plate + % children([plates{:}], {'contained_in', 'member_of'}, 'Table', true) + arguments + obj + path {mustBeText} + options.Table (1,1) logical = false + end + e = obj.walk(path, 'in', options.Table); + end + + function e = descendants(obj, options) + % DESCENDANTS - the entities reached by following relations down, of a type or kind + % + % E = DESCENDANTS(OBJ, 'Type', TYPE) follows every relation + % inward (to the entities that point to OBJ: its members, its + % parts, what was contained in it), then theirs, and so on, and + % returns the nearest entities whose subject type is TYPE -- + % without knowing which relations lead there: + % + % descendants([plates{:}], 'Type', 'organism') % the worms that were on them + % + % OBJ may be an array of entities (of one class; from a cell + % array C, call DESCENDANTS([C{:}], ...) as above). Options: + % 'Type' a subject type: 'organism', 'group', 'material', + % 'culture', ... (the entity's `type`) + % 'Kind' a document class the entity is (isa): 'subject', + % 'person', 'strain', ... + % 'Relation' follow only these relations (default: all) + % 'MaxDepth' steps to take at most (default 10) + % 'Table' true: a table instead, as PARENTS returns, with + % a `depth` column (the steps taken) + % An entity that matches is returned and not followed further. + % With neither 'Type' nor 'Kind', every entity reached is + % returned. Each step is one search over everything reached so + % far; each entity is visited once. With 'Type' or 'Kind', the + % search picks out the matches, and only they are read: the + % entities passed through on the way are followed by id alone. + arguments + obj + options.Type (1,:) char = '' + options.Kind (1,:) char = '' + options.Relation {mustBeText} = {} + options.MaxDepth (1,1) double {mustBePositive} = 10 + options.Table (1,1) logical = false + end + e = obj.reach('in', options); + end + + function e = ancestors(obj, options) + % ANCESTORS - the entities reached by following relations up, of a type or kind + % + % E = ANCESTORS(OBJ, 'Type', TYPE): as DESCENDANTS, following + % every relation outward (to what OBJ is a member of, part of, + % contained in, ...): + % + % w.ancestors('Type', 'material') % the plates a worm was on + % w.ancestors('Type', 'group') % its cohort + arguments + obj + options.Type (1,:) char = '' + options.Kind (1,:) char = '' + options.Relation {mustBeText} = {} + options.MaxDepth (1,1) double {mustBePositive} = 10 + options.Table (1,1) logical = false + end + e = obj.reach('out', options); + end + end + + methods (Access = protected) + function out = walk(obj, path, direction, asTable) + % WALK - follow PATH from every entity in OBJ, one search per step + path = cellstr(path); + if isempty(obj) + out = {}; + if asTable, out = ndi.entity.emptyWalkTable(); end + return; + end + container = obj(1).container_; + ids = arrayfun(@(x) x.document_id, obj, 'UniformOutput', false); + ids = ids(:); + % pairs: one row per (start, current) still being followed + pairs = unique(table(ids, ids, 'VariableNames', {'start', 'at'}), 'stable'); + if strcmp(direction, 'out') + from = 'child_id'; to = 'parent_id'; + else + from = 'parent_id'; to = 'child_id'; + end + for s = 1:numel(path) + E = ndi.entity.edges(container, unique(pairs.at, 'stable'), path{s}, from, to); + pairs = innerjoin(pairs, E, 'LeftKeys', 'at', 'RightKeys', 'from'); + pairs = unique(table(pairs.start, pairs.to, 'VariableNames', {'start', 'at'}), 'stable'); + if height(pairs) == 0 + break; + end + end + endIds = unique(pairs.at, 'stable'); + ents = ndi.entity.fetch(container, endIds); + found = ~cellfun(@isempty, ents); + if ~asTable + out = ents(found)'; + return; + end + byId = containers.Map(endIds(found), ents(found)); + [u, iu] = unique(ids, 'stable'); + nameOf = containers.Map(u, arrayfun(@(x) string(x.name), obj(iu), 'UniformOutput', false)); + keep = isKey(byId, pairs.at); + pairs = pairs(keep, :); + out = ndi.entity.emptyWalkTable(); + n = height(pairs); + if n == 0, return; end + ends = values(byId, pairs.at); + out = table(string(pairs.start), string(cellfun(@(i) char(nameOf(i)), pairs.start, 'UniformOutput', false)), ... + string(pairs.at), string(cellfun(@(x) x.name, ends(:), 'UniformOutput', false)), ends(:), ... + 'VariableNames', {'start_id', 'start_name', 'end_id', 'end_name', 'end'}); + end + + function out = reach(obj, direction, options) + % REACH - breadth-first over relations from every entity in OBJ + filtered = ~isempty(options.Type) || ~isempty(options.Kind); + filterQuery = []; + if ~isempty(options.Type) + filterQuery = ndi.v2.blockQuery('subject.type.name', 'exact_string', options.Type, ''); + end + if ~isempty(options.Kind) + k = ndi.v2.isaQuery(options.Kind); + if isempty(filterQuery), filterQuery = k; else, filterQuery = filterQuery & k; end + end + rel = cellstr(options.Relation); + empty = ndi.entity.emptyWalkTable(); + empty.depth = zeros(0, 1); + if isempty(obj) + out = {}; + if options.Table, out = empty; end + return; + end + container = obj(1).container_; + ids = arrayfun(@(x) x.document_id, obj, 'UniformOutput', false); + ids = ids(:); + [u, iu] = unique(ids, 'stable'); + nameOf = containers.Map(u, arrayfun(@(x) string(x.name), obj(iu), 'UniformOutput', false)); + if strcmp(direction, 'out') + from = 'child_id'; to = 'parent_id'; + else + from = 'parent_id'; to = 'child_id'; + end + seen = unique(ids); % never followed twice, never returned as a start + frontier = unique(table(ids, ids, 'VariableNames', {'start', 'at'}), 'stable'); + byId = containers.Map(); + hits = table(cell(0, 1), cell(0, 1), zeros(0, 1), 'VariableNames', {'start', 'at', 'depth'}); + for depth = 1:options.MaxDepth + E = ndi.entity.edges(container, unique(frontier.at, 'stable'), rel, from, to); + if height(E) == 0, break; end + next = innerjoin(frontier, E, 'LeftKeys', 'at', 'RightKeys', 'from'); + next = unique(table(next.start, next.to, 'VariableNames', {'start', 'at'}), 'stable'); + next = next(~ismember(next.at, seen), :); + if height(next) == 0, break; end + newIds = unique(next.at, 'stable'); + seen = [seen; newIds]; %#ok + % with a filter, the search itself says which ids match, and + % only those are read; the others are followed by id alone + % (on the Haley plates: 7,422 patches never read) + ents = ndi.entity.fetch(container, newIds, filterQuery); + match = ~cellfun(@isempty, ents); + for k = find(match)' + byId(newIds{k}) = ents{k}; + end + isHit = ismember(next.at, newIds(match)); + h = next(isHit, :); + h.depth = repmat(depth, height(h), 1); + hits = [hits; h]; %#ok + if filtered + frontier = next(~isHit, :); % a match is not followed further + else + frontier = next(isHit, :); % every id found in this container + end + if height(frontier) == 0, break; end + end + hitIds = unique(hits.at, 'stable'); + if ~options.Table + out = cellfun(@(i) byId(i), hitIds, 'UniformOutput', false)'; + return; + end + out = empty; + if height(hits) == 0, return; end + ends = values(byId, hits.at); + out = table(string(hits.start), string(cellfun(@(i) char(nameOf(i)), hits.start, 'UniformOutput', false)), ... + string(hits.at), string(cellfun(@(x) x.name, ends(:), 'UniformOutput', false)), ends(:), hits.depth, ... + 'VariableNames', {'start_id', 'start_name', 'end_id', 'end_name', 'end', 'depth'}); + end + end + + methods (Static, Access = protected) + function E = edges(container, ids, relationName, from, to) + % EDGES - table from, to, distributive: the relations whose FROM + % end is one of IDS, and whether each is marked distributive + % RELATIONNAME: a name, a cellstr of names, or empty for every relation + fromIds = cell(0, 1); toIds = cell(0, 1); dist = false(0, 1); + chunk = 200; + for c = 1:chunk:numel(ids) + part = ids(c:min(c + chunk - 1, numel(ids))); + q = ndi.entity.anyOf(cellfun(@(i) ndi.query('', 'depends_on', from, i), part, ... + 'UniformOutput', false)); + docs = container.database_search(ndi.query('', 'isa', 'directed_relation', '') & q); + for i = 1:numel(docs) + p = ndi.v2.props(docs{i}); + r = ndi.v2.termName(ndi.v2.blockOf(p, 'directed_relation', 'relation', '')); + if ~isempty(relationName) && ~any(strcmp(r, cellstr(relationName))) + continue; + end + a = ndi.v2.edgeIds(p, from); + b = ndi.v2.edgeIds(p, to); + a = a(ismember(a, part)); + d = ndi.v2.blockOf(p, 'directed_relation', 'distributive', false); + d = ~isempty(d) && (islogical(d) || isnumeric(d)) && logical(d(1)); + for x = 1:numel(a) + for y = 1:numel(b) + fromIds{end+1, 1} = a{x}; %#ok + toIds{end+1, 1} = b{y}; %#ok + dist(end+1, 1) = d; %#ok + end + end + end + end + E = table(fromIds, toIds, dist, 'VariableNames', {'from', 'to', 'distributive'}); + end + + function ents = fetch(container, ids, extra) + % FETCH - the entity for each id ({} where not found), a few searches in all + % EXTRA, an ndi.query or [], is ANDed in: ids not matching it are {} too + if nargin < 3, extra = []; end + ents = cell(numel(ids), 1); + if isempty(ids), return; end + where = containers.Map(ids, num2cell(1:numel(ids))); + chunk = 200; + for c = 1:chunk:numel(ids) + part = ids(c:min(c + chunk - 1, numel(ids))); + q = ndi.entity.anyOf(cellfun(@(i) ndi.query('base.id', 'exact_string', i, ''), part, ... + 'UniformOutput', false)); + if ~isempty(extra), q = extra & q; end + docs = container.database_search(q); + for i = 1:numel(docs) + p = ndi.v2.props(docs{i}); + k = where(char(p.base.id)); + ents{k} = ndi.entity.fromDocument(container, docs{i}); + end + end + end + + function q = anyOf(qs) + % ANYOF - the OR of the queries in cell array QS (ndi.v2.anyOf) + q = ndi.v2.anyOf(qs); + end + + function T = emptyWalkTable() + T = table(strings(0, 1), strings(0, 1), strings(0, 1), strings(0, 1), cell(0, 1), ... + 'VariableNames', {'start_id', 'start_name', 'end_id', 'end_name', 'end'}); + end + end + + methods (Static) + function obj = fromDocument(container, doc) + % FROMDOCUMENT - the right object for an entity document + % + % OBJ = ndi.entity.fromDocument(CONTAINER, DOC): an ndi.entity for + % any entity document (a subject's too). DOC may be an + % ndi.document or a document id. + arguments + container + doc + end + if ischar(doc) || isstring(doc) + d = ndi.v2.getDocument(container, char(doc)); + if isempty(d) + error('ndi:entity:notFound', 'No document %s in this %s.', char(doc), class(container)); + end + doc = d; + end + obj = ndi.entity(container, doc); + end + + function T = summaryOf(entities, options) + % SUMMARYOF - the summary table of a cell array of entities (see SUMMARY) + arguments + entities + options.nodes (1,1) logical = false + end + n = numel(entities); + name = strings(n, 1); kind = strings(n, 1); id = strings(n, 1); + type = strings(n, 1); local_identifier = strings(n, 1); + isSubj = false(n, 1); % entities read from a document (any kind) + for i = 1:n + e = entities{i}; + name(i) = string(e.name); kind(i) = string(e.kind); id(i) = string(e.document_id); + if ~isempty(e.document_id) && ~isempty(e.container_) + isSubj(i) = true; + type(i) = string(e.type); local_identifier(i) = string(e.local_identifier); + end + end + T = table(name, kind, id); + if ~any(isSubj), return; end + T.type = type; T.local_identifier = local_identifier; + container = entities{find(isSubj, 1)}.container_; + L = ndi.entity.statementLinks(container, cellstr(id(isSubj)), ... + struct('kind', 'assertion', 'filt', struct()), true); + cols = containers.Map('KeyType', 'char', 'ValueType', 'any'); + for k = 1:numel(L.doc) + p = ndi.v2.props(L.doc{k}); + v = ndi.v2.termName(ndi.v2.blockOf(p, 'statement', 'variable', '')); + [vk, vv, txt] = ndi.v2.statementValue(p); + if isempty(v), continue; end + nd = ''; + if strcmp(vk, 'term') && isstruct(vv) && isfield(vv, 'node'), nd = char(vv(1).node); end + if ~isKey(cols, v), cols(v) = containers.Map('KeyType', 'char', 'ValueType', 'any'); end + m = cols(v); + if isKey(m, L.about{k}), m(L.about{k}) = [m(L.about{k}), {{txt, nd}}]; + else, m(L.about{k}) = {{txt, nd}}; end + end + % its types (instance_of: a strain, a product), each under its + % type, named as the type entity is named (T17) + S = ndi.entity.inheritanceStates(container, cellstr(id(isSubj)), true); + S = S([S.i]); + if ~isempty(S) + tdocs = typeDocuments(container, unique({S.at}, 'stable')); + for k = 1:numel(S) + if ~isKey(tdocs, S(k).at), continue; end + p = tdocs(S(k).at); + [v, blk] = ndi.v2.kindOf(p); + txt = char(ndi.v2.blockOf(p, blk, 'name', '')); + if isempty(txt), txt = char(ndi.v2.blockOf(p, blk, 'local_identifier', '')); end + if isempty(v) || isempty(txt), continue; end + nd = cellstr(ndi.v2.blockOf(p, blk, 'global_identifier', {})); + if isempty(nd), nd = ''; else, nd = nd{1}; end + if ~isKey(cols, v), cols(v) = containers.Map('KeyType', 'char', 'ValueType', 'any'); end + m = cols(v); + if isKey(m, S(k).start), m(S(k).start) = [m(S(k).start), {{txt, nd}}]; + else, m(S(k).start) = {{txt, nd}}; end + end + end + vars = sort(keys(cols)); + for k = 1:numel(vars) + m = cols(vars{k}); + c = strings(n, 1); cn = strings(n, 1); + for i = 1:n + if ~isKey(m, char(id(i))), continue; end + pairs = m(char(id(i))); + txts = cellfun(@(x) x{1}, pairs, 'UniformOutput', false); + nds = cellfun(@(x) x{2}, pairs, 'UniformOutput', false); + [txts, keep] = unique(txts, 'stable'); + c(i) = string(strjoin(txts, '; ')); + cn(i) = string(strjoin(nds(keep), '; ')); + end + name_ = matlab.lang.makeValidName(vars{k}); + T.(name_) = c; + if options.nodes, T.([name_ '_node']) = cn; end + end + end + + function reached = walkIds(container, ids, relationNames, direction, maxDepth) + % WALKIDS - the ids reached from IDS across relations, without reading entities + % + % REACHED = ndi.entity.walkIds(CONTAINER, IDS, RELATIONNAMES, DIRECTION) + % follows RELATIONNAMES (a name, a cellstr, or {} for all) from + % every id in IDS, 'in' (to what points at them: members, parts) + % or 'out' (to what they point at), step after step until nothing + % new is reached or MAXDEPTH (default Inf) steps. REACHED (a row + % cellstr) leaves out IDS themselves. One search per step for + % every 200 ids; nothing is read but the relations. + arguments + container + ids + relationNames = {} + direction (1,:) char {mustBeMember(direction, {'in', 'out'})} = 'in' + maxDepth (1,1) double = Inf + end + if strcmp(direction, 'out') + from = 'child_id'; to = 'parent_id'; + else + from = 'parent_id'; to = 'child_id'; + end + frontier = unique(cellstr(ids), 'stable'); + frontier = frontier(:); + seen = frontier; + reached = {}; + depth = 0; + while ~isempty(frontier) && depth < maxDepth + depth = depth + 1; + E = ndi.entity.edges(container, frontier, relationNames, from, to); + next = unique(E.to, 'stable'); + next = next(~ismember(next, seen)); + seen = [seen; next]; %#ok + reached = [reached, next(:)']; %#ok + frontier = next; + end + end + + function e = fetchMany(container, ids) + % FETCHMANY - the entities with these document ids, a few searches in all + % + % E = ndi.entity.fetchMany(CONTAINER, IDS): a cell array (row), one + % entity per id found, in the order of IDS; ids not found are + % left out. + arguments + container + ids + end + ids = unique(cellstr(ids), 'stable'); + e = ndi.entity.fetch(container, ids(:)); + e = reshape(e(~cellfun(@isempty, e)), 1, []); + end + + function s = search(container, conditions, options) + % SEARCH - the entities in a session or dataset, by what is true of them + % + % S = ndi.entity.search(CONTAINER, CONDITIONS, ...) returns a cell + % array of entities (ndi.entity objects): those for which every + % condition holds. CONDITIONS is a cell of PROPERTY, VALUE pairs + % (default {}: every entity); options follow as name, value. + % ndi.entity.search(ds, {'kind', 'person', 'name', 'Jess*'}) + % ndi.entity.search(ds, {'type', 'organism', ... + % 'species', 'Caenorhabditis elegans', 'strain', {'N2', 'CB*'}}) + % ndi.entity.search(ds, {'manipulation', {'method', 'heating'}}) + % ndi.entity.search(ds, {'type', 'organism', ... + % 'contained_in', {'manipulation', {'variable', 'NGM agar'}}}) + % ndi.entity.search(ds, {'type', 'organism'}, 'inherited', false) + % + % A PROPERTY is one of: + % 'type', 'name', 'local_identifier', 'id', 'kind' + % the entity's own fields ('id': its document id; 'kind': + % its document class or, one entity class, its type -- + % 'person' under either schema; 'subject' is not a kind: + % the subjects are 'type', ndi.v2.subjectTypes()) + % 'statement', 'assertion', 'interaction', 'observation', + % 'manipulation', 'calculation' + % a statement of that kind (or a child kind) about the + % subject; VALUE is a cell of filters on ONE statement: + % 'variable', 'method' (not an assertion's), 'value', + % 'formulation' (a dose's), and when it held: 'at', 'during', + % 'before', 'after', 'duration' (ndi.v2.timeFilter). 'during' + % also takes statements, {'observation', {...}}: a statement + % of the subject's that overlaps one of those of the same + % subject (by 'inherited' and 'context'). The key twice is + % two statements. + % 'relation', 'directed_relation', 'undirected_relation' + % a relation the subject is in; VALUE is a cell: + % 'name' the relation ('contained_in', 'paired_with', ...) + % 'parent' the subject is the child; the parent is ... + % 'child' the subject is the parent; a child is ... + % 'with' the other end, either way, is ... + % 'at', 'during', 'before', 'after', 'duration' + % when the relation held (ndi.v2.timeFilter) + % where ... is a subject (an ndi.entity, several in a cell, + % or a document id) or a cell describing subjects -- anything + % ndi.entity.search takes, nested as deep as needed. + % a relation's name ('member_of', 'contained_in', 'part_of', + % 'paired_with', ...): short for 'directed_relation', + % {'name', NAME, 'parent', VALUE} ('with' for an undirected one) + % + % Times typed without a zone are read in the zone the lab + % recorded each time in; 'during' takes two times, a date or a + % month; 'duration' takes '>=3h', '<30min', ... + % anything else: an asserted variable -- 'strain', 'N2' is + % 'assertion', {'variable', 'strain', 'value', 'N2'} + % Property names ignore case. + % + % A pattern matches a term's name ignoring case or its node + % exactly ('NCBITaxon:6239'); a cell array is any of them; '*' is + % a wildcard, '\*' a literal star. A value is compared by its kind: + % numbers and dates take '>', '>=', '<', '<=' ('>=0.02', + % '>2023-11-16'); see ndi.v2.searchStatements. All pairs must hold. + % + % Options (name, value after CONDITIONS): + % 'inherited' true (default) or false, below + % 'explain' true: print what the search means before it runs + % 'strict' true (default): an unknown property is an error and + % values matching nothing a warning, each naming what + % there is (with a "did you mean"); false: an empty + % answer, quietly (for scripts over many datasets) + % 'tolerant' false (default): times are compared as stated; + % true: a time matches when its tolerance allows + % 'synonyms' true (default): a value that matches no term by + % its name or node also matches a term it is a + % synonym of -- {'species', 'C. elegans'} finds + % NCBITaxon:6239 (ndi.v2.synonyms, from + % ndi.ontology.lookup, looked up once per term + % and kept on disk; nothing when offline) + % 'context' false (default); true: also the entities a + % matching observation, manipulation or calculation + % was CONTEXT for -- what its subject contained at + % the time (V_eta tenet T17), below + % A nested description takes the outer search's 'strict' and + % 'inherited'. + % + % 'inherited' (default true): + % - a statement or relation about a group holds of its members + % when it is marked distributive, all the way down member_of + % (a strain stated on a cohort; a cohort contained_in a plate); + % - an assertion about a whole holds of its parts and samples, + % down part_of, sample_of, aliquot_of and passage_of (an + % animal's strain is its slice's). + % contained_in is not followed for statements (a plate's are not + % its worms'). false: only what is stated of the subject itself. + % 'context' (default false): a container's observations, + % manipulations and calculations reach what it contained, down + % contained_in (and nested containers), but only when the + % statement's time overlaps the stay's; a distributive stay of a + % group reaches its members. A plate's 22 C reading finds the + % worms on the plate during it. A stay or statement whose times + % cannot be compared is left out. Assertions never pass. + % The group or whole a statement was about matches too; 'type' + % narrows to what you want. + % + % An unknown asserted variable, relation name, or a kind / + % variable / method no statement has, is an error; values none of + % which match is a warning naming the values there are. + arguments + container + conditions cell = {} + options.inherited (1,1) logical = true + options.context (1,1) logical = false + options.strict (1,1) logical = true + options.explain (1,1) logical = false + options.tolerant (1,1) logical = false + options.synonyms (1,1) logical = true + end + spec = ndi.entity.parseSearch(conditions); + for f = fieldnames(options)' + spec.(f{1}) = options.(f{1}); + end + s = ndi.entity.searchSpec(container, spec); + end % search() + + end + + methods (Static, Hidden) + function s = searchSpec(container, spec) + % SEARCHSPEC - SEARCH for a parsed search spec (parseSearch) + % + % A single 'kind' condition narrows the database search to that + % class (or merged type); the condition is still checked below. + arguments + container + spec (1,1) struct + end + kind = 'entity'; + k = spec.own(strcmp(spec.own(:, 1), 'kind'), 2); + if isscalar(k) && ischar(k{1}) && ~ndi.v2.hasWildcard(k{1}) ... + && ~any(strcmp(k{1}, ndi.v2.entityTypesFor('subject'))) + kind = k{1}; + end + spec.kind = kind; + if spec.explain + fprintf('%s\n', ndi.entity.explainSearch(spec, '', containerWord(container))); + end + ids = ndi.entity.searchIds(container, spec); + if iscell(ids) + s = ndi.entity.fetchMany(container, ids); + else + q = ndi.v2.isaQuery(kind); % a merged entity by its type (2026-10-08) + lid = spec.own(strcmp(spec.own(:, 1), 'local_identifier'), 2); + if isscalar(lid) && ischar(lid{1}) && ~ndi.v2.hasWildcard(lid{1}) + if ~strcmp(kind, 'entity') % (every entity: compared below) + q = q & ndi.v2.blockQuery([kind '.local_identifier'], 'exact_string_anycase', ... + strrep(lid{1}, '\*', '*'), ''); + end + end + docs = container.database_search(q); + s = cellfun(@(d) ndi.entity.fromDocument(container, d), docs, 'UniformOutput', false); + end + s = reshape(s, 1, []); + keep = cellfun(@(x) strcmp(kind, 'entity') || strcmp(x.kind, kind), s); + own = spec.own; + for k = 1:size(own, 1) + for i = find(keep) + x = s{i}; + switch own{k, 1} + case 'type', keep(i) = ndi.v2.matchTerm(x.type, own{k, 2}); + case 'name', keep(i) = ndi.v2.matchTerm(x.name, own{k, 2}); + case 'kind', keep(i) = ndi.v2.matchTerm(x.kind, own{k, 2}); + case 'local_identifier', keep(i) = ndi.v2.matchTerm(x.local_identifier, own{k, 2}); + case 'id', keep(i) = any(strcmp(x.document_id, cellstr(own{k, 2}))); + end + end + end + s = reshape(s(keep), 1, []); + end % searchSpec() + + function t = explainNested(c, indent) + % EXPLAINNESTED - a description (a cell of search's pairs) in words + t = ndi.entity.explainSearch(ndi.entity.parseSearch(c), indent, ''); + end + + function spec = parseSearchNested(c) + % PARSESEARCHNESTED - parseSearch, for the explanation of a nested description + spec = ndi.entity.parseSearch(c); + end + + end + + methods (Static, Access = protected) + function spec = parseSearch(args) + % PARSESEARCH - search's CONDITIONS (PROPERTY, VALUE pairs) as a + % search spec: own fields, statement filters, relation filters; + % the switches at their defaults (search sets them from its options) + if mod(numel(args), 2) + error('ndi:entity:search:pairs', ... + 'Conditions are PROPERTY, VALUE pairs in a cell, e.g. {''type'', ''organism'', ''strain'', ''N2''}.'); + end + statementKinds = {'statement', 'assertion', 'interaction', 'observation', 'manipulation', 'calculation'}; + relationKinds = {'relation', 'directed_relation', 'undirected_relation'}; + [directed, undirected] = ndi.v2.relationNames(); + spec = struct('own', {cell(0, 2)}, ... + 'filters', {struct('kind', {}, 'filt', {})}, ... + 'relations', {struct('kind', {}, 'name', {}, 'side', {}, 'target', {}, 'time', {})}, ... + 'inherited', true, 'explain', false, 'strict', true, 'tolerant', false, ... + 'synonyms', true, ... + 'context', false); + for k = 1:2:numel(args) + p = args{k}; + if ~(ischar(p) || (isstring(p) && isscalar(p))) + error('ndi:entity:search:pairs', 'A property name must be text (argument %d).', k + 1); + end + p = char(p); + v = args{k + 1}; + if isstring(v), v = cellstr(v); if isscalar(v), v = v{1}; end, end + key = lower(strrep(p, '_', '')); + lp = lower(p); + if any(strcmp(key, {'inherited', 'explain', 'strict', 'tolerant', 'context', 'synonyms'})) + error('ndi:entity:search:option', ... + ['''%s'' is an option, not a condition: give it after the cell, ' ... + 'ndi.entity.search(S, {...}, ''%s'', %s).'], p, key, 'true'); + elseif strcmp(key, 'kind') && any(strcmpi(cellstr(v), 'subject')) + error('ndi:entity:search:subjectKind', ... + ['''subject'' is not a kind: give the type, e.g. {''type'', ' ... + '{''organism'', ''culture'', ''tissue'', ''cell'', ''group''}} (ndi.v2.subjectTypes).']); + elseif any(strcmp(key, {'type', 'name', 'id', 'kind'})) + spec.own(end+1, :) = {key, v}; + elseif strcmp(key, 'localidentifier') + spec.own(end+1, :) = {'local_identifier', v}; + elseif any(strcmp(lp, statementKinds)) + spec.filters(end+1) = struct('kind', lp, 'filt', ndi.entity.statementFilter(lp, v)); + elseif any(strcmp(lp, relationKinds)) + spec.relations(end+1) = ndi.entity.relationFilter(lp, v); + elseif any(strcmpi(p, directed)) + spec.relations(end+1) = struct('kind', 'directed_relation', 'name', {{p}}, ... + 'side', 'parent', 'target', {v}, 'time', struct()); + elseif any(strcmpi(p, undirected)) + spec.relations(end+1) = struct('kind', 'undirected_relation', 'name', {{p}}, ... + 'side', 'with', 'target', {v}, 'time', struct()); + else + if ~iscell(v), v = {v}; end % a cell is already "any of" + spec.filters(end+1) = struct('kind', 'assertion', ... + 'filt', struct('variable', p, 'value', {v})); + end + end + end + + function filt = statementFilter(kind, c) + % STATEMENTFILTER - {'variable', V, 'method', M, ...} as a struct + if ~iscell(c) || mod(numel(c), 2) + error('ndi:entity:search:statementFilter', ... + '''%s'' takes a cell of filters, e.g. {''variable'', ''NGM agar'', ''value'', ''>=0.02''}.', kind); + end + filt = struct(); + for k = 1:2:numel(c) + n = lower(char(c{k})); + if ~any(strcmp(n, {'variable', 'method', 'value', 'formulation', ... + 'at', 'during', 'before', 'after', 'duration'})) + error('ndi:entity:search:statementFilter', ... + ['Unknown filter ''%s'' for ''%s'': variable, method, value, formulation, ' ... + 'at, during, before, after, duration.'], c{k}, kind); + end + v = c{k + 1}; + if isstring(v), v = cellstr(v); if isscalar(v), v = v{1}; end, end + if strcmp(n, 'during') && ndi.entity.isStatementDescription(v) + % 'during', {'observation', {...}}: when a statement + % matching that holds of the same subject (statementLinks) + filt.during_statements = ndi.entity.statementKinds(v); + continue; + end + filt.(n) = v; + end + if strcmp(kind, 'assertion') && isfield(filt, 'during_statements') + error('ndi:entity:search:assertionTime', 'An assertion has no time, so it is never ''during'' anything.'); + end + if strcmp(kind, 'assertion') && isfield(filt, 'method') + error('ndi:entity:search:assertionMethod', 'An assertion has no method.'); + end + end + + function tf = isStatementDescription(v) + % ISSTATEMENTDESCRIPTION - is V {'observation', {...}, ...}, a + % description of statements, rather than times? + kinds = {'statement', 'interaction', 'observation', 'manipulation', 'calculation'}; + tf = iscell(v) && ~isempty(v) && (ischar(v{1}) || (isstring(v{1}) && isscalar(v{1}))) ... + && any(strcmpi(char(v{1}), [kinds, {'assertion'}])); + end + + function kinds = statementKinds(v) + % STATEMENTKINDS - {'observation', {...}, 'manipulation', {...}} + % (a kind alone is any of that kind) as the kinds struct + % statementLinks takes; several are any of them + kinds = struct('kind', {}, 'filt', {}); + k = 1; + while k <= numel(v) + kind = lower(char(v{k})); + if ~ndi.entity.isStatementDescription({kind}) + error('ndi:entity:search:statementFilter', ... + '''during'' takes times or statements, e.g. {''observation'', {''variable'', ''ambient temperature'', ''value'', ''>22''}}.'); + end + f = struct(); + if k < numel(v) && iscell(v{k + 1}) + f = ndi.entity.statementFilter(kind, v{k + 1}); + k = k + 1; + end + if strcmp(kind, 'assertion') + error('ndi:entity:search:assertionTime', 'An assertion has no time, so nothing is ''during'' it.'); + end + kinds(end+1) = struct('kind', kind, 'filt', f); %#ok + k = k + 1; + end + end + + function r = relationFilter(kind, c) + % RELATIONFILTER - {'name', N, 'parent'|'child'|'with', X} as a struct + if ~iscell(c) || mod(numel(c), 2) + error('ndi:entity:search:relationFilter', ... + '''%s'' takes a cell, e.g. {''name'', ''contained_in'', ''parent'', {''type'', ''material''}}.', kind); + end + r = struct('kind', kind, 'name', {{}}, 'side', '', 'target', {[]}, 'time', struct()); + for k = 1:2:numel(c) + n = lower(char(c{k})); + v = c{k + 1}; + switch n + case 'name' + if ~iscell(v), v = {v}; end + r.name = cellfun(@char, v, 'UniformOutput', false); + case {'parent', 'child', 'with'} + if ~isempty(r.side) + error('ndi:entity:search:relationFilter', ... + 'Give one of ''parent'', ''child'' and ''with'' in a relation (got ''%s'' and ''%s'').', r.side, n); + end + if strcmp(kind, 'undirected_relation') && ~strcmp(n, 'with') + error('ndi:entity:search:relationFilter', ... + 'An undirected relation has no %s: use ''with''.', n); + end + r.side = n; + r.target = v; + case {'at', 'during', 'before', 'after', 'duration'} + r.time.(n) = v; + case 'depth' + error('ndi:entity:search:notYet', ... + 'Relation filter ''%s'' is not built yet.', n); + otherwise + error('ndi:entity:search:relationFilter', ... + ['Unknown filter ''%s'' for ''%s'': name, parent, child, with, ' ... + 'at, during, before, after, duration.'], c{k}, kind); + end + end + if ~isempty(r.name) + [directed, undirected] = ndi.v2.relationNames(); + known = [directed, undirected]; + for k = 1:numel(r.name) + if ~isempty(known) && ~any(cellfun(@(x) ndi.v2.matchTerm(x, r.name{k}), known)) + error('ndi:entity:search:unknownRelation', 'No relation is called ''%s''. Relations: %s.', ... + r.name{k}, strjoin(known, ', ')); + end + end + end + end + + function ids = searchIds(container, spec) + % SEARCHIDS - the subject ids every statement and relation filter + % of SPEC holds of ([] when SPEC has none) + ids = []; + for k = 1:numel(spec.filters) + these = ndi.entity.subjectsOf(container, spec.filters(k), spec); + ids = narrow(ids, these); + if isempty(ids), ids = {}; return; end + end + for k = 1:numel(spec.relations) + these = ndi.entity.relatedIds(container, spec.relations(k), spec); + ids = narrow(ids, these); + if isempty(ids), ids = {}; return; end + end + if isempty(ids) && ~iscell(ids) + id = spec.own(strcmp(spec.own(:, 1), 'id'), 2); + if isscalar(id), ids = unique(cellstr(id{1}), 'stable'); end + end + end + + function ids = subjectsOf(container, f, spec) + % SUBJECTSOF - ids of the subjects statement filter F holds of: the + % statements' subjects; and, SPEC.inherited, the members (down + % member_of) of a group whose statement is distributive, and the + % parts and samples of a subject an assertion is about. With + % SPEC.strict, an unknown variable or method is an error and + % values that match nothing a warning (each naming what there is). + if isfield(f.filt, 'during_statements') + % the subjects with such a statement, then those for which it + % overlaps one matching the description (statementLinks) + g = f; g.filt = rmfield(g.filt, 'during_statements'); + ids = ndi.entity.subjectsOf(container, g, spec); + if isempty(ids), return; end + k = f; k.filt.tolerant = spec.tolerant; k.filt.synonyms = isfield(spec, 'synonyms') && spec.synonyms; + L = ndi.entity.statementLinks(container, ids, k, spec.inherited, ... + isfield(spec, 'context') && spec.context); + ids = ids(ismember(ids, L.about)); + return; + end + inherited = spec.inherited; + filt = f.filt; + filt.tolerant = spec.tolerant; + filt.synonyms = isfield(spec, 'synonyms') && spec.synonyms; + [docs, info] = ndi.v2.searchStatements(container, f.kind, filt); + if info.structural == 0 && strcmp(f.kind, 'assertion') + % with one entity class, a strain is not asserted: the + % subject is an instance_of the strain entity + via = ndi.entity.instanceIds(container, filt, spec); + if iscell(via), ids = via; return; end + end + if info.structural == 0 + ndi.entity.noSuchStatement(container, f, spec.strict, kindOfSpec(spec)); % errors unless each part exists + ids = {}; + return; + elseif isempty(docs) && spec.strict && info.timed && info.beforeTime > 0 + warning('ndi:entity:search:noSuchTime', '%s', ... + noTimeMessage(f.kind, f.filt, info.time, containerWord(container))); + elseif isempty(docs) && spec.strict + warning('ndi:entity:search:noSuchValue', '%s', ... + noValueMessage(f, info.values, containerWord(container), kindOfSpec(spec))); + end + ids = {}; groups = {}; asserted = {}; + for i = 1:numel(docs) + p = ndi.v2.props(docs{i}); + sid = ndi.v2.edgeIds(p, 'entity_id'); + ids = [ids, sid]; %#ok + if ~inherited, continue; end + if isDistributive(ndi.v2.blockOf(p, 'statement', 'distributive', false)) + groups = [groups, sid]; %#ok + end + if any(strcmp(ndi.v2.classChain(p), 'assertion')) + asserted = [asserted, sid]; %#ok + end + end + ids = unique(ids, 'stable'); + if isfield(spec, 'context') && spec.context + ids = unique([ids, ndi.entity.contextIds(container, docs, spec.tolerant)], 'stable'); + end + if ~inherited, return; end + if ~isempty(groups) + ids = unique([ids, ndi.entity.walkIds(container, unique(groups, 'stable'), ... + 'member_of', 'in')], 'stable'); + end + if ~isempty(asserted) + % an assertion holds of the parts and samples of what it is + % about, and of those of the members it holds of + g = unique(groups(ismember(groups, asserted)), 'stable'); + whole = asserted; + if ~isempty(g) + whole = [whole, ndi.entity.walkIds(container, g, 'member_of', 'in')]; + end + whole = unique(whole, 'stable'); + parts = ndi.entity.walkIds(container, whole, ... + {'part_of', 'sample_of', 'aliquot_of', 'passage_of'}, 'in'); + ids = unique([ids, parts], 'stable'); + end + end + + function ids = relatedIds(container, r, spec) + % RELATEDIDS - ids of the subjects in a relation matching R: the + % relation kind and name, and the other end (a subject, ids, or a + % description, searched first). INHERITED: a distributive relation + % whose child is a group holds of the group's members too. + inherited = spec.inherited; + T = ndi.entity.targetIds(container, r.target, spec); % [] = any + if iscell(T) && isempty(T), ids = {}; return; end + ids = {}; + if ~strcmp(r.kind, 'undirected_relation') + sides = {r.side}; + if isempty(r.side) || strcmp(r.side, 'with'), sides = {'parent', 'child'}; end + for k = 1:numel(sides) + if strcmp(sides{k}, 'parent') % this subject is the child + mine = 'child_id'; theirs = 'parent_id'; + else + mine = 'parent_id'; theirs = 'child_id'; + end + docs = ndi.entity.relationDocs(container, 'directed_relation', r.name, theirs, T); + docs = ndi.entity.timed(container, docs, r, spec); + groups = {}; + for i = 1:numel(docs) + p = ndi.v2.props(docs{i}); + m = ndi.v2.edgeIds(p, mine); + ids = [ids, m]; %#ok + if inherited && strcmp(mine, 'child_id') && ... + isDistributive(ndi.v2.blockOf(p, 'directed_relation', 'distributive', false)) + groups = [groups, m]; %#ok + end + end + if ~isempty(groups) + ids = [ids, ndi.entity.walkIds(container, unique(groups, 'stable'), 'member_of', 'in')]; %#ok + end + end + end + if any(strcmp(r.kind, {'relation', 'undirected_relation'})) && ... + (isempty(r.side) || strcmp(r.side, 'with')) + docs = ndi.entity.relationDocs(container, 'undirected_relation', r.name, 'entity_id', T); + docs = ndi.entity.timed(container, docs, r, spec); + for i = 1:numel(docs) + pair = ndi.v2.edgeIds(ndi.v2.props(docs{i}), 'entity_id'); + for j = 1:numel(pair) + others = pair([1:j-1, j+1:end]); + if ~iscell(T) || any(ismember(others, T)) + ids{end+1} = pair{j}; %#ok + end + end + end + end + ids = unique(ids, 'stable'); + end + + function docs = timed(container, docs, r, spec) + % TIMED - the relation documents DOCS whose time meets R's time + % filters (all of them when R has none) + if isempty(fieldnames(r.time)) || isempty(docs), return; end + [keep, tinfo] = ndi.v2.timeFilter(container, docs, r.time, spec.tolerant); + if ~any(keep) && spec.strict + warning('ndi:entity:search:noSuchTime', '%s', ... + noTimeMessage(r.kind, r.time, tinfo, containerWord(container))); + end + docs = docs(keep); + end + + function docs = relationDocs(container, className, names, edge, T) + % RELATIONDOCS - relation documents of CLASSNAME named one of NAMES + % ({} any) whose EDGE is one of the ids T ([] any) + base = ndi.query('', 'isa', className, ''); + if ~isempty(names) + t = ndi.v2.termQuery([className '.relation'], names); + if ~isempty(t), base = base & t; end + end + if ~iscell(T) + docs = container.database_search(base); + else + docs = {}; + for c = 1:200:numel(T) + part = T(c:min(c + 199, numel(T))); + q = ndi.entity.anyOf(cellfun(@(i) ndi.query('', 'depends_on', edge, i), part, ... + 'UniformOutput', false)); + docs = [docs, reshape(container.database_search(base & q), 1, [])]; %#ok + end + end + if isempty(names), return; end + keep = cellfun(@(d) ndi.v2.matchTerm(ndi.v2.blockOf(ndi.v2.props(d), className, 'relation', []), ... + names), docs); + docs = docs(keep); + end + + function ids = instanceIds(container, filt, spec) + % INSTANCEIDS - the subjects that are an instance_of an entity of + % the type FILT's variable names ('strain', 'product'), the + % entity matching FILT's value by name or global identifier. With + % one entity class (did-schema V_eta_entity_composition_plan.md + % sec. 5) that is how a subject has a strain. [] when the variable + % names no such type or the container holds no entity of it -- + % the caller then reports the property as unknown. A session + % opened from its dataset also looks in the dataset's documents, + % where the strain entities are (entitiesOfType). + ids = []; + v = cellstr(filt.variable); + if ~isscalar(v), return; end + type = lower(v{1}); + types = ndi.v2.entityTypesFor(type); + if ~isscalar(types) || ~strcmp(types{1}, type), return; end + targets = entitiesOfType(container, type); + if isempty(targets), return; end + if isfield(filt, 'value') && ~isempty(filt.value) + keep = cellfun(@(d) entityMatches(ndi.v2.props(d), type, filt.value), targets); + if ~any(keep) && isfield(spec, 'synonyms') && spec.synonyms + % no name or identifier matched: the names its identifiers go by + keep = cellfun(@(d) entitySynonymMatches(ndi.v2.props(d), type, filt.value), targets); + end + targets = targets(keep); + end + if isempty(targets) + if spec.strict + warning('ndi:entity:search:noSuchValue', 'No %s in this %s is called %s.', ... + type, containerWord(container), strjoin(cellstr(filt.value), ' or ')); + end + ids = {}; + return; + end + tid = cellfun(@(d) d.document_properties.base.id, targets, 'UniformOutput', false); + r = struct('kind', 'directed_relation', 'name', {{'instance_of'}}, 'side', 'parent', ... + 'target', struct('ids', {reshape(tid, 1, [])}), 'time', struct()); + ids = ndi.entity.relatedIds(container, r, spec); + end + + function T = targetIds(container, target, spec) + % TARGETIDS - the ids a relation's other end may be: [] for any; + % an ndi.entity (or several, or a cell of them); a document id; or + % a cell describing subjects, searched with ndi.entity.search + if isempty(target) && ~iscell(target) + T = []; + elseif isstruct(target) && isfield(target, 'ids') + T = target.ids; % document ids, already resolved + elseif isa(target, 'ndi.entity') + T = arrayfun(@(x) x.document_id, target, 'UniformOutput', false); + elseif ischar(target) || (isstring(target) && isscalar(target)) + T = {char(target)}; + elseif iscell(target) && ~isempty(target) && all(cellfun(@(x) isa(x, 'ndi.entity'), target)) + T = cellfun(@(x) x.document_id, target, 'UniformOutput', false); + elseif iscell(target) + % a description: the outer search's strict and inherited + % hold inside it + s = ndi.entity.search(container, target, 'strict', spec.strict, ... + 'inherited', spec.inherited); + T = cellfun(@(x) x.document_id, s, 'UniformOutput', false); + else + error('ndi:entity:search:relationTarget', ... + 'The other end of a relation is an entity, a document id, or a cell describing entities.'); + end + if iscell(T), T = reshape(T, 1, []); end + end + + function t = explainSearch(spec, indent, where) + % EXPLAINSEARCH - SPEC in words, one line per condition; WHERE is + % 'dataset' or 'session' for the first line ('' when nested) + lines = {}; + [plural, ~, named] = nounOf(spec); + for k = 1:size(spec.own, 1) + if k == named, continue; end % said by the noun ("organisms that") + v = valueText(spec.own{k, 2}); + switch spec.own{k, 1} + case 'type', lines{end+1} = sprintf('are of type %s', v); %#ok + case 'kind', lines{end+1} = sprintf('are of kind %s', v); %#ok + case 'name', lines{end+1} = sprintf('are named %s', v); %#ok + case 'local_identifier', lines{end+1} = sprintf('have local identifier %s', v); %#ok + case 'id', lines{end+1} = sprintf('have id %s', v); %#ok + end + end + for k = 1:numel(spec.filters) + lines{end+1} = statementText(spec.filters(k), spec.inherited); %#ok + if isfield(spec, 'context') && spec.context && ~strcmp(spec.filters(k).kind, 'assertion') + lines{end} = [lines{end} ... + ', or were contained in something that did while they were in it']; + end + end + for k = 1:numel(spec.relations) + lines{end+1} = relationText(spec.relations(k), indent, spec.inherited); %#ok + end + if isempty(where) + head = sprintf('%s that', plural); + else + head = sprintf('Searching this %s for %s that', where, plural); + if ~spec.inherited + head = [head ' (counting only what is stated about each one itself)']; + end + end + if isempty(lines) + if isempty(where), t = sprintf('%sany of the %s', indent, plural); + else, t = sprintf('Searching this %s for all %s', where, plural); end + return; + end + t = [indent head]; + for k = 1:numel(lines) + t = [t newline indent ' ' char(8226) ' ' lines{k}]; %#ok + end + end + + function noSuchStatement(container, f, strict, noun) + % NOSUCHSTATEMENT - no statement matched F's kind, variable and + % method together. When one of them matches nothing on its own, + % that is an error (for an asserted variable, naming the + % properties there are, with the nearest as a suggestion); when + % each exists but never on one statement, the answer is simply + % none. STRICT false: never an error. + if ~strict, return; end + where = containerWord(container); + for part = {'variable', 'method'} + if ~isfield(f.filt, part{1}), continue; end + [~, info] = ndi.v2.searchStatements(container, f.kind, struct(part{1}, {f.filt.(part{1})})); + if info.structural > 0, continue; end + pats = cellstr(f.filt.(part{1})); + if strcmp(f.kind, 'assertion') && strcmp(part{1}, 'variable') + docs = container.database_search(ndi.v2.isaQuery('assertion')); + names = cellfun(@(d) ndi.v2.termName(ndi.v2.blockOf(ndi.v2.props(d), ... + 'statement', 'variable', '')), docs, 'UniformOutput', false); + % and the entity types a subject can be an instance_of + % (one entity class: a strain is a relation) + for t = {'strain', 'product'} + if ~isempty(entitiesOfType(container, t{1})) + names{end+1} = t{1}; %#ok + end + end + props = unique([names, {'id', 'local_identifier', 'name', 'type'}]); + [~, o] = sort(lower(props)); + props = props(o); + msg = sprintf('No %s in this %s has a property %s.', noun, where, quoted(pats)); + msg = [msg didYouMean(pats, props)]; + error('ndi:entity:search:unknownVariable', '%s\nProperties in this %s: %s', ... + msg, where, strjoin(props, ', ')); + end + error('ndi:entity:search:noSuchStatement', 'No %s in this %s has %s %s.', ... + f.kind, where, part{1}, quoted(pats)); + end + end + end + + methods (Static, Hidden) + function all_ = inheritanceStates(container, ids, inherited) + % INHERITANCESTATES - (internal) what the subjects IDS inherit from + % + % A struct array, one element per (start, at) pair: START is one + % of IDS, AT an entity it inherits from (itself included), PATH + % the relations walked ('' for itself), and the flags M (a + % member_of was walked), L (a part_of, sample_of, aliquot_of or + % passage_of was) and I (the last step was instance_of: AT is a + % type of START). Groups and wholes are walked up to 16 steps; + % a type is reached from a subject, a whole, or a group whose + % instance_of is distributive, and is not walked past. Without + % INHERITED, only the subjects themselves. + ids = unique(cellstr(ids), 'stable'); + ids = reshape(ids, 1, []); + lineage = {'part_of', 'sample_of', 'aliquot_of', 'passage_of'}; + % states: start, at, hasMember, hasLineage, isInstance, path + S = struct('start', ids, 'at', ids, 'm', false, 'l', false, 'i', false, 'path', {''}); + all_ = S; + frontier = S; + depth = 0; + while inherited && ~isempty(frontier) && depth < 16 + depth = depth + 1; + frontier = frontier(~[frontier.i]); % a type is not walked past + if isempty(frontier), break; end + at = unique({frontier.at}, 'stable'); + next = struct('start', {}, 'at', {}, 'm', {}, 'l', {}, 'i', {}, 'path', {}); + for r = [{'member_of'}, lineage, {'instance_of'}] + step = r{1}; + E = ndi.entity.edges(container, at(:), step, 'child_id', 'parent_id'); + if height(E) == 0, continue; end + isType = strcmp(step, 'instance_of'); + for f = 1:numel(frontier) + hit = find(strcmp(E.from, frontier(f).at)); + for h = reshape(hit, 1, []) + % a group's type is its members' only when the + % group's instance_of is distributive + if isType && frontier(f).m && ~E.distributive(h), continue; end + p = step; + if ~isempty(frontier(f).path), p = [frontier(f).path ' > ' step]; end + next(end+1) = struct('start', frontier(f).start, 'at', E.to{h}, ... + 'm', frontier(f).m || strcmp(step, 'member_of'), ... + 'l', frontier(f).l || any(strcmp(step, lineage)), ... + 'i', isType, 'path', p); %#ok + end + end + end + % a (start, at) reached before is not followed again + keep = true(1, numel(next)); + for j = 1:numel(next) + keep(j) = ~any(strcmp({all_.start}, next(j).start) & strcmp({all_.at}, next(j).at)); + if keep(j), all_(end+1) = next(j); end %#ok + end + frontier = next(keep); + end + end + + function L = statementLinks(container, ids, kinds, inherited, context) + % STATEMENTLINKS - (internal) which statements hold of the subjects IDS + % + % L has fields about, doc, via (cell arrays, one entry per + % subject-statement pair). The subjects' groups (member_of, up) + % and wholes (part_of, sample_of, aliquot_of, passage_of, up) are + % walked together, one search per step; then one statement search + % per kind for every 200 subjects reached. A statement on a group + % holds when it is distributive; one on a whole when it is an + % assertion; on both, both. A subject's types (instance_of: its + % strain, its product) are walked too, from the subject or from + % a group whose instance_of is distributive, and an assertion + % about the type holds of each instance (T17); the walk stops at + % a type, which a session opened from its dataset finds among + % the dataset's documents. CONTEXT: then the containers of + % every subject reached (contextLinks). + if nargin < 5, context = false; end + described = arrayfun(@(k) isfield(k.filt, 'during_statements'), kinds); + if any(described) + L = struct('about', {{}}, 'doc', {{}}, 'via', {{}}); + if any(~described) + L = ndi.entity.statementLinks(container, ids, kinds(~described), inherited, context); + end + for k = find(described) + kk = kinds(k); + inner = kk.filt.during_statements; + kk.filt = rmfield(kk.filt, 'during_statements'); + Lk = ndi.entity.statementLinks(container, ids, kk, inherited, context); + tolerant = isfield(kk.filt, 'tolerant') && kk.filt.tolerant; + for i = 1:numel(inner), inner(i).filt.tolerant = tolerant; end + Lk = ndi.entity.duringLinks(container, Lk, inner, inherited, context, tolerant); + have = strcat(L.about, '|', cellfun(@docIdOf, L.doc, 'UniformOutput', false)); + for i = 1:numel(Lk.doc) + key = [Lk.about{i} '|' docIdOf(Lk.doc{i})]; + if any(strcmp(have, key)), continue; end + have{end+1} = key; %#ok + L.about{end+1} = Lk.about{i}; L.doc{end+1} = Lk.doc{i}; L.via{end+1} = Lk.via{i}; + end + end + return; + end + all_ = ndi.entity.inheritanceStates(container, ids, inherited); + reached = unique({all_.at}, 'stable'); + L = struct('about', {{}}, 'doc', {{}}, 'via', {{}}); + seen = containers.Map('KeyType', 'char', 'ValueType', 'logical'); + for k = 1:numel(kinds) + f = kinds(k).filt; + f.subject = reached; + f.withDataset = any([all_.i]); + docs = ndi.v2.searchStatements(container, kinds(k).kind, f); + for i = 1:numel(docs) + p = ndi.v2.props(docs{i}); + sid = ndi.v2.edgeIds(p, 'entity_id'); + if isempty(sid), continue; end + isAssertion = any(strcmp(ndi.v2.classChain(p), 'assertion')); + d = ndi.v2.blockOf(p, 'statement', 'distributive', false); + distributive = ~isempty(d) && (islogical(d) || isnumeric(d)) && logical(d(1)); + for j = find(strcmp({all_.at}, sid{1})) + st = all_(j); + if st.i + % a type's assertions hold of its instances; + % its interactions do not (T17) + if ~isAssertion, continue; end + elseif (st.m && ~distributive) || (st.l && ~isAssertion) + continue; + end + key = [st.start '|' char(p.base.id)]; + if isKey(seen, key), continue; end + seen(key) = true; + via = st.path; + if isempty(via), via = 'own'; end + L.about{end+1} = st.start; + L.doc{end+1} = docs{i}; + L.via{end+1} = via; + end + end + end + if context + % a type (a strain) is in no container + L = ndi.entity.contextLinks(container, all_(~[all_.i]), kinds, L, seen); + end + end + + function L = contextLinks(container, states, kinds, L, seen) + % CONTEXTLINKS - (internal) add to L the interactions of the + % containers the subjects of STATES were in, while they were in + % them (V_eta tenet T17). STATES: start (the subject asked for), + % at (a subject it reaches), m (reached through member_of), + % path. A stay reached through member_of must be distributive. + % Nested containers narrow the window to when both held. + cache = containers.Map('KeyType', 'char', 'ValueType', 'any'); + tolerant = false; + for k = 1:numel(kinds) + if isfield(kinds(k).filt, 'tolerant') && kinds(k).filt.tolerant, tolerant = true; end + end + frontier = struct('start', {states.start}, 'at', {states.at}, 'm', {states.m}, ... + 'path', {states.path}, 'win', {[]}); + C = struct('start', {}, 'at', {}, 'm', {}, 'path', {}, 'win', {}); + undecided = 0; + for depth = 1:4 + if isempty(frontier), break; end + S = ndi.v2.stays(container, unique({frontier.at}, 'stable'), 'child', cache); + next = struct('start', {}, 'at', {}, 'm', {}, 'path', {}, 'win', {}); + for i = 1:numel(S) + for f = find(strcmp({frontier.at}, S(i).child)) + fr = frontier(f); + if fr.m && ~S(i).distributive, continue; end + if ~S(i).decided, undecided = undecided + 1; continue; end + w = S(i); + if ~isempty(fr.win) + w.start = max(w.start, fr.win.start); w.end = min(w.end, fr.win.end); + if w.start > w.end, continue; end + end + p = 'contained_in'; + if ~isempty(fr.path), p = [fr.path ' > contained_in']; end + next(end+1) = struct('start', fr.start, 'at', S(i).parent, 'm', false, ... + 'path', p, 'win', w); %#ok + end + end + C = [C, next]; %#ok + frontier = next; + end + if undecided > 0 + warning('ndi:entity:context:undecided', ['%d stay(s) in a container had no ' ... + 'start or no end and were left out of the context.'], undecided); + end + if isempty(C), return; end + parents = unique({C.at}, 'stable'); + for k = 1:numel(kinds) + kind = kinds(k).kind; + if strcmp(kind, 'assertion'), continue; end % assertions never pass + if strcmp(kind, 'statement'), kind = 'interaction'; end + f = kinds(k).filt; + f.subject = parents; + docs = ndi.v2.searchStatements(container, kind, f); + refs = cell(1, numel(docs)); + for i = 1:numel(docs) + refs{i} = ndi.v2.edgeIds(ndi.v2.props(docs{i}), 'time_reference_id'); + end + all_ = unique([refs{:}]); + times = containers.Map('KeyType', 'char', 'ValueType', 'any'); + if ~isempty(all_), times = ndi.v2.timesOf(container, all_, cache); end + for i = 1:numel(docs) + p = ndi.v2.props(docs{i}); + sid = ndi.v2.edgeIds(p, 'entity_id'); + if isempty(sid), continue; end + t = cellfun(@(r) ifKeyAny(times, r), refs{i}, 'UniformOutput', false); + for j = find(strcmp({C.at}, sid{1})) + key = [C(j).start '|' char(p.base.id)]; + if isKey(seen, key), continue; end + if ~ndi.v2.overlaps(t, C(j).win, tolerant), continue; end + seen(key) = true; + L.about{end+1} = C(j).start; + L.doc{end+1} = docs{i}; + L.via{end+1} = C(j).path; + end + end + end + end + + function L = duringLinks(container, L, inner, inherited, context, tolerant) + % DURINGLINKS - (internal) keep the links of L whose statement's + % time overlaps that of a statement matching INNER (a kinds + % struct) that holds of the same subject -- by the same rules, + % INHERITED and CONTEXT. A statement or reading whose time + % cannot be compared does not match. + if isempty(L.doc), return; end + abouts = unique(L.about, 'stable'); + Li = ndi.entity.statementLinks(container, abouts, inner, inherited, context); + cache = containers.Map('KeyType', 'char', 'ValueType', 'any'); + refsOf = @(d) ndi.v2.edgeIds(ndi.v2.props(d), 'time_reference_id'); + ro = cellfun(refsOf, L.doc, 'UniformOutput', false); + ri = cellfun(refsOf, Li.doc, 'UniformOutput', false); + all_ = unique([ro{:}, ri{:}]); + times = containers.Map('KeyType', 'char', 'ValueType', 'any'); + if ~isempty(all_), times = ndi.v2.timesOf(container, all_, cache); end + % each subject's windows: the times of the statements matching INNER + wins = containers.Map('KeyType', 'char', 'ValueType', 'any'); + for i = 1:numel(Li.doc) + for r = 1:numel(ri{i}) + w = windowOf(ifKeyAny(times, ri{i}{r})); + if isempty(w), continue; end + if isKey(wins, Li.about{i}), wins(Li.about{i}) = [wins(Li.about{i}), w]; + else, wins(Li.about{i}) = w; end + end + end + keep = false(1, numel(L.doc)); + for i = 1:numel(L.doc) + if ~isKey(wins, L.about{i}), continue; end + t = cellfun(@(r) ifKeyAny(times, r), ro{i}, 'UniformOutput', false); + W = wins(L.about{i}); + for j = 1:numel(W) + if ndi.v2.overlaps(t, W(j), tolerant), keep(i) = true; break; end + end + end + L.about = L.about(keep); L.doc = L.doc(keep); L.via = L.via(keep); + end + + function ids = contextIds(container, docs, tolerant) + % CONTEXTIDS - (internal) the subjects the interactions DOCS were + % context for: what each one's subject contained while the + % statement held (V_eta tenet T17), down nested containers, and + % the members of a group in a distributive stay. Assertions + % never pass. + ids = {}; + keep = cellfun(@(d) ~any(strcmp(ndi.v2.classChain(ndi.v2.props(d)), 'assertion')), docs); + docs = docs(keep); + if isempty(docs), return; end + cache = containers.Map('KeyType', 'char', 'ValueType', 'any'); + refs = cell(1, numel(docs)); subj = cell(1, numel(docs)); + for i = 1:numel(docs) + p = ndi.v2.props(docs{i}); + refs{i} = ndi.v2.edgeIds(p, 'time_reference_id'); + s = ndi.v2.edgeIds(p, 'entity_id'); + if isempty(s), s = {''}; end + subj{i} = s{1}; + end + all_ = unique([refs{:}]); + times = containers.Map('KeyType', 'char', 'ValueType', 'any'); + if ~isempty(all_), times = ndi.v2.timesOf(container, all_, cache); end + T = cell(1, numel(docs)); + for i = 1:numel(docs) + T{i} = cellfun(@(r) ifKeyAny(times, r), refs{i}, 'UniformOutput', false); + end + frontier = struct('at', subj, 'doc', num2cell(1:numel(docs)), 'win', {[]}); + frontier = frontier(~cellfun(@isempty, subj)); + groups = {}; + for depth = 1:4 + if isempty(frontier), break; end + S = ndi.v2.stays(container, unique({frontier.at}, 'stable'), 'parent', cache); + next = struct('at', {}, 'doc', {}, 'win', {}); + for i = 1:numel(S) + if ~S(i).decided, continue; end + for f = find(strcmp({frontier.at}, S(i).parent)) + fr = frontier(f); + w = S(i); + if ~isempty(fr.win) + w.start = max(w.start, fr.win.start); w.end = min(w.end, fr.win.end); + if w.start > w.end, continue; end + end + if ~ndi.v2.overlaps(T{fr.doc}, w, tolerant), continue; end + ids{end+1} = S(i).child; %#ok + if S(i).distributive, groups{end+1} = S(i).child; end %#ok + next(end+1) = struct('at', S(i).child, 'doc', fr.doc, 'win', w); %#ok + end + end + frontier = next; + end + if ~isempty(groups) + ids = [ids, ndi.entity.walkIds(container, unique(groups, 'stable'), 'member_of', 'in')]; + end + ids = unique(ids, 'stable'); + end + end + +end + +function id = docIdOf(d) +% a document's base.id +p = ndi.v2.props(d); +id = char(p.base.id); +end + +function w = windowOf(t) +% a resolved time (ndi.v2.timeOf) as a window for ndi.v2.overlaps, or [] +% when it cannot be placed (no time, or only a bound on its anchor) +w = []; +if isempty(t) || isnat(t.start) || strcmp(t.resolved_by, 'relation'), return; end +e = t.end; +if isnat(e), e = t.start; end +tol = t.tolerance; etol = t.end_tolerance; +if isempty(tol) || all(isnan(tol)), tol = [0 0]; end +if isempty(etol) || all(isnan(etol)), etol = tol; end +if isscalar(tol), tol = [tol tol]; end +if isscalar(etol), etol = [etol etol]; end +tol(isnan(tol)) = 0; etol(isnan(etol)) = 0; +w = struct('start', t.start, 'end', e, 'tol', tol, 'end_tol', etol); +end + +function v = ifKeyAny(m, k) +% the value at key K of map M, or [] when absent +if isKey(m, k), v = m(k); else, v = []; end +end + +function t = describe(filt) +% 'variable = strain, value = N2' for messages +f = fieldnames(filt); +parts = cell(1, numel(f)); +for k = 1:numel(f) + v = filt.(f{k}); + if iscell(v) + v = strjoin(cellfun(@toText, v, 'UniformOutput', false), ' or '); + else + v = toText(v); + end + parts{k} = sprintf('%s = %s', f{k}, v); +end +t = strjoin(parts, ', '); +if isempty(t), t = 'anything'; end +end + +function t = toText(v) +if ischar(v), t = ['''' v '''']; elseif isnumeric(v) || islogical(v), t = mat2str(v); else, t = class(v); end +end + + +function ids = narrow(ids, these) +% the intersection so far ([] = not narrowed yet) +if isempty(ids) && ~iscell(ids) + ids = these; +else + ids = ids(ismember(ids, these)); +end +end + +function tf = isDistributive(d) +tf = ~isempty(d) && (islogical(d) || isnumeric(d)) && logical(d(1)); +end + +function t = relationText(r, indent, inherited) +% one condition line for a relation filter, with its target nested below +name = 'related to'; +if ~isempty(r.name), name = strjoin(r.name, ' or '); end +through = ''; +if inherited && (strcmp(r.side, 'parent') || isempty(r.side) || strcmp(r.side, 'with')) + through = ' (directly, or through a group they are members of)'; +end +target = r.target; +named = ''; +if isempty(target) && ~iscell(target) + named = 'anything'; +elseif isa(target, 'ndi.entity') + named = strjoin(arrayfun(@(x) ['"' char(x.name) '"'], target, 'UniformOutput', false), ' or '); +elseif ischar(target) || isstring(target) + named = ['the document ' char(target)]; +elseif iscell(target) && all(cellfun(@(x) isa(x, 'ndi.entity'), target)) + named = strjoin(cellfun(@(x) ['"' char(x.name) '"'], target, 'UniformOutput', false), ' or '); +end +switch r.side + case 'child' + lead = sprintf('are the %s parent of', name); + case 'parent' + lead = sprintf('are %s', name); + otherwise + if isempty(r.name), lead = 'are related to'; else, lead = sprintf('are in a %s relation with', name); end +end +when = timeText(r.time); +if ~isempty(named) + t = sprintf('%s %s%s%s', lead, named, when, through); +else + [~, singular] = nounOf(ndi.entity.parseSearchNested(target)); + inner = ndi.entity.explainNested(target, [indent ' ']); + if ~contains(inner, newline) + % no conditions of its own beyond what the noun says + t = sprintf('%s %s %s%s%s', lead, article(singular), singular, when, through); + else + inner = inner(find(inner == newline, 1):end); % the nested head reads "an organism that" + t = sprintf('%s %s %s%s%s that%s', lead, article(singular), singular, when, through, inner); + end +end +end + +function t = statementText(f, inherited) +% one condition line for a statement filter: "have strain N2", "had a +% manipulation with variable NGM agar and value >= 0.02" +kind = f.kind; +verb = 'had'; +if any(strcmp(kind, {'assertion', 'statement'})), verb = 'have'; end +fl = f.filt; +names = intersect({'variable', 'method', 'value', 'formulation'}, fieldnames(fl), 'stable'); +when = timeText(fl); +if isfield(fl, 'during_statements') + inner = arrayfun(@(k) statementText(k, false), fl.during_statements, 'UniformOutput', false); + when = [when ', at a time when they also ' strjoin(inner, ' or ')]; +end +if strcmp(kind, 'assertion') && isequal(sort(names), sort({'value', 'variable'})) && ischar(fl.variable) + t = sprintf('have %s %s%s', fl.variable, valueText(fl.value), when); +else + parts = cellfun(@(n) sprintf('%s %s', n, valueText(fl.(n))), names, 'UniformOutput', false); + t = sprintf('%s %s %s', verb, article(kind), kind); + if ~isempty(parts) + if numel(parts) > 1 + parts = [strjoin(parts(1:end-1), ', ') ' and ' parts{end}]; + else + parts = parts{1}; + end + t = sprintf('%s with %s', t, parts); + end + t = [t when]; +end +if inherited + if strcmp(kind, 'assertion') + t = [t ' (stated on them, on a group they are members of, or on what they are part of)']; + else + t = [t ' (on them, or on a group they are members of)']; + end +end +end + +function t = timeText(f) +% ', at 2023-11-16T14:00, lasting >=3h' for the time filters in F ('' when none) +t = ''; +if ~isstruct(f), return; end +words = struct('at', 'at', 'during', 'during', 'before', 'before', 'after', 'after', 'duration', 'lasting'); +for n = {'at', 'during', 'before', 'after', 'duration'} + if ~isfield(f, n{1}) || isempty(f.(n{1})), continue; end + v = f.(n{1}); + if isa(v, 'datetime'), v = arrayfun(@char, v, 'UniformOutput', false); end + if strcmp(n{1}, 'during') && iscell(v) && numel(v) == 2 + txt = sprintf('%s to %s', valueText(v{1}), valueText(v{2})); + else + txt = valueText(v); + end + t = sprintf('%s, %s %s', t, words.(n{1}), txt); +end +end + +function m = noTimeMessage(kind, filt, tinfo, where) +% the warning when statements or relations matched but none at that time +m = sprintf('No %s in this %s matches %s at the time asked.', kind, where, describe(filt)); +why = {}; +if tinfo.unresolved > 0, why{end+1} = sprintf('%d time(s) could not be resolved here', tinfo.unresolved); end +if tinfo.nozone > 0, why{end+1} = sprintf(['%d time(s) have no recorded time zone to read the times ' ... + 'you typed in (give a zone, e.g. ''2023-11-16T14:00-08:00'')'], tinfo.nozone); end +if tinfo.bound > 0, why{end+1} = sprintf(['%d time(s) are known only as before or after another ' ... + 'event, which cannot decide this'], tinfo.bound); end +if tinfo.docs > 0, why{end+1} = sprintf('%d have no time at all', tinfo.docs); end +if ~isempty(why), m = sprintf('%s\nNot compared: %s.', m, strjoin(why, '; ')); end +end + +function t = valueText(v) +% a pattern or a list of them, as typed: N2 or CB* +if iscell(v) + t = strjoin(cellfun(@valueText, v, 'UniformOutput', false), ' or '); +elseif ischar(v) || isstring(v) + t = char(v); +elseif isnumeric(v) || islogical(v) + t = mat2str(v); +else + t = class(v); +end +end + +function q = quoted(pats) +q = strjoin(cellfun(@(x) ['''' char(x) ''''], cellstr(pats), 'UniformOutput', false), ' or '); +end + +function m = didYouMean(pats, candidates) +% ' Did you mean ''strain''?' for the first pattern with a near candidate +m = ''; +for k = 1:numel(pats) + if ~ischar(pats{k}) || ndi.v2.hasWildcard(pats{k}), continue; end + c = ndi.v2.closest(pats{k}, candidates); + if ~isempty(c) + m = sprintf(' Did you mean ''%s''?', c); + return; + end +end +end + +function w = containerWord(container) +if isa(container, 'ndi.dataset'), w = 'dataset'; else, w = 'session'; end +end + +function m = noValueMessage(f, values, where, noun) +% the warning when a variable is there and none of the values is +vals = f.filt.value; +if ~iscell(vals), vals = {vals}; end +shown = values(1:min(end, 30)); +more = ''; +if numel(values) > 30, more = sprintf(' (and %d more)', numel(values) - 30); end +pats = vals(cellfun(@ischar, vals)); +if strcmp(f.kind, 'assertion') && isfield(f.filt, 'variable') && ischar(f.filt.variable) + var = f.filt.variable; + m = sprintf('No %s in this %s has %s %s.', noun, where, var, quoted(pats)); + label = [upper(var(1)) var(2:end)]; +else + m = sprintf('No %s in this %s matches %s.', f.kind, where, describe(f.filt)); + label = 'Values'; +end +m = [m didYouMean(pats, values)]; +m = sprintf('%s\n%s in this %s: %s%s', m, label, where, strjoin(shown, ', '), more); +end + +function [plural, singular, named] = nounOf(spec) +% what a search is for, in words, from a single 'kind' or 'type' condition: +% 'organisms', 'people', or for several types the list of them +% ('organisms, cultures, tissues, cells or groups'); else 'entities'. +% NAMED: the spec.own row the noun says (0 when none) +plural = 'entities'; singular = 'entity'; named = 0; +for key = {'kind', 'type'} + k = find(strcmp(spec.own(:, 1), key{1})); + if ~isscalar(k), continue; end + v = spec.own{k, 2}; + if isstring(v), v = cellstr(v); end + if ischar(v), v = {v}; end + if ~iscell(v) || isempty(v) || ~all(cellfun(@ischar, v)) || any(cellfun(@ndi.v2.hasWildcard, v)) + continue; + end + words = lower(reshape(v, 1, [])); + plural = listOf(cellfun(@pluralOf, words, 'UniformOutput', false)); + singular = listOf(words); + named = k; + return; +end +end + +function t = listOf(w) +% 'a', 'a or b', 'a, b or c' +if isscalar(w), t = w{1}; +else, t = [strjoin(w(1:end-1), ', ') ' or ' w{end}]; +end +end + +function p = pluralOf(w) +% an English plural, for the nouns entity kinds and types are +if strcmp(w, 'person'), p = 'people'; +elseif strcmp(w, 'software'), p = 'software'; +elseif endsWith(w, 'y') && ~any(w(end-1) == 'aeiou'), p = [w(1:end-1) 'ies']; +elseif endsWith(w, {'s', 'sh', 'ch', 'x'}), p = [w 'es']; +else, p = [w 's']; +end +end + +function a = article(word) +if any(lower(word(1)) == 'aeiou'), a = 'an'; else, a = 'a'; end +end + +function tf = entityMatches(p, type, patterns) +% does entity P of TYPE match any of PATTERNS by name or global identifier +tf = ndi.v2.matchTerm(char(ndi.v2.blockOf(p, type, 'name', '')), patterns); +ids = cellstr(ndi.v2.blockOf(p, type, 'global_identifier', {})); +for k = 1:numel(ids) + if tf, return; end + tf = ndi.v2.matchTerm(struct('name', '', 'node', ids{k}), patterns); +end +end + +function m = typeDocuments(container, ids) +% the properties of the documents IDS (a strain, a product), by id: the +% container's own, and for a session opened from a dataset the dataset's +m = containers.Map('KeyType', 'char', 'ValueType', 'any'); +qs = cellfun(@(i) ndi.query('base.id', 'exact_string', i, ''), ids, 'UniformOutput', false); +q = ndi.v2.anyOf(qs); +docs = container.database_search(q); +if numel(docs) < numel(ids) && isa(container, 'ndi.session') + docs = container.database_search_with_dataset(q); +end +for i = 1:numel(docs) + p = ndi.v2.props(docs{i}); + m(char(p.base.id)) = p; +end +end + +function c = namedArgs(options) +% an options struct as NAME, VALUE pairs, to pass on +n = fieldnames(options); +c = cell(1, 2 * numel(n)); +c(1:2:end) = n; +c(2:2:end) = struct2cell(options); +end + +function tf = entitySynonymMatches(p, type, patterns) +% does a name one of the entity's global identifiers goes by (a CURIE's +% label or synonym, ndi.v2.synonyms) match PATTERNS? +tf = false; +ids = cellstr(ndi.v2.blockOf(p, type, 'global_identifier', {})); +for k = 1:numel(ids) + names = ndi.v2.synonyms(ids{k}); + for j = 1:numel(names) + if ndi.v2.matchTerm(struct('name', names{j}, 'node', ''), patterns) + tf = true; + return; + end + end +end +end + +function docs = entitiesOfType(container, type) +% the entities of TYPE (merged `entity` documents only) CONTAINER can reach: +% its own, and for a session opened from a dataset the dataset's too, since +% a strain or product entity is written once, at dataset level +q = ndi.v2.isaQuery(type) & ndi.query('', 'isa', 'entity', ''); +docs = container.database_search(q); +if isempty(docs) && isa(container, 'ndi.session') + docs = container.database_search_with_dataset(q); +end +end + +function n = kindOfSpec(spec) +% the noun a search's messages use: its kind ('subject', 'person', ...) +n = 'entity'; +if isfield(spec, 'kind') && ~isempty(spec.kind), n = char(spec.kind); end +end diff --git a/src/ndi/+ndi/interaction.m b/src/ndi/+ndi/interaction.m new file mode 100644 index 0000000000..2c124f4308 --- /dev/null +++ b/src/ndi/+ndi/interaction.m @@ -0,0 +1,117 @@ +classdef (Abstract) interaction < ndi.statement + % ndi.interaction - a statement about something done to, with or about a subject + % + % The shared parent of ndi.observation, ndi.manipulation and + % ndi.calculation: each happened at a time, by a method, possibly with an + % instrument. It is abstract: ndi.statement.fromDocument always returns + % one of the three. + % + % ndi.interaction Methods (beyond ndi.statement): + % time - when: a struct array, one per time reference + % method - the method's term ({node, name}); method_name, its name + % method_parameters - table of the method's parameters + % instrument - the instrument subject(s) (a cell array of ndi.entity) + % software - the software entity (a cell array of ndi.entity) + % + % See also ndi.statement, ndi.v2.timeOf. + + methods + function obj = interaction(container, doc) + arguments + container = [] + doc = [] + end + obj = obj@ndi.statement(container, doc); + end + + function t = time(obj) + % TIME - when the statement holds, one struct per time reference + % + % T = TIME(OBJ) is a struct array (see ndi.v2.timeOf): kind, + % start, end (datetime, UTC, NaT when it cannot be resolved), + % offsets, relation, clock, referent_id, approximate, tolerance, + % resolved_by. Empty when the statement has no time reference. + t = []; + ids = ndi.v2.edgeIds(obj.document_properties(), 'time_reference_id'); + for i = 1:numel(ids) + d = ndi.v2.getDocument(obj.container_, ids{i}); + if isempty(d), continue; end + r = ndi.v2.timeOf(obj.container_, d); + if isempty(t), t = r; else, t(end+1) = r; end %#ok + end + end + + function m = method(obj) + % METHOD - the method's term ({node, name}); [] when none + m = ndi.v2.blockOf(obj.document_properties(), 'interaction', 'method', []); + end + + function n = method_name(obj) + % METHOD_NAME - the method's name ('' when none) + n = ndi.v2.termName(obj.method()); + end + + function T = method_parameters(obj) + % METHOD_PARAMETERS - table: variable, value, unit, source_unit + % + % One row per parameter stored on the statement. VALUE is the + % number in `unit` (the canonical unit), a term's name, or text; + % SOURCE_VALUE is the number as the source wrote it, in + % SOURCE_UNIT ('' for a term or text). + m = ndi.v2.blockOf(obj.document_properties(), 'interaction', 'method_parameters', []); + m = ndi.v2.entries(m); + variable = strings(0, 1); value = cell(0, 1); unit = strings(0, 1); + source_value = strings(0, 1); source_unit = strings(0, 1); + for i = 1:numel(m) + x = m{i}; + variable(end+1, 1) = string(ndi.v2.termName(fieldOf(x, 'variable'))); %#ok + unit(end+1, 1) = string(ndi.v2.termName(fieldOf(x, 'unit'))); %#ok + source_unit(end+1, 1) = string(char(fieldOf(x, 'source_unit'))); %#ok + sv = ""; + if ~isempty(fieldOf(x, 'term')) + value{end+1, 1} = ndi.v2.termName(fieldOf(x, 'term')); %#ok + elseif ~isempty(fieldOf(x, 'text')) + value{end+1, 1} = char(fieldOf(x, 'text')); %#ok + else + % a number: {value, source_value} + q = fieldOf(x, 'value'); + n = fieldOf(q, 'value'); + if isempty(n) && isnumeric(q), n = q; end + value{end+1, 1} = double(n); %#ok + sv = string(fieldOf(q, 'source_value')); + if isempty(sv) || ismissing(sv), sv = ""; end + end + source_value(end+1, 1) = sv; %#ok + end + T = table(variable, value, unit, source_value, source_unit); + end + + function s = instrument(obj) + % INSTRUMENT - the instrument subject(s), a cell array of ndi.entity + s = obj.entitiesAt('instrument_id'); + end + + function s = software(obj) + % SOFTWARE - the software entity(s), a cell array of ndi.entity + s = obj.entitiesAt('software_id'); + end + end + + methods (Access = protected) + function e = entitiesAt(obj, edge) + e = {}; + ids = ndi.v2.edgeIds(obj.document_properties(), edge); + for i = 1:numel(ids) + d = ndi.v2.getDocument(obj.container_, ids{i}); + if ~isempty(d) + e{end+1} = ndi.entity.fromDocument(obj.container_, d); %#ok + end + end + end + end +end + +function v = fieldOf(s, name) +v = []; +if isstruct(s) && isfield(s, name), v = s.(name); end +end diff --git a/src/ndi/+ndi/manipulation.m b/src/ndi/+ndi/manipulation.m new file mode 100644 index 0000000000..d337203b6f --- /dev/null +++ b/src/ndi/+ndi/manipulation.m @@ -0,0 +1,34 @@ +classdef manipulation < ndi.interaction + % ndi.manipulation - a statement of what was done to a subject + % + % ndi.manipulation Methods (beyond ndi.interaction): + % formulation - for a dose: what was given (an ndi.value of class + % 'formulation'; [] when the statement names none) + % + % See also ndi.interaction, ndi.statement. + + methods + function obj = manipulation(container, doc) + % MANIPULATION - use ndi.statement.fromDocument + arguments + container = [] + doc = [] + end + obj = obj@ndi.interaction(container, doc); + end + + function f = formulation(obj) + % FORMULATION - what a dose gave: an ndi.value ([] when none) + % + % A formulation is a data_type, not an entity: its value holds + % the ingredients (and their own formulations) as stored. + f = []; + ids = ndi.v2.edgeIds(obj.document_properties(), 'formulation_id'); + if isempty(ids), return; end + d = ndi.v2.getDocument(obj.container_, ids{1}); + if isempty(d), return; end + p = ndi.v2.props(d); + f = ndi.value('formulation', ndi.v2.blockOf(p, 'formulation', 'value', [])); + end + end +end diff --git a/src/ndi/+ndi/observation.m b/src/ndi/+ndi/observation.m new file mode 100644 index 0000000000..f817a175f2 --- /dev/null +++ b/src/ndi/+ndi/observation.m @@ -0,0 +1,18 @@ +classdef observation < ndi.interaction + % ndi.observation - a statement of what was observed or measured of a subject + % + % Everything it does comes from ndi.interaction and ndi.statement. + % + % See also ndi.interaction, ndi.statement. + + methods + function obj = observation(container, doc) + % OBSERVATION - use ndi.statement.fromDocument + arguments + container = [] + doc = [] + end + obj = obj@ndi.interaction(container, doc); + end + end +end diff --git a/src/ndi/+ndi/query.m b/src/ndi/+ndi/query.m index 20f6a86b88..5924e8cec3 100644 --- a/src/ndi/+ndi/query.m +++ b/src/ndi/+ndi/query.m @@ -85,7 +85,7 @@ 'exact_number', 'lessthan', 'lessthaneq', ... 'greaterthan', 'greaterthaneq', 'hasfield', ... 'hasanysubfield_contains_string', 'isa', 'depends_on', ... - 'hasmember','exact_string_anycase', ... + 'hasmember','exact_string_anycase','wildcard', ... 'or', ... % Keep 'or' (cannot be negated) '~regexp', '~exact_string', '~contains_string', ... '~exact_number', '~lessthan', '~lessthaneq', ... @@ -127,10 +127,37 @@ % issue #781 and ndi.compat.translateQueryPaths. ndi_query_obj.searchstructure = ndi.compat.translateQueryPaths( ... ndi_query_obj.searchstructure); + + % `isa subject` is the biological subjects (ndi.v2.subjectTypes): + % a v1 subject document always was one, while V_eta also makes + % probes, cameras (device) and plates (material) subjects, or + % (one entity class, 2026-10-08) entities with a type. So a + % `subject` of another type is left out, and an `entity` of a + % biological type is found; a subject with no type (v1) stays. + if nargin > 2 && strcmp(op, 'isa') && (ischar(param1) || isstring(param1)) ... + && strcmp(char(param1), 'subject') + ndi_query_obj.searchstructure = ndi.query.subjectSearch(); + end end % query() constructor end % methods methods (Static) + function ss = subjectSearch() + % SUBJECTSEARCH - the search structure of ndi.query('', 'isa', 'subject') + others = setdiff(ndi.v2.entityTypesFor('subject'), ndi.v2.subjectTypes(), 'stable'); + q = did.query('', 'isa', 'subject'); + for k = 1:numel(others) + q = q & did.query('subject.type.name', '~exact_string', others{k}); + end + types = ndi.v2.subjectTypes(); + t = did.query('entity.type.name', 'exact_string', types{1}); + for k = 2:numel(types) + t = t | did.query('entity.type.name', 'exact_string', types{k}); + end + q = q | (did.query('', 'isa', 'entity') & t); + ss = q.searchstructure; + end + function q = all() % ALL - return a query that matches all documents % diff --git a/src/ndi/+ndi/session.m b/src/ndi/+ndi/session.m index f8ffe988e8..bca4291816 100755 --- a/src/ndi/+ndi/session.m +++ b/src/ndi/+ndi/session.m @@ -13,6 +13,13 @@ database % An ndi.database associated with this session autoclose_listeners % A map of listeners for auto-closing binary documents end + properties (GetAccess={?session, ?ndi.dataset}, SetAccess={?session, ?ndi.dataset}, Transient) + % datasetDatabase - for a LINKED session opened through an ndi.dataset, + % the dataset's own database ([] otherwise), so that + % database_search_with_dataset reaches the dataset-level documents + % (a strain, a study, a person) a session in its own folder does not hold + datasetDatabase = [] + end methods function ndi_session_obj = session(reference) @@ -176,12 +183,21 @@ q2 = ndi.query('base.session_id','exact_string',ndi_session_obj.id(),''); q = q1 & q2; if numel(varargin)>0 + rest = {}; for i=1:2:numel(varargin) if strcmpi(varargin{i},'name') - varargin{i} = 'base.name'; % case matters here + % A name lives in base.name (v1, and V_eta as the + % migrators write it) or in acquisition_system.name + % (did-schema #73 item 54), so ask for either. + q = q & ( ndi.query('base.name','exact_string',varargin{i+1},'') | ... + ndi.query('acquisition_system.name','exact_string',varargin{i+1},'') ); + else + rest(end+1:end+2) = varargin(i:i+1); end end - q = q & ndi.query(varargin); + if ~isempty(rest) + q = q & ndi.query(rest); + end end dev_doc = ndi_session_obj.database_search(q); % dev is cell list of ndi.document objects @@ -384,6 +400,36 @@ ndi_document_obj = ndi_session_obj.database.search(searchparameters&inSession); end % database_search(); + function ndi_document_obj = database_search_with_dataset(ndi_session_obj, searchparameters) + % DATABASE_SEARCH_WITH_DATASET - search this session's database, dataset documents included + % + % NDI_DOCUMENT_OBJ = DATABASE_SEARCH_WITH_DATASET(NDI_SESSION_OBJ, SEARCHPARAMETERS) + % + % Like DATABASE_SEARCH, but without restricting the answer to this + % session's own documents. A session opened from an ndi.dataset + % lives in the dataset's database, so this also finds the + % dataset-level documents (a strain, a person, a study). So does a + % LINKED session opened through its dataset (ndi.dataset/open_session): + % its own folder's database, then the dataset's. A session opened + % from its own folder on its own holds only its own documents, so + % the answer is the same as DATABASE_SEARCH's. + % + % See also: ndi.session/database_search, ndi.dataset/open_session + arguments + ndi_session_obj (1,1) ndi.session + searchparameters {mustBeA(searchparameters,{'ndi.query','did.query'})} + end + ndi_document_obj = ndi_session_obj.database.search(searchparameters); + if ~isempty(ndi_session_obj.datasetDatabase) + % a linked session opened through its dataset: the dataset's + % own documents too, each once + more = ndi_session_obj.datasetDatabase.search(searchparameters); + have = cellfun(@(d) d.id(), ndi_document_obj, 'UniformOutput', false); + keep = ~ismember(cellfun(@(d) d.id(), more, 'UniformOutput', false), have); + ndi_document_obj = [reshape(ndi_document_obj, 1, []), reshape(more(keep), 1, [])]; + end + end % database_search_with_dataset() + function database_clear(ndi_session_obj, areyousure) % DATABASE_CLEAR - deletes/removes all entries from the database associated with an session % diff --git a/src/ndi/+ndi/statement.m b/src/ndi/+ndi/statement.m new file mode 100644 index 0000000000..556cfbf07a --- /dev/null +++ b/src/ndi/+ndi/statement.m @@ -0,0 +1,509 @@ +classdef statement + % ndi.statement - something stated about a subject + % + % A V2 dataset is a set of statements about subjects: a worm's speed + % (calculated), a plate's temperature (manipulated), a recording + % (observed), a patch's strain (asserted). An ndi.statement is a read-only + % view of one statement document, with the session or dataset it was + % read from. It comes back as the right child: + % + % ndi.statement + % |-- ndi.assertion (species, strain, exclusion: no time or method) + % `-- ndi.interaction (time, method, instrument) + % |-- ndi.observation + % |-- ndi.manipulation (formulation, for a dose) + % `-- ndi.calculation (inputs, software) + % + % READ THROUGH THE DATASET. A statement's software, formulation, + % strain and instruments are stored with the dataset, and an + % ndi.session searches only its own documents, so a statement read + % through a session cannot reach them (software() and formulation() + % come back empty). Read through the ndi.dataset, which reaches every + % document; find things in a session first if that is quicker, then + % re-read them through the dataset with fromDocument: + % + % ds = ndi.dataset.dir(datasetPath); + % w = ndi.entity.search(ds, {'LocalIdentifier', 'concentration_worm0012'}); + % st = w{1}.statements('Variable', 'midpoint speed'); + % v = st{1}.value(); % an ndi.value + % speed = double(v); % metres per second, by video frame + % v.axes() % what each dimension indexes + % + % ndi.statement Methods: + % kind, document, document_properties, document_id + % subject - the ndi.entity the statement is about + % variable - the term; variable_name, its name + % value - an ndi.value (decoded from its body when stored there) + % raw_value - the inline value exactly as stored + % composite - the value's composite class, e.g. 'velocity' + % axes - table of the value's keys + % conditions - table of the conditions it holds under + % notes - text + % bodies - the data-body documents holding the value + % about, via - the subject it was asked for, and how it holds of it + % summary - a table, one row per statement (ndi.summary for a cell) + % fromDocument, search - (static) make statements + % + % See also ndi.entity, ndi.value, ndi.interaction. + + properties (SetAccess = protected, GetAccess = public, Hidden) + statement_document_ = [] % the ndi.document this statement reads + container_ = [] % the ndi.session or ndi.dataset it was read from + about_ = '' % the subject it was asked for (ndi.entity/statements) + via_ = '' % how it holds of that subject ('own', 'member_of', ...) + end + + methods + function obj = statement(container, doc) + % STATEMENT - a statement for DOC, read from CONTAINER + % + % Use ndi.statement.fromDocument, which returns the right child. + arguments + container = [] + doc = [] + end + if nargin == 0 + return; + end + obj.container_ = container; + obj.statement_document_ = doc; + end + + function id = about(obj) + % ABOUT - the document id of the subject this statement was asked for + % + % A statement read through ndi.entity/statements is about the + % subject it was asked for, which may not be its own subject (a + % cohort's strain, read for one of its worms). '' when the + % statement was not read for a subject; its own subject's id is + % then what it is about (see VIA). + id = obj.about_; + if isempty(id) + ids = ndi.v2.edgeIds(obj.document_properties(), 'entity_id'); + if ~isempty(ids), id = ids{1}; end + end + end + + function v = via(obj) + % VIA - how the statement holds of the subject it was asked for + % + % 'own' (stated about it), 'member_of' (stated about a group it + % belongs to, distributively), 'part_of', 'sample_of', ... (an + % assertion about a whole it is part or a sample of), or a path + % such as 'member_of > part_of'. + v = obj.via_; + if isempty(v), v = 'own'; end + end + + function T = summary(obj) + % SUMMARY - one table row per statement + % + % T = SUMMARY(S) for a statement or an array of one class; for a + % cell array of statements of any kind use ndi.summary. Columns: + % subject the subject it is about (see ABOUT) + % kind assertion, observation, manipulation, calculation + % class the document class, e.g. 'temperature_manipulation' + % variable, variable_node, method, method_node, value, + % value_node each term's name and its ontology node ('' when it + % has none yet, or the value is not a term); a value + % of several numbers is '(array)', one kept in a data + % body '(data body)' + % unit + % start, end its time, the first of several references, shown + % in the zone it was recorded in when every row + % shares one (else UTC); NaT when it has none or it + % cannot be resolved + % timezone the zone each time was recorded in + % stated_on the subject it was stated about + % via how it holds of SUBJECT (see VIA) + % id the statement document's id + T = ndi.statement.summaryOf(num2cell(obj)); + end + + function obj = withContext(obj, aboutId, via) + % WITHCONTEXT - (internal) the same statement, read for subject ABOUTID by VIA + arguments + obj + aboutId char + via char + end + obj.about_ = char(aboutId); + obj.via_ = char(via); + end + + function d = document(obj) + % DOCUMENT - the statement's ndi.document + d = obj.statement_document_; + end + + function p = document_properties(obj) + % DOCUMENT_PROPERTIES - the statement document's properties (a struct) + p = ndi.v2.props(obj.statement_document_); + end + + function id = document_id(obj) + % DOCUMENT_ID - the statement document's base.id + p = obj.document_properties(); + id = char(p.base.id); + end + + function k = kind(obj) + % KIND - the statement's document class, e.g. 'velocity_calculation' + p = obj.document_properties(); + k = char(p.document_class.class_name); + end + + function s = subject(obj) + % SUBJECT - the entity this statement is about ([] when not found): + % an ndi.entity (a statement may + % be about any entity since 2026-10-08). The same as ENTITY. + s = obj.entity(); + end + + function s = entity(obj) + % ENTITY - the entity this statement is about ([] when not found) + s = []; + ids = ndi.v2.edgeIds(obj.document_properties(), 'entity_id'); + if isempty(ids), return; end + d = ndi.v2.getDocument(obj.container_, ids{1}); + if ~isempty(d) + s = ndi.entity.fromDocument(obj.container_, d); + end + end + + function t = variable(obj) + % VARIABLE - the statement's variable, a term ({node, name}) + t = ndi.v2.blockOf(obj.document_properties(), 'statement', 'variable', []); + end + + function n = variable_name(obj) + % VARIABLE_NAME - the variable's name (its node when it has no name) + n = ndi.v2.termName(obj.variable()); + end + + function c = composite(obj) + % COMPOSITE - the class that holds the value, e.g. 'velocity', 'term' + % + % The class in the statement's chain whose own superclass is + % data_type ('' when there is none). + c = ''; + p = obj.document_properties(); + chain = ndi.v2.classChain(p); + for i = 1:numel(chain) + if any(strcmp(ndi.v2.directParents(chain{i}), 'value')) + c = chain{i}; + return; + end + end + end + + function v = raw_value(obj) + % RAW_VALUE - the inline value exactly as stored ([] when in a body) + v = []; + c = obj.composite(); + if ~isempty(c) + v = ndi.v2.blockOf(obj.document_properties(), c, 'value', []); + end + end + + function v = value(obj) + % VALUE - the statement's value, an ndi.value + % + % Inline, or read from the statement's data body: a sampled body + % is decoded into an array (its datum type, byte order and keys); + % an opaque body (a video, an image file) gives its file paths. + p = obj.document_properties(); + c = obj.composite(); + % value.data_type since 2026-10-08, value.datum_type before + dt = char(ndi.v2.blockOf(p, 'value', 'data_type', ndi.v2.blockOf(p, 'value', 'datum_type', ''))); + ids = ndi.v2.edgeIds(p, 'value_id'); + if isempty(c) && ~isempty(ids) + % the value is a document of its own (a shared item, a model fit) + d = ndi.v2.getDocument(obj.container_, ids{1}); + if isempty(d) + error('ndi:statement:noValue', 'Statement %s names value %s, which is not in this %s.', ... + obj.document_id(), ids{1}, class(obj.container_)); + end + q = ndi.v2.props(d); + k = char(q.document_class.class_name); + v = ndi.value(k, ndi.v2.blockOf(q, k, 'value', []), ... + 'Keys', ndi.v2.blockOf(q, 'data', 'keys', [])); + return; + end + if ~logical(firstOr(ndi.v2.blockOf(p, 'value', 'data_body', false), false)) + v = ndi.value(c, obj.raw_value(), 'Keys', ndi.v2.blockOf(p, 'data', 'keys', []), ... + 'DatumType', dt); + return; + end + b = obj.bodies(); + if isempty(b) + error('ndi:statement:noBody', ['Statement %s keeps its value in a data body, ' ... + 'and no body pointing at it was found in this %s.'], obj.document_id(), ... + class(obj.container_)); + end + bp = ndi.v2.props(b{1}); + % the keys are the statement's; a body that states its own is the fallback + keys = ndi.v2.blockOf(p, 'data', 'keys', []); + if isempty(keys), keys = ndi.v2.blockOf(bp, 'data', 'keys', []); end + if any(strcmp(ndi.v2.classChain(bp), 'sampled_body')) + vals = ndi.v2.readBody(obj.container_, b{1}, dt, keys); + v = ndi.value(c, obj.raw_value(), 'Data', vals, 'Keys', keys, 'DatumType', dt); + else + files = {}; + for i = 1:numel(b) + q = ndi.v2.props(b{i}); + fi = []; + if isfield(q, 'files') && isfield(q.files, 'file_info'), fi = q.files.file_info; end + for j = 1:numel(fi) + [tf, f] = obj.container_.database_existbinarydoc(char(q.base.id), char(fi(j).name)); + if tf, files{end+1} = f; end %#ok + end + end + v = ndi.value(c, obj.raw_value(), 'Files', files, 'Keys', keys, 'DatumType', dt); + end + end + + function b = bodies(obj) + % BODIES - the data-body documents holding this statement's value (a cell array) + b = obj.container_.database_search(ndi.query('', 'isa', 'data_body', '') & ... + ndi.query('', 'depends_on', 'owner_id', obj.document_id())); + end + + function T = axes(obj) + % AXES - table of the value's keys: variable, unit, n, coordinates + T = ndi.v2.axesTable(ndi.v2.blockOf(obj.document_properties(), 'data', 'keys', [])); + end + + function T = conditions(obj) + % CONDITIONS - table of the conditions the statement holds under + % + % One row per condition: variable, value (a number in `unit`, a + % count, or a term's name), unit, source_value, source_unit. + c = ndi.v2.blockOf(obj.document_properties(), 'statement', 'conditions', []); + c = ndi.v2.entries(c); + variable = strings(0, 1); value = cell(0, 1); unit = strings(0, 1); + source_value = cell(0, 1); source_unit = strings(0, 1); + for i = 1:numel(c) + x = c{i}; + variable(end+1, 1) = string(ndi.v2.termName(fieldOr(x, 'variable', ''))); %#ok + unit(end+1, 1) = string(ndi.v2.termName(fieldOr(x, 'unit', ''))); %#ok + source_unit(end+1, 1) = string(fieldOr(x, 'source_unit', '')); %#ok + if ~isempty(fieldOr(x, 'term', [])) + value{end+1, 1} = ndi.v2.termName(fieldOr(fieldOr(x, 'term', []), 'value', fieldOr(x, 'term', []))); %#ok + source_value{end+1, 1} = []; %#ok + elseif ~isempty(fieldOr(x, 'count', [])) + value{end+1, 1} = double(fieldOr(fieldOr(x, 'count', []), 'value', NaN)); %#ok + source_value{end+1, 1} = []; %#ok + else + % a quantity: {value: {value, source_value}} + q = fieldOr(fieldOr(x, 'quantity', []), 'value', []); + value{end+1, 1} = double(fieldOr(q, 'value', NaN)); %#ok + source_value{end+1, 1} = fieldOr(q, 'source_value', []); %#ok + end + end + T = table(variable, value, unit, source_value, source_unit); + end + + function tf = distributive(obj) + % DISTRIBUTIVE - true when the statement, made about a group, + % holds of each of the group's members (subject_statement.distributive) + d = ndi.v2.blockOf(obj.document_properties(), 'statement', 'distributive', false); + tf = ~isempty(d) && logical(d(1)); + end + + function n = notes(obj) + % NOTES - the statement's notes ('' when none) + n = char(ndi.v2.blockOf(obj.document_properties(), 'interaction', 'notes', '')); + end + end + + methods (Static) + function T = summaryOf(statements) + % SUMMARYOF - the summary table of a cell array of statements (see SUMMARY) + arguments + statements cell + end + n = numel(statements); + subject = strings(n, 1); kind = strings(n, 1); class = strings(n, 1); + variable = strings(n, 1); method = strings(n, 1); value = strings(n, 1); + variable_node = strings(n, 1); method_node = strings(n, 1); value_node = strings(n, 1); + unit = strings(n, 1); stated_on = strings(n, 1); via = strings(n, 1); id = strings(n, 1); + timezone = strings(n, 1); + start = NaT(n, 1, 'TimeZone', 'UTC'); stop = NaT(n, 1, 'TimeZone', 'UTC'); + columns = {'subject', 'kind', 'class', 'variable', 'variable_node', 'method', 'method_node', ... + 'value', 'value_node', 'unit', 'start', 'end', 'timezone', 'stated_on', 'via', 'id'}; + if n == 0 + T = table(subject, kind, class, variable, variable_node, method, method_node, value, ... + value_node, unit, start, stop, timezone, stated_on, via, id, 'VariableNames', columns); + return; + end + container = statements{1}.container_; + cache = containers.Map('KeyType', 'char', 'ValueType', 'any'); + refs = cell(n, 1); own = cell(n, 1); + for i = 1:n + st = statements{i}; + p = st.document_properties(); + chain = ndi.v2.classChain(p); + kinds = {'assertion', 'observation', 'manipulation', 'calculation'}; + k = find(ismember(kinds, chain), 1); + if ~isempty(k), kind(i) = string(kinds{k}); else, kind(i) = "statement"; end + class(i) = string(p.document_class.class_name); + vr = ndi.v2.blockOf(p, 'statement', 'variable', ''); + variable(i) = string(ndi.v2.termName(vr)); variable_node(i) = string(nodeOf(vr)); + mr = ndi.v2.blockOf(p, 'interaction', 'method', ''); + method(i) = string(ndi.v2.termName(mr)); method_node(i) = string(nodeOf(mr)); + [vk, vv, vt, vu] = ndi.v2.statementValue(p); + if strcmp(vk, 'term'), value_node(i) = string(nodeOf(vv)); end + if strcmp(vk, 'none') && logical(firstOr(ndi.v2.blockOf(p, 'value', 'data_body', false), false)) + vt = '(data body)'; + elseif strcmp(vk, 'none') + c = st.composite(); + if ~isempty(c) && numel(ndi.v2.blockOf(p, c, 'value', [])) > 1 + vt = '(array)'; % several values inline, e.g. one per encounter + end + end + value(i) = string(vt); unit(i) = string(vu); + o = ndi.v2.edgeIds(p, 'entity_id'); + if isempty(o), o = {''}; end + own{i} = o{1}; + via(i) = string(st.via()); + id(i) = string(p.base.id); + refs{i} = ndi.v2.edgeIds(p, 'time_reference_id'); + end + allRefs = unique([refs{:}]); + if ~isempty(allRefs) + times = ndi.v2.timesOf(container, allRefs, cache); + for i = 1:n + for r = 1:numel(refs{i}) + if ~isKey(times, refs{i}{r}), continue; end + t = times(refs{i}{r}); + if ~isnat(t.start) + start(i) = t.start; stop(i) = t.end; + timezone(i) = string(t.timezone); + break; + end + end + end + end + aboutIds = cellfun(@(s) s.about(), statements, 'UniformOutput', false); + ids = unique([reshape(aboutIds, 1, []), reshape(own, 1, [])]); + ids = ids(~cellfun(@isempty, ids)); + names = containers.Map('KeyType', 'char', 'ValueType', 'char'); + if ~isempty(ids) + ents = ndi.entity.fetchMany(container, ids); + for e = 1:numel(ents), names(ents{e}.document_id) = char(ents{e}.name); end + end + nameOf = @(x) string(ifKey(names, x)); + for i = 1:n + subject(i) = nameOf(aboutIds{i}); + stated_on(i) = nameOf(own{i}); + end + zones = unique(timezone(timezone ~= "")); + if isscalar(zones) + try + start.TimeZone = char(zones); stop.TimeZone = char(zones); + catch + end + end + T = table(subject, kind, class, variable, variable_node, method, method_node, value, ... + value_node, unit, start, stop, timezone, stated_on, via, id, 'VariableNames', columns); + end + + function obj = fromDocument(container, doc) + % FROMDOCUMENT - the right statement object for a statement document + % + % OBJ = ndi.statement.fromDocument(CONTAINER, DOC): an + % ndi.observation, ndi.manipulation, ndi.calculation or + % ndi.assertion by the document's class chain (never a list of + % leaf classes, so a new leaf needs no code), else an + % ndi.statement. DOC may be an ndi.document or an id. + arguments + container + doc + end + if ischar(doc) || isstring(doc) + d = ndi.v2.getDocument(container, char(doc)); + if isempty(d) + error('ndi:statement:notFound', 'No document %s in this %s.', char(doc), class(container)); + end + doc = d; + end + chain = ndi.v2.classChain(ndi.v2.props(doc)); + if any(strcmp(chain, 'observation')) + obj = ndi.observation(container, doc); + elseif any(strcmp(chain, 'manipulation')) + obj = ndi.manipulation(container, doc); + elseif any(strcmp(chain, 'calculation')) + obj = ndi.calculation(container, doc); + elseif any(strcmp(chain, 'assertion')) + obj = ndi.assertion(container, doc); + else + obj = ndi.statement(container, doc); + end + end + + function s = search(container, options) + % SEARCH - the statements in a session or dataset + % + % S = ndi.statement.search(CONTAINER, ...) returns a cell array. + % Filters: + % 'Subject' an ndi.entity (or an entity document id) + % 'Variable' the variable's name, e.g. 'midpoint speed' + % 'Class' 'observation', 'manipulation', 'calculation', + % 'assertion', 'interaction', or a document class + % (e.g. 'velocity_calculation') + % 'Method' the method's name, e.g. 'tracking by WormLab' + arguments + container + options.Subject = [] + options.Variable (1,:) char = '' + options.Class (1,:) char = '' + options.Method (1,:) char = '' + end + cls = 'statement'; + if ~isempty(options.Class) + cls = ndi.v2.vetaName(options.Class); % 'subject_observation' still works + end + q = ndi.v2.isaQuery(cls); + if ~isempty(options.Subject) + sid = options.Subject; + if isa(sid, 'ndi.entity'), sid = sid.document_id; end % any entity (2026-10-08) + q = q & ndi.v2.entityQuery(char(sid)); + end + if ~isempty(options.Variable) + q = q & ndi.v2.fieldQuery('statement.variable.name', 'exact_string', options.Variable); + end + if ~isempty(options.Method) + q = q & ndi.v2.fieldQuery('interaction.method.name', 'exact_string', options.Method); + end + docs = container.database_search(q); + s = cell(1, numel(docs)); + for i = 1:numel(docs) + s{i} = ndi.statement.fromDocument(container, docs{i}); + end + end + end +end + +function v = firstOr(x, default) +if isempty(x), v = default; else, v = x(1); end +end + +function v = fieldOr(s, name, default) +v = default; +if isstruct(s) && isfield(s, name) && ~isempty(s(1).(name)) + v = s(1).(name); +end +end + +function v = ifKey(m, k) +v = ''; +if ~isempty(k) && isKey(m, k), v = m(k); end +end + +function n = nodeOf(t) +% a term's node ('' when none) +n = ''; +if iscell(t) && ~isempty(t), t = t{1}; end +if isstruct(t) && ~isempty(t) && isfield(t, 'node') && ~isempty(t(1).node), n = char(t(1).node); end +end diff --git a/src/ndi/+ndi/summary.m b/src/ndi/+ndi/summary.m new file mode 100644 index 0000000000..848c13c8db --- /dev/null +++ b/src/ndi/+ndi/summary.m @@ -0,0 +1,52 @@ +function T = summary(x, options) +%SUMMARY A table of subjects, entities or statements, one row each. +% +% T = ndi.summary(X): X is what ndi.entity.search, +% ndi.statement.search or ndi.entity/statements returns -- a cell array, +% any mix of classes -- or an array, or one object. +% subjects and other entities ndi.entity/summary: name, kind, id, and +% for subjects type, local_identifier and a +% column per asserted variable (inherited) +% statements ndi.statement/summary: subject, kind, +% class, variable, method, value, unit, +% start, end, stated_on, via, id +% Entities and statements make different tables, so X holds one or the +% other. For one object, x.summary() does the same. Every term in a +% statement summary comes with its node (variable_node, method_node, +% value_node); for entities, ndi.summary(X, 'nodes', true) adds a +% _node column beside each asserted variable. +% +% ws = ndi.entity.search(ds, {'type', 'organism', 'strain', 'N2'}); +% ndi.summary(ws) % one row per worm +% ndi.summary(statements([ws{:}])) % everything about them +% +% See also ndi.entity/summary, ndi.statement/summary. + +arguments + x + options.nodes (1,1) logical = false +end +if ~iscell(x) + x = num2cell(x); +end +x = reshape(x, 1, []); +isEnt = cellfun(@(e) isa(e, 'ndi.entity'), x); +isSt = cellfun(@(e) isa(e, 'ndi.statement'), x); +if ~all(isEnt | isSt) + bad = find(~(isEnt | isSt), 1); + error('ndi:summary:what', 'ndi.summary takes entities or statements (item %d is a %s).', ... + bad, class(x{bad})); +end +if any(isEnt) && any(isSt) + error('ndi:summary:mixed', ... + 'Entities and statements make different tables: summarise them separately.'); +end +if all(isSt) && ~isempty(x) + if options.nodes + error('ndi:summary:options', 'A statement summary takes no options (its nodes are always shown).'); + end + T = ndi.statement.summaryOf(x); +else + T = ndi.entity.summaryOf(x, 'nodes', options.nodes); +end +end diff --git a/src/ndi/+ndi/value.m b/src/ndi/+ndi/value.m new file mode 100644 index 0000000000..e034919092 --- /dev/null +++ b/src/ndi/+ndi/value.m @@ -0,0 +1,200 @@ +classdef value + % ndi.value - a statement's value: what was measured, set or computed + % + % Every V2 statement's value is a `value` composite (`data_type` until + % 2026-10-08) (length, time, + % intensity, velocity, temperature, term, label, score, dose, position, + % model_fit, ...). A dimensioned one stores each value as a cell: the + % number in its canonical unit (e.g. `meters`), the source value and + % unit it came from, a tolerance and whether it is approximate (tenet + % T14). A value is stored INLINE on the statement or, when it is an array, + % in a data body; an ndi.value reads the same way either way. + % + % Get one from a statement: v = st.value(); + % + % ndi.value Methods: + % canonical - the values in the canonical unit (numbers), or the + % names (terms, labels), or the raw struct (dose, model + % fit, ...); double(v) is the same for numbers + % unit - the canonical unit's field name, e.g. 'meters' + % source - table: source_value, source_unit (as the lab wrote it) + % tolerance - [minus plus] per value, in the canonical unit + % approximate - true per value that is approximate + % axes - table of the value's keys (what each dimension indexes) + % files - for a value kept as files (a video, an image): paths + % + % See also ndi.statement. + + properties (SetAccess = protected) + class_name = '' % the composite, e.g. 'length', 'term', 'dose' + raw = [] % the inline value as stored (struct or array) + data = [] % the decoded array, when the value is in a sampled body + file_list = {} % file paths, when the value is in an opaque body + keys = [] % the value's keys (data.keys) + datum_type = '' % how the body's bytes are encoded + canonical_field = '' % the cell's canonical field, e.g. 'meters' + end + + methods + function obj = value(className, raw, options) + % VALUE - wrap a composite value + % + % OBJ = ndi.value(CLASSNAME, RAW, 'Data', A, 'Files', F, + % 'Keys', K, 'DatumType', T). Made by ndi.statement/value. + arguments + className (1,:) char = '' + raw = [] + options.Data = [] + options.Files = {} + options.Keys = [] + options.DatumType (1,:) char = '' + end + obj.class_name = className; + obj.raw = raw; + obj.data = options.Data; + obj.file_list = options.Files; + obj.keys = options.Keys; + obj.datum_type = options.DatumType; + obj.canonical_field = ndi.value.canonicalField(className); + end + + function v = canonical(obj) + % CANONICAL - the values in the canonical unit + % + % Numbers for a dimensioned value (from the body, or each cell's + % canonical field), names for terms and labels, the raw struct + % for a structured value (dose, item, position, model_fit). + if ~isempty(obj.data) || ~isempty(obj.file_list) + v = obj.data; + return; + end + r = obj.raw; + if iscell(r) && ~isempty(r) && all(cellfun(@isstruct, r)) + try r = [r{:}]; catch, end + end + if isstruct(r) && ~isempty(obj.canonical_field) && isfield(r, obj.canonical_field) + v = arrayfun(@(x) double(firstOr(x.(obj.canonical_field), NaN)), r); + elseif isstruct(r) && (isfield(r, 'name') || isfield(r, 'node')) && ... + all(ismember(fieldnames(r), {'node', 'name'})) + v = strings(size(r)); + for i = 1:numel(r), v(i) = string(ndi.v2.termName(r(i))); end + elseif isstruct(r) && isfield(r, 'text') + % the `text` kind {text, language} (2026-10-08): the text + v = strings(size(r)); + for i = 1:numel(r), v(i) = string(char(r(i).text)); end + else + v = r; + end + end + + function d = double(obj) + % DOUBLE - the canonical values as numbers + d = double(obj.canonical()); + end + + function u = unit(obj) + % UNIT - the canonical unit's field name, e.g. 'meters' ('' when none) + u = obj.canonical_field; + end + + function T = source(obj) + % SOURCE - table: source_value, source_unit, per inline value + [r, ok] = obj.cells(); + source_value = nan(numel(r), 1); source_unit = strings(numel(r), 1); + for i = 1:numel(r) + if ~ok, break; end + if isfield(r, 'source_value'), source_value(i) = double(firstOr(r(i).source_value, NaN)); end + if isfield(r, 'source_unit'), source_unit(i) = string(r(i).source_unit); end + end + T = table(source_value, source_unit); + end + + function t = tolerance(obj) + % TOLERANCE - [minus plus] per inline value (NaN when none), canonical unit + [r, ok] = obj.cells(); + t = nan(numel(r), 2); + for i = 1:numel(r) + if ~ok || ~isfield(r, 'tolerance') || isempty(r(i).tolerance), continue; end + tl = r(i).tolerance; + if isfield(tl, 'minus'), t(i, 1) = double(firstOr(tl.minus, NaN)); end + if isfield(tl, 'plus'), t(i, 2) = double(firstOr(tl.plus, NaN)); end + end + end + + function a = approximate(obj) + % APPROXIMATE - true per inline value marked approximate + [r, ok] = obj.cells(); + a = false(numel(r), 1); + for i = 1:numel(r) + if ok && isfield(r, 'approximate') && ~isempty(r(i).approximate) + a(i) = logical(r(i).approximate); + end + end + end + + function T = axes(obj) + % AXES - table of the value's keys: variable, unit, n, coordinates + T = ndi.v2.axesTable(obj.keys); + end + + function f = files(obj) + % FILES - the value's file paths, when it is kept as files + f = obj.file_list; + end + end + + methods (Access = private) + function [r, ok] = cells(obj) + r = obj.raw; + if iscell(r) && ~isempty(r) && all(cellfun(@isstruct, r)) + try r = [r{:}]; catch, end + end + ok = isstruct(r); + if ~ok, r = []; end + end + end + + methods (Static) + function f = canonicalField(className) + % CANONICALFIELD - the numeric field of CLASSNAME's value cell, e.g. 'meters' + % + % Read from the V2 schema: the first numeric sub-field of the + % class's `value` that is not source_value, scale_min or scale_max. + arguments + className (1,:) char + end + f = ''; + if isempty(className), return; end + try + s = did2.schema.cache.shared().getClass(className); + catch + return; + end + if ~isfield(s, 'fields'), return; end + % field definitions differ in their keys, so they are walked as a + % cell array rather than concatenated + fs = asCell(s.fields); + v = []; + for k = 1:numel(fs) + if isfield(fs{k}, 'name') && strcmp(fs{k}.name, 'value'), v = fs{k}; break; end + end + if isempty(v) || ~isfield(v, 'fields') || isempty(v.fields), return; end + sub = asCell(v.fields); + for k = 1:numel(sub) + if isfield(sub{k}, 'type') && any(strcmp(sub{k}.type, {'double', 'integer', 'matrix'})) && ... + ~any(strcmp(sub{k}.name, {'source_value', 'scale_min', 'scale_max'})) + f = char(sub{k}.name); + return; + end + end + end + end +end + +function v = firstOr(x, default) +if isempty(x), v = default; else, v = x(1); end +end + +function c = asCell(x) +if iscell(x), c = reshape(x, 1, []); else, c = num2cell(reshape(x, 1, [])); end +end diff --git a/src/ndi/docs/NDI-matlab/manual/developer/V2_Object_Layer.md b/src/ndi/docs/NDI-matlab/manual/developer/V2_Object_Layer.md new file mode 100644 index 0000000000..bea4243074 --- /dev/null +++ b/src/ndi/docs/NDI-matlab/manual/developer/V2_Object_Layer.md @@ -0,0 +1,354 @@ +# NDI V2 object layer: subjects, statements and entities + +**Status: PARTLY BUILT (2026-10-06), for review by Jess Haley and Steve Van Hooser.** +Built (read side): `ndi.entity`, the statement tree, +`ndi.value`, and the `ndi.v2` helpers. **Not built, on purpose:** any change to +`ndi.session` or `ndi.dataset` (section 4, "Entry points"); `ndi.strain`, `ndi.study`; +`subject.location(t)`; the `'During'` filter. Section 9 lists what was built. + +## 1. Why + +A V2 (V_eta) dataset is a set of statements about subjects: who or what was +observed, manipulated, calculated about or asserted about, when, how and with +what result. NDI's objects do not know this yet. Today a user who wants one worm's +speed from a V2 dataset has to: + +1. search for `velocity_calculation` documents whose `subject_id` edge is the worm; +2. pick the one whose `variable` is `midpoint speed`; +3. follow its `depends_on` to the `sampled_body` that holds the values; +4. open that body's file, gunzip it, and decode it by the body's `datum_type`, + `byte_order` and `keys` (the axes); +5. follow the statement's time-reference edges to absolute or relative time + references, and resolve them to a start and end; +6. read units and tolerances from the value cell. + +That is schema knowledge, not science. This document proposes objects that do it +once, so the user writes `w.statements('statement', {'variable', 'midpoint speed'}).value()`. + +## 2. What exists today (NDI-matlab, branch `claude/haley-import-v2`) + +| object | what it does | V2 status | +|---|---|---| +| `ndi.dataset` | opens a dataset, lists and opens its sessions, searches its database | opens a V2 dataset and lists its sessions (import stage 13, decision #66); no subjects, statements, studies or entities | +| `ndi.session` | database front end; DAQ systems, probes, elements, syncgraph | opens a V2 session; no subjects, statements or entities | +| `ndi.subject` | `local_identifier`, `description`; `newdocument`, `searchquery`; a string check | requires an `@` in `local_identifier` (`subject.m:120`); no navigation | +| `ndi.element`, `ndi.probe` | acquisition objects, raw-data reading | V2 read through `ndi.vintage` (the v1/V2 name map) | +| statements, entities | none | only as raw documents from `database_search` | + +There is no method today that lists a session's subjects. + +## 3. Decisions (agreed with Jess Haley, 2026-10-06) + +- **D1 Read first.** Build the side that opens an existing dataset and answers + questions before any side that creates documents through these objects. Writing + stays with `did2.build` (as the Haley import does) until reading is done. +- **D2 One statement tree, mirroring the schema.** + ``` + ndi.statement statement + ├── ndi.assertion assertion (e.g. term_assertion: species, strain, exclusion) + └── ndi.interaction interaction (abstract: never returned on its own) + ├── ndi.observation observation + ├── ndi.manipulation manipulation + └── ndi.calculation calculation + ``` + There is no class per leaf (`velocity_calculation`, `dose_manipulation`, ...): + about forty leaves would differ only by name. A statement object is returned as + the right child automatically, from its document's class chain, so a new leaf in + the schema needs no new code. +- **D3 `value()`, not `data()`.** A statement has one value, stored inline or in a + data body; where it is stored is the schema's business. `value()` returns it + either way, decoding the body when the value lives there. `body()` stays for + the raw file. +- **D4 (WITHDRAWN 2026-10-08, see section 8) `ndi.subject` is an `ndi.entity`.** The schema already says so: `subject`, + `session`, `dataset`, `study`, `person`, `strain`, ... are all `entity` + (`formulation` is a `data_type`, not an entity). +- **D5 Back compatibility with v1** (section 6). +- **D6 The `@` in `local_identifier` is a v1 rule, not a V2 rule.** `ndi.subject` + keeps requiring it when it CREATES a v1 subject, unchanged. A V2 subject read + from a document is accepted without one. Reasons: + - uniqueness is already given: every subject has a globally unique `id`, and the + schema makes `local_identifier` unique within its dataset; + - the lab is structure, and V2 declares structure rather than encoding it in + strings (tenet T14). The lab is an `organization` entity reached through + relations, not a suffix that code splits on (four places in NDI split a + subject name on `@` today: the Birren, Dabrowska and Haley v1 converters and + the metadata app); + - a suffix goes stale when a lab moves or merges, and a relation can be updated. + +## 4. The classes + +### `ndi.entity` + +Built from any entity document. + +| member | returns | +|---|---| +| `id`, `kind` (the document class), `name`, `local_identifier` | text | +| `global_identifiers()` | table: scheme (ORCID, ROR, RRID, DOI, ...), value | +| `relations(name, 'Direction', 'out' or 'in')` | table: relation, the other entity, role(s), time | +| `parents(name)`, `children(name)` | entities across one relation (`part_of`, `member_of`, ...), or a path of them (`{'contained_in', 'member_of'}`); called on an array of entities, one search per step, each entity returned once; `'Table', true` gives one row per (start, end) | +| `ancestors(...)`, `descendants(...)` | the nearest entities of a `'Type'` (`organism`, `material`, ...) or `'Kind'` (document class), following any relation up or down: `descendants([plates{:}], 'Type', 'organism')` is the worms that were on them (function form: `[c{:}].method(...)` is not valid MATLAB), without knowing the relations. `'Relation'` limits the relations followed, `'MaxDepth'` the steps, `'Table'` adds a `depth` column | +| `document()` | the underlying `ndi.document` | +| `ndi.entity.fromDocument(container, doc)` (static) | an `ndi.entity` for any entity document, a subject's included | +| `ndi.entity.search(container, conditions, options)` (static) | a cell array of entities; `conditions` a cell of pairs, e.g. `{'kind', 'person', 'name', 'Jess*'}` (`'kind'`: a document class or, one entity class, an entity type); `{}` is every entity | +| `type`, `description` | the entity's type and description, from either schema | +| `statements(...)`, `assertions(...)`, `observations(...)`, ... | what is stated of the entity, as for a subject below: since 2026-10-08 (one entity class) any entity can be the subject of a statement, so these live on `ndi.entity` (a person's given name, a strain's genetic strain type) | + +`ndi.entity` is a CONCRETE value class: a person, an organization or a piece of +software is an `ndi.entity` with no subclass. It holds its document and the +session or dataset it was read from, and nothing else; every question is a query. + +Subclasses only where there is real behaviour: + +- `ndi.strain`: `species()`, `lineage()` (parental strains), `genotype()`; +- `ndi.study`: `sessions()`, `factors()`, `design()`. + +Persons, organizations, products, software, funding and publications are plain +`ndi.entity`. A formulation is not an entity, so it is not here: it is reached +from a dose (`ndi.manipulation.formulation()`), see open question Q4. + +### Subjects (`ndi.entity`) + +The members below are `ndi.entity`'s and work for any entity; they are +written for a subject (a worm, a cohort, a plate) because that is where they +are used most. `ndi.subject` is v1's class, unchanged on this branch. + +| member | returns | +|---|---| +| `type` | the coarse kind: organism, group, culture, material, ... | +| `statements(kind, filters, options)` | statement objects about this subject; `kind` (default `'statement'`): observation, manipulation, calculation, assertion, interaction, or a leaf class; `filters` a cell: `'variable'`, `'method'`, `'value'`, `'formulation'`, `'at'`, `'during'`, `'before'`, `'after'`, `'duration'`; options `'inherited'`, `'context'`, `'tolerant'` | +| `assertions(filters, options)` | `statements('assertion', filters, options)`: assertion objects (species `Caenorhabditis elegans`, strain `N2`, `inclusion in analysis` `excluded`); `ndi.summary` of them is the table | +| `observations(filters, options)`, `manipulations(...)`, `calculations(...)`, `interactions(...)` | `statements('', filters, options)`: objects of that kind, inherited by default; the cell of filters first, e.g. `w.observations({'variable', 'temperature'})` | +| `...('context', true)` | also what the subject was IN (V_eta tenet T17): the interactions of each container it was `contained_in` (through a distributive stay of its group, and nested containers) whose time overlaps the stay. Marked `via() = '... contained_in'`; assertions never pass; a stay or statement whose times cannot be compared is left out. `ndi.entity.search(..., 'context', true)` finds them the same way | +| `'during', {'observation', {...}}` | a time filter that takes statements as well as times: a statement is kept when its time overlaps one matching the description that holds of the same subject, by the same rules (`'inherited'`, `'context'`). `w.observations({'variable', 'image intensity', 'during', {'observation', {'variable', 'ambient temperature', 'value', '>22'}}}, 'context', true)`: the worm's plate's videos while that plate read over 22 C | +| `members()` / `memberOf()` | a group's members / the groups it belongs to (`member_of`) | +| `parts()` / `partOf()` | `part_of`: a plate's patches / a patch's plate | +| `location(t)` | where the subject was at time `t` (`contained_in`) | + +The v1 constructor and its `@` check stay as they are (D6). + +### `ndi.statement` + +| member | returns | +|---|---| +| `subject()`, `entity()` | the `ndi.entity` it is about | +| `variable` | the term (name and node) | +| `value()` | the value, with units, decoded from the body when stored there (array, with axes) | +| `axes()` | the value's keys: name, unit, values (e.g. video frame, metre) | +| `conditions()` | table: variable, value, unit | +| `notes` | text | +| `body()` | the raw data-body document and file, if any | +| `ndi.statement.fromDocument(doc)` (static) | the right child, from the document's class chain | + +### `ndi.assertion < ndi.statement` + +No additions: an assertion is a variable and value about a subject, with no time, +method or instrument. + +### `ndi.interaction < ndi.statement` (abstract) + +| member | returns | +|---|---| +| `time()` | start and end, resolved through the time references (absolute UTC or relative to another document), with tolerance | +| `method()`, `method_parameters()` | the term; a table of parameters | +| `instrument()` | the instrument subject, if any | + +### `ndi.observation`, `ndi.manipulation`, `ndi.calculation < ndi.interaction` + +- `ndi.observation`: nothing more today. +- `ndi.manipulation`: `formulation()` for a dose: what was given, with its + ingredient tree. +- `ndi.calculation`: `inputs()` (the statements and documents it was computed + from), `software()` (software, interpreter, operating system). + +### `ndi.value` + +A statement's value, the same object whether it was stored inline or in a data +body (answers Q3: a small value class). + +| member | returns | +|---|---| +| `canonical()`, `double()` | the values in the canonical unit; a term's name; the raw struct for a structured value (dose, item, model fit) | +| `unit()` | the canonical field, e.g. `meters`, read from the schema | +| `source()` | table: source_value, source_unit | +| `tolerance()`, `approximate()` | per value | +| `axes()` | the keys: variable, unit, n, coordinates | +| `files()` | file paths, for a value kept in an opaque body (a video) | + +### Entry points (instead of `ndi.session` / `ndi.dataset` additions) + +`ndi.session` and `ndi.dataset` are NOT changed (Jess Haley, 2026-10-06: "I'm not +sure about the session and dataset changes"). The entry points are static methods +that take the session or dataset as their first argument: + +| call | returns | +|---|---| +| `ndi.entity.search(S, {'type', 'organism', 'species', 'Caenorhabditis elegans', 'strain', {'N2', 'CB*'}, 'manipulation', {'method', 'heating', 'value', '>=0.02'}}, 'inherited', true)` | entities for which every condition holds. The conditions are ONE cell of PROPERTY, VALUE pairs (`{}`: every entity; `{'type', ndi.v2.subjectTypes()}` keeps the subjects, Q2); the switches `'inherited'`, `'context'`, `'strict'`, `'explain'`, `'tolerant'`, `'synonyms'` are declared options after it (an `arguments` block validates them), and a switch inside the cell is an error. A nested description is a cell too and takes the outer `'strict'` and `'inherited'`. `'synonyms'` (default true): a value that matches no term by its name or node also matches a term it is a synonym of, `{'species', 'C. elegans'}` finding `NCBITaxon:6239`; the names come from `ndi.ontology.lookup`, looked up once per term node and kept on disk (`ndi.v2.synonyms`, folder `ndi.v2.synonymFolder`, or `NDI_SYNONYM_CACHE`), and nothing is found offline. A strain matches through its global identifiers' synonyms the same way. Only a value that matched nothing directly triggers a lookup. A property is one of the subject's own fields (`type`, `name`, `local_identifier`); a statement kind (`statement`, `assertion`, `interaction`, `observation`, `manipulation`, `calculation`, each including its children) with a cell of filters on ONE statement (`variable`, `method`, `value`, `formulation`; the key twice is two statements); or any other name, an asserted variable (`'strain', 'N2'` is `'assertion', {'variable', 'strain', 'value', 'N2'}`). Names ignore case. A pattern matches a term's name ignoring case or its node exactly; a cell array is any of; `*` is a wildcard (in the database with DID-matlab #218's `wildcard` operator, else rechecked in MATLAB). Numbers and dates compare with `>`, `>=`, `<`, `<=`; an array or body value has no single value. `'inherited'` (default true) adds the members, down `member_of`, of a group whose statement is `distributive`; `contained_in` and `part_of` are not followed. A variable or method no statement has is an error; values matching nothing, a warning (`ndi.v2.searchStatements`) Relations: `'relation'`, `'directed_relation'`, `'undirected_relation'` take `{'name', R, 'parent'|'child'|'with', X}` (the subject is the child / the parent / either end), X a subject, several, a document id, or a nested description; a relation's own name is a key (`'contained_in', X` = child of a contained_in whose parent is X; `'with'` for undirected). Inherited also carries a distributive relation from a group to its members, an assertion from a whole to its parts and samples (part_of, sample_of, aliquot_of, passage_of), and an assertion from a type to its instances (`instance_of`: a strain's or a product's assertions hold of each worm or bottle, reached from the subject, a whole, or a group whose `instance_of` is distributive; V_eta T17). `'strain', 'N2'` is a subject that is an `instance_of` the strain named N2 (one entity class). `'explain', true` prints the search in words. Not built yet: `'at'`, `'during'`, `'depth'`. | +| `ndi.statement.search(S, 'Subject', s, 'Variable', v, 'Class', c, 'Method', m)` | statements | +| `ndi.entity.search(S, {'kind', K})` | entities of a kind | +| `ndi.entity.fromDocument(S, doc)`, `ndi.statement.fromDocument(S, doc)` | the object for one document | + +**Read through the dataset.** A session searches only its own documents, and the +shared documents (people, organizations, studies, strains, software, products, +instruments, formulations) are stored with the dataset (Haley decision #20). An +object read through a session therefore cannot reach them: `software()` and +`formulation()` come back empty. An object read through the `ndi.dataset` reaches +everything, including every session's documents. Measured on the full Haley dataset +(58 sessions, 2026-10-06, `tools/v2_object_layer_tryout.m`): one worm's 22 +statements took 0.19 s through its session and 0.52 s through the dataset; the +dose's formulation was not found through the session and was found through the +dataset. To narrow a search to one session, find things in the session, then read +them through the dataset with `fromDocument(ds, id)`. + +A session opened FROM its dataset (`ds.open_session(id)`) lives in the dataset's +database, and `ndi.session/database_search_with_dataset` searches it without the +session filter. The readers use it for the dataset-level entities a subject +points at: a strain found by name (`'strain', 'N2'`), and a strain's assertions +and name reaching its worms (`ndi.summary` shows a `strain` column). A session +folder opened on its own holds the `instance_of` relation but not the strain, so +it cannot name it. + +## 5. How it is built + +- **Objects wrap documents.** Each object holds its document and its session or + dataset. It caches nothing it cannot rebuild, and reads lazily: `value()` + touches the body file only when called. +- **Every navigation is an indexed query.** "This worm's statements" is a query + on the `depends_on` table (`name = 'subject_id' AND document_id = `), + which is indexed in `did2.database.sqlitedb`, not a scan of the documents. On + the Haley dataset (187,673 documents, 889,957 edges) it should take + milliseconds; the tests will time it. +- **Class from the class chain.** `fromDocument` reads the document's superclasses + (the database's `superclasses` table) to pick `ndi.observation`, + `ndi.manipulation`, ..., never a hard-coded list of leaves. +- **Body decoding in one place.** A single reader turns a `sampled_body` into a + MATLAB array from its `datum_type`, `byte_order`, `datum_order`, `fill_value`, + `compression` and `keys`. The Haley import's `ingestedBody` is its writing + counterpart, so the tests can round-trip. +- **Time resolution in one place.** A single resolver follows a statement's time + references, including chains (relative to an epoch relative to a session), + to a start and end in UTC where an absolute anchor exists, else relative to the + anchor it names. + +## 6. Back compatibility with v1 + +1. **Every existing method keeps its exact behaviour on v1 data:** + `ndi.subject`'s constructor and `@` check, `ndi.session` and `ndi.dataset` + search and open, elements, probes, DAQ, syncgraph. +2. **New methods work on V2, and on v1 where `ndi.vintage` defines an + equivalent** (for example, a v1 `subject` document becomes an `ndi.subject` + with no statements). **Where v1 has no equivalent, they return empty, never + an error.** +3. **No v1 document is rewritten or reinterpreted on read.** Migration stays + `ndi.migrate`'s job. +4. **Every new class is tested on a v1 session and a V2 session**, as + `TestVintageMap` already does for the read path. +5. **`ndi.session` and `ndi.dataset` gain entity behaviour last** (Q1), because + their class trees are the deepest and the most used. + +## 7. Build order + +1. `ndi.entity`, `ndi.subject` (V2 read; `@` relaxed for V2, D6), + `session.subjects()` / `dataset.subjects()`. +2. `ndi.statement` and its tree; `value()` with the body reader; `axes()`, + `conditions()`. +3. `time()` with the time resolver; `method()`, `instrument()`. +4. `ndi.calculation.inputs()`, `ndi.manipulation.formulation()`, + `ndi.strain`, `ndi.study`, `entities()`, `studies()`. +5. `ndi.session` / `ndi.dataset` as entities (Q1). + +Each step lands with tests on the Haley CI fixture (V2) and a v1 session, and is +tried on the real Haley dataset before the next. + +## 8. Open questions + +- **Q1** (Reading does not need it: read through the dataset, section 4.) How `ndi.session` and `ndi.dataset` become entities: a mixin class, or + composition (`session.entity()` returning an `ndi.entity`)? A mixin changes + their class hierarchy; composition does not. +- **Q2** ANSWERED, and changed 2026-10-08 (Jess Haley): the subjects are the + biological types, organism, culture, tissue, cell and group + (`ndi.v2.subjectTypes`), as a v1 `subject` document always was. + `ndi.query('', 'isa', 'subject')` finds those (and a v1 subject, which has no + type), so every existing `isa subject` search keeps meaning the animals; + A probe, camera (device) or plate + (material) is an entity: `ndi.entity.search(S, {'type', 'material'})`. + `'subject'` is not a kind `ndi.entity.search` accepts. +- **2026-10-08 (Jess Haley): `ndi.subject` is not touched on this branch.** It is + v1's class exactly as on `V2`; every entity, a worm included, is read as an + `ndi.entity`, and D4 below is withdrawn. +- **Q3** ANSWERED: `ndi.value`, a small value class. +- **Q4** ANSWERED: a formulation is a data_type, so `formulation()` returns an + `ndi.value` of class `formulation`; no `ndi.formulation`. +- **Q5** The writing side (D1 defers it): when it comes, does `ndi.statement` + get constructors wrapping `did2.build`, or does `did2.build` stay the only + writer? + +## 9. What is built (2026-10-06) + +| file | what | +|---|---| +| `+ndi/entity.m` | `ndi.entity` (concrete value class) | +| `+ndi/subject.m` | now `ndi.ido & ndi.documentservice & ndi.entity`; adds `type`, `statements`, `assertions`, `members`, `memberOf`, `parts`, `partOf`, static `fromDocument`, `search`. The v1 constructor, `newdocument`, `searchquery` and the `@` check are unchanged | +| `+ndi/statement.m` | `ndi.statement`: `subject`, `variable`, `variable_name`, `composite`, `raw_value`, `value`, `bodies`, `axes`, `conditions`, `notes`, static `fromDocument`, `search` | +| `+ndi/assertion.m` | `ndi.assertion` | +| `+ndi/interaction.m` | `ndi.interaction` (abstract): `time`, `method`, `method_name`, `method_parameters`, `instrument`, `software` | +| `+ndi/observation.m` | `ndi.observation` | +| `+ndi/manipulation.m` | `ndi.manipulation`: `formulation` | +| `+ndi/calculation.m` | `ndi.calculation`: `inputs`, `interpreter`, `operating_system` | +| `+ndi/data_type.m` | `ndi.value` | +| `+ndi/+v2/` | helpers: `props`, `classChain`, `directParents`, `edgeIds`, `getDocument`, `blockOf`, `termName`, `readBody` (the sampled-body reader), `axesTable`, `parseUtc`, `timeOf` (the time resolver) | + +Tests: `tests/+ndi/+unittest/+setup/+V2/TestObjectLayer.m`, over the Haley CI +fixture, including the v1 subject constructor. Not yet: a chunked body (an +error for now), a v1 session beyond the subject constructor, timing on the full +Haley dataset. + +## 10. One entity class (2026-10-08) + +did-schema `V_eta_entity_composition_plan.md` (signed 2026-10-08) merged the +entity classes into one `entity` with a bound `type`, made statements a +direction class plus one value-kind mixin, and made a strain an `instance_of` +relation. The readers read both schemas: + +| file | what changed | +|---|---| +| `+ndi/entity.m` | `type`, `local_identifier`, `description` read either schema; the statement members, `search` and its helpers moved here from `+ndi/subject.m` (error ids `ndi:entity:...`); `searchKind`; `inheritanceStates` (the walk up `member_of`, the lineage relations and `instance_of`); `summaryOf` adds a column per type a subject is an instance of | +| `+ndi/subject.m` | unchanged from `V2` (v1 only) | +| `+ndi/session.m` | `database_search_with_dataset` | +| `+ndi/+v2/` | `kindOf`, `blockOf` (falls back to the `entity` block), `isaQuery`, `blockQuery`, `leafName`; `classChain` appends the value kind; `matchTerm` / `termQuery` match a CURIE prefix ignoring case; `statementValue` reads the `text` kind; `searchStatements` takes `withDataset` | + +Recording keys (decision #70): a Haley video or image statement's `axes()` gives +the image's rows and columns in pixels (centres at 0.5, 1.5, ...) and, for a +video, `time`, one per frame from the recording's start at 1 / frame rate. They +are written only when the file was opened (`'ReadVideos'`, or a TIFF). + +## 11. Linked and ingested sessions (2026-10-09) + +did-schema `V_eta_linked_session_plan.md` (signed 2026-10-09): a session is in +a dataset when its `session` entity is `part_of` the dataset, or `part_of` a +study that is `part_of` the dataset; the relation is in the dataset's database. +An INGESTED session's documents are in the dataset's database too; a LINKED +session's are in its own folder, which a `linked_session` document (`path`, +relative to the dataset's folder when inside it, and an `entity_id` edge to the +session's entity) names. v1's `session_in_a_dataset` is not used. + +| method / function | what it does on a V2 dataset | +|---|---| +| `session_list`, `open_session`, `database_search` | list the members (`ndi.v2.datasetSessions`), open a linked one from its folder (checked to hold that session), search the dataset and each linked session | +| `session_notes` | the listing's denominator, and every session left out and why (not `part_of` the dataset; a linked folder missing, unopenable or holding another session; also a warning, `ndi:dataset:sessionsLeftOut`) | +| `add_linked_session(S, 'PartOf', id)` | a `part_of` relation (to the dataset, or the study `PartOf` names) and a `linked_session` document | +| `add_ingested_session(S, 'PartOf', id)` | copies every document of S, with the files it ingested; adds a `part_of` relation unless S's own relations already make it a member | +| `convertLinkedSessionToIngested` | copies the linked session's documents in, deletes the `linked_session` | +| `convertIngestedSessionToLinked(id, folder)` | writes the session's documents to a new V2 session in `folder`, deletes them from the dataset, adds a `linked_session`; the `part_of` relations stay | +| `unlink_session` | deletes the `linked_session` and the `part_of` relations naming the session; the folder stays | +| `makeSelfContained('DryRun', tf)` | copies every file the dataset's documents record by location into its file store | + +Documents are copied with `ndi.v2.copyDocuments`, as stored +(`ndi.v2.sessionDocuments` reads the did2 database, not the ndi read path, +which adds class blocks): ids, edges and session ids unchanged; a file a +document ingested is ingested again from the source's copy; a file recorded by +location stays where it is. `ndi.dataset.dir`'s constructor now reads its own +database for its lookups. Tests: `TestLinkedSessions`. + +Not decided here, and not built: merging shared entities (a strain, people, +software) when a session made on its own joins a dataset. diff --git a/src/ndi/docs/NDI-matlab/manual/developer/cloud.md b/src/ndi/docs/NDI-matlab/manual/developer/cloud.md index 4e732b0dcf..0bdc1fb120 100644 --- a/src/ndi/docs/NDI-matlab/manual/developer/cloud.md +++ b/src/ndi/docs/NDI-matlab/manual/developer/cloud.md @@ -453,13 +453,24 @@ pyramid level is an O(1) uid lookup rather than another round trip. | Function | Signature | Description | |---|---|---| -| `ndi.cloud.api.files.getFileDetails` | `getFileDetails(cloudDatasetID, cloudFileUID)` | Retrieves metadata for a single file, including a pre-signed download URL. | +| `ndi.cloud.api.files.getFileDetails` | `getFileDetails(cloudDatasetID, cloudFileUID)` | Retrieves metadata for a single file, including a pre-signed download URL and `crc32` (see below). | | `ndi.cloud.api.files.getFile` | `getFile(downloadURL, downloadedFile, 'useCurl', true)` | Downloads a file from a pre-signed URL and saves it to a local path. `useCurl` defaults to `true` (curl requests identity encoding so already-compressed archives are not corrupted); set it to `false` to use MATLAB's native `websave`. | -| `ndi.cloud.api.files.getFileUploadURL` | `getFileUploadURL(cloudDatasetID, cloudFileUID)` | Returns a pre-signed URL for uploading a single file. | +| `ndi.cloud.api.files.getFileUploadURL` | `getFileUploadURL(cloudDatasetID, cloudFileUID, 'checksumCRC32', crc)` | Returns a pre-signed URL for uploading a single file. With `checksumCRC32` (from `ndi.fun.file.crc32`) the URL signs the `x-amz-checksum-crc32` header, so pass the same value to `putFiles(..., 'checksumCRC32', crc)`; S3 then rejects an upload whose content does not match, and the cloud records the value as the file's `crc32`. `ndi.cloud.uploadSingleFile` does both by default. | | `ndi.cloud.api.files.getFileCollectionUploadURL` | `getFileCollectionUploadURL(cloudDatasetID)` | Returns a pre-signed URL for uploading a ZIP archive of multiple files. | | `ndi.cloud.api.files.putFiles` | `putFiles(preSignedURL, filePath, 'useCurl', true)` | Uploads a local file to a pre-signed URL via HTTP PUT. `useCurl` defaults to `true` (consistent S3 object headers); set it to `false` to use MATLAB's native HTTP client. | | `ndi.cloud.api.files.listFiles` | `listFiles(cloudDatasetId, 'limit', L, 'after', cursor)` | Lists a single keyset page of a dataset's files. Returns the page plus a `cursor`/`hasMore` envelope; pass `cursor` back as `after` for the next page. | | `ndi.cloud.api.files.listFilesAll` | `listFilesAll(cloudDatasetId, 'limit', L)` | Lists **all** of a dataset's files by following the keyset cursor across pages, with optional polling for newly uploaded files. The whole-dataset counterpart to `listFiles`. | +| `ndi.cloud.api.files.lookupFiles` | `lookupFiles(cloudDatasetID, uids)` | Returns `uploaded`, `size` and `crc32` for the given file uids (`files`), and the uids the dataset has no record of (`missing`). No URLs are signed. Lists longer than the server's 5000-uid limit are split into several calls. Use it to check one document's or import batch's files, e.g. to resume an interrupted upload. | + +#### File checksums (`crc32`) + +NDI Cloud records a CRC-32 for each uploaded file: the value sent as +`checksumCRC32` for single-file uploads, or the zip entry's CRC-32 for bulk +uploads. It is the base64 of the 4-byte big-endian CRC-32 (IEEE), the same +encoding S3 uses, and `ndi.fun.file.crc32(filePath)` computes it for a local +file. `getFileDetails`, `listFiles`/`listFilesAll`, `lookupFiles` and the +`fileInfo` array of `getSignedURLSet`/`getSignedURLSetAll` return it. Files +uploaded without a checksum have `crc32` empty (`''`); compare `size` instead. ### Compute (`ndi.cloud.api.compute.*`) diff --git a/src/ndi/ndi_common/database_documents/data/lightsheetZarrLevel.json b/src/ndi/ndi_common/database_documents/data/lightsheetZarrLevel.json new file mode 100644 index 0000000000..83c301cf7a --- /dev/null +++ b/src/ndi/ndi_common/database_documents/data/lightsheetZarrLevel.json @@ -0,0 +1,56 @@ +{ + "document_class": { + "definition": "$NDIDOCUMENTPATH\/data\/lightsheetZarrLevel.json", + "validation": "$NDISCHEMAPATH\/data\/lightsheetZarrLevel_schema.json", + "class_name": "lightsheetZarrLevel", + "property_list_name": "lightsheetZarrLevel", + "class_version": 1, + "superclasses": [ + { + "definition": "$NDIDOCUMENTPATH\/base.json" + } + ] + }, + "depends_on": [ + { + "name": "lightsheetZarrPyramid_id", + "value": "" + }, + { + "name": "subject_id", + "value": "" + }, + { + "name": "source_file_id", + "value": "" + } + ], + "files": { + "file_list": [ + "chunk.bin" + ], + "file_series": [ + "chunk.bin" + ] + }, + "lightsheetZarrLevel": { + "label": "", + "level": 0, + "reduction_function": "none", + "axes_order": "tczyx", + "shape": [], + "chunks": [], + "chunk_grid": [], + "n_chunks_stored": 0, + "chunk_index_origin": 1, + "chunk_order": "C", + "dtype": "", + "fill_value": 0, + "byte_order": "little", + "codec": "raw", + "codec_params": "{}", + "voxel_size": [], + "voxel_size_units": "micrometer", + "translation": [] + } +} diff --git a/src/ndi/ndi_common/database_documents/data/lightsheetZarrLevel.md b/src/ndi/ndi_common/database_documents/data/lightsheetZarrLevel.md new file mode 100644 index 0000000000..f14e93b4fe --- /dev/null +++ b/src/ndi/ndi_common/database_documents/data/lightsheetZarrLevel.md @@ -0,0 +1,84 @@ +# lightsheetZarrLevel + +One resolution level of a `lightsheetZarrPyramid`. Owns the chunk bytes +for that level as a `chunk.bin_#` file series, plus enough metadata for +a reader to reconstruct the level as a lazy multiscale array. + +NDI documents are immutable, so each level is a document of its own. +Adding a new reduction (a `max` pyramid alongside a mean one, for +example) means writing more `lightsheetZarrLevel` documents — no parent +edit, no rewrite. The parent describes the source volume; the ladder is +discovered from its children. + +Level 0 is often shared across reductions: two NGFF multiscales entries +(mean and max) whose `datasets[0].path` is the same string are the same +underlying array. This document class represents that honestly — one +level doc with `reduction_function: 'none'` — instead of writing the +raw bytes twice. + +## What this document is + +- **The chunk container.** The `files.file_list` names one file series + `chunk.bin_#`, one file per stored chunk. Chunk index numbering is + 1-based to match NDI's file-series convention, and the mapping from + the file's 1-based number to the C-order chunk coordinate is defined + by `chunk_grid` and `chunk_order`. +- **Level-local metadata.** `shape`, `chunks`, `voxel_size`, and + `translation` describe this level as it sits inside the parent's + coordinate system. A viewer that wants only one level does not have + to touch the parent's fields. + +## Fields + +- `label` — human-readable name for the level (e.g. + `"mean level 2 (12.8um)"`). Empty allowed. +- `level` — 0-based level index within the parent pyramid. 0 is the + finest. +- `reduction_function` — the function used to produce this level from + level 0: `'mean'`, `'max'`, ... `'none'` means the array is the raw + source (typical for level 0 when it is shared across reductions). A + reader asking for reduction `R` takes levels with + `reduction_function in {'none', R}`, sorted by `level` ascending. +- `axes_order` — copied from the parent; carried here for + self-describing reads. +- `shape` — per-axis size of this level's array, in `axes_order`. +- `chunks` — per-axis chunk size, in `axes_order`. This is the NGFF / + Zarr chunk shape. +- `chunk_grid` — per-axis count of chunks along each axis = + `ceil(shape ./ chunks)`. +- `n_chunks_stored` — number of `chunk.bin_#` files that were written. + May be less than `prod(chunk_grid)` when trailing empty (fill-value) + chunks were elided. +- `chunk_index_origin` — the first `#` in the `chunk.bin_#` series. + Always 1 by convention (NDI file series are 1-based). +- `chunk_order` — the order in which the chunk grid is flattened to + the 1-based file series. `"C"` for C-order (last axis fastest); + `"F"` for Fortran-order. +- `dtype` — the Zarr dtype string (e.g. `) is not a file of +% this dataset" +% while getFileDetails(datasetId, ) for the same uid returns 200 OK +% right before and right after. Two server code paths disagree on +% whether the manifest is a file of the dataset. The failing dataset has +% 121,441 members; the existing 15-file lightsheet fixture does NOT +% trigger it. This test walks N up in a bisectable ramp so the failure +% (if reproducible from a fresh upload at all) is bracketed. +% +% What each row does, per N: +% a) Build a synthetic OME-Zarr blob whose level-0 chunk.bin series +% has approximately N members (via ndi.test.lightsheet.makeBlobFixture), +% ingest it into a fresh session and dataset, upload with +% uploadAsNew, wait for server-side bulk-upload extraction. +% b) Extract the manifest UID locally from the level document's +% chunk.bin binary path (ndi.database.internal.list_binary_files +% does the same thing at upload time). +% c) Call createSignedURLSetJob(datasetId, docId, 'ndi', 'fileSeries', +% 'chunk.bin') and waitForSignedURLSetJob to a terminal state. +% d) INDEPENDENTLY call getFileDetails(datasetId, manifestUid); a +% 200 here in the same instant createSignedURLSetJob says "not a +% file of this dataset" is the disagreement we are bisecting. +% e) Record: N (intended), actualMemberCount, jobId, jobState, +% jobElapsedSec, mapSize (ready) or errorMessage (failed), +% manifestFetchable, datasetId. +% +% Skips cleanly without NDI_CLOUD_USERNAME / NDI_CLOUD_PASSWORD (same +% idiom as FileSeriesRoundTripTest). Every dataset the test creates is +% deleted on teardown so the test account does not accumulate cruft. +% +% What we want the run to tell us: +% * If a size triggers the bug: this is a reproducer created by a +% fresh upload using only test-account state -- attach the results +% to the bug report and the API team can bisect on N. +% * If no size triggers it: the failing dataset's issue is +% upload-path-specific (workflow / MATLAB version / ingest history), +% not size-specific. Compare its provenance to what this test does +% to find the drift. +% +% Either outcome is actionable. Once the server bug is fixed the test +% becomes a permanent regression guard. +% +% See VH-Lab/NDI-matlab#1010 addendum. Related Python analog: +% Waltham-Data-Science/NDI-python 5ab9c04. + + properties (Constant) + DatasetNamePrefix = 'NDI_UNITTEST_SIGNED_URL_SCALING_'; + % chunk.bin_# member ≈ (fixed) uint16 bytes per spatial chunk. + % Shape is chosen at runtime per N so total spatial chunks ≈ N + % for a single-channel, single-level blob store. See setupPer + % method. + BaseChunkShape = [16 16 16]; + end + + properties (TestParameter) + % Intended member count. The actual count comes back from the + % local level document after ingest and is what the table + % reports -- Shape/ChunkShape only aims for the target. + % + % A run with credentials but a small time budget can restrict + % via the MATLAB unittest -Selector infrastructure or just let + % the small sizes run and Ctrl-C before the big one. + % + % Size regimes and their purpose: + % N5, N100, N500 -- exercise the plumbing (upload, + % extract, signed-URL job) at + % fixture-scale where the API path + % used to time out on its own. + % N2000, N10000 -- push into the "signing rate + % matters" regime where the earlier + % fix (see NDI-matlab#1010 / + % ndi-cloud-node#149) moved the + % floor from ~24 URLs/s. + % N50000, N100000 -- exercise the >100k regime that + % real lightsheet levels live in + % (a level of a 121,441-URL pyramid + % motivated Waltham-Data-Science/ + % NDI-python#320 / ndi-cloud-node + % #151). Currently gated behind + % NDI_SCALINGPROBE_HUGE=1 so the + % default CI run does not swallow a + % 30+ minute upload per row: the + % size is what proves the fix lands, + % but the upload is a cloud-budget + % hit until then. Flip the gate off + % (i.e. remove the assumeTrue below) + % once ndi-cloud-node#151 is fixed + % AND the upload path is fast + % enough that CI can afford these + % rows on every push. + targetMemberCount = struct( ... + 'N5', 5, ... + 'N100', 100, ... + 'N500', 500, ... + 'N2000', 2000, ... + 'N10000', 10000, ... + 'N50000', 50000, ... + 'N100000', 100000); + end + + properties + WorkDir char = '' + DatasetIDs string = string.empty % for class-level cleanup + Results struct = struct( ... + 'N', {}, ... + 'actualMemberCount',{}, ... + 'jobId', {}, ... + 'jobState', {}, ... + 'jobElapsedSec', {}, ... + 'mapSize', {}, ... + 'errorMessage', {}, ... + 'manifestFetchable',{}, ... + 'manifestUid', {}, ... + 'datasetId', {}) + end + + methods (TestClassSetup) + function checkCredentials(testCase) + username = getenv("NDI_CLOUD_USERNAME"); + password = getenv("NDI_CLOUD_PASSWORD"); + diagMsg = ['Missing NDI Cloud credentials ' ... + '(NDI_CLOUD_USERNAME/NDI_CLOUD_PASSWORD). ' ... + 'Skipping TestSignedUrlSetJobScaling.']; + testCase.assumeNotEmpty(username, diagMsg); + testCase.assumeNotEmpty(password, diagMsg); + end + + function checkLightsheetToolchain(testCase) + % A run without the lightsheet builder skips rather than + % errors: the fixture path leans on it heavily and the + % failure mode we are hunting is server-side, not local. + % which() is the reliable probe for a package function's + % presence; exist(..., 'file') does not always resolve + % dotted package names across MATLAB releases. + testCase.assumeNotEmpty(... + which('ndi.test.lightsheet.makeBlobFixture'), ... + ['ndi.test.lightsheet.makeBlobFixture is not on the ' ... + 'path. This test needs the lightsheet fixture builder.']); + testCase.assumeNotEmpty(... + which('ndi.fun.doc.lightsheet.fromOMEZarr'), ... + ['ndi.fun.doc.lightsheet.fromOMEZarr is not on the ' ... + 'path. This test needs the OME-Zarr -> pyramid+level ' ... + 'ingest.']); + end + end + + methods (TestClassTeardown) + function cleanupUploadedDatasets(testCase) + % Belt-and-braces: every method's own teardown already tries + % to delete its dataset, but a mid-method crash may leave + % one behind. Sweep at class teardown too, silently. + for i = 1:numel(testCase.DatasetIDs) + did = testCase.DatasetIDs(i); + if strlength(did) == 0, continue, end + try + ndi.cloud.api.datasets.deleteDataset( ... + char(did), 'when', 'now'); %#ok + catch + end + end + end + end + + methods (TestMethodSetup) + function setupWorkFolder(testCase) + import matlab.unittest.fixtures.TemporaryFolderFixture + fx = testCase.applyFixture(TemporaryFolderFixture); + testCase.WorkDir = fx.Folder; + end + end + + methods (Test, ParameterCombination = 'sequential') + + function testCreateSignedURLSetJobForNMemberSeries(testCase, targetMemberCount) + % Runs one (build + upload + createSignedURLSetJob) round and + % records the outcome. Assertions are DELIBERATELY LOOSE: a + % failure is a DATA POINT, not a red test. The test only fails + % if setup / upload / local extraction goes wrong, since those + % would poison the row and mask the server-side outcome we are + % here to observe. The row is what the reader reads. + + % Huge-N gate. 50k / 100k rows upload ~50-100k files each, + % which currently takes tens of minutes and burns cloud + % budget on every CI push; skip them unless + % NDI_SCALINGPROBE_HUGE=1 is explicitly set. Retire this + % gate once ndi-cloud-node#151 is fixed and CI can afford + % the upload cost on every run. + if targetMemberCount >= 50000 + hugeGate = getenv('NDI_SCALINGPROBE_HUGE'); + testCase.assumeTrue(strcmp(hugeGate, '1'), ... + sprintf(['N=%d skipped by default: set ' ... + 'NDI_SCALINGPROBE_HUGE=1 to run the ' ... + '>=50k rows. See the comment on ' ... + 'targetMemberCount for the rationale.'], ... + targetMemberCount)); + end + + row = testCase.freshRow(targetMemberCount); + + % --- 1. Build the fixture at the size that yields ~N members + [shape, chunkShape] = pickShapeForTarget(targetMemberCount, ... + testCase.BaseChunkShape); + zarrParent = fullfile(testCase.WorkDir, 'zarr'); + mkdir(zarrParent); + [zarrPath, ~] = ndi.test.lightsheet.makeBlobFixture(zarrParent, ... + 'Shape', shape, ... + 'NumChannels', 1, ... + 'NumLevels', 1, ... + 'ChunkShape', chunkShape); + + % --- 2. Ingest into a fresh session + dataset + sessionDir = fullfile(testCase.WorkDir, 'session'); + mkdir(sessionDir); + S = ndi.session.dir('scaleprobe', sessionDir); + subject = ndi.document('subject', ... + 'base.session_id', S.id(), ... + 'subject.local_identifier', 'scaleprobe@vhlab'); + S.database_add(subject); + + % Pass ``chunks`` through explicitly so makePyramid does + % NOT fall through to chooseTileShape's default 8 MB + % target. Without this the [16 16 16] chunk shape + % pickShapeForTarget hands makeBlobFixture gets rechunked + % downstream to a much larger tile (observed empirically + % as [161 161 161], ~4 MB per chunk), so a "target 100000 + % members" fixture only produces ~125 members and the + % scaling axis this test exists to sweep is silently + % collapsed to a constant. Handing the same chunkShape to + % fromOMEZarr keeps the level-0 chunk shape at what the + % fixture wrote, so ceil(shape ./ chunkShape) member count + % actually matches the target. + [~, levelDocs, ~] = ndi.fun.doc.lightsheet.fromOMEZarr( ... + S, zarrPath, ... + 'subjectID', subject.id(), ... + 'materializeChunks', true, ... + 'codec', 'raw', ... + 'chunks', chunkShape); + testCase.assertNotEmpty(levelDocs, ... + 'fromOMEZarr produced no level documents; bail before the cloud call.'); + + % The bug is triggered by ONE chunk.bin series scope on ONE + % level document, so pick level 0 (highest resolution -> most + % members) and use it for the rest of the run. + levelDoc = firstLevelDoc(levelDocs); + + % --- 3. Wrap in a dataset and upload + datasetDir = fullfile(testCase.WorkDir, 'dataset'); + mkdir(datasetDir); + D = ndi.dataset.dir('scaleprobe_ds', datasetDir); + D = D.add_ingested_session(S); + + uniqueName = char(sprintf('%s%s', ... + testCase.DatasetNamePrefix, did.ido.unique_id())); + [ok, cloudDatasetId, msg] = ndi.cloud.uploadDataset(D, ... + 'uploadAsNew', true, ... + 'skipMetadataEditorMetadata', true, ... + 'remoteDatasetName', uniqueName); + testCase.assertTrue(ok, ... + sprintf('uploadDataset failed for N=%d: %s', ... + targetMemberCount, char(string(msg)))); + testCase.DatasetIDs(end+1) = string(cloudDatasetId); + row.datasetId = string(cloudDatasetId); + % Register per-method cleanup so a failed run still tears + % this dataset down. + testCase.addTeardown(@() safeDeleteDataset(cloudDatasetId)); + + % --- 4. Wait for the server to finish extracting the bulk zip. + % Size-adaptive timeout: waitForAllBulkUploads' default 300 s + % is fine at fixture scale but the extract time swings + % wildly with server load -- on b0dcce11 N=2000 finished + % in 22 s while on e3d5f51c the same N sat past 300 s. + % Rather than chase that variance with an ever-larger + % ceiling (and fail CI whenever the server is warm), + % observe the outcome and press on: the probe's job is to + % record server behaviour at scale, and "bulk extract did + % not complete in s" is one such observation. If the + % extract times out we cannot call createSignedURLSetJob + % (files are not there yet), so the row records + % state='bulkTimeout' and the next N runs. This matches + % the philosophy stated at the failed-job branch below: + % the probe does not fail on things the server does; it + % records them. + bulkWaitSec = max(300, ceil(targetMemberCount / 20)); + [waitOk, waitInfo] = ndi.cloud.api.files.waitForAllBulkUploads( ... + cloudDatasetId, 'timeout', bulkWaitSec); + if ~waitOk + row.jobState = "bulkTimeout"; + row.errorMessage = sprintf( ... + 'waitForAllBulkUploads did not confirm completion (state=%s, elapsed=%.1fs, timeout=%ds).', ... + char(string(waitInfo.state)), waitInfo.elapsed, bulkWaitSec); + row.jobElapsedSec = waitInfo.elapsed; + testCase.Results(end+1) = row; + printRow(row); + return + end + + % --- 5. Extract the manifest uid from the LOCAL level doc + [manifestExists, manifestLocalPath] = ... + D.database_existbinarydoc(levelDoc.id(), 'chunk.bin'); + testCase.assertTrue(manifestExists, ... + sprintf('chunk.bin manifest for level doc %s was not found locally.', ... + levelDoc.id())); + [~, manifestUid, ~] = fileparts(char(manifestLocalPath)); + row.manifestUid = string(manifestUid); + + % Actual member count from the local manifest -- what the + % table reports so N vs. reality is legible. + try + localManifest = did.file.readSeriesManifest(char(manifestLocalPath)); + row.actualMemberCount = localManifest.count; + catch + row.actualMemberCount = -1; + end + + % Regression guard: an actual count far below target means + % the fixture builder is silently rechunking (as it did + % before we started passing ``chunks`` through to + % fromOMEZarr), so the scaling axis is not being exercised + % at the intended size and every row degenerates into a + % test of the small-N case. Warn loudly so a future + % regression is visible in the run log rather than only + % legible by squinting at the [scaleprobe] rows. + if targetMemberCount >= 100 && row.actualMemberCount > 0 && ... + row.actualMemberCount < 0.25 * targetMemberCount + warning('NDI:test:SignedUrlSetJobScaling:FixtureCollapsed', ... + ['N=%d requested but fixture produced only %d members ' ... + '(< 25%%). The scaling axis is not being exercised as ' ... + 'intended -- check that fromOMEZarr is honoring the ' ... + 'requested chunk shape and not falling through to ' ... + 'chooseTileShape''s default byte budget.'], ... + targetMemberCount, row.actualMemberCount); + end + + % --- 6. createSignedURLSetJob + wait + ndiDocumentId = levelDoc.id(); + tStart = tic; + [createOk, createAnswer] = ndi.cloud.api.files.createSignedURLSetJob( ... + string(cloudDatasetId), string(ndiDocumentId), ... + 'idNamespace', "ndi", ... + 'fileSeries', "chunk.bin"); + if ~createOk + row.jobState = "createRejected"; + row.jobId = ""; + row.errorMessage = extractMessage(createAnswer); + else + row.jobId = string(createAnswer.jobId); + % Size-adaptive timeout. At the observed per-URL rate the + % small sizes finish well inside 300 s; the >=50k rows + % need a longer ceiling to reach 'ready' at all once the + % server-side fix lands. A generous ceiling costs + % nothing when the fix is fast (the wait returns as + % soon as the job is ready) and lets the row report + % the actual wall time rather than "timed out". + jobTimeoutSec = max(300, ceil(targetMemberCount / 100)); + [readyOk, jobAnswer] = ndi.cloud.api.files.waitForSignedURLSetJob( ... + row.jobId, 'timeout', jobTimeoutSec); + if readyOk + row.jobState = "ready"; + if isfield(jobAnswer, 'fileCount') && ~isempty(jobAnswer.fileCount) + row.mapSize = double(jobAnswer.fileCount); + end + row.errorMessage = ""; + else + row.jobState = string(safeField(jobAnswer, 'state', 'failed')); + row.errorMessage = extractMessage(jobAnswer); + end + end + row.jobElapsedSec = toc(tStart); + + % --- 7. Independent getFileDetails for the same manifest uid + [detailsOk, ~] = ndi.cloud.api.files.getFileDetails( ... + string(cloudDatasetId), row.manifestUid); + row.manifestFetchable = detailsOk; + + testCase.Results(end+1) = row; + printRow(row); + + % The test itself does not fail on a job failure -- that IS + % the observation the run is here to make. Only a wildly + % inconsistent state fails: the manifest is not fetchable AND + % the job also failed, which contradicts the reported cloud + % bug (job fails while manifest fetch works). That signals + % the bug reproduction went sideways, not that the bug + % reproduced. + if strcmp(row.jobState, "failed") && ~row.manifestFetchable + testCase.verifyTrue(false, ... + sprintf(['Both the job and the direct manifest ' ... + 'fetch failed for uid %s (N=%d, actual=%d). ' ... + 'The reported bug is a DISAGREEMENT between those ' ... + 'two -- this row is not the disagreement, so ' ... + 'something else broke setup.'], ... + char(row.manifestUid), targetMemberCount, ... + row.actualMemberCount)); + end + end + + end + + methods (Access = private) + function row = freshRow(testCase, N) %#ok + row = struct( ... + 'N', N, ... + 'actualMemberCount', 0, ... + 'jobId', "", ... + 'jobState', "unstarted", ... + 'jobElapsedSec', 0, ... + 'mapSize', NaN, ... + 'errorMessage', "", ... + 'manifestFetchable', false, ... + 'manifestUid', "", ... + 'datasetId', ""); + end + end +end + +function [shape, chunkShape] = pickShapeForTarget(target, baseChunkShape) + % Aim for level-0 spatial chunks ≈ target. NumChannels=1, + % NumLevels=1 in the fixture builder, so total series members + % ≈ prod(ceil(shape ./ chunkShape)). + % + % Factor target as k*k*(target/k^2) with k = round(target^(1/3)) + % so shapes stay roughly cubic. A cubic-ish layout matters because + % the lightsheet levels the failing dataset carries are cubic-ish + % too; a stringy 1xN layout could miss a shape-dependent bug the + % server has. + chunkShape = baseChunkShape; + if target <= 1 + chunks = [1 1 1]; + else + k = max(1, round(target^(1/3))); + c1 = k; + c2 = k; + c3 = max(1, ceil(target / (c1*c2))); + chunks = [c1 c2 c3]; + end + shape = chunks .* chunkShape; +end + +function d = firstLevelDoc(levelDocs) + % levelDocs can be a cell array or a struct-of-arrays or a single + % doc depending on the fromOMEZarr version. Normalize to a scalar + % ndi.document. + if iscell(levelDocs) + d = levelDocs{1}; + elseif numel(levelDocs) > 1 + d = levelDocs(1); + else + d = levelDocs; + end +end + +function printRow(r) + % One row per N to the MATLAB console. Printed immediately (not + % held for a class-teardown summary) because parametrized test + % cases run in fresh test instances -- rows do not accumulate on + % one testCase across parameter values, so the log itself IS the + % table. Grep for [scaleprobe] in a CI run to pull the rows out. + detail = char(r.errorMessage); + if numel(detail) > 60, detail = [detail(1:57) '...']; end + fprintf(['[scaleprobe] N=%-6d actual=%-6d state=%-10s ' ... + 'elapsed=%6.1fs mapSize=%-8s manifestFetchable=%d ' ... + 'manifestUid=%s dataset=%s jobId=%s detail="%s"\n'], ... + r.N, r.actualMemberCount, char(r.jobState), r.jobElapsedSec, ... + mapSizeAsText(r.mapSize), r.manifestFetchable, ... + char(r.manifestUid), char(r.datasetId), char(r.jobId), detail); +end + +function s = mapSizeAsText(value) + if isnumeric(value) && isscalar(value) && ~isnan(value) + s = sprintf('%d', round(value)); + else + s = '-'; + end +end + +function m = extractMessage(payload) + % Server-side error payloads travel in a few shapes. Try the common + % ones so the table row carries whatever the server said, verbatim. + m = ""; + if isstruct(payload) + if isfield(payload, 'message') && ~isempty(payload.message) + m = string(payload.message); + elseif isfield(payload, 'error') && ~isempty(payload.error) + m = string(payload.error); + elseif isfield(payload, 'state') && ~isempty(payload.state) + m = string(payload.state); + end + elseif ischar(payload) || isstring(payload) + m = string(payload); + end +end + +function v = safeField(s, name, defaultValue) + if isstruct(s) && isfield(s, name) && ~isempty(s.(name)) + v = s.(name); + else + v = defaultValue; + end +end + +function safeDeleteDataset(cloudDatasetId) + try + ndi.cloud.api.datasets.deleteDataset(char(cloudDatasetId), ... + 'when', 'now'); %#ok + catch + end +end diff --git a/tests/+ndi/+unittest/+cloud/+sync/TestFetchManifest.m b/tests/+ndi/+unittest/+cloud/+sync/TestFetchManifest.m new file mode 100644 index 0000000000..ad84cb6a86 --- /dev/null +++ b/tests/+ndi/+unittest/+cloud/+sync/TestFetchManifest.m @@ -0,0 +1,178 @@ +classdef TestFetchManifest < matlab.unittest.TestCase +% TESTFETCHMANIFEST - manifest fetch bypasses the batch cache. +% +% A manifest is a SINGLE KNOWN uid, so its URL is resolved by ONE direct +% getFileDetails call. Going through the per-document +% batchSignedUrlLookup here would force the batch to walk the whole +% document's signed-URL set -- ~243 pages at ~25 s each for a 121k-chunk +% lightsheet level (~100 min per manifest, seven levels ≈ 12 h before any +% bytes hit disk). The batch is worth its cost when many uids share the +% document; for a single known uid it is pure overhead. See +% VH-Lab/NDI-matlab#1010 and Waltham-Data-Science/NDI-python commit +% 5ab9c04 (the Python analog fix). + + properties + WorkDir + end + + methods (TestMethodSetup) + function useWorkingFolder(testCase) + testCase.applyFixture(matlab.unittest.fixtures.WorkingFolderFixture); + testCase.WorkDir = pwd; + % Clear the batch cache so a prior test's signer calls + % cannot show up in this test's counters. Reading the + % counters back with an empty documentId does not disturb + % the cache. + ndi.cloud.download.internal.batchSignedUrlLookup( ... + "", "", "", "", 'clearCache', true); + end + end + + methods (Test) + + function testManifestFetchGoesDirectAndSkipsTheBatch(testCase) + % The whole point: fetchManifest for a single known uid + % must issue exactly one getFileDetails call and must NOT + % call batchSignedUrlLookup at all. Before this fix the + % batch would answer that one-uid question by paging the + % whole document's signed-URL set -- a ~12 h hang on a + % lightsheet level's manifest (NDI-matlab#1010). + destPath = fullfile(testCase.WorkDir, 'manifest.bin'); + recorded = containers.Map('KeyType','char','ValueType','any'); + recorded('detailsCalls') = 0; + recorded('detailsArgs') = {}; + recorded('fileCalls') = 0; + recorded('fileArgs') = {}; + detailsFcn = @(datasetId, fileUid) localScriptedDetails( ... + recorded, datasetId, fileUid, 'https://s3.example/manifest'); + fileFcn = @(url, path, varargin) localScriptedFile( ... + recorded, url, path, varargin); + + ndi.cloud.sync.internal.fetchManifest( ... + string(destPath), "ds1", "manifest_uid_A", ... + "doc1", "chunkdata.bin", [], ... + 'DetailsFetcher', detailsFcn, ... + 'FileFetcher', fileFcn); + + testCase.verifyEqual(recorded('detailsCalls'), 1, ... + 'exactly one getFileDetails call for one manifest uid'); + args = recorded('detailsArgs'); + testCase.verifyEqual(char(args{1}), 'ds1'); + testCase.verifyEqual(char(args{2}), 'manifest_uid_A', ... + 'the direct call must carry the manifest uid'); + + testCase.verifyEqual(recorded('fileCalls'), 1, ... + 'the resolved URL must be fetched once'); + fargs = recorded('fileArgs'); + testCase.verifyEqual(char(fargs{1}), 'https://s3.example/manifest'); + testCase.verifyEqual(char(fargs{2}), char(destPath)); + + % The batch must not have been touched. Read the counters + % back with an empty documentId -- that path returns early + % without incrementing anything, so it is safe to call as a + % pure read. + [~, batchStats] = ndi.cloud.download.internal.batchSignedUrlLookup( ... + "", "", "", ""); + testCase.verifyEqual(batchStats.signerCalls, 0, ... + 'batchSignedUrlLookup must NOT be reached on a manifest fetch'); + testCase.verifyEqual(batchStats.uidHits, 0); + testCase.verifyEqual(batchStats.uidMisses, 0); + end + + function testManifestFetchGoesThroughHandlerWhenProvided(testCase) + % When a customFileHandler is provided (DID contract), the + % fetch is dispatched through it and neither getFileDetails + % nor getFile is called. This preserves the existing DID + % #201 shape and is what the offline manifest-resolution + % test in TestSeriesWithCloudOnlyManifestResolvesThroughHandler + % exercises at the integration layer. Asserted here at the + % unit layer so a regression in the branch order shows up + % without spinning up the full DID stack. + destPath = fullfile(testCase.WorkDir, 'manifest.bin'); + recorded = containers.Map('KeyType','char','ValueType','any'); + recorded('detailsCalls') = 0; + recorded('fileCalls') = 0; + recorded('handlerCalls') = 0; + recorded('handlerCtx') = {}; + + detailsFcn = @(varargin) localFailUnexpectedCall(recorded, 'detailsCalls'); %#ok + fileFcn = @(varargin) localFailUnexpectedCall(recorded, 'fileCalls'); %#ok + + handler = @(dest, src, ctx) localRecordHandlerCall( ... + recorded, dest, src, ctx); + + ndi.cloud.sync.internal.fetchManifest( ... + string(destPath), "ds1", "manifest_uid_B", ... + "docB", "chunkdata.bin", handler); + + testCase.verifyEqual(recorded('handlerCalls'), 1, ... + 'the handler path must fire when a handler is supplied'); + testCase.verifyEqual(recorded('detailsCalls'), 0, ... + 'getFileDetails must not be called on the handler path'); + testCase.verifyEqual(recorded('fileCalls'), 0, ... + 'getFile must not be called on the handler path'); + + ctx = recorded('handlerCtx'); + testCase.verifyEqual(char(ctx.uid), 'manifest_uid_B'); + testCase.verifyEqual(char(ctx.filename), 'chunkdata.bin'); + testCase.verifyEqual(char(ctx.seriesName), '', ... + 'the manifest context must carry seriesName="" so the handler treats uid as the manifest itself, not a member'); + testCase.verifyEqual(char(ctx.mode), 'open'); + end + + function testDetailsFetcherFailureIsReported(testCase) + % Preserve the pre-existing error shape: a getFileDetails + % that says ok=false raises NDI:cloud:sync:ManifestFetchFailed + % with the manifest uid in the message. Callers upstream + % (updateFileInfoForRemoteFiles) rely on catching this to + % leave the entry un-reconstructed so DID#185's guard fires + % on add_docs. + destPath = fullfile(testCase.WorkDir, 'manifest.bin'); + failingFcn = @(varargin) deal(false, ... + struct('message','server said no')); + + testCase.verifyError(@() ndi.cloud.sync.internal.fetchManifest( ... + string(destPath), "ds1", "manifest_uid_C", ... + "docC", "chunkdata.bin", [], ... + 'DetailsFetcher', failingFcn), ... + 'NDI:cloud:sync:ManifestFetchFailed'); + end + + end +end + +function [ok, answer] = localScriptedDetails(recorded, datasetId, fileUid, url) + recorded('detailsCalls') = recorded('detailsCalls') + 1; + recorded('detailsArgs') = {datasetId, fileUid}; + ok = true; + answer = struct('downloadUrl', url); +end + +function [ok, msg] = localScriptedFile(recorded, url, path, varargin) + recorded('fileCalls') = recorded('fileCalls') + 1; + recorded('fileArgs') = {url, path, varargin}; + ok = true; + msg = 'ok'; +end + +function [ok, msg] = localFailUnexpectedCall(recorded, key) + % A stub that any test using the handler path expects NOT to be + % called. Record it so the assertion fires with a clear counter, + % rather than raising here (which would leave the test in a less + % informative state). + recorded(key) = recorded(key) + 1; + ok = false; + msg = 'unexpected call'; +end + +function localRecordHandlerCall(recorded, dest, src, ctx) %#ok + recorded('handlerCalls') = recorded('handlerCalls') + 1; + recorded('handlerCtx') = ctx; + % Write a tiny placeholder so the caller sees the handler landed + % something at destPath. + fid = fopen(char(dest), 'w'); + if fid > 0 + fwrite(fid, uint8(0), 'uint8'); + fclose(fid); + end +end diff --git a/tests/+ndi/+unittest/+cloud/AuthTest.m b/tests/+ndi/+unittest/+cloud/AuthTest.m index 55bf7c640a..b9a43b4fca 100644 --- a/tests/+ndi/+unittest/+cloud/AuthTest.m +++ b/tests/+ndi/+unittest/+cloud/AuthTest.m @@ -59,7 +59,13 @@ function testLoginLogout(testCase) narrative(end+1) = "Login call successful. Verifying token and organization ID."; testCase.verifyClass(answer_login.token, 'char', login_message); - testCase.verifyClass(answer_login.user.organizations.id, 'char', login_message); + % `organizations` may be a struct array on a multi-org + % account -- `answer_login.user.organizations.id` then + % expands to a comma-separated list and verifyClass reads + % it as extra positional args (MATLAB:narginchk: + % tooManyInputs). ndi.cloud.authenticate already picks the + % first (extractFirstOrganizationId); mirror that here. + testCase.verifyClass(answer_login.user.organizations(1).id, 'char', login_message); narrative(end+1) = "Token and organization ID are of the correct class."; % Step 3: Test Logout diff --git a/tests/+ndi/+unittest/+cloud/DecodeJwtTest.m b/tests/+ndi/+unittest/+cloud/DecodeJwtTest.m new file mode 100644 index 0000000000..51c4a11975 --- /dev/null +++ b/tests/+ndi/+unittest/+cloud/DecodeJwtTest.m @@ -0,0 +1,104 @@ +classdef DecodeJwtTest < matlab.unittest.TestCase + % DECODEJWTTEST - offline tests for ndi.cloud.internal.decodeJwt + % + % decodeJwt used javax.xml.bind.DatatypeConverter to Base64-decode the JWT + % payload. That class is gone from the Java shipped with recent MATLAB + % releases, so every cloud login failed in getTokenExpiration. It now uses + % matlab.net.base64decode. These tests build a JWT locally from a known + % JSON payload, so they need no network or credentials. + % + % Authored without a local MATLAB runtime; needs MATLAB to validate/run. + + methods (Test) + + function testPayloadWithoutPadding(testCase) + % Payload byte count is a multiple of 3: no '=' to restore. + payload = ndi.unittest.cloud.DecodeJwtTest.examplePayload(); + [token, payloadSegment] = ... + ndi.unittest.cloud.DecodeJwtTest.makeJwt(payload, 0); + testCase.verifyEqual(mod(numel(payloadSegment), 4), 0); + + decoded = ndi.cloud.internal.decodeJwt(token); + ndi.unittest.cloud.DecodeJwtTest.verifyPayload(testCase, decoded, payload); + end + + function testPayloadNeedingOnePadCharacter(testCase) + % Payload byte count mod 3 == 2: one '=' was stripped. + payload = ndi.unittest.cloud.DecodeJwtTest.examplePayload(); + [token, payloadSegment] = ... + ndi.unittest.cloud.DecodeJwtTest.makeJwt(payload, 2); + testCase.verifyEqual(mod(numel(payloadSegment), 4), 3); + + decoded = ndi.cloud.internal.decodeJwt(token); + ndi.unittest.cloud.DecodeJwtTest.verifyPayload(testCase, decoded, payload); + end + + function testPayloadNeedingTwoPadCharacters(testCase) + % Payload byte count mod 3 == 1: two '=' were stripped. + payload = ndi.unittest.cloud.DecodeJwtTest.examplePayload(); + [token, payloadSegment] = ... + ndi.unittest.cloud.DecodeJwtTest.makeJwt(payload, 1); + testCase.verifyEqual(mod(numel(payloadSegment), 4), 2); + + decoded = ndi.cloud.internal.decodeJwt(token); + ndi.unittest.cloud.DecodeJwtTest.verifyPayload(testCase, decoded, payload); + end + + function testReturnsTokenExpiration(testCase) + % The real caller: getTokenExpiration reads the 'exp' claim. + payload = ndi.unittest.cloud.DecodeJwtTest.examplePayload(); + token = ndi.unittest.cloud.DecodeJwtTest.makeJwt(payload, 0); + + decoded = ndi.cloud.internal.decodeJwt(token); + testCase.verifyTrue(isfield(decoded, 'exp')); + testCase.verifyEqual(decoded.exp, 1893456000); + end + + end + + methods (Static, Access = private) + + function payload = examplePayload() + % A JWT-like payload. The name includes non-ASCII characters + % (e-acute and the euro sign) to exercise the UTF-8 decode. + payload = struct( ... + 'exp', 1893456000, ... + 'email', 'test.user@example.com', ... + 'name', ['Ren', char(233), ' ', char(8364)], ... + 'nonce', ''); + end + + function [token, payloadSegment] = makeJwt(payload, targetMod3) + % Build HEADER.PAYLOAD.SIGNATURE with Base64Url segments. The + % 'nonce' field is lengthened until the UTF-8 payload byte count + % mod 3 equals targetMod3, which fixes how much padding the + % decoder has to restore. + payloadBytes = unicode2native(jsonencode(payload), 'UTF-8'); + while mod(numel(payloadBytes), 3) ~= targetMod3 + payload.nonce = [payload.nonce, 'x']; + payloadBytes = unicode2native(jsonencode(payload), 'UTF-8'); + end + + headerBytes = unicode2native('{"alg":"HS256","typ":"JWT"}', 'UTF-8'); + headerSegment = ndi.unittest.cloud.DecodeJwtTest.base64UrlEncode(headerBytes); + payloadSegment = ndi.unittest.cloud.DecodeJwtTest.base64UrlEncode(payloadBytes); + token = [headerSegment, '.', payloadSegment, '.', 'dummySignature']; + end + + function encoded = base64UrlEncode(bytes) + encoded = char(matlab.net.base64encode(uint8(bytes))); + encoded = strrep(encoded, '+', '-'); + encoded = strrep(encoded, '/', '_'); + encoded = strrep(encoded, '=', ''); + end + + function verifyPayload(testCase, decoded, payload) + testCase.verifyClass(decoded, 'struct'); + testCase.verifyEqual(decoded.exp, payload.exp); + testCase.verifyEqual(decoded.email, payload.email); + testCase.verifyEqual(decoded.name, payload.name); + end + + end + +end diff --git a/tests/+ndi/+unittest/+cloud/FileSeriesRoundTripTest.m b/tests/+ndi/+unittest/+cloud/FileSeriesRoundTripTest.m index 439ae59e9a..ea79b4b0c7 100644 --- a/tests/+ndi/+unittest/+cloud/FileSeriesRoundTripTest.m +++ b/tests/+ndi/+unittest/+cloud/FileSeriesRoundTripTest.m @@ -596,20 +596,23 @@ function testMembersSurviveADownloadFromTheCloud(testCase) "still arrived, which is why every other assertion here passes, " + ... "but the batch presign path did not answer -- and at 28,000 " + ... "members that fallback is 28,000 API calls. " + msg); - % One presign call for the scope, plus at most one retry if - % the first answer was a partial map (VH-Lab/NDI-matlab#991 - % / Waltham-Data-Science/NDI-python#309 -- some environments - % lag briefly after the bulk upload). Anything beyond that - % would be per-uid fallback, which the uidMisses assertion - % above already rules out; this is just for a clearer - % failure message. + % One presign call for the scope, plus up to N partial-map + % retries where N is the length of the default retry schedule + % (VH-Lab/NDI-matlab#991 / Waltham-Data-Science/NDI-python#309 + % and #320 -- User 1 prod can lag long enough after + % ``waitForAllBulkUploads`` that a single retry is not + % enough, so the default schedule is now 1 s / 3 s / 9 s). + % Anything beyond that would be per-uid fallback, which the + % uidMisses assertion above already rules out; this is just + % for a clearer failure message. + defaultRetryWaves = 3; testCase.verifyEqual(batchStats.signerCalls, 1 + batchStats.partialMapRetries, ... "the members of one series should cost ONE presign call, plus " + ... - "at most one retry if the first map was partial. Got " + ... + "at most one signer call per partial-map wave. Got " + ... "signerCalls=" + batchStats.signerCalls + ", partialMapRetries=" + ... batchStats.partialMapRetries + ". " + msg); - testCase.verifyLessThanOrEqual(batchStats.partialMapRetries, 1, ... - "the retry is bounded to once per scope; got partialMapRetries=" + ... + testCase.verifyLessThanOrEqual(batchStats.partialMapRetries, defaultRetryWaves, ... + "the retry is bounded by the delay schedule; got partialMapRetries=" + ... batchStats.partialMapRetries + ". " + msg); % The bytes, compared in full rather than by length. Setup gives @@ -920,6 +923,7 @@ function testTheBatchScopeIsTheSeriesAndNotTheWholeDocument(testCase) "not in the series, so the 'fileSeries' scope was not honored. " + msg); end + end end diff --git a/tests/+ndi/+unittest/+cloud/FilesTest.m b/tests/+ndi/+unittest/+cloud/FilesTest.m index f28989944e..8c4f10ec6b 100644 --- a/tests/+ndi/+unittest/+cloud/FilesTest.m +++ b/tests/+ndi/+unittest/+cloud/FilesTest.m @@ -48,6 +48,17 @@ function setupNewDataset(testCase) end end methods (Access = private) + function [localFilePath, fileUID] = makeLocalTestFile(testCase) + % A temporary file named by a fresh uid, containing TestFileContent. + tempFolder = testCase.applyFixture(matlab.unittest.fixtures.TemporaryFolderFixture); + fileUID = string(did.ido.unique_id()); + localFilePath = fullfile(tempFolder.Folder, fileUID); + fid = fopen(localFilePath, 'w'); + testCase.assertGreaterThan(fid, 0, "Could not create local test file."); + fprintf(fid, '%s', testCase.TestFileContent); + fclose(fid); + end + % Private helper for teardown, called by addTeardown. function deleteDatasetAfterTest(testCase) if testCase.KeepDataset @@ -114,7 +125,9 @@ function testSingleFileUploadAndDownloadUseCurl(testCase) testCase.verifyTrue(b_put, "File upload (PUT request) failed. " + msg_put); if ~b_put, return; end narrative(end+1) = "File uploaded successfully."; - pause(10); % Give server time to process the file + % The PUT returning 200 only means S3 has the object; wait for the + % server to record it (it can lag when other uploads are queued). + ndi.cloud.api.files.waitForFileUpload(testCase.DatasetID, fileUID); % Step 3.5: Verify the file appears in the dataset's file list narrative(end+1) = "Preparing to check dataset file list for the newly uploaded file."; [b_list, file_list, resp_list, url_list] = ndi.cloud.api.files.listFilesAll(testCase.DatasetID, 'checkForUpdates', true); @@ -203,7 +216,9 @@ function testSingleFileUploadAndDownloadUseMatlabNotCurl(testCase) testCase.verifyTrue(b_put, "File upload (PUT request) failed. " + msg_put); if ~b_put, return; end narrative(end+1) = "File uploaded successfully."; - pause(10); % Give server time to process the file + % The PUT returning 200 only means S3 has the object; wait for the + % server to record it (it can lag when other uploads are queued). + ndi.cloud.api.files.waitForFileUpload(testCase.DatasetID, fileUID); % Step 3.5: Verify the file appears in the dataset's file list narrative(end+1) = "Preparing to check dataset file list for the newly uploaded file."; [b_list, file_list, resp_list, url_list] = ndi.cloud.api.files.listFilesAll(testCase.DatasetID, 'checkForUpdates', true); @@ -289,7 +304,9 @@ function testSingleFileUploadAndDownloadUseCurlStop(testCase) testCase.verifyTrue(b_put, "File upload (PUT request) failed. " + msg_put); if ~b_put, return; end narrative(end+1) = "File uploaded successfully."; - pause(10); % Give server time to process the file + % The PUT returning 200 only means S3 has the object; wait for the + % server to record it (it can lag when other uploads are queued). + ndi.cloud.api.files.waitForFileUpload(testCase.DatasetID, fileUID); % Step 3.5: Verify the file appears in the dataset's file list narrative(end+1) = "Preparing to check dataset file list for the newly uploaded file."; [b_list, file_list, resp_list, url_list] = ndi.cloud.api.files.listFilesAll(testCase.DatasetID, 'checkForUpdates', true); @@ -395,7 +412,9 @@ function testSingleFileUploadAndDownloadUseCurlFileFieldTest(testCase) testCase.verifyTrue(b_put, "File upload (PUT request) failed. " + msg_put); if ~b_put, return; end narrative(end+1) = "File uploaded successfully."; - pause(10); % Give server time to process the file + % The PUT returning 200 only means S3 has the object; wait for the + % server to record it (it can lag when other uploads are queued). + ndi.cloud.api.files.waitForFileUpload(testCase.DatasetID, fileUID); % Step 3.5: Verify the file appears in the dataset's file list narrative(end+1) = "Preparing to check dataset file list for the newly uploaded file."; [b_list, file_list, resp_list, url_list] = ndi.cloud.api.files.listFilesAll(testCase.DatasetID, 'checkForUpdates', true); @@ -839,6 +858,203 @@ function testBulkUploadWithWaitForCompletion(testCase) testCase.Narrative = narrative; end + function testUploadWithChecksumRoundTrip(testCase) + % NDI Cloud #162: a checksum sent with the upload URL request is + % recorded as the file's crc32 and returned by getFileDetails, + % listFiles and lookupFiles. + testCase.Narrative = "Begin testUploadWithChecksumRoundTrip"; + narrative = testCase.Narrative; + + [localFilePath, fileUID] = testCase.makeLocalTestFile(); + localCrc = ndi.fun.file.crc32(localFilePath); + narrative(end+1) = "Local file " + fileUID + " has crc32 " + localCrc; + + [b_url, ans_url, resp_url, url_url] = ndi.cloud.api.files.getFileUploadURL(... + testCase.DatasetID, fileUID, 'checksumCRC32', localCrc); + msg_url = ndi.unittest.cloud.APIMessage(narrative, b_url, ans_url, resp_url, url_url); + testCase.verifyTrue(b_url, "Failed to get a checksummed upload URL. " + msg_url); + if ~b_url, testCase.Narrative = narrative; return; end + + [b_put, ans_put, resp_put, url_put] = ndi.cloud.api.files.putFiles(ans_url, localFilePath, ... + "useCurl", true, "checksumCRC32", localCrc); + msg_put = ndi.unittest.cloud.APIMessage(narrative, b_put, ans_put, resp_put, url_put); + testCase.verifyTrue(b_put, "Upload with a matching checksum failed. " + msg_put); + if ~b_put, testCase.Narrative = narrative; return; end + narrative(end+1) = "Uploaded with checksumCRC32."; + % The PUT returning 200 only means S3 has the object; wait for the + % server to record it (it can lag when other uploads are queued). + ndi.cloud.api.files.waitForFileUpload(testCase.DatasetID, fileUID); + + % getFileDetails + [b_d, ans_d, resp_d, url_d] = ndi.cloud.api.files.getFileDetails(testCase.DatasetID, fileUID); + msg_d = ndi.unittest.cloud.APIMessage(narrative, b_d, ans_d, resp_d, url_d); + testCase.verifyTrue(b_d, "getFileDetails failed. " + msg_d); + if b_d + testCase.verifyTrue(isfield(ans_d, 'crc32'), "getFileDetails has no crc32. " + msg_d); + if isfield(ans_d, 'crc32') + testCase.verifyEqual(char(ans_d.crc32), localCrc, "getFileDetails crc32 differs from the local file. " + msg_d); + end + end + + % listFiles (keyset page) + [b_l, ans_l, resp_l, url_l] = ndi.cloud.api.files.listFiles(testCase.DatasetID, 'limit', 10); + msg_l = ndi.unittest.cloud.APIMessage(narrative, b_l, ans_l, resp_l, url_l); + testCase.verifyTrue(b_l, "listFiles failed. " + msg_l); + if b_l + row = ans_l(strcmp({ans_l.uid}, char(fileUID))); + testCase.verifyNumElements(row, 1, "listFiles does not list the file. " + msg_l); + if isscalar(row) + testCase.verifyEqual(row.crc32, localCrc, "listFiles crc32 differs from the local file. " + msg_l); + end + end + + % lookupFiles, with one uid the dataset does not have + missingUID = string(did.ido.unique_id()); + [b_k, ans_k, resp_k, url_k] = ndi.cloud.api.files.lookupFiles(testCase.DatasetID, [fileUID missingUID]); + msg_k = ndi.unittest.cloud.APIMessage(narrative, b_k, ans_k, resp_k, url_k); + testCase.verifyTrue(b_k, "lookupFiles failed. " + msg_k); + if b_k + testCase.verifyNumElements(ans_k.files, 1, "lookupFiles should find exactly the uploaded file. " + msg_k); + if isscalar(ans_k.files) + testCase.verifyEqual(ans_k.files(1).uid, char(fileUID), msg_k); + testCase.verifyTrue(ans_k.files(1).uploaded, "lookupFiles: file not marked uploaded. " + msg_k); + testCase.verifyEqual(ans_k.files(1).size, double(strlength(testCase.TestFileContent)), ... + "lookupFiles size differs from the local file. " + msg_k); + testCase.verifyEqual(ans_k.files(1).crc32, localCrc, "lookupFiles crc32 differs from the local file. " + msg_k); + end + testCase.verifyEqual(ans_k.missing, missingUID, "lookupFiles should report the unknown uid as missing. " + msg_k); + end + + testCase.Narrative = narrative; + end + + function testUploadWithWrongChecksumIsRejected(testCase) + % NDI Cloud #162: the upload URL is signed with the checksum, so + % S3 refuses content that does not match it. + testCase.Narrative = "Begin testUploadWithWrongChecksumIsRejected"; + narrative = testCase.Narrative; + + [localFilePath, fileUID] = testCase.makeLocalTestFile(); + wrongCrc = ndi.fun.file.crc32ToBase64(0); + testCase.assertNotEqual(ndi.fun.file.crc32(localFilePath), wrongCrc); + + [b_url, ans_url, resp_url, url_url] = ndi.cloud.api.files.getFileUploadURL(... + testCase.DatasetID, fileUID, 'checksumCRC32', wrongCrc); + msg_url = ndi.unittest.cloud.APIMessage(narrative, b_url, ans_url, resp_url, url_url); + testCase.verifyTrue(b_url, "Failed to get a checksummed upload URL. " + msg_url); + if ~b_url, testCase.Narrative = narrative; return; end + + [b_put, ans_put, resp_put, url_put] = ndi.cloud.api.files.putFiles(ans_url, localFilePath, ... + "useCurl", true, "checksumCRC32", wrongCrc); + msg_put = ndi.unittest.cloud.APIMessage(narrative, b_put, ans_put, resp_put, url_put); + testCase.verifyFalse(b_put, "S3 accepted content whose CRC-32 does not match the signed checksum. " + msg_put); + + testCase.Narrative = narrative; + end + + function testUploadSingleFileRecordsChecksum(testCase) + % ndi.cloud.uploadSingleFile sends the checksum by default. + testCase.Narrative = "Begin testUploadSingleFileRecordsChecksum"; + narrative = testCase.Narrative; + + [localFilePath, fileUID] = testCase.makeLocalTestFile(); + [b_up, err_up] = ndi.cloud.uploadSingleFile(testCase.DatasetID, fileUID, localFilePath); + testCase.verifyTrue(b_up, "uploadSingleFile failed: " + string(err_up)); + if ~b_up, testCase.Narrative = narrative; return; end + % uploadSingleFile waits for the server to record the upload, so + % the file must already be marked uploaded here. + + [b_k, ans_k, resp_k, url_k] = ndi.cloud.api.files.lookupFiles(testCase.DatasetID, fileUID); + msg_k = ndi.unittest.cloud.APIMessage(narrative, b_k, ans_k, resp_k, url_k); + testCase.verifyTrue(b_k, "lookupFiles failed. " + msg_k); + if b_k + testCase.verifyNumElements(ans_k.files, 1, msg_k); + if isscalar(ans_k.files) + testCase.verifyTrue(ans_k.files(1).uploaded, ... + "uploadSingleFile returned before the server recorded the upload. " + msg_k); + testCase.verifyEqual(ans_k.files(1).crc32, ndi.fun.file.crc32(localFilePath), ... + "uploadSingleFile did not record the file's crc32. " + msg_k); + end + end + testCase.Narrative = narrative; + end + + function testLookupFilesRejectsAnInvalidBody(testCase) + % The server refuses more than 5000 uids; the client splits longer + % lists, so drive the server check with a raw request. + testCase.Narrative = "Begin testLookupFilesRejectsAnInvalidBody"; + + token = ndi.cloud.authenticate(); + apiURL = ndi.cloud.api.url('lookup_dataset_files', 'dataset_id', testCase.DatasetID); + headers = [matlab.net.http.HeaderField('accept','application/json'), ... + matlab.net.http.field.ContentTypeField(matlab.net.http.MediaType('application/json')), ... + matlab.net.http.HeaderField('Authorization', ['Bearer ' token])]; + body = matlab.net.http.MessageBody(struct('uids', "u" + string(1:5001))); + resp = send(matlab.net.http.RequestMessage(matlab.net.http.RequestMethod.POST, headers, body), apiURL); + testCase.verifyEqual(double(resp.StatusCode), 400, "5001 uids should be refused with 400."); + + body = matlab.net.http.MessageBody(struct('notUids', 1)); + resp = send(matlab.net.http.RequestMessage(matlab.net.http.RequestMethod.POST, headers, body), apiURL); + testCase.verifyEqual(double(resp.StatusCode), 400, "A body without uids should be refused with 400."); + end + + function testLookupFilesSplitsLongLists(testCase) + % More than one server call's worth of uids, none uploaded. + testCase.Narrative = "Begin testLookupFilesSplitsLongLists"; + + uids = "nouid" + string(1:5003); + [b, answer, resp, url] = ndi.cloud.api.files.lookupFiles(testCase.DatasetID, uids); + msg = ndi.unittest.cloud.APIMessage(testCase.Narrative, b, answer, resp, url); + testCase.verifyTrue(b, "lookupFiles over two calls failed. " + msg); + if b + testCase.verifyEmpty(answer.files, msg); + testCase.verifyEqual(sort(answer.missing), sort(uids), msg); + end + end + + function testWaitForFileUpload(testCase) + % waitForFileUpload returns once the server has recorded a + % single-file upload, and times out on a uid that never uploads. + testCase.Narrative = "Begin testWaitForFileUpload"; + narrative = testCase.Narrative; + + [localFilePath, fileUID] = testCase.makeLocalTestFile(); + [b_url, ans_url, resp_url, url_url] = ndi.cloud.api.files.getFileUploadURL(testCase.DatasetID, fileUID); + msg_url = ndi.unittest.cloud.APIMessage(narrative, b_url, ans_url, resp_url, url_url); + testCase.verifyTrue(b_url, "Failed to get file upload URL. " + msg_url); + if ~b_url, testCase.Narrative = narrative; return; end + + [b_put, ans_put, resp_put, url_put] = ndi.cloud.api.files.putFiles(ans_url, localFilePath, "useCurl", true); + msg_put = ndi.unittest.cloud.APIMessage(narrative, b_put, ans_put, resp_put, url_put); + testCase.verifyTrue(b_put, "File upload (PUT request) failed. " + msg_put); + if ~b_put, testCase.Narrative = narrative; return; end + + narrative(end+1) = "Waiting for the server to record the upload."; + [b_w, ans_w, resp_w, url_w] = ndi.cloud.api.files.waitForFileUpload(testCase.DatasetID, fileUID); + msg_w = ndi.unittest.cloud.APIMessage(narrative, b_w, ans_w, resp_w, url_w); + testCase.verifyTrue(b_w, "waitForFileUpload did not see the file uploaded. " + msg_w); + testCase.verifyEqual(ans_w.state, 'complete', msg_w); + testCase.verifyEmpty(ans_w.pending, msg_w); + if isscalar(ans_w.files) + testCase.verifyEqual(ans_w.files(1).uid, char(fileUID), msg_w); + testCase.verifyEqual(ans_w.files(1).size, double(strlength(testCase.TestFileContent)), ... + "waitForFileUpload reported the wrong size. " + msg_w); + else + testCase.verifyFail("waitForFileUpload should report exactly one file. " + msg_w); + end + + narrative(end+1) = "Waiting, briefly, on a uid that was never uploaded."; + neverUID = string(did.ido.unique_id()); + [b_n, ans_n, resp_n, url_n] = ndi.cloud.api.files.waitForFileUpload(testCase.DatasetID, neverUID, ... + 'timeout', 3); + msg_n = ndi.unittest.cloud.APIMessage(narrative, b_n, ans_n, resp_n, url_n); + testCase.verifyFalse(b_n, "waitForFileUpload succeeded for a uid that was never uploaded. " + msg_n); + testCase.verifyEqual(ans_n.state, 'timeout', msg_n); + testCase.verifyEqual(ans_n.pending, neverUID, msg_n); + + testCase.Narrative = narrative; + end + function testListFilesWithOptions(testCase) testCase.Narrative = "Begin testListFilesWithOptions"; narrative = testCase.Narrative; @@ -868,7 +1084,9 @@ function testListFilesWithOptions(testCase) narrative(end+1) = "File uploaded successfully."; - pause(10); % Give server time to process + % The PUT returning 200 only means S3 has the object; wait for the + % server to record it (it can lag when other uploads are queued). + ndi.cloud.api.files.waitForFileUpload(testCase.DatasetID, fileUID); % Step 2: Call listFiles with checkForUpdates enabled narrative(end+1) = "Calling listFiles with checkForUpdates=true."; diff --git a/tests/+ndi/+unittest/+cloud/SignedURLSetTest.m b/tests/+ndi/+unittest/+cloud/SignedURLSetTest.m index 7f38a5a5b3..371d1fa622 100644 --- a/tests/+ndi/+unittest/+cloud/SignedURLSetTest.m +++ b/tests/+ndi/+unittest/+cloud/SignedURLSetTest.m @@ -274,6 +274,37 @@ function testGetSignedURLSetContentRoundTrip(testCase) testCase.Narrative = narrative; end + % ------------------------------------------------------------------ + % Sync path: per-file metadata beside the URL map (NDI Cloud #162) + % ------------------------------------------------------------------ + function testGetSignedURLSetFileInfo(testCase) + testCase.Narrative = "Begin testGetSignedURLSetFileInfo"; + narrative = testCase.Narrative; + + [b, page, resp, url] = ndi.cloud.api.files.getSignedURLSet(... + testCase.DatasetID, testCase.CloudDocumentID); + msg = ndi.unittest.cloud.APIMessage(narrative, b, page, resp, url); + testCase.verifyTrue(b, "getSignedURLSet returned failure. " + msg); + if ~b, testCase.Narrative = narrative; return; end + + % An environment without fileInfo (NDI Cloud #162) fails here: the + % empty list does not match the URL map. + urlUIDs = sort(ndi.unittest.cloud.SignedURLSetTest.returnedUIDs(page.files)); + infoUIDs = sort(string({page.fileInfo.uid})); + testCase.verifyEqual(infoUIDs, urlUIDs, ... + "fileInfo should describe exactly the files on the page. " + msg); + for i = 1:numel(page.fileInfo) + f = page.fileInfo(i); + testCase.verifyTrue(f.uploaded, "fileInfo " + f.uid + " not marked uploaded. " + msg); + testCase.verifyGreaterThan(f.size, 0, "fileInfo " + f.uid + " has no size. " + msg); + % The files were uploaded after #162, through the bulk path, + % which records each zip entry's CRC-32. + testCase.verifyNotEmpty(f.crc32, "fileInfo " + f.uid + " has no crc32. " + msg); + end + + testCase.Narrative = narrative; + end + % ------------------------------------------------------------------ % Sync path: oversized limit is server-clamped % ------------------------------------------------------------------ @@ -355,18 +386,10 @@ function testCreateAndWaitForSignedURLSetJob(testCase) testCase.DatasetID, testCase.CloudDocumentID); msg = ndi.unittest.cloud.APIMessage(narrative, b, job, resp, url); - % Environments where the worker isn't provisioned return - % 500 CONFIGURATION_ERROR / ENQUEUE_ERROR / JOB_CREATION_ERROR. - % Skip rather than fail so this test stays meaningful on - % partial deployments. + % An environment without the worker returns 500 + % CONFIGURATION_ERROR / ENQUEUE_ERROR / JOB_CREATION_ERROR, + % which fails here. if ~b - if isstruct(job) && isfield(job, 'code') && ... - any(strcmp(job.code, {'CONFIGURATION_ERROR','ENQUEUE_ERROR','JOB_CREATION_ERROR'})) - testCase.assumeFail(... - "Signed URL set worker is not configured on this environment; skipping async path. " + msg); - testCase.Narrative = narrative; - return; - end testCase.verifyTrue(b, "createSignedURLSetJob returned failure. " + msg); testCase.Narrative = narrative; return; diff --git a/tests/+ndi/+unittest/+cloud/SignedUrlDiskCacheEndToEndTest.m b/tests/+ndi/+unittest/+cloud/SignedUrlDiskCacheEndToEndTest.m new file mode 100644 index 0000000000..8327572fdc --- /dev/null +++ b/tests/+ndi/+unittest/+cloud/SignedUrlDiskCacheEndToEndTest.m @@ -0,0 +1,152 @@ +classdef SignedUrlDiskCacheEndToEndTest < matlab.unittest.TestCase +% SIGNEDURLDISKCACHEENDTOENDTEST - the persistent signed-URL disk cache +% seen through batchSignedUrlLookup, end-to-end. +% +% This is the "live cell" test-cloud-api.yml runs (testCloudApi -> +% ndi.unittest.cloud recursive). Placed in +cloud so the daily cloud +% CI picks it up alongside the async-job fixture test; the injected +% signer keeps it self-contained -- the disk cache is a client-side +% concern, so a real cloud round trip proves nothing the local run +% does not, and paying for a fresh lightsheet upload on every daily +% CI run to inject a counter around a signer we already control is +% pure cost. +% +% Proves: two consecutive batchSignedUrlLookup calls for the same +% scope cost ONE signer invocation. The first populates the disk +% cache; the second, with a fresh in-process cache, reads from disk +% and never touches the signer. Whether that signer would have been +% getSignedURLSetAll's paging walk or the createSignedURLSetJob +% async round-trip, the result is the same shape: signer counter +% doesn't move. + + properties + TmpDir char = '' + SavedCacheDirEnv char = '' + end + + methods (TestMethodSetup) + function isolateCacheDir(testCase) + testCase.SavedCacheDirEnv = getenv('NDI_SIGNED_URL_CACHE_DIR'); + testCase.TmpDir = tempname; + setenv('NDI_SIGNED_URL_CACHE_DIR', testCase.TmpDir); + testCase.addTeardown(@() localCleanup( ... + testCase.SavedCacheDirEnv, testCase.TmpDir)); + end + end + + methods (Access = private) + function [signer, counter] = countingSigner(~, filesMap) + % Emitted as a NESTED handle (not an anonymous @()... wrap) + % so nargout(signer) reads back as a fixed 2 rather than -1 + % -- which is what batchSignedUrlLookup's four-vs-two output + % probe expects to see for an injected mock. The same + % pattern BatchSignedUrlLookupTest already uses. + counter = containers.Map('KeyType','char','ValueType','any'); + counter('n') = 0; + % 23 h ahead of now mirrors the server's 24 h URL lifetime, + % leaving the default 30 min safety buffer comfortably valid. + futureDt = datetime('now','TimeZone','UTC') + hours(23); + futureDt.Format = 'yyyy-MM-dd''T''HH:mm:ss''Z'''; + futureIso = char(futureDt); + signer = @doSign; + function [ok, answer] = doSign(~, ~, varargin) %#ok + counter('n') = counter('n') + 1; + ok = true; + answer = struct( ... + 'files', filesMap, ... + 'filesExpireAt', futureIso, ... + 'expiresAt', futureIso, ... + 'generatedAt', futureIso, ... + 'fileCount', double(filesMap.Count), ... + 'pages', 1); + end + end + end + + methods (Test) + + function testReopenPaysZeroSignerCallsFromDiskCache(testCase) + % The end-to-end promise: sign the scope once, save it to + % disk, then a reopen (fresh in-process cache) resolves + % every uid without another signer round trip. For a + % 121k-member chunk.bin series that means "one 85 min job + % ever" instead of "one per viewer open all day". + filesMap = containers.Map('KeyType','char','ValueType','any'); + filesMap('4192a3c0dd1b4e00_3fe8a1b2c3d4e5f6') = 'https://s3/one'; + filesMap('4192a3c0dd1b4e00_4fe8a1b2c3d4e5f6') = 'https://s3/two'; + [signer, counter] = testCase.countingSigner(filesMap); + + % --- Run 1: cold. Expect one signer call, both uids hit + % the in-memory cache, and the disk file lands. + u1a = ndi.cloud.download.internal.batchSignedUrlLookup( ... + "ds-e2e", "doc-e2e", "chunk.bin", ... + "4192a3c0dd1b4e00_3fe8a1b2c3d4e5f6", ... + 'signer', signer, 'clearCache', true, 'diskCache', true); + u1b = ndi.cloud.download.internal.batchSignedUrlLookup( ... + "ds-e2e", "doc-e2e", "chunk.bin", ... + "4192a3c0dd1b4e00_4fe8a1b2c3d4e5f6", ... + 'signer', signer, 'diskCache', true); + testCase.verifyEqual(u1a, 'https://s3/one'); + testCase.verifyEqual(u1b, 'https://s3/two'); + testCase.verifyEqual(counter('n'), 1, ... + 'run 1 must cost exactly one signer call (in-memory batch)'); + + diskFile = fullfile(testCase.TmpDir, 'ds-e2e', ... + 'doc-e2e_chunk.bin.json.gz'); + testCase.assertTrue(isfile(diskFile), ... + 'run 1 must have written the scope to disk'); + + % --- Run 2: fresh in-process cache; disk should carry us. + % clearCache wipes the persistent in-memory cache -- this + % is what "a new viewer opens the same dataset" looks like + % from the batch's point of view. If the disk cache is + % working, the signer never runs again. + u2a = ndi.cloud.download.internal.batchSignedUrlLookup( ... + "ds-e2e", "doc-e2e", "chunk.bin", ... + "4192a3c0dd1b4e00_3fe8a1b2c3d4e5f6", ... + 'signer', signer, 'clearCache', true, 'diskCache', true); + u2b = ndi.cloud.download.internal.batchSignedUrlLookup( ... + "ds-e2e", "doc-e2e", "chunk.bin", ... + "4192a3c0dd1b4e00_4fe8a1b2c3d4e5f6", ... + 'signer', signer, 'diskCache', true); + testCase.verifyEqual(u2a, 'https://s3/one'); + testCase.verifyEqual(u2b, 'https://s3/two'); + testCase.verifyEqual(counter('n'), 1, ... + ['run 2 must have hit the disk cache -- no additional ' ... + 'signer call, whether it was going to be a paging ' ... + 'walk or an async job.']); + end + + function testDiskOptOutStillCallsSignerOnEveryColdRun(testCase) + % Guard: diskCache=false is the default, and it means "no + % on-disk state, ever" -- existing tests and callers that + % have not opted in must see the pre-cache behaviour where + % a fresh in-process cache re-signs the scope. + filesMap = containers.Map('KeyType','char','ValueType','any'); + filesMap('aa_bb') = 'https://s3/x'; + [signer, counter] = testCase.countingSigner(filesMap); + + for i = 1:3 + ndi.cloud.download.internal.batchSignedUrlLookup( ... + "ds-optout", "doc-optout", "", "aa_bb", ... + 'signer', signer, 'clearCache', true); % diskCache=false + end + testCase.verifyEqual(counter('n'), 3, ... + 'diskCache=false must call the signer per cold run'); + + % And nothing should have landed on disk either. + noFile = fullfile(testCase.TmpDir, 'ds-optout', 'doc-optout.json.gz'); + testCase.verifyFalse(isfile(noFile), ... + 'diskCache=false must not write any cache file'); + end + end +end + +function localCleanup(savedEnv, tmp) + setenv('NDI_SIGNED_URL_CACHE_DIR', savedEnv); + if ~isempty(tmp) && isfolder(tmp) + try %#ok + rmdir(tmp, 's'); + end + end +end diff --git a/tests/+ndi/+unittest/+cloud/TestPublishWithDocsAndFiles.m b/tests/+ndi/+unittest/+cloud/TestPublishWithDocsAndFiles.m index 6a3baef57a..8ad0843203 100644 --- a/tests/+ndi/+unittest/+cloud/TestPublishWithDocsAndFiles.m +++ b/tests/+ndi/+unittest/+cloud/TestPublishWithDocsAndFiles.m @@ -83,10 +83,22 @@ function deleteDatasetAfterTest(testCase) end function [b, ans_details, resp, url] = getFileDetailsWithRetry(testCase, fileUID) + % Poll getFileDetails until the server marks the file + % uploaded and returns a downloadUrl, or until we exceed + % the timeout. getFileDetails itself returns b=false while + % the file document is still 'uploaded: false' (its + % response body carries size=0 and no downloadUrl), so a + % retry loop on ~b covers the "PUT to S3 succeeded but the + % backend hasn't committed the file document yet" window + % without treating it as a hard failure. When the backend + % is under load this window can stretch well past the + % previous 10 s single-retry budget. + timeoutSec = 300; + intervalSec = 2; + deadline = tic; [b, ans_details, resp, url] = ndi.cloud.api.files.getFileDetails(testCase.DatasetID, fileUID); - if ~b - % Retry once after a brief wait to ride out transient backend 500s. - pause(10); + while ~b && toc(deadline) < timeoutSec + pause(intervalSec); [b, ans_details, resp, url] = ndi.cloud.api.files.getFileDetails(testCase.DatasetID, fileUID); end end @@ -145,8 +157,7 @@ function testSerialUploadPublishDownload(testCase) end narrative(end+1) = "All files uploaded successfully."; - narrative(end+1) = "Pausing for 20 seconds to allow for processing before pre-publish verification..."; - pause(20); + narrative(end+1) = "Per-file getFileDetails polling (getFileDetailsWithRetry) will gate each verification on uploaded=true rather than a fixed pause."; % Step 2.5: Pre-Publish Verification narrative(end+1) = "VERIFICATION (PRE-PUBLISH): Checking documents before publishing."; @@ -304,8 +315,7 @@ function testMixedUploadPublishDownload(testCase) end narrative(end+1) = "All files uploaded successfully."; - narrative(end+1) = "Pausing for 20 seconds to allow for processing before pre-publish verification..."; - pause(20); + narrative(end+1) = "Per-file getFileDetails polling (getFileDetailsWithRetry) will gate each verification on uploaded=true rather than a fixed pause."; % Step 2.5: Pre-Publish Verification narrative(end+1) = "VERIFICATION (PRE-PUBLISH): Checking documents before publishing."; diff --git a/tests/+ndi/+unittest/+cloud/TestSignedUrlSetJobFixture.m b/tests/+ndi/+unittest/+cloud/TestSignedUrlSetJobFixture.m new file mode 100644 index 0000000000..65a1849f00 --- /dev/null +++ b/tests/+ndi/+unittest/+cloud/TestSignedUrlSetJobFixture.m @@ -0,0 +1,546 @@ +classdef TestSignedUrlSetJobFixture < matlab.unittest.TestCase +% TESTSIGNEDURLSETJOBFIXTURE - pin the async signed-URL-set-job bug +% with a self-contained fresh upload. +% +% The bug: for a lightsheetZarrLevel document, +% POST /datasets/{d}/documents/{doc}/signed-url-set-jobs +% ?fileSeries=chunk.bin +% reaches state=failed with +% "Manifest for file series 'chunk.bin' (uid ) is not a file +% of this dataset" +% while adjacent +% GET /datasets/{d}/files/{UID}/detail +% for the same UID returns 200 OK. Two server code paths disagree +% on whether the manifest UID is a file of the dataset. +% +% Confirmed 2026-09-27 against both the tiny lightsheet fixture on +% User 1 prod (6ab84b549852a120dbcb22bc, 5-of-5 level docs) AND the +% 121k-member production dataset (6ab034e430a0f8d0e461dfcc). See +% VH-Lab/NDI-matlab#1010 and the Python analog on the same branch +% of NDI-python (test_cloud_signed_url_set_job.py, commit 5ab9c04). +% +% This test builds its own fresh lightsheet fixture on each run -- +% no hardcoded dataset id, no dependency on any environment's fixture +% state -- so it runs the same probe against whichever environment +% CLOUD_API_ENVIRONMENT / NDI_CLOUD_USERNAME points at. +% +% Diagnostics are narrated the same way FileSeriesRoundTripTest and +% ndi.unittest.cloud.APIMessage produce them: every step appends a +% string to Narrative, and any assertion that touches the cloud +% wraps the message in APIMessage so a failure prints as a JSON +% report the server-side team can read without a MATLAB session. +% See ndi.unittest.cloud.APIMessage and VH-Lab/NDI-matlab#945. + + properties (Constant) + DatasetNamePrefix = "NDI_UNITTEST_SIGNED_URL_FIXTURE_" + LevelClassName = "lightsheetZarrLevel" + SeriesName = "chunk.bin" + WaitTimeoutSeconds = 120 + end + + properties + WorkDir char = '' + DatasetIDs string = string.empty + Narrative (1,:) string = string.empty + end + + methods (TestClassSetup) + function checkCredentials(testCase) + username = getenv("NDI_CLOUD_USERNAME"); + password = getenv("NDI_CLOUD_PASSWORD"); + diagMsg = ['Missing NDI Cloud credentials ' ... + '(NDI_CLOUD_USERNAME/NDI_CLOUD_PASSWORD). ' ... + 'Skipping TestSignedUrlSetJobFixture.']; + testCase.assumeNotEmpty(username, diagMsg); + testCase.assumeNotEmpty(password, diagMsg); + end + + function checkLightsheetToolchain(testCase) + % which() is the reliable probe for a package function's + % presence on the path; exist(..., 'file') does not always + % resolve dotted package names across MATLAB releases. + testCase.assumeNotEmpty(... + which('ndi.test.lightsheet.makeBlobFixture'), ... + ['ndi.test.lightsheet.makeBlobFixture is not on the ' ... + 'path. This test needs the lightsheet fixture builder.']); + testCase.assumeNotEmpty(... + which('ndi.fun.doc.lightsheet.fromOMEZarr'), ... + ['ndi.fun.doc.lightsheet.fromOMEZarr is not on the path.']); + end + end + + methods (TestClassTeardown) + function cleanupUploadedDatasets(testCase) + % Sweep every dataset this class created, even if a + % per-test teardown already tried; a failed run may have + % left one behind. Best-effort, silent. + for i = 1:numel(testCase.DatasetIDs) + did = testCase.DatasetIDs(i); + if strlength(did) == 0, continue, end + try + ndi.cloud.api.datasets.deleteDataset(char(did), ... + 'when', 'now'); %#ok + catch + end + end + end + end + + methods (TestMethodSetup) + function setupWorkFolder(testCase) + import matlab.unittest.fixtures.TemporaryFolderFixture + fx = testCase.applyFixture(TemporaryFolderFixture); + testCase.WorkDir = fx.Folder; + testCase.Narrative = "Begin TestSignedUrlSetJobFixture setup " + ... + "in " + string(fx.Folder) + "."; + end + end + + methods (Test) + + function testLevelDocsAgreeOnManifestReachability(testCase) + % Build and upload a fresh lightsheet fixture, then probe + % every lightsheetZarrLevel document the upload produced + % for the async-job vs. getFileDetails disagreement. On a + % healthy environment every row should be ready+200. Any + % row where async=failed but detail=200 IS the reported + % server-side bug and this test fails with each offending + % triplet named. + + cloudDatasetId = testCase.buildAndUploadTinyFixture(); + + % --- 1. Enumerate lightsheetZarrLevel documents + testCase.Narrative(end+1) = "Enumerating documents on the " + ... + "freshly uploaded dataset via " + ... + "ndi.cloud.api.documents.listDatasetDocumentsAll."; + [okList, docSummaries, listResp, listUrl] = ... + ndi.cloud.api.documents.listDatasetDocumentsAll(cloudDatasetId); + listMsg = ndi.unittest.cloud.APIMessage(testCase.Narrative, ... + okList, docSummaries, listResp, listUrl); + testCase.assertTrue(okList, ... + "listDatasetDocumentsAll failed for the freshly-uploaded " + ... + "dataset. " + listMsg); + + classNames = arrayfun(@(s) string(safeField(s,'className','')), ... + docSummaries); + levelIdx = find(classNames == testCase.LevelClassName); + testCase.Narrative(end+1) = "Filtered to " + numel(levelIdx) + ... + " " + testCase.LevelClassName + " document(s) out of " + ... + numel(docSummaries) + " total."; + noneMsg = ndi.unittest.cloud.APIMessage(testCase.Narrative, ... + true, docSummaries, listResp, listUrl); + testCase.assertNotEmpty(levelIdx, ... + "Uploaded dataset has no " + testCase.LevelClassName + ... + " documents; fixture upload must have dropped them. " + noneMsg); + + disagreements = string.empty; + + for k = 1:numel(levelIdx) + summary = docSummaries(levelIdx(k)); + cloudDocId = string(safeField(summary, 'id', '')); + ndiDocId = string(safeField(summary, 'ndiId', '')); + + testCase.Narrative(end+1) = "--- Probing " + ... + testCase.LevelClassName + " " + (k) + "/" + ... + numel(levelIdx) + " (cloud id " + cloudDocId + ... + ", ndi id " + ndiDocId + ") ---"; + + [manifestUid, docNarrative, docFatal] = ... + testCase.resolveManifestUid(cloudDatasetId, cloudDocId); + testCase.Narrative = docNarrative; + if docFatal + % resolveManifestUid already appended a reason to + % the narrative; skip this doc without abandoning + % the whole test. + continue + end + + [row, probeNarrative] = testCase.probeOneLevelDoc( ... + cloudDatasetId, ndiDocId, manifestUid, ... + cloudDocId, testCase.Narrative); + testCase.Narrative = probeNarrative; + + printFixtureProbeRow(row); + + if row.jobOk && row.detailsOk + % Healthy: both paths agree. Real fileCount comes + % from the async job's result blob (fetched inside + % probeOneLevelDoc when the job was ready), so a + % ready-with-zero-files is legible. + successMsg = ndi.unittest.cloud.APIMessage( ... + testCase.Narrative, true, row.jobResultBlob, ... + row.jobResultResponse, row.jobResultUrl); + testCase.verifyGreaterThan(row.jobMapSize, 0, ... + "The async job reported 'ready' but its result " + ... + "blob's fileCount was " + row.jobMapSize + ... + ". A ready manifest scope should carry at least " + ... + "one member. " + successMsg); + elseif ~row.jobOk && ~row.detailsOk + testCase.Narrative(end+1) = "Both paths refused the " + ... + "same manifest UID -- consistent, not the " + ... + "disagreement we are probing for."; + elseif ~row.jobOk && row.detailsOk + disagreements(end+1) = sprintf( ... + 'doc=%s ndi=%s manifestUid=%s: async job %s (%s) but getFileDetails returned 200', ... + char(cloudDocId), char(ndiDocId), char(manifestUid), ... + char(row.jobState), char(row.jobDetail)); %#ok + testCase.Narrative(end+1) = "DISAGREEMENT: " + ... + "async job state=" + row.jobState + ... + " (" + row.jobDetail + ") but getFileDetails " + ... + "returned 200 for manifest uid " + manifestUid + "."; + else % row.jobOk && ~row.detailsOk + disagreements(end+1) = sprintf( ... + 'doc=%s: async job ready but getFileDetails failed for manifestUid=%s (%s)', ... + char(cloudDocId), char(manifestUid), ... + char(row.detailsDetail)); %#ok + testCase.Narrative(end+1) = "INVERSE DISAGREEMENT: " + ... + "async job ready but getFileDetails FAILED for " + ... + "manifest uid " + manifestUid + " (" + ... + row.detailsDetail + ")."; + end + end + + if ~isempty(disagreements) + finalMsg = ndi.unittest.cloud.APIMessage( ... + testCase.Narrative, false, ... + struct('disagreements', {cellstr(disagreements)}, ... + 'datasetId', char(cloudDatasetId)), ... + matlab.net.http.ResponseMessage.empty, ... + "createSignedURLSetJob and getFileDetails cross-check"); + testCase.verifyTrue(false, sprintf( ... + ['createSignedURLSetJob and getFileDetails disagree ' ... + 'on %d document(s) of freshly-uploaded fixture %s. ' ... + 'This is the server-side bug tracked in ' ... + 'VH-Lab/NDI-matlab#1010.\n%s\n\n%s'], ... + numel(disagreements), char(cloudDatasetId), ... + strjoin(" " + disagreements, newline), finalMsg)); + end + end + + end + + methods (Access = private) + function cloudDatasetId = buildAndUploadTinyFixture(testCase) + % Build a tiny lightsheet OME-Zarr on disk, ingest into a + % fresh session, wrap in a dataset, upload uploadAsNew. + % Small shape / small chunks -> a handful of level docs, + % each carrying its own chunk.bin series -- enough surface + % for the disagreement to fire on a per-level basis. + + zarrParent = fullfile(testCase.WorkDir, 'zarr'); + mkdir(zarrParent); + testCase.Narrative(end+1) = "Building tiny lightsheet blob " + ... + "fixture at " + string(zarrParent) + " " + ... + "(Shape [32 32 32], NumChannels 1, NumLevels 3, " + ... + "ChunkShape [16 16 16])."; + [zarrPath, ~] = ndi.test.lightsheet.makeBlobFixture(zarrParent, ... + 'Shape', [32 32 32], ... + 'NumChannels', 1, ... + 'NumLevels', 3, ... + 'ChunkShape', [16 16 16]); + testCase.Narrative(end+1) = "Zarr fixture built at " + ... + string(zarrPath) + "."; + + sessionDir = fullfile(testCase.WorkDir, 'session'); + mkdir(sessionDir); + S = ndi.session.dir('probe', sessionDir); + subject = ndi.document('subject', ... + 'base.session_id', S.id(), ... + 'subject.local_identifier', 'probe@vhlab'); + S.database_add(subject); + testCase.Narrative(end+1) = "Session opened at " + ... + string(sessionDir) + " with subject " + string(subject.id()) + "."; + + testCase.Narrative(end+1) = "Ingesting the OME-Zarr into the " + ... + "session via ndi.fun.doc.lightsheet.fromOMEZarr " + ... + "(materializeChunks=true, codec=raw)."; + ndi.fun.doc.lightsheet.fromOMEZarr(S, zarrPath, ... + 'subjectID', subject.id(), ... + 'materializeChunks', true, ... + 'codec', 'raw'); + + datasetDir = fullfile(testCase.WorkDir, 'dataset'); + mkdir(datasetDir); + D = ndi.dataset.dir('probe_ds', datasetDir); + D = D.add_ingested_session(S); + testCase.Narrative(end+1) = "Dataset created at " + ... + string(datasetDir) + "; ingested session " + string(S.id()) + "."; + + uniqueName = char(testCase.DatasetNamePrefix + ... + string(did.ido.unique_id())); + testCase.Narrative(end+1) = "Uploading to NDI Cloud as " + ... + "remote name '" + string(uniqueName) + "' " + ... + "(uploadAsNew=true, skipMetadataEditorMetadata=true)."; + [ok, cloudDatasetId, msg] = ndi.cloud.uploadDataset(D, ... + 'uploadAsNew', true, ... + 'skipMetadataEditorMetadata', true, ... + 'remoteDatasetName', uniqueName); + uploadMsg = ndi.unittest.cloud.APIMessage(testCase.Narrative, ... + ok, msg, matlab.net.http.ResponseMessage.empty, ... + "ndi.cloud.uploadDataset"); + testCase.assertTrue(ok, ... + "uploadDataset failed: " + string(msg) + ". " + uploadMsg); + cloudDatasetId = string(cloudDatasetId); + testCase.DatasetIDs(end+1) = cloudDatasetId; + testCase.addTeardown(@() safeDeleteDataset(cloudDatasetId)); + testCase.Narrative(end+1) = "Upload accepted; cloud dataset id is " + ... + cloudDatasetId + "."; + + testCase.Narrative(end+1) = "Waiting for server-side " + ... + "bulk-upload extraction (waitForAllBulkUploads) " + ... + "before probing files."; + [waitOk, waitInfo, waitResp, waitUrl] = ... + ndi.cloud.api.files.waitForAllBulkUploads(cloudDatasetId); + waitMsg = ndi.unittest.cloud.APIMessage(testCase.Narrative, ... + waitOk, waitInfo, waitResp, waitUrl); + testCase.assertTrue(waitOk, ... + "waitForAllBulkUploads did not confirm completion " + ... + "(state=" + string(safeField(waitInfo,'state','')) + ", " + ... + "elapsed=" + safeField(waitInfo,'elapsed',NaN) + "s). " + waitMsg); + testCase.Narrative(end+1) = "Bulk-upload extraction " + ... + "confirmed (elapsed=" + safeField(waitInfo,'elapsed',NaN) + "s)."; + end + + function [manifestUid, narrative, fatal] = resolveManifestUid( ... + testCase, cloudDatasetId, cloudDocId) + % Fetch the cloud document's files.file_info block and pull + % the chunk.bin manifest UID out of it. If we can't reach + % that state, append a narrative line and return fatal=true + % so the caller skips this document without failing the + % whole test. + narrative = testCase.Narrative; + manifestUid = ""; + fatal = false; + + narrative(end+1) = "Fetching full document via " + ... + "ndi.cloud.api.documents.getDocument to read " + ... + "files.file_info."; + [okDoc, docProps, docResp, docUrl] = ... + ndi.cloud.api.documents.getDocument(cloudDatasetId, cloudDocId); + if ~okDoc + docMsg = ndi.unittest.cloud.APIMessage(narrative, ... + okDoc, docProps, docResp, docUrl); + narrative(end+1) = "getDocument failed for cloud id " + ... + cloudDocId + "; skipping this doc. Diagnostic: " + docMsg; + fatal = true; + return + end + manifestUid = extractManifestUidFromDoc(docProps, testCase.SeriesName); + if strlength(manifestUid) == 0 + narrative(end+1) = "Document has no " + testCase.SeriesName + ... + " file_info entry; skipping (not a level with a " + ... + "materialized manifest)."; + fatal = true; + return + end + narrative(end+1) = "Resolved manifest UID for series '" + ... + testCase.SeriesName + "' = " + manifestUid + "."; + end + + function [row, narrative] = probeOneLevelDoc(testCase, ... + cloudDatasetId, ndiDocId, manifestUid, cloudDocId, narrative) + % Do the two adjacent API calls: createSignedURLSetJob + + % waitForSignedURLSetJob (async path) and getFileDetails + % (single-uid path). Return a row struct summarising the + % outcome, plus the extended narrative. + + row = struct( ... + 'cloudDocId', cloudDocId, ... + 'ndiDocId', ndiDocId, ... + 'manifestUid', manifestUid, ... + 'jobId', "", ... + 'jobState', "unstarted", ... + 'jobDetail', "", ... + 'jobElapsedSec', 0, ... + 'jobOk', false, ... + 'jobMapSize', NaN, ... + 'jobResultBlob', [], ... + 'jobResultResponse', matlab.net.http.ResponseMessage.empty, ... + 'jobResultUrl', "", ... + 'detailsOk', false, ... + 'detailsDetail', ""); + + % --- Async job path + narrative(end+1) = "Calling " + ... + "ndi.cloud.api.files.createSignedURLSetJob(" + ... + "idNamespace='ndi', fileSeries='" + testCase.SeriesName + ... + "') on NDI doc id " + ndiDocId + "."; + tStart = tic; + [okCreate, createAnswer, createResp, createUrl] = ... + ndi.cloud.api.files.createSignedURLSetJob( ... + cloudDatasetId, ndiDocId, ... + 'idNamespace', "ndi", ... + 'fileSeries', testCase.SeriesName); + if ~okCreate + row.jobState = "createRejected"; + row.jobDetail = extractMessage(createAnswer); + narrative(end+1) = "createSignedURLSetJob was REJECTED " + ... + "(HTTP-level). Detail: " + row.jobDetail; + else + row.jobId = string(createAnswer.jobId); + narrative(end+1) = "Job accepted; jobId=" + row.jobId + ... + ". Polling with waitForSignedURLSetJob (timeout " + ... + testCase.WaitTimeoutSeconds + "s)."; + [okReady, jobAnswer, waitResp, waitUrl] = ... + ndi.cloud.api.files.waitForSignedURLSetJob( ... + row.jobId, 'timeout', testCase.WaitTimeoutSeconds); + if okReady + row.jobState = "ready"; + row.jobDetail = ""; + row.jobResultUrl = string(safeField(jobAnswer, ... + 'resultUrl', '')); + narrative(end+1) = "Job reached state 'ready'. " + ... + "Fetching the result blob via " + ... + "ndi.cloud.api.files.getSignedURLSetResult to " + ... + "read fileCount (the poll answer does not carry it)."; + if strlength(row.jobResultUrl) > 0 && ... + ~isempty(which('ndi.cloud.api.files.getSignedURLSetResult')) + try + [okResult, resultBlob, resultResp, ~] = ... + ndi.cloud.api.files.getSignedURLSetResult( ... + row.jobResultUrl); + row.jobResultBlob = resultBlob; + row.jobResultResponse = resultResp; + if okResult + row.jobMapSize = double(safeField( ... + resultBlob, 'fileCount', NaN)); + narrative(end+1) = "Result blob parsed; " + ... + "fileCount=" + row.jobMapSize + "."; + else + narrative(end+1) = "Result blob fetch/parse " + ... + "FAILED (blob URL " + row.jobResultUrl + ")."; + end + catch resErr + narrative(end+1) = "Result blob fetch RAISED: " + ... + string(resErr.identifier) + " -- " + ... + string(resErr.message); + end + else + narrative(end+1) = "No resultUrl on the poll " + ... + "answer (or getSignedURLSetResult not on path); " + ... + "leaving fileCount unset."; + end + else + row.jobState = string(safeField(jobAnswer, 'state', 'failed')); + row.jobDetail = extractMessage(jobAnswer); + narrative(end+1) = "Job reached terminal state '" + ... + row.jobState + "'. Detail: " + row.jobDetail; + % Keep the poll response around so the failure JSON + % carries the server's own words verbatim. + row.jobResultResponse = waitResp; + row.jobResultUrl = string(waitUrl); + end + end + row.jobElapsedSec = toc(tStart); + row.jobOk = row.jobState == "ready"; + + % --- Single-uid detail path + narrative(end+1) = "Independently calling " + ... + "ndi.cloud.api.files.getFileDetails(datasetId, " + ... + "manifestUid=" + manifestUid + ")."; + [detailsOk, detailsAnswer, detailsResp, detailsUrl] = ... + ndi.cloud.api.files.getFileDetails( ... + cloudDatasetId, manifestUid); + row.detailsOk = detailsOk; + row.detailsDetail = extractMessage(detailsAnswer); + if detailsOk + narrative(end+1) = "getFileDetails returned 200 for " + ... + "manifest uid " + manifestUid + "."; + else + narrative(end+1) = "getFileDetails FAILED for manifest " + ... + "uid " + manifestUid + ". Detail: " + row.detailsDetail; + end + + % Stash the createSignedURLSetJob response as a fallback for + % the JSON report when the job path is the interesting one. + if ~row.jobOk + row.jobResultResponse = createResp; + row.jobResultUrl = string(createUrl); + if isempty(row.jobResultBlob) + row.jobResultBlob = createAnswer; + end + end + + % Also stash the details response when it is the interesting + % one -- keeps the row self-contained even if the caller + % never reaches the JSON assembly path. + if ~row.detailsOk && isempty(row.jobResultBlob) + row.jobResultBlob = detailsAnswer; + row.jobResultResponse = detailsResp; + row.jobResultUrl = string(detailsUrl); + end + end + end +end + +function uid = extractManifestUidFromDoc(docProps, seriesName) + % Manifest UID for a series named NAME lives at + % document.files.file_info entry.name==NAME .locations(1).uid. + uid = ""; + if ~isstruct(docProps), return, end + files = safeField(docProps, 'files', struct()); + fileInfo = safeField(files, 'file_info', []); + if isempty(fileInfo), return, end + for i = 1:numel(fileInfo) + entry = fileInfo(i); + name = string(safeField(entry, 'name', '')); + if name ~= seriesName, continue, end + locs = safeField(entry, 'locations', []); + if isempty(locs), return, end + uid = string(safeField(locs(1), 'uid', '')); + return + end +end + +function v = safeField(s, name, defaultValue) + if isstruct(s) && isfield(s, name) && ~isempty(s.(name)) + v = s.(name); + else + v = defaultValue; + end +end + +function m = extractMessage(payload) + m = ""; + if isstruct(payload) + if isfield(payload, 'message') && ~isempty(payload.message) + m = string(payload.message); + elseif isfield(payload, 'error') && ~isempty(payload.error) + m = string(payload.error); + elseif isfield(payload, 'state') && ~isempty(payload.state) + m = string(payload.state); + end + elseif ischar(payload) || isstring(payload) + m = string(payload); + end +end + +function printFixtureProbeRow(row) + % Single-line summary per level doc, kept for grep-ability in CI + % logs. The JSON narrative report is what a human reads on a + % failed assertion; this line is the sequence marker. + detail = char(string(row.jobDetail)); + if numel(detail) > 60, detail = [detail(1:57) '...']; end + fprintf(['[fixture-probe] doc=%s ndi=%s manifestUid=%s ' ... + 'jobState=%s elapsed=%.1fs mapSize=%s manifestFetchable=%d ' ... + 'detail="%s"\n'], ... + char(row.cloudDocId), char(row.ndiDocId), char(row.manifestUid), ... + char(row.jobState), row.jobElapsedSec, mapSizeAsText(row.jobMapSize), ... + row.detailsOk, detail); +end + +function s = mapSizeAsText(value) + if isnumeric(value) && isscalar(value) && ~isnan(value) + s = sprintf('%d', round(value)); + else + s = '-'; + end +end + +function safeDeleteDataset(cloudDatasetId) + try + ndi.cloud.api.datasets.deleteDataset(char(cloudDatasetId), ... + 'when', 'now'); %#ok + catch + end +end diff --git a/tests/+ndi/+unittest/+cloud/feedbackTest.m b/tests/+ndi/+unittest/+cloud/feedbackTest.m new file mode 100644 index 0000000000..0e64a15c52 --- /dev/null +++ b/tests/+ndi/+unittest/+cloud/feedbackTest.m @@ -0,0 +1,259 @@ +classdef feedbackTest < matlab.unittest.TestCase +% feedbackTest - Test suite for the ndi.cloud.api.feedback commands. +% +% Exercises the client wrappers around the in-app problem report +% endpoints (ndi-cloud-node#158): +% +% POST /feedback +% DELETE /feedback/{feedbackId} +% +% Every report a test submits is deleted again (by the test itself, or +% by a teardown if the test fails part-way), so the suite does not +% leave reports in the admin triage queue. Deleted reports do not count +% against the server's limit of 10 reports per user per hour. +% +% Each test follows the narrative + APIMessage pattern used throughout +% ndi.unittest.cloud so a failure carries every step and the URL of the +% last API call. +% +% Requirements: +% - NDI_CLOUD_USERNAME and NDI_CLOUD_PASSWORD environment variables +% - CLOUD_API_ENVIRONMENT=dev to run against the dev API + + properties (Constant) + AppName = "NDI-matlab-unittest"; + % Valid 24-hex id that will not match any stored report. + NonexistentID = "000000000000000000000000"; + end + + properties + Narrative (1,:) string + end + + methods (TestClassSetup) + function checkCredentials(testCase) + username = getenv("NDI_CLOUD_USERNAME"); + password = getenv("NDI_CLOUD_PASSWORD"); + diagMsg = ['Missing NDI Cloud credentials (NDI_CLOUD_USERNAME/', ... + 'NDI_CLOUD_PASSWORD). Skipping cloud-dependent tests.']; + testCase.assumeNotEmpty(username, diagMsg); + testCase.assumeNotEmpty(password, diagMsg); + end + + function checkEndpointDeployed(testCase) + % The cloud suite runs against both prod and dev. Skip (rather + % than fail) in an environment that does not have the feedback + % endpoint yet: once deployed, a malformed id is answered 400 by + % the route; before that, the API's catch-all answers 404. + [~, ~, resp] = ndi.cloud.api.feedback.deleteFeedback("not-a-valid-id"); + testCase.assumeNotEqual(double(resp.StatusCode), 404, ... + "The /feedback endpoint is not deployed in this API environment. Skipping feedbackTest."); + end + end + + methods (TestMethodSetup) + function resetNarrative(testCase) + testCase.Narrative = "Begin feedbackTest"; + end + end + + methods (Access = private) + function registerCleanup(testCase, feedbackID) + % Delete the report at teardown if the test did not get to it. + % deleteFeedback returning 404 here just means it is already gone. + testCase.addTeardown(@() ndi.cloud.api.feedback.deleteFeedback(feedbackID)); + end + end + + methods (Test) + % ------------------------------------------------------------------ + % Happy path: submit a minimal report, delete it, and confirm a + % second delete reports 404. + function testSubmitAndDelete(testCase) + narrative = testCase.Narrative; + + narrative(end+1) = "Preparing to call submitFeedback with only the required fields."; + [b, feedbackID, resp, url] = ndi.cloud.api.feedback.submitFeedback( ... + testCase.AppName, "other", "feedbackTest: minimal report (safe to ignore)"); + narrative(end+1) = "Attempted to call API with URL " + string(url); + msg = ndi.unittest.cloud.APIMessage(narrative, b, feedbackID, resp, url); + testCase.assertTrue(b, msg); + testCase.verifyEqual(double(resp.StatusCode), 201, msg); + testCase.verifyClass(feedbackID, 'char', msg); + testCase.verifyNotEmpty(feedbackID, msg); + testCase.registerCleanup(feedbackID); + narrative(end+1) = "Report stored with id " + string(feedbackID); + + narrative(end+1) = "Preparing to call deleteFeedback on the new report."; + [b_del, ans_del, resp_del, url_del] = ndi.cloud.api.feedback.deleteFeedback(feedbackID); + msg_del = ndi.unittest.cloud.APIMessage(narrative, b_del, ans_del, resp_del, url_del); + testCase.verifyTrue(b_del, msg_del); + testCase.verifyEqual(double(resp_del.StatusCode), 204, msg_del); + + narrative(end+1) = "Deleting the same report again; expecting 404."; + [b_again, ans_again, resp_again, url_again] = ndi.cloud.api.feedback.deleteFeedback(feedbackID); + msg_again = ndi.unittest.cloud.APIMessage(narrative, b_again, ans_again, resp_again, url_again); + testCase.verifyFalse(b_again, msg_again); + testCase.verifyEqual(double(resp_again.StatusCode), 404, msg_again); + end + + % ------------------------------------------------------------------ + % Every optional field, including a log that carries credentials the + % server is expected to redact. (The reporter cannot read reports + % back, so redaction itself is checked in the server's own tests.) + function testSubmitFullReport(testCase) + narrative = testCase.Narrative; + + environment = struct('os', string(computer()), ... + 'matlab', string(version()), 'launchedFrom', "matlab"); + state = struct('level', 3, 'fetchStats', struct('ok', 10, 'failed', 2)); + fakeToken = "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ1bml0dGVzdCJ9.c2lnbmF0dXJlX3VuaXR0ZXN0"; + logText = "[lightsheet] GET https://bucket.s3.amazonaws.com/k?X-Amz-Signature=deadbeef01" ... + + newline + "Authorization: Bearer " + fakeToken; + + narrative(end+1) = "Preparing to call submitFeedback with all optional fields."; + [b, feedbackID, resp, url] = ndi.cloud.api.feedback.submitFeedback( ... + testCase.AppName, "bug", "feedbackTest: full report (safe to ignore)", ... + 'appVersion', "NDI-matlab unittest", ... + 'datasetId', testCase.NonexistentID, ... + 'documentId', "unittest-document", ... + 'environment', environment, ... + 'state', state, ... + 'log', logText, ... + 'traceback', "feedbackTest: no real traceback"); + narrative(end+1) = "Attempted to call API with URL " + string(url); + msg = ndi.unittest.cloud.APIMessage(narrative, b, feedbackID, resp, url); + testCase.assertTrue(b, msg); + testCase.verifyEqual(double(resp.StatusCode), 201, msg); + testCase.registerCleanup(feedbackID); + + narrative(end+1) = "Deleting report " + string(feedbackID); + [b_del, ans_del, resp_del, url_del] = ndi.cloud.api.feedback.deleteFeedback(feedbackID); + msg_del = ndi.unittest.cloud.APIMessage(narrative, b_del, ans_del, resp_del, url_del); + testCase.verifyTrue(b_del, msg_del); + end + + % ------------------------------------------------------------------ + % Server-side validation (400), sent through the implementation + % class so the wrapper's own argument validation does not stop it. + function testInvalidCategoryIsRejected(testCase) + narrative = testCase.Narrative; + body = struct('schemaVersion', 1, 'app', testCase.AppName, ... + 'category', "feature", 'description', "feedbackTest: invalid category"); + + narrative(end+1) = "Submitting a report with category 'feature'; expecting 400."; + call = ndi.cloud.api.implementation.feedback.SubmitFeedback('feedbackStruct', body); + [b, answer, resp, url] = call.execute(); + msg = ndi.unittest.cloud.APIMessage(narrative, b, answer, resp, url); + if b + testCase.registerCleanup(answer); + end + testCase.verifyFalse(b, msg); + testCase.verifyEqual(double(resp.StatusCode), 400, msg); + end + + function testMissingDescriptionIsRejected(testCase) + narrative = testCase.Narrative; + body = struct('schemaVersion', 1, 'app', testCase.AppName, 'category', "bug"); + + narrative(end+1) = "Submitting a report without a description; expecting 400."; + call = ndi.cloud.api.implementation.feedback.SubmitFeedback('feedbackStruct', body); + [b, answer, resp, url] = call.execute(); + msg = ndi.unittest.cloud.APIMessage(narrative, b, answer, resp, url); + if b + testCase.registerCleanup(answer); + end + testCase.verifyFalse(b, msg); + testCase.verifyEqual(double(resp.StatusCode), 400, msg); + end + + function testUnsupportedSchemaVersionIsRejected(testCase) + narrative = testCase.Narrative; + body = struct('schemaVersion', 99, 'app', testCase.AppName, ... + 'category', "bug", 'description', "feedbackTest: bad schemaVersion"); + + narrative(end+1) = "Submitting a report with schemaVersion 99; expecting 400."; + call = ndi.cloud.api.implementation.feedback.SubmitFeedback('feedbackStruct', body); + [b, answer, resp, url] = call.execute(); + msg = ndi.unittest.cloud.APIMessage(narrative, b, answer, resp, url); + if b + testCase.registerCleanup(answer); + end + testCase.verifyFalse(b, msg); + testCase.verifyEqual(double(resp.StatusCode), 400, msg); + end + + % ------------------------------------------------------------------ + % Size cap (413): a log over the 500,000-character field cap but + % under the 1 MB request cap. + function testOversizeLogIsRejected(testCase) + narrative = testCase.Narrative; + + narrative(end+1) = "Submitting a report with a 600,000-character log; expecting 413."; + [b, feedbackID, resp, url] = ndi.cloud.api.feedback.submitFeedback( ... + testCase.AppName, "performance", "feedbackTest: oversize log", ... + 'log', string(repmat('x', 1, 600000))); + msg = ndi.unittest.cloud.APIMessage(narrative, b, feedbackID, resp, url); + if b + testCase.registerCleanup(feedbackID); + end + testCase.verifyFalse(b, msg); + testCase.verifyEqual(double(resp.StatusCode), 413, msg); + end + + % ------------------------------------------------------------------ + % Requests without a token are refused. The API answers missing or + % invalid tokens with 403 throughout; 401 is accepted too. + function testUnauthenticatedSubmitIsRejected(testCase) + narrative = testCase.Narrative; + body = struct('schemaVersion', 1, 'app', testCase.AppName, ... + 'category', "bug", 'description', "feedbackTest: no token"); + + narrative(end+1) = "POSTing a report with no Authorization header; expecting 401/403."; + contentTypeField = matlab.net.http.field.ContentTypeField(matlab.net.http.MediaType('application/json')); + acceptField = matlab.net.http.HeaderField('accept', 'application/json'); + request = matlab.net.http.RequestMessage(matlab.net.http.RequestMethod.POST, ... + [acceptField contentTypeField], matlab.net.http.MessageBody(body)); + url = ndi.cloud.api.url("submit_feedback"); + resp = request.send(url); + msg = ndi.unittest.cloud.APIMessage(narrative, false, [], resp, url); + testCase.verifyTrue(ismember(double(resp.StatusCode), [401 403]), msg); + end + + % ------------------------------------------------------------------ + % Delete edge cases. + function testDeleteNonexistentReport(testCase) + narrative = testCase.Narrative; + + narrative(end+1) = "Deleting a well-formed id that matches no report; expecting 404."; + [b, answer, resp, url] = ndi.cloud.api.feedback.deleteFeedback(testCase.NonexistentID); + msg = ndi.unittest.cloud.APIMessage(narrative, b, answer, resp, url); + testCase.verifyFalse(b, msg); + testCase.verifyEqual(double(resp.StatusCode), 404, msg); + end + + function testDeleteMalformedID(testCase) + narrative = testCase.Narrative; + + narrative(end+1) = "Deleting a malformed id; expecting 400."; + [b, answer, resp, url] = ndi.cloud.api.feedback.deleteFeedback("not-a-valid-id"); + msg = ndi.unittest.cloud.APIMessage(narrative, b, answer, resp, url); + testCase.verifyFalse(b, msg); + testCase.verifyEqual(double(resp.StatusCode), 400, msg); + end + + % ------------------------------------------------------------------ + % Client-side validation in the wrapper (no network call). + function testWrapperRejectsUnknownCategory(testCase) + testCase.verifyError(@() ndi.cloud.api.feedback.submitFeedback( ... + testCase.AppName, "feature", "never sent"), ... + 'MATLAB:validators:mustBeMember'); + end + + function testWrapperRejectsEmptyDescription(testCase) + testCase.verifyError(@() ndi.cloud.api.feedback.submitFeedback( ... + testCase.AppName, "bug", ""), ... + 'MATLAB:validators:mustBeNonzeroLengthText'); + end + end +end diff --git a/tests/+ndi/+unittest/+database/TestDid2SqliteHierarchy.m b/tests/+ndi/+unittest/+database/TestDid2SqliteHierarchy.m index d34f77c554..6c617fc151 100644 --- a/tests/+ndi/+unittest/+database/TestDid2SqliteHierarchy.m +++ b/tests/+ndi/+unittest/+database/TestDid2SqliteHierarchy.m @@ -124,7 +124,9 @@ function testAnAndedQueryKeepsBothTermsInOrder(testCase) % dropped here would narrow nothing and silently WIDEN a % search -- more documents come back, not fewer, so nothing % downstream errors. - q = and(ndi.query('', 'isa', 'subject'), ... + % (not `isa subject`: ndi.query rewrites that one into the + % biological entity types, an `or`, so it is not a plain isa term) + q = and(ndi.query('', 'isa', 'directed_relation'), ... ndi.query('base.name', 'exact_string', 'abc')); q2 = ndi.database.implementations.database.did2sqlite.toDid2Query(q); testCase.verifyEqual(numel(q2.searchstructure), 2, sprintf( ... diff --git a/tests/+ndi/+unittest/+database/TestMetadataAppOpenMindsInstances.m b/tests/+ndi/+unittest/+database/TestMetadataAppOpenMindsInstances.m new file mode 100644 index 0000000000..b15c6841f6 --- /dev/null +++ b/tests/+ndi/+unittest/+database/TestMetadataAppOpenMindsInstances.m @@ -0,0 +1,35 @@ +classdef TestMetadataAppOpenMindsInstances < matlab.unittest.TestCase + % TestMetadataAppOpenMindsInstances - Read openMINDS controlled instances for the metadata app. + % + % Description: + % The metadata app lists openMINDS controlled instances (e.g. species, + % licenses) and reads each instance document from the local openMINDS + % instance library. This test checks that the documents are found and + % decoded with the installed openMINDS_MATLAB version. + + methods (Test) + function testGetOpenMindsInstances(testCase) + [names, labels] = ndi.database.metadata_app.fun.getOpenMindsInstances('BiologicalSex'); + + testCase.verifyNotEmpty(names, ... + 'Diagnostic: Expected controlled instances of BiologicalSex.'); + testCase.verifyNumElements(labels, numel(names), ... + 'Diagnostic: Expected one label per instance name.'); + testCase.verifyTrue(all(strlength(string(labels)) > 0), ... + 'Diagnostic: Every instance document should have a name.'); + end + + function testGetCCByLicences(testCase) + [names, shortNames] = ndi.database.metadata_app.fun.getCCByLicences(); + + testCase.verifyNotEmpty(names, ... + 'Diagnostic: Expected CC BY licenses in the instance library.'); + testCase.verifyTrue(all(contains(string(names), 'CC-BY')), ... + 'Diagnostic: Only CC BY licenses should be returned.'); + testCase.verifyNumElements(shortNames, numel(names), ... + 'Diagnostic: Expected one short name per license.'); + testCase.verifyTrue(all(strlength(shortNames) > 0), ... + 'Diagnostic: Every license document should have a short name.'); + end + end +end diff --git a/tests/+ndi/+unittest/+database/TestV1SubjectSearch.m b/tests/+ndi/+unittest/+database/TestV1SubjectSearch.m new file mode 100644 index 0000000000..ecfed4fe4d --- /dev/null +++ b/tests/+ndi/+unittest/+database/TestV1SubjectSearch.m @@ -0,0 +1,48 @@ +classdef TestV1SubjectSearch < matlab.unittest.TestCase + % TestV1SubjectSearch - `isa subject` finds a v1 subject on a v1 database. + % + % ndi.query('', 'isa', 'subject') leaves out subjects of non-biological + % types with '~exact_string' terms on subject.type.name. The v1 database + % (didsqlite) searches in SQL, where such a term needs the field, so it + % left out every v1 subject. didsqlite now searches with the plain + % `isa subject` on a v1 database. + + properties + session + subjectID + end + + methods (TestMethodSetup) + function build(testCase) + d = fullfile(tempname, 'v1subj'); + mkdir(d); + testCase.addTeardown(@() rmdir(fileparts(d), 's')); + S = ndi.session.dir('v1subj', d); + sub = ndi.document('subject', 'base.session_id', S.id(), ... + 'subject.local_identifier', 'mouse@vhlab'); + S.database_add(sub); + testCase.session = S; + testCase.subjectID = sub.id(); + end + end + + methods (Test) + function testIsaSubjectFindsAV1Subject(testCase) + docs = testCase.session.database_search(ndi.query('', 'isa', 'subject')); + testCase.verifyTrue(any(cellfun(@(d) strcmp(d.id(), testCase.subjectID), docs))); + end + + function testIsaSubjectAndedWithAnotherTerm(testCase) + q = ndi.query('', 'isa', 'subject') & ... + ndi.query('base.session_id', 'exact_string', testCase.session.id()); + docs = testCase.session.database_search(q); + testCase.verifyTrue(any(cellfun(@(d) strcmp(d.id(), testCase.subjectID), docs))); + end + + function testIsaSubjectInsideAnOr(testCase) + q = ndi.query('', 'isa', 'subject') | ndi.query('', 'isa', 'probe'); + docs = testCase.session.database_search(q); + testCase.verifyTrue(any(cellfun(@(d) strcmp(d.id(), testCase.subjectID), docs))); + end + end +end diff --git a/tests/+ndi/+unittest/+fun/+doc/+lightsheet/TestChooseLevel.m b/tests/+ndi/+unittest/+fun/+doc/+lightsheet/TestChooseLevel.m new file mode 100644 index 0000000000..0b188d96e2 --- /dev/null +++ b/tests/+ndi/+unittest/+fun/+doc/+lightsheet/TestChooseLevel.m @@ -0,0 +1,96 @@ +classdef TestChooseLevel < matlab.unittest.TestCase + % TestChooseLevel - pure per-axis voxel-size level picker. + % + % chooseLevel is called by any reader that wants the coarsest level + % whose voxel size still meets a target; a viewport at 12 um/pixel + % has no business decoding a chunk at 0.5 um/pixel. Pure function + % over the level table, no session needed. + + methods (Static) + function levels = fourLevelPyramid() + % A 4-level dyadic ladder at 0.5, 1, 2, 4 um. + levels(1) = struct('level', 0, 'voxel_size', [0.5 0.5 0.5]); + levels(2) = struct('level', 1, 'voxel_size', [1.0 1.0 1.0]); + levels(3) = struct('level', 2, 'voxel_size', [2.0 2.0 2.0]); + levels(4) = struct('level', 3, 'voxel_size', [4.0 4.0 4.0]); + end + + function t = fourLevelTable() + L = ndi.unittest.fun.doc.lightsheet.TestChooseLevel.fourLevelPyramid(); + vs = arrayfun(@(l) l.voxel_size, L, 'UniformOutput', false).'; + level = arrayfun(@(l) l.level, L).'; + t = table(level, vs, 'VariableNames', {'level','voxel_size'}); + end + end + + methods (Test) + + function testScalarTargetPicksCoarsestMeeting(testCase) + L = ndi.unittest.fun.doc.lightsheet.TestChooseLevel.fourLevelPyramid(); + [idx, row] = ndi.fun.doc.lightsheet.chooseLevel(L, 2.0); + % coarsest level with all axes <= 2 um is level 2 (2 um/axis) + testCase.verifyEqual(idx, 2); + testCase.verifyEqual(row.voxel_size, [2 2 2]); + end + + function testScalarTargetPickerBetweenLevels(testCase) + L = ndi.unittest.fun.doc.lightsheet.TestChooseLevel.fourLevelPyramid(); + % target 1.5 um: level 1 (1 um) fits, level 2 (2 um) does not + idx = ndi.fun.doc.lightsheet.chooseLevel(L, 1.5); + testCase.verifyEqual(idx, 1); + end + + function testTargetFinerThanEveryLevelReturnsFinest(testCase) + L = ndi.unittest.fun.doc.lightsheet.TestChooseLevel.fourLevelPyramid(); + % target 0.1 um: no level meets it; the coarsest that does + % NOT exist, so bestK stays at the initial finest level. + idx = ndi.fun.doc.lightsheet.chooseLevel(L, 0.1); + testCase.verifyEqual(idx, 0); + end + + function testTargetCoarserThanEveryLevelReturnsCoarsest(testCase) + L = ndi.unittest.fun.doc.lightsheet.TestChooseLevel.fourLevelPyramid(); + idx = ndi.fun.doc.lightsheet.chooseLevel(L, 1000); + testCase.verifyEqual(idx, 3); + end + + function testPerAxisTarget(testCase) + L = ndi.unittest.fun.doc.lightsheet.TestChooseLevel.fourLevelPyramid(); + % permitted per-axis: [1, 3, 3] -- level 1 fits (1,1,1), + % level 2 fails on the first axis (2 > 1). + idx = ndi.fun.doc.lightsheet.chooseLevel(L, [1 3 3]); + testCase.verifyEqual(idx, 1); + end + + function testTableInput(testCase) + t = ndi.unittest.fun.doc.lightsheet.TestChooseLevel.fourLevelTable(); + idx = ndi.fun.doc.lightsheet.chooseLevel(t, 2.0); + testCase.verifyEqual(idx, 2); + end + + function testTableInputReturnsRowNotStruct(testCase) + t = ndi.unittest.fun.doc.lightsheet.TestChooseLevel.fourLevelTable(); + [~, row] = ndi.fun.doc.lightsheet.chooseLevel(t, 2.0); + testCase.verifyClass(row, 'table'); + testCase.verifyEqual(height(row), 1); + end + + function testEmptyLevelsErrors(testCase) + testCase.verifyError( ... + @() ndi.fun.doc.lightsheet.chooseLevel( ... + struct('level',{},'voxel_size',{}), 1.0), ... + 'NDI:lightsheet:chooseLevel:emptyLevels'); + end + + function testTargetShorterThanVoxelUsesShorterAxes(testCase) + % A 3-D level with a scalar target checks every axis; a + % 3-D level with a 2-D target checks the first two axes + % only. Documents the behavior that a reader passing a + % 2-D XY target against a ZYX level compares X and Y and + % ignores Z. + L = ndi.unittest.fun.doc.lightsheet.TestChooseLevel.fourLevelPyramid(); + idx = ndi.fun.doc.lightsheet.chooseLevel(L, [2 2]); + testCase.verifyEqual(idx, 2); + end + end +end diff --git a/tests/+ndi/+unittest/+fun/+doc/+lightsheet/TestChooseTileShape.m b/tests/+ndi/+unittest/+fun/+doc/+lightsheet/TestChooseTileShape.m new file mode 100644 index 0000000000..a36a3db3a0 --- /dev/null +++ b/tests/+ndi/+unittest/+fun/+doc/+lightsheet/TestChooseTileShape.m @@ -0,0 +1,173 @@ +classdef TestChooseTileShape < matlab.unittest.TestCase + % TestChooseTileShape - byte-budgeted chunk sizer. + % + % Verifies chooseTileShape hits a byte budget with tiles that are + % isotropic in world space, clamps to level shape at coarse levels, + % keeps non-spatial axes at 1, and falls back gracefully when voxel + % size is missing. + + methods (Test) + + function testIsotropicUint16HitsBudget(testCase) + % 8 MB uncompressed, uint16, isotropic 1 um -> ~161^3. + budget = 8 * 2^20; + c = ndi.fun.doc.lightsheet.chooseTileShape( ... + [1200 2000 2000], 'zyx', [1 1 1], 'uint16', budget); + testCase.verifyEqual(numel(c), 3); + % Each side within 5% of the cube root of voxelsPerTile. + expectedSide = (budget / 2)^(1/3); + testCase.verifyLessThan(abs(c(1) - expectedSide) / expectedSide, 0.05); + testCase.verifyEqual(c(1), c(2)); + testCase.verifyEqual(c(2), c(3)); + % Bytes within 20% of the budget. + bytes = prod(c) * 2; + testCase.verifyGreaterThan(bytes, 0.8 * budget); + testCase.verifyLessThan(bytes, 1.2 * budget); + end + + function testAnisotropicVoxelProducesWorldCube(testCase) + % 1 um XY, 5 um Z -> chunk_z * 5 == chunk_x * 1 (approx). + budget = 8 * 2^20; + c = ndi.fun.doc.lightsheet.chooseTileShape( ... + [1200 2000 2000], 'zyx', [5 1 1], 'uint16', budget); + worldZ = c(1) * 5; + worldY = c(2) * 1; + worldX = c(3) * 1; + testCase.verifyLessThan(abs(worldZ - worldY) / worldY, 0.05); + testCase.verifyLessThan(abs(worldX - worldY) / worldY, 0.05); + testCase.verifyLessThan(c(1), c(2)); % thin Z + % Bytes still near budget. + bytes = prod(c) * 2; + testCase.verifyGreaterThan(bytes, 0.7 * budget); + testCase.verifyLessThan(bytes, 1.3 * budget); + end + + function testChannelAxisStaysOne(testCase) + % 4D czyx: c is not spatial and must stay 1. + c = ndi.fun.doc.lightsheet.chooseTileShape( ... + [3 1200 2000 2000], 'czyx', [1 1 1 1], 'uint16', 8 * 2^20); + testCase.verifyEqual(c(1), 1); + testCase.verifyGreaterThan(c(2), 1); + end + + function testTimeAxisStaysOne(testCase) + c = ndi.fun.doc.lightsheet.chooseTileShape( ... + [10 3 1200 2000 2000], 'tczyx', [1 1 1 1 1], 'uint16', 8 * 2^20); + testCase.verifyEqual(c(1), 1); + testCase.verifyEqual(c(2), 1); + end + + function testClampToShapeAtCoarseLevel(testCase) + % A level whose spatial shape is already smaller than a + % naive cube: one whole chunk, no larger than the shape. + c = ndi.fun.doc.lightsheet.chooseTileShape( ... + [32 32 32], 'zyx', [1 1 1], 'uint16', 8 * 2^20); + testCase.verifyEqual(c, [32 32 32]); + end + + function testUint8BudgetGivesLargerTile(testCase) + % Halving bytes-per-voxel doubles voxel count. + budget = 8 * 2^20; + c16 = ndi.fun.doc.lightsheet.chooseTileShape( ... + [1200 2000 2000], 'zyx', [1 1 1], 'uint16', budget); + c8 = ndi.fun.doc.lightsheet.chooseTileShape( ... + [1200 2000 2000], 'zyx', [1 1 1], 'uint8', budget); + testCase.verifyGreaterThan(prod(c8), prod(c16)); + % Ratio approx 2 (uint16 half the voxels of uint8). + testCase.verifyLessThan(abs(prod(c8) / prod(c16) - 2) / 2, 0.1); + end + + function testFloat32BudgetGivesSmallerTile(testCase) + budget = 8 * 2^20; + c16 = ndi.fun.doc.lightsheet.chooseTileShape( ... + [1200 2000 2000], 'zyx', [1 1 1], 'uint16', budget); + cf = ndi.fun.doc.lightsheet.chooseTileShape( ... + [1200 2000 2000], 'zyx', [1 1 1], 'float32', budget); + testCase.verifyLessThan(prod(cf), prod(c16)); + end + + function testNumpyDtypeStringWorks(testCase) + % Endian-prefixed forms ('f4') should be accepted. + budget = 8 * 2^20; + cA = ndi.fun.doc.lightsheet.chooseTileShape( ... + [1200 2000 2000], 'zyx', [1 1 1], ' isotropic voxel-space fallback. + budget = 8 * 2^20; + c = ndi.fun.doc.lightsheet.chooseTileShape( ... + [1200 2000 2000], 'zyx', [0 0 0], 'uint16', budget); + % Fell back to iso -> cube of ~161. + expectedSide = (budget / 2)^(1/3); + testCase.verifyLessThan(abs(c(1) - expectedSide) / expectedSide, 0.05); + testCase.verifyEqual(c(1), c(2)); + testCase.verifyEqual(c(2), c(3)); + end + + function testBudgetScales(testCase) + % Doubling the budget doubles the voxel count. + small = ndi.fun.doc.lightsheet.chooseTileShape( ... + [1200 2000 2000], 'zyx', [1 1 1], 'uint16', 4 * 2^20); + big = ndi.fun.doc.lightsheet.chooseTileShape( ... + [1200 2000 2000], 'zyx', [1 1 1], 'uint16', 8 * 2^20); + testCase.verifyLessThan(abs(prod(big) / prod(small) - 2) / 2, 0.1); + end + + function testRedistributionWhenOneAxisClamps(testCase) + % Level whose Z is very short but XY is huge: Z clamps and + % the budget should shift into XY rather than being wasted. + budget = 8 * 2^20; + c = ndi.fun.doc.lightsheet.chooseTileShape( ... + [8 4096 4096], 'zyx', [1 1 1], 'uint16', budget); + testCase.verifyEqual(c(1), 8); % Z clamped to shape + bytes = prod(c) * 2; + % After redistribution should be near budget rather than at + % the naive isotropic-cube figure (which would leave a lot + % on the table when Z is short). + testCase.verifyGreaterThan(bytes, 0.7 * budget); + testCase.verifyLessThan(bytes, 1.3 * budget); + end + + function testTileCountMonotonicallyDecreasesAcrossLadder(testCase) + % Across a dyadic ladder each level halves shape and + % doubles voxel size, so chunk-in-voxels stays constant and + % chunk_grid drops by 2 per axis -> 8x fewer tiles per + % level for a 3D pyramid. Codified because a future + % refactor that dropped voxel-size scaling would break it + % silently. + budget = 8 * 2^20; + shape0 = [1200 2000 2000]; + vox0 = [1 1 1]; + tileCounts = zeros(1, 5); + byteSizes = zeros(1, 5); + for L = 0:4 + shp = max(1, floor(shape0 / 2^L)); + vs = vox0 * 2^L; + c = ndi.fun.doc.lightsheet.chooseTileShape( ... + shp, 'zyx', vs, 'uint16', budget); + tileCounts(L+1) = prod(ceil(shp ./ c)); + byteSizes(L+1) = prod(c) * 2; + end + % Strictly non-increasing. + testCase.verifyEqual(tileCounts, sort(tileCounts, 'descend')); + % Every level's per-tile bytes at or below the budget + % (coarse levels may fall well below when clamped, but no + % level oversizes). + testCase.verifyLessThanOrEqual(byteSizes, 1.3 * budget); + % Coarsest level should be one tile. + testCase.verifyEqual(tileCounts(end), 1); + end + + function testShapeAxesLengthMismatchErrors(testCase) + testCase.verifyError( ... + @() ndi.fun.doc.lightsheet.chooseTileShape( ... + [1200 2000 2000], 'yx', [1 1 1], 'uint16', 8 * 2^20), ... + 'NDI:lightsheet:chooseTileShape:axesShapeMismatch'); + end + + end +end diff --git a/tests/+ndi/+unittest/+fun/+doc/+lightsheet/TestFromOMEZarr.m b/tests/+ndi/+unittest/+fun/+doc/+lightsheet/TestFromOMEZarr.m new file mode 100644 index 0000000000..e70ff20c3b --- /dev/null +++ b/tests/+ndi/+unittest/+fun/+doc/+lightsheet/TestFromOMEZarr.m @@ -0,0 +1,210 @@ +classdef TestFromOMEZarr < matlab.unittest.TestCase + % TestFromOMEZarr - end-to-end ingest against a synthetic OME-Zarr. + % + % The fixture is a metadata-only dual-pyramid store (mean + max + % sharing level 0) written in the test setup itself, so this suite + % does not depend on NDR being installed. NDR is still the source + % of truth for reading OME-Zarr; when NDR is present we call it + % here as ingest does. When NDR is not on the path, every test + % filters rather than failing. + + properties + session + subjectID + fixtureDir + end + + methods (TestMethodSetup) + function build(testCase) + % `exist(pkg.fun, 'file')` returns 0 for package functions + % even when they resolve, so guard on `which` instead. + if isempty(which('ndr.format.omezarr.listPyramids')) + testCase.assumeFail(['NDR reader not on path; skipping ' ... + 'end-to-end OME-Zarr ingest tests.']); + end + d = fullfile(tempname, 'fromome'); + mkdir(d); + testCase.addTeardown(@() rmdir(fileparts(d), 's')); + S = ndi.session.dir('fromome', d); + sub = ndi.document('subject', 'base.session_id', S.id(), ... + 'subject.local_identifier', 'lightsheet@vhlab'); + S.database_add(sub); + testCase.session = S; + testCase.subjectID = sub.id(); + testCase.fixtureDir = writeSyntheticZarr(testCase); + end + end + + methods (Test) + + function testWritesOneParentAndFiveLevels(testCase) + [pdoc, lds, info] = ndi.fun.doc.lightsheet.fromOMEZarr( ... + testCase.session, testCase.fixtureDir, ... + 'subjectID', testCase.subjectID); + testCase.verifyClass(pdoc, 'ndi.document'); + % mean + max with shared level 0 -> 5 unique paths. + testCase.verifyEqual(numel(lds), 5); + testCase.verifyTrue(info.sharedLevel0); + testCase.verifyEqual(sort(info.reductions(:).'), {'max','mean'}); + end + + function testMissingSubjectIDIsRefused(testCase) + testCase.verifyError(@() ndi.fun.doc.lightsheet.fromOMEZarr( ... + testCase.session, testCase.fixtureDir), ... + 'NDI:lightsheet:fromOMEZarr:noSubject'); + end + + function testMissingStoreErrors(testCase) + testCase.verifyError(@() ndi.fun.doc.lightsheet.fromOMEZarr( ... + testCase.session, '/no/such/store.ome.zarr', ... + 'subjectID', testCase.subjectID), ... + 'NDI:lightsheet:fromOMEZarr:noSuchStore'); + end + + function testNonZarrDirectoryRefused(testCase) + d = fullfile(tempname, 'not_a_zarr'); + mkdir(d); + testCase.addTeardown(@() rmdir(fileparts(d), 's')); + testCase.verifyError(@() ndi.fun.doc.lightsheet.fromOMEZarr( ... + testCase.session, d, 'subjectID', testCase.subjectID), ... + 'NDI:lightsheet:fromOMEZarr:notOMEZarr'); + end + + function testPyramidNamesFilter(testCase) + % Restricting to `mean` alone means only that pyramid is + % ingested. Its level 0 is not shared with anything, so no + % dedupe (3 levels: 0, mean/1, mean/2). + [~, lds, info] = ndi.fun.doc.lightsheet.fromOMEZarr( ... + testCase.session, testCase.fixtureDir, ... + 'subjectID', testCase.subjectID, ... + 'pyramidNames', {'mean'}); + testCase.verifyEqual(numel(lds), 3); + testCase.verifyFalse(info.sharedLevel0); + end + + function testUnknownPyramidNameErrors(testCase) + testCase.verifyError(@() ndi.fun.doc.lightsheet.fromOMEZarr( ... + testCase.session, testCase.fixtureDir, ... + 'subjectID', testCase.subjectID, ... + 'pyramidNames', {'nope'}), ... + 'NDI:lightsheet:fromOMEZarr:noNamedPyramid'); + end + + function testSourceFileIDReused(testCase) + % Passing an explicit sourceFileID keeps a caller-created + % fileReference rather than making a new one. + src = ndi.fun.doc.lightsheet.makeSourceFile( ... + testCase.session, testCase.fixtureDir); + testCase.session.database_add(src); + [~, ~, info] = ndi.fun.doc.lightsheet.fromOMEZarr( ... + testCase.session, testCase.fixtureDir, ... + 'subjectID', testCase.subjectID, ... + 'sourceFileID', src.id()); + testCase.verifyEqual(info.sourceFileID, src.id()); + end + + function testTileBudgetForwardedToLevels(testCase) + % A tiny budget forces chunks smaller than the source + % zarr's chunks -- proof that fromOMEZarr forwards the + % option and makePyramid re-chooses per level. + budget = 512; % bytes, tiny + [~, lds] = ndi.fun.doc.lightsheet.fromOMEZarr( ... + testCase.session, testCase.fixtureDir, ... + 'subjectID', testCase.subjectID, ... + 'tileBudgetBytes', budget); + for k = 1:numel(lds) + p = lds{k}.document_properties.lightsheetZarrLevel; + bytes = prod(p.chunks) * 2; % uint16 + testCase.verifyLessThanOrEqual(bytes, 2 * budget, ... + 'per-tile bytes should track the budget'); + end + end + + end +end + +% --------------------------------------------------------------------- + +function fixtureDir = writeSyntheticZarr(testCase) +% Write a metadata-only OME-Zarr with a mean + max ladder sharing '0'. +% Directory layout: +% /.zattrs (multiscales, both pyramids) +% /0/.zarray (shared level 0) +% /mean/1/.zarray +% /mean/2/.zarray +% /max/1/.zarray +% /max/2/.zarray +% +% No chunk bytes; fromOMEZarr is metadata-only. + parent = tempname; + mkdir(parent); + testCase.addTeardown(@() rmdir(parent, 's')); + fixtureDir = fullfile(parent, 'example.zarr'); + mkdir(fixtureDir); + + dtype = ' 53 -> 27 -> 14 -> 7 along y. + testCase.verifyEqual(numel(meanLevels), 4); + end + + function testAnisotropicZIsNotHalvedFirst(testCase) + [~, ~, info] = testCase.ingest('voxelSizeUm', [4 1 1]); + first = info.levels(find(strcmp({info.levels.reduction}, 'mean'), 1)); + testCase.verifyEqual(first.factor, [1 1 1 2 2]); + end + + function testChannelSubsetAndOrder(testCase) + P = testCase.ingest('channels', {'Ex_640_Em_640F_Ch3'}); + p = P.document_properties.lightsheetZarrPyramid; + testCase.verifyEqual(p.n_channels, 1); + testCase.verifyEqual(p.channel_names, 'Ex_640_Em_640F_Ch3'); + end + + function testMeanOnlyLadder(testCase) + [~, L] = testCase.ingest('reductions', {'mean'}); + reds = cellfun(@(d) d.document_properties.lightsheetZarrLevel.reduction_function, ... + L, 'UniformOutput', false); + testCase.verifyFalse(any(strcmp(reds, 'max'))); + end + + function testNoAllChannelsIsAClearError(testCase) + plain = ndr.test.format.smartspim.makeExampleFixture( ... + fullfile(tempname, 'plain')); + testCase.addTeardown(@() rmdir(fileparts(plain), 's')); + testCase.verifyError(@() ndi.fun.doc.lightsheet.fromSPIM( ... + testCase.session, plain, 'subjectID', testCase.subjectID, ... + 'progressFcn', []), 'NDI:lightsheet:fromSPIM:noAllChannels'); + end + + function testSubjectIsRequired(testCase) + testCase.verifyError(@() ndi.fun.doc.lightsheet.fromSPIM( ... + testCase.session, testCase.root), 'NDI:lightsheet:fromSPIM:noSubject'); + end + end +end diff --git a/tests/+ndi/+unittest/+fun/+doc/+lightsheet/TestLaunchViewer.m b/tests/+ndi/+unittest/+fun/+doc/+lightsheet/TestLaunchViewer.m new file mode 100644 index 0000000000..a0123175b8 --- /dev/null +++ b/tests/+ndi/+unittest/+fun/+doc/+lightsheet/TestLaunchViewer.m @@ -0,0 +1,111 @@ +classdef TestLaunchViewer < matlab.unittest.TestCase + % TestLaunchViewer - the napari viewer is launched with its log kept. + % + % A viewer launched from MATLAB has no terminal, so its stdout and + % stderr -- which tiles it fetched, what failed and why -- go to + % lightsheet_YYYYMMDDHHMMSS.txt in NDI's temporary folder, with + % NDI_LIGHTSHEET_DEBUG=1 so the debugging lines are in it. + % viewerShellCommand and viewerLogFile are pure and tested on every + % platform; launchViewer runs a stand-in launcher on POSIX. + + methods (Test) + + function testLogFileIsNamedByLaunchTimeInNdiTemp(testCase) + when = datetime(2026, 10, 12, 9, 5, 7); + f = ndi.fun.doc.lightsheet.viewerLogFile(when); + testCase.verifyEqual(f, fullfile(ndi.common.PathConstants.TempFolder, ... + 'lightsheet_20261012090507.txt')); + end + + function testPosixDetachedDebugCommand(testCase) + c = ndi.fun.doc.lightsheet.viewerShellCommand( ... + '/usr/local/bin/napariViewLightsheet ''/s'' --pyramid p', ... + '/tmp/nditemp/lightsheet_1.txt', false, true, false); + testCase.verifyEqual(c, ['NDI_LIGHTSHEET_DEBUG=1 ' ... + '/usr/local/bin/napariViewLightsheet ''/s'' --pyramid p ' ... + '>>''/tmp/nditemp/lightsheet_1.txt'' 2>&1 &']); + end + + function testPosixWaitWithoutDebug(testCase) + c = ndi.fun.doc.lightsheet.viewerShellCommand('v', '/tmp/l.txt', ... + true, false, false); + testCase.verifyEqual(c, 'v >>''/tmp/l.txt'' 2>&1'); + end + + function testPosixQuotesALogPathWithAQuote(testCase) + c = ndi.fun.doc.lightsheet.viewerShellCommand('v', '/tmp/it''s.txt', ... + true, false, false); + testCase.verifyEqual(c, 'v >>''/tmp/it''\''''s.txt'' 2>&1'); + end + + function testWindowsDetachedDebugCommand(testCase) + c = ndi.fun.doc.lightsheet.viewerShellCommand( ... + '"C:\ndi\napariViewLightsheet.bat" "C:\s" --pyramid p', ... + 'C:\Temp\nditemp\lightsheet_1.txt', false, true, true); + testCase.verifyEqual(c, ['start "napari" /min cmd /c "' ... + 'set NDI_LIGHTSHEET_DEBUG=1&& ' ... + '"C:\ndi\napariViewLightsheet.bat" "C:\s" --pyramid p ' ... + '>>"C:\Temp\nditemp\lightsheet_1.txt" 2>&1"']); + end + + function testWindowsWaitRunsInPlace(testCase) + c = ndi.fun.doc.lightsheet.viewerShellCommand('v', 'C:\l.txt', ... + true, true, true); + testCase.verifyEqual(c, 'set NDI_LIGHTSHEET_DEBUG=1&& v >>"C:\l.txt" 2>&1'); + end + + function testLaunchWritesHeaderOutputAndDebugFlag(testCase) + testCase.assumeFalse(ispc, 'Runs a POSIX stand-in launcher.'); + folder = tempname; + mkdir(folder); + testCase.addTeardown(@() rmdir(folder, 's')); + launcher = fullfile(folder, 'fakeViewer.sh'); + fid = fopen(launcher, 'w'); + fprintf(fid, '#!/bin/sh\necho "debug=$NDI_LIGHTSHEET_DEBUG args=$*"\necho "to stderr" >&2\n'); + fclose(fid); + system(['chmod +x ''' launcher '''']); + logFile = fullfile(folder, 'sub', 'lightsheet_test.txt'); + + [status, got, fullCmd] = ndi.fun.doc.lightsheet.launchViewer( ... + [launcher ' --pyramid p1'], 'wait', true, ... + 'logFile', logFile, 'verbose', false); + + testCase.verifyEqual(status, 0); + testCase.verifyEqual(got, logFile); + testCase.verifySubstring(fullCmd, 'NDI_LIGHTSHEET_DEBUG=1'); + text = fileread(logFile); + testCase.verifySubstring(text, '=== napari lightsheet viewer:'); + testCase.verifySubstring(text, ['command : ' launcher ' --pyramid p1']); + testCase.verifySubstring(text, 'debug=1 args=--pyramid p1'); + testCase.verifySubstring(text, 'to stderr'); + end + + function testASecondLaunchAppendsRatherThanOverwrites(testCase) + testCase.assumeFalse(ispc, 'Runs a POSIX command.'); + logFile = [tempname '.txt']; + testCase.addTeardown(@() delete(logFile)); + ndi.fun.doc.lightsheet.launchViewer('echo first', 'wait', true, ... + 'logFile', logFile, 'verbose', false); + ndi.fun.doc.lightsheet.launchViewer('echo second', 'wait', true, ... + 'logFile', logFile, 'verbose', false); + text = fileread(logFile); + testCase.verifySubstring(text, 'first'); + testCase.verifySubstring(text, 'second'); + end + + function testTheManagerLaunchesThroughLaunchViewer(testCase) + % The View dialog must keep the log too, not just the + % command-line helper, and its preview must be the exact line + % that runs: the shell command built for the same log file + % the launch then writes to. + src = fileread(which('ndi.gui.app.LightsheetZarrManager')); + testCase.verifySubstring(src, ... + 'ndi.fun.doc.lightsheet.launchViewer(cmd, ''logFile'', logFile)'); + testCase.verifySubstring(src, ... + 'ndi.fun.doc.lightsheet.viewerShellCommand( ...'); + testCase.verifySubstring(src, 'cmd, logFile, false, true, ispc)'); + src = fileread(which('ndi.fun.doc.lightsheet.view')); + testCase.verifySubstring(src, 'ndi.fun.doc.lightsheet.launchViewer(cmd'); + end + end +end diff --git a/tests/+ndi/+unittest/+fun/+doc/+lightsheet/TestLevelTable.m b/tests/+ndi/+unittest/+fun/+doc/+lightsheet/TestLevelTable.m new file mode 100644 index 0000000000..19cbb5f7e3 --- /dev/null +++ b/tests/+ndi/+unittest/+fun/+doc/+lightsheet/TestLevelTable.m @@ -0,0 +1,119 @@ +classdef TestLevelTable < matlab.unittest.TestCase + % TestLevelTable - level enumeration + reduction filter. + % + % Exercises levelTable across the empty case, the mean+max ladder + % with a shared level 0 (5 rows: 'none', mean/1, mean/2, max/1, + % max/2), and the reduction filter that returns raw + one reduction. + + properties + session + subjectID + pyramidDoc + end + + methods (TestMethodSetup) + function build(testCase) + d = fullfile(tempname, 'leveltable'); + mkdir(d); + testCase.addTeardown(@() rmdir(fileparts(d), 's')); + S = ndi.session.dir('leveltable', d); + sub = ndi.document('subject', 'base.session_id', S.id(), ... + 'subject.local_identifier', 'lightsheet@vhlab'); + S.database_add(sub); + testCase.session = S; + testCase.subjectID = sub.id(); + end + end + + methods (Access = private) + function buildTwoPyramidsWithLevels(testCase) + pys = ndi.unittest.fun.doc.lightsheet.TestMakePyramid.twoPyramidsWithSharedLevel0(); + [pdoc, ~, ~] = ndi.fun.doc.lightsheet.makePyramid( ... + testCase.session, pys, {'mean','max'}, ... + 'subjectID', testCase.subjectID); + testCase.pyramidDoc = pdoc; + end + end + + methods (Test) + + function testEmptyPyramidReturnsEmptyTable(testCase) + % Build a parent with NO levels, verify the table is empty + % but has the expected columns and frame. + L = ndi.unittest.fun.doc.lightsheet.TestMakePyramid.dyadicLevels( ... + {'0'}, [16 32 32], [1 1 1], 'uint16'); + py = ndi.unittest.fun.doc.lightsheet.TestMakePyramid.pyramidEntry( ... + 'mean', 'box', L); + [pdoc, ~, ~] = ndi.fun.doc.lightsheet.makePyramid( ... + testCase.session, py, {'mean'}, ... + 'subjectID', testCase.subjectID); + % Delete the single level, leaving the parent with no + % children -- levelTable's empty path. + docs = testCase.session.database_search( ... + ndi.query('', 'isa', 'lightsheetZarrLevel') & ... + ndi.query('depends_on', 'depends_on', ... + 'lightsheetZarrPyramid_id', pdoc.id())); + for k = 1:numel(docs) + testCase.session.database_rm(docs{k}); + end + t = ndi.fun.doc.lightsheet.levelTable(testCase.session, pdoc); + testCase.verifyEqual(height(t), 0); + testCase.verifyTrue(ismember('reduction_function', ... + t.Properties.VariableNames)); + % UserData still carries the parent frame. + f = t.Properties.UserData; + testCase.verifyEqual(f.axes_order, 'zyx'); + end + + function testFullLadderReturnsFiveRowsSortedFinestFirst(testCase) + testCase.buildTwoPyramidsWithLevels(); + t = ndi.fun.doc.lightsheet.levelTable( ... + testCase.session, testCase.pyramidDoc); + testCase.verifyEqual(height(t), 5); + testCase.verifyEqual(t.level(1), 0); + testCase.verifyEqual(t.reduction_function{1}, 'none'); + testCase.verifyTrue(all(diff(t.level) >= 0)); + % Frame carries the reductions present in the ladder. + f = t.Properties.UserData; + testCase.verifyEqual( ... + sort(f.reductions(:).'), {'max','mean'}); + end + + function testAcceptsIDInsteadOfDoc(testCase) + testCase.buildTwoPyramidsWithLevels(); + t = ndi.fun.doc.lightsheet.levelTable( ... + testCase.session, testCase.pyramidDoc.id()); + testCase.verifyEqual(height(t), 5); + end + + function testUnknownIDErrors(testCase) + testCase.verifyError(@() ndi.fun.doc.lightsheet.levelTable( ... + testCase.session, '00000000-0000-0000-0000-000000000000'), ... + 'NDI:lightsheet:levelTable:noParent'); + end + + function testReductionFilterKeepsSharedAndOneReduction(testCase) + testCase.buildTwoPyramidsWithLevels(); + t = ndi.fun.doc.lightsheet.levelTable( ... + testCase.session, testCase.pyramidDoc, ... + 'reduction', 'max'); + % raw level 0 + max/1 + max/2 = 3. + testCase.verifyEqual(height(t), 3); + testCase.verifyTrue(all( ... + strcmp(t.reduction_function, 'none') | ... + strcmp(t.reduction_function, 'max'))); + testCase.verifyFalse(any(strcmp(t.reduction_function, 'mean'))); + end + + function testTableColumnsMatchSchema(testCase) + testCase.buildTwoPyramidsWithLevels(); + t = ndi.fun.doc.lightsheet.levelTable( ... + testCase.session, testCase.pyramidDoc); + expected = {'level','reduction_function','shape','chunks', ... + 'chunk_grid','n_chunks_stored','voxel_size', ... + 'translation','dtype','id'}; + testCase.verifyEqual(t.Properties.VariableNames, expected); + end + + end +end diff --git a/tests/+ndi/+unittest/+fun/+doc/+lightsheet/TestMakePyramid.m b/tests/+ndi/+unittest/+fun/+doc/+lightsheet/TestMakePyramid.m new file mode 100644 index 0000000000..e7d3e573fb --- /dev/null +++ b/tests/+ndi/+unittest/+fun/+doc/+lightsheet/TestMakePyramid.m @@ -0,0 +1,249 @@ +classdef TestMakePyramid < matlab.unittest.TestCase + % TestMakePyramid - the pyramid/level document writer. + % + % Exercises makePyramid over synthetic listPyramids-style structs so + % the whole schema-level path (parent doc, per-level doc, dedupe of + % shared level 0, dependency wiring, chunk-shape choice) is covered + % without a real OME-Zarr store. Also drives the auto tile-shape + % chooser through makePyramid to catch integration regressions that + % the standalone TestChooseTileShape suite would miss. + + properties + session + subjectID + end + + methods (TestMethodSetup) + function build(testCase) + d = fullfile(tempname, 'makepyramid'); + mkdir(d); + testCase.addTeardown(@() rmdir(fileparts(d), 's')); + S = ndi.session.dir('makepyramid', d); + sub = ndi.document('subject', 'base.session_id', S.id(), ... + 'subject.local_identifier', 'lightsheet@vhlab'); + S.database_add(sub); + testCase.session = S; + testCase.subjectID = sub.id(); + end + end + + methods (Static) + function ax = axes3D() + ax = struct('name', {'z','y','x'}, 'type', {'space','space','space'}, ... + 'unit', {'micrometer','micrometer','micrometer'}); + end + + function levels = dyadicLevels(paths, shape0, vox0, dtype) + n = numel(paths); + levels = repmat(struct( ... + 'path', '', 'shape', [], 'chunks', [], 'dtype', dtype, ... + 'scale', [], 'translation', [0 0 0]), n, 1); + for k = 1:n + levels(k).path = paths{k}; + levels(k).shape = max(1, floor(shape0 / 2^(k-1))); + % Source zarr's own chunks are irrelevant to the doc -- + % makePyramid re-chooses them -- but supply something + % plausible so the input struct is well-formed. + levels(k).chunks = min(levels(k).shape, [64 64 64]); + levels(k).scale = vox0 * 2^(k-1); + levels(k).translation = [0 0 0]; + end + end + + function py = pyramidEntry(name, typ, levels) + py = struct('name', name, 'type', typ, ... + 'axes', ndi.unittest.fun.doc.lightsheet.TestMakePyramid.axes3D(), ... + 'levels', levels); + end + + function pys = twoPyramidsWithSharedLevel0() + L0 = ndi.unittest.fun.doc.lightsheet.TestMakePyramid.dyadicLevels( ... + {'0','mean/1','mean/2'}, [128 512 512], [1 1 1], 'uint16'); + L1 = ndi.unittest.fun.doc.lightsheet.TestMakePyramid.dyadicLevels( ... + {'0','max/1','max/2'}, [128 512 512], [1 1 1], 'uint16'); + pys = [ndi.unittest.fun.doc.lightsheet.TestMakePyramid.pyramidEntry( ... + 'mean', 'box', L0); ... + ndi.unittest.fun.doc.lightsheet.TestMakePyramid.pyramidEntry( ... + 'max', 'max', L1)]; + end + end + + methods (Test) + + function testOnePyramidOneLevelWritesTwoDocs(testCase) + L = ndi.unittest.fun.doc.lightsheet.TestMakePyramid.dyadicLevels( ... + {'0'}, [64 128 128], [1 1 1], 'uint16'); + py = ndi.unittest.fun.doc.lightsheet.TestMakePyramid.pyramidEntry( ... + 'raw', 'box', L); + [pdoc, lds, shared] = ndi.fun.doc.lightsheet.makePyramid( ... + testCase.session, py, {'mean'}, ... + 'subjectID', testCase.subjectID); + testCase.verifyFalse(shared); + testCase.verifyClass(pdoc, 'ndi.document'); + testCase.verifyEqual(numel(lds), 1); + % The level document is written with reduction_function = + % the entry's reduction, since only one pyramid uses this + % path. + p = lds{1}.document_properties.lightsheetZarrLevel; + testCase.verifyEqual(p.reduction_function, 'mean'); + testCase.verifyEqual(p.level, 0); + end + + function testTwoPyramidsSharedLevel0DedupesToFiveDocs(testCase) + pys = ndi.unittest.fun.doc.lightsheet.TestMakePyramid.twoPyramidsWithSharedLevel0(); + [pdoc, lds, shared] = ndi.fun.doc.lightsheet.makePyramid( ... + testCase.session, pys, {'mean','max'}, ... + 'subjectID', testCase.subjectID); + testCase.verifyTrue(shared); + % 5 unique paths: shared '0', mean/1, mean/2, max/1, max/2. + testCase.verifyEqual(numel(lds), 5); + reductions = cellfun(@(d) ... + d.document_properties.lightsheetZarrLevel.reduction_function, ... + lds, 'UniformOutput', false); + testCase.verifyEqual(sum(strcmp(reductions, 'none')), 1); + testCase.verifyEqual(sum(strcmp(reductions, 'mean')), 2); + testCase.verifyEqual(sum(strcmp(reductions, 'max')), 2); + end + + function testChunksAutoChosenByBudget(testCase) + % Level 0 shape 128 x 512 x 512 uint16 with the 32 MB + % default budget lands on the whole volume as one chunk + % (128*512*512*2 = 64 MB uncompressed, past the budget, + % but every axis is clamped so chunks == shape). The + % stamped chunks should be within 1.3x of the budget OR + % equal to the level shape, which is the shape-clamp + % case. + pys = ndi.unittest.fun.doc.lightsheet.TestMakePyramid.twoPyramidsWithSharedLevel0(); + [~, lds] = ndi.fun.doc.lightsheet.makePyramid( ... + testCase.session, pys, {'mean','max'}, ... + 'subjectID', testCase.subjectID); + for k = 1:numel(lds) + p = lds{k}.document_properties.lightsheetZarrLevel; + bytes = prod(p.chunks) * 2; + withinBudget = bytes <= 1.3 * 32 * 2^20; + clampedToShape = isequal(p.chunks(:).', p.shape(:).'); + testCase.verifyTrue(withinBudget || clampedToShape, ... + sprintf(['chunks %s produced %d bytes; neither ' ... + 'within 1.3x of 32 MB budget nor clamped to ' ... + 'shape %s'], mat2str(p.chunks), bytes, ... + mat2str(p.shape))); + testCase.verifyEqual(numel(p.chunks), numel(p.shape)); + testCase.verifyEqual(p.chunk_grid, ceil(p.shape ./ p.chunks)); + end + end + + function testExplicitChunksOverrideBudget(testCase) + L = ndi.unittest.fun.doc.lightsheet.TestMakePyramid.dyadicLevels( ... + {'0','1'}, [64 128 128], [1 1 1], 'uint16'); + py = ndi.unittest.fun.doc.lightsheet.TestMakePyramid.pyramidEntry( ... + 'mean', 'box', L); + [~, lds] = ndi.fun.doc.lightsheet.makePyramid( ... + testCase.session, py, {'mean'}, ... + 'subjectID', testCase.subjectID, ... + 'chunks', [32 32 32]); + % Explicit chunks used verbatim on every level (clamped by + % shape). + p0 = lds{1}.document_properties.lightsheetZarrLevel; + p1 = lds{2}.document_properties.lightsheetZarrLevel; + testCase.verifyEqual(p0.chunks, [32 32 32]); + testCase.verifyEqual(p1.chunks, [32 32 32]); + end + + function testExplicitChunksClampToShape(testCase) + L = ndi.unittest.fun.doc.lightsheet.TestMakePyramid.dyadicLevels( ... + {'0'}, [16 32 32], [1 1 1], 'uint16'); + py = ndi.unittest.fun.doc.lightsheet.TestMakePyramid.pyramidEntry( ... + 'mean', 'box', L); + [~, lds] = ndi.fun.doc.lightsheet.makePyramid( ... + testCase.session, py, {'mean'}, ... + 'subjectID', testCase.subjectID, ... + 'chunks', [64 64 64]); + % Explicit chunks larger than shape are clamped to shape. + p = lds{1}.document_properties.lightsheetZarrLevel; + testCase.verifyEqual(p.chunks, [16 32 32]); + end + + function testExplicitChunksArityMismatchErrors(testCase) + L = ndi.unittest.fun.doc.lightsheet.TestMakePyramid.dyadicLevels( ... + {'0'}, [16 32 32], [1 1 1], 'uint16'); + py = ndi.unittest.fun.doc.lightsheet.TestMakePyramid.pyramidEntry( ... + 'mean', 'box', L); + testCase.verifyError(@() ndi.fun.doc.lightsheet.makePyramid( ... + testCase.session, py, {'mean'}, ... + 'subjectID', testCase.subjectID, ... + 'chunks', [32 32]), ... + 'NDI:lightsheet:makePyramid:chunksArity'); + end + + function testReductionCellArityIsChecked(testCase) + L = ndi.unittest.fun.doc.lightsheet.TestMakePyramid.dyadicLevels( ... + {'0'}, [16 32 32], [1 1 1], 'uint16'); + py = ndi.unittest.fun.doc.lightsheet.TestMakePyramid.pyramidEntry( ... + 'mean', 'box', L); + testCase.verifyError(@() ndi.fun.doc.lightsheet.makePyramid( ... + testCase.session, py, {'mean','max'}, ... + 'subjectID', testCase.subjectID), ... + 'NDI:lightsheet:makePyramid:reductionArity'); + end + + function testParentCarriesLevel0Frame(testCase) + pys = ndi.unittest.fun.doc.lightsheet.TestMakePyramid.twoPyramidsWithSharedLevel0(); + [pdoc, ~, ~] = ndi.fun.doc.lightsheet.makePyramid( ... + testCase.session, pys, {'mean','max'}, ... + 'subjectID', testCase.subjectID, ... + 'pipelineVersion', 'unit-test-1', ... + 'label', 'my volume'); + p = pdoc.document_properties.lightsheetZarrPyramid; + testCase.verifyEqual(p.axes_order, 'zyx'); + testCase.verifyEqual(reshape(p.shape_level0, 1, []), [128 512 512]); + testCase.verifyEqual(reshape(p.voxel_size_level0, 1, []), [1 1 1]); + testCase.verifyEqual(char(p.dtype), 'uint16'); + testCase.verifyEqual(char(p.pipeline_version), 'unit-test-1'); + testCase.verifyEqual(char(p.label), 'my volume'); + end + + function testDependencyWiringSubjectAndPyramid(testCase) + pys = ndi.unittest.fun.doc.lightsheet.TestMakePyramid.twoPyramidsWithSharedLevel0(); + [pdoc, lds, ~] = ndi.fun.doc.lightsheet.makePyramid( ... + testCase.session, pys, {'mean','max'}, ... + 'subjectID', testCase.subjectID); + testCase.verifyEqual( ... + pdoc.dependency_value('subject_id'), testCase.subjectID); + for k = 1:numel(lds) + testCase.verifyEqual( ... + lds{k}.dependency_value('lightsheetZarrPyramid_id'), ... + pdoc.id()); + testCase.verifyEqual( ... + lds{k}.dependency_value('subject_id'), testCase.subjectID); + end + end + + function testPyramidTypeCarried(testCase) + % The parent's pyramid_type is the FIRST entry's type. Two + % pyramids with different types don't collide (they share + % the parent, since a store is one source volume). + pys = ndi.unittest.fun.doc.lightsheet.TestMakePyramid.twoPyramidsWithSharedLevel0(); + [pdoc, ~, ~] = ndi.fun.doc.lightsheet.makePyramid( ... + testCase.session, pys, {'mean','max'}, ... + 'subjectID', testCase.subjectID); + p = pdoc.document_properties.lightsheetZarrPyramid; + testCase.verifyEqual(char(p.pyramid_type), 'box'); + end + + function testLevelDocsWrittenToDatabase(testCase) + L = ndi.unittest.fun.doc.lightsheet.TestMakePyramid.dyadicLevels( ... + {'0','1'}, [64 128 128], [1 1 1], 'uint16'); + py = ndi.unittest.fun.doc.lightsheet.TestMakePyramid.pyramidEntry( ... + 'mean', 'box', L); + [~, lds] = ndi.fun.doc.lightsheet.makePyramid( ... + testCase.session, py, {'mean'}, ... + 'subjectID', testCase.subjectID); + for k = 1:numel(lds) + back = testCase.session.database_search( ... + ndi.query('base.id', 'exact_string', lds{k}.id())); + testCase.verifyEqual(numel(back), 1); + end + end + + end +end diff --git a/tests/+ndi/+unittest/+fun/+doc/+lightsheet/TestMakeSourceFile.m b/tests/+ndi/+unittest/+fun/+doc/+lightsheet/TestMakeSourceFile.m new file mode 100644 index 0000000000..13ce79d148 --- /dev/null +++ b/tests/+ndi/+unittest/+fun/+doc/+lightsheet/TestMakeSourceFile.m @@ -0,0 +1,117 @@ +classdef TestMakeSourceFile < matlab.unittest.TestCase + % TestMakeSourceFile - describe (fileReference) vs attach (generic_file). + % + % Mirrors +gene's TestGeneFromFiles source-file coverage, but for the + % OME-Zarr case: source stores are directory trees, and describing + % rather than attaching is the default so a 100 GB tree does not get + % copied alongside the pyramid summarising it. Checksum is taken + % over .zattrs (small, identifies the multiscales layout uniquely) + % rather than every chunk in the tree. + + properties + session + end + + methods (TestMethodSetup) + function build(testCase) + d = fullfile(tempname, 'srcfile'); + mkdir(d); + testCase.addTeardown(@() rmdir(fileparts(d), 's')); + testCase.session = ndi.session.dir('srcfile', d); + end + end + + methods (Access = private) + function f = aFile(testCase, name, contents) + f = fullfile(tempname, name); + mkdir(fileparts(f)); + testCase.addTeardown(@() rmdir(fileparts(f), 's')); + fid = fopen(f, 'w'); fwrite(fid, contents); fclose(fid); + end + + function d = aZarrStore(testCase, name) + d = fullfile(tempname, name); + mkdir(d); + testCase.addTeardown(@() rmdir(fileparts(d), 's')); + zattrs = fullfile(d, '.zattrs'); + fid = fopen(zattrs, 'w'); + fwrite(fid, '{"multiscales":[{"name":"raw","axes":[]}]}'); + fclose(fid); + end + end + + methods (Test) + + function testFileDescribeMakesFileReference(testCase) + f = testCase.aFile('a.bin', 'abcdef'); + doc = ndi.fun.doc.lightsheet.makeSourceFile(testCase.session, f); + testCase.verifyEqual( ... + doc.document_properties.document_class.class_name, ... + 'fileReference'); + r = doc.document_properties.fileReference; + testCase.verifyEqual(r.filename, 'a.bin'); + testCase.verifyEqual(r.fileSize, 6); + testCase.verifyNotEmpty(r.checksum); + testCase.verifyEqual(r.checksumAlgorithm, 'MD5'); + testCase.verifyEmpty(doc.current_file_list(), ... + 'describing must not attach the file'); + end + + function testFileAttachMakesGenericFile(testCase) + f = testCase.aFile('a.bin', 'abcdef'); + doc = ndi.fun.doc.lightsheet.makeSourceFile( ... + testCase.session, f, 'attachFile', true); + testCase.verifyEqual( ... + doc.document_properties.document_class.class_name, ... + 'generic_file'); + files = doc.current_file_list(); + testCase.verifyEqual(numel(files), 1); + testCase.verifyEqual(char(files{1}), 'generic_file.ext'); + end + + function testDirectoryDescribeChecksumsZattrs(testCase) + d = testCase.aZarrStore('vol.ome.zarr'); + doc = ndi.fun.doc.lightsheet.makeSourceFile(testCase.session, d); + r = doc.document_properties.fileReference; + testCase.verifyEqual(r.filename, 'vol.ome.zarr'); + testCase.verifyNotEmpty(r.checksum); + testCase.verifyEqual(r.checksumAlgorithm, 'MD5'); + % fileSize is 0 for a directory (only for plain files). + testCase.verifyEqual(r.fileSize, 0); + end + + function testAttachingADirectoryIsRefused(testCase) + d = testCase.aZarrStore('vol.ome.zarr'); + testCase.verifyError(@() ndi.fun.doc.lightsheet.makeSourceFile( ... + testCase.session, d, 'attachFile', true), ... + 'NDI:lightsheet:makeSourceFile:cannotAttachDirectory'); + end + + function testMissingPathErrors(testCase) + testCase.verifyError(@() ndi.fun.doc.lightsheet.makeSourceFile( ... + testCase.session, '/no/such/thing.zarr'), ... + 'NDI:lightsheet:makeSourceFile:noSuchPath'); + end + + function testChecksumFalseLeavesFieldEmpty(testCase) + f = testCase.aFile('nochk.bin', 'x'); + doc = ndi.fun.doc.lightsheet.makeSourceFile( ... + testCase.session, f, 'checksum', false); + r = doc.document_properties.fileReference; + testCase.verifyEmpty(r.checksum); + testCase.verifyEmpty(r.checksumAlgorithm); + end + + function testDescribedDocIsAValidDocument(testCase) + % The regression test that started this whole file: a + % described document has to validate against its schema. A + % fileReference has no `file` slot, so this is why the + % class was chosen over generic_file. + f = testCase.aFile('valid.txt', 'hi'); + doc = ndi.fun.doc.lightsheet.makeSourceFile(testCase.session, f); + testCase.verifyWarningFree(@() ... + testCase.session.database_add(doc)); + end + + end +end diff --git a/tests/+ndi/+unittest/+fun/+doc/+lightsheet/TestReduceBlocks.m b/tests/+ndi/+unittest/+fun/+doc/+lightsheet/TestReduceBlocks.m new file mode 100644 index 0000000000..a4ffb671e3 --- /dev/null +++ b/tests/+ndi/+unittest/+fun/+doc/+lightsheet/TestReduceBlocks.m @@ -0,0 +1,54 @@ +classdef TestReduceBlocks < matlab.unittest.TestCase + % TestReduceBlocks - vectorized block reduction for building pyramid levels. + % + % Must agree with ndr.format.omezarr.reduce (the per-voxel reference): + % output i covers inputs (i-1)*f+1 .. i*f, trailing partial block + % reduced over what it has. + + methods (Test) + + function testMeanMatchesNdrReduce(testCase) + rng(1); + a = uint16(randi(65535, [5 7 9])); + got = ndi.fun.doc.lightsheet.reduceBlocks(a, [2 2 2], 'mean', 'double'); + ref = ndr.format.omezarr.reduce(a, [2 2 2], 'mean'); + testCase.verifyEqual(got, ref, 'AbsTol', 1e-2); + end + + function testMaxMatchesNdrReduce(testCase) + rng(2); + a = uint16(randi(65535, [6 5 3])); + got = ndi.fun.doc.lightsheet.reduceBlocks(a, [2 2 2], 'max'); + ref = ndr.format.omezarr.reduce(a, [2 2 2], 'max'); + testCase.verifyEqual(got, ref); + testCase.verifyClass(got, 'uint16'); + end + + function testPartialTrailingBlock(testCase) + % Length 5, factor 2: blocks of 2, 2 and 1. + a = uint16([1 3 5 7 9]); + got = ndi.fun.doc.lightsheet.reduceBlocks(a, [1 2], 'mean', 'double'); + testCase.verifyEqual(got, [2 6 9]); + end + + function testMeanIntoIntegerRounds(testCase) + a = uint16([1 2]); + got = ndi.fun.doc.lightsheet.reduceBlocks(a, [1 2], 'mean'); + testCase.verifyEqual(got, uint16(2)); % 1.5 rounds to 2 + end + + function testTrailingSingletonAxesInFiveD(testCase) + % t, c, z, y, x with a singleton x: factor entries for every axis. + a = reshape(uint16(1:16), [1 2 4 2 1]); + got = ndi.fun.doc.lightsheet.reduceBlocks(a, [1 1 2 2 2], 'max'); + testCase.verifyEqual(size(got, 1:5), [1 2 2 1 1]); + ref = ndr.format.omezarr.reduce(a, [1 1 2 2], 'max'); + testCase.verifyEqual(got(:), ref(:)); + end + + function testFactorOneIsIdentity(testCase) + a = uint16(magic(4)); + testCase.verifyEqual(ndi.fun.doc.lightsheet.reduceBlocks(a, [1 1], 'mean'), a); + end + end +end diff --git a/tests/+ndi/+unittest/+fun/+doc/+lightsheet/TestViewCommand.m b/tests/+ndi/+unittest/+fun/+doc/+lightsheet/TestViewCommand.m new file mode 100644 index 0000000000..39163d67f5 --- /dev/null +++ b/tests/+ndi/+unittest/+fun/+doc/+lightsheet/TestViewCommand.m @@ -0,0 +1,144 @@ +classdef TestViewCommand < matlab.unittest.TestCase + % TestViewCommand - the pure command builder ndi.gui.app.LightsheetZarrManager + % calls to hand a pyramid to the napari viewer. + % + % viewCommand does not launch anything or touch the filesystem, so a + % test can assert the exact command string without a display, without + % Python, and without a session. Every branch of the flag surface is + % exercised so a UI change that quietly drops --no-controls or + % --reduction fails here rather than in a demo. + + properties (Constant) + Launcher = '/usr/local/bin/napariViewLightsheet' + Session = '/tmp/session1' + Pyramid = '4126a1b2c3d4e5f6_a1b2c3d4e5f6a1b2' + end + + methods (Test) + + function testMinimalCommand(testCase) + cmd = ndi.fun.doc.lightsheet.viewCommand( ... + testCase.Launcher, testCase.Session, testCase.Pyramid); + testCase.verifyEqual(cmd, ... + sprintf('''%s'' ''%s'' --pyramid ''%s''', ... + testCase.Launcher, testCase.Session, testCase.Pyramid)); + end + + function testReductionFlag(testCase) + cmd = ndi.fun.doc.lightsheet.viewCommand( ... + testCase.Launcher, testCase.Session, testCase.Pyramid, ... + 'reduction', 'max'); + testCase.verifyTrue(contains(cmd, '--reduction ''max''')); + end + + function testReductionMeanFlag(testCase) + cmd = ndi.fun.doc.lightsheet.viewCommand( ... + testCase.Launcher, testCase.Session, testCase.Pyramid, ... + 'reduction', 'mean'); + testCase.verifyTrue(contains(cmd, '--reduction ''mean''')); + end + + function testChannelFlag(testCase) + cmd = ndi.fun.doc.lightsheet.viewCommand( ... + testCase.Launcher, testCase.Session, testCase.Pyramid, ... + 'channel', 2); + testCase.verifyTrue(contains(cmd, '--channel 2')); + end + + function testChannelDefaultIsOmitted(testCase) + cmd = ndi.fun.doc.lightsheet.viewCommand( ... + testCase.Launcher, testCase.Session, testCase.Pyramid); + testCase.verifyFalse(contains(cmd, '--channel'), ... + 'channel=-1 (default) must not be emitted.'); + end + + function testLevelFlag(testCase) + cmd = ndi.fun.doc.lightsheet.viewCommand( ... + testCase.Launcher, testCase.Session, testCase.Pyramid, ... + 'level', 0); + testCase.verifyTrue(contains(cmd, '--level 0')); + end + + function testLevelDefaultIsOmitted(testCase) + cmd = ndi.fun.doc.lightsheet.viewCommand( ... + testCase.Launcher, testCase.Session, testCase.Pyramid); + testCase.verifyFalse(contains(cmd, '--level'), ... + 'level=-1 (default) must not be emitted.'); + end + + function testNoControls(testCase) + cmd = ndi.fun.doc.lightsheet.viewCommand( ... + testCase.Launcher, testCase.Session, testCase.Pyramid, ... + 'controls', false); + testCase.verifyTrue(contains(cmd, '--no-controls')); + end + + function testControlsDefaultIsOn(testCase) + cmd = ndi.fun.doc.lightsheet.viewCommand( ... + testCase.Launcher, testCase.Session, testCase.Pyramid); + testCase.verifyFalse(contains(cmd, '--no-controls'), ... + 'controls=true (default) must not emit --no-controls.'); + end + + function testNameFlag(testCase) + cmd = ndi.fun.doc.lightsheet.viewCommand( ... + testCase.Launcher, testCase.Session, testCase.Pyramid, ... + 'name', 'my layer'); + testCase.verifyTrue(contains(cmd, '--name')); + testCase.verifyTrue(contains(cmd, 'my layer'), ... + 'the layer name must appear in the command.'); + end + + function testEmptyLauncherErrors(testCase) + testCase.verifyError( ... + @() ndi.fun.doc.lightsheet.viewCommand('', ... + testCase.Session, testCase.Pyramid), ... + 'NDI:lightsheet:viewCommand:noLauncher'); + end + + function testEmptySessionPathErrors(testCase) + testCase.verifyError( ... + @() ndi.fun.doc.lightsheet.viewCommand( ... + testCase.Launcher, '', testCase.Pyramid), ... + 'NDI:lightsheet:viewCommand:noSessionPath'); + end + + function testEmptyPyramidIDErrors(testCase) + % viewCommand refuses to build a command without a pyramid id; + % the viewer will not guess which pyramid to open on a session + % that holds more than one. + testCase.verifyError( ... + @() ndi.fun.doc.lightsheet.viewCommand( ... + testCase.Launcher, testCase.Session, ''), ... + 'NDI:lightsheet:viewCommand:noPyramidID'); + end + + function testInvalidReductionErrors(testCase) + testCase.verifyError( ... + @() ndi.fun.doc.lightsheet.viewCommand( ... + testCase.Launcher, testCase.Session, testCase.Pyramid, ... + 'reduction', 'median'), ... + 'MATLAB:validators:mustBeMember'); + end + + function testFullFlagSurface(testCase) + % Every flag together; order matters for a shell command, and + % downstream (napariViewLightsheet argparse) accepts the flags + % in any order, so the assertion is on presence + values. + cmd = ndi.fun.doc.lightsheet.viewCommand( ... + testCase.Launcher, testCase.Session, testCase.Pyramid, ... + 'reduction', 'max', 'channel', 1, 'level', 2, ... + 'controls', false, 'name', 'V1'); + testCase.verifyTrue(startsWith(cmd, ... + sprintf('''%s'' ''%s'' --pyramid ''%s''', ... + testCase.Launcher, testCase.Session, testCase.Pyramid)), ... + 'launcher, session, --pyramid must lead the command.'); + testCase.verifyTrue(contains(cmd, '--reduction ''max''')); + testCase.verifyTrue(contains(cmd, '--channel 1')); + testCase.verifyTrue(contains(cmd, '--level 2')); + testCase.verifyTrue(contains(cmd, '--no-controls')); + testCase.verifyTrue(contains(cmd, '--name')); + testCase.verifyTrue(contains(cmd, 'V1')); + end + end +end diff --git a/tests/+ndi/+unittest/+gui/+app/TestLightsheetZarrManager.m b/tests/+ndi/+unittest/+gui/+app/TestLightsheetZarrManager.m new file mode 100644 index 0000000000..7c0d7bf0f4 --- /dev/null +++ b/tests/+ndi/+unittest/+gui/+app/TestLightsheetZarrManager.m @@ -0,0 +1,110 @@ +classdef TestLightsheetZarrManager < matlab.unittest.TestCase + % TestLightsheetZarrManager - non-GUI paths of the manager. + % + % The GUI (uifigure construction, callbacks) is not exercised here; + % headless CI cannot open a figure. What is testable is the + % launcherPath resolver (preference fallback) and deletionPlan (a + % pure query wrapper used before database_rm cascades). + + properties + session + subjectID + end + + methods (TestMethodSetup) + function build(testCase) + d = fullfile(tempname, 'lszmgr'); + mkdir(d); + testCase.addTeardown(@() rmdir(fileparts(d), 's')); + S = ndi.session.dir('lszmgr', d); + sub = ndi.document('subject', 'base.session_id', S.id(), ... + 'subject.local_identifier', 'lightsheet@vhlab'); + S.database_add(sub); + testCase.session = S; + testCase.subjectID = sub.id(); + end + end + + methods (Test) + + function testLauncherPathFallsBackToDefault(testCase) + % Without a preference set, the launcher is the class's + % declared default. Should be an absolute path ending in + % 'napariViewLightsheet'. + p = ndi.gui.app.LightsheetZarrManager.launcherPath(); + testCase.verifyClass(p, 'char'); + testCase.verifyNotEmpty(p); + [~, name] = fileparts(p); + testCase.verifyEqual(name, 'napariViewLightsheet'); + end + + function testDeletionPlanCollectsLevelsAndParent(testCase) + % Deletion is a cascade; the plan enumerates everything + % the manager will remove so the UI can name the counts + % before doing it. + pys = ndi.unittest.fun.doc.lightsheet.TestMakePyramid.twoPyramidsWithSharedLevel0(); + [pdoc, lds, ~] = ndi.fun.doc.lightsheet.makePyramid( ... + testCase.session, pys, {'mean','max'}, ... + 'subjectID', testCase.subjectID); + plan = ndi.gui.app.LightsheetZarrManager.deletionPlan( ... + testCase.session, pdoc.id()); + testCase.verifyEqual(plan.pyramidID, pdoc.id()); + testCase.verifyEqual(plan.nLevels, numel(lds)); + testCase.verifyEqual(numel(plan.docsToRemove), ... + numel(lds) + 1, ... + 'plan should include every level plus the parent'); + end + + function testDeletionPlanOnAbsentPyramidReportsZero(testCase) + % A never-created id has no children and no parent; the + % plan should return an empty removal list. + plan = ndi.gui.app.LightsheetZarrManager.deletionPlan( ... + testCase.session, ... + '00000000-0000-0000-0000-000000000000'); + testCase.verifyEqual(plan.nLevels, 0); + testCase.verifyEmpty(plan.docsToRemove); + end + + function testDeletionMessageNamesTheCounts(testCase) + % The confirmation must distinguish 1 from many. The pyramid + % counts as 1, plus every level. + plan = struct('pyramidID','abcd','nLevels',7,'docsToRemove',{{}}); + msg = ndi.gui.app.LightsheetZarrManager.deletionMessage( ... + plan, 'my-pyramid'); + testCase.verifySubstring(msg, '8 document'); + testCase.verifySubstring(msg, '7 level'); + testCase.verifySubstring(msg, 'my-pyramid'); + end + + function testSubjectNameMapUsesLocalIdentifier(testCase) + % pyramidRows shows the subject's local_identifier, not its + % raw id. subjectNameMap is where that lookup lives; a + % subject with a local_identifier should be findable by its + % document id. + m = ndi.gui.app.LightsheetZarrManager.subjectNameMap( ... + testCase.session); + testCase.verifyTrue(isKey(m, testCase.subjectID)); + testCase.verifyEqual(m(testCase.subjectID), 'lightsheet@vhlab'); + end + + function testPyramidRowsCarriesSubjectNameAndDoc(testCase) + % One row per pyramid parent. Subject is the local_identifier + % (not the raw id), and the row keeps the raw doc alongside + % the id so the delete/view callbacks can act on it. + pys = ndi.unittest.fun.doc.lightsheet.TestMakePyramid.twoPyramidsWithSharedLevel0(); + [pdoc, ~, ~] = ndi.fun.doc.lightsheet.makePyramid( ... + testCase.session, pys, {'mean','max'}, ... + 'subjectID', testCase.subjectID); + rows = ndi.gui.app.LightsheetZarrManager.pyramidRows( ... + testCase.session); + testCase.verifyNotEmpty(rows); + match = find(strcmp({rows.id}, pdoc.id()), 1); + testCase.verifyNotEmpty(match); + r = rows(match); + testCase.verifyEqual(r.subject, 'lightsheet@vhlab'); + testCase.verifyEqual(r.subjectID, testCase.subjectID); + testCase.verifyNotEmpty(r.doc); + end + + end +end diff --git a/tests/+ndi/+unittest/+gui/+component/TestProgressBarWindow.m b/tests/+ndi/+unittest/+gui/+component/TestProgressBarWindow.m index ca708fc25d..c25facdff4 100644 --- a/tests/+ndi/+unittest/+gui/+component/TestProgressBarWindow.m +++ b/tests/+ndi/+unittest/+gui/+component/TestProgressBarWindow.m @@ -123,6 +123,40 @@ function testAddOneBar(testCase) testCase.verifyNotEmpty(app.ProgressBars(1).Axes, 'Axes handle should not be empty.'); testCase.verifyNotEmpty(app.ProgressBars(1).Patch, 'Patch handle should not be empty.'); end + + function testBarFillsItsRow(testCase) + % The bar's axes sit in a borderless panel that the grid lays + % out, the axes' INNER box BarMargin px inside it. In a grid cell + % a uiaxes keeps room for hidden tick labels; on R2026a that + % left a 2.7 px bar in a 28 px row. + app = ndi.gui.component.ProgressBarWindow('Fill Test', 'Visible', 'off'); + testCase.addTeardown(@delete, app.ProgressFigure); + app.addBar('Label', 'Task 1', 'Tag', 'T1'); + app.addBar('Label', 'Task 2', 'Tag', 'T2'); + ax = app.ProgressBars(2).Axes; + testCase.verifyClass(ax.Parent, 'matlab.ui.container.Panel'); + testCase.verifyEqual(ax.Parent.Layout.Row, 4, 'the panel takes the bar row'); + testCase.verifyEqual(ax.PositionConstraint, 'innerposition'); + testCase.verifyEqual(ax.Units, 'pixels'); + testCase.verifyNotEmpty(ax.Parent.SizeChangedFcn, 'the bar follows its panel''s size'); + testCase.verifyEqual(char(ax.Parent.AutoResizeChildren), 'off', ... + 'SizeChangedFcn does not run while AutoResizeChildren is on'); + % BarMargin px inside the panel on every side + m = ndi.gui.component.ProgressBarWindow.BarMargin; + drawnow + ndi.gui.component.ProgressBarWindow.fitBarAxes(ax, ax.Parent); + pos = getpixelposition(ax.Parent); % the grid sets the panel's size + testCase.verifyEqual(ax.InnerPosition, ... + [1+m, 1+m, max(pos(3)-2*m, 1), max(pos(4)-2*m, 1)], 'AbsTol', 1e-6); + + % removing the first bar moves the second panel up and deletes + % the first bar's panel along with its axes + firstPanel = app.ProgressBars(1).Axes.Parent; + app.updateBar('T1', 1); + app.removeBar('T1'); + testCase.verifyFalse(isvalid(firstPanel), 'the removed bar''s panel is deleted'); + testCase.verifyEqual(ax.Parent.Layout.Row, 2, 'the remaining bar''s panel moves up'); + end function testAddMultipleBars(testCase) app = ndi.gui.component.ProgressBarWindow('Multi Bar Test', 'Visible', 'off'); testCase.addTeardown(@delete, app.ProgressFigure); @@ -248,7 +282,7 @@ function testRemoveBar(testCase) testCase.verifyEqual(app.ProgressBars(2).State, 'Open', 'Remaining bar state incorrect.'); testCase.verifyNumElements(app.ProgressGrid.RowHeight, initialRowCount - 2, 'Grid rows not removed correctly.'); testCase.verifyEqual(app.ProgressBars(2).Label.Layout.Row, 1); - testCase.verifyEqual(app.ProgressBars(2).Axes.Layout.Row, 2); + testCase.verifyEqual(app.ProgressBars(2).Axes.Parent.Layout.Row, 2); % the bar's panel % Verify properties of removed bar testCase.verifyEqual(app.ProgressBars(1).Tag, 'TRemove', 'Removed bar is incorrect.'); testCase.verifyEqual(app.ProgressBars(1).State, 'Closed', 'Removed bar state incorrect.'); @@ -408,10 +442,16 @@ function testAutoCloseOnTimeout(testCase) testCase.addTeardown(@delete, app.ProgressFigure); originalTimeout = app.Timeout; app.setTimeout(duration(0,0,0.1)); % 0.1 seconds - testCase.addTeardown(@() setfield(app,'Timeout',originalTimeout)); + % Guard the restore teardown against AutoDelete: this test's + % updateBar call auto-closes both bars, and with AutoDelete + % on (the constructor default) the app is then deleted. + % Touching a deleted AppBase handle raises + % MATLAB:class:InvalidHandle, which would fail the test + % even though the assertion above succeeded. + testCase.addTeardown(@() restoreTimeoutIfAlive(app, originalTimeout)); app.addBar('Tag', 'TAutoTimeout', 'Auto', true); app.addBar('Tag', 'TUpdated', 'Auto', true); - + pause(0.2); % Wait for timeout testCase.verifyWarning(@() app.updateBar('TUpdated', 0.2), 'ProgressBarWindow:AutoCloseOnTimeout'); end @@ -589,4 +629,13 @@ function verifyButtonIconIsError(testCase, buttonHandle) end end end +end + +function restoreTimeoutIfAlive(app, originalTimeout) + % Teardown helper: restore the app's Timeout only if the app is + % still a live handle. A test that drives AutoDelete to tear the + % window down cannot then reach through app to write a property. + if isvalid(app) + app.Timeout = originalTimeout; + end end \ No newline at end of file diff --git a/tests/+ndi/+unittest/+gui/+component/TestProgressBarWindowDocked.m b/tests/+ndi/+unittest/+gui/+component/TestProgressBarWindowDocked.m index 35c624ef09..3031777701 100644 --- a/tests/+ndi/+unittest/+gui/+component/TestProgressBarWindowDocked.m +++ b/tests/+ndi/+unittest/+gui/+component/TestProgressBarWindowDocked.m @@ -75,8 +75,9 @@ function testDockedBarRenders(testCase) testCase.verifyEqual(app.ProgressBars(1).Progress, 0.5); testCase.verifyEqual(app.ProgressBars(1).Percent.Text, '50%'); - testCase.verifySameHandle(app.ProgressBars(1).Axes.Parent, testCase.Pane.BarGrid, ... - 'Bar axes should be parented to the pane bar grid.'); + % the axes sit in the bar's panel, which the pane's grid lays out + testCase.verifySameHandle(app.ProgressBars(1).Axes.Parent.Parent, testCase.Pane.BarGrid, ... + 'Bar panel should be parented to the pane bar grid.'); end function testCascadeMultipleBars(testCase) diff --git a/tests/+ndi/+unittest/+setup/+V2/TestCreateSession.m b/tests/+ndi/+unittest/+setup/+V2/TestCreateSession.m new file mode 100644 index 0000000000..fee3bbdaba --- /dev/null +++ b/tests/+ndi/+unittest/+setup/+V2/TestCreateSession.m @@ -0,0 +1,91 @@ +classdef TestCreateSession < matlab.unittest.TestCase +%TESTCREATESESSION ndi.setup.V2.createSession makes a session NDI can open. +% +% The point of the function is the round trip: a V2 (did2) database is +% written, and the ordinary ndi.session.dir(PATH) then opens it with the +% same id and reference, through the did2sqlite backend. Filtered (never +% passed) when did2.build or a V2 schema with `study` is missing. +% +% UNVERIFIED: written without MATLAB; test-import-v2.yml is the first run. + + properties + Dir + end + + methods (TestClassSetup) + function needV2(testCase) + testCase.assumeTrue(~isempty(which('did2.build.document')), ... + 'did2.build (DID-matlab V2) is not on the path'); + schemaPath = getenv('DID_SCHEMA_PATH'); + testCase.assumeTrue(~isempty(schemaPath) && ... + ndi.setup.V2.schemaHasClass('study', schemaPath), ... + 'DID_SCHEMA_PATH does not hold a V2 schema with `study`'); + end + end + + methods (TestMethodSetup) + function makeDir(testCase) + testCase.Dir = tempname; + mkdir(testCase.Dir); + testCase.addTeardown(@() rmdir(testCase.Dir, 's')); + end + end + + methods (Test) + function testRoundTrip(testCase) + sid = ndi.ido.unique_id(); + session = ndi.setup.V2.createSession(testCase.Dir, 'day 22-02-01', ... + 'SessionId', sid); + testCase.verifyEqual(session.id(), sid); + testCase.verifyEqual(session.reference, 'day 22-02-01'); + testCase.verifyTrue(isfile(fullfile(testCase.Dir, '.ndi', 'V_eta.sqlite'))); + + reopened = ndi.session.dir(testCase.Dir); + testCase.verifyEqual(reopened.id(), sid); + testCase.verifyEqual(reopened.reference, 'day 22-02-01'); + s = reopened.database_search(ndi.v2.isaQuery('session')); + testCase.verifyNumElements(s, 1); + testCase.verifyEqual(ndi.v2.blockOf(s{1}.document_properties, 'session', 'local_identifier'), 'day 22-02-01'); + end + + function testStudyRelationIsStoredInTheSession(testCase) + study = ndi.ido.unique_id(); + [session, docs] = ndi.setup.V2.createSession(testCase.Dir, 'd1', ... + 'StudyIds', {study}); + testCase.verifyNumElements(docs, 2); + rel = session.database_search(ndi.query('', 'isa', 'directed_relation')); + testCase.verifyNumElements(rel, 1); + r = rel{1}.document_properties; + testCase.verifyEqual(r.directed_relation.relation.name, 'part_of'); + parent = r.depends_on(strcmp({r.depends_on.name}, 'parent_id')); + testCase.verifyEqual(parent.document_id, study); + end + + function testRefusesToOverwriteByDefault(testCase) + ndi.setup.V2.createSession(testCase.Dir, 'd1'); + testCase.verifyError(@() ndi.setup.V2.createSession(testCase.Dir, 'd2'), ... + 'ndi:setup:V2:sessionExists'); + s = ndi.setup.V2.createSession(testCase.Dir, 'd2', 'Overwrite', true); + testCase.verifyEqual(s.reference, 'd2'); + end + + function testWritesInBatches(testCase) + % 1 session document + 4 relations, written 2 at a time, not + % re-validated on insert: every document arrives, and the + % batch count is what writeDocuments reports + studies = arrayfun(@(~) ndi.ido.unique_id(), 1:4, 'UniformOutput', false); + [session, docs] = ndi.setup.V2.createSession(testCase.Dir, 'd1', ... + 'StudyIds', studies, 'BatchSize', 2, 'Validate', false, 'Progress', true); + testCase.verifyNumElements(docs, 5); + testCase.verifyNumElements(session.database_search( ... + ndi.query('', 'isa', 'directed_relation')), 4); + + file = fullfile(testCase.Dir, 'more.sqlite'); + stats = ndi.setup.V2.writeDocuments(file, docs, 'BatchSize', 2, 'Progress', false); + testCase.verifyEqual([stats.documents, stats.batches, stats.files], [5, 3, 0]); + db = did2.database.sqlitedb(file); + testCase.verifyEqual(db.count(), 5); + db.close(); + end + end +end diff --git a/tests/+ndi/+unittest/+setup/+V2/TestDatasetMetadataHaley.m b/tests/+ndi/+unittest/+setup/+V2/TestDatasetMetadataHaley.m new file mode 100644 index 0000000000..50c7c7c26e --- /dev/null +++ b/tests/+ndi/+unittest/+setup/+V2/TestDatasetMetadataHaley.m @@ -0,0 +1,271 @@ +classdef TestDatasetMetadataHaley < matlab.unittest.TestCase +%TESTDATASETMETADATAHALEY Stage 2 of the Haley V2 import, over its real spec. +% +% Builds every dataset-level document from +% +ndi/+setup/+conv/+haley/import_V2_spec.json with +% ndi.setup.V2.datasetMetadata (which validates each one through +% did2.build) and checks what came out: the counts, the links, and that +% spec content with no V2 home is REPORTED rather than dropped. +% +% Needs DID-matlab's did2.build and a V2 schema that has `study`, +% `instance_of`, `awarded_to` and `directed_relation.roles` +% (Waltham-Data-Science/did-schema PR #78). Without them every test is +% FILTERED with a message, never passed; test-import-v2.yml runs this with +% them and fails if anything was filtered. +% +% UNVERIFIED: written without MATLAB; test-import-v2.yml is the first run. + + properties + Spec + Result + end + + methods (TestClassSetup) + function build(testCase) + testCase.assumeTrue(~isempty(which('did2.build.document')), ... + 'did2.build (DID-matlab V2) is not on the path'); + schemaPath = getenv('DID_SCHEMA_PATH'); + testCase.assumeTrue(~isempty(schemaPath) && ... + ndi.setup.V2.schemaHasClass('study', schemaPath), ... + 'DID_SCHEMA_PATH does not hold a V2 schema with `study` (did-schema PR #78)'); + specFile = fullfile(fileparts(which('ndi.setup.conv.haley.import_V2')), ... + 'import_V2_spec.json'); + testCase.Spec = jsondecode(fileread(specFile)); + testCase.Result = ndi.setup.V2.datasetMetadata(testCase.Spec, did.ido.unique_id()); + end + end + + methods (Test) + function testCountsFollowTheSpec(testCase) + s = testCase.Spec; + c = testCase.Result.census; + n = @(cls) sum(c.count(strcmp(c.class, cls))); + testCase.verifyEqual(n('organization'), numel(s.organizations)); + testCase.verifyEqual(n('person'), numel(s.people)); + testCase.verifyEqual(n('funding'), numel(s.funding)); + testCase.verifyEqual(n('study'), numel(s.studies)); + testCase.verifyEqual(n('strain'), numel(s.strains)); + testCase.verifyEqual(n('subject'), numel(s.instruments)); + testCase.verifyEqual(n('chemical'), numel(s.chemicals)); + testCase.verifyEqual(n('formulation'), numel(s.formulations)); + testCase.verifyEqual(n('dataset'), 1); + % every strain has a source, so each adds one stock product + testCase.verifyEqual(n('product'), numel(s.products) + numel(s.strains)); + % 47 before decision #56; + the media kitchen's suborganization_of + % Salk + 6 recipes documented_by WormBook + if ndi.setup.V2.mergedEntities() + % one entity class (2026-10-08): the entity fields become + % assertions and relations, so these totals are a different + % count, not a moved one. What must hold on either schema: + % every document is an entity of a known type, a statement or + % a relation. + kinds = {'organization', 'person', 'funding', 'study', 'strain', ... + 'subject', 'chemical', 'formulation', 'dataset', 'product', ... + 'software', 'publication', 'protocol', 'directed_relation'}; + other = c(~ismember(c.class, kinds), :); + testCase.verifyTrue(all(endsWith(other.class, ... + {'_assertion', '_observation', '_manipulation', '_calculation'})), ... + strjoin(other.class, ', ')); + testCase.verifyFalse(any(strcmp(c.class, 'web_resource')), ... + 'web_resource retired: its URL is an identifier of what it names'); + return; + end + testCase.verifyEqual(n('directed_relation'), 54, ... + 'relation count moved: re-derive it from the spec, do not bump it'); + % 133 before decision #56; + 3 organizations, 1 web resource, + % 4 products, 20 chemicals, 9 formulations, 7 relations; + the + % temperature probe (decision #59: an instrument with no product, + % so no instance_of relation); + the lab's analysis package and + % macOS, stage 10's run environment (decision #60) + testCase.verifyEqual(numel(testCase.Result.documents), 180); + end + + function testStudiesCanBeMintedApartFromTheRest(testCase) + % import_V2 builds the studies in stage 3, beside the sessions + % (decision #50): "exclude" + "only" together are "include". + sid = did.ido.unique_id(); + datasetId = did.ido.unique_id(); + rest = ndi.setup.V2.datasetMetadata(testCase.Spec, sid, ... + 'DatasetId', datasetId, 'Studies', "exclude"); + studies = ndi.setup.V2.datasetMetadata(testCase.Spec, sid, ... + 'DatasetId', datasetId, 'Studies', "only"); + classOf = @(r) cellfun(@ndi.v2.leafName, r.documents, 'UniformOutput', false); + testCase.verifyEqual(sum(strcmp(classOf(rest), 'study')), 0); + testCase.verifyEqual(sum(strcmp(classOf(studies), 'study')), numel(testCase.Spec.studies)); + testCase.verifyEqual(numel(rest.documents) + numel(studies.documents), ... + numel(testCase.Result.documents)); + % each study is part_of the dataset the other call built + rel = studies.documents(strcmp(classOf(studies), 'directed_relation')); + testCase.verifyEqual(numel(rel), numel(testCase.Spec.studies)); + for k = 1:numel(rel) + dep = rel{k}.depends_on; + testCase.verifyTrue(any(strcmp({dep.document_id}, datasetId))); + end + ds = rest.documents(strcmp(classOf(rest), 'dataset')); + testCase.verifyEqual(ds{1}.base.id, datasetId); + testCase.verifyError(@() ndi.setup.V2.datasetMetadata(testCase.Spec, sid, ... + 'Studies', "only"), 'ndi:setup:V2:noDatasetId'); + end + + function testEveryEdgeNamesABuiltDocument(testCase) + docs = testCase.Result.documents; + ids = cellfun(@(d) d.base.id, docs, 'UniformOutput', false); + nEdges = 0; + for k = 1:numel(docs) + for e = 1:numel(docs{k}.depends_on) + nEdges = nEdges + 1; + testCase.verifyTrue(any(strcmp(docs{k}.depends_on(e).document_id, ids)), ... + sprintf('%s edge %s names no document built here', ... + docs{k}.document_class.class_name, docs{k}.depends_on(e).name)); + end + end + testCase.verifyGreaterThan(nEdges, 0, 'no edges were inspected'); + end + + function testAuthorsAreOrderedWithRoles(testCase) + rels = relationsNamed(testCase.Result.documents, 'has_author'); + testCase.verifyEqual(numel(rels), 4); + seq = cellfun(@(r) r.directed_relation.sequence, rels); + testCase.verifyEqual(sort(seq), 1:4); + first = rels{seq == 1}; + testCase.verifyEqual(parentOf(first), testCase.Result.ids('haley')); + testCase.verifyEqual(numel(first.directed_relation.roles), 14); + corr = rels(cellfun(@(r) any(strcmp({r.directed_relation.roles.name}, ... + 'corresponding author')), rels)); + testCase.verifyEqual(sort(cellfun(@parentOf, corr, 'UniformOutput', false)), ... + sort({testCase.Result.ids('aoi'), testCase.Result.ids('chalasani')})); + end + + function testRelationTermsAreCompletedFromTheirValueSet(testCase) + partOf = relationsNamed(testCase.Result.documents, 'part_of'); + testCase.verifyEqual(numel(partOf), numel(testCase.Spec.studies)); + % prefixes are lowercase from 2026-10-08 (CURIE_lookups_meta.json) + testCase.verifyEqual(lower(partOf{1}.directed_relation.relation.node), 'bfo:0000050'); + end + + function testStrainLineageAndStock(testCase) + docs = testCase.Result.documents; + gfp = docs{strcmp(cellfun(@(d) d.base.id, docs, 'UniformOutput', false), ... + testCase.Result.ids('OP50-GFP'))}; + if ndi.setup.V2.mergedEntities() + % one entity class: the background and the stock are relations + from = relationsNamed(docs, 'derived_from'); + from = from(cellfun(@(r) any(strcmp({r.depends_on.document_id}, ... + testCase.Result.ids('OP50-GFP'))), from)); + testCase.verifyTrue(any(cellfun(@(r) strcmp(parentOf(r), ... + testCase.Result.ids('OP50')), from)), 'OP50-GFP derived_from OP50'); + if ndi.setup.V2.schemaHasRelation('distributed_as') + % decision #17: the stock is `distributed_as`, the background + % alone is `derived_from` + testCase.verifyEqual(numel(from), 1, 'its background only'); + stock = relationsNamed(docs, 'distributed_as'); + stock = stock(cellfun(@(r) any(strcmp({r.depends_on.document_id}, ... + testCase.Result.ids('OP50-GFP'))), stock)); + testCase.verifyNumElements(stock, 1, 'OP50-GFP distributed_as its stock'); + else + testCase.verifyEqual(numel(from), 2, 'its background and its stock'); + end + return; + end + bg = gfp.depends_on(strcmp({gfp.depends_on.name}, 'background_strain_id')); + testCase.verifyEqual({bg.document_id}, {testCase.Result.ids('OP50')}); + testCase.verifyTrue(any(strcmp({gfp.depends_on.name}, 'product_id'))); + end + + function testRecipesAreBuiltFromTheirIngredients(testCase) + % decision #56: NGM and the WormBook media, dataset-level + docs = testCase.Result.documents; + ids = testCase.Result.ids; + byId = containers.Map(cellfun(@(d) d.base.id, docs, 'UniformOutput', false), docs); + sc = byId(ids('s_complete')); + ing = sc.depends_on(strcmp({sc.depends_on.name}, 'ingredient_id')); + testCase.verifyEqual({ing.document_id}, {ids('s_basal'), ids('k_citrate_1m_ph6'), ... + ids('trace_metals'), ids('cacl2_1m'), ids('mgso4_1m')}, 'in the recipe''s order'); + testCase.verifyEqual(numel(sc.formulation.value.ingredients), 5, ... + 'one amount per ingredient edge'); + testCase.verifyEqual(sc.depends_on(strcmp({sc.depends_on.name}, 'product_id')).document_id, ... + ids('s_complete_kitchen'), 'made by the Salk media kitchen'); + ngm = byId(ids('ngm_no_peptone')); + ing = ngm.depends_on(strcmp({ngm.depends_on.name}, 'ingredient_id')); + testCase.verifyFalse(any(strcmp({ing.document_id}, ids('peptone_bd')))); + testCase.verifyEqual(numel(ing), 7); + cited = relationsNamed(docs, 'documented_by'); + cited = cited(cellfun(@(r) strcmp(parentOf(r), ids('wormbook_maintenance')), cited)); + testCase.verifyEqual(numel(cited), 6, ... + 'S Basal, S-Complete, trace metals, potassium citrate, potassium phosphate, LB'); + if ndi.setup.V2.schemaHasField('formulation', 'value.type') + testCase.verifyEqual(sc.formulation.value.type.name, 'S-Complete'); + end + end + + function testUnrepresentedContentIsReportedNotDropped(testCase) + u = testCase.Result.unrepresented; + % software.vendor has no V2 home. An instrument's `name` has one only + % once the schema declares subject.name (did-schema PR #80); before + % that it is reported here, one row per instrument. Nothing else in + % the spec should be. + hasName = ndi.setup.V2.schemaHasField('subject', 'name'); + isName = strcmp({u.field}, 'name'); + if hasName + testCase.verifyFalse(any(isName), 'subject.name exists, so names are built'); + else + testCase.verifyEqual(nnz(isName), numel(testCase.Spec.instruments)); + end + u = u(~isName); + % formulation.value.type likewise, once the schema declares it + % (did-schema PR #84); before that, one row per formulation + isType = strcmp({u.field}, 'type'); + if ndi.setup.V2.schemaHasField('formulation', 'value.type') + testCase.verifyFalse(any(isType), 'formulation.value.type exists, so types are built'); + else + testCase.verifyEqual(nnz(isType), numel(testCase.Spec.formulations)); + end + u = u(~isType); + % with one entity class a web resource's host has no relation + % (hosted_by retired, 2026-10-08): one row per resource naming one -- + % EXCEPT a stored copy's host, which is the dataset's `hosted_by` on a + % schema that has it again (did-schema 9bedceb, 2026-10-15) + isHost = strcmp({u.field}, 'host'); + wr = testCase.Spec.web_resources; + if isstruct(wr), wr = num2cell(wr); end + stored = cellstr(string(testCase.Spec.dataset.stored_at)); + hasHosted = ndi.setup.V2.schemaHasRelation('hosted_by'); + if ndi.setup.V2.mergedEntities() + expect = sum(cellfun(@(x) isfield(x, 'host') && ... + ~(hasHosted && any(strcmp(x.key, stored))), wr)); + testCase.verifyEqual(nnz(isHost), expect); + else + testCase.verifyFalse(any(isHost)); + end + u = u(~isHost); + % software.vendor: a `developed_by` relation on a schema that has it + % (did-schema 9bedceb), else one row per software naming a vendor + % (the analysis package and macOS name none) + sw = testCase.Spec.software; + if isstruct(sw), sw = num2cell(sw); end + nVendor = sum(cellfun(@(x) isfield(x, 'vendor') && ~isempty(x.vendor), sw)); + if ndi.setup.V2.schemaHasRelation('developed_by') + testCase.verifyEmpty(u); + testCase.verifyNumElements(relationsNamed(testCase.Result.documents, ... + 'developed_by'), nVendor); + else + testCase.verifyEqual(unique({u.field}), {'vendor'}); + testCase.verifyEqual(numel(u), nVendor); + end + if hasHosted && ndi.setup.V2.mergedEntities() + testCase.verifyNumElements(relationsNamed(testCase.Result.documents, ... + 'hosted_by'), 1, 'the NDI Cloud copy names its host'); + end + end + end +end + +function rels = relationsNamed(docs, name) +isRel = cellfun(@(d) strcmp(d.document_class.class_name, 'directed_relation'), docs); +rels = docs(isRel); +rels = rels(cellfun(@(r) strcmp(r.directed_relation.relation.name, name), rels)); +end + +function id = parentOf(r) +id = r.depends_on(strcmp({r.depends_on.name}, 'parent_id')).document_id; +end diff --git a/tests/+ndi/+unittest/+setup/+V2/TestHaleyPatchTemplate.m b/tests/+ndi/+unittest/+setup/+V2/TestHaleyPatchTemplate.m new file mode 100644 index 0000000000..77db2add95 --- /dev/null +++ b/tests/+ndi/+unittest/+setup/+V2/TestHaleyPatchTemplate.m @@ -0,0 +1,70 @@ +classdef TestHaleyPatchTemplate < matlab.unittest.TestCase +%TESTHALEYPATCHTEMPLATE Placing detected E. coli patches by the acetate template (decision #80). +% +% The layout is the spec's `ecoli_patch_layout` (learned by patchLayout +% from Jess's full images, 2026-10-14). Centroids are made from it with a +% known template centre, angle and scale -- the inverse of the transform +% matchTemplatePatches applies -- so the right answer is known. +% +% UNVERIFIED: written without MATLAB; test-import-v2.yml is the first run. + + properties + Layout + end + + methods (TestClassSetup) + function readLayout(testCase) + f = fullfile(fileparts(which('ndi.setup.conv.haley.import_V2')), 'import_V2_spec.json'); + spec = jsondecode(fileread(f)); + testCase.Layout = spec.ecoli_patch_layout; + end + end + + methods (Test) + function testElevenPatchesGetTheirNumbers(testCase) + keep = setdiff(1:12, 4); % row 1, column 2 not detected + c = testCase.centroids(keep, [1100 1400], 2, 30, 0.3); + [slot, why] = ndi.setup.conv.haley.matchTemplatePatches(c, [1100 1400], 2, 30, testCase.Layout); + testCase.verifyEmpty(why); + testCase.verifyEqual(slot(:)', keep); + end + + function testThreePatchesAreStillPlaced(testCase) + % nothing depends on which rows or how many were detected + keep = [3 9 12]; + c = testCase.centroids(keep, [1000 1300], -1.5, 28, 0.3); + slot = ndi.setup.conv.haley.matchTemplatePatches(c, [1000 1300], -1.5, 28, testCase.Layout); + testCase.verifyEqual(slot(:)', keep); + end + + function testAPatchFarFromEveryPositionIsNotPlaced(testCase) + c = testCase.centroids(1:11, [1100 1400], 0, 30, 0); + c(5, :) = c(5, :) + [0 3 * 30]; % 3 mm off its position + [slot, why] = ndi.setup.conv.haley.matchTemplatePatches(c, [1100 1400], 0, 30, testCase.Layout); + testCase.verifyEmpty(slot); + testCase.verifySubstring(why, 'from the nearest template position'); + end + + function testTwoPatchesOnOnePositionAreNotPlaced(testCase) + c = testCase.centroids([1 1], [1100 1400], 0, 30, 0); + c(2, :) = c(2, :) + [10 0]; + [slot, why] = ndi.setup.conv.haley.matchTemplatePatches(c, [1100 1400], 0, 30, testCase.Layout); + testCase.verifyEmpty(slot); + testCase.verifySubstring(why, 'same template position'); + end + end + + methods + function c = centroids(testCase, slots, centerYX, angleDeg, scale, jitterMm) + % pixel centroids of the given patch numbers in an image whose + % template has CENTERYX ([y x]), ANGLEDEG and SCALE (px/mm) + pos = reshape(testCase.Layout.positions_mm, [], 2); + th = testCase.Layout.angle_sign * angleDeg * pi / 180; + rng(1); + p = pos(slots, :) + jitterMm * (rand(numel(slots), 2) - 0.5); + u = p(:, 1) * scale; + v = p(:, 2) * scale; + c = [centerYX(2) + cos(th) * u - sin(th) * v, centerYX(1) + sin(th) * u + cos(th) * v]; + end + end +end diff --git a/tests/+ndi/+unittest/+setup/+V2/TestHaleyRecordings.m b/tests/+ndi/+unittest/+setup/+V2/TestHaleyRecordings.m new file mode 100644 index 0000000000..7d35a75f33 --- /dev/null +++ b/tests/+ndi/+unittest/+setup/+V2/TestHaleyRecordings.m @@ -0,0 +1,346 @@ +classdef TestHaleyRecordings < matlab.unittest.TestCase +%TESTHALEYRECORDINGS Stage 5 of the Haley V2 import (acquisition listing), over a synthetic fixture. +% +% Writes a stand-in for the raw data: a tableOfContents.xlsx and +% experimentInfo.mat per C. elegans folder, placeholder .mp4 files in one +% day's videos folder, and ecoli/bacteria.mat with placeholder .tiff +% images. Checks the recordings listed, how they are matched to files, +% plates and cameras, and the source checks. Videos are not opened +% ('ReadVideos' stays false), so the placeholders need no content. + + properties + Root + Spec + end + + methods (TestClassSetup) + function fixture(testCase) + testCase.Root = tempname; + testCase.addTeardown(@() rmdir(testCase.Root, 's')); + specFile = fullfile(fileparts(which('ndi.setup.conv.haley.import_V2')), ... + 'import_V2_spec.json'); + testCase.Spec = jsondecode(fileread(specFile)); + ndi.unittest.setup.V2.TestHaleyRecordings.writeFixture(testCase.Root); + end + end + + methods (Static) + function writeFixture(root) + % The raw-data stand-in, also used by TestHaleyWrite. + ce = fullfile(root, 'haley', 'celegans'); + + % foragingConcentration, one day (22-02-04): + % plate 11 camera 1 video + lawn clip; filmed twice (an + % hourly continuation), the lawn clip shared + % plate 12 camera 2 video + lawn clip + % plate 13 camera 1 video named but not on disk + % plate 14 camera 1 table says camera 1, file name says _2; + % numFrames 0 + % plus a video on disk that no row names. + f = fullfile(ce, 'foragingConcentration'); + writeToc(f, {'0001', '22-02-04', 'nb', '0001-0016', 'grid [OD600 = 1]', 'yes', ''}); + t = @(h, m, s) datetime(2022, 2, 4, h, m, s); + writeInfo(f, [1 1 1 1 1], [11 11 12 13 14], [1 2 1 1 1], [1 1 2 1 1], ... + {'2022-02-04_12-10-51_1.avi', '2022-02-04_13-12-02_1.avi', '2022-02-04_12-10-51_2.avi', ... + '2022-02-04_14-16-36_1.avi', '2022-02-04_15-17-10_2.avi'}, ... + {'2022-02-04_11-49-08_1.avi', '2022-02-04_11-49-08_1.avi', '2022-02-04_11-49-08_2.avi', ... + '', ''}, ... + [t(12, 10, 51), t(13, 12, 2), t(12, 10, 51), t(14, 16, 36), t(15, 17, 10)], ... + [10797 10797 10796 10797 0], [2.9991 2.9991 2.9991 2.9991 0]); + writeTracks(f); + writeEncounters(ce); + v = fullfile(f, 'videos', '22-02-04'); + mkdir(v); + for name = {'2022-02-04_12-10-51_1', '2022-02-04_13-12-02_1', '2022-02-04_12-10-51_2', ... + '2022-02-04_15-17-10_2', '2022-02-04_11-49-08_1', '2022-02-04_11-49-08_2', ... + '2022-02-04_11-58-24_1'} + placeholder(fullfile(v, [name{1} '.mp4']), 100); + end + + % The other folders: a day with no rows for it. + for name = {'foragingMatching', 'foragingMini', 'foragingMutants', 'foragingSensory'} + f = fullfile(ce, name{1}); + writeToc(f, {'0001', '23-02-24', 'nb', '0001-0004', 'grid [OD600 = 1]', 'yes', ''}); + writeInfo(f, 99, 1, 1, 1, {'x.avi'}, {''}, datetime(2023, 2, 24), 1, 1); + end + + % E. coli (decision #64): raw images 2 and 3 and their empty-plate + % background image 9 on disk, 1 and 4 not (4 is fluorescence); + % images/ holds 2 and 3 background-normalised. + ec = fullfile(root, 'haley', 'ecoli'); + mkdir(fullfile(ec, 'images')); + mkdir(fullfile(ec, 'raw')); + % real (tiny) 16-bit TIFFs: the write path reads their headers + imwrite(uint16(magic(4)), fullfile(ec, 'images', '0002.tiff')); + imwrite(uint16(magic(5)), fullfile(ec, 'images', '0003.tiff')); + imwrite(uint16(magic(4)) + 1, fullfile(ec, 'raw', '0002.tiff')); + imwrite(uint16(magic(5)) + 1, fullfile(ec, 'raw', '0003.tiff')); + imwrite(uint16(ones(4)), fullfile(ec, 'raw', '0009.tiff')); + % stage 8: poured 25 Dec, cold room that evening, seeded 28 Dec, + % cold room that afternoon, room temperature the morning of 30 Dec; + % plate 2 without peptone + d = @(day, h) datetime(2023, 12, day, h, 0, 0); + info = table([1; 1], [1; 2], {'rectangle'; 'none'}, [1; 1], [0.5; 200], ... + [d(25, 10); d(25, 10)], [d(25, 18); d(25, 18)], [d(28, 11); d(28, 11)], ... + [d(28, 15); d(28, 15)], [d(30, 8); d(30, 8)], {'with'; 'without'}, ... + [10; 10], [600; 600], ... + 'VariableNames', {'expNum', 'plateNum', 'template', 'OD600', 'lawnVolume', ... + 'timePoured', 'timePouredColdRoom', 'timeSeed', 'timeSeedColdRoom', ... + 'timeRoomTemp', 'peptone', 'OD600Real', 'CFU'}); %#ok + metaData = table([1; 1; 1; 1; 1], [1; 2; 3; 4; 9], [1; 1; 2; 2; 2], ... + datetime(2023, 12, 30, 9, [38; 39; 40; 41; 42], 0), ... + {'a.tif'; 'b.tif'; 'c.tif'; 'd.tif'; 'e.tif'}, [0; 1; 1; 1; 1], [30; 300; 300; 3000; 300], ... + [NaN; 9; 9; 9; 9], [NaN; 1; NaN; NaN; NaN], ... + 'VariableNames', {'expNum', 'imageNum', 'plateNum', 'acquisitionTime', ... + 'fileName', 'fluorescence', 'exposureTime', 'backgroundImageNum', ... + 'brightfieldImageNum'}); %#ok + % stage 10 part C (decision #62): image 2 (plate 1, 12 patches) + % and image 3 (plate 2, one patch) were analysed. Image 2's mask is + % the staggered 3 x 4 grid, its bottom-left patch shifted a column + % left so bwlabel numbers it FIRST (as on the real image 2) while + % the grid order makes the top-left patch 1; lawnAnalysis row j has + % borderAmplitude 100 + j. + mkdir(fullfile(ec, 'mask')); + mkdir(fullfile(ec, 'closest')); + m = false(40, 80); + for x = [6 26 46 66] + m(7:9, x-1:x+1) = true; % top row + m(31:33, x-1:x+1) = true; % bottom row + m(19:21, x+9:x+11) = true; % middle row, offset + end + m(31:33, 5:7) = false; m(31:33, 4:6) = true; % bottom-left, a column left + imwrite(m, fullfile(ec, 'mask', '0002.png')); + imwrite(uint8(ones(40, 80)), fullfile(ec, 'closest', '0002.png')); + m = false(10, 10); m(4:6, 4:6) = true; + imwrite(m, fullfile(ec, 'mask', '0003.png')); + j = (1:12)'; + lawnAnalysis = table([2 * ones(12, 1); 3], [0.1 * ones(12, 1); 0.2], ... + [1000 + j; 2000], [900 * ones(12, 1); 1900], [0.9 * ones(12, 1); 0.8], ... + [100 + j; 500], [50 + j; 250], ... + 'VariableNames', {'imageNum', 'xPeak', 'yPeak', 'yOuterEdge', 'circularity', ... + 'borderAmplitude', 'centerAmplitude'}); + % stage 12 (decision #65): image 2's patches at growth times 60 + + % 10j min, border / exposure = (94 + t/10) / 300, a line; row 12's + % border / centre ratio is 40, so plotForagingGFP leaves it out + lawnAnalysis.plateNum = [ones(12, 1); 2]; + lawnAnalysis.fileName = [repmat({'b.tif'}, 12, 1); {'c.tif'}]; + lawnAnalysis.acquisitionTime = [repmat(datetime(2023, 12, 30, 9, 39, 0), 12, 1); ... + datetime(2023, 12, 30, 9, 40, 0)]; + lawnAnalysis.exposureTime = [300 * ones(12, 1); 300]; + lawnAnalysis.peptone = [repmat({'with'}, 12, 1); {'without'}]; + lawnAnalysis.OD600 = [ones(12, 1); 1]; + lawnAnalysis.lawnVolume = [0.5 * ones(12, 1); 200]; + lawnAnalysis.growthTimeCondition = zeros(13, 1); + lawnAnalysis.growthTimeTotal = minutes([60 + 10 * j; 60]); + lawnAnalysis.borderCenterRatio = [(100 + j) ./ (50 + j); 2]; + lawnAnalysis.borderCenterRatio(12) = 40; + save(fullfile(ec, 'bacteria.mat'), 'info', 'metaData', 'lawnAnalysis'); + % the published fits: the line above; the joint low-OD fit over the + % same values (OD600 1, so a = 94/300, b = 1/3000); a 200 uL fit + writeFits(fullfile(ec, 'borderAmplitude.csv'), { ... + 'with', 0.5, 0, 0.05, 1 / 3000, 0.05 * 94 / 300, 0, 1, 9 + 'with', 0.5, 0, 0.1, 1 / 3000, 0.1 * 94 / 300, 0, 1, 9 + 'with', 0.5, 0, 1, 1 / 3000, 94 / 300, 0, 1, 9 + 'with', 200, 48, 1, -0.0068, 55.7, 380, 0.06, 43}); + % analyzeGFP's workspace (decision #64), -v7.3 as the real one: the + % profile curves of images 2 and 3 (curve j of image 2 is 100*j + + % its distance index; NaN beyond the patch's last pixel) and image + % 9's background fit + x = -0.002:0.001:0.002; + Y = (100 * (1:12)') + (1:5); + Y(:, 5) = NaN; + lawnProfiles = struct('imageNum', [2 3], 'plateNum', [1 2], 'expNum', [1 1], ... + 'image', {{[], []}}, 'imageNormalized', {{[], []}}, 'labeled', {{[], []}}, ... + 'distances', {{x, x}}, 'pixelValues', {[]}, ... + 'pixelValuesNormalized', {{Y, 7:11}}); %#ok + backgroundInfo = table(9, 7, 0.95, 'VariableNames', {'imageNum', 'numOutliers', 'rsquare'}); %#ok + save(fullfile(ec, 'analyzeGFP_24-04-04.mat'), 'lawnProfiles', 'backgroundInfo', '-v7.3'); + end + end + + methods (Test) + function testOneRowPerRecordingOnDisk(testCase) + [R, ~] = testCase.list(); + testCase.verifyEqual(sum(strcmp(R.kind, 'behaviour')), 4, ... + 'plate 11 twice, 12, and 14; plate 13 has no file'); + testCase.verifyEqual(sum(strcmp(R.kind, 'lawn')), 2, 'plate 11''s clip is listed once'); + testCase.verifyEqual(sum(strcmp(R.kind, 'image')), 3, 'raw images 2, 3 and background 9'); + testCase.verifyEqual(numel(unique(R.epoch)), height(R)); + end + + function testEpochsSystemsAndPlates(testCase) + [R, ~] = testCase.list(); + r = R(strcmp(R.epoch, 'concentration_2022-02-04_12-10-51_2'), :); + testCase.verifyEqual(r.system{1}, 'camera2'); + testCase.verifyEqual(r.plate{1}, 'concentration_assayPlate0012'); + testCase.verifyEqual(r.source_name{1}, '2022-02-04_12-10-51_2.avi'); + testCase.verifyEqual(r.duration, 10796 / 2.9991, 'AbsTol', 1e-9); + l = R(strcmp(R.epoch, 'concentration_2022-02-04_11-49-08_1'), :); + testCase.verifyEqual(l.kind{1}, 'lawn'); + testCase.verifyEqual(l.local_start, datetime(2022, 2, 4, 11, 49, 8), ... + 'a lawn clip starts at the time in its name'); + testCase.verifyTrue(isnan(l.duration), 'not read without ReadVideos'); + e = R(strcmp(R.epoch, 'ecoli_image0002'), :); + testCase.verifyEqual(e.system{1}, 'microscope'); + testCase.verifyEqual(e.plate{1}, 'ecoli_plate0001'); + testCase.verifyEqual(e.local_start, datetime(2023, 12, 30, 9, 39, 0)); + testCase.verifySubstring(e.note{1}, 'background image 9'); + testCase.verifySubstring(e.note{1}, 'brightfield image 1'); + testCase.verifyEqual(e.file{1}, fullfile('ecoli', 'raw', '0002.tiff'), ... + 'the recording is the raw image (decision #64)'); + end + + function testSourceChecks(testCase) + [~, checks] = testCase.list(); + testCase.verifyTrue(any(contains(checks.rowWithoutFile, '2022-02-04_14-16-36_1'))); + testCase.verifyTrue(any(contains(checks.fileWithoutRow, '2022-02-04_11-58-24_1.mp4'))); + testCase.verifyTrue(any(contains(checks.cameraDisagrees, '2022-02-04_15-17-10_2'))); + testCase.verifyTrue(any(contains(checks.noFrames, '2022-02-04_15-17-10_2'))); + testCase.verifyTrue(any(contains(checks.imagesWithoutFile, '2 of 5'))); + testCase.verifyTrue(any(contains(checks.imagesWithoutFile, '1 of them fluorescence'))); + end + end + + methods + function [R, checks] = list(testCase) + T = ndi.setup.conv.haley.sessionList(testCase.Root, testCase.Spec); + [R, checks] = ndi.setup.conv.haley.recordingList(testCase.Root, T); + end + end +end + +function writeToc(folder, rows) +if ~isfolder(folder), mkdir(folder); end +[~, name] = fileparts(folder); +T = cell2table([repmat({name}, size(rows, 1), 1), rows], 'VariableNames', ... + {'experimentName', 'experimentNumber', 'directoryName', 'notebookName', ... + 'wormNumber', 'conditions', 'include', 'notes'}); +writetable(T, fullfile(folder, 'tableOfContents.xlsx')); +end + +function writeInfo(folder, expNum, plateNum, videoNum, camera, video, lawn, t, nFrames, rate) +% The recording columns, plus the subject columns (subjectList needs them) +% filled with plain values: two worms per plate (plate*10 + 1, + 2), N2, +% picked the day before, one patch. Plate 14 is food-deprived: moved to an +% unseeded plate (`starvedTime`) 3 h before it was filmed. Plate 13 is +% excluded (the source's `exclude`). Stage 8's columns (hand-written, to the +% minute): seeded two days before filming at OD600 1, 0.5 uL, from a solution +% read at OD600 10.2 with CFU 500; to the cold room an hour later; to room +% temperature 67 min before filming -- plate 12 exactly 60 min before, the +% rule that marks an estimated time; its acclimation plate seeded 30 min after +% it, the same day, at growthOD600 1, cold room an hour later, room +% temperature an hour before the pick. Stage 9's probe readings, the same on +% every video of a plate: temp 21.5 + plate/100 C, humidity 40 + plate %RH. +% Stage 10 part A's columns, on 8x8 images rather than 1024x1024: 33 pixels per +% mm; an arena mask of the inner 6x6; a reference mark at (2,2) except on +% plate 14 (empty, as foragingMini's are); a lawn mask of the centre 2x2; every +% pixel's nearest patch is the plate's one patch; patches found by template; +% the lawn clip registered at 0.98 where the plate has one; the patch's +% circularity 0.9. +n = numel(expNum); +arena = false(8); arena(2:7, 2:7) = true; +ref = false(8); ref(2, 2) = true; +lawnMask = false(8); lawnMask(4:5, 4:5) = true; +refs = repmat({ref}, n, 1); +refs(plateNum(:) == 14) = {false(8)}; +fit = NaN(n, 1); +fit(~cellfun(@isempty, lawn(:))) = 0.98; +minute = @(x) dateshift(x, 'start', 'minute'); +seed = minute(t(:) - days(2)); +roomTemp = minute(t(:) - minutes(67)); +roomTemp(plateNum(:) == 12) = t(plateNum(:) == 12) - hours(1); +growthSeed = seed + minutes(30); +starved = NaT(n, 1); +starved(plateNum(:) == 14) = t(plateNum(:) == 14) - hours(3); +worms = arrayfun(@(p) double(p) * 10 + [1 2], plateNum(:), 'UniformOutput', false); +info = table(uint16(expNum(:)), uint16(plateNum(:)), uint16(videoNum(:)), double(camera(:)), ... + video(:), lawn(:), t(:), double(nFrames(:)), double(rate(:)), ... + worms, repmat({'N2'}, n, 1), t(:) - days(1), repmat({zeros(1, 2)}, n, 1), ... + repmat({zeros(1, 1)}, n, 1), plateNum(:) == 13, starved, ... + seed, seed + hours(1), roomTemp, ones(n, 1), 0.5 * ones(n, 1), 10.2 * ones(n, 1), ... + 500 * ones(n, 1), growthSeed, growthSeed + hours(1), minute(t(:) - days(1) - hours(1)), ... + ones(n, 1), 21.5 + double(plateNum(:)) / 100, 40 + double(plateNum(:)), ... + 33 * ones(n, 1), 30 * ones(n, 1), 1.5 * ones(n, 1), 5 * ones(n, 1), ... + repmat({arena}, n, 1), refs, repmat({lawnMask}, n, 1), repmat({ones(8)}, n, 1), ... + repmat({{'template'}}, n, 1), fit, repmat({0.9}, n, 1), ... + 'VariableNames', {'expNum', 'plateNum', 'videoNum', 'camera', 'videoFileName', ... + 'lawnFileName', 'timeRecord', 'numFrames', 'frameRate', 'wormNum', 'strainID', ... + 'growthTimePicked', 'lawnCenters', 'lawnRadii', 'exclude', 'starvedTime', ... + 'timeSeed', 'timeColdRoom', 'timeRoomTemp', 'OD600', 'lawnVolume', 'OD600Real', ... + 'CFU', 'growthTimeSeed', 'growthTimeColdRoom', 'growthTimeRoomTemp', 'growthOD600', ... + 'temp', 'humidity', 'scale', 'arenaDiameter', 'lawnDiameter', 'lawnSpacing', ... + 'arenaMask', 'refMask', 'lawnMask', 'lawnClosest', 'lawnMethod', 'lawnRegistration', ... + 'lawnCircularity'}); %#ok +save(fullfile(folder, 'experimentInfo.mat'), 'info'); +end + +function writeTracks(folder) +% Stage 10 part B's midpoint.mat: 5 frames for each worm of each filmed video +% (plate 11 videos 1 and 2, 12, 14; worms plate*10 + 1, + 2). Worm w is at +% x = frame + w/10, y = 4; speed w um/s; 2 pixels inside the patch edge; its +% nearest patch is patch 1, except frame 3 of worm 121, which the source +% marks noTrack (and reads its nearest patch at pixel (1,1)). +rows = {}; +for pv = [11 1; 11 2; 12 1; 14 1]' + for w = pv(1) * 10 + [1 2] + f = (1:5)'; + noTrack = false(5, 1); + if w == 121, noTrack(3) = true; end + rows{end+1} = table(uint16(ones(5, 1)), uint16(pv(1) * ones(5, 1)), ... + uint16(pv(2) * ones(5, 1)), w * ones(5, 1), f, f + w / 10, 4 * ones(5, 1), ... + noTrack, w * ones(5, 1), 2 * ones(5, 1), ones(5, 1), ... + 'VariableNames', {'expNum', 'plateNum', 'videoNum', 'wormNum', 'frameNum', ... + 'xPosition', 'yPosition', 'noTrack', 'velocitySmooth', 'distanceLawnEdge', ... + 'closestLawnID'}); %#ok + end +end +data = vertcat(rows{:}); %#ok +save(fullfile(folder, 'midpoint.mat'), 'data'); +end + +function writeFits(file, rows) +% borderAmplitude.csv as plotForagingGFP writes it +fid = fopen(file, 'w'); +fprintf(fid, 'peptone,lawnVolume,growthTimeCondition,OD600,slope,intercept,sse,rsquare,dfe,adjrsquare,rmse\r\n'); +for r = 1:size(rows, 1) + fprintf(fid, '%s,%.17g,%.17g,%.17g,%.17g,%.17g,%.17g,%.17g,%d,%.17g,%.17g\r\n', rows{r, :}, 0, 0); +end +fclose(fid); +end + +function writeEncounters(folder) +% Stage 11's encounter.mat: worm 121 (plate 12, one video) -- the gap before its +% first encounter (id 0), then encounters at rows 2-3 and 4-5 on patch 1; +% worm 111 (plate 11, two videos of 5 frames each, joined) -- one encounter at +% rows 7-9, frames 2-4 of the second video. Values: speeds w*1 um/s on, ... +n = 4; +name = repmat({'foragingConcentration'}, n, 1); +encounter = table(uint16(ones(n, 1)), uint16([12; 12; 12; 11]), [1; 1; 1; 2], ... + [121; 121; 121; 111], [0; 1; 2; 1], [NaN; 2; 4; 7], [NaN; 3; 5; 9], ... + [NaN; 1; 1; 1], [NaN; 10; 20; 30], [NaN; 5; 6; 7], [NaN; 0.1; 0.2; 0.3], ... + [NaN; 100; 110; 120], [NaN; 8; 9; 10], [NaN; 1.5; 2.5; 3.5], [NaN; -40; -50; -60], ... + [50; 60; NaN; NaN], [1.0; 1.1; NaN; NaN], name, [NaN; 0.9; 0.2; 0.7], [NaN; 0.8; 0.6; 0.01], ... + {''; 'exploit'; 'sample'; 'exploit'}, ... + 'VariableNames', {'expNum', 'plateNum', 'videoNum', 'wormNum', 'id', 'enter', 'exit', ... + 'lawnID', 'velocityOn', 'velocityOnMin', 'distanceOnMax', 'velocityBeforeEnter', ... + 'velocityAfterEnter', 'timeSlowDown', 'decelerate', 'velocityOff', 'distanceOffMax', ... + 'expName', 'exploitPosterior', 'sensePosterior', 'label'}); +% stage 12 (decision #65): worm 121's patches at OD600 1, worm 111's at 0.05 +% (the joint fit); the values labelEncounters computed from the fixture's fits +encounter.peptone = repmat({'with'}, n, 1); +encounter.lawnOD600 = [NaN; 1; 1; 0.05]; +encounter.lawnVolume = 0.5 * ones(n, 1); +encounter.growthCondition = [NaN; 0; 0; 0]; +encounter.lawnGrowth = [NaN; 74; 80; 90]; +encounter.growthLawnGrowth = hours(51) * ones(n, 1); +encounter.borderAmplitude = [NaN; 94 / 300 + 74 / 3000; 94 / 300 + 80 / 3000; ... + 0.05 * 94 / 300 + 90 / 3000]; +encounter.borderAmplitudeGrowth = (55.7 - 0.0068 * 51 * 60) * ones(n, 1); %#ok +save(fullfile(folder, 'encounter.mat'), 'encounter'); +end + +function placeholder(file, n) +fid = fopen(file, 'w'); +fwrite(fid, zeros(1, n, 'uint8')); +fclose(fid); +end diff --git a/tests/+ndi/+unittest/+setup/+V2/TestHaleySeeding.m b/tests/+ndi/+unittest/+setup/+V2/TestHaleySeeding.m new file mode 100644 index 0000000000..1ce1084c0f --- /dev/null +++ b/tests/+ndi/+unittest/+setup/+V2/TestHaleySeeding.m @@ -0,0 +1,175 @@ +classdef TestHaleySeeding < matlab.unittest.TestCase +%TESTHALEYSEEDING Stage 8 part B of the Haley V2 import: the seeding suspensions. +% +% Writes the seeding columns of three C. elegans studies and the E. coli +% table (column names as in the real files) and checks +% ndi.setup.conv.haley.seedingSuspensions against the real spec's `seeding` +% and `corrections` (decisions #56, #57): +% 2023-02-17 Mini and Matching read identically: ONE solution, and since +% their diluents differ (LB, S-Complete) it lists none +% 2023-03-27 Matching (S-Complete, CFU corrected away) and Mini (LB) read +% differently: two solutions, _a and _b; Matching's +% acclimation plates use Mini's LB solution +% Mutants a growth seeding day with no solution is reported +% E. coli OP50-GFP in LB with carbenicillin; CFU corrected away; OD600 +% 0 is the diluent itself +% The last test builds the entries with ndi.setup.V2.datasetMetadata (needs +% did2.build and a V2 schema); the others need neither. +% +% UNVERIFIED: written without MATLAB; test-import-v2.yml is the first run. + + properties + Root + Spec + S + Checks + end + + methods (TestClassSetup) + function fixture(testCase) + testCase.Root = tempname; + testCase.addTeardown(@() rmdir(testCase.Root, 's')); + specFile = fullfile(fileparts(which('ndi.setup.conv.haley.import_V2')), ... + 'import_V2_spec.json'); + testCase.Spec = jsondecode(fileread(specFile)); + ce = fullfile(testCase.Root, 'haley', 'celegans'); + d1 = datetime(2023, 2, 17, 10, 0, 0); + d2 = datetime(2023, 3, 27, 10, 0, 0); + writeInfo(fullfile(ce, 'foragingMini'), [d1; d2], [10.1; 10.4], [1117; 846], ... + [1; 0.5], [d1; d2] + hours(1), [1; 1]); + writeInfo(fullfile(ce, 'foragingMatching'), [d1; d2], [10.1; 9.9375], [1117; 846], ... + {[1 5]; [1 10]}, [d1; d2] + hours(2), [1; 1]); + d3 = datetime(2023, 10, 30, 10, 0, 0); + writeInfo(fullfile(ce, 'foragingMutants'), [d3; d3], [10.3125; 10.3125], [585; 585], ... + [1; 1], [d3 - days(1); d3], [1; 1]); + e = fullfile(testCase.Root, 'haley', 'ecoli'); + mkdir(e); + d4 = datetime(2022, 4, 28, 10, 0, 0); + info = table([d4; d4], [10.275; 10.275], [457; 457], [1; 0], ... + 'VariableNames', {'timeSeed', 'OD600Real', 'CFU', 'OD600'}); %#ok + save(fullfile(e, 'bacteria.mat'), 'info'); + [testCase.S, testCase.Checks] = ndi.setup.conv.haley.seedingSuspensions( ... + testCase.Root, testCase.Spec); + end + end + + methods (Test) + function testSolutionsAndDilutions(testCase) + keys = cellfun(@(x) x.key, testCase.S.entries, 'UniformOutput', false); + testCase.verifyEqual(sort(keys), sort({ ... + 'suspension_op50_20230217', 'suspension_op50_20230217_od1', 'suspension_op50_20230217_od5', ... + 'suspension_op50_20230327_a', 'suspension_op50_20230327_a_od1', ... + 'suspension_op50_20230327_b', 'suspension_op50_20230327_b_od0p5', 'suspension_op50_20230327_b_od1', ... + 'suspension_op50_20231030', 'suspension_op50_20231030_od1', ... + 'suspension_op50gfp_20220428', 'suspension_op50gfp_20220428_od1', ... + 'suspension_op50_20231029_unmeasured_od1'})); + % each solution comes before its dilutions (they name it); an + % unmeasured day has no stock to come first (decision #82) + for k = 1:numel(keys) + if contains(keys{k}, '_od') && ~contains(keys{k}, '_unmeasured') + base = regexprep(keys{k}, '_od[0-9p]+$', ''); + testCase.verifyLessThan(find(strcmp(keys, base)), k, keys{k}); + end + end + end + + function testASharedSolutionWithMixedDiluentsListsNone(testCase) + x = testCase.entry('suspension_op50_20230217'); + testCase.verifyNumElements(x.ingredients, 1, 'Mini (LB) and Matching (S-Complete) shared it'); + g = x.ingredients{1}; + testCase.verifyEqual(g.ingredient, 'OP50'); + testCase.verifyEqual(g.concentration.particles_per_liter, 1117 * 2e10); + testCase.verifyEqual(g.concentration.source_value, 10.1); + testCase.verifyEqual(g.concentration.source_unit, 'OD600'); + d = testCase.entry('suspension_op50_20230217_od5'); + testCase.verifyNumElements(d.ingredients, 1); + testCase.verifyEqual(d.ingredients{1}.ingredient, 'suspension_op50_20230217'); + testCase.verifyEqual(d.ingredients{1}.concentration.volume_fraction, 0.5, 'AbsTol', 1e-12); + testCase.verifyEqual(d.ingredients{1}.concentration.source_value, 5); + end + + function testCorrectedCfuAndDiluents(testCase) + a = testCase.entry('suspension_op50_20230327_a'); % Matching + testCase.verifyFalse(isfield(a.ingredients{1}.concentration, 'particles_per_liter'), ... + 'the S-Complete seeding of 2023-03-27 measured no CFU (correction)'); + testCase.verifyEqual(a.ingredients{1}.concentration.source_value, 9.9375); + testCase.verifyEqual(a.ingredients{2}.ingredient, 's_complete'); + b = testCase.entry('suspension_op50_20230327_b'); % Mini + testCase.verifyEqual(b.ingredients{1}.concentration.particles_per_liter, 846 * 2e10); + testCase.verifyEqual(b.ingredients{2}.ingredient, 'lb'); + e = testCase.entry('suspension_op50gfp_20220428'); + testCase.verifyEqual(e.ingredients{1}.ingredient, 'OP50-GFP'); + testCase.verifyFalse(isfield(e.ingredients{1}.concentration, 'particles_per_liter')); + testCase.verifyEqual(e.ingredients{2}.ingredient, 'lb_carb'); + end + + function testIndexPointsEachSeedingAtItsFormulation(testCase) + I = testCase.S.index; + at = @(folder, day, od, kind) I.key(strcmp(I.folder, folder) & I.day == day & ... + I.od600 == od & strcmp(I.kind, kind)); + d2 = datetime(2023, 3, 27); + testCase.verifyEqual(at('foragingMatching', d2, 10, 'assay'), {'suspension_op50_20230327_a'}); + testCase.verifyEqual(at('foragingMatching', d2, 1, 'growth'), {'suspension_op50_20230327_b_od1'}, ... + 'Matching''s acclimation plates use that day''s LB solution (Mini''s)'); + testCase.verifyEqual(at('foragingMini', d2, 1, 'growth'), {'suspension_op50_20230327_b_od1'}); + testCase.verifyEqual(at('ecoli', datetime(2022, 4, 28), 0, 'assay'), {'lb_carb'}); + testCase.verifyEqual(at('foragingMutants', datetime(2023, 10, 30), 1, 'growth'), ... + {'suspension_op50_20231030_od1'}); + end + + function testAGrowthDayWithNoReadingIsRecordedUnmeasured(testCase) + % decision #82: Mutants 2023-10-29 seeded acclimation plates only + x = testCase.entry('suspension_op50_20231029_unmeasured_od1'); + testCase.verifyEqual(x.ingredients{1}.ingredient, 'OP50'); + testCase.verifyEqual(x.ingredients{1}.concentration.source_value, 1); + testCase.verifyEqual(x.ingredients{1}.concentration.source_unit, 'OD600'); + testCase.verifyFalse(isfield(x.ingredients{1}.concentration, 'particles_per_liter'), ... + 'no CFU was measured'); + I = testCase.S.index; + r = strcmp(I.folder, 'foragingMutants') & I.day == datetime(2023, 10, 29) & ... + strcmp(I.kind, 'growth'); + testCase.verifyEqual(I.key(r), {'suspension_op50_20231029_unmeasured_od1'}); + testCase.verifySubstring(I.note{r}, 'not measured'); + end + + function testChecks(testCase) + c = testCase.Checks; + testCase.verifyNumElements(c.growthWithoutSolution, 1); + testCase.verifyTrue(contains(c.growthWithoutSolution{1}, 'foragingMutants 2023-10-29')); + testCase.verifyEmpty(c.missingColumns); + testCase.verifyEmpty(c.twoReadings); + testCase.verifyEmpty(c.noReading); + end + + function testTheEntriesBuild(testCase) + testCase.assumeTrue(~isempty(which('did2.build.document')), ... + 'did2.build (DID-matlab V2) is not on the path'); + testCase.assumeTrue(ndi.setup.V2.schemaHasField('formulation', 'value'), ... + 'DID_SCHEMA_PATH does not hold a V2 schema with `formulation`'); + spec = testCase.Spec; + f = spec.formulations; + if isstruct(f), f = num2cell(f); end + n0 = numel(f); + spec.formulations = [reshape(f, 1, []), testCase.S.entries]; + r = ndi.setup.V2.datasetMetadata(spec, did.ido.unique_id()); + c = r.census; + testCase.verifyEqual(sum(c.count(strcmp(c.class, 'formulation'))), ... + n0 + numel(testCase.S.entries)); + end + end + + methods + function x = entry(testCase, key) + k = find(cellfun(@(e) strcmp(e.key, key), testCase.S.entries)); + testCase.assertNumElements(k, 1, key); + x = testCase.S.entries{k}; + end + end +end + +function writeInfo(folder, timeSeed, od, cfu, od600, growthTimeSeed, growthOD600) +mkdir(folder); +info = table(timeSeed, od, cfu, od600, growthTimeSeed, growthOD600, 'VariableNames', ... + {'timeSeed', 'OD600Real', 'CFU', 'OD600', 'growthTimeSeed', 'growthOD600'}); %#ok +save(fullfile(folder, 'experimentInfo.mat'), 'info'); +end diff --git a/tests/+ndi/+unittest/+setup/+V2/TestHaleySessions.m b/tests/+ndi/+unittest/+setup/+V2/TestHaleySessions.m new file mode 100644 index 0000000000..3c385e0363 --- /dev/null +++ b/tests/+ndi/+unittest/+setup/+V2/TestHaleySessions.m @@ -0,0 +1,167 @@ +classdef TestHaleySessions < matlab.unittest.TestCase +%TESTHALEYSESSIONS Stage 3 of the Haley V2 import, over a synthetic fixture. +% +% Writes a small stand-in for the raw data -- a tableOfContents.xlsx per +% C. elegans study folder (column names and value formats copied from the +% real foragingConcentration file) and an ecoli/bacteria.mat with `info` +% and `metaData` tables -- then checks the session list and that +% ndi.setup.V2.makeSessions creates openable sessions with the right +% name, description and study. +% +% Filtered (never passed) without did2.build or a V2 schema that has +% `study` and `session.name` (did-schema PRs #78 and #79). +% +% UNVERIFIED: written without MATLAB; test-import-v2.yml is the first run. + + properties + Root + Spec + end + + methods (TestClassSetup) + function fixture(testCase) + testCase.assumeTrue(~isempty(which('did2.build.document')), ... + 'did2.build (DID-matlab V2) is not on the path'); + sp = getenv('DID_SCHEMA_PATH'); + testCase.assumeTrue(~isempty(sp) && ndi.setup.V2.schemaHasClass('study', sp), ... + 'DID_SCHEMA_PATH does not hold a V2 schema with `study`'); + testCase.assumeTrue(ndi.setup.V2.schemaHasField('session', 'name'), ... + 'the V2 schema has no session.name (did-schema PR #79)'); + + testCase.Root = tempname; + testCase.addTeardown(@() rmdir(testCase.Root, 's')); + specFile = fullfile(fileparts(which('ndi.setup.conv.haley.import_V2')), ... + 'import_V2_spec.json'); + testCase.Spec = jsondecode(fileread(specFile)); + + ce = fullfile(testCase.Root, 'haley', 'celegans'); + writeToc(fullfile(ce, 'foragingConcentration'), { ... + '0001', '22-02-01', '220128_foraging_behavior', '0001-0040', ... + 'grid [OD600 = 0, 0.05, 0.1]', 'no', 'no contrast video, so patches are poorly aligned'; ... + '0003', '22-02-16', '220210_foraging_behavior', '0081-0120', ... + 'single [OD600 = 0, 0.05]', 'yes', 'no L4 plate information'; ... + '0014', '22-04-26', '220407_foraging_behavior', '0449-0546', ... + 'grid [OD600 = 0, 0.05, 0.1, 0.5, 1, 2, 3, 4, 5, 10, 1 (48H)]', 'yes', ''}); + for f = {'foragingMatching', 'foragingMini', 'foragingMutants', 'foragingSensory'} + writeToc(fullfile(ce, f{1}), {'0001', '23-02-24', 'nb', '0001-0010', ... + 'grid [OD600 = 1]', 'yes', ''}); + end + ec = fullfile(testCase.Root, 'haley', 'ecoli'); + mkdir(ec); + info = table([1; 1; 2], [1; 2; 3], 'VariableNames', {'expNum', 'plateNum'}); %#ok + metaData = table([1; 1; 2], [1; 2; 3], ... + datetime({'30-Dec-2023 09:39:10'; '31-Dec-2023 10:00:00'; '05-Jan-2024 08:00:00'}), ... + 'VariableNames', {'expNum', 'imageNum', 'acquisitionTime'}); %#ok + save(fullfile(ec, 'bacteria.mat'), 'info', 'metaData'); + end + end + + methods (Test) + function testOneRowPerExperiment(testCase) + T = ndi.setup.conv.haley.sessionList(testCase.Root, testCase.Spec); + testCase.verifyEqual(height(T), 3 + 4 + 2); + testCase.verifyEqual(numel(unique(T.local_identifier)), height(T)); + testCase.verifyTrue(ismember('concentration_0001', T.local_identifier)); + testCase.verifyTrue(ismember('ecoli_0002', T.local_identifier)); + end + + function testConcentrationSplitsIntoTwoStudies(testCase) + T = ndi.setup.conv.haley.sessionList(testCase.Root, testCase.Spec); + r = @(id) T(strcmp(T.local_identifier, id), :); + testCase.verifyEqual(r('concentration_0001').study_key{1}, 'single_density_multi_patch'); + testCase.verifyEqual(r('concentration_0003').study_key{1}, 'large_single_patch'); + testCase.verifyEqual(r('sensory_0001').study_key{1}, 'sensory_mutants'); + end + + function testRow0014IsCorrectedToTheSingleStudy(testCase) + % decision #72: the row says grid, its plates are single + [T, checks] = ndi.setup.conv.haley.sessionList(testCase.Root, testCase.Spec); + r = T(strcmp(T.local_identifier, 'concentration_0014'), :); + testCase.verifyEqual(r.study_key{1}, 'large_single_patch'); + testCase.verifyEqual(numel(checks.corrected), 1); + testCase.verifySubstring(checks.corrected{1}, 'conditions_concentration_experiment_14'); + testCase.verifyEmpty(checks.correctionNotApplied); + end + + function testACorrectionWhoseSourceChangedIsNotApplied(testCase) + spec = testCase.Spec; + c = spec.corrections; + if isstruct(c), c = num2cell(c); end + for k = 1:numel(c) + if strcmp(c{k}.key, 'conditions_concentration_experiment_14') + c{k}.was = 'grid [something else]'; + end + end + spec.corrections = c; + [T, checks] = ndi.setup.conv.haley.sessionList(testCase.Root, spec); + r = T(strcmp(T.local_identifier, 'concentration_0014'), :); + testCase.verifyEqual(r.study_key{1}, 'single_density_multi_patch', ... + 'a cell that no longer reads `was` is left as it is'); + testCase.verifyEmpty(checks.corrected); + testCase.verifyEqual(numel(checks.correctionNotApplied), 1); + end + + function testDayMetadataIsKept(testCase) + T = ndi.setup.conv.haley.sessionList(testCase.Root, testCase.Spec); + r = T(strcmp(T.local_identifier, 'concentration_0001'), :); + testCase.verifyFalse(r.include); + testCase.verifySubstring(r.description{1}, 'no contrast video'); + testCase.verifySubstring(r.description{1}, '220128_foraging_behavior'); + testCase.verifySubstring(r.name{1}, '1 Feb 2022'); + typo = T(strcmp(T.local_identifier, 'concentration_0014'), :); + testCase.verifyEqual([typo.worm_first, typo.worm_last], [449 546], ... + 'the range is carried as written; stage 5 checks it against the worms'); + end + + function testChecksFlagTheSourceIndexWithoutChangingIt(testCase) + ce = fullfile(testCase.Root, 'haley', 'celegans', 'foragingConcentration', 'videos'); + mkdir(fullfile(ce, '22-02-01')); % the day row 0001 names + mkdir(fullfile(ce, '22-12-31')); % a day no row names + [T, checks] = ndi.setup.conv.haley.sessionList(testCase.Root, testCase.Spec); + testCase.verifyEqual(height(T), 9, 'checks must not drop or alter rows'); + testCase.verifyTrue(any(contains(checks.unusedVideoDay, '22-12-31'))); + testCase.verifyTrue(any(contains(checks.noVideoFolder, '22-02-16'))); + testCase.verifyFalse(any(contains(checks.noVideoFolder, '22-02-01'))); + testCase.verifyFalse(any(contains(checks.ecoliSpread, 'experiment 1')), ... + 'fixture experiment 1 spans one day'); + end + + function testEcoliSessionsAreDatedByTheirFirstImage(testCase) + T = ndi.setup.conv.haley.sessionList(testCase.Root, testCase.Spec); + r = T(strcmp(T.local_identifier, 'ecoli_0001'), :); + testCase.verifyEqual(r.date, datetime(2023, 12, 30)); + testCase.verifySubstring(r.description{1}, '2 plate(s)'); + end + + function testMakeSessionsCreatesOpenableSessions(testCase) + studyIds = containers.Map({'single_density_multi_patch'}, {ndi.ido.unique_id()}); + T = ndi.setup.conv.haley.sessionList(testCase.Root, testCase.Spec, ... + 'StudyIds', studyIds, 'OutputRoot', fullfile(testCase.Root, 'out')); + T = T(strcmp(T.local_identifier, 'concentration_0001'), :); + [T, sessions] = ndi.setup.V2.makeSessions(T); + s = ndi.session.dir(T.path{1}); + testCase.verifyEqual(s.id(), sessions{1}.id()); + testCase.verifyEqual(s.reference, 'concentration_0001'); + d = s.database_search(ndi.v2.isaQuery('session')); + testCase.verifySubstring(ndi.v2.blockOf(d{1}.document_properties, 'session', 'name'), 'experiment 1'); + rel = s.database_search(ndi.query('', 'isa', 'directed_relation')); + testCase.verifyNumElements(rel, 1); + end + + function testABadTableCreatesNothing(testCase) + T = table({'a'; 'a'}, {fullfile(testCase.Root, 'x1'); fullfile(testCase.Root, 'x2')}, ... + 'VariableNames', {'local_identifier', 'path'}); + testCase.verifyError(@() ndi.setup.V2.makeSessions(T), 'ndi:setup:V2:badSessionTable'); + testCase.verifyFalse(isfolder(fullfile(testCase.Root, 'x1'))); + end + end +end + +function writeToc(folder, rows) +if ~isfolder(folder), mkdir(folder); end +[~, name] = fileparts(folder); +T = cell2table([repmat({name}, size(rows, 1), 1), rows], 'VariableNames', ... + {'experimentName', 'experimentNumber', 'directoryName', 'notebookName', ... + 'wormNumber', 'conditions', 'include', 'notes'}); +writetable(T, fullfile(folder, 'tableOfContents.xlsx')); +end diff --git a/tests/+ndi/+unittest/+setup/+V2/TestHaleySubjects.m b/tests/+ndi/+unittest/+setup/+V2/TestHaleySubjects.m new file mode 100644 index 0000000000..7b58780335 --- /dev/null +++ b/tests/+ndi/+unittest/+setup/+V2/TestHaleySubjects.m @@ -0,0 +1,343 @@ +classdef TestHaleySubjects < matlab.unittest.TestCase +%TESTHALEYSUBJECTS Stage 4 of the Haley V2 import (subjects), over a synthetic fixture. +% +% Writes a small stand-in for the raw data -- a tableOfContents.xlsx and +% an experimentInfo.mat `info` table per C. elegans folder (column names +% and types as in the real files), and ecoli/bacteria.mat with `info` and +% `metaData` -- then checks the subjects listed for each session, their +% identifiers, the acclimation-plate numbering and the source checks. +% +% Needs no schema and no did2: it builds no documents. + + properties + Root + Spec + end + + methods (TestClassSetup) + function fixture(testCase) + testCase.Root = tempname; + testCase.addTeardown(@() rmdir(testCase.Root, 's')); + specFile = fullfile(fileparts(which('ndi.setup.conv.haley.import_V2')), ... + 'import_V2_spec.json'); + testCase.Spec = jsondecode(fileread(specFile)); + ce = fullfile(testCase.Root, 'haley', 'celegans'); + + % foragingConcentration: day 1 has plates 1 and 2 (worms 1-8, one + % acclimation plate); day 2 has plate 3, filmed twice (worms 9-12), and + % a tableOfContents range that is wrong (9-14). Plate 4's expNum + % has no tableOfContents row. + f = fullfile(ce, 'foragingConcentration'); + writeToc(f, {'0001', '22-02-01', 'nb1', '0001-0008', 'grid [OD600 = 1]', 'yes', ''; ... + '0002', '22-02-04', 'nb2', '0009-0014', 'grid [OD600 = 1]', 'yes', ''}); + pick1 = datetime(2022, 1, 31, 13, 4, 0); + pick2 = datetime(2022, 2, 3, 10, 0, 0); + writeInfo(f, [1 1 2 2 9], [1 2 3 3 4], [1 1 1 2 1], ... + {[1 2 3 4], [5 6 7 8], [9 10 11 12], [9 10 11 12], [99 100]}, ... + {'N2', 'N2', 'N2', 'N2', 'N2'}, [pick1 pick1 pick2 pick2 pick2], ... + {zeros(3, 2), zeros(2, 2), zeros(1, 2), zeros(1, 2), zeros(1, 2)}, ... + {zeros(3, 1), zeros(1, 1), zeros(1, 1), zeros(1, 1), zeros(1, 1)}, ... + [true false false false false]); + + % foragingMutants: one day, three strains picked at two times + % (TU253 at 10:46 on two plates, MT15434 at 10:07), and a plate + % with no lawnCenters. + f = fullfile(ce, 'foragingMutants'); + writeToc(f, {'0001', '23-11-03', 'nb', '0001-0020', 'grid [OD600 = 1]', 'yes', ''}); + d = datetime(2023, 11, 2); + % Plate 5 is filmed twice and its FIRST video is a failed start + % with no lawnCenters (as Mutants plate 1 is, 3 Nov 2023). + writeInfo(f, [1 1 1 1 1 1], [1 2 3 4 5 5], [1 1 1 1 1 2], ... + {[1 2 3 4], [5 6 7 8], [9 10 11 12], [13 14 15 16], [17 18 19 20], [17 18 19 20]}, ... + {'TU253', 'MT15434', 'TU253', 'N2', 'N2', 'N2'}, ... + [d + duration(10, 46, 0), d + duration(10, 7, 0), d + duration(10, 46, 0), ... + d + duration(10, 39, 0), d + duration(10, 39, 0), d + duration(10, 39, 0)], ... + {zeros(2, 2), zeros(2, 2), zeros(2, 2), [], [], zeros(3, 2)}, ... + {zeros(2, 1), zeros(2, 1), zeros(2, 1), [], [], zeros(3, 1)}, false(1, 6), ... + struct('condition', {{'grid'; 'grid'; 'single'; 'grid'; 'grid'; 'grid'}}, ... + 'growthCondition', [24; 24; 24; NaN; 24; 24], ... + 'arenaDiameter', [30; 30; 30; 0; 0; 30], ... + 'growthTimeSeed', d - days(4) + duration([12 39 0; 12 0 0; 12 41 0; 12 0 0; 12 0 0; 12 0 0]))); + % ^ plate 3 is a single-patch plate on a mutant screen (single-density, + % multi-patch) day: the assay type check reports it. Plate 4 has + % no growthCondition and is not excluded: doImport excluded it. + % Plate 4 was never filmed (arena diameter 0 on its one row); + % plate 5's failed first video also reads 0, its second 30. + % Plates 1 and 3 share TU253's acclimation plate and give its + % seeding time as 12:39 and 12:41 (as concentration_0015 does). + + % The other three folders: one day, one plate, worms matching. + for name = {'foragingMatching', 'foragingMini', 'foragingSensory'} + f = fullfile(ce, name{1}); + writeToc(f, {'0001', '23-02-24', 'nb', '0001-0004', 'grid [OD600 = 1]', 'yes', ''}); + writeInfo(f, 1, 1, 1, {[1 2 3 4]}, {'N2'}, datetime(2023, 2, 23, 12, 50, 0), ... + {zeros(1, 2)}, {zeros(1, 1)}, false); + end + % Matching is a grid template (multi-density): OD600 has one value + % per template position, more than there are patches (the real + % plates: 19 positions, the empty centre OD600 0, 18 patches), and + % lawnClosestOD600 maps each pixel to its nearest patch's OD600. + % Two patches, at [x y] = [10 20] and [30 40]; template [5 0 10]. + f = fullfile(ce, 'foragingMatching'); + M = zeros(50, 50); M(20, 10) = 5; M(40, 30) = 10; + writeInfo(f, 1, 1, 1, {[1 2 3 4]}, {'N2'}, datetime(2023, 2, 23, 12, 50, 0), ... + {[10 20; 30 40]}, {[1; 1]}, false, ... + struct('OD600', {{[5 0 10]}}, 'lawnClosestOD600', {{M}})); + + % E. coli: a rectangle plate, a 'none' plate with one lawn, a + % 'none' blank plate, and a rectangle plate with no bacteria. + ec = fullfile(testCase.Root, 'haley', 'ecoli'); + mkdir(ec); + info = table([1; 1; 1; 2; 2], [1; 2; 3; 4; 5], {'rectangle'; 'none'; 'none'; 'rectangle'; 'none'}, ... + [1; 1; 0; 1; 0], [0.5; 200; 0; 0; 20], ... + 'VariableNames', {'expNum', 'plateNum', 'template', 'OD600', 'lawnVolume'}); %#ok + metaData = table([1; 2], [1; 2], ... + datetime({'30-Dec-2023 09:39:10'; '05-Jan-2024 08:00:00'}), ... + 'VariableNames', {'expNum', 'imageNum', 'acquisitionTime'}); %#ok + save(fullfile(ec, 'bacteria.mat'), 'info', 'metaData'); + end + end + + methods (Test) + function testIdentifiersUseTheFolderPrefix(testCase) + [S, ~] = testCase.list(); + ids = S.local_identifier; + for want = {'concentration_assayPlate0001', 'concentration_assayPlate0001_patch0003', ... + 'concentration_worm0012', 'concentration_0001_acclimationPlate0001', ... + 'mutants_assayPlate0002', 'ecoli_plate0001_patch0012', 'ecoli_plate0002_patch0001'} + testCase.verifyTrue(ismember(want{1}, ids), want{1}); + end + testCase.verifyFalse(any(startsWith(ids, 'foraging')), ... + 'no local_identifier keeps the `foraging` prefix'); + testCase.verifyEqual(numel(unique(ids)), height(S)); + end + + function testAPlateFilmedTwiceIsOneSubject(testCase) + [S, ~] = testCase.list(); + p3 = S(strcmp(S.local_identifier, 'concentration_assayPlate0003'), :); + testCase.verifyEqual(height(p3), 1); + testCase.verifySubstring(p3.description{1}, 'filmed in 2 video(s)'); + testCase.verifyEqual(sum(strcmp(S.kind, 'worm') & S.plate == 3 ... + & strcmp(S.folder, 'foragingConcentration')), 4, 'its worms are listed once'); + end + + function testGridTemplatePatchesTakeTheirOwnDensity(testCase) + % Matching: OD600 is the template (3 positions, 2 patches); each + % patch's OD600 is the lawnClosestOD600 map at its centre, and the + % template's non-zero positions agree, so nothing is reported + [S, checks] = testCase.list(); + pk = S(strcmp(S.kind, 'patch') & strcmp(S.folder, 'foragingMatching'), :); + testCase.verifyEqual(height(pk), 2); + testCase.verifyEqual(arrayfun(@(j) pk.prep(j).od600, 1:2), [5 10]); + testCase.verifyEmpty(checks.patchOD600, strjoin(checks.patchOD600, newline)); + end + + function testPatchesFollowLawnCenters(testCase) + [S, checks] = testCase.list(); + n = @(folder, p) sum(strcmp(S.kind, 'patch') & strcmp(S.folder, folder) & S.plate == p); + testCase.verifyEqual(n('foragingConcentration', 1), 3); + testCase.verifyEqual(n('foragingMutants', 4), 0, 'no lawnCenters -> no patches'); + testCase.verifyEqual(n('foragingMutants', 5), 3, ... + 'a failed first video: patches come from the restart'); + testCase.verifyFalse(any(contains(checks.noLawnCenters, 'plate 5'))); + testCase.verifyTrue(any(contains(checks.noLawnCenters, 'plate 4'))); + testCase.verifyTrue(any(contains(checks.patchCountDisagrees, 'plate 2')), ... + 'Concentration plate 2 lists 2 centres and 1 radius'); + end + + function testAcclimationPlatesAreNumberedByPickTime(testCase) + [S, ~] = testCase.list(); + g = S(strcmp(S.kind, 'acclimation_plate') & strcmp(S.session, 'mutants_0001'), :); + testCase.verifyEqual(height(g), 3, 'one per (strain, pick time)'); + g1 = g(strcmp(g.local_identifier, 'mutants_0001_acclimationPlate0001'), :); + testCase.verifyEqual(g1.strain{1}, 'MT15434', 'the earliest pick (10:07) is 0001'); + p1 = S(strcmp(S.local_identifier, 'mutants_assayPlate0001'), :); + p3 = S(strcmp(S.local_identifier, 'mutants_assayPlate0003'), :); + testCase.verifyEqual(p1.acclimation{1}, p3.acclimation{1}, 'same strain and pick -> same acclimation plate'); + w = S(strcmp(S.local_identifier, 'mutants_worm0005'), :); + testCase.verifyEqual(w.acclimation{1}, 'mutants_0001_acclimationPlate0001'); + end + + function testSourceChecksReportWithoutChanging(testCase) + [S, checks] = testCase.list(); + testCase.verifyTrue(any(contains(checks.plateWithoutSession, 'plate 4 (expNum 9)'))); + testCase.verifyFalse(any(strcmp(S.local_identifier, 'concentration_assayPlate0004'))); + testCase.verifyTrue(any(contains(checks.wormRange, 'concentration_0002')), ... + 'the 9-14 range against worms 9-12'); + testCase.verifyFalse(any(contains(checks.wormRange, 'concentration_0001'))); + testCase.verifyTrue(any(contains(checks.ecoliSeeding, 'plate 4')), ... + 'rectangle template with lawnVolume 0'); + testCase.verifyFalse(any(contains(checks.ecoliSeeding, 'plate 5')), ... + 'an LB-only patch is not a finding'); + end + + function testAPlateDoImportExcludedForNoGrowthConditionIsListed(testCase) + % doImport also excluded rows with NaN growthCondition + % (identifyValidRows); V2 lists them instead, for a decision + [S, checks] = testCase.list(); + testCase.verifyEqual(checks.noGrowthCondition, ... + {'foragingMutants: plate 4 (expNum 1) has NaN growthCondition and is not excluded; doImport excluded it'}); + testCase.verifyFalse(S.exclude(strcmp(S.local_identifier, 'mutants_assayPlate0004')), ... + 'listed, not excluded'); + end + + function testAPlateNeverFilmedIsFlaggedAndKeepsItsWorms(testCase) + % decision #73: arena diameter 0 on every row = not filmed + [S, checks] = testCase.list(); + p4 = S(strcmp(S.local_identifier, 'mutants_assayPlate0004'), :); + p5 = S(strcmp(S.local_identifier, 'mutants_assayPlate0005'), :); + testCase.verifyFalse(p4.filmed); + testCase.verifyTrue(p5.filmed, 'one filmed video is enough'); + testCase.verifySubstring(p4.description{1}, 'not filmed'); + testCase.verifyEqual(checks.notFilmed, ... + {'foragingMutants: plate 4 (expNum 1) was not filmed'}); + testCase.verifyFalse(p4.exclude, 'not filmed is not excluded'); + testCase.verifyEqual(sum(strcmp(S.kind, 'worm') & S.plate == 4 & ... + strcmp(S.folder, 'foragingMutants')), 4, 'its worms are kept'); + end + + function testASeedingTimeSpanIsRecordedAsItsMidpoint(testCase) + % decision #75: assay plates disagreeing only on their acclimation + % plate's seeding time give one seeding, at the span's midpoint + [S, checks] = testCase.list(); + a = find(strcmp(S.kind, 'acclimation_plate') & strcmp(S.folder, 'foragingMutants') ... + & strcmp(S.strain, 'TU253')); + testCase.verifyNumElements(a, 1); + d = datetime(2023, 11, 2) - days(4); + testCase.verifyEqual(S.prep(a).seeded, d + duration(12, 40, 0)); + testCase.verifyEqual(S.prep(a).seeded_spread_s, 60); + testCase.verifyEqual(numel(checks.growthSeedSpread), 1); + testCase.verifyFalse(any(contains(checks.growthDisagrees, 'foragingMutants')), ... + 'a span is reported once, as a span'); + end + + function testAnExperimentCorrectionReplacesTheAcclimationRoomTemperature(testCase) + % decision #70e: a named correction by folder and experiment sets + % that day's acclimation plate room temperature time + T = ndi.setup.conv.haley.sessionList(testCase.Root, testCase.Spec); + fix = struct('key', 'test_fix', 'folder', 'foragingMini', 'experiment', 1, ... + 'column', 'growthTimeRoomTemp', 'value', '2023-02-23T09:02', 'reason', 'a test'); + [S, checks] = ndi.setup.conv.haley.subjectList(testCase.Root, T, 'Corrections', {fix}); + a = find(strcmp(S.kind, 'acclimation_plate') & strcmp(S.folder, 'foragingMini')); + testCase.verifyNumElements(a, 1); + testCase.verifyEqual(S.prep(a).room_temp, datetime(2023, 2, 23, 9, 2, 0)); + testCase.verifyEqual(S.prep(a).room_temp_note, 'a test', 'the reason goes with the time'); + testCase.verifyEmpty(checks.correctionUnmatched); + % one naming a day not in the sessions is reported + fix.experiment = 99; + [~, checks] = ndi.setup.conv.haley.subjectList(testCase.Root, T, 'Corrections', {fix}); + testCase.verifyEqual(checks.correctionUnmatched, {'test_fix: no foragingMini experiment 99 in the selected sessions'}); + end + + function testAssayTypeIsCheckedAgainstTheStudy(testCase) + % the original import's per-plate assay type (folder + condition) + % against the study the plate's session is part_of; only folders + % whose experimentInfo has a `condition` column are compared + [~, checks] = testCase.list(); + d = checks.assayTypeDisagrees; + testCase.verifyNumElements(d, 1, strjoin(d, newline)); + testCase.verifySubstring(d{1}, 'foragingMutants: plate 3 is LargeSinglePatch'); + testCase.verifySubstring(d{1}, 'mutant_screen (SingleDensityMultiPatch)'); + out = evalc('testCase.list();'); + testCase.verifySubstring(out, 'ASSAY TYPE: 5 assay plate(s) compared with their study, 1 disagree'); + end + + function testEcoliPatchesFollowTheTemplate(testCase) + [S, ~] = testCase.list(); + n = @(p) sum(strcmp(S.kind, 'patch') & strcmp(S.folder, 'ecoli') & S.plate == p); + testCase.verifyEqual(n(1), 12, 'rectangle: a 3 x 4 grid'); + testCase.verifyEqual(n(2), 1, 'none with bacteria: one lawn'); + testCase.verifyEqual(n(3), 0, 'none without bacteria: blank'); + p3 = S(strcmp(S.local_identifier, 'ecoli_plate0003'), :); + testCase.verifySubstring(p3.description{1}, 'no bacteria'); + testCase.verifyEqual(n(5), 1, '20 ul of LB alone (OD600 0) is a patch'); + p5 = S(strcmp(S.local_identifier, 'ecoli_plate0005'), :); + testCase.verifySubstring(p5.description{1}, 'LB alone'); + end + + function testNamesFollowThePaper(testCase) + [S, ~] = testCase.list(); + nm = @(id) S.name{strcmp(S.local_identifier, id)}; + testCase.verifyEqual(nm('concentration_assayPlate0001'), 'Assay Plate 0001'); + testCase.verifyEqual(nm('concentration_assayPlate0001_patch0003'), 'Patch 0003 on Assay Plate 0001'); + testCase.verifyEqual(nm('concentration_worm0012'), 'Worm 0012'); + testCase.verifyEqual(nm('mutants_0001_acclimationPlate0001'), 'Acclimation Plate 0001'); + testCase.verifyEqual(nm('ecoli_plate0001'), 'Plate 0001'); + testCase.verifyEqual(nm('ecoli_plate0001_patch0012'), 'Patch 0012 on Plate 0001'); + testCase.verifyEqual(nm('concentration_assayPlate0001_worms'), 'Worms on Assay Plate 0001'); + testCase.verifyTrue(all(ismember(S.kind, {'assay_plate', 'acclimation_plate', 'plate', ... + 'patch', 'cohort', 'worm'}))); + end + + function testEachPlateOfWormsIsOneCohort(testCase) + % decision #55: the worms of an assay plate, moved together, are one + % group subject; each worm names it + [S, ~] = testCase.list(); + c = S(strcmp(S.kind, 'cohort') & strcmp(S.folder, 'foragingConcentration'), :); + testCase.verifyEqual(sort(c.local_identifier'), {'concentration_assayPlate0001_worms', ... + 'concentration_assayPlate0002_worms', 'concentration_assayPlate0003_worms'}); + c1 = c(strcmp(c.local_identifier, 'concentration_assayPlate0001_worms'), :); + testCase.verifyEqual(c1.strain{1}, 'N2'); + testCase.verifyEqual(c1.acclimation{1}, 'concentration_0001_acclimationPlate0001', ... + 'the cohort carries the plates its worms were on'); + w = S(strcmp(S.local_identifier, 'concentration_worm0003'), :); + testCase.verifyEqual(w.cohort{1}, 'concentration_assayPlate0001_worms'); + testCase.verifyEmpty(S.cohort{strcmp(S.local_identifier, 'concentration_assayPlate0001')}); + end + + function testTypesAndBacteria(testCase) + % subject.type and each patch's bacteria (decisions #54, #55) + [S, ~] = testCase.list(); + row = @(id) S(strcmp(S.local_identifier, id), :); + testCase.verifyEqual(row('concentration_worm0003').type{1}, 'organism'); + testCase.verifyEqual(row('concentration_assayPlate0001_worms').type{1}, 'group'); + testCase.verifyEqual(row('concentration_assayPlate0001').type{1}, 'material'); + testCase.verifyEqual(row('mutants_0001_acclimationPlate0001').type{1}, 'material'); + testCase.verifyEqual(row('ecoli_plate0001').type{1}, 'material'); + p = row('concentration_assayPlate0001_patch0001'); + testCase.verifyEqual(p.bacteria{1}, 'OP50', 'OP50 on every C. elegans plate'); + testCase.verifyEqual(p.type{1}, 'culture'); + p = row('ecoli_plate0002_patch0001'); + testCase.verifyEqual(p.bacteria{1}, 'OP50-GFP', 'OP50-GFP on the E. coli plates'); + testCase.verifyEqual(p.type{1}, 'culture'); + p = row('ecoli_plate0005_patch0001'); + testCase.verifyEmpty(p.bacteria{1}, 'OD600 0: LB alone, no bacteria'); + testCase.verifyEqual(p.type{1}, 'material'); + end + + function testExcludeIsCarriedForTheAssertionsStage(testCase) + [S, ~] = testCase.list(); + testCase.verifyTrue(S.exclude(strcmp(S.local_identifier, 'concentration_assayPlate0001'))); + testCase.verifyFalse(S.exclude(strcmp(S.local_identifier, 'concentration_assayPlate0002'))); + end + end + + methods + function [S, checks] = list(testCase) + T = ndi.setup.conv.haley.sessionList(testCase.Root, testCase.Spec); + [S, checks] = ndi.setup.conv.haley.subjectList(testCase.Root, T); + end + end +end + +function writeToc(folder, rows) +if ~isfolder(folder), mkdir(folder); end +[~, name] = fileparts(folder); +T = cell2table([repmat({name}, size(rows, 1), 1), rows], 'VariableNames', ... + {'experimentName', 'experimentNumber', 'directoryName', 'notebookName', ... + 'wormNumber', 'conditions', 'include', 'notes'}); +writetable(T, fullfile(folder, 'tableOfContents.xlsx')); +end + +function writeInfo(folder, expNum, plateNum, videoNum, wormNum, strainID, picked, centers, radii, exclude, extra) +info = table(uint16(expNum(:)), uint16(plateNum(:)), uint16(videoNum(:)), wormNum(:), ... + strainID(:), picked(:), centers(:), radii(:), logical(exclude(:)), ... + 'VariableNames', {'expNum', 'plateNum', 'videoNum', 'wormNum', 'strainID', ... + 'growthTimePicked', 'lawnCenters', 'lawnRadii', 'exclude'}); +if nargin > 10 + for f = reshape(fieldnames(extra), 1, []) + info.(f{1}) = reshape(extra.(f{1}), [], 1); + end +end +save(fullfile(folder, 'experimentInfo.mat'), 'info'); +end diff --git a/tests/+ndi/+unittest/+setup/+V2/TestHaleyTransferTime.m b/tests/+ndi/+unittest/+setup/+V2/TestHaleyTransferTime.m new file mode 100644 index 0000000000..9f14490def --- /dev/null +++ b/tests/+ndi/+unittest/+setup/+V2/TestHaleyTransferTime.m @@ -0,0 +1,118 @@ +classdef TestHaleyTransferTime < matlab.unittest.TestCase +%TESTHALEYTRANSFERTIME ndi.setup.conv.haley.transferTime over the real spec's +%transfer_protocol (decision #52): which video start stands for the transfer. + + properties + Protocol + end + + methods (TestClassSetup) + function readSpec(testCase) + specFile = fullfile(fileparts(which('ndi.setup.conv.haley.import_V2')), ... + 'import_V2_spec.json'); + spec = jsondecode(fileread(specFile)); + testCase.Protocol = spec.transfer_protocol; + end + end + + methods (Test) + function testLawnFirstUsesTheRecordingStart(testCase) + b = datetime(2022, 3, 18, 10, 0, 0); + [T, rule] = ndi.setup.conv.haley.transferTime(testCase.Protocol, ... + 'foragingConcentration', b, b - minutes(9), '2022-03-18_10-00-00_1'); + testCase.verifyEqual(T, b); + testCase.verifyEqual(rule.contrast_video, 'before_worms'); + testCase.verifySubstring(rule.method, 'agar plug'); + end + + function testWormsFirstUsesTheLawnClip(testCase) + b = datetime(2023, 1, 6, 10, 0, 0); + [T, rule] = ndi.setup.conv.haley.transferTime(testCase.Protocol, ... + 'foragingMini', b, b - minutes(0.2), '2023-01-06_10-00-00_1'); + testCase.verifyEqual(T, b - minutes(0.2)); + testCase.verifyEqual(rule.used, 'lawn clip'); + end + + function testMatchingSwitchesByDate(testCase) + b = datetime(2023, 3, 31, 10, 0, 0); + [T, rule] = ndi.setup.conv.haley.transferTime(testCase.Protocol, ... + 'foragingMatching', b, b - minutes(6), '2023-03-31_10-00-00_1'); + testCase.verifyEqual(T, b, 'days 2-3: lawn first'); + testCase.verifySubstring(rule.method, 'eyelash'); + b = datetime(2023, 5, 18, 10, 0, 0); + T = ndi.setup.conv.haley.transferTime(testCase.Protocol, ... + 'foragingMatching', b, b - minutes(1), '2023-05-18_10-00-00_1'); + testCase.verifyEqual(T, b - minutes(1), 'days 4-5: worms first'); + b = datetime(2023, 2, 24, 10, 0, 0); + T = ndi.setup.conv.haley.transferTime(testCase.Protocol, ... + 'foragingMatching', b, b - minutes(0.6), '2023-02-24_10-00-00_1'); + testCase.verifyEqual(T, b - minutes(0.6), 'day 1: worms first'); + end + + function testTheNamedExceptionIsWormsFirst(testCase) + % 2023-04-04_16-10-14_2 is the plate's LAWN CLIP; its first + % behaviour video is 2023-04-04_16-11-36_2. The exception matches + % whichever of the plate's videos it names. + b = datetime(2023, 4, 4, 16, 11, 36); + [T, rule] = ndi.setup.conv.haley.transferTime(testCase.Protocol, ... + 'foragingMatching', b, b - seconds(82), ... + {'2023-04-04_16-11-36_2', '2023-04-04_16-10-14_2'}); + testCase.verifyEqual(T, b - seconds(82)); + testCase.verifyEqual(rule.source, 'exception'); + [T, rule] = ndi.setup.conv.haley.transferTime(testCase.Protocol, ... + 'foragingMatching', b, b - seconds(82), '2023-04-04_16-11-36_2'); + testCase.verifyEqual(T, b, 'named by neither of its videos: the day''s rule (lawn first)'); + testCase.verifyEqual(rule.source, 'rule'); + b = datetime(2023, 4, 4, 16, 11, 30); + [T, rule] = ndi.setup.conv.haley.transferTime(testCase.Protocol, ... + 'foragingMatching', b, b - minutes(1.4), '2023-04-04_16-10-14_2'); + testCase.verifyEqual(T, b - minutes(1.4)); + testCase.verifyEqual(rule.source, 'exception'); + % Concentration, 2022-09-02 08:48:41: the contrast video was taken + % late, once the worms were on (both cameras) + b = datetime(2022, 9, 2, 8, 48, 41); + for cam = ["_1", "_2"] + [T, rule] = ndi.setup.conv.haley.transferTime(testCase.Protocol, ... + 'foragingConcentration', b, b - seconds(33), char("2022-09-02_08-48-41" + cam)); + testCase.verifyEqual(T, b - seconds(33)); + testCase.verifyEqual(rule.source, 'exception'); + end + end + + function testAFarOrLateLawnClipIsNotUsed(testCase) + b = datetime(2023, 11, 3, 10, 0, 0); + T = ndi.setup.conv.haley.transferTime(testCase.Protocol, 'foragingMutants', ... + b, b - hours(2), 'x'); + testCase.verifyEqual(T, b, 'more than an hour before: the clip is not this transfer'); + T = ndi.setup.conv.haley.transferTime(testCase.Protocol, 'foragingMutants', ... + b, b + minutes(1), 'x'); + testCase.verifyEqual(T, b, 'after the recording started: not the transfer either'); + end + + function testTheBoundIsEarlyOnlyAndAtMostFiveMinutes(testCase) + % CHANGE 7 / decision #52: T is read off the first video after the + % transfer, so it bounds [minus 0]; minus is the protocol's 300 s, + % or less on a lawn-first plate whose lawn clip ended closer + b = datetime(2022, 4, 22, 10, 0, 0); + [~, rule] = ndi.setup.conv.haley.transferTime(testCase.Protocol, ... + 'foragingConcentration', b, b - minutes(7), 'x', b - minutes(6.5)); + testCase.verifyEqual(rule.tolerance, [300 0], 'lawn clip ended 6.5 min before: the cap'); + [~, rule] = ndi.setup.conv.haley.transferTime(testCase.Protocol, ... + 'foragingConcentration', b, b - minutes(3), 'x', b - minutes(2.5)); + testCase.verifyEqual(rule.tolerance, [150 0], 'AbsTol', 1e-9, ... + 'lawn clip ended 2.5 min before: the transfer came after it'); + [~, rule] = ndi.setup.conv.haley.transferTime(testCase.Protocol, ... + 'foragingConcentration', b, b - minutes(3), 'x'); + testCase.verifyEqual(rule.tolerance, [180 0], 'AbsTol', 1e-9, ... + 'no lawn end known: its start, a wider but true bound'); + [~, rule] = ndi.setup.conv.haley.transferTime(testCase.Protocol, ... + 'foragingMutants', b, b - minutes(1), 'x', b - minutes(0.5)); + testCase.verifyEqual(rule.tolerance, [300 0], 'worms first: T is the lawn clip; the cap'); + end + + function testNoVideoNoTime(testCase) + T = ndi.setup.conv.haley.transferTime(testCase.Protocol, 'foragingMini', NaT, NaT, ''); + testCase.verifyTrue(isnat(T)); + end + end +end diff --git a/tests/+ndi/+unittest/+setup/+V2/TestHaleyWrite.m b/tests/+ndi/+unittest/+setup/+V2/TestHaleyWrite.m new file mode 100644 index 0000000000..3fca6197f0 --- /dev/null +++ b/tests/+ndi/+unittest/+setup/+V2/TestHaleyWrite.m @@ -0,0 +1,1068 @@ +classdef TestHaleyWrite < matlab.unittest.TestCase +%TESTHALEYWRITE The Haley V2 import, written: one C. elegans session and one +%E. coli session, over TestHaleyRecordings' synthetic fixture. +% +% Runs import_V2 with 'Write', true and 'Sessions' set, then reads the +% session back: the documents in its V2 database (each recording's file +% recorded by location, not held), an otherwise empty session folder, and +% the acquisition systems as NDI rebuilds them (daqsystem_load), whose +% navigator (ndi.file.navigator.bodies) finds the epochs, raw files and +% probe maps through the documents. The placeholder videos are not real +% videos, so nothing reads frames. + + properties + Root + Out + end + + methods (TestClassSetup) + function fixture(testCase) + testCase.Root = tempname; + testCase.addTeardown(@() rmdir(testCase.Root, 's')); + ndi.unittest.setup.V2.TestHaleyRecordings.writeFixture(testCase.Root); + % the output beside the raw data (nothing is written there) + testCase.Out = fullfile(testCase.Root, 'haley_V2'); + end + end + + methods (Test) + function testExclusionReasonAndAPlateNotFilmed(testCase) + % decision #73: an excluded plate carries its day's note when the + % day is marked `include = no`; a plate never filmed gets + % `behavior recording` = Not Performed and is not excluded + session = table({ndi.ido.unique_id()}, {'concentration_0001'}, false, ... + {'no contrast video, so patches are poorly aligned'}, ... + 'VariableNames', {'session_id', 'local_identifier', 'include', 'notes'}); + S = table({'concentration_0001'; 'concentration_0001'}, {'assay_plate'; 'assay_plate'}, ... + {'p1'; 'p2'}, [true; false], [true; false], ... + 'VariableNames', {'session', 'kind', 'local_identifier', 'exclude', 'filmed'}); + ids = containers.Map({'p1', 'p2'}, {ndi.ido.unique_id(), ndi.ido.unique_id()}); + out = ndi.setup.conv.haley.assertionDocuments(session, S, ids); + testCase.verifyEqual(out.counts.plate_excluded, 1); + testCase.verifyEqual(out.counts.plate_not_filmed, 1); + docs = cellfun(@newNames, out.documents, 'UniformOutput', false); + pick = @(var) docs(cellfun(@(d) strcmp(d.statement.variable.name, var), docs)); + % the reason is the tag's notes; with a schema whose statements + % have no notes, a text statement beside it + ex = pick('inclusion in analysis'); + testCase.verifyNumElements(ex, 1); + nf = pick('behavior recording'); + testCase.verifyNumElements(nf, 1); + testCase.verifyEqual(nf{1}.term.value.node, 'NCIT:C141477'); + testCase.verifyEqual(nf{1}.term.value.name, 'Not Performed'); + if ndi.setup.V2.schemaHasField('statement', 'notes') + testCase.verifyEqual(ex{1}.statement.notes, 'no contrast video, so patches are poorly aligned'); + testCase.verifyEqual(nf{1}.statement.notes, 'Prepared but not filmed: no video.'); + testCase.verifyEmpty(pick('reason for inclusion in analysis')); + elseif ~ndi.setup.V2.schemaHasClass('text') + % neither notes nor a text kind: the reasons are reported + testCase.verifyEmpty(pick('reason for inclusion in analysis')); + testCase.verifyTrue(any(contains(out.skipped, 'no contrast video'))); + else + why = pick('reason for inclusion in analysis'); + testCase.verifyNumElements(why, 1); + testCase.verifyEqual(why{1}.text.value(1).text, 'no contrast video, so patches are poorly aligned'); + why = pick('reason for behavior recording'); + testCase.verifyNumElements(why, 1); + testCase.verifyEqual(why{1}.text.value(1).text, 'Prepared but not filmed: no video.'); + end + + % a day marked `include = yes` gives an exclusion no reason + session.include = true; + out = ndi.setup.conv.haley.assertionDocuments(session, S, ids); + docs = cellfun(@newNames, out.documents, 'UniformOutput', false); + pick = @(var) docs(cellfun(@(d) strcmp(d.statement.variable.name, var), docs)); + ex = pick('inclusion in analysis'); + testCase.verifyNumElements(ex, 1); + testCase.verifyEmpty(pick('reason for inclusion in analysis')); + testCase.verifyFalse(isfield(ex{1}.statement, 'notes') && ~isempty(ex{1}.statement.notes)); + end + + function testConcentrationSession(testCase) + result = testCase.write("concentration_0001"); + testCase.verifyEqual(height(result.sessions), 1); + sessionPath = result.sessions.path{1}; + + docs = testCase.documents(sessionPath); + classes = cellfun(@(d) d.document_class.class_name, docs, 'UniformOutput', false); + count = @(c) sum(strcmp(classes, c)); + % plates 11-14, one patch each, two worms each, one cohort each + % (decision #55); three acclimation plates (the fixture's plates + % were picked at three times); one food deprivation plate (plate + % 14's worms) + testCase.verifyEqual(count('subject'), 4 + 4 + 8 + 4 + 3 + 1); + % plate 11 twice + 12 + 14 (behaviour) + 2 lawn clips + testCase.verifyEqual(count('epoch'), 6); + testCase.verifyEqual(count('intensity_observation'), 6); + testCase.verifyEqual(count('opaque_body'), 6); + testCase.verifyEqual(count('acquisition_system'), 2, 'camera1 and camera2'); + testCase.verifyEqual(count('session'), 1); + + s = docs{strcmp(classes, 'session')}; + testCase.verifyNotEmpty(s.depends_on, 'the session names its UTC extent'); + + b = docs(strcmp(classes, 'opaque_body')); + b = b{cellfun(@(d) strcmp(d.data_body.filename, '2022-02-04_12-10-51_2.mp4'), b)}; + % an EDAM format term (did-schema 877bbde); the media type on an + % older schema copy + [~, decl] = ndi.setup.V2.schemaHasField('data_body', 'format'); + if ~isempty(decl) && strcmp(decl.type, 'ontology_term') + testCase.verifyEqual(b.data_body.format.node, 'EDAM:format_3997'); + testCase.verifyEqual(b.data_body.format.name, 'MPEG-4'); + else + testCase.verifyEqual(b.data_body.format, 'video/mp4'); + end + testCase.verifyEqual(b.data_body.size_bytes, 100); + testCase.verifyEqual(b.data_body.hash_algorithm, 'MD5'); + testCase.verifyEqual(numel(b.data_body.content_hash), 32); + % the file is recorded BY LOCATION, not held: the raw file, in place + raw = fullfile(testCase.Root, 'haley', 'celegans', 'foragingConcentration', ... + 'videos', '22-02-04', '2022-02-04_12-10-51_2.mp4'); + [tf, where] = ndi.database.fun.externalFileLocation(b, 'body_data_0'); + testCase.verifyTrue(tf); + testCase.verifyEqual(where, raw); + testCase.verifyEqual(b.files.file_info.locations.ingest, 0); + + % nothing is written into the session folder but its database, + % and nothing next to the raw data + listing = dir(sessionPath); + testCase.verifyEqual(sort(setdiff({listing.name}, {'.', '..'})), {'.ndi'}); + testCase.verifyEmpty(dir(fullfile(fileparts(raw), '.*.epochid.ndi'))); + + % NDI rebuilds both cameras from the session's own database, and + % each finds its epochs, files and probe maps through the documents + session = ndi.session.dir(sessionPath); + sys = session.daqsystem_load(); + if ~iscell(sys), sys = {sys}; end + names = cellfun(@(x) x.name, sys, 'UniformOutput', false); + testCase.verifyEqual(sort(names), {'camera1', 'camera2'}); + cam2 = sys{strcmp(names, 'camera2')}; + testCase.verifyClass(cam2, 'ndi.daq.system.image'); + testCase.verifyClass(cam2.filenavigator, 'ndi.file.navigator.bodies'); + et = cam2.filenavigator.epochtable(); + testCase.verifyEqual(sort({et.epoch_id}), ... + {'concentration_2022-02-04_11-49-08_2', 'concentration_2022-02-04_12-10-51_2', ... + 'concentration_2022-02-04_15-17-10_2'}, ... + 'camera 2: plate 12 and its lawn clip, and plate 14 (camera 2 by its file name)'); + e = et(strcmp({et.epoch_id}, 'concentration_2022-02-04_12-10-51_2')); + testCase.verifyEqual(e.underlying_epochs.underlying, {raw}); + subj = docs(strcmp(classes, 'subject')); + plate = subj{cellfun(@(d) strcmp(d.subject.local_identifier, ... + 'concentration_assayPlate0012'), subj)}; + testCase.verifyEqual(e.epochprobemap.name, 'camera2'); + testCase.verifyEqual(e.epochprobemap.type, 'brightfield-imaging'); + testCase.verifyEqual(e.epochprobemap.subjectstring, plate.base.id); + end + + function testRelations(testCase) + % stage 6 (decision #52), over the fixture's concentration_0001: + % plates 11-14 (one patch each, two worms each, worms N = plate*10 + % + 1, 2), three acclimation plates; plate 13 has no video. + result = testCase.write("concentration_0001"); + docs = testCase.documents(result.sessions.path{1}); + classes = cellfun(@(d) d.document_class.class_name, docs, 'UniformOutput', false); + byId = containers.Map(cellfun(@(d) d.base.id, docs, 'UniformOutput', false), docs); + subj = docs(strcmp(classes, 'subject')); + idOf = containers.Map(cellfun(@(d) d.subject.local_identifier, subj, 'UniformOutput', false), ... + cellfun(@(d) d.base.id, subj, 'UniformOutput', false)); + rel = docs(strcmp(classes, 'directed_relation')); + % only relations between subjects (the session is also part_of + % its study, minted in stage 3 since decision #50) + subjectIdSet = values(idOf); + rel = rel(cellfun(@(r) any(strcmp(edge(r, 'child_id'), subjectIdSet)), rel)); + testCase.verifyEqual(numel(rel), 4 + 8 + 4 + 4 + 1, ['4 patch part_of plate, 8 worms ' ... + 'member_of their cohort, 4 cohorts in their assay plate, 4 in their acclimation ' ... + 'plate, 1 (plate 14) in a food deprivation plate (decision #55)']); + find1 = @(child, parent, name) rel(cellfun(@(r) ... + strcmp(edge(r, 'child_id'), idOf(child)) && strcmp(edge(r, 'parent_id'), idOf(parent)) ... + && strcmp(r.directed_relation.relation.name, name), rel)); + + r = find1('concentration_assayPlate0012_patch0001', 'concentration_assayPlate0012', 'part_of'); + testCase.verifyNumElements(r, 1); + testCase.verifyEmpty(edgeAll(r{1}, 'time_reference_id'), 'a patch is part of its plate, timeless'); + + % each worm is a member of its plate's cohort, timelessly; the + % plates and moves are stated once, on the cohort (decision #55) + r = find1('concentration_worm0121', 'concentration_assayPlate0012_worms', 'member_of'); + testCase.verifyNumElements(r, 1); + testCase.verifyEmpty(edgeAll(r{1}, 'time_reference_id')); + testCase.verifyEmpty(find1('concentration_worm0121', 'concentration_assayPlate0012', ... + 'contained_in'), 'no per-worm plate relation: the cohort carries it'); + + % worm 121's cohort was on assay plate 12 while it was filmed: from + % its first behaviour video, 2022-02-04 12:10:51 Los Angeles = 20:10:51 UTC + r = find1('concentration_assayPlate0012_worms', 'concentration_assayPlate0012', 'contained_in'); + testCase.verifyNumElements(r, 1); + % it holds of each worm: `distributive`, once the schema has it (#84) + if ndi.setup.V2.schemaHasField('directed_relation', 'distributive') + testCase.verifyTrue(logical(r{1}.directed_relation.distributive)); + else + testCase.verifyFalse(isfield(r{1}.directed_relation, 'distributive')); + end + t = edgeAll(r{1}, 'time_reference_id'); + testCase.verifyNumElements(t, 1); + ref = byId(t{1}); + testCase.verifyEqual(ref.absolute_time_reference.value.start.utc, '2022-02-04T20:10:51.000Z'); + testCase.verifyTrue(logical(ref.absolute_time_reference.value.start.approximate), ... + 'the transfer was a few minutes before filming: the start is approximate'); + testCase.verifyTrue(logical(ref.absolute_time_reference.value.duration.approximate), ... + 'an approximate start makes the extent approximate'); + testCase.verifyNotEmpty(ref.absolute_time_reference.value.end.utc, ... + 'the window records its end: the end of filming'); + testCase.verifyFalse(logical(ref.absolute_time_reference.value.end.approximate), ... + 'the end of filming is a video time: exact (CHANGE 6)'); + % CHANGE 7: T is only ever EARLY of the video it is read off -- + % [minus 0], minus at most the protocol's 5 min (lawn first: or + % less, to the lawn clip); the end of filming states no bound + v = ref.absolute_time_reference.value; + testCase.verifyEqual(v.start.tolerance.plus, 0, 'never later than the video'); + testCase.verifyGreaterThan(v.start.tolerance.minus, 0); + testCase.verifyLessThanOrEqual(v.start.tolerance.minus, 300); + testCase.verifyFalse(isfield(v.end, 'tolerance'), 'the end of filming is exact'); + testCase.verifyEqual(v.duration.tolerance.minus, 0, ... + 'the start can only be earlier, so the window only longer'); + testCase.verifyEqual(v.duration.tolerance.plus, v.start.tolerance.minus, 'AbsTol', 1e-9); + + % ... and on its acclimation plate from the pick time (the day + % before, 12:10:51) until filming, that end approximate + r = find1('concentration_assayPlate0012_worms', 'concentration_0001_acclimationPlate0001', 'contained_in'); + testCase.verifyNumElements(r, 1); + t = edgeAll(r{1}, 'time_reference_id'); + ref = byId(t{1}); + testCase.verifyEqual(ref.absolute_time_reference.value.start.utc, '2022-02-03T20:10:51.000Z'); + testCase.verifyEqual(ref.absolute_time_reference.value.duration.seconds, 86400, 'AbsTol', 1e-6); + testCase.verifyEqual(ref.absolute_time_reference.value.end.utc, '2022-02-04T20:10:51.000Z'); + testCase.verifyTrue(logical(ref.absolute_time_reference.value.end.approximate), ... + 'it ends at the transfer T, which is approximate'); + testCase.verifyTrue(logical(ref.absolute_time_reference.value.duration.approximate)); + testCase.verifyTrue(logical(ref.absolute_time_reference.value.start.approximate), ... + 'the pick time was written by hand: approximate'); + v = ref.absolute_time_reference.value; + testCase.verifyEqual(v.start.tolerance, struct('minus', 60, 'plus', 60), ... + 'read off a clock (perhaps analog) and written to the minute'); + testCase.verifyEqual(v.start.source_value, '2022-02-03T12:10', ... + 'source_value at the resolution it was written: no invented seconds'); + testCase.verifyEqual(v.end.tolerance.plus, 0, 'it ends at T: early only'); + testCase.verifyEqual(v.duration.tolerance.minus, 60 + v.end.tolerance.minus, 'AbsTol', 1e-9, ... + 'shortest: picked a minute late and moved T-minus early'); + testCase.verifyEqual(v.duration.tolerance.plus, 60, 'AbsTol', 1e-9, ... + 'longest: picked a minute early, moved at T'); + + % plate 14 was food-deprived: acclimation plate (picked the day + % before, 15:17:10) -> food deprivation plate (12:17:10, 3 h before + % filming) -> assay plate (filmed from 15:17:10) + r = find1('concentration_assayPlate0014_worms', 'concentration_0001_foodDeprivationPlate0001', 'contained_in'); + testCase.verifyNumElements(r, 1); + t = edgeAll(r{1}, 'time_reference_id'); + ref = byId(t{1}); + testCase.verifyEqual(ref.absolute_time_reference.value.start.utc, '2022-02-04T20:17:10.000Z'); + testCase.verifyEqual(ref.absolute_time_reference.value.duration.seconds, 3 * 3600, 'AbsTol', 1e-6); + testCase.verifyEqual(ref.absolute_time_reference.value.end.utc, '2022-02-04T23:17:10.000Z'); + testCase.verifyTrue(logical(ref.absolute_time_reference.value.end.approximate), ... + 'it ends at the transfer T, which is approximate'); + testCase.verifyTrue(logical(ref.absolute_time_reference.value.start.approximate), ... + 'starvedTime was read off a clock too: approximate'); + testCase.verifyEqual(ref.absolute_time_reference.value.start.tolerance, ... + struct('minus', 60, 'plus', 60)); + r = find1('concentration_assayPlate0014_worms', 'concentration_0001_acclimationPlate0003', 'contained_in'); + testCase.verifyNumElements(r, 1); + t = edgeAll(r{1}, 'time_reference_id'); + ref = byId(t{1}); + testCase.verifyEqual(ref.absolute_time_reference.value.duration.seconds, 21 * 3600, 'AbsTol', 1e-6, ... + 'the acclimation window ends when the worms move to the food deprivation plate'); + testCase.verifyTrue(logical(ref.absolute_time_reference.value.duration.approximate), ... + 'the pick (its start) is approximate, so the extent is'); + + % plate 13 was never filmed: its cohort is in it, with no time + r = find1('concentration_assayPlate0013_worms', 'concentration_assayPlate0013', 'contained_in'); + testCase.verifyNumElements(r, 1); + testCase.verifyEmpty(edgeAll(r{1}, 'time_reference_id')); + end + + function testAssertionsAndTypes(testCase) + % stage 7 (decision #55), over the fixture's concentration_0001, + % with stage 2 run so the strains have documents + result = testCase.write("concentration_0001", ["metadata", "assertions"]); + docs = testCase.documents(result.sessions.path{1}); + classes = cellfun(@(d) d.document_class.class_name, docs, 'UniformOutput', false); + subj = docs(strcmp(classes, 'subject')); + idOf = containers.Map(cellfun(@(d) d.subject.local_identifier, subj, 'UniformOutput', false), ... + cellfun(@(d) d.base.id, subj, 'UniformOutput', false)); + byLocal = containers.Map(cellfun(@(d) d.subject.local_identifier, subj, 'UniformOutput', false), subj); + ta = docs(strcmp(classes, 'term_assertion')); + about = @(local, variable) ta(cellfun(@(a) strcmp(edge(a, 'entity_id'), idOf(local)) ... + && strcmp(a.statement.variable.name, variable), ta)); + + % 4 cohorts, each with a species, a strain and a sex; 4 seeded + % patches, each with a species and a strain; plate 13 excluded. With + % one entity class (2026-10-08) the strain is an `instance_of` + % relation to the strain entity instead of an assertion. + merged = ndi.setup.V2.mergedEntities(); + perStrain = double(~merged); + testCase.verifyNumElements(ta, 4 * (2 + perStrain) + 4 * (1 + perStrain) + 1); + rel = docs(strcmp(classes, 'directed_relation')); + instanceOf = @(local) rel(cellfun(@(r) strcmp(r.directed_relation.relation.name, ... + 'instance_of') && strcmp(edge(r, 'child_id'), idOf(local)), rel)); + a = about('concentration_assayPlate0012_worms', 'species'); + testCase.verifyNumElements(a, 1); + testCase.verifyEqual(a{1}.term.value.node, 'NCBITaxon:6239'); + if merged + r = instanceOf('concentration_assayPlate0012_worms'); + testCase.verifyNumElements(r, 1); + testCase.verifyEqual(edge(r{1}, 'parent_id'), result.metadata.ids('N2'), ... + 'the cohort is an instance of the dataset-level N2 strain entity'); + testCase.verifyTrue(logical(r{1}.directed_relation.distributive), ... + 'stated of the cohort, it holds of each worm'); + testCase.verifyEmpty(about('concentration_assayPlate0012_worms', 'strain')); + else + a = about('concentration_assayPlate0012_worms', 'strain'); + testCase.verifyNumElements(a, 1); + testCase.verifyEqual(a{1}.term.value.name, 'N2'); + testCase.verifyEqual(a{1}.term.value.node, 'WBStrain:00000001', ... + 'the strain is its WormBase term, not a name alone'); + testCase.verifyEqual(edge(a{1}, 'strain_id'), result.metadata.ids('N2'), ... + 'the strain assertion names the dataset-level strain document'); + if ndi.setup.V2.schemaHasField('statement', 'distributive') + testCase.verifyTrue(logical(a{1}.statement.distributive), ... + 'stated on the cohort, it holds of each worm'); + end + end + a = about('concentration_assayPlate0012_worms', 'biological sex'); + testCase.verifyNumElements(a, 1); + testCase.verifyEqual(a{1}.statement.variable.node, 'PATO:0000047'); + testCase.verifyEqual(a{1}.term.value.node, 'PATO:0001340'); + testCase.verifyEqual(a{1}.term.value.name, 'hermaphrodite'); + testCase.verifyEmpty(about('concentration_assayPlate0012_patch0001', 'biological sex'), ... + 'bacteria have no sex'); + testCase.verifyEmpty(about('concentration_worm0121', 'species'), ... + 'a worm inherits its cohort''s species; it is not repeated'); + if merged + r = instanceOf('concentration_assayPlate0012_patch0001'); + testCase.verifyNumElements(r, 1); + testCase.verifyEqual(edge(r{1}, 'parent_id'), result.metadata.ids('OP50')); + else + a = about('concentration_assayPlate0012_patch0001', 'strain'); + testCase.verifyNumElements(a, 1); + testCase.verifyEqual(a{1}.term.value.name, 'OP50'); + testCase.verifyEqual(a{1}.term.value.node, 'WBStrain:00041969'); + end + a = about('concentration_assayPlate0012_patch0001', 'species'); + testCase.verifyEqual(a{1}.term.value.node, 'NCBITaxon:562'); + a = about('concentration_assayPlate0013', 'inclusion in analysis'); + testCase.verifyNumElements(a, 1); + testCase.verifyEqual(a{1}.term.value.name, 'excluded'); + testCase.verifyEmpty(about('concentration_assayPlate0012', 'inclusion in analysis')); + + % subject.type, once the schema has it (#84) + if ndi.setup.V2.schemaHasField('subject', 'type') + type = @(local) subjectType(byLocal, local); + testCase.verifyEqual(type('concentration_worm0121'), 'organism'); + testCase.verifyEqual(type('concentration_assayPlate0012_worms'), 'group'); + testCase.verifyEqual(type('concentration_assayPlate0012_patch0001'), 'culture'); + testCase.verifyEqual(type('concentration_assayPlate0012'), 'material'); + testCase.verifyEqual(type('concentration_0001_acclimationPlate0001'), 'material'); + else + w = byLocal('concentration_worm0121'); + testCase.verifyFalse(isfield(w.subject, 'type')); + end + end + + function testManipulations(testCase) + % stage 8 (decisions #54, #56, #57) over the fixture's + % concentration_0001, with stage 2 run so the recipes and the + % seeding suspensions have documents + result = testCase.write("concentration_0001", ["metadata", "assertions", "manipulations"]); + c = result.written{1}.manipulations.counts; + testCase.verifyEqual(c.pour, 4 + 3 + 1, '4 assay, 3 acclimation, 1 food deprivation plate'); + testCase.verifyEqual(c.seed_patch, 4, 'one patch per assay plate'); + testCase.verifyEqual(c.seed_acclimation_plate, 3); + testCase.verifyEqual(c.temperature, 4 * 2 + 3 * 3 + 1, ['assay: cold room, room ' ... + 'temperature; acclimation: cold room, room temperature, incubator; food ' ... + 'deprivation: incubator']); + testCase.verifyEqual(c.transfer, 4 + 1 + 3, ['each cohort onto its acclimation ' ... + 'plate, plate 14''s onto its food deprivation plate, the 3 filmed plates'' assay transfers']); + testCase.verifyEqual(c.food_deprivation, 1); + testCase.verifyTrue(any(contains(result.subjectChecks.roomTempLooksEstimated, ... + 'concentration_assayPlate0012')), 'plate 12''s timeRoomTemp is recording - 1 h'); + + docs = testCase.documents(result.sessions.path{1}); + classes = cellfun(@(d) d.document_class.class_name, docs, 'UniformOutput', false); + byId = containers.Map(cellfun(@(d) d.base.id, docs, 'UniformOutput', false), docs); + subj = docs(strcmp(classes, 'subject')); + idOf = containers.Map(cellfun(@(d) d.subject.local_identifier, subj, 'UniformOutput', false), ... + cellfun(@(d) d.base.id, subj, 'UniformOutput', false)); + on = @(set, local) set(cellfun(@(a) strcmp(edge(a, 'entity_id'), idOf(local)), set)); + liters = @(d) d.dose.value.volume.liters; + dm = docs(strcmp(classes, 'dose_manipulation')); + + % the patch: 0.5 uL of that day's OD600 1 dilution, at the seeding + seed = on(dm, 'concentration_assayPlate0012_patch0001'); + testCase.verifyNumElements(seed, 1); + testCase.verifyEqual(liters(seed{1}), 0.5e-6, 'AbsTol', 1e-15); + testCase.verifyEqual(edge(seed{1}, 'formulation_id'), ... + result.metadata.ids('suspension_op50_20220202_od1')); + % the plate: 25 mL of NGM, poured before that seeding + pour = on(dm, 'concentration_assayPlate0012'); + testCase.verifyNumElements(pour, 1); + testCase.verifyEqual(liters(pour{1}), 0.025, 'AbsTol', 1e-12); + testCase.verifyEqual(edge(pour{1}, 'formulation_id'), result.metadata.ids('ngm')); + testCase.verifyEqual(pour{1}.statement.variable.name, 'NGM agar'); + when = byId(edge(pour{1}, 'time_reference_id')); + testCase.verifyEqual(when.document_class.class_name, 'relative_time_reference'); + testCase.verifyEqual(edge(when, 'referent_id'), edge(seed{1}, 'time_reference_id')); + % the acclimation plate: 200 uL of the same dilution + acc = on(dm, 'concentration_0001_acclimationPlate0001'); + acc = acc(cellfun(@(d) abs(liters(d) - 200e-6) < 1e-12, acc)); + testCase.verifyNumElements(acc, 1); + testCase.verifyEqual(edge(acc{1}, 'formulation_id'), ... + result.metadata.ids('suspension_op50_20220202_od1')); + + % plate 12: into the cold room (4 C), then room temperature (20 C) + tm = on(docs(strcmp(classes, 'temperature_manipulation')), 'concentration_assayPlate0012'); + testCase.verifyEqual(reshape(sort(cellfun(@(d) d.temperature.value.celsius, tm)), 1, []), [4 20]); + methods = sort(cellfun(@(d) d.interaction.method.name, tm, 'UniformOutput', false)); + testCase.verifyEqual(reshape(methods, 1, []), {'ambient exposure', 'refrigeration'}); + + % the cohort's moves, at its contained_in times + tmn = docs(strcmp(classes, 'term_manipulation')); + mine = on(tmn, 'concentration_assayPlate0012_worms'); + toAssay = mine(cellfun(@(d) strcmp(d.term.value.name, 'assay plate'), mine)); + testCase.verifyNumElements(toAssay, 1); + testCase.verifyEqual(toAssay{1}.statement.variable.name, 'location'); + testCase.verifyEqual(toAssay{1}.interaction.method.name, 'agar plug transfer'); + mp = toAssay{1}.interaction.method_parameters; + if iscell(mp), mp = [mp{:}]; end + testCase.verifyTrue(any(arrayfun(@(q) strcmp(q.variable.name, 'cleaning step'), mp))); + dr = docs(strcmp(classes, 'directed_relation')); + rel = dr(cellfun(@(r) strcmp(edge(r, 'child_id'), idOf('concentration_assayPlate0012_worms')) ... + && strcmp(edge(r, 'parent_id'), idOf('concentration_assayPlate0012')), dr)); + testCase.verifyNumElements(rel, 1); + testCase.verifyEqual(edge(toAssay{1}, 'time_reference_id'), edge(rel{1}, 'time_reference_id'), ... + 'the move is stated at the time of the relation it makes'); + fd = on(tmn, 'concentration_assayPlate0014_worms'); + fd = fd(cellfun(@(d) strcmp(d.statement.variable.name, 'food availability'), fd)); + testCase.verifyNumElements(fd, 1); + testCase.verifyEqual(fd{1}.term.value.name, 'none'); + toDep = on(tmn, 'concentration_assayPlate0014_worms'); + toDep = toDep(cellfun(@(d) strcmp(d.term.value.name, 'food deprivation plate'), toDep)); + testCase.verifyNumElements(toDep, 1); + testCase.verifyEqual(toDep{1}.interaction.method.name, 'agar plug transfer'); + end + + function testObservations(testCase) + % stage 9 (decision #59): one ambient temperature and one relative + % humidity reading per filmed assay plate, by the probe, over the + % plate's recordings. Filmed: 11 (twice), 12, 14 (13's video is not + % on disk). + testCase.assumeTrue(ndi.setup.V2.schemaHasField('humidity', 'value'), ... + 'DID_SCHEMA_PATH does not hold a schema with `humidity` (did-schema PR #86)'); + result = testCase.write("concentration_0001", ["metadata", "observations"]); + c = result.written{1}.observations.counts; + testCase.verifyEqual([c.temperature, c.humidity], [3 3]); + testCase.verifyEmpty(result.written{1}.observations.skipped); + + docs = testCase.documents(result.sessions.path{1}); + classes = cellfun(@(d) d.document_class.class_name, docs, 'UniformOutput', false); + byId = containers.Map(cellfun(@(d) d.base.id, docs, 'UniformOutput', false), docs); + subj = docs(strcmp(classes, 'subject')); + idOf = containers.Map(cellfun(@(d) d.subject.local_identifier, subj, 'UniformOutput', false), ... + cellfun(@(d) d.base.id, subj, 'UniformOutput', false)); + on = @(set, local) set(cellfun(@(a) strcmp(edge(a, 'entity_id'), idOf(local)), set)); + tobs = docs(strcmp(classes, 'temperature_observation')); + hobs = docs(strcmp(classes, 'humidity_observation')); + probe = result.metadata.ids('temperature_probe_1'); + + % plate 12, filmed once: its video's own UTC reference + t12 = on(tobs, 'concentration_assayPlate0012'); + h12 = on(hobs, 'concentration_assayPlate0012'); + testCase.verifyNumElements(t12, 1); + testCase.verifyNumElements(h12, 1); + testCase.verifyEqual(t12{1}.temperature.value.celsius, 21.62, 'AbsTol', 1e-9); + testCase.verifyEqual(h12{1}.humidity.value.percent_relative_humidity, 52, 'AbsTol', 1e-9); + testCase.verifyEqual(t12{1}.statement.variable.name, 'ambient temperature'); + testCase.verifyEqual(h12{1}.statement.variable.name, 'relative humidity'); + testCase.verifyEqual(edge(t12{1}, 'instrument_id'), probe); + testCase.verifyEqual(edge(t12{1}, 'time_reference_id'), edge(h12{1}, 'time_reference_id')); + recs = on(docs(strcmp(classes, 'intensity_observation')), 'concentration_assayPlate0012'); + testCase.verifyTrue(any(cellfun(@(d) ismember(edge(t12{1}, 'time_reference_id'), ... + edgeAll(d, 'time_reference_id')), recs)), 'the reading is timed by its video'); + % and its recordings name their camera (stage 5; decision #59's fix) + cams = unique(cellfun(@(d) edge(d, 'instrument_id'), recs, 'UniformOutput', false)); + testCase.verifyEqual(reshape(cams, 1, []), {result.metadata.ids('camera_2')}); + + % plate 11, filmed twice: one reading, over both videos + t11 = on(tobs, 'concentration_assayPlate0011'); + testCase.verifyNumElements(t11, 1); + w = byId(edge(t11{1}, 'time_reference_id')); + testCase.verifyEqual(w.document_class.class_name, 'absolute_time_reference'); + vids = on(docs(strcmp(classes, 'intensity_observation')), 'concentration_assayPlate0011'); + testCase.verifyFalse(any(cellfun(@(d) ismember(w.base.id, edgeAll(d, 'time_reference_id')), vids)), ... + 'a window of its own, not either video''s'); + + testCase.verifyEmpty(on(tobs, 'concentration_assayPlate0013'), 'its video is not on disk'); + end + + function testGeometry(testCase) + % stage 10 part A (decision #60): per behaviour video on disk (plate + % 11 twice, 12, 14), the lab's coordinate system, masks, nearest + % patch and lawn clip registration, each a calculation of the + % plate from its video, the masks and maps INGESTED into the session + result = testCase.write("concentration_0001", ["metadata", "calculations"]); + ge = result.written{1}.geometry; + c = ge.counts; + testCase.verifyEqual([c.coordinate_system, c.arena, c.reference_mark, c.lawn], [4 4 3 4], ... + 'plate 14''s reference mark mask is empty, so none'); + testCase.verifyEqual(c.registration, 3, 'the three videos of plates with a lawn clip'); + hasItem = ndi.setup.V2.schemaHasField('item', 'value'); + testCase.verifyEqual(c.nearest_patch, 4 * hasItem); + + sessionPath = result.sessions.path{1}; + docs = testCase.documents(sessionPath); + classes = cellfun(@(d) d.document_class.class_name, docs, 'UniformOutput', false); + byId = containers.Map(cellfun(@(d) d.base.id, docs, 'UniformOutput', false), docs); + subj = docs(strcmp(classes, 'subject')); + idOf = containers.Map(cellfun(@(d) d.subject.local_identifier, subj, 'UniformOutput', false), ... + cellfun(@(d) d.base.id, subj, 'UniformOutput', false)); + ids = result.metadata.ids; + + % plate 12's video: its coordinate system is on the recording + rec = docs(strcmp(classes, 'intensity_observation')); + rec = rec(cellfun(@(d) strcmp(edge(d, 'entity_id'), idOf('concentration_assayPlate0012')), rec)); + cs = docs(strcmp(classes, 'coordinate_system')); + cs = cs(cellfun(@(d) any(strcmp(edge(d, 'referent_id'), cellfun(@(r) r.base.id, rec, ... + 'UniformOutput', false))), cs)); + testCase.verifyNumElements(cs, 1); + dims = cs{1}.coordinate_system.dimensions; + testCase.verifyEqual(dims(1).positive_direction.name, 'image right'); + testCase.verifyEqual(dims(2).positive_direction.name, 'image down'); + testCase.verifyEqual(dims(1).spacing.meters, 1e-3 / 33, 'AbsTol', 1e-15); + video = byId(edge(cs{1}, 'referent_id')); + + % each patch's centre, radius and circularity, per video (decision + % #70): one patch per plate, at lawnCenters [0 0] (pixel centres are + % 1-based, the system's origin the corner: -0.5), radius 0 px, 0.9 + testCase.verifyEqual([c.patch_centre, c.patch_radius, c.patch_circularity], [4 4 4]); + patch = idOf('concentration_assayPlate0012_patch0001'); + about = @(cls, var) docs(cellfun(@(d) strcmp(d.document_class.class_name, cls) ... + && strcmp(d.statement.variable.name, var) && strcmp(edge(d, 'entity_id'), patch), docs)); + centre = about('position_calculation', 'patch centre'); + testCase.verifyNumElements(centre, 1); + testCase.verifyEqual(centre{1}.position.value(1).coordinates(:)', [-0.5 -0.5]); + testCase.verifyEqual(edge(centre{1}, 'coordinate_system_id'), cs{1}.base.id, ... + 'in the video''s own coordinate system'); + testCase.verifyEqual(edgeAll(centre{1}, 'input_id'), {video.base.id}); + radius = about('length_calculation', 'patch radius'); + testCase.verifyNumElements(radius, 1); + testCase.verifyEqual(radius{1}.length.value(1).source_unit, 'pixel'); + circ = about('score_calculation', 'patch circularity'); + testCase.verifyEqual(circ{1}.score.value(1).score, 0.9, 'AbsTol', 1e-12); + + % its arena mask: a label calculation of the plate from the video, + % run by the analysis package in MATLAB on macOS + masks = docs(strcmp(classes, 'label_calculation')); + arena = masks(cellfun(@(d) strcmp(d.statement.variable.name, 'arena region') ... + && strcmp(edge(d, 'entity_id'), idOf('concentration_assayPlate0012')), masks)); + testCase.verifyNumElements(arena, 1); + a = arena{1}; + testCase.verifyEqual(edgeAll(a, 'input_id'), {video.base.id}); + testCase.verifyEqual(edge(a, 'software_id'), ids('haley_analysis')); + testCase.verifyEqual(edge(a, 'interpreter_id'), ids('matlab')); + testCase.verifyEqual(edge(a, 'operating_system_id'), ids('macos')); + % `datum_type` was renamed `data_type` on 2026-10-08 + testCase.verifyEqual(ndi.v2.blockOf(a, 'value', 'data_type', ndi.v2.blockOf(a, 'value', 'datum_type')), 'bool'); + mp = entries(a.interaction.method_parameters); + testCase.verifyEqual(mp{1}.variable.name, 'arena diameter'); + + % its body is held by the session's database, not left in a temp file + b = docs(strcmp(classes, 'sampled_body')); + b = b{cellfun(@(d) strcmp(edge(d, 'owner_id'), a.base.id), b)}; + testCase.verifyEqual(b.data_body.compression, 'gzip'); + testCase.verifyEqual(b.files.file_info.locations.ingest, 1); + testCase.verifyFalse(isfile(b.files.file_info.locations.location), 'the original is deleted'); + db = did2.database.sqlitedb(fullfile(sessionPath, '.ndi', ... + ndi.database.implementations.database.did2sqlite.DEFAULTFILENAME())); + held = db.filePath(b.base.id, 'body_data_0'); + db.close(); + testCase.verifyTrue(startsWith(held, fullfile(sessionPath, '.ndi'))); + tmp = [tempname '.bin.gz']; + copyfile(held, tmp); + raw = gunzip(tmp); + fid = fopen(raw{1}, 'r'); + bytes = fread(fid, Inf, 'uint8=>uint8'); + fclose(fid); + delete(tmp); delete(raw{1}); + expected = false(8); expected(2:7, 2:7) = true; + testCase.verifyEqual(reshape(bytes, 8, 8), uint8(expected), 'column-major, as written'); + + % the lawn mask's method: every patch was found by template + lawn = masks(cellfun(@(d) strcmp(d.statement.variable.name, 'bacterial lawn region') ... + && strcmp(edge(d, 'entity_id'), idOf('concentration_assayPlate0012')), masks)); + testCase.verifyEqual(lawn{1}.interaction.method.name, 'patch detection by template'); + + % the registration fit: from both the video and the lawn clip + sc = docs(strcmp(classes, 'score_calculation')); + sc = sc(cellfun(@(d) strcmp(edge(d, 'entity_id'), idOf('concentration_assayPlate0012')), sc)); + testCase.verifyNumElements(sc, 1); + testCase.verifyEqual(sc{1}.score.value.score, 0.98, 'AbsTol', 1e-12); + testCase.verifyNumElements(edgeAll(sc{1}, 'input_id'), 2); + + if hasItem + near = docs(strcmp(classes, 'item_calculation')); + near = near(cellfun(@(d) strcmp(edge(d, 'entity_id'), idOf('concentration_assayPlate0012')), near)); + testCase.verifyNumElements(near, 1); + testCase.verifyEqual(edgeAll(near{1}, 'item_id'), {idOf('concentration_assayPlate0012_patch0001')}); + end + end + + function testTracks(testCase) + % stage 10 part B (decision #61): per worm and video, the midpoint + % track, speed, distance to the patch edge and nearest patch, keyed + % by the 0-based video frame. Videos on disk: plate 11 twice, 12, + % 14, two worms each -> 8 tracks. + result = testCase.write("concentration_0001", ["metadata", "calculations"]); + c = result.written{1}.tracks.counts; + testCase.verifyEqual([c.position, c.speed, c.patch_edge_distance], [8 8 8]); + hasItem = ndi.setup.V2.schemaHasField('item', 'value'); + testCase.verifyEqual(c.nearest_patch, 8 * hasItem); + + sessionPath = result.sessions.path{1}; + docs = testCase.documents(sessionPath); + classes = cellfun(@(d) d.document_class.class_name, docs, 'UniformOutput', false); + subj = docs(strcmp(classes, 'subject')); + idOf = containers.Map(cellfun(@(d) d.subject.local_identifier, subj, 'UniformOutput', false), ... + cellfun(@(d) d.base.id, subj, 'UniformOutput', false)); + of = @(cls, local) docs(strcmp(classes, cls) & cellfun(@(d) ... + strcmp(edge(d, 'entity_id'), idOf(local)), docs)); + + % worm 121, filmed once (plate 12) + pos = of('position_calculation', 'concentration_worm0121'); + testCase.verifyNumElements(pos, 1); + p = pos{1}; + testCase.verifyEqual(p.statement.variable.name, 'midpoint position'); + cs = docs(strcmp(classes, 'coordinate_system')); + testCase.verifyTrue(any(cellfun(@(d) strcmp(d.base.id, edge(p, 'coordinate_system_id')), cs)), ... + 'the positions are in their video''s coordinate system'); + k1 = entries(p.data.keys); + testCase.verifyEqual(k1{1}.variable.name, 'video frame'); + testCase.verifyEqual(k1{1}.n, 5); + names = cellfun(@(m) m.variable.name, entries(p.interaction.method_parameters), ... + 'UniformOutput', false); + testCase.verifyEqual(sort(names(:)'), {'gap filling', 'longest gap filled'}); + + % its bytes: x = frame + 12.1 - 0.5, y = 3.5 (pixels from the corner) + xy = testCase.bodyValues(sessionPath, docs, classes, p.base.id, 'double'); + testCase.verifyEqual(reshape(xy, 5, 2), [(1:5)' + 12.1 - 0.5, 3.5 * ones(5, 1)], 'AbsTol', 1e-12); + + sp = of('velocity_calculation', 'concentration_worm0121'); + testCase.verifyNumElements(sp, 1); + testCase.verifyEqual(edgeAll(sp{1}, 'input_id'), {p.base.id}); + v = testCase.bodyValues(sessionPath, docs, classes, sp{1}.base.id, 'double'); + testCase.verifyEqual(v(:)', 121e-6 * ones(1, 5), 'AbsTol', 1e-15, 'm/s'); + + de = of('length_calculation', 'concentration_worm0121'); + testCase.verifyNumElements(de, 1); + testCase.verifyNumElements(edgeAll(de{1}, 'input_id'), 2, 'the positions and the lawn mask'); + v = testCase.bodyValues(sessionPath, docs, classes, de{1}.base.id, 'double'); + testCase.verifyEqual(v(:)', 2 * 1e-3 / 33 * ones(1, 5), 'AbsTol', 1e-15); + + if hasItem + np = of('item_calculation', 'concentration_worm0121'); + testCase.verifyNumElements(np, 1); + v = testCase.bodyValues(sessionPath, docs, classes, np{1}.base.id, 'uint8'); + testCase.verifyEqual(v(:)', uint8([0 0 255 0 0]), 'frame 3 is noTrack: the fill value'); + end + end + + function testEcoliSession(testCase) + result = testCase.write("ecoli_0001"); + docs = testCase.documents(result.sessions.path{1}); + classes = cellfun(@(d) d.document_class.class_name, docs, 'UniformOutput', false); + testCase.verifyEqual(sum(strcmp(classes, 'epoch')), 3, 'raw images 2, 3 and background 9'); + testCase.verifyEqual(sum(strcmp(classes, 'acquisition_system')), 1, 'the microscope'); + r = docs(strcmp(classes, 'acquisition_reader')); + testCase.verifyEqual(r{1}.acquisition_reader.reader_string, 'tiffstack'); + session = ndi.session.dir(result.sessions.path{1}); + sys = session.daqsystem_load(); + et = sys.filenavigator.epochtable(); + testCase.verifyEqual(sort({et.epoch_id}), {'ecoli_image0002', 'ecoli_image0003', 'ecoli_image0009'}); + e2 = et(strcmp({et.epoch_id}, 'ecoli_image0002')); + testCase.verifyEqual(e2.epochprobemap.type, 'wide-field-imaging'); + testCase.verifyEqual(e2.underlying_epochs.underlying, ... + {fullfile(testCase.Root, 'haley', 'ecoli', 'raw', '0002.tiff')}, ... + 'the epoch is the raw image (decision #64)'); + % the recording's size, read back through the object layer + % (decision #70): rows and columns in pixels, centres at 0.5, 1.5, ... + obs = session.database_search(ndi.v2.isaQuery('observation')); + sizes = {}; + for i = 1:numel(obs) + A = ndi.statement.fromDocument(session, obs{i}).axes(); + if height(A) ~= 2 || ~all(A.unit == "pixel"), continue; end + testCase.verifyEqual(A.variable, ["image vertical position"; "image horizontal position"]); + testCase.verifyEqual(A.coordinates{1}, (0.5:1:A.n(1) - 0.5)'); + testCase.verifyEqual(A.n(1), A.n(2), 'the fixture''s images are square'); + sizes{end+1} = A.n(1); %#ok + end + testCase.verifyEqual(unique([sizes{:}]), [4 5], ... + 'the raw images 0002 and 0009 (4 x 4) and 0003 (5 x 5)'); + end + + function testEncounters(testCase) + % stage 11 (decision #63): per worm, an encounter-onset list and the + % per-encounter values keyed by it. Worm 121 (plate 12): two + % encounters and the gap before them; worm 111 (plate 11, filmed + % twice): one encounter in the second video. + testCase.assumeTrue(ndi.setup.V2.schemaHasLeaf('time_calculation'), ... + 'DID_SCHEMA_PATH does not hold time_calculation (did-schema PR #87)'); + result = testCase.write("concentration_0001", ... + ["metadata", "observations", "calculations", "encounters"]); + c = result.written{1}.encounters.counts; + testCase.verifyEqual([c.worms, c.encounters], [2 3]); + testCase.verifyEmpty(result.written{1}.encounters.skipped); + + docs = testCase.documents(result.sessions.path{1}); + classes = cellfun(@(d) d.document_class.class_name, docs, 'UniformOutput', false); + subj = docs(strcmp(classes, 'subject')); + idOf = containers.Map(cellfun(@(d) d.subject.local_identifier, subj, 'UniformOutput', false), ... + cellfun(@(d) d.base.id, subj, 'UniformOutput', false)); + named = @(cls, local, var) docs(strcmp(classes, cls) & cellfun(@(d) ... + strcmp(edge(d, 'entity_id'), idOf(local)) && isfield(d, 'statement') ... + && strcmp(d.statement.variable.name, var), docs)); + + fr = 2.9991; + list = named('time_calculation', 'concentration_worm0121', 'encounter onset'); + testCase.verifyNumElements(list, 1); + testCase.verifyEqual([list{1}.time.value.seconds], [1 3] / fr, 'AbsTol', 1e-9, ... + 'rows 2 and 4 of a one-video track: frames 2 and 4'); + l111 = named('time_calculation', 'concentration_worm0111', 'encounter onset'); + testCase.verifyEqual(l111{1}.time.value.seconds, 3671 + 1 / fr, 'AbsTol', 1e-6, ... + 'row 7 = frame 2 of the second video, which started 3671 s after the first'); + + sp = named('velocity_calculation', 'concentration_worm0121', 'median speed on patch'); + testCase.verifyEqual([sp{1}.velocity.value.meters_per_second], [10 20] * 1e-6, 'AbsTol', 1e-15); + testCase.verifyTrue(ismember(list{1}.base.id, edgeAll(sp{1}, 'input_id')), ... + 'a per-encounter value is computed over the encounter list'); + de = named('acceleration_calculation', 'concentration_worm0121', 'deceleration on entry'); + testCase.verifyEqual([de{1}.acceleration.value.meters_per_second_squared], [-40 -50] * 1e-6, 'AbsTol', 1e-15); + lb = named('label_calculation', 'concentration_worm0121', 'encounter type'); + testCase.verifyEqual({lb{1}.label.value.name}, {'exploit', 'sample'}); + pr = named('score_calculation', 'concentration_worm0121', 'probability of exploitation'); + testCase.verifyEqual([pr{1}.score.value.score], [0.9 0.2], 'AbsTol', 1e-12); + g0 = named('velocity_calculation', 'concentration_worm0121', ... + 'median speed off patch before the first encounter'); + testCase.verifyEqual(g0{1}.velocity.value.meters_per_second, 50e-6, 'AbsTol', 1e-15); + + % the key takes its positions from the list, once both repositories have key_id + keys = entries(sp{1}.data.keys); + if isfield(keys{1}, 'positions_from') && ~isempty(keys{1}.positions_from) + testCase.verifyEqual(edgeAll(sp{1}, 'key_id'), {list{1}.base.id}); + end + testCase.verifyEqual(keys{1}.n, 2); + end + + function testEcoliProfiles(testCase) + % stage 10 part C (decision #62): each analysed image's mask, and + % each detected patch's values on its patch SUBJECT, matched by + % position. Image 2's bwlabel order puts the bottom-left patch + % first; the grid makes the top-left patch 1. + result = testCase.write("ecoli_0001", ["metadata", "calculations"]); + c = result.written{1}.ecoli.counts; + testCase.verifyEqual([c.mask, c.matched_images, c.unmatched_images], [2 2 0]); + testCase.verifyEqual(c.patch_values, 13 * 6, '13 patches x 6 values'); + hasItem = ndi.setup.V2.schemaHasField('item', 'value'); + testCase.verifyEqual(c.nearest_patch, double(hasItem), 'only image 2 has a closest map'); + + docs = testCase.documents(result.sessions.path{1}); + classes = cellfun(@(d) d.document_class.class_name, docs, 'UniformOutput', false); + subj = docs(strcmp(classes, 'subject')); + idOf = containers.Map(cellfun(@(d) d.subject.local_identifier, subj, 'UniformOutput', false), ... + cellfun(@(d) d.base.id, subj, 'UniformOutput', false)); + border = docs(strcmp(classes, 'intensity_calculation')); + border = border(cellfun(@(d) strcmp(d.statement.variable.name, 'patch border amplitude'), border)); + amp = @(local) cellfun(@(d) d.intensity.value.arbitrary_units, border(cellfun(@(d) ... + strcmp(edge(d, 'entity_id'), idOf(local)), border))); + testCase.verifyEqual(amp('ecoli_plate0001_patch0001'), 102, 'row 2 is the top-left patch'); + testCase.verifyEqual(amp('ecoli_plate0001_patch0002'), 101, 'row 1 is the bottom-left patch'); + testCase.verifyEqual(amp('ecoli_plate0002_patch0001'), 500, 'the one-patch plate'); + + len = docs(strcmp(classes, 'length_calculation')); + l1 = len(cellfun(@(d) strcmp(edge(d, 'entity_id'), idOf('ecoli_plate0002_patch0001')), len)); + testCase.verifyEqual(l1{1}.length.value.meters, 0.2e-3, 'AbsTol', 1e-15); + + masks = docs(strcmp(classes, 'label_calculation')); + testCase.verifyNumElements(masks, 2); + plates = {idOf('ecoli_plate0001'), idOf('ecoli_plate0002')}; + testCase.verifyTrue(all(cellfun(@(d) ismember(edge(d, 'entity_id'), plates), masks)), ... + 'a mask is a calculation of its plate'); + + % decision #64: the TIFFs in images/ are background-normalised + % images, calculated from the raw image and its background image + testCase.verifyEqual([c.normalised_image, c.background_fit, c.profiles], [2 1 13]); + ic = docs(strcmp(classes, 'intensity_calculation')); + byVar = @(name) ic(cellfun(@(d) strcmp(d.statement.variable.name, name), ic)); + norm = byVar('background-normalised fluorescence image'); + testCase.verifyNumElements(norm, 2); + obs = docs(strcmp(classes, 'intensity_observation')); + rawOf = @(n) obs{cellfun(@(d) any(cellfun(@(t) strcmp(t, epochRefOf(docs, classes, n)), ... + edgeAll(d, 'time_reference_id'))), obs)}.base.id; + n2 = norm{cellfun(@(d) strcmp(edge(d, 'entity_id'), idOf('ecoli_plate0001')), norm)}; + testCase.verifyEqual(sort(edgeAll(n2, 'input_id')), sort({rawOf(2), rawOf(9)}), ... + 'from the raw image and its background image'); + m2 = masks{cellfun(@(d) strcmp(edge(d, 'entity_id'), idOf('ecoli_plate0001')), masks)}; + testCase.verifyEqual(edgeAll(m2, 'input_id'), {n2.base.id}, ... + 'the mask is from the normalised image'); + + % the background fit's quality, on the background's plate + sc = docs(strcmp(classes, 'score_calculation')); + rs = sc(cellfun(@(d) strcmp(d.statement.variable.name, 'background fit r-squared'), sc)); + testCase.verifyNumElements(rs, 1); + testCase.verifyEqual(rs{1}.score.value.score, 0.95); + cc = docs(strcmp(classes, 'count_calculation')); + testCase.verifyEqual(cc{1}.count.value.count, 7); + + % the profile curves: patch 1 (top-left) is row 2 of image 2 + curves = byVar('intensity by distance from patch edge'); + testCase.verifyNumElements(curves, 13); + p1 = curves{cellfun(@(d) strcmp(edge(d, 'entity_id'), idOf('ecoli_plate0001_patch0001')), curves)}; + k = entries(p1.data.keys); + testCase.verifyEqual(k{1}.variable.name, 'distance from patch edge'); + testCase.verifyEqual(k{1}.n, 5); + v = testCase.bodyValues(result.sessions.path{1}, docs, classes, p1.base.id, 'double'); + testCase.verifyEqual(v(:)', [201 202 203 204 NaN]); + testCase.verifyEqual(edgeAll(p1, 'input_id'), {n2.base.id, m2.base.id}); + end + + function testDensity(testCase) + % stage 12 (decision #65): the E. coli border amplitude fits, each + % about a group of patches, and each worm's per-encounter estimates + testCase.assumeTrue(ndi.setup.V2.schemaHasLeaf('time_calculation'), ... + 'DID_SCHEMA_PATH does not hold time_calculation (did-schema PR #87)'); + result = testCase.write(["ecoli_0001", "concentration_0001"], ... + ["metadata", "observations", "calculations", "encounters", "density"]); + de = result.density; + c = de.counts; + testCase.verifyEqual([c.fits, c.groups, c.members, c.inputs, c.refit_checked], ... + [2 2 24 22 2], ['the OD600 1 fit and the joint low-OD fit, each about the 12 ' ... + 'patches of plate 1, from the 11 included values; the 200 uL fit has no image']); + testCase.verifyTrue(any(contains(de.skipped, '200 uL'))); + testCase.verifyEqual([c.worms, c.estimates], [2 4]); + testCase.verifyEqual(c.encounters_checked, 6); + testCase.verifyEmpty(de.checks.encounterDisagrees, 'encounter.mat reproduces from the fits'); + + dd = de.datasetDocuments; + dc = cellfun(@ndi.v2.leafName, dd, 'UniformOutput', false); + fits = dd(strcmp(dc, 'model_fit_calculation')); + groups = dd(strcmp(dc, 'subject')); + testCase.verifyNumElements(groups, 2); + testCase.verifyEqual(sum(strcmp(dc, 'directed_relation')), 24); + lin = fits{cellfun(@(d) strcmp(d.model_fit.value.equation, 'f = slope*t + intercept'), fits)}; + testCase.verifyTrue(ismember(edge(lin, 'entity_id'), ... + cellfun(@(d) d.base.id, groups, 'UniformOutput', false)), 'the fit is about a group'); + co = entries(lin.model_fit.value.coefficients); + testCase.verifyEqual(co{1}.value, 1 / 3000, 'AbsTol', 1e-15); + testCase.verifyEqual(co{2}.value, 94 / 300, 'AbsTol', 1e-15); + testCase.verifyNumElements(edgeAll(lin, 'input_id'), 11, 'row 12 is excluded'); + + % the estimates, in the C. elegans session + k = find(strcmp(result.sessions.local_identifier, 'concentration_0001')); + docs = testCase.documents(result.sessions.path{k}); + classes = cellfun(@(d) d.document_class.class_name, docs, 'UniformOutput', false); + subj = docs(strcmp(classes, 'subject')); + idOf = containers.Map(cellfun(@(d) d.subject.local_identifier, subj, 'UniformOutput', false), ... + cellfun(@(d) d.base.id, subj, 'UniformOutput', false)); + named = @(local, var) docs(strcmp(classes, 'intensity_calculation') & cellfun(@(d) ... + strcmp(edge(d, 'entity_id'), idOf(local)) && isfield(d, 'statement') ... + && strcmp(d.statement.variable.name, var), docs)); + e121 = named('concentration_worm0121', 'estimated patch border amplitude'); + testCase.verifyNumElements(e121, 1); + testCase.verifyEqual([e121{1}.intensity.value.arbitrary_units], ... + 94 / 300 + [74 80] / 3000, 'AbsTol', 1e-12); + testCase.verifyTrue(ismember(lin.base.id, edgeAll(e121{1}, 'input_id')), ... + 'from the OD600 1 fit'); + g111 = named('concentration_worm0111', 'estimated cultivation plate border amplitude'); + testCase.verifyNumElements(g111, 1); + testCase.verifyEqual(g111{1}.intensity.value.arbitrary_units, 55.7 - 0.0068 * 3060, 'AbsTol', 1e-9); + end + + function testDataset(testCase) + % stage 13 (decision #66): one V2 database holding the dataset-level + % documents and both sessions, ingested; checked, and opened as NDI does + testCase.assumeTrue(ndi.setup.V2.schemaHasLeaf('time_calculation'), ... + 'DID_SCHEMA_PATH does not hold time_calculation (did-schema PR #87)'); + result = testCase.write(["ecoli_0001", "concentration_0001"], ... + ["metadata", "observations", "calculations", "encounters", "density", "dataset"]); + r = result.dataset.report; + testCase.verifyEqual(r.sessions.found, 2); + testCase.verifyEmpty(r.sessions.notInDataset, 'each session is part_of a study of the dataset'); + testCase.verifyEmpty(r.duplicateIds); + testCase.verifyGreaterThan(r.edges.checked, 0); + testCase.verifyEqual(height(r.edges.dangling), 0, evalc('disp(r.edges.dangling)')); + + ds = ndi.dataset.dir(result.dataset.path); + testCase.verifyEqual(ds.id(), result.datasetSessionId); + [refs, ids] = ds.session_list(); + testCase.verifyEqual(sort(refs), {'concentration_0001', 'ecoli_0001'}); + s = ds.open_session(ids{strcmp(refs, 'ecoli_0001')}); + testCase.verifyNotEmpty(s.database_search(ndi.v2.isaQuery('subject')), ... + 'an ingested session reads its own documents from the dataset''s database'); + again = ndi.dataset.dir(result.dataset.path); + testCase.verifyEqual(again.id(), result.datasetSessionId, ... + 'still the dataset after a session was opened in its folder'); + testCase.verifyNumElements(again.database_search(ndi.v2.isaQuery('model_fit_calculation')), 2, ... + 'stage 12''s fits are in the dataset'); + + % the run's own record, beside the dataset + testCase.verifyTrue(isfolder(result.runFolder), 'a run folder is made with ''Write'''); + for f = {'import_log.txt', 'import_report.mat', 'skipped.txt'} + testCase.verifyTrue(isfile(fullfile(result.runFolder, f{1})), f{1}); + end + rep = load(fullfile(result.runFolder, 'import_report.mat')); + testCase.verifyFalse(rep.report.failed); + testCase.verifyEqual(rep.report.dataset.id, result.dataset.id); + testCase.verifySubstring(fileread(fullfile(result.runFolder, 'import_log.txt')), ... + '== stage 13: check & write the dataset =='); + + % the verification the import's user runs afterwards: every check passes, + % and its census is the import's own count + v = ndi.setup.conv.haley.verifyDataset(result.dataset.path, 'Expected', result); + testCase.verifyEmpty(v.failed, evalc('disp(v.census.differences)')); + testCase.verifyEqual(v.census.total, sum(r.documents.byClass.GroupCount)); + testCase.verifyEqual(v.edges.dangling, 0); + testCase.verifyGreaterThan(v.files.ingested, 0); + testCase.verifyGreaterThan(v.hashes.checked, 0, 'ingested bodies record an MD5'); + + % the term inventory: plain-text and ontology terms, with where they sit + csv = [tempname '.csv']; + T = ndi.setup.conv.haley.termInventory(result.dataset.path, 'OutFile', csv); + testCase.addTeardown(@() delete(csv)); + fprintf('TERM INVENTORY (test dataset):\n%s\nEND TERM INVENTORY\n', fileread(csv)); + testCase.verifyTrue(isfile(csv)); + testCase.verifyTrue(any(T.kind == "plain text" & T.name == "image intensity"), ... + 'the recordings'' variable is listed as plain text'); + testCase.verifyTrue(any(T.kind == "ontology"), 'species carry ontology nodes'); + testCase.verifyEqual(sum(T.kind == "ontology" & T.node == ""), 0); + end + + function testOccupiedFolderStopsTheRunFirst(testCase) + % a session folder that already holds a database is found + % before anything is built, not after the dataset is written + testCase.write("concentration_0001"); + testCase.verifyError(@() ndi.setup.conv.haley.import_V2(testCase.Root, ... + 'Stages', ["sessions", "subjects", "acquisition"], ... + 'Sessions', "concentration_0001", 'OutputRoot', testCase.Out, ... + 'Write', true, 'ReadVideos', false), 'ndi:setup:conv:haley:foldersTaken'); + end + + function testUnknownSessionIsAnError(testCase) + testCase.verifyError(@() ndi.setup.conv.haley.import_V2(testCase.Root, ... + 'Stages', "sessions", 'Sessions', "concentration_0099", ... + 'OutputRoot', testCase.Out), 'ndi:setup:conv:haley:unknownSession'); + end + end + + methods + function result = write(testCase, which, more) + if nargin < 3, more = string.empty; end + result = ndi.setup.conv.haley.import_V2(testCase.Root, ... + 'Stages', ["sessions", "subjects", "acquisition", "relations", more], 'Sessions', which, ... + 'OutputRoot', testCase.Out, 'Write', true, 'Overwrite', true, ... + 'ReadVideos', false); + end + + function v = bodyValues(~, sessionPath, docs, classes, ownerId, precision) + % the values of the ingested body owned by OWNERID, read back + b = docs(strcmp(classes, 'sampled_body')); + b = b{cellfun(@(d) strcmp(edge(d, 'owner_id'), ownerId), b)}; + db = did2.database.sqlitedb(fullfile(sessionPath, '.ndi', ... + ndi.database.implementations.database.did2sqlite.DEFAULTFILENAME())); + held = db.filePath(b.base.id, 'body_data_0'); + db.close(); + tmp = [tempname '.bin.gz']; + copyfile(held, tmp); + raw = gunzip(tmp); + fid = fopen(raw{1}, 'r', 'ieee-le'); + v = fread(fid, Inf, [precision '=>' precision]); + fclose(fid); + delete(tmp); delete(raw{1}); + end + + function docs = documents(~, sessionPath) + db = did2.database.sqlitedb(fullfile(sessionPath, '.ndi', ... + ndi.database.implementations.database.did2sqlite.DEFAULTFILENAME())); + ids = db.allIds(); + docs = cellfun(@(i) newNames(db.get(i).documentProperties), ids, 'UniformOutput', false); + db.close(); + end + end +end + +function d = newNames(d) +% a document as V_eta names it since 2026-10-08 (did-schema V_eta_tenets.md, T2 +% amendment), whichever schema wrote it: the statement blocks lose `subject_`, +% `data_type` is `value`, and a statement's `subject_id` edge is `entity_id` +for old = {'subject_statement', 'subject_assertion', 'subject_interaction', ... + 'subject_observation', 'subject_manipulation', 'subject_calculation', 'data_type'} + if isfield(d, old{1}) && isstruct(d.(old{1})) + d.(ndi.v2.vetaName(old{1})) = d.(old{1}); + d = rmfield(d, old{1}); + end +end +if isfield(d, 'statement') && isfield(d, 'depends_on') && isstruct(d.depends_on) + for k = 1:numel(d.depends_on) + if strcmp(d.depends_on(k).name, 'subject_id'), d.depends_on(k).name = 'entity_id'; end + end +end +% ... and as the per-leaf classes name it (did-schema +% V_eta_entity_composition_plan.md, 2026-10-08): an `observation` listing +% `temperature` is a temperature_observation, and an `entity` of type organism a +% `subject` (its fields under `subject`), so these tests read both schemas +[name, block] = ndi.v2.leafName(d); +if strcmp(block, 'entity') && ~strcmp(name, 'entity') + d.(name) = d.entity; + d = rmfield(d, 'entity'); +end +d.document_class.class_name = name; +end + +function v = edge(doc, name) +v = edgeAll(doc, name); +if isempty(v) + v = ''; +else + v = v{1}; +end +end + +function id = epochRefOf(docs, classes, n) +% the id of the relative time reference onto the epoch of E. coli image N +ep = docs(strcmp(classes, 'epoch')); +e = ep{cellfun(@(d) strcmp(d.epoch.local_identifier, sprintf('ecoli_image%04d', n)), ep)}; +rr = docs(strcmp(classes, 'relative_time_reference')); +r = rr(cellfun(@(d) strcmp(edge(d, 'referent_id'), e.base.id), rr)); +id = r{1}.base.id; +end + +function c = entries(x) +% a decoded list as a cell array (jsondecode gives a struct array when its +% entries share fields, a cell array when they do not) +if iscell(x), c = reshape(x, 1, []); else, c = num2cell(reshape(x, 1, [])); end +end + +function v = edgeAll(doc, name) +v = {}; +if ~isfield(doc, 'depends_on') || isempty(doc.depends_on) + return; +end +d = doc.depends_on; +if iscell(d), d = [d{:}]; end +hit = d(strcmp({d.name}, name)); +if isempty(hit) && strcmp(name, 'entity_id') + hit = d(strcmp({d.name}, 'subject_id')); % the edge's name before 2026-10-08 +end +v = {hit.document_id}; +v = v(~cellfun(@isempty, v)); +end + +function t = subjectType(byLocal, local) +d = byLocal(local); +t = d.subject.type.name; +end diff --git a/tests/+ndi/+unittest/+setup/+V2/TestLinkedSessions.m b/tests/+ndi/+unittest/+setup/+V2/TestLinkedSessions.m new file mode 100644 index 0000000000..01bbdf4e93 --- /dev/null +++ b/tests/+ndi/+unittest/+setup/+V2/TestLinkedSessions.m @@ -0,0 +1,253 @@ +classdef TestLinkedSessions < matlab.unittest.TestCase +%TESTLINKEDSESSIONS Linked and ingested sessions in a V2 dataset. +% +% V_eta_linked_session_plan.md (signed 2026-10-09): a session is in a +% dataset when it is `part_of` the dataset or a study of it; an INGESTED +% session's documents are in the dataset's database, a LINKED one's in its +% own folder, which a `linked_session` document names. Writes +% TestHaleyRecordings' synthetic concentration_0001 into a dataset once, +% then each test works on its own copy of that dataset. +% +% Tests that link need the `linked_session` class; with a schema that +% does not have it (the `current` CI leg) they return early (the workflow +% fails a run in which any test is filtered). + + properties + Root + Master % the dataset the import wrote, every session ingested + Result + Linked % the import's result with 'SessionFolders': sessions linked + Path % this test's copy of the dataset + Dataset + end + + methods (TestClassSetup) + function fixture(testCase) + testCase.Root = tempname; + testCase.addTeardown(@() rmdir(testCase.Root, 's')); + ndi.unittest.setup.V2.TestHaleyRecordings.writeFixture(testCase.Root); + testCase.Result = ndi.setup.conv.haley.import_V2(testCase.Root, ... + 'Stages', ["sessions", "subjects", "acquisition", "relations", "metadata", ... + "assertions", "manipulations", "observations", "calculations", "dataset"], ... + 'Sessions', "concentration_0001", ... + 'OutputRoot', fullfile(testCase.Root, 'haley_V2'), 'Write', true, ... + 'Overwrite', true, 'ReadVideos', false, 'SessionFolders', false); + testCase.Master = testCase.Result.dataset.path; + % 'SessionFolders' (decision #71): each session written once, to its + % folder, and linked from the dataset + testCase.Linked = ndi.setup.conv.haley.import_V2(testCase.Root, ... + 'Stages', ["sessions", "subjects", "acquisition", "relations", "metadata", ... + "assertions", "dataset"], ... + 'Sessions', "concentration_0001", ... + 'OutputRoot', fullfile(testCase.Root, 'haley_V2_linked'), 'Write', true, ... + 'Overwrite', true, 'ReadVideos', false, 'SessionFolders', true); + end + end + + methods (TestMethodSetup) + function copyDataset(testCase) + testCase.Path = fullfile(testCase.Root, ['ds_' char(ndi.ido.unique_id())]); + copyfile(testCase.Master, testCase.Path); + testCase.Dataset = ndi.dataset.dir(testCase.Path); + end + end + + methods (Test) + function testIngestedSessionIsListedFromPartOf(testCase) + [refs, ids] = testCase.Dataset.session_list(); + testCase.verifyEqual(refs, {'concentration_0001'}); + testCase.verifyEqual(numel(ids), 1); + notes = testCase.Dataset.session_notes(); + testCase.verifyTrue(startsWith(notes{1}, 'DENOMINATOR:'), ... + 'The listing states its denominator first.'); + testCase.verifyTrue(contains(notes{1}, '1 ingested and 0 linked'), notes{1}); + S = testCase.Dataset.open_session(ids{1}); + testCase.verifyEqual(S.reference, 'concentration_0001'); + end + + function testIngestedToLinkedAndBack(testCase) + if ~canLink(), return; end + [~, ids] = testCase.Dataset.session_list(); + sid = ids{1}; + before = testCase.sessionDocuments(sid); + [docId, name] = ingestedFile(before); + + folder = fullfile(testCase.Root, ['linked_' char(ndi.ido.unique_id())]); + testCase.Dataset.convertIngestedSessionToLinked(sid, folder, 'areYouSure', true); + testCase.verifyTrue(isfile(fullfile(folder, '.ndi', ... + ndi.database.implementations.database.did2sqlite.DEFAULTFILENAME()))); + + % opened again from disk, the session is linked and in the folder + ds = ndi.dataset.dir(testCase.Path); + [refs, ids2] = ds.session_list(); + testCase.verifyEqual(refs, {'concentration_0001'}); + testCase.verifyEqual(ids2, {sid}); + notes = ds.session_notes(); + testCase.verifyTrue(contains(notes{1}, '0 ingested and 1 linked'), notes{1}); + S = ds.open_session(sid); + testCase.verifyEqual(canonical(S.path), canonical(folder)); + % every document is still reached through the dataset + testCase.verifyEqual(sort(idsOf(testCase.sessionDocuments(sid, ds))), sort(idsOf(before))); + if ~isempty(docId) + testCase.verifyTrue(S.database_existbinarydoc(docId, name), ... + 'An ingested file moves out with its document.'); + end + + ds.convertLinkedSessionToIngested(sid, 'areYouSure', true); + ds = ndi.dataset.dir(testCase.Path); + notes = ds.session_notes(); + testCase.verifyTrue(contains(notes{1}, '1 ingested and 0 linked'), notes{1}); + testCase.verifyEqual(sort(idsOf(testCase.sessionDocuments(sid, ds))), sort(idsOf(before))); + if ~isempty(docId) + testCase.verifyTrue(ds.database_existbinarydoc(docId, name), ... + 'An ingested file comes back with its document.'); + end + end + + function testAddAndUnlinkALinkedSession(testCase) + if ~canLink(), return; end + folder = fullfile(testCase.Root, ['extra_' char(ndi.ido.unique_id())]); + mkdir(folder); + S2 = ndi.setup.V2.createSession(folder, 'extra'); + testCase.Dataset.add_linked_session(S2); + [refs, ids] = testCase.Dataset.session_list(); + testCase.verifyEqual(sort(refs), sort({'concentration_0001', 'extra'})); + % its documents are found through the dataset + found = testCase.Dataset.database_search(ndi.query('base.session_id', 'exact_string', S2.id(), '')); + testCase.verifyNotEmpty(found); + % and the dataset opened again from disk still has it + ds = ndi.dataset.dir(testCase.Path); + [refs2, ~] = ds.session_list(); + testCase.verifyEqual(sort(refs2), sort(refs)); + + ds.unlink_session(S2.id(), 'areYouSure', true); + [refs3, ~] = ds.session_list(); + testCase.verifyEqual(refs3, {'concentration_0001'}); + testCase.verifyTrue(isfolder(fullfile(folder, '.ndi')), 'Unlinking leaves the folder.'); + testCase.verifyNotEmpty(ids); + end + + function testASessionAlreadyInTheDatasetIsNotAddedTwice(testCase) + if ~canLink(), return; end + % the linked import's session folder holds a session its dataset has + ds = ndi.dataset.dir(testCase.Linked.dataset.path); + S = ndi.session.dir(testCase.Linked.sessions.path{1}); + testCase.verifyError(@() ds.add_linked_session(S), ?MException); + end + + function testTheImportLinksItsSessions(testCase) + if ~canLink(), return; end + L = testCase.Linked; + ds = ndi.dataset.dir(L.dataset.path); + [refs, ids] = ds.session_list(); + testCase.verifyEqual(refs, {'concentration_0001'}); + notes = ds.session_notes(); + testCase.verifyTrue(contains(notes{1}, '0 ingested and 1 linked'), notes{1}); + % written once: the dataset's database holds no document of the + % session but its part_of relation + files = ndi.setup.V2.datasetDatabaseFiles(L.dataset.path); + testCase.verifyNumElements(files, 2, 'the dataset''s database and the session folder''s'); + db = did2.database.sqlitedb(files{1}); + n = mksqlite(db.testHookDbId(), sprintf(['SELECT COUNT(*) AS n FROM documents ' ... + 'WHERE session_id = ''%s'' AND classname <> ''directed_relation'''], ids{1})); + db.close(); + testCase.verifyEqual(double(n.n), 0, 'no session document is copied into the dataset'); + % opened through the dataset, the session reaches the dataset's documents + S = ds.open_session(ids{1}); + testCase.verifyEqual(canonical(S.path), canonical(L.sessions.path{1})); + testCase.verifyNotEmpty(S.database_search_with_dataset(ndi.v2.isaQuery('study')), ... + 'a linked session opened through its dataset finds the dataset''s studies'); + alone = ndi.session.dir(L.sessions.path{1}); + testCase.verifyEmpty(alone.database_search_with_dataset(ndi.v2.isaQuery('study')), ... + 'opened on its own, the folder holds only the session''s documents'); + % and the dataset passes its checks, over both databases + v = ndi.setup.conv.haley.verifyDataset(L.dataset.path, 'Expected', L, 'Hashes', false); + testCase.verifyEmpty(v.failed, evalc('disp(v.census.differences)')); + testCase.verifyEqual(v.edges.dangling, 0); + end + + function testAddAnIngestedSession(testCase) + folder = fullfile(testCase.Root, ['ingest_' char(ndi.ido.unique_id())]); + mkdir(folder); + S2 = ndi.setup.V2.createSession(folder, 'ingested_extra'); + testCase.Dataset.add_ingested_session(S2); + ds = ndi.dataset.dir(testCase.Path); + [refs, ids] = ds.session_list(); + testCase.verifyEqual(sort(refs), sort({'concentration_0001', 'ingested_extra'})); + notes = ds.session_notes(); + testCase.verifyTrue(contains(notes{1}, '2 ingested and 0 linked'), notes{1}); + S = ds.open_session(ids{strcmp(refs, 'ingested_extra')}); + testCase.verifyEqual(canonical(S.path), canonical(testCase.Path), ... + 'An ingested session opens from the dataset''s folder.'); + end + + function testAFolderThatMovedIsReported(testCase) + if ~canLink(), return; end + [~, ids] = testCase.Dataset.session_list(); + folder = fullfile(testCase.Root, ['moving_' char(ndi.ido.unique_id())]); + testCase.Dataset.convertIngestedSessionToLinked(ids{1}, folder, 'areYouSure', true); + movefile(folder, [folder '_moved']); + ds = ndi.dataset.dir(testCase.Path); + refs = testCase.verifyWarning(@() ds.session_list(), 'ndi:dataset:sessionsLeftOut'); + testCase.verifyEmpty(refs); + notes = ds.session_notes(); + testCase.verifyTrue(any(contains(notes, 'no such folder')), strjoin(notes, newline)); + end + + function testMakeSelfContained(testCase) + dry = testCase.Dataset.makeSelfContained('DryRun', true); + testCase.verifyGreaterThan(dry.documents, 0); + testCase.verifyGreaterThan(dry.ingested, 0, ... + 'The fixture records its videos by location, so there is something to copy in.'); + done = testCase.Dataset.makeSelfContained(); + testCase.verifyEqual(done.ingested, dry.ingested); + testCase.verifyEqual(done.withByLocation, dry.withByLocation); + again = testCase.Dataset.makeSelfContained('DryRun', true); + testCase.verifyEqual(again.ingested, 0, 'Nothing is left by location.'); + testCase.verifyEqual(again.documents, dry.documents, 'No document was lost.'); + end + end + + methods + function docs = sessionDocuments(testCase, sid, ds) + if nargin < 3, ds = testCase.Dataset; end + found = ds.database_search(ndi.query('base.session_id', 'exact_string', sid, '')); + docs = cellfun(@(d) d.document_properties, found, 'UniformOutput', false); + end + end +end + +function tf = canLink() +% the schema in use has the linked_session class +tf = ndi.setup.V2.schemaHasClass('linked_session'); +end + +function ids = idsOf(docs) +ids = cellfun(@(d) d.base.id, docs, 'UniformOutput', false); +end + +function p = canonical(p) +p = char(java.io.File(p).getCanonicalPath()); +end + +function [docId, name] = ingestedFile(docs) +% a document of DOCS with an ingested file, and that file's name ('' if none) +docId = ''; name = ''; +for i = 1:numel(docs) + d = docs{i}; + if ~isfield(d, 'files') || ~isfield(d.files, 'file_info') || isempty(d.files.file_info) + continue; + end + fi = d.files.file_info; + if iscell(fi), fi = [fi{:}]; end + for a = 1:numel(fi) + L = fi(a).locations; + if iscell(L), L = [L{:}]; end + if any(arrayfun(@(x) isfield(x, 'ingest') && logical(x.ingest), L)) + docId = d.base.id; + name = char(fi(a).name); + return; + end + end +end +end diff --git a/tests/+ndi/+unittest/+setup/+V2/TestObjectLayer.m b/tests/+ndi/+unittest/+setup/+V2/TestObjectLayer.m new file mode 100644 index 0000000000..9aafbdf320 --- /dev/null +++ b/tests/+ndi/+unittest/+setup/+V2/TestObjectLayer.m @@ -0,0 +1,1007 @@ +classdef TestObjectLayer < matlab.unittest.TestCase +%TESTOBJECTLAYER The V2 object layer (ndi.entity, ndi.statement and its +%children, ndi.value) over a written Haley session. +% +% Writes TestHaleyRecordings' synthetic concentration_0001 with its +% assertions, manipulations and calculations, opens it with +% ndi.session.dir, and asks the questions through the objects that +% TestHaleyWrite answers by reading documents. ndi.subject is v1's and +% is not used: every entity, a worm included, is an ndi.entity. + + properties + Root + Session + Dataset + Result + end + + methods (TestClassSetup) + function fixture(testCase) + testCase.Root = tempname; + testCase.addTeardown(@() rmdir(testCase.Root, 's')); + ndi.unittest.setup.V2.TestHaleyRecordings.writeFixture(testCase.Root); + testCase.Result = ndi.setup.conv.haley.import_V2(testCase.Root, ... + 'Stages', ["sessions", "subjects", "acquisition", "relations", "metadata", ... + "assertions", "manipulations", "observations", "calculations", "dataset"], ... + 'Sessions', "concentration_0001", ... + 'OutputRoot', fullfile(testCase.Root, 'haley_V2'), 'Write', true, ... + 'Overwrite', true, 'ReadVideos', false, 'SessionFolders', true); + testCase.Session = ndi.session.dir(testCase.Result.sessions.path{1}); + testCase.Dataset = ndi.dataset.dir(testCase.Result.dataset.path); + end + end + + methods (Test) + function testSubjectIsAnEntity(testCase) + w = testCase.subject('concentration_worm0121'); + testCase.verifyClass(w, 'ndi.entity'); + testCase.verifyEqual(w.kind, 'subject'); + p = w.document_properties(); + testCase.verifyEqual(w.name, char(ndi.v2.blockOf(p, 'subject', 'name')), 'a subject''s display name is its own name'); + testCase.verifyEqual(w.name(), 'Worm 0121'); + testCase.verifyEqual(w.local_identifier, 'concentration_worm0121', ... + 'no @ is required of a V2 subject read from a document (D6)'); + if ndi.setup.V2.schemaHasField('subject', 'type') + testCase.verifyEqual(w.type, 'organism'); + g = ndi.entity.search(testCase.Session, {'Type', 'group'}); + testCase.verifyNumElements(g, 4, 'one cohort per assay plate'); + end + e = ndi.entity.fromDocument(testCase.Session, w.document_id); + testCase.verifyClass(e, 'ndi.entity'); + testCase.verifyEqual(e.document_id, w.document_id); + end + + function testNameTypeKindAndIdAreProperties(testCase) + % read from the document when asked, shown when the object is + % displayed, and not settable + w = testCase.subject('concentration_worm0121'); + for p = {'name', 'type', 'kind', 'document_id'} + testCase.verifyTrue(isprop(w, p{1}), p{1}); + end + testCase.verifyEqual(w.name(), w.name, 'the method-call form still works'); + shown = evalc('disp(w)'); + testCase.verifySubstring(shown, 'Worm 0121'); + testCase.verifySubstring(shown, 'organism'); + testCase.verifyError(@() setName(w), ?MException); + % an entity with no document: all four are empty + s = ndi.entity(); + testCase.verifyEqual({s.name, s.type, s.kind, s.document_id}, {'', '', '', ''}); + end + + function testRelations(testCase) + w = testCase.subject('concentration_worm0121'); + g = w.memberOf(); + testCase.verifyNumElements(g, 1); + testCase.verifyEqual(g{1}.local_identifier, 'concentration_assayPlate0012_worms'); + m = g{1}.members(); + names = sort(cellfun(@(x) x.local_identifier, m, 'UniformOutput', false)); + testCase.verifyEqual(names, {'concentration_worm0121', 'concentration_worm0122'}); + T = g{1}.relations('member_of', 'Direction', 'in'); + testCase.verifyEqual(height(T), 2); + testCase.verifyEqual(unique(T.direction), "in"); + + p = testCase.subject('concentration_assayPlate0012_patch0001'); + w = p.partOf(); + testCase.verifyNumElements(w, 1); + testCase.verifyEqual(w{1}.local_identifier, 'concentration_assayPlate0012'); + parts = w{1}.parts(); + testCase.verifyEqual(cellfun(@(x) x.local_identifier, parts, 'UniformOutput', false), ... + {'concentration_assayPlate0012_patch0001'}); + end + + function testRelationPathsFromOneOrManyEntities(testCase) + % parents/children take a path of relations and an array of + % starting entities; each entity reached is returned once + w = testCase.subject('concentration_worm0121'); + plates = w.parents({'member_of', 'contained_in'}); + names = cellfun(@(x) x.local_identifier, plates, 'UniformOutput', false); + testCase.verifyTrue(all(ismember({'concentration_assayPlate0012', ... + 'concentration_0001_acclimationPlate0001'}, names)), strjoin(names, ', ')); + + a = testCase.subject('concentration_assayPlate0012'); + worms = a.children({'contained_in', 'member_of'}); + testCase.verifyEqual(sort(local(worms)), {'concentration_worm0121', 'concentration_worm0122'}); + + b = testCase.subject('concentration_0001_acclimationPlate0001'); + % the acclimation plate also held other cohorts (0011's worms): + % the array call is the union of the one-plate calls, each worm + % once, though 0121 and 0122 were on both plates + onB = local(b.children({'contained_in', 'member_of'})); + testCase.verifyTrue(all(ismember({'concentration_worm0121', 'concentration_worm0122'}, onB))); + both = children([a b], {'contained_in', 'member_of'}); + testCase.verifyEqual(sort(local(both)), sort(unique([local(worms), onB])), ... + 'a worm on both plates is returned once'); + T = children([a b], {'contained_in', 'member_of'}, 'Table', true); + testCase.verifyEqual(height(T), numel(worms) + numel(onB), 'one row per (plate, worm)'); + testCase.verifyEqual(sort(unique(T.start_name)), sort(string({a.name; b.name}))); + testCase.verifyEmpty(a.children({'member_of', 'member_of'}), 'nothing at the end of that path'); + end + + function testRelationsOfManyEntities(testCase) + a = testCase.subject('concentration_assayPlate0012'); + b = testCase.subject('concentration_0001_acclimationPlate0001'); + Ta = a.relations(); + Tb = b.relations(); + T = relations([a b]); + testCase.verifyEqual(height(T), height(Ta) + height(Tb), 'the same rows as one at a time'); + testCase.verifyEqual(sort(T.relation_id(T.start_id == a.document_id)), sort(Ta.relation_id)); + testCase.verifyEqual(unique(T.start_name(T.start_id == b.document_id)), string(b.name)); + testCase.verifyTrue(any(T.relation == "contained_in" & T.direction == "in" & ... + T.start_id == a.document_id), 'the cohort contained_in the assay plate'); + testCase.verifyTrue(any(T.relation == "part_of" & T.direction == "in" & ... + T.start_id == a.document_id), 'its patches part_of it'); + In = relations([a b], 'contained_in', 'Direction', 'in'); + testCase.verifyEqual(unique(In.relation), "contained_in"); + testCase.verifyEqual(sort(unique(In.start_id)), sort(string({a.document_id; b.document_id}))); + testCase.verifyEqual(height(relations([a a])), height(Ta), 'an entity given twice is read once'); + end + + function testDescendantsAndAncestorsNeedNoRelationNames(testCase) + a = testCase.subject('concentration_assayPlate0012'); + worms = a.descendants('Type', 'organism'); + testCase.verifyEqual(sort(local(worms)), {'concentration_worm0121', 'concentration_worm0122'}); + T = a.descendants('Type', 'organism', 'Table', true); + testCase.verifyEqual(T.depth, [2; 2], 'plate <- cohort <- worm'); + testCase.verifyEqual(T.start_id, string(repmat({a.document_id}, 2, 1))); + + w = testCase.subject('concentration_worm0121'); + g = w.ancestors('Type', 'group'); + testCase.verifyEqual(local(g), {'concentration_assayPlate0012_worms'}); + m = local(w.ancestors('Type', 'material')); + testCase.verifyTrue(all(ismember({'concentration_assayPlate0012', ... + 'concentration_0001_acclimationPlate0001'}, m)), strjoin(m, ', ')); + testCase.verifyEqual(local(w.ancestors('Kind', 'subject')), {'concentration_assayPlate0012_worms'}, ... + 'a match is not followed further: the nearest subject is the cohort'); + + testCase.verifyEmpty(w.ancestors('Type', 'material', 'Kind', 'subject'), ... + 'Type and Kind together: a plate is not a subject (isa subject is the biological types)'); + testCase.verifyEmpty(w.ancestors('Type', 'material', 'Kind', 'person')); + + parts = local(a.descendants('Relation', 'part_of')); + testCase.verifyNotEmpty(parts); + testCase.verifyTrue(all(startsWith(parts, 'concentration_assayPlate0012_patch')), strjoin(parts, ', ')); + all_ = local(a.descendants()); + testCase.verifyTrue(all(ismember([parts, {'concentration_assayPlate0012_worms', ... + 'concentration_worm0121', 'concentration_worm0122'}], all_)), ... + 'with no filter, everything reached'); + testCase.verifyEmpty(w.descendants('Type', 'organism'), 'nothing points to a worm'); + end + + function testFindByWhatIsTrueOfASubject(testCase) + % strain and species are stated on each cohort, distributive + % (decision #55): a search finds the worms without knowing that + S = testCase.Session; + if ndi.setup.V2.mergedEntities() + % one entity class (2026-10-08): a cohort is an instance_of the + % dataset-level strain entity, so 'strain', 'N2' needs that + % entity in reach. A session opened from its dataset reaches it + % (ndi.session/database_search_with_dataset); a session folder + % opened on its own holds only the relation and cannot name it. + testCase.verifyError(@() ndi.entity.search(S, {'strain', 'N2'}), ... + 'ndi:entity:search:unknownVariable'); + [refs, ids] = testCase.Dataset.session_list(); + inDataset = testCase.Dataset.open_session(ids{strcmp(refs, S.reference)}); + testCase.verifyTrue(all(ismember({'concentration_worm0121', 'concentration_worm0122'}, ... + local(ndi.entity.search(inDataset, {'type', 'organism', 'strain', 'N2'})))), ... + 'a session opened from its dataset finds the strain entity there'); + S = testCase.Dataset; + end + w = ndi.entity.search(S, {'type', 'organism', 'species', 'Caenorhabditis elegans', 'strain', 'N2'}); + names = local(w); + testCase.verifyTrue(all(ismember({'concentration_worm0121', 'concentration_worm0122'}, names)), ... + strjoin(names, ', ')); + testCase.verifyTrue(all(cellfun(@(x) strcmp(x.type, 'organism'), w))); + testCase.verifyTrue(ismember('concentration_assayPlate0012_worms', ... + local(ndi.entity.search(S, {'strain', 'N2'}))), 'the cohort it was stated on matches too'); + + % property names and value names ignore case; nodes are exact + same = @(varargin) isequal(sort(local(ndi.entity.search(S, varargin))), sort(names)); + testCase.verifyTrue(same('Type', 'Organism', 'STRAIN', 'n2')); + testCase.verifyTrue(same('type', 'organism', 'species', 'NCBITaxon:6239', 'strain', 'N2')); + testCase.verifyTrue(same('type', 'organism', 'strain', {'N2', 'no such strain'}), 'a cell is any of'); + testCase.verifyTrue(same('type', 'organism', 'strain', 'N*'), 'a wildcard'); + testCase.verifyTrue(same('type', 'organism', 'species', '*ELEGANS', 'strain', 'N2'), ... + 'a wildcard on a name ignores case'); + testCase.verifyTrue(same('type', 'organism', 'species', 'NCBITaxon:*', 'strain', 'N2'), ... + 'a wildcard on a node'); + + % inherited false: only what was stated on the subject itself + testCase.verifyEmpty(ndi.entity.search(S, {'type', 'organism', 'strain', 'N2'}, 'inherited', false), ... + 'strain is stated on no worm itself'); + testCase.verifyEqual(sort(local(ndi.entity.search(S, {'strain', 'N2'}, 'inherited', false))), ... + sort(local(ndi.entity.search(S, {'strain', 'N2', 'type', 'group'}))), 'the cohorts alone'); + + ecoli = ndi.entity.search(S, {'species', 'NCBITaxon:562'}); + testCase.verifyNotEmpty(ecoli); + testCase.verifyFalse(any(cellfun(@(x) strcmp(x.type, 'organism'), ecoli)), 'lawns, not worms'); + testCase.verifyEqual(local(ndi.entity.search(S, {'strain', 'N2', ... + 'local_identifier', 'concentration_worm0121'})), {'concentration_worm0121'}); + testCase.verifyEqual(local(ndi.entity.search(S, {'LocalIdentifier', 'concentration_worm0121'})), ... + {'concentration_worm0121'}, 'the old spelling still works'); + + % a misspelled variable is an error; a value no one has, a warning + testCase.verifyError(@() ndi.entity.search(S, {'stran', 'N2'}), 'ndi:entity:search:unknownVariable'); + testCase.verifyWarning(@() ndi.entity.search(S, {'species', 'Caenorhabdiits elegans'}), ... + 'ndi:entity:search:noSuchValue'); + testCase.verifyEmpty(testCase.verifyWarning(@() ndi.entity.search(S, {'strain', 'no such strain'}), ... + 'ndi:entity:search:noSuchValue')); + % the conditions are one cell of pairs; the switches are options after it + testCase.verifyError(@() ndi.entity.search(S, {'strain'}), 'ndi:entity:search:pairs'); + testCase.verifyError(@() ndi.entity.search(S, {'strain', 'N2', 'inherited', false}), ... + 'ndi:entity:search:option'); + testCase.verifyError(@() ndi.entity.search(S, {'strain', 'N2'}, 'inheritd', false), ... + ?MException, 'a misspelt option is caught by the arguments block'); + end + + function testSynonymsFindATermByAnotherName(testCase) + % a value that matches no term by its name or node matches a term + % it is a synonym of (ndi.v2.synonyms); the names are read from + % the on-disk cache, prepared here so no lookup goes online + S = testCase.Session; + folder = tempname; + mkdir(folder); + testCase.addTeardown(@() rmdir(folder, 's')); + old = getenv('NDI_SYNONYM_CACHE'); + setenv('NDI_SYNONYM_CACHE', folder); + testCase.addTeardown(@() setenv('NDI_SYNONYM_CACHE', old)); + fid = fopen(fullfile(folder, 'ncbitaxon_6239.json'), 'w'); + fwrite(fid, jsonencode(struct('node', 'NCBITaxon:6239', 'names', ... + {{'Caenorhabditis elegans', 'C. elegans', 'nematode worm'}})), 'char'); + fclose(fid); + testCase.verifyEqual(ndi.v2.synonyms('ncbitaxon:6239'), ... + {'Caenorhabditis elegans', 'C. elegans', 'nematode worm'}, 'read from the cache, any case'); + worms = {'concentration_worm0121', 'concentration_worm0122'}; + w = local(ndi.entity.search(S, {'type', 'organism', 'species', 'C. elegans'})); + testCase.verifyTrue(all(ismember(worms, w)), strjoin(w, ', ')); + w = local(ndi.entity.search(S, {'type', 'organism', 'species', 'nematode*'})); + testCase.verifyTrue(all(ismember(worms, w)), 'a wildcard over the synonyms'); + testCase.verifyEmpty(testCase.verifyWarning(@() ndi.entity.search(S, ... + {'type', 'organism', 'species', 'C. elegans'}, 'synonyms', false), ... + 'ndi:entity:search:noSuchValue'), 'synonyms off: the name only'); + a = testCase.subject('concentration_assayPlate0012_worms').assertions( ... + {'variable', 'species', 'value', 'C. elegans'}); + testCase.verifyNotEmpty(a, 'statement filters use them too'); + testCase.verifyEmpty(ndi.v2.synonyms('not a curie')); + end + + function testFindByAStatementOfAKind(testCase) + % the six statement keys: the filters in one cell are one + % statement; a key given twice is two statements + S = testCase.Session; + plate = 'concentration_assayPlate0012'; + has = @(varargin) ismember(plate, local(ndi.entity.search(S, varargin))); + testCase.verifyTrue(has('manipulation', {'variable', 'NGM agar'})); + testCase.verifyTrue(has('manipulation', {'method', 'refrigeration'})); + testCase.verifyTrue(has('interaction', {'method', 'refrigeration'}), 'an interaction is any of the three'); + testCase.verifyTrue(has('statement', {'variable', 'NGM agar'})); + testCase.verifyTrue(has('manipulation', {'method', 'REFRIG*'}), 'a wildcard, any case'); + % a dose compares by its amount, in its canonical unit (liters) + testCase.verifyTrue(has('manipulation', {'variable', 'NGM agar', 'value', '>=0.02'})); + testCase.verifyTrue(has('manipulation', {'variable', 'NGM agar', 'value', '<=0.025'})); + w = warning('off', 'ndi:entity:search:noSuchValue'); + restore = onCleanup(@() warning(w)); + testCase.verifyFalse(has('manipulation', {'variable', 'NGM agar', 'value', '>0.03'})); + % one statement that is both, versus two statements + testCase.verifyEmpty(ndi.entity.search(S, {'manipulation', {'method', 'refrigeration', ... + 'variable', 'NGM agar'}}), 'no single manipulation is both: none, not an error'); + testCase.verifyTrue(has('manipulation', {'method', 'refrigeration'}, ... + 'manipulation', {'variable', 'NGM agar'})); + % the shorthand is an assertion (with one entity class a strain is + % a relation, not an assertion: testFindByWhatIsTrueOfASubject) + if ~ndi.setup.V2.mergedEntities() + testCase.verifyEqual(sort(local(ndi.entity.search(S, {'assertion', {'variable', 'strain', 'value', 'N2'}}))), ... + sort(local(ndi.entity.search(S, {'strain', 'N2'})))); + testCase.verifyEqual(sort(local(ndi.entity.search(S, {'str*', 'N2'}))), ... + sort(local(ndi.entity.search(S, {'strain', 'N2'}))), 'a wildcard in the property name'); + end + testCase.verifyError(@() ndi.entity.search(S, {'assertion', {'method', 'x'}}), ... + 'ndi:entity:search:assertionMethod'); + testCase.verifyError(@() ndi.entity.search(S, {'manipulation', {'method', 'no such method'}}), ... + 'ndi:entity:search:noSuchStatement'); + testCase.verifyError(@() ndi.entity.search(S, {'manipulation', {'colour', 'x'}}), ... + 'ndi:entity:search:statementFilter'); + testCase.verifyError(@() ndi.entity.search(S, {'manipulation', 'NGM agar'}), ... + 'ndi:entity:search:statementFilter'); + % a plate's statements are not its worms': contained_in is not followed + testCase.verifyEmpty(ndi.entity.search(S, {'type', 'organism', 'manipulation', {'variable', 'NGM agar'}})); + end + + function testFindByRelation(testCase) + S = testCase.Session; + worms = {'concentration_worm0121', 'concentration_worm0122'}; + cohort = testCase.subject('concentration_assayPlate0012_worms'); + plate = testCase.subject('concentration_assayPlate0012'); + names = @(varargin) sort(local(ndi.entity.search(S, varargin))); + + % the shortcut: a relation's name, the subject is the child + testCase.verifyEqual(names('member_of', cohort), worms); + testCase.verifyEqual(names('directed_relation', {'parent', cohort}), worms, 'no name: any relation'); + testCase.verifyEqual(names('directed_relation', {'name', 'member_of', 'parent', cohort.document_id}), ... + worms, 'the other end as a document id'); + % the cohort is contained_in the plate, distributive: so is each worm + testCase.verifyEqual(names('type', 'organism', 'contained_in', plate), worms); + testCase.verifyEmpty(ndi.entity.search(S, {'type', 'organism', 'contained_in', plate}, 'inherited', false), ... + 'not inherited: no worm is contained_in anything itself'); + % the other end described, not named + onNGM = names('type', 'organism', 'contained_in', {'manipulation', {'variable', 'NGM agar'}}); + testCase.verifyTrue(all(ismember(worms, onNGM)), strjoin(onNGM, ', ')); + % the reverse: the subject is the parent + p = names('directed_relation', {'name', 'part_of', 'child', ... + {'local_identifier', 'concentration_assayPlate0012_patch0001'}}); + testCase.verifyEqual(p, {'concentration_assayPlate0012'}); + % nested: worms on the plate that patch 0001 is part of + testCase.verifyEqual(names('type', 'organism', 'contained_in', {'directed_relation', ... + {'name', 'part_of', 'child', {'local_identifier', 'concentration_assayPlate0012_patch0001'}}}), worms); + % 'with': either end + w = names('relation', {'with', cohort}); + testCase.verifyTrue(all(ismember([worms, {'concentration_assayPlate0012', ... + 'concentration_0001_acclimationPlate0001'}], w)), strjoin(w, ', ')); + testCase.verifyEqual(local(ndi.entity.search(S, {'id', cohort.document_id})), ... + {'concentration_assayPlate0012_worms'}); + + % explain says what it will do + out = evalc('ndi.entity.search(S, {''type'', ''organism'', ''contained_in'', {''manipulation'', {''variable'', ''NGM agar''}}}, ''explain'', true);'); + testCase.verifySubstring(out, 'contained_in'); + testCase.verifySubstring(out, 'NGM agar'); + testCase.verifySubstring(out, 'Searching this session for organisms that', ... + 'the search names what it is for'); + testCase.verifySubstring(out, 'are contained_in an entity'); + testCase.verifySubstring(out, 'had a manipulation with variable NGM agar'); + % the noun follows a single kind or type condition; several types are listed + out = evalc('ndi.entity.search(S, {''kind'', ''person''}, ''explain'', true);'); + testCase.verifySubstring(out, 'Searching this session for all people'); + out = evalc('ndi.entity.search(S, {''type'', ndi.v2.subjectTypes(), ''species'', ''Caenorhabditis elegans''}, ''explain'', true);'); + testCase.verifySubstring(out, ... + 'Searching this session for organisms, cultures, tissues, cells or groups that', ... + 'several types are listed'); + out = evalc('ndi.entity.search(S, {''species'', ''Caenorhabditis elegans''}, ''explain'', true);'); + testCase.verifySubstring(out, 'Searching this session for entities that'); + out = evalc(['ndi.entity.search(S, {''type'', ''organism'', ''contained_in'', ' ... + '{''type'', ''material''}}, ''explain'', true);']); + testCase.verifySubstring(out, 'are contained_in a material'); + + testCase.verifyError(@() ndi.entity.search(S, {'directed_relation', {'name', 'containd_in'}}), ... + 'ndi:entity:search:unknownRelation'); + testCase.verifyError(@() ndi.entity.search(S, {'contained_in', plate, 'directed_relation', ... + {'name', 'contained_in', 'parent', plate, 'depth', 2}}), 'ndi:entity:search:notYet'); + testCase.verifyError(@() ndi.entity.search(S, {'undirected_relation', {'parent', plate}}), ... + 'ndi:entity:search:relationFilter'); + testCase.verifyError(@() ndi.entity.search(S, {'directed_relation', {'parent', plate, 'child', plate}}), ... + 'ndi:entity:search:relationFilter'); + end + + function testAnAssertionAboutAWholeHoldsOfItsParts(testCase) + % plate 0013 is excluded (an assertion on the plate): its patches, + % part_of it, are too; its worms, contained_in it, are not + S = testCase.Session; + direct = local(ndi.entity.search(S, {'inclusion in analysis', 'excluded'}, 'inherited', false)); + testCase.verifyTrue(ismember('concentration_assayPlate0013', direct)); + all_ = local(ndi.entity.search(S, {'inclusion in analysis', 'excluded'})); + extra = setdiff(all_, direct); + testCase.verifyTrue(all(startsWith(extra, 'concentration_assayPlate0013_patch')), strjoin(extra, ', ')); + patches = local(ndi.entity.search(S, {'directed_relation', {'name', 'part_of', 'parent', ... + {'local_identifier', 'concentration_assayPlate0013'}}})); + testCase.verifyEqual(sort(extra), sort(patches), 'exactly its parts'); + testCase.verifyEmpty(ndi.entity.search(S, {'type', 'organism', 'inclusion in analysis', 'excluded'}), ... + 'contained_in does not carry statements'); + end + + function testContextIsWhatTheSubjectWasIn(testCase) + % V_eta tenet T17: a container's interactions reach what it + % contained only with 'context', true, only while it was in it, + % and marked as reached through contained_in. Assertions never. + if ~ndi.setup.V2.schemaHasField('statement', 'distributive') + return; % the stays are stated on the cohort, distributive + end + S = testCase.Session; + w = testCase.subject('concentration_worm0121'); + c = testCase.subject('concentration_assayPlate0012_worms'); + plates = cellfun(@(x) x.document_id, c.parents('contained_in'), 'UniformOutput', false); + testCase.assertNotEmpty(plates, 'the cohort was on plates'); + isCtx = @(x) endsWith(x.via(), 'contained_in'); + + % off by default: nothing about the worm comes from its plates + testCase.verifyFalse(any(cellfun(isCtx, w.statements()))); + testCase.verifyEmpty(w.manipulations({'variable', 'ambient temperature'})); + + m = w.manipulations('context', true); + ctx = m(cellfun(isCtx, m)); + testCase.verifyNotEmpty(ctx, 'its plates'' manipulations while it was on them'); + testCase.verifyTrue(all(cellfun(@(x) ismember(x.subject().document_id, plates), ctx)), ... + 'only the plates it was on; stated_on is the plate'); + testCase.verifyTrue(all(cellfun(@(x) strcmp(x.via(), 'member_of > contained_in'), ctx)), ... + 'through its cohort''s distributive stay'); + vars = cellfun(@(x) x.variable_name(), ctx, 'UniformOutput', false); + methods_ = cellfun(@(x) char(ndi.v2.termName(ndi.v2.blockOf(x.document_properties(), ... + 'interaction', 'method', ''))), ctx, 'UniformOutput', false); + testCase.verifyTrue(any(strcmp(vars, 'ambient temperature')), ... + 'the bench and the incubator, while it was there'); + testCase.verifyFalse(any(strcmp(vars, 'NGM agar')), ... + 'the pour is only known to be before the seeding: it cannot be placed in the stay'); + testCase.verifyFalse(any(strcmp(methods_, 'refrigeration')), ... + 'the cold room was before the worms arrived'); + own = w.manipulations(); + testCase.verifyEqual(numel(m) - numel(ctx), numel(own), 'context adds; it removes nothing'); + + % the videos of the plate it was filmed on + o = w.observations('context', true); + testCase.verifyNotEmpty(o(cellfun(isCtx, o)), 'the assay plate''s recordings'); + + % assertions never pass: plate 0013 is excluded, its worms are not + testCase.verifyEqual(numel(w.assertions('context', true)), numel(w.assertions())); + testCase.verifyEmpty(ndi.entity.search(S, {'type', 'organism', ... + 'inclusion in analysis', 'excluded'}, 'context', true)); + + % search: a plate's room-temperature period finds the worms on it then + found = local(ndi.entity.search(S, {'type', 'organism', ... + 'manipulation', {'method', 'ambient exposure'}}, 'context', true)); + testCase.verifyTrue(ismember('concentration_worm0121', found)); + testCase.verifyEmpty(ndi.entity.search(S, {'type', 'organism', ... + 'manipulation', {'method', 'ambient exposure'}}), 'not without context'); + testCase.verifyEmpty(ndi.entity.search(S, {'type', 'organism', ... + 'manipulation', {'method', 'refrigeration'}}, 'context', true), ... + 'every cold room was before the worms arrived'); + out = evalc(['ndi.entity.search(S, {''type'', ''organism'', ''manipulation'', ' ... + '{''method'', ''ambient exposure''}}, ''context'', true, ''explain'', true);']); + testCase.verifySubstring(out, 'contained in something that did while they were in it'); + end + + function testDuringAnotherStatement(testCase) + % 'during' takes a description of statements as well as times: + % kept when it overlaps one matching it that holds of the same + % subject, by the same rules ('inherited', 'context'). The + % fixture's plates read 21.5 + plate/100 C over their videos, so + % '>21.615' is plates 12 and 14 (21.62, 21.64), not plate 11 (21.61). + S = testCase.Session; + plate = testCase.subject('concentration_assayPlate0012'); + hot = {'observation', {'variable', 'ambient temperature', 'value', '>21.615'}}; + cold = {'observation', {'variable', 'ambient temperature', 'value', '>30'}}; + v = plate.observations({'variable', 'image intensity'}); + testCase.assertNotEmpty(v, 'plate 12 was filmed'); + during = plate.observations({'variable', 'image intensity', 'during', hot}); + testCase.verifyNotEmpty(during, 'plate 12''s reading is timed by one of its videos'); + testCase.verifyLessThanOrEqual(numel(during), numel(v)); + testCase.verifyEmpty(plate.observations({'variable', 'image intensity', 'during', cold})); + other = testCase.subject('concentration_assayPlate0011'); + testCase.verifyEmpty(other.observations({'variable', 'image intensity', 'during', hot}), ... + 'plate 11 read 21.61 C; plate 12''s reading is not plate 11''s'); + + if ndi.setup.V2.schemaHasField('statement', 'distributive') + % a worm: its plate's videos, while that plate was over 21.615 C + w = testCase.subject('concentration_worm0121'); + testCase.verifyEmpty(w.observations({'variable', 'image intensity', 'during', hot}), ... + 'without context the worm has neither the videos nor the reading'); + o = w.observations({'variable', 'image intensity', 'during', hot}, 'context', true); + testCase.verifyNotEmpty(o); + testCase.verifyTrue(all(cellfun(@(x) strcmp(x.subject().local_identifier, ... + 'concentration_assayPlate0012'), o))); + t = w.observations({'variable', 'ambient temperature'}, 'context', true); + testCase.verifyNumElements(t, 1, 'the one reading of the plate it was filmed on'); + testCase.verifyEqual(double(t{1}.value()), 21.62, 'AbsTol', 1e-9); + + % search: worms filmed while their plate was over 21.615 C + found = local(ndi.entity.search(S, {'type', 'organism', ... + 'observation', {'variable', 'image intensity', 'during', hot}}, 'context', true)); + testCase.verifyTrue(ismember('concentration_worm0121', found)); + warm = local(ndi.entity.search(S, {'type', 'organism', 'contained_in', ... + {'local_identifier', {'concentration_assayPlate0012', 'concentration_assayPlate0014'}}})); + testCase.verifyEmpty(setdiff(found, warm), 'only the worms on plates 12 and 14'); + cool = local(ndi.entity.search(S, {'type', 'organism', 'contained_in', other})); + testCase.verifyEmpty(intersect(found, cool), 'none of the worms on plate 11'); + testCase.verifyEmpty(ndi.entity.search(S, {'type', 'organism', ... + 'observation', {'variable', 'image intensity', 'during', cold}}, 'context', true)); + out = evalc(['ndi.entity.search(S, {''type'', ''organism'', ' ... + '''observation'', {''variable'', ''image intensity'', ''during'', hot}}, ''context'', true, ''explain'', true);']); + testCase.verifySubstring(out, 'at a time when they also had an observation with variable ambient temperature'); + end + testCase.verifyError(@() plate.observations({'during', {'assertion', {'variable', 'strain'}}}), ... + 'ndi:entity:search:assertionTime'); + end + + function testTimeFilters(testCase) + % the fixture's windows (TestHaleyWrite pins them), Los Angeles: + % cohort 0012 on acclimation plate 0001 from 2022-02-03 12:10:51 to + % 2022-02-04 12:10:51; cohort 0014 food-deprived for exactly 3 h, + % 2022-02-04 12:17:10 to 15:17:10 (the end, T, up to 5 min early) + S = testCase.Session; + w = warning('off', 'ndi:entity:search:noSuchTime'); + restore = onCleanup(@() warning(w)); + acc = testCase.subject('concentration_0001_acclimationPlate0001'); + on = @(varargin) local(ndi.entity.search(S, {'type', 'organism', 'directed_relation', ... + [{'name', 'contained_in', 'parent', acc}, varargin]})); + worms = {'concentration_worm0121', 'concentration_worm0122'}; + testCase.verifyTrue(all(ismember(worms, on('at', '2022-02-03T18:00'))), 'local time, the lab''s zone'); + testCase.verifyTrue(all(ismember(worms, on('at', '2022-02-04T02:00Z'))), 'the same instant in UTC'); + testCase.verifyTrue(all(ismember(worms, on('at', datetime(2022, 2, 3, 18, 0, 0)))), 'a datetime'); + testCase.verifyFalse(any(ismember(worms, on('at', '2022-02-02T18:00'))), 'before the pick'); + testCase.verifyTrue(all(ismember(worms, on('during', '2022-02-03'))), 'a whole day'); + testCase.verifyTrue(all(ismember(worms, on('during', {'2022-02-01', '2022-02-03T13:00'})))); + testCase.verifyFalse(any(ismember(worms, on('during', '2022-02-01')))); + testCase.verifyTrue(all(ismember(worms, on('before', '2022-02-05')))); + testCase.verifyTrue(all(ismember(worms, on('after', '2022-02-03')))); + testCase.verifyFalse(any(ismember(worms, on('after', '2022-02-04')))); + + % duration, and tolerance + fd = local(ndi.entity.search(S, {'member_of', testCase.subject('concentration_assayPlate0014_worms')})); + testCase.assertNotEmpty(fd); + deprived = @(varargin) local(ndi.entity.search(S, {'type', 'organism', 'manipulation', ... + [{'variable', 'food availability'}, varargin{1}]}, varargin{2:end})); + testCase.verifyEqual(sort(deprived({'duration', '>=3h'})), sort(fd)); + testCase.verifyEqual(sort(deprived({'duration', '>=180min'})), sort(fd)); + testCase.verifyEmpty(deprived({'duration', '>3h'}), 'exactly 3 h as stated'); + testCase.verifyEqual(sort(deprived({'duration', '>3h'}, 'tolerant', true)), sort(fd), ... + 'the hand-written start could be a minute early'); + warning('on', 'ndi:entity:search:noSuchTime'); + testCase.verifyWarning(@() ndi.entity.search(S, {'manipulation', ... + {'variable', 'food availability', 'duration', '>4h'}}), 'ndi:entity:search:noSuchTime'); + warning('off', 'ndi:entity:search:noSuchTime'); + + % a time known only as "before the seeding" (a plate's pour) is a bound + plate = 'concentration_assayPlate0012'; + poured = @(varargin) local(ndi.entity.search(S, {'manipulation', [{'variable', 'NGM agar'}, varargin]})); + testCase.verifyTrue(ismember(plate, poured('before', '2030-01-01')), 'the bound decides it'); + testCase.verifyFalse(ismember(plate, poured('at', '2022-02-04T12:00')), 'a bound cannot place it'); + + out = evalc(['ndi.entity.search(S, {''type'', ''organism'', ''manipulation'', ' ... + '{''variable'', ''food availability'', ''duration'', ''>=3h''}}, ''explain'', true);']); + testCase.verifySubstring(out, 'lasting >=3h'); + testCase.verifyError(@() ndi.entity.search(S, {'directed_relation', ... + {'name', 'contained_in', 'at', 'sometime after lunch'}}), 'ndi:v2:timeFilter:badTime'); + testCase.verifyError(@() ndi.entity.search(S, {'manipulation', ... + {'variable', 'food availability', 'duration', '3 fortnights'}}), 'ndi:v2:timeFilter:badDuration'); + end + + function testAssertions(testCase) + c = testCase.subject('concentration_assayPlate0012_worms'); + a = c.assertions(); + testCase.verifyTrue(all(cellfun(@(x) isa(x, 'ndi.assertion'), a)), 'objects, not a table'); + T = ndi.summary(a); + testCase.verifyEqual(lower(T.value_node(T.variable == "species")), "ncbitaxon:6239"); + if ndi.setup.V2.mergedEntities() + % one entity class: the strain is an instance_of relation + testCase.verifyEmpty(c.assertions({'variable', 'strain'})); + x = testCase.subject('concentration_assayPlate0013').assertions({'variable', 'inclusion in analysis'}); + testCase.verifyNumElements(x, 1); + testCase.verifyEqual(x{1}.value().canonical(), "excluded"); + return; + end + testCase.verifyEqual(T.value(T.variable == "strain"), "N2"); + st = c.assertions({'variable', 'strain'}); + testCase.verifyNumElements(st, 1, 'a cell of filters comes first'); + testCase.verifyEqual(st{1}.variable_name(), 'strain'); + a = c.statements('assertion', {'variable', 'strain'}); + testCase.verifyNumElements(a, 1); + testCase.verifyClass(a{1}, 'ndi.assertion'); + testCase.verifyEqual(a{1}.composite(), 'term'); + testCase.verifyEqual(a{1}.value().canonical(), "N2"); + testCase.verifyEqual(a{1}.subject().document_id, c.document_id); + x = testCase.subject('concentration_assayPlate0013').assertions({'variable', 'inclusion in analysis'}); + testCase.verifyNumElements(x, 1); + testCase.verifyEqual(x{1}.value().canonical(), "excluded"); + end + + function testInheritedFollowsMemberOfAndDistributive(testCase) + % decision #55: species and strain are stated once, on the + % cohort, marked distributive -- they hold of each worm, and are + % read that way by default + w = testCase.subject('concentration_worm0121'); + own = w.assertions('inherited', false); + testCase.verifyFalse(any(cellfun(@(x) strcmp(x.variable_name(), 'species'), own)), ... + 'nothing is stated on the worm itself'); + testCase.verifyTrue(all(cellfun(@(x) strcmp(x.subject().document_id, w.document_id), own))); + T = w.assertions(); + st = w.statements(); + mine = w.statements('inherited', false); + testCase.verifyTrue(all(cellfun(@(x) strcmp(x.via(), 'own'), mine))); + testCase.verifyEqual(sum(cellfun(@(x) strcmp(x.via(), 'own'), st)), numel(mine), ... + 'its own statements are kept'); + if ndi.setup.V2.schemaHasField('statement', 'distributive') + sp = T(cellfun(@(x) strcmp(x.variable_name(), 'species'), T)); + testCase.verifyNumElements(sp, 1); + testCase.verifyEqual(lower(sp{1}.raw_value().node), 'ncbitaxon:6239'); + testCase.verifyEqual(sp{1}.subject().local_identifier, 'concentration_assayPlate0012_worms'); + testCase.verifyEqual(sp{1}.via(), 'member_of'); + strain = T(cellfun(@(x) strcmp(x.variable_name(), 'strain'), T)); + if ndi.setup.V2.mergedEntities() + testCase.verifyEmpty(strain, 'one entity class: a strain is a relation'); + else + testCase.verifyEqual(strain{1}.value().canonical(), "N2"); + end + extra = st(cellfun(@(x) ~strcmp(x.via(), 'own'), st)); + testCase.verifyNotEmpty(extra); + testCase.verifyTrue(all(cellfun(@(x) x.distributive(), extra)), 'only distributive ones'); + testCase.verifyTrue(all(cellfun(@(x) strcmp(x.subject().local_identifier, ... + 'concentration_assayPlate0012_worms'), extra)), 'each says where it was stated'); + testCase.verifyTrue(all(cellfun(@(x) strcmp(x.about(), w.document_id), st)), ... + 'each says which subject it was asked for'); + else + testCase.verifyEqual(numel(st), numel(mine), 'no distributive flag in this schema: nothing inherits'); + end + % contained_in is not followed: the plate's temperature is not the worm's + testCase.verifyEmpty(w.statements('temperature_manipulation')); + testCase.verifyEmpty(w.statements('manipulation', {'variable', 'ambient temperature'})); + end + + function testSubjectsAreTheBiologicalTypes(testCase) + % `isa subject` is the organisms, cultures, tissues, cells and + % groups (ndi.v2.subjectTypes), as a v1 subject always was; a + % plate (material) or an instrument (device) is an entity, not a + % subject. 'subject' is not a kind ndi.entity.search takes. + S = testCase.Session; + bio = ndi.v2.subjectTypes(); + all_ = ndi.entity.search(S, {'type', '*'}); + types = cellfun(@(x) x.type, all_, 'UniformOutput', false); + testCase.assertTrue(any(strcmp(types, 'material')), 'the fixture has plates'); + s = ndi.entity.search(S, {'type', bio}); + testCase.verifyEqual(numel(s), sum(ismember(types, bio))); + docs = S.database_search(ndi.query('', 'isa', 'subject')); + testCase.verifyEqual(sort(cellfun(@(d) char(d.document_properties.base.id), docs, ... + 'UniformOutput', false)), sort(cellfun(@(x) x.document_id, s, 'UniformOutput', false)), ... + 'isa subject finds the biological types'); + plate = ndi.entity.search(S, {'LocalIdentifier', 'concentration_assayPlate0012'}); + testCase.verifyNumElements(plate, 1, 'a plate is an entity'); + testCase.verifyFalse(ismember(plate{1}.document_id, ... + cellfun(@(d) char(d.document_properties.base.id), docs, 'UniformOutput', false)), ... + 'and not a subject'); + testCase.verifyError(@() ndi.entity.search(S, {'kind', 'subject'}), 'ndi:entity:search:subjectKind'); + end + + function testTypesPassTheirAssertionsToInstances(testCase) + % one entity class (2026-10-08): a cohort is an instance_of its + % strain, distributive, so each worm is one too; what is + % asserted of the strain holds of each worm (T17) and the + % summary names the strain under its type + if ~ndi.setup.V2.mergedEntities() + return; % a strain is an assertion there (testAssertions) + end + [refs, ids] = testCase.Dataset.session_list(); + S = testCase.Dataset.open_session(ids{strcmp(refs, testCase.Session.reference)}); + plate = ndi.entity.search(S, {'LocalIdentifier', 'concentration_assayPlate0012'}); + testCase.assertNumElements(plate, 1); + ws = ndi.entity.search(S, {'type', 'organism', 'contained_in', plate{1}}); + W = [ws{:}]; + testCase.assertNumElements(W, 2); + T = ndi.summary(ws); + testCase.assertTrue(ismember('strain', T.Properties.VariableNames), ... + strjoin(T.Properties.VariableNames, ', ')); + testCase.verifyEqual(T.strain, ["N2"; "N2"], 'the cohort''s strain, through instance_of'); + a = W(1).assertions(); + via = cellfun(@(x) x.via(), a, 'UniformOutput', false); + fromType = a(contains(via, 'instance_of')); + testCase.verifyNotEmpty(fromType, ['the strain''s assertions hold of the worm: ' strjoin(via, ', ')]); + testCase.verifyTrue(all(contains(via(contains(via, 'instance_of')), 'member_of > instance_of')), ... + 'reached through the cohort'); + i = W(1).interactions(); + testCase.verifyFalse(any(cellfun(@(x) contains(x.via(), 'instance_of'), i)), ... + 'a type''s interactions do not pass to its instances'); + % a session folder opened on its own holds the relation, not the strain + T0 = ndi.summary(ndi.entity.search(testCase.Session, {'type', 'organism', 'contained_in', ... + testCase.subject('concentration_assayPlate0012')})); + testCase.verifyFalse(ismember('strain', T0.Properties.VariableNames)); + end + + function testStatementsOfManySubjectsAndSummaries(testCase) + S = testCase.Session; + ws = ndi.entity.search(S, {'type', 'organism', 'contained_in', ... + testCase.subject('concentration_assayPlate0012')}); + W = [ws{:}]; + testCase.assertNumElements(W, 2); + st = statements(W, 'assertion'); + about = cellfun(@(x) x.about(), st, 'UniformOutput', false); + testCase.verifyEqual(sort(unique(about)), sort({W.document_id}), 'one set per worm'); + one = W(1).statements('assertion'); + testCase.verifyEqual(sum(strcmp(about, W(1).document_id)), numel(one), ... + 'the same as asking one worm at a time'); + A = assertions(W); + testCase.verifyEqual(numel(A), numel(st), 'assertions is statements of one kind'); + % the kind shortcuts are statements of that kind, filters included + kinds = {'observation', 'manipulation', 'calculation', 'interaction'}; + short = {@observations, @manipulations, @calculations, @interactions}; + for k = 1:numel(kinds) + testCase.verifyEqual(cellfun(@(x) x.document_id(), short{k}(W), 'UniformOutput', false), ... + cellfun(@(x) x.document_id(), statements(W, kinds{k}), 'UniformOutput', false), kinds{k}); + end + if ndi.setup.V2.schemaHasField('statement', 'distributive') + m = manipulations(W(1), {'variable', 'location'}); + testCase.verifyNotEmpty(m, 'the moves onto plates, stated on the cohort'); + testCase.verifyTrue(all(cellfun(@(x) strcmp(x.variable_name(), 'location'), m))); + end + + T = ndi.summary(ws); + testCase.verifyEqual(height(T), 2); + testCase.verifyTrue(all(ismember({'name', 'kind', 'id', 'type', 'local_identifier'}, ... + T.Properties.VariableNames))); + if ismember('strain', T.Properties.VariableNames) + testCase.verifyEqual(T.strain, ["N2"; "N2"], 'the cohort''s strain, inherited'); + end + testCase.verifyEqual(W(1).summary().id, string(W(1).document_id), 'one object: one row'); + N = ndi.summary(ws, 'nodes', true); + if ismember('species', N.Properties.VariableNames) + testCase.verifyEqual(lower(N.species_node), ["ncbitaxon:6239"; "ncbitaxon:6239"]); + testCase.verifyFalse(ismember('species_node', T.Properties.VariableNames), 'nodes only when asked'); + end + + U = ndi.summary(statements(W)); + testCase.verifyEqual(height(U), numel(statements(W))); + testCase.verifyTrue(all(ismember({'subject', 'kind', 'class', 'variable', 'variable_node', ... + 'method', 'method_node', 'value', 'value_node', 'unit', 'start', 'end', 'timezone', ... + 'stated_on', 'via', 'id'}, U.Properties.VariableNames))); + sp = U(U.variable == "species", :); + if height(sp) > 0 + testCase.verifyEqual(unique(lower(sp.value_node)), "ncbitaxon:6239", 'a term value carries its node'); + end + timed = U(~isnat(U.start), :); + testCase.verifyNotEmpty(timed, 'the transfers have times'); + testCase.verifyEqual(unique(timed.timezone), "America/Los_Angeles"); + testCase.verifyEqual(U.start.TimeZone, 'America/Los_Angeles', 'shown in the zone recorded'); + testCase.verifyClass(U.start, 'datetime'); + testCase.verifyError(@() ndi.summary(statements(W), 'nodes', true), 'ndi:summary:options'); + r = U(U.variable == "strain", :); + if height(r) > 0 + testCase.verifyEqual(unique(r.via), "member_of"); + testCase.verifyEqual(unique(r.stated_on), "Worms on Assay Plate 0012"); + end + m = U(U.kind == "manipulation" & U.via == "member_of", :); + testCase.verifyTrue(all(~isnat(m.start) | m.value ~= ""), 'transfers have times or values'); + testCase.verifyError(@() ndi.summary([ws, statements(W)]), 'ndi:summary:mixed'); + testCase.verifyEqual(height(ndi.summary({})), 0); + end + + function testStrictAndTheMessages(testCase) + S = testCase.Session; + testCase.verifyError(@() ndi.entity.search(S, {'stran', 'N2'}), 'ndi:entity:search:unknownVariable'); + try + ndi.entity.search(S, {'stran', 'N2'}); + catch err + testCase.verifySubstring(err.message, 'No entity in this session has a property ''stran''.'); + if ~ndi.setup.V2.mergedEntities() + % one entity class: the strain entity is not in a session + % (testFindByWhatIsTrueOfASubject), so it is not suggested + testCase.verifySubstring(err.message, 'Did you mean ''strain''?'); + end + testCase.verifySubstring(err.message, 'Properties in this session: '); + testCase.verifySubstring(err.message, 'local_identifier'); + end + testCase.verifyWarning(@() ndi.entity.search(S, {'species', 'Caenorhabdiits elegans'}), ... + 'ndi:entity:search:noSuchValue'); + % the message, whole: as an error (evalc wraps it at the console width) + b = warning('error', 'ndi:entity:search:noSuchValue'); %#ok + restore = onCleanup(@() warning(b)); + msg = ''; + try + ndi.entity.search(S, {'species', 'Caenorhabdiits elegans'}); + catch err + msg = err.message; + end + clear restore + testCase.verifySubstring(msg, 'Did you mean ''Caenorhabditis elegans''?'); + testCase.verifySubstring(msg, 'Species in this session: '); + % strict false: nothing, quietly + testCase.verifyEmpty(testCase.verifyWarningFree(@() ... + ndi.entity.search(S, {'stran', 'N2'}, 'strict', false))); + testCase.verifyEmpty(testCase.verifyWarningFree(@() ... + ndi.entity.search(S, {'species', 'Caenorhabdiits elegans'}, 'strict', false))); + testCase.verifyEmpty(testCase.verifyWarningFree(@() ... + ndi.entity.search(S, {'type', 'organism', 'contained_in', {'stran', 'N2'}}, 'strict', false)), ... + 'a nested description is strict as the outer search is'); + testCase.verifyEqual(ndi.v2.closest('stran', {'strain', 'species'}), 'strain'); + testCase.verifyEqual(ndi.v2.closest('zzz', {'strain', 'species'}), ''); + end + + function testCalculationValueFromItsBody(testCase) + w = testCase.subject('concentration_worm0121'); + sp = w.statements('velocity_calculation'); + testCase.verifyNumElements(sp, 1); + testCase.verifyClass(sp{1}, 'ndi.calculation'); + testCase.verifyTrue(isa(sp{1}, 'ndi.interaction')); + v = sp{1}.value(); + testCase.verifyClass(v, 'ndi.value'); + testCase.verifyEqual(v.class_name, 'velocity'); + testCase.verifyEqual(double(v), 121e-6 * ones(5, 1), 'AbsTol', 1e-15, ... + 'm/s by video frame, decoded from the sampled body'); + ax = sp{1}.axes(); + testCase.verifyEqual(ax.variable(1), "video frame"); + testCase.verifyEqual(ax.n(1), 5); + + in = sp{1}.inputs(); + testCase.verifyNumElements(in, 1); + testCase.verifyClass(in{1}, 'ndi.calculation'); + testCase.verifyEqual(in{1}.variable_name(), 'midpoint position'); + xy = double(in{1}.value()); + testCase.verifyEqual(reshape(xy, 5, 2), [(1:5)' + 12.1 - 0.5, 3.5 * ones(5, 1)], 'AbsTol', 1e-12); + P = in{1}.method_parameters(); + testCase.verifyEqual(sort(P.variable)', ["gap filling", "longest gap filled"]); + testCase.verifyFalse(any(cellfun(@isstruct, P.value)), ... + 'a numeric parameter is its number, not its stored record'); + P = sp{1}.method_parameters(); + num = P(cellfun(@isnumeric, P.value), :); + testCase.verifyNotEmpty(num, 'the speed''s window and fastest step are numbers'); + testCase.verifyTrue(all(cellfun(@isscalar, num.value))); + + de = w.statements('length_calculation'); + testCase.verifyNumElements(de, 1); + testCase.verifyNumElements(de{1}.inputs(), 2, 'the positions and the lawn mask'); + testCase.verifyEqual(double(de{1}.value()), 2 * 1e-3 / 33 * ones(5, 1), 'AbsTol', 1e-15); + end + + function testManipulations(testCase) + plate = testCase.subject('concentration_assayPlate0012'); + pour = plate.statements('manipulation', {'variable', 'NGM agar'}); + testCase.verifyNumElements(pour, 1); + testCase.verifyClass(pour{1}, 'ndi.manipulation'); + v = pour{1}.value(); + testCase.verifyEqual(v.class_name, 'dose'); + testCase.verifyEqual(v.raw.volume.liters, 0.025, 'AbsTol', 1e-12); + t = pour{1}.time(); + testCase.verifyNumElements(t, 1); + testCase.verifyEqual(t.kind, 'relative', 'poured relative to the seeding'); + testCase.verifyNotEmpty(t.referent_id); + % the formulation is reached when its document is in this session + fid = ndi.v2.edgeIds(pour{1}.document_properties(), 'formulation_id'); + testCase.verifyEqual(fid, {testCase.Result.metadata.ids('ngm')}); + f = pour{1}.formulation(); + if isempty(ndi.v2.getDocument(testCase.Session, fid{1})) + testCase.verifyEmpty(f, 'a dataset-level formulation is not found from a session'); + else + testCase.verifyEqual(f.class_name, 'formulation'); + end + + tm = plate.statements('temperature_manipulation'); + testCase.verifyNumElements(tm, 2); + m = sort(cellfun(@(s) s.method_name(), tm, 'UniformOutput', false)); + testCase.verifyEqual(m, {'ambient exposure', 'refrigeration'}); + for i = 1:2 + v = tm{i}.value(); + testCase.verifyNotEmpty(v.unit()); + testCase.verifyEqual(double(v), double(v.raw.(v.unit()))); + end + + moves = ndi.statement.search(testCase.Session, 'Variable', 'location', 'Method', 'agar plug transfer'); + testCase.verifyNotEmpty(moves); + testCase.verifyTrue(all(cellfun(@(s) isa(s, 'ndi.manipulation'), moves))); + P = moves{1}.method_parameters(); + testCase.verifyTrue(any(P.variable == "cleaning step")); + end + + function testANestedEdgeTargetReachesDid2(testCase) + % a depends_on whose target is a query (and one inside an or) + % is converted for did2 as a query, not dropped or stringified. + % Searching with it needs DID-matlab #218; the search itself is + % tested there (testSqliteDb/testSearchDependsOnAQueryOrAList). + D = 'ndi.database.implementations.database.did2sqlite'; + inner = ndi.query('chemical.value.substance.name', 'exact_string', 'peptone', ''); + q = ndi.query('', 'isa', 'formulation', '') & ndi.query('', 'depends_on', 'ingredient_id', inner); + q2 = feval([D '.toDid2Query'], q); + ss = q2.searchstructure; + k = find(strcmp({ss.operation}, 'depends_on')); + testCase.assertNumElements(k, 1); + testCase.verifyClass(ss(k).param2, 'did2.query'); + testCase.verifyEqual(ss(k).param2.searchstructure.field, 'chemical.value.substance.name'); + testCase.verifyEqual(ss(k).param2.searchstructure.param1, 'peptone'); + + o = feval([D '.toDid2Query'], ndi.query('', 'depends_on', 'ingredient_id', inner) | ... + ndi.query('base.name', 'exact_string', 'x', '')); + testCase.verifyEqual(o.searchstructure.operation, 'or'); + testCase.verifyClass(o.searchstructure.param1(1).param2, 'did2.query', 'inside an or'); + end + + function testEntitiesOfAnyKindAreSearchedAndDescribed(testCase) + % statements may be about any entity (2026-10-08): ndi.entity has + % search and statements + D = testCase.Dataset; + p = ndi.entity.search(D, {'kind', 'person', 'name', '*Haley'}); + testCase.verifyNumElements(p, 1, 'search by kind and name'); + testCase.verifyEqual(p{1}.kind, 'person'); + testCase.verifyClass(p{1}, 'ndi.entity'); + old = ndi.entity.search(D, {'kind', 'person', 'Name', p{1}.name}); + testCase.verifyEqual(cellfun(@(x) x.document_id, old, 'UniformOutput', false), ... + {p{1}.document_id}, 'the older (KIND, ''Name'', NAME) form still works'); + testCase.verifyEmpty(ndi.entity.search(D, {'kind', 'person', 'type', 'organism'}), ... + 'a person is not an organism'); + w = ndi.entity.search(D, {'local_identifier', 'concentration_worm0121'}); + testCase.verifyEqual(cellfun(@(x) x.document_id, ndi.entity.search(D, ... + {'local_identifier', 'concentration_worm0121'}), 'UniformOutput', false), ... + cellfun(@(x) x.document_id, w, 'UniformOutput', false), 'a worm is a subject'); + testCase.verifyError(@() ndi.entity.search(D, {'kind', 'subject', 'local_identifier', 'x'}), ... + 'ndi:entity:search:subjectKind'); + if ndi.setup.V2.mergedEntities() + % a person's names are `text` assertions about it: read, and searched + g = p{1}.assertions({'variable', 'given name'}); + testCase.verifyNumElements(g, 1); + testCase.verifyEqual(g{1}.value().canonical(), "Jessica A"); + testCase.verifyEqual(g{1}.subject().document_id, p{1}.document_id, ... + 'a statement''s subject may be any entity'); + testCase.verifyEqual(cellfun(@(x) x.document_id, ... + ndi.entity.search(D, {'kind', 'person', 'given name', 'jess*'}), 'UniformOutput', false), ... + {p{1}.document_id}, 'a text value matches like a name: any case, wildcards'); + T = ndi.summary(p); + testCase.verifyEqual(T.familyName, "Haley", 'a person''s assertions are summary columns'); + end + end + + function testDatasetLevelDocumentsAreReachedThroughTheDataset(testCase) + % software, formulations, strains and people are stored with the + % dataset (decision #20): read through the dataset, a statement + % reaches them; read through a session, it does not + local = 'concentration_worm0121'; + ws = ndi.entity.search(testCase.Session, {'LocalIdentifier', local}); + w = ndi.entity.fromDocument(testCase.Dataset, ws{1}.document_id()); + sp = w.statements('velocity_calculation'); + testCase.verifyNumElements(sp, 1); + testCase.verifyEqual(double(sp{1}.value()), 121e-6 * ones(5, 1), 'AbsTol', 1e-15); + testCase.assertNotEmpty(ndi.v2.edgeIds(sp{1}.document_properties(), 'software_id'), ... + 'with stage 2 run, a calculation names its software (import_V2, stage 10)'); + sw = sp{1}.software(); + testCase.verifyNumElements(sw, 1, 'the analysis package, from the dataset'); + testCase.verifyEqual(sw{1}.kind(), 'software'); + spS = ws{1}.statements('velocity_calculation'); + testCase.verifyEmpty(spS{1}.software(), 'not reached through the session'); + + plate = ndi.entity.fromDocument(testCase.Dataset, ... + testCase.subject('concentration_assayPlate0012').document_id()); + pour = plate.statements('manipulation', {'variable', 'NGM agar'}); + f = pour{1}.formulation(); + testCase.verifyNotEmpty(f, 'the formulation is found through the dataset'); + testCase.verifyEqual(f.class_name, 'formulation'); + end + + function testReadingAV2DocumentGivesTheFullPassResult(testCase) + % applyReadNormalization takes a shortcut for documents already + % at the read target (when DID-matlab has it). Every document of + % the written session must come out exactly as the full + % v1_to_v2 pass makes it. + db = did2.database.sqlitedb(fullfile(testCase.Result.sessions.path{1}, '.ndi', ... + ndi.database.implementations.database.did2sqlite.DEFAULTFILENAME())); + closer = onCleanup(@() db.close()); + ids = db.allIds(); + testCase.assertGreaterThan(numel(ids), 100, 'the written session has documents'); + nDiffer = 0; first = ''; + for i = 1:numel(ids) + d = db.get(ids{i}); + got = ndi.database.internal.applyReadNormalization(d); + full = did2.convert.v1_to_v2(d.toStruct(), 'Validate', false, ... + 'RenameClassNames', false, 'TargetVersion', 'V_delta'); + want = ndi.document(full.migrated{1}.toStruct()); + if ~isequaln(got.document_properties, want.document_properties) + nDiffer = nDiffer + 1; + if isempty(first), first = ids{i}; end + end + end + testCase.verifyEqual(nDiffer, 0, sprintf(['%d of %d document(s) read differently ' ... + 'from the full pass; first: %s'], nDiffer, numel(ids), first)); + end + + function testInteractionIsAbstract(testCase) + testCase.verifyTrue(meta.class.fromName('ndi.interaction').Abstract); + end + + function testV1SubjectUnchanged(testCase) + % ndi.subject is v1's, as on V2: the @ is required, and it is not + % an ndi.entity + testCase.verifyError(@() ndi.subject('nolab', 'a subject'), ?MException); + s = ndi.subject('anteater23@nosuchlab.org', 'a subject'); + testCase.verifyEqual(s.local_identifier, 'anteater23@nosuchlab.org'); + testCase.verifyEqual(s.description, 'a subject'); + testCase.verifyFalse(isa(s, 'ndi.entity')); + d = s.newdocument(); + testCase.verifyEqual(d.document_properties.subject.local_identifier, 'anteater23@nosuchlab.org'); + end + end + + methods + function s = subject(testCase, local) + x = ndi.entity.search(testCase.Session, {'LocalIdentifier', local}); + testCase.assertNumElements(x, 1, local); + s = x{1}; + end + end +end + +function n = local(c) +n = cellfun(@(x) x.local_identifier, c, 'UniformOutput', false); +n = reshape(n, 1, []); +end + +function setName(w) +w.name = 'x'; +end diff --git a/tests/+ndi/+unittest/+test/+lightsheet/TestBlobFixture.m b/tests/+ndi/+unittest/+test/+lightsheet/TestBlobFixture.m new file mode 100644 index 0000000000..4f146bc2a8 --- /dev/null +++ b/tests/+ndi/+unittest/+test/+lightsheet/TestBlobFixture.m @@ -0,0 +1,581 @@ +classdef TestBlobFixture < matlab.unittest.TestCase + % TestBlobFixture - end-to-end fixture + ingest smoke. + % + % Small shapes are used here (32x32x32 by default rather than the + % public 300x300x300 default) so the test fixture stays fast. + % Everything still exercises the real path -- .zattrs and .zarray + % on disk, raw chunk bytes, mean and max ladder writing, ingest + % into a session, and the pyramid + level document round-trip. + + properties + parentDir + zarrPath + gt + end + + methods (TestMethodSetup) + function build(testCase) + testCase.parentDir = tempname; + mkdir(testCase.parentDir); + testCase.addTeardown(@() rmdir(testCase.parentDir, 's')); + % Existing tests pin NumChannels=1 so they keep exercising + % the 3-D 'z,y,x' layout the assertions were written for. + % A separate testMultiChannelFixture below covers the 2-D + % default (2 channels, axes 'c,z,y,x'). + [testCase.zarrPath, testCase.gt] = ... + ndi.test.lightsheet.makeBlobFixture(testCase.parentDir, ... + 'Shape', [32 32 32], ... + 'NumChannels', 1, ... + 'NumLevels', 3, ... + 'ChunkShape', [16 16 16]); + end + end + + methods (Test) + + function testFixtureDirectoryLooksLikeAnOMEZarr(testCase) + testCase.verifyTrue(isfolder(testCase.zarrPath)); + testCase.verifyTrue(isfile(fullfile(testCase.zarrPath, '.zattrs'))); + testCase.verifyTrue(isfile(fullfile(testCase.zarrPath, '0', '.zarray'))); + end + + function testZattrsCarriesTwoPyramidsWithSharedLevel0(testCase) + attrs = jsondecode(fileread( ... + fullfile(testCase.zarrPath, '.zattrs'))); + testCase.verifyTrue(isfield(attrs, 'multiscales')); + % multiscales may deserialise as a struct array or cell. + n = numel(attrs.multiscales); + testCase.verifyEqual(n, 2); + names = collectPyramidNames(attrs.multiscales); + testCase.verifyEqual(sort(names), {'max','mean'}); + % Both mean[0] and max[0] point at '0'. + firsts = collectFirstPaths(attrs.multiscales); + testCase.verifyEqual(firsts, {'0','0'}); + end + + function testEachLevelHasZArrayAndChunkFiles(testCase) + gtLevels = testCase.gt.levels; + for k = 1:numel(gtLevels) + levelDir = fullfile(testCase.zarrPath, gtLevels(k).path); + testCase.verifyTrue(isfolder(levelDir), ... + sprintf('level dir missing: %s', gtLevels(k).path)); + testCase.verifyTrue(isfile(fullfile(levelDir, '.zarray')), ... + sprintf('.zarray missing: %s', gtLevels(k).path)); + % At least one chunk file present (name starts with a + % digit, the Zarr v2 flat layout). + listing = dir(levelDir); + names = {listing.name}; + digits = '0123456789'; + nChunks = 0; + for m = 1:numel(names) + nm = names{m}; + if ~isempty(nm) && any(nm(1) == digits) + nChunks = nChunks + 1; + end + end + testCase.verifyGreaterThan(nChunks, 0, ... + sprintf('no chunk files in: %s', gtLevels(k).path)); + end + end + + function testChunkFileSizeMatchesRawBytes(testCase) + % Zarr v2 raw chunk: size = prod(chunks) * bytesPerElement, + % edge chunks padded with 0 up to the same size. + chunkShape = testCase.gt.chunkShape; + expected = prod(chunkShape) * 2; % uint16 + path = fullfile(testCase.zarrPath, '0', '0.0.0'); + testCase.verifyTrue(isfile(path)); + info = dir(path); + testCase.verifyEqual(info.bytes, expected); + end + + function testFromOMEZarrIngestsTheFixture(testCase) + % `exist(pkg.fun, 'file')` returns 0 for package functions + % even when they resolve, so guard on `which` instead. + if isempty(which('ndr.format.omezarr.listPyramids')) + testCase.assumeFail(['NDR reader not on path; skipping ' ... + 'end-to-end ingest.']); + end + sDir = fullfile(tempname, 'blobsession'); + mkdir(sDir); + testCase.addTeardown(@() rmdir(fileparts(sDir), 's')); + S = ndi.session.dir('blob', sDir); + sub = ndi.document('subject', 'base.session_id', S.id(), ... + 'subject.local_identifier', 'blob@vhlab'); + S.database_add(sub); + + [pdoc, lds, info] = ndi.fun.doc.lightsheet.fromOMEZarr( ... + S, testCase.zarrPath, 'subjectID', sub.id()); + testCase.verifyTrue(info.sharedLevel0); + % 3 levels * 2 reductions, deduped on level 0 -> 5 docs. + testCase.verifyEqual(numel(lds), 5); + % Parent describes the source volume, axes zyx. + p = pdoc.document_properties.lightsheetZarrPyramid; + testCase.verifyEqual(char(p.axes_order), 'zyx'); + testCase.verifyEqual(reshape(p.shape_level0, 1, []), [32 32 32]); + end + + function testIngestBlobFixtureIsOneCallEndToEnd(testCase) + % The one-line demo that ships the fixture and ingests it. + % Uses a tiny shape so it stays quick; the point is that + % session + subject + fromOMEZarr all happen in one call + % and INFO carries the fixture ground truth alongside the + % ingest info. + if isempty(which('ndr.format.omezarr.listPyramids')) + testCase.assumeFail(['NDR reader not on path; skipping ' ... + 'end-to-end ingest.']); + end + sDir = fullfile(tempname, 'blobingest'); + testCase.addTeardown(@() rmdir(fileparts(sDir), 's')); + zDir = fullfile(tempname, 'blobingest_zarr'); + testCase.addTeardown(@() rmdir(zDir, 's')); + [S, pdoc, info] = ndi.test.lightsheet.ingestBlobFixture( ... + 'sessionDir', sDir, ... + 'zarrDir', zDir, ... + 'Shape', [32 32 32], ... + 'NumChannels', 1, ... + 'NumLevels', 3, ... + 'ChunkShape', [16 16 16]); + testCase.verifyClass(S, 'ndi.session.dir'); + testCase.verifyClass(pdoc, 'ndi.document'); + testCase.verifyTrue(info.sharedLevel0); + testCase.verifyNotEmpty(info.subjectID); + testCase.verifyEqual(info.fixture.shape, [32 32 32]); + testCase.verifyEqual(info.fixture.chunkShape, [16 16 16]); + end + + function testStreamingReTileHitsSmallerBudget(testCase) + % Force the streaming per-chunk read path: build a fixture + % with chunk shape [16 16 16] and ingest with a + % tileBudgetBytes small enough that chooseTileShape picks a + % SMALLER target chunk (~8 KB -> ~16 uint16 voxels per axis + % in 3-D). Each target chunk overlaps <=1 source chunk, so + % this exercises the per-chunk readArray Region call rather + % than reading the whole level -- the fix for the memory + % ceiling on lab-scale volumes. + if isempty(which('ndr.format.omezarr.listPyramids')) + testCase.assumeFail(['NDR reader not on path; skipping ' ... + 'end-to-end ingest.']); + end + sDir = fullfile(tempname, 'streamsession'); + testCase.addTeardown(@() rmdir(fileparts(sDir), 's')); + zDir = fullfile(tempname, 'streamzarr'); + testCase.addTeardown(@() rmdir(zDir, 's')); + [S, pdoc, info] = ndi.test.lightsheet.ingestBlobFixture( ... + 'sessionDir', sDir, ... + 'zarrDir', zDir, ... + 'Shape', [32 32 32], ... + 'NumChannels', 1, ... + 'NumLevels', 2, ... + 'ChunkShape', [16 16 16], ... + 'materializeChunks', true); + testCase.verifyClass(pdoc, 'ndi.document'); + % Every level document should have chunks attached (files + % ingested via add_file BEFORE database_add), proving the + % per-chunk readArray path wrote all target chunks. + q = ndi.query('','isa','lightsheetZarrLevel',''); + levels = S.database_search(q); + testCase.verifyNotEmpty(levels); + for k = 1:numel(levels) + L = levels{k}; + p = L.document_properties.lightsheetZarrLevel; + if p.n_chunks_stored > 0 + fh = S.database_openbinarydoc(L, 'chunk.bin_1'); + testCase.verifyNotEmpty(fh, ... + sprintf('chunk.bin_1 missing on level %d', p.level)); + S.database_closebinarydoc(fh); + end + end + testCase.verifyNotEmpty(info.subjectID); + end + + function testProgressCallbackReceivesMonotoneFractionsEndingAtOne(testCase) + % progressFcn is called before each level starts and after + % each chunk write. The fractions should be monotone + % non-decreasing (never step backwards) and end at exactly + % 1.0 with the 'Done.' label -- so a bar in the UI is safe + % to leave at the last reported fraction. + if isempty(which('ndr.format.omezarr.listPyramids')) + testCase.assumeFail(['NDR reader not on path; skipping ' ... + 'end-to-end ingest.']); + end + sDir = fullfile(tempname, 'progsession'); + testCase.addTeardown(@() rmdir(fileparts(sDir), 's')); + zDir = fullfile(tempname, 'progzarr'); + testCase.addTeardown(@() rmdir(zDir, 's')); + + log = struct('frac', {}, 'text', {}); + function record(f, t) + log(end+1).frac = f; %#ok + log(end).text = t; + end + captureFcn = @(f, t) record(f, t); + + % Small volume with 2 pyramid levels and materialized + % chunks so several chunks fire the callback. + ndi.test.lightsheet.ingestBlobFixture( ... + 'sessionDir', sDir, ... + 'zarrDir', zDir, ... + 'Shape', [32 32 32], ... + 'NumChannels', 1, ... + 'NumLevels', 2, ... + 'ChunkShape', [16 16 16], ... + 'materializeChunks', true, ... + 'progressFcn', captureFcn); + + testCase.verifyNotEmpty(log, ... + 'The callback should have fired at least once.'); + fracs = [log.frac]; + testCase.verifyTrue(all(fracs >= 0 & fracs <= 1), ... + 'All fractions must be in [0,1].'); + testCase.verifyTrue(all(diff(fracs) >= -1e-12), ... + 'Fractions must be monotone non-decreasing.'); + testCase.verifyEqual(fracs(end), 1, ... + 'Final tick should be fraction 1.0.'); + testCase.verifyEqual(log(end).text, 'Done.', ... + 'Final tick should carry the Done label.'); + end + + function testProgressFcnEmptyStaysSilent(testCase) + % [] means silent -- no NDI bar opened, no callbacks + % anywhere. The ingest should complete cleanly all the + % same. This is what CI passes to keep the run truly + % headless without depending on the auto-headless mode. + if isempty(which('ndr.format.omezarr.listPyramids')) + testCase.assumeFail(['NDR reader not on path; skipping ' ... + 'end-to-end ingest.']); + end + sDir = fullfile(tempname, 'silentsession'); + testCase.addTeardown(@() rmdir(fileparts(sDir), 's')); + zDir = fullfile(tempname, 'silentzarr'); + testCase.addTeardown(@() rmdir(zDir, 's')); + [~, pdoc, ~] = ndi.test.lightsheet.ingestBlobFixture( ... + 'sessionDir', sDir, ... + 'zarrDir', zDir, ... + 'Shape', [32 32 32], ... + 'NumChannels', 1, ... + 'NumLevels', 2, ... + 'ChunkShape', [16 16 16], ... + 'materializeChunks', true, ... + 'progressFcn', []); + testCase.verifyClass(pdoc, 'ndi.document'); + end + + function testProgressFcnBadSpecErrors(testCase) + % A stray string that is not 'default' or 'auto' is a + % user error and must be named -- not silently treated as + % silent. fromOMEZarr's resolver raises with the id + % NDI:lightsheet:fromOMEZarr:badProgressFcn. + sDir = fullfile(tempname, 'errsession'); + testCase.addTeardown(@() rmdir(fileparts(sDir), 's')); + zDir = fullfile(tempname, 'errzarr'); + testCase.addTeardown(@() rmdir(zDir, 's')); + % A tiny fixture is enough because argument parsing + % raises before the read starts. + testCase.verifyError(@() ndi.test.lightsheet.ingestBlobFixture( ... + 'sessionDir', sDir, ... + 'zarrDir', zDir, ... + 'Shape', [16 16 16], ... + 'NumChannels', 1, ... + 'NumLevels', 1, ... + 'ChunkShape', [16 16 16], ... + 'materializeChunks', false, ... + 'progressFcn', 'banana'), ... + 'NDI:lightsheet:fromOMEZarr:badProgressFcn'); + end + + function testParallelWorkersProduceIdenticalChunks(testCase) + % Serial and parallel ingest MUST agree byte-for-byte on + % every materialized chunk. The point of parallelising is + % to run the same encode faster on more cores, not to + % change the pyramid. Skipped when the Parallel Computing + % Toolbox is not on the runner. + if isempty(which('ndr.format.omezarr.listPyramids')) + testCase.assumeFail(['NDR reader not on path; ' ... + 'skipping end-to-end ingest.']); + end + if isempty(ver('parallel')) + testCase.assumeFail(['Parallel Computing Toolbox ' ... + 'not available on this runner; skipping.']); + end + + % Two runs, same fixture (fixed rng inside makeBlobFixture), + % different session dirs so both sets of chunks survive + % long enough to be compared. + sDirS = fullfile(tempname, 'serialsession'); + testCase.addTeardown(@() rmdir(fileparts(sDirS), 's')); + zDirS = fullfile(tempname, 'serialzarr'); + testCase.addTeardown(@() rmdir(zDirS, 's')); + [Ss, pS, ~] = ndi.test.lightsheet.ingestBlobFixture( ... + 'sessionDir', sDirS, 'zarrDir', zDirS, ... + 'Shape', [32 32 32], 'NumChannels', 1, ... + 'NumLevels', 2, 'ChunkShape', [16 16 16], ... + 'materializeChunks', true, 'progressFcn', [], ... + 'codec', 'blosc-zstd', 'clevel', 3, ... + 'numWorkers', 0); + + sDirP = fullfile(tempname, 'parallelsession'); + testCase.addTeardown(@() rmdir(fileparts(sDirP), 's')); + zDirP = fullfile(tempname, 'parallelzarr'); + testCase.addTeardown(@() rmdir(zDirP, 's')); + [Sp, pP, ~] = ndi.test.lightsheet.ingestBlobFixture( ... + 'sessionDir', sDirP, 'zarrDir', zDirP, ... + 'Shape', [32 32 32], 'NumChannels', 1, ... + 'NumLevels', 2, 'ChunkShape', [16 16 16], ... + 'materializeChunks', true, 'progressFcn', [], ... + 'codec', 'blosc-zstd', 'clevel', 3, ... + 'numWorkers', 2); + + testCase.verifyChunksEqual(Ss, pS, Sp, pP, ... + 'Serial and parallel must produce identical chunks.'); + end + + function verifyChunksEqual(testCase, S1, p1, S2, p2, msg) + % Shared helper: match up level docs by (level, reduction) + % and byte-compare every materialized chunk between two + % ingested sessions. Used by tests that vary a performance + % knob (numWorkers, prefetchBytes, ...) and need to prove + % it did not change the produced pyramid. + levels1 = S1.database_search( ... + ndi.query('','depends_on', ... + 'lightsheetZarrPyramid_id', p1.id()) & ... + ndi.query('','isa','lightsheetZarrLevel')); + levels2 = S2.database_search( ... + ndi.query('','depends_on', ... + 'lightsheetZarrPyramid_id', p2.id()) & ... + ndi.query('','isa','lightsheetZarrLevel')); + testCase.verifyEqual(numel(levels1), numel(levels2), ... + sprintf('%s (level count differs)', msg)); + keyOf = @(L) sprintf('%d-%s', ... + L.document_properties.lightsheetZarrLevel.level, ... + L.document_properties.lightsheetZarrLevel.reduction_function); + keys1 = cellfun(keyOf, levels1, 'UniformOutput', false); + keys2 = cellfun(keyOf, levels2, 'UniformOutput', false); + for i = 1:numel(levels1) + j = find(strcmp(keys2, keys1{i}), 1); + testCase.verifyNotEmpty(j, ... + sprintf('No matching level for %s', keys1{i})); + lA = levels1{i}; + lB = levels2{j}; + nc = lA.document_properties.lightsheetZarrLevel.n_chunks_stored; + testCase.verifyEqual( ... + lB.document_properties.lightsheetZarrLevel.n_chunks_stored, ... + nc, sprintf('%s (n_chunks_stored differs for %s)', ... + msg, keys1{i})); + for k = 1:nc + nm = sprintf('chunk.bin_%d', k); + fA = S1.database_openbinarydoc(lA, nm); + bA = fread(fA.fid, Inf, 'uint8=>uint8'); + S1.database_closebinarydoc(fA); + fB = S2.database_openbinarydoc(lB, nm); + bB = fread(fB.fid, Inf, 'uint8=>uint8'); + S2.database_closebinarydoc(fB); + testCase.verifyEqual(bA, bB, ... + sprintf('%s (chunk %s of level %s differs)', ... + msg, nm, keys1{i})); + end + end + end + + function testPrefetchBytesDoesNotChangeChunks(testCase) + % Reading one big super-block per level (default) and + % reading many tiny super-blocks MUST produce identical + % chunks. Prefetch is a performance knob, not a semantic + % one. Forcing prefetchBytes tiny drives the outer super- + % block loop through many iterations on a 32^3 fixture: + % 32^3 uint16 = 64 KiB total, so prefetch=8 KiB means + % ~8 super-blocks per level. + if isempty(which('ndr.format.omezarr.listPyramids')) + testCase.assumeFail(['NDR reader not on path; ' ... + 'skipping end-to-end ingest.']); + end + sDirBig = fullfile(tempname, 'bigprefetch'); + testCase.addTeardown(@() rmdir(fileparts(sDirBig), 's')); + zDirBig = fullfile(tempname, 'bigprefetchzarr'); + testCase.addTeardown(@() rmdir(zDirBig, 's')); + [Sb, pB, ~] = ndi.test.lightsheet.ingestBlobFixture( ... + 'sessionDir', sDirBig, 'zarrDir', zDirBig, ... + 'Shape', [32 32 32], 'NumChannels', 1, ... + 'NumLevels', 2, 'ChunkShape', [16 16 16], ... + 'materializeChunks', true, 'progressFcn', [], ... + 'codec', 'blosc-zstd', 'clevel', 3, ... + 'numWorkers', 0); + + sDirSm = fullfile(tempname, 'smallprefetch'); + testCase.addTeardown(@() rmdir(fileparts(sDirSm), 's')); + zDirSm = fullfile(tempname, 'smallprefetchzarr'); + testCase.addTeardown(@() rmdir(zDirSm, 's')); + [Ss, pS, ~] = ndi.test.lightsheet.ingestBlobFixture( ... + 'sessionDir', sDirSm, 'zarrDir', zDirSm, ... + 'Shape', [32 32 32], 'NumChannels', 1, ... + 'NumLevels', 2, 'ChunkShape', [16 16 16], ... + 'materializeChunks', true, 'progressFcn', [], ... + 'codec', 'blosc-zstd', 'clevel', 3, ... + 'numWorkers', 0, ... + 'prefetchBytes', 8 * 1024); + + testCase.verifyChunksEqual(Sb, pB, Ss, pS, ... + 'Prefetch size must not change the produced chunks.'); + end + + function testAsyncPrefetchDoesNotChangeChunks(testCase) + % Overlapping reads with compression must not change the + % produced pyramid: asyncPrefetch is a performance knob. + % Skipped if PCT is missing (no way to exercise parfeval). + if isempty(which('ndr.format.omezarr.listPyramids')) + testCase.assumeFail(['NDR reader not on path; ' ... + 'skipping end-to-end ingest.']); + end + if isempty(ver('parallel')) + testCase.assumeFail(['Parallel Computing Toolbox ' ... + 'not available on this runner; skipping.']); + end + + % Force multiple super-blocks so async has real work. + sDirSync = fullfile(tempname, 'syncsession'); + testCase.addTeardown(@() rmdir(fileparts(sDirSync), 's')); + zDirSync = fullfile(tempname, 'synczarr'); + testCase.addTeardown(@() rmdir(zDirSync, 's')); + [Ssync, pSync, ~] = ndi.test.lightsheet.ingestBlobFixture( ... + 'sessionDir', sDirSync, 'zarrDir', zDirSync, ... + 'Shape', [32 32 32], 'NumChannels', 1, ... + 'NumLevels', 2, 'ChunkShape', [16 16 16], ... + 'materializeChunks', true, 'progressFcn', [], ... + 'codec', 'blosc-zstd', 'clevel', 3, ... + 'numWorkers', 2, 'prefetchBytes', 8 * 1024, ... + 'asyncPrefetch', false); + + sDirAsync = fullfile(tempname, 'asyncsession'); + testCase.addTeardown(@() rmdir(fileparts(sDirAsync), 's')); + zDirAsync = fullfile(tempname, 'asynczarr'); + testCase.addTeardown(@() rmdir(zDirAsync, 's')); + [Sasync, pAsync, ~] = ndi.test.lightsheet.ingestBlobFixture( ... + 'sessionDir', sDirAsync, 'zarrDir', zDirAsync, ... + 'Shape', [32 32 32], 'NumChannels', 1, ... + 'NumLevels', 2, 'ChunkShape', [16 16 16], ... + 'materializeChunks', true, 'progressFcn', [], ... + 'codec', 'blosc-zstd', 'clevel', 3, ... + 'numWorkers', 2, 'prefetchBytes', 8 * 1024, ... + 'asyncPrefetch', true); + + testCase.verifyChunksEqual(Ssync, pSync, Sasync, pAsync, ... + 'asyncPrefetch must not change the produced chunks.'); + end + + function testMaxKeepsBrightSpikeMeanDoesNot(testCase) + % At the spike's downsampled position, max preserves the + % 60k spike and mean dilutes it (averages the one spike + % voxel with factor^3-1 darker neighbours). Verified on + % the ground-truth ladder so the assertion is about the + % downsampler, not the on-disk writer. + L0 = testCase.gt.volumes.level0; + spikePos = max(1, round(testCase.gt.shape / 6)); + % The wide Gaussian bleeds a fraction of a unit into the + % spike voxel, so L0 rounds to 60000 or 60001; a >= check + % holds either way, and 60000 is the sensible lower bound. + testCase.verifyGreaterThanOrEqual( ... + L0(spikePos(1), spikePos(2), spikePos(3)), uint16(60000)); + top = testCase.gt.numLevels; + factor = 2^(top - 1); + blk = floor((spikePos - 1) / factor) + 1; + maxTop = testCase.gt.volumes.max{top}; + meanTop = testCase.gt.volumes.mean{top}; + testCase.verifyGreaterThanOrEqual( ... + maxTop(blk(1), blk(2), blk(3)), uint16(60000)); + testCase.verifyLessThan( ... + double(meanTop(blk(1), blk(2), blk(3))), ... + double(maxTop(blk(1), blk(2), blk(3)))); + end + + function testMultiChannelFixtureWritesCZYXLayout(testCase) + % The default (NumChannels=2) fixture writes a 4-D + % (C, Z, Y, X) OME-Zarr with a leading channel axis, a + % 4-tuple chunk shape (channel dim = 1), and an OMERO + % channels block naming Ch1 / Ch2. + multiDir = tempname; + mkdir(multiDir); + testCase.addTeardown(@() rmdir(multiDir, 's')); + [zp, gt] = ndi.test.lightsheet.makeBlobFixture(multiDir, ... + 'Shape', [32 32 32], ... + 'NumLevels', 2, ... + 'ChunkShape', [16 16 16]); + testCase.verifyEqual(gt.numChannels, 2); + testCase.verifyEqual(gt.channelNames, {'Ch1','Ch2'}); + % Level 0 shape has C prepended. + level0Shape = gt.levels(1).shape; + testCase.verifyEqual(level0Shape, [2 32 32 32]); + % .zarray chunks is 4-tuple with 1 on the channel axis. + za = jsondecode(fileread(fullfile(zp, '0', '.zarray'))); + testCase.verifyEqual(reshape(za.chunks, 1, []), [1 16 16 16]); + testCase.verifyEqual(reshape(za.shape, 1, []), [2 32 32 32]); + % One chunk file per (c, z, y, x) tile with a 4-dot key. + testCase.verifyTrue(isfile(fullfile(zp, '0', '0.0.0.0'))); + testCase.verifyTrue(isfile(fullfile(zp, '0', '1.0.0.0'))); + % .zattrs axes list carries c first, and an OMERO block + % names both channels. jsondecode may return multiscales / + % axes / channels as either a struct array or a cell array + % depending on field homogeneity, so pick the first entry + % through a helper that handles both. + attrs = jsondecode(fileread(fullfile(zp, '.zattrs'))); + firstMultiscale = firstEntry(attrs.multiscales); + firstAxis = firstEntry(firstMultiscale.axes); + testCase.verifyEqual(char(firstAxis.name), 'c'); + testCase.verifyEqual(char(firstAxis.type), 'channel'); + testCase.verifyTrue(isfield(attrs, 'omero')); + testCase.verifyEqual(entryCount(attrs.omero.channels), 2); + firstOmero = firstEntry(attrs.omero.channels); + testCase.verifyEqual(char(firstOmero.label), 'Ch1'); + end + + end +end + +% --------------------------------------------------------------------- + +function e = firstEntry(coll) +% First element of either a cell array or a struct array. jsondecode +% chooses one or the other depending on field homogeneity, so callers +% that want "element 1" without caring can go through this helper. + if iscell(coll) + e = coll{1}; + else + e = coll(1); + end +end + +function n = entryCount(coll) + if iscell(coll) + n = numel(coll); + else + n = numel(coll); + end +end + +function names = collectPyramidNames(ms) + if iscell(ms) + names = cellfun(@(e) char(e.name), ms, 'UniformOutput', false); + else + names = arrayfun(@(e) char(e.name), ms, 'UniformOutput', false); + end + names = names(:).'; +end + +function firsts = collectFirstPaths(ms) + n = numel(ms); + firsts = cell(1, n); + for k = 1:n + if iscell(ms) + entry = ms{k}; + else + entry = ms(k); + end + ds = entry.datasets; + if iscell(ds) + first = ds{1}; + else + first = ds(1); + end + firsts{k} = char(first.path); + end +end + diff --git a/tests/+ndi/+unittest/+vintage/FakeFileSession.m b/tests/+ndi/+unittest/+vintage/FakeFileSession.m new file mode 100644 index 0000000000..b6eae12478 --- /dev/null +++ b/tests/+ndi/+unittest/+vintage/FakeFileSession.m @@ -0,0 +1,31 @@ +classdef FakeFileSession +%FAKEFILESESSION The two session calls ndi.database.fun.extract_docs_files +% makes, and nothing more. DATABASE_SEARCH returns the documents it was +% built with; DATABASE_EXISTBINARYDOC answers from a name -> path map, +% [false, ''] for any other name -- so a test sees exactly which member +% names the walker asked for by which ones it recorded. + + properties + Docs + Files + end + + methods + function obj = FakeFileSession(docs, files) + obj.Docs = docs; + obj.Files = files; + end + + function d = database_search(obj, ~) + d = obj.Docs; + end + + function [tf, p] = database_existbinarydoc(obj, ~, name) + tf = isKey(obj.Files, name); + p = ''; + if tf + p = obj.Files(name); + end + end + end +end diff --git a/tests/+ndi/+unittest/+vintage/LightsheetFixtures.m b/tests/+ndi/+unittest/+vintage/LightsheetFixtures.m new file mode 100644 index 0000000000..c4d4ed1fd3 --- /dev/null +++ b/tests/+ndi/+unittest/+vintage/LightsheetFixtures.m @@ -0,0 +1,214 @@ +classdef LightsheetFixtures +%LIGHTSHEETFIXTURES Lightsheet documents in both vintages, built by hand. +% +% The V_eta builders here are shaped as DID-matlab's lightsheet migration +% emits them (+migrators_j/lightsheet_zarr_pyramid.m, +% lightsheet_zarr_level.m, private/jLightsheetKeys.m (through +% did2.build.key: a regular axis's origin and spacing are {value, +% source_value} pairs) and the store-record +% half of +did2/+convert/foldGenericFiles.m, DID-matlab +% claude/ecstatic-pascal-md91fw at f363d98), AFTER the batch pass: +% +% pyramid an `observation` (value kind `intensity`) with the v1 pyramid's +% id; entity_id = the subject, instrument_id = the element, +% software_id = the builder `entity`; value.data_type 'uint16', +% value.source_data_type ' + end + end + + end +end diff --git a/tests/+ndi/+unittest/+vintage/TestLightsheetMigratedSession.m b/tests/+ndi/+unittest/+vintage/TestLightsheetMigratedSession.m new file mode 100644 index 0000000000..f224368c5a --- /dev/null +++ b/tests/+ndi/+unittest/+vintage/TestLightsheetMigratedSession.m @@ -0,0 +1,157 @@ +classdef TestLightsheetMigratedSession < matlab.unittest.TestCase +%TESTLIGHTSHEETMIGRATEDSESSION The lightsheet readers over a V2 session holding a MIGRATED pyramid. +% +% Writes, into a V2 (did2) session made by ndi.setup.V2.createSession and +% reopened with the ordinary ndi.session.dir, one pyramid as DID-matlab's +% lightsheet migration leaves it (ndi.unittest.vintage.LightsheetFixtures): +% the intensity observation, its builder software entity, its OME-Zarr +% store's unheld opaque_body, and three level sampled_bodies -- level 0 +% (two chunks, recorded by location as body_data_0 and body_data_1), and +% one reduced level each of max and mean. Then asks the readers: +% levelTable, chooseLevel, LightsheetZarrManager's rows and deletion plan, +% and whether a level's member 0 resolves to its bytes. +% +% Filtered (never passed) when did2.build or a V2 schema with `study` is +% missing, as TestCreateSession is; test-import-v2.yml fails on a +% filtered test, so a green run means these ran. +% +% UNVERIFIED: written without MATLAB; test-import-v2.yml is the first run. + + properties + Dir + Session + Member0 + end + + methods (TestClassSetup) + function needV2(testCase) + testCase.assumeTrue(~isempty(which('did2.build.document')), ... + 'did2.build (DID-matlab V2) is not on the path'); + schemaPath = getenv('DID_SCHEMA_PATH'); + testCase.assumeTrue(~isempty(schemaPath) && ... + ndi.setup.V2.schemaHasClass('study', schemaPath), ... + 'DID_SCHEMA_PATH does not hold a V2 schema with `study`'); + % One method, not two: TestClassSetup methods run in no + % guaranteed order, and the write needs the checks above first. + F = ndi.unittest.vintage.LightsheetFixtures; + testCase.Dir = tempname; + mkdir(testCase.Dir); + testCase.addTeardown(@() rmdir(testCase.Dir, 's')); + bytes = fullfile(testCase.Dir, 'bytes'); + mkdir(bytes); + testCase.Member0 = F.tempFile(bytes, 'uid_level0_chunk0', uint8(1:8)); + m1 = F.tempFile(bytes, 'uid_level0_chunk1', uint8(9:16)); + % A DECOY: an intensity observation with the same builder + % software but no light-sheet method. The finder keys on the + % method (Jess 2026-10-15), so it must NOT be listed. + decoy = F.pyramid('obs_not_lightsheet', 'sw_ls_1'); + decoy.interaction = struct('method', struct('node', '', 'name', '')); + decoy.statement.notes = ''; + docs = {F.software('sw_ls_1'), ... + F.pyramid('pyr_ls_1', 'sw_ls_1'), decoy, ... + F.store('fref_store_1', 'pyr_ls_1'), ... + F.level('lvl_0', 'pyr_ls_1', 1, '', {testCase.Member0, m1}), ... + F.level('lvl_max1', 'pyr_ls_1', 2, 'max', {}), ... + F.level('lvl_mean1', 'pyr_ls_1', 2, 'mean', {})}; + sessionDir = fullfile(testCase.Dir, 'session'); + mkdir(sessionDir); + ndi.setup.V2.createSession(sessionDir, 'lightsheet migrated', ... + 'SessionId', F.sessionId(), 'Documents', docs, 'Validate', false); + testCase.Session = ndi.session.dir(sessionDir); + end + end + + methods (Test) + + function testTheMigratedPyramidIsFound(testCase) + docs = ndi.vintage.lightsheetPyramidDocs(testCase.Session); + ids = cellfun(@(d) d.id(), docs, 'UniformOutput', false); + testCase.verifyEqual(ids, {'pyr_ls_1'}, ... + ['exactly the observation whose method is light-sheet ' ... + 'microscopy, not every observation of the builder software']); + end + + function testLevelTableReadsTheMigratedLevels(testCase) + t = ndi.fun.doc.lightsheet.levelTable(testCase.Session, 'pyr_ls_1'); + testCase.verifyEqual(height(t), 3); + testCase.verifyEqual(t.level.', [0 1 1]); + testCase.verifyEqual(t.id{1}, 'lvl_0', 'the level keeps its v1 id'); + testCase.verifyEqual(sort(t.reduction_function(2:3)).', {'max', 'mean'}); + testCase.verifyEqual(t.reduction_function{1}, 'none'); + testCase.verifyEqual(t.shape{1}, [1 2 400 2048 2048]); + testCase.verifyEqual(t.shape{2}, [1 2 200 1024 1024]); + testCase.verifyEqual(t.chunks{1}, [1 1 64 256 256]); + testCase.verifyEqual(t.chunk_grid{1}, [1 2 7 8 8]); + testCase.verifyEqual(t.n_chunks_stored(1), 2); + testCase.verifyEqual(t.voxel_size{1}, [1 1 2 0.5 0.5], 'AbsTol', 1e-12, ... + 'micrometre, as v1 -- not the canonical meter'); + testCase.verifyEqual(t.voxel_size{2}, [1 1 4 1 1], 'AbsTol', 1e-12); + testCase.verifyEqual(t.translation{1}, [0 0 10 0 0], 'AbsTol', 1e-12); + testCase.verifyEqual(t.dtype{1}, ' + % file_pattern, acquisition_metadata_reader -> + % epoch_parameter_reader, metadata_file_pattern -> file_regex. + % A document built to today's schema (did2.build, import_V2) + % carries the new names; a migrated one carries the old. Both + % must read. + cur = ndi.unittest.vintage.TestVintageMap.doc( ... + 'epoch_file_pattern', 'epoch_file_pattern', ... + struct('file_pattern', {{'#.mp4'}})); + [v, found] = ndi.vintage.field(cur, 'fileparameters'); + testCase.verifyTrue(found, 'fileparameters -> file_pattern did not resolve'); + testCase.verifyEqual(v, {'#.mp4'}); + + % When a block somehow carries both, the CURRENT name wins. + both = ndi.unittest.vintage.TestVintageMap.doc( ... + 'epoch_file_pattern', 'epoch_file_pattern', ... + struct('file_pattern', {{'new'}}, 'data_file_pattern', {{'old'}})); + [v, ~] = ndi.vintage.field(both, 'fileparameters'); + testCase.verifyEqual(v, {'new'}); + + [entry, vintage] = ndi.vintage.entryFor('epoch_parameter_reader'); + testCase.verifyNotEmpty(entry, 'epoch_parameter_reader has no map row'); + testCase.verifyEqual(entry.concept, 'daqmetadatareader'); + testCase.verifyEqual(vintage, 'V_eta'); + [entry2, ~] = ndi.vintage.entryFor('acquisition_metadata_reader'); + testCase.verifyEqual(entry2.concept, 'daqmetadatareader', ... + 'the pre-#73 class name must still resolve'); + + epr = ndi.unittest.vintage.TestVintageMap.doc( ... + 'epoch_parameter_reader', 'epoch_parameter_reader', ... + struct('file_regex', '.*\.tsv\>')); + [v, found] = ndi.vintage.field(epr, 'tab_separated_file_parameter'); + testCase.verifyTrue(found, 'tab_separated_file_parameter -> file_regex did not resolve'); + testCase.verifyEqual(v, '.*\.tsv\>'); + + % isaQuery asks for the v1 class AND every V_eta name. + txt = jsonencode(ndi.vintage.isaQuery('daqmetadatareader').searchstructure); + testCase.verifySubstring(txt, '"daqmetadatareader"'); + testCase.verifySubstring(txt, '"epoch_parameter_reader"'); + testCase.verifySubstring(txt, '"acquisition_metadata_reader"'); + end + + function testARepeatedEdgeNameIsAFamily(testCase) + % did-schema T15: a family is ONE name repeated, not `_1`, `_2`. + % dependency_value_n alone returns only the first of these. + deps = struct('name', {'clock_alignment_configuration_id', ... + 'clock_alignment_configuration_id', 'software_id'}, ... + 'value', {'c1', 'c2', 's1'}); + pol = ndi.unittest.vintage.TestVintageMap.doc( ... + 'clock_alignment_policy', '', [], deps); + ids = ndi.vintage.edge_n(pol, 'syncrule_id'); + testCase.verifyEqual(ids, {'c1', 'c2'}); + + % ...while the numbered form the migrators write still reads. + deps = struct('name', {'clock_alignment_configuration_1', ... + 'clock_alignment_configuration_2'}, 'value', {'c1', 'c2'}); + old = ndi.unittest.vintage.TestVintageMap.doc( ... + 'clock_alignment_policy', '', [], deps); + testCase.verifyEqual(ndi.vintage.edge_n(old, 'syncrule_id'), {'c1', 'c2'}); + end + + function testObjectNameReadsEitherHome(testCase) + % #73 item 54 moved a system's name from base.name to + % acquisition_system.name. + cur = ndi.unittest.vintage.TestVintageMap.doc( ... + 'acquisition_system', 'acquisition_system', struct('name', 'camera1')); + testCase.verifyEqual(ndi.vintage.objectName(cur), 'camera1'); + old = ndi.unittest.vintage.TestVintageMap.doc('acquisition_system', ... + '', [], [], 'intan1'); + testCase.verifyEqual(ndi.vintage.objectName(old), 'intan1'); + end + function testTheV1ObjectClassPathIsByteForByteTheOldOne(testCase) % ndi_document2ndi_object's inline read moved into % ndi.vintage.objectClass. If the v1 branch changed at all, @@ -306,7 +390,7 @@ function testAMissingObjectClassIsNamedRatherThanEmpty(testCase) methods (Static, Access = private) - function d = doc(className, blockName, blockStruct) + function d = doc(className, blockName, blockStruct, deps, baseName) % A minimal ndi.document of a named class, optionally carrying % one property block. Built from a struct so these tests need no % schema on the path. @@ -329,9 +413,15 @@ function testAMissingObjectClassIsNamedRatherThanEmpty(testCase) s.base = struct('id', 'test_id_0001', 'session_id', 'test_sess', ... 'name', '', 'datestamp', '2024-01-01T00:00:00.000Z'); s.depends_on = struct('name', {}, 'value', {}); - if nargin >= 3 + if nargin >= 3 && ~isempty(blockName) s.(blockName) = blockStruct; end + if nargin >= 4 && ~isempty(deps) + s.depends_on = deps; + end + if nargin >= 5 + s.base.name = baseName; + end d = ndi.document(s); end diff --git a/tests/+ndi/+unittest/BatchSignedUrlLookupTest.m b/tests/+ndi/+unittest/BatchSignedUrlLookupTest.m index 7b8ae82782..7ab59366e7 100644 --- a/tests/+ndi/+unittest/BatchSignedUrlLookupTest.m +++ b/tests/+ndi/+unittest/BatchSignedUrlLookupTest.m @@ -84,6 +84,28 @@ function quietTheExpectedFallbackWarning(testCase) error('BatchSignedUrlLookupTest:retryBlewUp', 'retry blew up'); end end + + function signer = flakyThenGoodSigner(~, filesMap, counterHandle, raiseCount) + % A signer that raises on the first `raiseCount` calls and + % returns `filesMap` thereafter -- exercises the transient- + % error backoff path. Named error id matches what + % matlab.net.http.send hands back on a real TLS blip, so + % the retry logic is being tested against the shape it will + % see in production. See VH-Lab/NDI-matlab#1010. + signer = @doSign; + function [ok, answer] = doSign(~, ~, varargin) + counterHandle('n') = counterHandle('n') + 1; + counterHandle('lastArgs') = varargin; + if counterHandle('n') <= raiseCount + error('MATLAB:webservices:ConnectionFailed', ... + 'Error downloading content from https://example.invalid'); + end + ok = true; + answer = struct('files', filesMap, ... + 'pages', 1, ... + 'expiresAt', '2026-12-31T00:00:00Z'); + end + end end methods (Test) @@ -482,6 +504,143 @@ function testPartialMapRetryThatRaisesStillFallsBack(testCase) stats.lastFailureReason)); end + function testTransientErrorRetriesBeforePerMemberFallback(testCase) + % One transient TLS blip at start of run must NOT cascade to + % per-member fallback for a whole series. Before the retry, + % a MATLAB:webservices:ConnectionFailed on the first batch + % call marked the scope failed and cost N getFileDetails + % calls for a 156k-member lightsheet series -- 4+ hours of + % wall clock. With the retry, one raise is absorbed by a + % second signer call and every uid in the scope resolves + % from the cached map. See VH-Lab/NDI-matlab#1010 and + % Waltham-Data-Science/NDI-python#322. + counter = containers.Map('KeyType','char','ValueType','any'); + counter('n') = 0; + filesMap = testCase.mapOf({'u_1', 'https://s3/u_1'; ... + 'u_2', 'https://s3/u_2'; ... + 'u_3', 'https://s3/u_3'}); + signer = testCase.flakyThenGoodSigner(filesMap, counter, 1); + + % Zero delays so the suite stays fast; the retry mechanics + % are what this test asserts, not the wait times. + url1 = ndi.cloud.download.internal.batchSignedUrlLookup( ... + "ds", "doc", "chunks", "u_1", ... + 'signer', signer, 'clearCache', true, ... + 'retryBackoffSeconds', [0 0 0]); + testCase.verifyEqual(url1, 'https://s3/u_1', ... + 'the first uid should resolve from the retried batch'); + testCase.verifyEqual(counter('n'), 2, ... + 'exactly one retry after the transient raise'); + + % Every other uid in the same scope must ride the cached + % map -- no more signer calls, no per-member fallback. + url2 = ndi.cloud.download.internal.batchSignedUrlLookup( ... + "ds", "doc", "chunks", "u_2", 'signer', signer); + url3 = ndi.cloud.download.internal.batchSignedUrlLookup( ... + "ds", "doc", "chunks", "u_3", 'signer', signer); + testCase.verifyEqual(url2, 'https://s3/u_2'); + testCase.verifyEqual(url3, 'https://s3/u_3'); + testCase.verifyEqual(counter('n'), 2, ... + 'later uids must not trigger another signer call'); + + [~, stats] = ndi.cloud.download.internal.batchSignedUrlLookup("", "", "", ""); + testCase.verifyEqual(stats.uidHits, 3); + testCase.verifyEqual(stats.uidMisses, 0, ... + 'no per-member fallback -- the retry rescued the scope'); + testCase.verifyEqual(stats.signerCalls, 2, ... + 'initial raise + one successful retry'); + testCase.verifyEqual(stats.transientRetries, 1, ... + 'one retry was needed to recover from the transient'); + end + + function testTransientRetriesAreBoundedThenFallBack(testCase) + % The retry is a safety net, not a busy loop. Exhaust the + % backoff and the caller is handed "" so the per-member + % fallback still runs -- the batch retry replaces the O(N) + % cascade with ~20 s of wall clock, but a scope that + % legitimately cannot be reached must not spin forever. + counter = containers.Map('KeyType','char','ValueType','any'); + counter('n') = 0; + filesMap = testCase.mapOf({'u_1', 'https://s3/u_1'}); + % 999 keeps every attempt raising -- more than any + % retryBackoffSeconds this test would set. + signer = testCase.flakyThenGoodSigner(filesMap, counter, 999); + + url = ndi.cloud.download.internal.batchSignedUrlLookup( ... + "ds", "doc", "chunks", "u_1", ... + 'signer', signer, 'clearCache', true, ... + 'retryBackoffSeconds', [0 0]); + + testCase.verifyEqual(url, "", ... + 'exhausted retries should return "" so the caller falls back'); + testCase.verifyEqual(counter('n'), 3, ... + 'three attempts total: one initial call, two retries'); + + [~, stats] = ndi.cloud.download.internal.batchSignedUrlLookup("", "", "", ""); + testCase.verifyEqual(stats.transientRetries, 2, ... + 'two retries were spent, matching numel(retryBackoffSeconds)'); + testCase.verifyEqual(stats.uidMisses, 1, ... + 'the final failure still counts as a miss'); + testCase.verifyTrue(contains(stats.lastFailureReason, ... + 'MATLAB:webservices:ConnectionFailed'), ... + sprintf('lastFailureReason should name the raise; got %s', ... + stats.lastFailureReason)); + end + + function testTransientRetryPausesBetweenAttempts(testCase) + % The backoff delays are what turn hammering a flaky + % endpoint into a graceful wait. Verify each delay is + % actually applied, and only between failed attempts (a + % successful attempt breaks the loop before its delay). + counter = containers.Map('KeyType','char','ValueType','any'); + counter('n') = 0; + filesMap = testCase.mapOf({'u_1', 'https://s3/u_1'}); + % Raise twice, succeed on the third call. Two delays should + % fire (between attempt 1 -> 2 and 2 -> 3). + signer = testCase.flakyThenGoodSigner(filesMap, counter, 2); + + box = containers.Map('KeyType','char','ValueType','any'); + box('sleeps') = []; + appendSleep = @(s) localAppendToMap(box, 'sleeps', s); + + ndi.cloud.download.internal.batchSignedUrlLookup( ... + "ds", "doc", "chunks", "u_1", ... + 'signer', signer, 'clearCache', true, ... + 'retryBackoffSeconds', [0.1 0.2 0.4], ... + 'sleepFcn', appendSleep); + + testCase.verifyEqual(box('sleeps'), [0.1 0.2], ... + 'exactly one delay per retry, in order'); + end + + function testTransientRetryDoesNotFireOnOkFalse(testCase) + % An ok=false response is a server-side no, not a transient. + % Retrying it hammers a dead endpoint for zero benefit. + % Verify the retry loop only reacts to a raise. + function [ok, answer] = failingSigner(varargin) %#ok + ok = false; + answer = struct('message','server said no'); + end + + box = containers.Map('KeyType','char','ValueType','any'); + box('sleeps') = []; + appendSleep = @(s) localAppendToMap(box, 'sleeps', s); + + url = ndi.cloud.download.internal.batchSignedUrlLookup( ... + "ds", "doc", "chunks", "u_1", ... + 'signer', @failingSigner, 'clearCache', true, ... + 'retryBackoffSeconds', [1 4 16], ... + 'sleepFcn', appendSleep); + testCase.verifyEqual(url, ""); + + [~, stats] = ndi.cloud.download.internal.batchSignedUrlLookup("", "", "", ""); + testCase.verifyEqual(stats.signerCalls, 1, ... + 'ok=false must not trigger a retry'); + testCase.verifyEqual(stats.transientRetries, 0); + testCase.verifyEmpty(box('sleeps'), ... + 'no backoff should have been slept -- nothing to retry'); + end + function testExpiredEntryIsRefetched(testCase) % Once TTL has elapsed a cached scope is dropped and the % next call goes back to the signer, so a MATLAB session diff --git a/tests/+ndi/+unittest/FileCrc32OfflineTest.m b/tests/+ndi/+unittest/FileCrc32OfflineTest.m new file mode 100644 index 0000000000..3eb79bec00 --- /dev/null +++ b/tests/+ndi/+unittest/FileCrc32OfflineTest.m @@ -0,0 +1,232 @@ +classdef FileCrc32OfflineTest < matlab.unittest.TestCase +% FILECRC32OFFLINETEST - offline tests for file CRC-32 support (NDI Cloud #162). +% +% Covers the parts that need no live cloud: computing a local file's CRC-32 +% in the cloud's encoding, the lookupFiles request and response handling, +% the checksumCRC32 query parameter on the upload URL request, and the +% signed-url-set fileInfo field. +% +% The live round trip (upload with a checksum, then read it back from +% getFileDetails, listFiles and lookupFiles) is in +% tests/+ndi/+unittest/+cloud/FilesTest.m. +% +% Expected CRC values were computed with Python's zlib.crc32, the same +% CRC-32 (IEEE) S3 uses. +% +% Authored without a local MATLAB runtime; needs MATLAB to validate/run. + + methods (Static, Access = private) + function filePath = writeTempFile(testCase, bytes) + folder = testCase.applyFixture(matlab.unittest.fixtures.TemporaryFolderFixture).Folder; + filePath = fullfile(folder, 'payload.bin'); + fid = fopen(filePath, 'w'); + fwrite(fid, uint8(bytes), 'uint8'); + fclose(fid); + end + end + + methods (Test) + + % ---- encoding ----------------------------------------------------- + + function testCrc32ToBase64IsBigEndian(testCase) + testCase.verifyEqual(ndi.fun.file.crc32ToBase64(hex2dec('3610A686')), 'NhCmhg=='); + testCase.verifyEqual(ndi.fun.file.crc32ToBase64(0), 'AAAAAA=='); + testCase.verifyEqual(ndi.fun.file.crc32ToBase64(4294967295), '/////w=='); + end + + function testCrc32ToBase64RejectsOutOfRange(testCase) + testCase.verifyError(@() ndi.fun.file.crc32ToBase64(-1), ?MException); + testCase.verifyError(@() ndi.fun.file.crc32ToBase64(2^32), ?MException); + end + + % ---- local file CRC ---------------------------------------------- + + function testCrc32OfKnownText(testCase) + p = ndi.unittest.FileCrc32OfflineTest.writeTempFile(testCase, double('hello')); + testCase.verifyEqual(ndi.fun.file.crc32(p), 'NhCmhg=='); + end + + function testCrc32OfEmptyFile(testCase) + p = ndi.unittest.FileCrc32OfflineTest.writeTempFile(testCase, []); + testCase.verifyEqual(ndi.fun.file.crc32(p), 'AAAAAA=='); + end + + function testCrc32OfBytesWithTheHighBitSet(testCase) + % Bytes 128-255 are read as negative int8 and handed to Java as + % byte[]; the CRC must still be of the unsigned bytes. + p = ndi.unittest.FileCrc32OfflineTest.writeTempFile(testCase, 0:255); + testCase.verifyEqual(ndi.fun.file.crc32(p), 'KQWMcw=='); + end + + function testCrc32DoesNotDependOnChunkSize(testCase) + p = ndi.unittest.FileCrc32OfflineTest.writeTempFile(testCase, ... + double('The quick brown fox jumps over the lazy dog')); + testCase.verifyEqual(ndi.fun.file.crc32(p), 'QU+jOQ=='); + testCase.verifyEqual(ndi.fun.file.crc32(p, 'chunkSize', 1), 'QU+jOQ=='); + testCase.verifyEqual(ndi.fun.file.crc32(p, 'chunkSize', 7), 'QU+jOQ=='); + end + + function testCrc32AcrossManyChunks(testCase) + p = ndi.unittest.FileCrc32OfflineTest.writeTempFile(testCase, repmat(0:255, 1, 1000)); + testCase.verifyEqual(ndi.fun.file.crc32(p, 'chunkSize', 4096), '/HCvGg=='); + end + + % ---- upload URL request ------------------------------------------ + + function testUploadUrlCarriesAnEncodedChecksum(testCase) + call = ndi.cloud.api.implementation.files.GetFileUploadURL(... + 'cloudDatasetID', "ds1", 'cloudFileUID', "uid1", ... + 'checksumCRC32', "QU+jOQ=="); + u = string(call.buildURL("org1")); + testCase.verifyTrue(contains(u, "/datasets/org1/ds1/files/uid1")); + testCase.verifyTrue(contains(u, "checksumCRC32=")); + % An unencoded '+' would reach the server as a space. + testCase.verifyFalse(contains(u, "QU+jOQ"), ... + 'the base64 checksum must be percent-encoded'); + end + + function testUploadUrlWithoutChecksumHasNoQuery(testCase) + call = ndi.cloud.api.implementation.files.GetFileUploadURL(... + 'cloudDatasetID', "ds1", 'cloudFileUID', "uid1"); + u = string(call.buildURL("org1")); + testCase.verifyFalse(contains(u, "checksumCRC32")); + end + + % ---- PUT with a checksum header ---------------------------------- + + function testPutSendsTheChecksumHeader(testCase) + p = ndi.unittest.FileCrc32OfflineTest.writeTempFile(testCase, double('hello')); + call = ndi.cloud.api.implementation.files.PutFiles(... + 'preSignedURL', "https://bucket.s3.amazonaws.com/k?X-Amz-Signature=abc", ... + 'filePath', p, 'checksumCRC32', "NhCmhg=="); + cmd = string(call.curlArguments()); + testCase.verifyTrue(contains(cmd, '-H "x-amz-checksum-crc32: NhCmhg=="'), ... + 'the signed x-amz-checksum-crc32 header must be sent'); + end + + function testPutWithoutChecksumSendsNoHeader(testCase) + p = ndi.unittest.FileCrc32OfflineTest.writeTempFile(testCase, double('hello')); + call = ndi.cloud.api.implementation.files.PutFiles(... + 'preSignedURL', "https://bucket.s3.amazonaws.com/k?X-Amz-Signature=abc", ... + 'filePath', p); + testCase.verifyFalse(contains(string(call.curlArguments()), "x-amz-checksum-crc32")); + end + + function testPutRefusesAMalformedChecksum(testCase) + % The value goes into a shell command, so only a real base64 + % CRC-32 is accepted. + p = ndi.unittest.FileCrc32OfflineTest.writeTempFile(testCase, double('hello')); + testCase.verifyError(@() ndi.cloud.api.implementation.files.PutFiles(... + 'preSignedURL', "https://bucket.s3.amazonaws.com/k", ... + 'filePath', p, 'checksumCRC32', "abc"" && rm -rf x"), ... + 'NDI:CloudApi:PutFiles:InvalidChecksum'); + end + + % ---- lookupFiles request ----------------------------------------- + + function testLookupEndpoint(testCase) + u = ndi.cloud.api.url('lookup_dataset_files', 'dataset_id', "ds1"); + testCase.verifyTrue(endsWith(string(u), "/datasets/ds1/files/lookup")); + end + + function testLookupBodyIsAJsonArrayEvenForOneUid(testCase) + call = ndi.cloud.api.implementation.files.LookupFiles(... + 'cloudDatasetID', "ds1", 'cloudFileUIDs', "4192a3c0dd1b4e00_3fe8a1b2c3d4e5f6"); + txt = jsonencode(call.requestBody()); + decoded = jsondecode(txt); + testCase.verifyTrue(iscell(decoded.uids) || numel(string(decoded.uids)) > 1, ... + 'uids must encode as a JSON array, not a bare string'); + testCase.verifyEqual(unique(string(decoded.uids)), "4192a3c0dd1b4e00_3fe8a1b2c3d4e5f6"); + end + + function testLookupBodyKeepsEveryUid(testCase) + uids = ["a1", "b2", "c3"]; + call = ndi.cloud.api.implementation.files.LookupFiles(... + 'cloudDatasetID', "ds1", 'cloudFileUIDs', uids); + decoded = jsondecode(jsonencode(call.requestBody())); + testCase.verifyEqual(reshape(string(decoded.uids), 1, []), uids); + end + + function testLookupRefusesMoreThanTheServerLimit(testCase) + uids = "u" + string(1:5001); + testCase.verifyError(@() ndi.cloud.api.implementation.files.LookupFiles(... + 'cloudDatasetID', "ds1", 'cloudFileUIDs', uids), ... + 'NDI:CloudApi:LookupFiles:TooManyUids'); + end + + function testLookupRefusesNoUids(testCase) + testCase.verifyError(@() ndi.cloud.api.implementation.files.LookupFiles(... + 'cloudDatasetID', "ds1", 'cloudFileUIDs', strings(1, 0)), ... + 'NDI:CloudApi:LookupFiles:NoUids'); + end + + function testLookupFilesWithNoUidsMakesNoCall(testCase) + % No request is sent, so this needs no credentials. + [b, answer, resp, url] = ndi.cloud.api.files.lookupFiles("ds1", strings(1, 0)); + testCase.verifyTrue(b); + testCase.verifyEmpty(answer.files); + testCase.verifyEmpty(answer.missing); + testCase.verifyEmpty(resp); + testCase.verifyEmpty(url); + end + + % ---- lookupFiles response ---------------------------------------- + + function testLookupResultNormalizesFilesAndMissing(testCase) + txt = ['{"files":[' ... + '{"uid":"4192a3c0dd1b4e00_3fe8a1b2c3d4e5f6","uploaded":true,"size":5,"crc32":"NhCmhg=="},' ... + '{"uid":"aa","uploaded":false,"size":0,"crc32":null}],' ... + '"missing":["zz"]}']; + r = ndi.cloud.api.implementation.files.lookupFilesResult(jsondecode(txt)); + + testCase.verifySize(r.files, [2 1]); + testCase.verifyEqual(r.files(1).uid, '4192a3c0dd1b4e00_3fe8a1b2c3d4e5f6'); + testCase.verifyEqual(r.files(1).uploaded, true); + testCase.verifyEqual(r.files(1).size, 5); + testCase.verifyEqual(r.files(1).crc32, 'NhCmhg=='); + testCase.verifyEqual(r.files(2).uploaded, false); + testCase.verifyEqual(r.files(2).crc32, '', 'a null crc32 becomes '''''); + testCase.verifyEqual(r.missing, "zz"); + end + + function testLookupResultWithNothingFound(testCase) + r = ndi.cloud.api.implementation.files.lookupFilesResult(... + jsondecode('{"files":[],"missing":["a","b"]}')); + testCase.verifyEmpty(r.files); + testCase.verifyTrue(all(isfield(r.files, {'uid','uploaded','size','crc32'}))); + testCase.verifyEqual(r.missing, ["a" "b"]); + end + + function testLookupResultWithNothingMissing(testCase) + r = ndi.cloud.api.implementation.files.lookupFilesResult(... + jsondecode('{"files":[{"uid":"a","uploaded":true,"size":1,"crc32":"AAAAAA=="}],"missing":[]}')); + testCase.verifySize(r.files, [1 1]); + testCase.verifyEqual(r.missing, strings(1, 0)); + end + + % ---- signed-url-set fileInfo ------------------------------------- + + function testSignedURLSetPageCarriesFileInfo(testCase) + uid = '4192a3c0dd1b4e00_3fe8a1b2c3d4e5f6'; + txt = ['{"files":{"' uid '":"https://s3/x"},' ... + '"fileInfo":[{"uid":"' uid '","uploaded":true,"size":5,"crc32":"NhCmhg=="}],' ... + '"nextCursor":null}']; + page = ndi.cloud.api.implementation.files.signedURLSetPage(jsondecode(txt), txt); + + testCase.verifySize(page.fileInfo, [1 1]); + testCase.verifyEqual(page.fileInfo(1).uid, uid, ... + 'a uid starting with a digit must survive (it is a value, not a key)'); + testCase.verifyEqual(page.fileInfo(1).crc32, 'NhCmhg=='); + testCase.verifyEqual(page.files(uid), 'https://s3/x'); + end + + function testSignedURLSetPageWithoutFileInfo(testCase) + % A server that predates fileInfo: the field is present and empty. + txt = '{"files":{"9a":"u1"},"nextCursor":null}'; + page = ndi.cloud.api.implementation.files.signedURLSetPage(jsondecode(txt), txt); + testCase.verifyTrue(isfield(page, 'fileInfo')); + testCase.verifyEmpty(page.fileInfo); + end + end +end diff --git a/tests/+ndi/+unittest/FileUploadWaitMockTest.m b/tests/+ndi/+unittest/FileUploadWaitMockTest.m new file mode 100644 index 0000000000..88159bcb54 --- /dev/null +++ b/tests/+ndi/+unittest/FileUploadWaitMockTest.m @@ -0,0 +1,136 @@ +classdef FileUploadWaitMockTest < matlab.unittest.TestCase +% FILEUPLOADWAITMOCKTEST - waitForFileUpload's polling logic, against mocks. +% +% A live server marks a fresh upload processed within a few seconds, so it +% cannot show what happens when it does not: a file that is still pending, +% a uid with no record yet, a transient lookup error, or one of several +% uids lagging the others. The live happy path is in +% tests/+ndi/+unittest/+cloud/FilesTest.m. +% +% Runs entirely through the pollStatus seam against ndi.test.helper. +% ScriptedFileUpload. +% +% Authored without a local MATLAB runtime; needs MATLAB to validate/run. + + methods (Test) + + function testAlreadyUploadedReturnsAfterOnePoll(testCase) + row = @ndi.test.helper.ScriptedFileUpload.fileRow; + results = { lookupResult({row("u1", true)}) }; + call = scriptedCall(results, "u1", 5); + + [b, answer] = call.execute(); + testCase.verifyTrue(b); + testCase.verifyEqual(answer.state, 'complete'); + testCase.verifyEmpty(answer.pending); + testCase.verifyEqual(numel(answer.files), 1); + testCase.verifyEqual(answer.files(1).uid, 'u1'); + testCase.verifyEqual(call.callCount, 1); + end + + function testPendingFileIsPolledUntilUploaded(testCase) + % The case that broke FilesTest: the PUT has returned but the + % server has not processed it yet (uploaded=false, size=0). + row = @ndi.test.helper.ScriptedFileUpload.fileRow; + results = { lookupResult({row("u1", false)}), ... + lookupResult({row("u1", false)}), ... + lookupResult({row("u1", true)}) }; + call = scriptedCall(results, "u1", 5); + + [b, answer] = call.execute(); + testCase.verifyTrue(b); + testCase.verifyEqual(answer.state, 'complete'); + testCase.verifyEqual(call.callCount, 3); + testCase.verifyEqual(numel(answer.files), 1, ... + 'a uid seen pending and then uploaded must be reported once'); + testCase.verifyTrue(answer.files(1).uploaded); + end + + function testMissingRecordIsTreatedAsPending(testCase) + % No record at all yet is not a failure: keep waiting. + row = @ndi.test.helper.ScriptedFileUpload.fileRow; + results = { lookupResult({}, "u1"), ... + lookupResult({row("u1", true)}) }; + call = scriptedCall(results, "u1", 5); + + [b, ~] = call.execute(); + testCase.verifyTrue(b); + testCase.verifyEqual(call.callCount, 2); + end + + function testNeverUploadedTimesOut(testCase) + row = @ndi.test.helper.ScriptedFileUpload.fileRow; + results = { lookupResult({row("u1", false)}) }; + call = scriptedCall(results, "u1", 0.05); + + [b, answer] = call.execute(); + testCase.verifyFalse(b); + testCase.verifyEqual(answer.state, 'timeout'); + testCase.verifyEqual(answer.pending, "u1"); + testCase.verifyTrue(isfield(answer, 'elapsed')); + testCase.verifyEqual(numel(answer.files), 1, ... + 'a timeout should report the last row seen for the pending uid'); + testCase.verifyFalse(answer.files(1).uploaded); + end + + function testAnApiErrorIsRetriedNotTerminal(testCase) + % A failed lookup is not a failed upload; keep polling. + row = @ndi.test.helper.ScriptedFileUpload.fileRow; + results = { struct('ok', false, 'result', struct('message', 'gateway')), ... + lookupResult({row("u1", true)}) }; + call = scriptedCall(results, "u1", 5); + + [b, ~] = call.execute(); + testCase.verifyTrue(b); + testCase.verifyEqual(call.callCount, 2); + end + + function testOnlyPendingUidsArePolledAgain(testCase) + % With several uids, the ones already uploaded drop out of the + % next lookup, and the result keeps every uploaded row. + row = @ndi.test.helper.ScriptedFileUpload.fileRow; + results = { lookupResult({row("u1", true), row("u2", false)}), ... + lookupResult({row("u2", true)}) }; + call = scriptedCall(results, ["u1" "u2"], 5); + + [b, answer] = call.execute(); + testCase.verifyTrue(b); + testCase.verifyEqual(call.callCount, 2); + testCase.verifyEqual(call.polledUids{1}, ["u1" "u2"]); + testCase.verifyEqual(call.polledUids{2}, "u2"); + testCase.verifyEqual(sort(string({answer.files.uid})), ["u1" "u2"]); + end + + function testDuplicateUidsArePolledOnce(testCase) + row = @ndi.test.helper.ScriptedFileUpload.fileRow; + results = { lookupResult({row("u1", true)}) }; + call = scriptedCall(results, ["u1" "u1"], 5); + + [b, ~] = call.execute(); + testCase.verifyTrue(b); + testCase.verifyEqual(call.polledUids{1}, "u1"); + end + end +end + +% The class lives in package ndi.unittest, so helpers are local functions +% rather than static methods (a bare FileUploadWaitMockTest.x does not +% resolve inside the package). + +function r = lookupResult(rows, missing) + % A scripted lookupFiles answer. ROWS is a cell array of fileRow structs. + if nargin < 2, missing = strings(1, 0); end + if isempty(rows) + files = struct('uid', {}, 'uploaded', {}, 'size', {}, 'crc32', {}); + files = files(:); + else + files = vertcat(rows{:}); + end + r = struct('ok', true, 'result', struct('files', files, 'missing', missing)); +end + +function call = scriptedCall(results, uids, timeout) + call = ndi.test.helper.ScriptedFileUpload(results, ... + 'cloudDatasetID', "ds1", 'cloudFileUIDs', uids, ... + 'timeout', timeout, 'initialInterval', 0.01, 'maxInterval', 0.01); +end diff --git a/tests/+ndi/+unittest/ReconstructSeriesIngestLocationsTest.m b/tests/+ndi/+unittest/ReconstructSeriesIngestLocationsTest.m index ce5643b859..33a7314177 100644 --- a/tests/+ndi/+unittest/ReconstructSeriesIngestLocationsTest.m +++ b/tests/+ndi/+unittest/ReconstructSeriesIngestLocationsTest.m @@ -87,6 +87,69 @@ function testReconstructsOneEntryPerPresentSlot(testCase) end end + function testV1DocumentNumbersMembersFromOne(testCase) + % A v1 document (MemberBase 1, the default): member NAME_1 is + % manifest slot 0, so `index` equals the 1-based manifest + % position. Passing the base explicitly gives exactly what the + % default gives -- v1 behaviour is unchanged. + [manifestUid, uids] = testCase.writeManifest(5, [1 3 5]); + fi = testCase.fileInfoFor('chunkdata.bin', manifestUid); + si = testCase.seriesInfoFor('chunkdata.bin', 5, 3); + + outDefault = ndi.cloud.sync.internal.reconstructSeriesIngestLocations( ... + si, fi, testCase.Dir, "ds"); + outV1 = ndi.cloud.sync.internal.reconstructSeriesIngestLocations( ... + si, fi, testCase.Dir, "ds", 'MemberBase', 1); + + testCase.verifyEqual([outV1(1).ingest_locations.index], [1 3 5]); + testCase.verifyEqual({outV1(1).ingest_locations.uid}, uids([1 3 5])); + testCase.verifyEqual(outV1, outDefault); + end + + function testVetaDocumentNumbersMembersFromZero(testCase) + % A V_eta document (MemberBase 0; DID-schema #73 item 17, + % DID-matlab 1a6bd9f): member NAME_0 is manifest slot 0. `index` + % is the number DID puts in the member's name + % (sprintf('%s_%d', name, index) in sqlitedb/do_add_doc), so the + % same manifest slots 1,3,5 (1-based) are members 0,2,4 -- with + % the SAME uids, because the manifest does not depend on vintage. + [manifestUid, uids] = testCase.writeManifest(5, [1 3 5]); + fi = testCase.fileInfoFor('chunkdata.bin', manifestUid); + si = testCase.seriesInfoFor('chunkdata.bin', 5, 3); + + out = ndi.cloud.sync.internal.reconstructSeriesIngestLocations( ... + si, fi, testCase.Dir, "ds", 'MemberBase', 0); + + il = out(1).ingest_locations; + testCase.assertNumElements(il, 3); + testCase.verifyEqual([il.index], [0 2 4]); + testCase.verifyEqual({il.uid}, uids([1 3 5])); + end + + function testMemberBaseReadFromDocumentVintage(testCase) + % The callers read the base with ndi.vintage.seriesMemberBase + % from the document itself. A V_eta schema_version gives 0, a + % document without one gives 1, and feeding that into the + % reconstruction names the first present member accordingly. + [manifestUid, ~] = testCase.writeManifest(2, [1 2]); + fi = testCase.fileInfoFor('body_data', manifestUid); + si = testCase.seriesInfoFor('body_data', 2, 2); + + veta = struct('document_class', struct('schema_version', 'V_eta')); + v1 = struct('document_class', struct('class_name', 'x')); + testCase.verifyEqual(ndi.vintage.seriesMemberBase(veta), 0); + testCase.verifyEqual(ndi.vintage.seriesMemberBase(v1), 1); + + outVeta = ndi.cloud.sync.internal.reconstructSeriesIngestLocations( ... + si, fi, testCase.Dir, "ds", ... + 'MemberBase', ndi.vintage.seriesMemberBase(veta)); + outV1 = ndi.cloud.sync.internal.reconstructSeriesIngestLocations( ... + si, fi, testCase.Dir, "ds", ... + 'MemberBase', ndi.vintage.seriesMemberBase(v1)); + testCase.verifyEqual([outVeta(1).ingest_locations.index], [0 1]); + testCase.verifyEqual([outV1(1).ingest_locations.index], [1 2]); + end + function testLeavesExistingIngestLocationsAlone(testCase) % A series that already carries ingest_locations (a doc built % locally and never round-tripped, say) is not touched. The diff --git a/tests/+ndi/+unittest/SignedUrlDiskCacheTest.m b/tests/+ndi/+unittest/SignedUrlDiskCacheTest.m new file mode 100644 index 0000000000..5f59c46983 --- /dev/null +++ b/tests/+ndi/+unittest/SignedUrlDiskCacheTest.m @@ -0,0 +1,284 @@ +classdef SignedUrlDiskCacheTest < matlab.unittest.TestCase +% SIGNEDURLDISKCACHETEST - the on-disk signed-URL-set cache. +% +% Persistent layer that sits beneath the in-process cache inside +% ndi.cloud.download.internal.batchSignedUrlLookup and turns a +% dataset-reopen "still ~85 min to sign 121k URLs" into "read a file". +% See ndi.cloud.download.internal.signedUrlDiskCache and the task +% brief on VH-Lab/NDI-matlab#1010 follow-on / Waltham-Data-Science/ +% NDI-python#322. +% +% Nothing here hits the network or the user's real cache directory: +% the class setup points NDI_SIGNED_URL_CACHE_DIR at a per-run temp +% dir and the teardown removes it. + + properties + TmpDir char = '' + SavedCacheDirEnv char = '' + SavedSafetyEnv char = '' + end + + methods (TestMethodSetup) + function isolateCacheDir(testCase) + % One temp dir per test so a leaked file cannot leak across + % assertions -- the class runs a save + load round-trip and + % a concurrent-write case in the same suite. + testCase.SavedCacheDirEnv = getenv('NDI_SIGNED_URL_CACHE_DIR'); + testCase.SavedSafetyEnv = getenv('NDI_SIGNED_URL_CACHE_SAFETY_SECONDS'); + testCase.TmpDir = tempname; + setenv('NDI_SIGNED_URL_CACHE_DIR', testCase.TmpDir); + % Clear the safety-seconds override so each test starts + % from the documented default (1800 s) unless it sets its + % own; a leftover override from a previous run of this + % suite would silently corrupt an expiry assertion. + setenv('NDI_SIGNED_URL_CACHE_SAFETY_SECONDS', ''); + end + + function restoreOnTeardown(testCase) + envKey = 'NDI_SIGNED_URL_CACHE_DIR'; + savedDir = testCase.SavedCacheDirEnv; + safetyKey = 'NDI_SIGNED_URL_CACHE_SAFETY_SECONDS'; + savedSaf = testCase.SavedSafetyEnv; + tmp = testCase.TmpDir; + testCase.addTeardown(@() localCleanup( ... + envKey, savedDir, safetyKey, savedSaf, tmp)); + end + end + + methods (Access = private) + function m = mapOf(~, uidUrlPairs) + m = containers.Map('KeyType','char','ValueType','any'); + for i = 1:size(uidUrlPairs, 1) + m(uidUrlPairs{i, 1}) = uidUrlPairs{i, 2}; + end + end + + function s = futureIso(~, secondsAhead) + % ISO-8601 UTC N seconds from now, in the same format the + % server uses in the getSignedURLSetJob ready-state payload. + dt = datetime('now','TimeZone','UTC') + seconds(secondsAhead); + dt.Format = 'yyyy-MM-dd''T''HH:mm:ss''Z'''; + s = char(dt); + end + end + + methods (Test) + + function testSaveLoadRoundTripPreservesFields(testCase) + % The point of the disk cache: what goes in comes back out + % byte-for-byte on the fields callers actually consume -- + % the uid->URL map, the server's authoritative expiry, and + % the generatedAt provenance. + m = testCase.mapOf({ ... + '4192a3c0dd1b4e00_3fe8a1b2c3d4e5f6', 'https://s3/one'; ... + '4192a3c0dd1b4e00_4fe8a1b2c3d4e5f6', 'https://s3/two?sig=abc%2F123'}); + payload = struct( ... + 'files', m, ... + 'filesExpireAt', testCase.futureIso(23*3600), ... + 'expiresAt', testCase.futureIso(23*3600), ... + 'generatedAt', testCase.futureIso(-30), ... + 'fileCount', 2); + ndi.cloud.download.internal.signedUrlDiskCache.save( ... + "ds1", "doc1", "chunk.bin", payload); + + got = ndi.cloud.download.internal.signedUrlDiskCache.load( ... + "ds1", "doc1", "chunk.bin"); + + testCase.assertNotEmpty(got, 'a just-written scope should load'); + testCase.verifyEqual(sort(keys(got.files)), ... + sort({'4192a3c0dd1b4e00_3fe8a1b2c3d4e5f6', '4192a3c0dd1b4e00_4fe8a1b2c3d4e5f6'})); + testCase.verifyEqual(got.files('4192a3c0dd1b4e00_3fe8a1b2c3d4e5f6'), ... + 'https://s3/one'); + testCase.verifyEqual(got.files('4192a3c0dd1b4e00_4fe8a1b2c3d4e5f6'), ... + 'https://s3/two?sig=abc%2F123', ... + 'URL with url-escaped bytes must survive JSON encoding'); + testCase.verifyEqual(char(got.filesExpireAt), payload.filesExpireAt); + testCase.verifyEqual(double(got.fileCount), 2); + end + + function testExpiredPayloadIsATreatedAsAMiss(testCase) + % A cached scope whose filesExpireAt is in the past must + % NOT be served -- the URLs are dead and would 403 on use. + m = testCase.mapOf({'aa_bb', 'https://s3/expired'}); + payload = struct( ... + 'files', m, ... + 'filesExpireAt', testCase.futureIso(-3600), ... % 1 h ago + 'expiresAt', "", ... + 'generatedAt', "", ... + 'fileCount', 1); + ndi.cloud.download.internal.signedUrlDiskCache.save( ... + "ds1", "doc1", "", payload); + + got = ndi.cloud.download.internal.signedUrlDiskCache.load( ... + "ds1", "doc1", ""); + testCase.verifyEmpty(got, ... + 'a filesExpireAt in the past is a miss'); + end + + function testSafetyBufferGuardsAgainstAlmostExpired(testCase) + % The whole point of the buffer: a URL that will expire in + % 20 minutes is a miss even though it is still technically + % valid, because a viewer opened with it will 403 before + % they get through the file. + m = testCase.mapOf({'aa_bb', 'https://s3/almost'}); + payload = struct( ... + 'files', m, ... + 'filesExpireAt', testCase.futureIso(20*60), ... + 'expiresAt', "", ... + 'generatedAt', "", ... + 'fileCount', 1); + ndi.cloud.download.internal.signedUrlDiskCache.save( ... + "ds1", "doc1", "", payload); + + % Default safety is 30 min (1800 s), so 20 min is inside + % the buffer and load must miss. + got = ndi.cloud.download.internal.signedUrlDiskCache.load( ... + "ds1", "doc1", ""); + testCase.verifyEmpty(got, ... + '20 min < 30 min safety buffer must be a miss'); + + % With a shorter buffer the same file loads -- proving the + % check is the buffer, not the write, doing the work. + setenv('NDI_SIGNED_URL_CACHE_SAFETY_SECONDS', '60'); + got2 = ndi.cloud.download.internal.signedUrlDiskCache.load( ... + "ds1", "doc1", ""); + testCase.verifyNotEmpty(got2, ... + 'with a 60 s buffer the same payload loads'); + end + + function testMissingExpiryIsNotCacheable(testCase) + % save() refuses to persist a payload with no expiry -- the + % cache would then have no way to age-check it. Load must + % see nothing on disk. + m = testCase.mapOf({'aa_bb', 'https://s3/no-expiry'}); + payload = struct( ... + 'files', m, ... + 'filesExpireAt', "", ... + 'expiresAt', "", ... + 'generatedAt', "", ... + 'fileCount', 1); + ndi.cloud.download.internal.signedUrlDiskCache.save( ... + "ds1", "doc1", "", payload); + + got = ndi.cloud.download.internal.signedUrlDiskCache.load( ... + "ds1", "doc1", ""); + testCase.verifyEmpty(got); + end + + function testForgetRemovesTheFile(testCase) + % forget is the S3-403 hook: an S3 refusal means the + % scope's cached URLs are dead, so drop them before the + % next uid in the scope pays the same 403. + m = testCase.mapOf({'aa_bb', 'https://s3/ok'}); + payload = struct( ... + 'files', m, ... + 'filesExpireAt', testCase.futureIso(3600), ... + 'expiresAt', "", ... + 'generatedAt', "", ... + 'fileCount', 1); + ndi.cloud.download.internal.signedUrlDiskCache.save( ... + "ds1", "doc1", "chunk.bin", payload); + testCase.assertNotEmpty( ... + ndi.cloud.download.internal.signedUrlDiskCache.load( ... + "ds1", "doc1", "chunk.bin"), ... + 'sanity: the scope should be on disk before forget'); + + ndi.cloud.download.internal.signedUrlDiskCache.forget( ... + "ds1", "doc1", "chunk.bin"); + + testCase.verifyEmpty( ... + ndi.cloud.download.internal.signedUrlDiskCache.load( ... + "ds1", "doc1", "chunk.bin"), ... + 'forget must remove the scope from disk'); + end + + function testForgetOnUnknownScopeDoesNotRaise(testCase) + % Best-effort: a forget on a scope we never cached must + % succeed silently so the S3-403 recovery path doesn't + % need to check-first. + testCase.verifyWarningFree(@() ... + ndi.cloud.download.internal.signedUrlDiskCache.forget( ... + "nosuch-ds", "nosuch-doc", "nosuch-series")); + end + + function testCacheDirRespectsEnvVar(testCase) + % The env-var override lets NDI-python and the MATLAB tests + % point the cache anywhere; the default falls under + % ~/.ndi/. Both are documented on-disk contract. + actual = ndi.cloud.download.internal.signedUrlDiskCache.cacheDir(); + testCase.verifyEqual(actual, testCase.TmpDir); + end + + function testConcurrentSavesDoNotCorruptTheFile(testCase) + % Two viewers signing the same scope back-to-back is the + % expected shape: last write wins, both reads afterward see + % a valid file. atomic rename is what makes this true. + baseMap = testCase.mapOf({'aa_bb', 'https://s3/first'}); + payload1 = struct( ... + 'files', baseMap, ... + 'filesExpireAt', testCase.futureIso(3600), ... + 'expiresAt', "", ... + 'generatedAt', "", ... + 'fileCount', 1); + payload2 = payload1; + payload2.files = testCase.mapOf({'cc_dd', 'https://s3/second'}); + + for i = 1:10 + ndi.cloud.download.internal.signedUrlDiskCache.save( ... + "ds1", "doc1", "chunks", payload1); + ndi.cloud.download.internal.signedUrlDiskCache.save( ... + "ds1", "doc1", "chunks", payload2); + end + + got = ndi.cloud.download.internal.signedUrlDiskCache.load( ... + "ds1", "doc1", "chunks"); + testCase.assertNotEmpty(got, ... + 'after alternating writes, the file must still be valid'); + % Whichever payload landed last is fine (last-writer wins); + % what matters is that we did NOT get a half-written blob. + testCase.verifyTrue(isa(got.files, 'containers.Map')); + testCase.verifyEqual(double(got.files.Count), 1); + end + + function testSeriesNameEscapingHandlesPathySpaces(testCase) + % Real series names can hold '/', spaces, unicode; the + % filesystem can't. The escape rule NDI-python mirrors is + % percent-encoding for non-safe bytes and a SHA-1 fallback + % above 96 chars. + m = testCase.mapOf({'aa_bb', 'https://s3/x'}); + payload = struct( ... + 'files', m, ... + 'filesExpireAt', testCase.futureIso(3600), ... + 'expiresAt', "", ... + 'generatedAt', "", ... + 'fileCount', 1); + + pathy = "level 3/chunk.bin"; + ndi.cloud.download.internal.signedUrlDiskCache.save( ... + "ds1", "doc1", pathy, payload); + got = ndi.cloud.download.internal.signedUrlDiskCache.load( ... + "ds1", "doc1", pathy); + testCase.assertNotEmpty(got, ... + 'a series name with a slash and a space must save and load'); + + % Two different names must land in two different files. + other = "level 3_chunk.bin"; + ndi.cloud.download.internal.signedUrlDiskCache.save( ... + "ds1", "doc1", other, payload); + listing = dir(fullfile(testCase.TmpDir, 'ds1')); + names = {listing(~[listing.isdir]).name}; + testCase.verifyEqual(sum(endsWith(names, '.json.gz')), 2, ... + 'two distinct series names must produce two distinct cache files'); + end + end +end + +function localCleanup(envKey, savedDir, safetyKey, savedSaf, tmp) + setenv(envKey, savedDir); + setenv(safetyKey, savedSaf); + if ~isempty(tmp) && isfolder(tmp) + try %#ok + rmdir(tmp, 's'); + end + end +end diff --git a/tests/requirements.txt b/tests/requirements.txt index 282f87af64..3bb32d121f 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -5,4 +5,4 @@ https://github.com/a-ma72/mksqlite@master https://github.com/VH-lab/vhlab-thirdparty-matlab@master https://github.com/Waltham-Data-Science/NDI-compress-matlabp https://github.com/Waltham-Data-Science/ndi-ontology-matlab -fex://134212-openminds-metadata-models-for-matlab/0.11.0 +https://github.com/openMetadataInitiative/openMINDS_MATLAB@v0.12.0 diff --git a/tools/v2_import_profile.m b/tools/v2_import_profile.m new file mode 100644 index 0000000000..41dcf5220b --- /dev/null +++ b/tools/v2_import_profile.m @@ -0,0 +1,124 @@ +function R = v2_import_profile(dataParentDir, options) +%V2_IMPORT_PROFILE Where the time goes when the Haley V2 import runs one session. +% +% R = v2_import_profile(DATAPARENTDIR) runs ndi.setup.conv.haley.import_V2 +% on one session ('Sessions', default concentration_0001), every stage, +% 'Write' true, into a FRESH output folder under tempdir (your real +% haley_V2 is not touched), under the MATLAB profiler. It prints: +% +% - the wall-clock time; +% - self time grouped by where it is spent: the schema cache, did2.build +% (making documents), the database and mksqlite (writing), the rest of +% did2, the Haley import's own code, the rest of NDI, reading files, +% MATLAB itself; +% - the 30 functions with the most self time. +% +% R has the numbers and is saved to /import_profile.mat. +% Run it once per DID-matlab version and pass the earlier .mat as +% 'Compare' to print the two side by side, e.g. +% +% a = v2_import_profile(d, 'Label', 'V2'); % DID-matlab V2 +% % ... check out the #218 branch, clear classes, set the path again ... +% b = v2_import_profile(d, 'Label', '218', 'Compare', a.matFile); +% +% Options: +% 'Sessions' default "concentration_0001" +% 'Label' a name for this run (default 'run'), part of the folder +% 'Checksums' default false, as in the full run +% 'ReadVideos' default true (import_V2's default) +% 'Compare' a .mat saved by an earlier call +% +% The profiler slows MATLAB down (it roughly doubled the read profiled +% earlier), so read the groups as shares, and the wall clock as an upper +% bound. +% +% See also ndi.setup.conv.haley.import_V2, v2_subject_listing_profile. + +arguments + dataParentDir (1,:) char {mustBeFolder} + options.Sessions (1,:) string = "concentration_0001" + options.Label (1,:) char = 'run' + options.Checksums (1,1) logical = false + options.ReadVideos (1,1) logical = true + options.Compare (1,:) char = '' +end + +out = fullfile(tempdir, ['haley_V2_profile_' options.Label]); +if isfolder(out), rmdir(out, 's'); end +mkdir(out); +fprintf('import of %s into %s (fresh)\nDID-matlab: %s\n', strjoin(options.Sessions, ', '), out, ... + fileparts(fileparts(fileparts(which('did2.schema.cache'))))); + +profile clear; profile on; +t0 = tic; +r = ndi.setup.conv.haley.import_V2(dataParentDir, 'Sessions', options.Sessions, ... + 'OutputRoot', out, 'Write', true, 'Overwrite', true, ... + 'Checksums', options.Checksums, 'ReadVideos', options.ReadVideos); +wall = toc(t0); +profile off; +info = profile('info'); +ft = info.FunctionTable; + +self = arrayfun(@(f) f.TotalTime - sum([f.Children.TotalTime]), ft); +names = string({ft.FunctionName})'; +files = string({ft.FileName})'; +group = strings(numel(names), 1); +for k = 1:numel(names), group(k) = groupOf(names(k), files(k)); end + +[g, ~, j] = unique(group); +selfByGroup = accumarray(j, self(:)); +G = table(g, selfByGroup, selfByGroup / sum(self), 'VariableNames', {'group', 'self_s', 'share'}); +G = sortrows(G, 'self_s', 'descend'); + +F = table(names, group, self(:), [ft.TotalTime]', [ft.NumCalls]', ... + 'VariableNames', {'fn', 'group', 'self_s', 'total_s', 'calls'}); +F = sortrows(F, 'self_s', 'descend'); + +nDocs = NaN; +try nDocs = sum(r.dataset.report.documents.byClass.GroupCount); catch, end +fprintf('\n== wall clock (under the profiler) ==\n %.1f s for %s; %g document(s) in the dataset\n', ... + wall, strjoin(options.Sessions, ', '), nDocs); +fprintf('\n== self time by group ==\n'); +disp(G); +fprintf('\n== top 30 functions by self time ==\n'); +disp(F(1:min(30, height(F)), :)); + +R = struct('label', options.Label, 'sessions', options.Sessions, 'wall', wall, ... + 'documents', nDocs, 'groups', G, 'functions', F, 'outputRoot', out, ... + 'matFile', fullfile(out, 'import_profile.mat')); +save(R.matFile, 'R'); +fprintf('saved %s\n', R.matFile); + +if ~isempty(options.Compare) + A = load(options.Compare); + A = A.R; + fprintf('\n== %s vs %s ==\n wall: %.1f s vs %.1f s\n', A.label, R.label, A.wall, R.wall); + C = outerjoin(A.groups(:, {'group', 'self_s'}), R.groups(:, {'group', 'self_s'}), ... + 'Keys', 'group', 'MergeKeys', true); + C.Properties.VariableNames = {'group', A.label, R.label}; + disp(C); +end +end + +function g = groupOf(name, file) +f = replace(file, '\', '/'); +if contains(f, '/+did2/+schema/') + g = "schema cache"; +elseif contains(f, '/+did2/+build/') + g = "did2.build (making documents)"; +elseif contains(f, '/+did2/+database/') || startsWith(name, "mksqlite") + g = "database + mksqlite (writing)"; +elseif contains(f, '/+did2/') + g = "did2, other"; +elseif contains(f, '/+haley/') || contains(f, '/+setup/+V2/') + g = "Haley import"; +elseif contains(f, '/+ndi/') + g = "NDI, other"; +elseif any(startsWith(name, ["readtable", "readmatrix", "readcell", "load", "fread", "fopen", ... + "imread", "imfinfo", "VideoReader", "read", "h5read", "xlsread", "fileread", "jsondecode", ... + "dir", "exist", "isfile", "isfolder"])) + g = "reading files"; +else + g = "MATLAB, other"; +end +end diff --git a/tools/v2_object_layer_tryout.m b/tools/v2_object_layer_tryout.m new file mode 100644 index 0000000000..abeeeb65d8 --- /dev/null +++ b/tools/v2_object_layer_tryout.m @@ -0,0 +1,274 @@ +function T = v2_object_layer_tryout(datasetPath, options) +%V2_OBJECT_LAYER_TRYOUT Try the V2 object layer on a real dataset, timing each step. +% +% T = v2_object_layer_tryout(DATASETPATH) opens the V2 dataset at +% DATASETPATH (the folder holding .ndi, e.g. /haley_V2/dataset) and +% asks it the questions the object layer exists for, through ndi.entity, +% ndi.statement, ndi.entity and ndi.value, READ THROUGH THE DATASET: +% things are found in one session (quick), then read through the +% ndi.dataset, which also reaches the shared documents (software, +% formulations, strains, people) that a session cannot. The last step +% reads the same statements through the session, to show the +% difference. Each step prints what it +% found and how long it took; a step that errors is reported and the rest +% still run. T is one row per step: step, ok, seconds, result. +% +% Options: +% 'Session' the session reference to open (default: the first +% concentration session the dataset lists) +% +% It writes nothing. +% +% See also ndi.statement, ndi.entity, ndi.value, +% src/ndi/docs/NDI-matlab/manual/developer/V2_Object_Layer.md. + +arguments + datasetPath (1,:) char {mustBeFolder} + options.Session (1,:) char = '' +end + +rows = cell(0, 4); +ctx = struct(); + + function step(name, fn) + fprintf('\n== %s ==\n', name); + t0 = tic; + try + r = fn(); + ok = true; + catch err + r = sprintf('ERROR %s: %s', err.identifier, err.message); + ok = false; + fprintf(' %s\n', r); + for k = 1:min(3, numel(err.stack)) + fprintf(' at %s:%d\n', err.stack(k).name, err.stack(k).line); + end + end + sec = toc(t0); + fprintf(' [%.3f s]\n', sec); + rows(end+1, :) = {string(name), ok, sec, string(r)}; + end + +% ---- the dataset and a session --------------------------------------------------- +step('open the dataset', @openDataset); + function r = openDataset() + ctx.ds = ndi.dataset.dir(datasetPath); + [ctx.refs, ctx.ids] = ctx.ds.session_list(); + r = sprintf('%d session(s)', numel(ctx.refs)); + fprintf(' %s\n', r); + end + +step('open a session', @openSession); + function r = openSession() + ref = options.Session; + if isempty(ref) + ref = ctx.refs{find(startsWith(ctx.refs, 'concentration'), 1)}; + end + ctx.S = ctx.ds.open_session(ctx.ids{strcmp(ctx.refs, ref)}); + r = ref; + fprintf(' %s\n', r); + end + +% ---- subjects --------------------------------------------------------------------- +step('ndi.entity.search: every subject in the session', @allSubjects); + function r = allSubjects() + s = ndi.entity.search(ctx.S); + types = cellfun(@(x) string(x.type()), s); + [u, ~, j] = unique(types); + n = accumarray(j(:), 1); + parts = arrayfun(@(k) sprintf('%s %d', u(k), n(k)), 1:numel(u), 'UniformOutput', false); + r = sprintf('%d subject(s): %s', numel(s), strjoin(parts, ', ')); + fprintf(' %s\n', r); + end + +step('ndi.entity.search: every subject in the dataset', @allSubjectsDataset); + function r = allSubjectsDataset() + s = ndi.entity.search(ctx.ds); + r = sprintf('%d subject(s) in %d session(s)', numel(s), numel(ctx.refs)); + fprintf(' %s\n', r); + end + +step('ndi.statement.search: the speed calculations in the session', @speeds); + function r = speeds() + ctx.sp = ndi.statement.search(ctx.S, 'Class', 'velocity_calculation'); + r = sprintf('%d velocity_calculation(s)', numel(ctx.sp)); + fprintf(' %s\n', r); + end + +step('the worm of the first one, read through the dataset', @worm); + function r = worm() + ctx.wS = ctx.sp{1}.subject(); + ctx.w = ndi.entity.fromDocument(ctx.ds, ctx.wS.document_id()); + r = sprintf('%s (%s), %s', ctx.w.local_identifier, ctx.w.type, ctx.w.document_id); + fprintf(' %s\n', r); + end + +step('ndi.entity.search by LocalIdentifier, in the dataset', @byLocal); + function r = byLocal() + x = ndi.entity.search(ctx.ds, {'LocalIdentifier', ctx.w.local_identifier}); + r = sprintf('%d found', numel(x)); + fprintf(' %s\n', r); + end + +step('the worm''s statements, by kind', @wormStatements); + function r = wormStatements() + st = ctx.w.statements(); + k = cellfun(@(x) string(x.kind()) + " / " + string(x.variable_name()), st); + [u, ~, j] = unique(k); + n = accumarray(j(:), 1); + for i = 1:numel(u), fprintf(' %3d %s\n', n(i), u(i)); end + r = sprintf('%d statement(s)', numel(st)); + end + +% ---- a value from its body ------------------------------------------------------- +step('its speed: value() decoded from the body', @speedValue); + function r = speedValue() + st = ctx.w.statements('velocity_calculation', {'variable', 'midpoint speed'}); + if isempty(st), st = ctx.w.statements('velocity_calculation'); end + ctx.speed = st{1}; + v = st{1}.value(); + d = double(v); + fprintf(' variable: %s; class %s; unit %s; size %s\n', st{1}.variable_name(), ... + v.class_name, v.unit(), mat2str(size(d))); + fprintf(' first values: %s\n', mat2str(d(1:min(5, end))', 4)); + fprintf(' median %.4g, max %.4g (%d NaN)\n', median(d, 'omitnan'), max(d), nnz(isnan(d))); + disp(v.axes()); + r = sprintf('%s %s, median %.4g %s', st{1}.variable_name(), mat2str(size(d)), ... + median(d, 'omitnan'), v.unit()); + end + +step('what the speed was computed from (inputs)', @inputs); + function r = inputs() + in = ctx.speed.inputs(); + names = cellfun(@describe, in, 'UniformOutput', false); + for i = 1:numel(names), fprintf(' %s\n', names{i}); end + r = strjoin(names, '; '); + end + +step('how: method, parameters, software, time', @how); + function r = how() + fprintf(' method: %s\n', ctx.speed.method_name()); + disp(ctx.speed.method_parameters()); + sw = ctx.speed.software(); + fprintf(' software: %s\n', strjoin(cellfun(@(x) x.name(), sw, 'UniformOutput', false), ', ')); + t = ctx.speed.time(); + for i = 1:numel(t) + fprintf(' time: %s, %s to %s (resolved by %s)\n', t(i).kind, string(t(i).start), ... + string(t(i).end), t(i).resolved_by); + end + r = sprintf('%s; %d time reference(s)', ctx.speed.method_name(), numel(t)); + end + +% ---- groups, parts, assertions ---------------------------------------------------- +step('the worm''s cohort and its members', @cohort); + function r = cohort() + g = ctx.w.memberOf(); + ctx.cohort = g{1}; + m = ctx.cohort.members(); + r = sprintf('%s: %d member(s)', ctx.cohort.local_identifier, numel(m)); + fprintf(' %s\n', r); + end + +step('the cohort''s assertions', @assertions); + function r = assertions() + A = ctx.cohort.assertions(); + disp(A); + r = strjoin(A.variable + " = " + A.value, '; '); + end + +step('the cohort''s relations (places over time)', @relations); + function r = relations() + R = ctx.cohort.relations(); + disp(R(:, {'relation', 'direction', 'other_id'})); + r = sprintf('%d relation(s)', height(R)); + end + +step('a plate and its patches (part_of)', @plate); + function r = plate() + p = ctx.cohort.parents('contained_in'); + names = cellfun(@(x) x.local_identifier, p, 'UniformOutput', false); + fprintf(' the cohort was in: %s\n', strjoin(names, ', ')); + pl = p{find(contains(names, 'assayPlate'), 1)}; + parts = pl.parts(); + r = sprintf('%s has %d part(s): %s', pl.local_identifier, numel(parts), ... + strjoin(cellfun(@(x) x.local_identifier, parts, 'UniformOutput', false), ', ')); + fprintf(' %s\n', r); + ctx.plate = pl; + end + +% ---- manipulations ------------------------------------------------------------------ +step('the plate''s manipulations, with time and dose', @manipulations); + function r = manipulations() + mm = ctx.plate.statements('manipulation'); + for i = 1:numel(mm) + v = mm{i}.value(); + t = mm{i}.time(); + when = 'no time'; + if ~isempty(t), when = sprintf('%s %s', t(1).kind, string(t(1).start)); end + fprintf(' %-26s %-24s %-22s %s\n', mm{i}.kind(), mm{i}.variable_name(), ... + mm{i}.method_name(), when); + if strcmp(v.class_name, 'dose'), ctx.dose = mm{i}; end + end + r = sprintf('%d manipulation(s)', numel(mm)); + end + +step('the dose''s formulation', @formulation); + function r = formulation() + f = ctx.dose.formulation(); + if isempty(f) + r = 'not found'; + else + r = sprintf('found: %s; fields %s', f.class_name, strjoin(fieldnames(f.raw), ', ')); + end + fprintf(' %s\n', r); + end + +% ---- dataset-level ------------------------------------------------------------------ +step('dataset-level entities (people, organizations, studies, strains)', @entities); + function r = entities() + out = {}; + for k = {'person', 'organization', 'study', 'strain', 'software'} + e = ndi.entity.search(ctx.ds, {'kind', k{1}}); + names = cellfun(@(x) x.name(), e, 'UniformOutput', false); + fprintf(' %-12s %3d %s\n', k{1}, numel(e), strjoin(names(1:min(5, end)), ', ')); + out{end+1} = sprintf('%s %d', k{1}, numel(e)); %#ok + end + r = strjoin(out, ', '); + end + +step('the dataset''s model fits (statement.find on the dataset)', @fits); + function r = fits() + f = ndi.statement.search(ctx.ds, 'Class', 'model_fit_calculation'); + r = sprintf('%d model_fit_calculation(s)', numel(f)); + fprintf(' %s\n', r); + end + +% ---- the same, through the session --------------------------------------------------- +step('the same worm read through the SESSION instead (for comparison)', @throughSession); + function r = throughSession() + st = ctx.wS.statements(); + sp = ctx.wS.statements('velocity_calculation', {'variable', ctx.speed.variable_name()}); + nSw = numel(sp{1}.software()); + nF = 0; + dm = ndi.statement.fromDocument(ctx.S, ctx.dose.document_id()); + if ~isempty(dm.formulation()), nF = 1; end + r = sprintf(['%d statement(s); speed software found: %d (dataset: %d); ' ... + 'formulation found: %d (dataset: 1)'], numel(st), nSw, numel(ctx.speed.software()), nF); + fprintf(' %s\n', r); + end + +% ---- summary -------------------------------------------------------------------------- +T = cell2table(rows, 'VariableNames', {'step', 'ok', 'seconds', 'result'}); +fprintf('\n== summary: %d step(s), %d ok, %d failed, %.1f s in all ==\n', height(T), ... + nnz(T.ok), nnz(~T.ok), sum(T.seconds)); +disp(T(:, {'step', 'ok', 'seconds'})); +end + +function s = describe(x) +if isa(x, 'ndi.statement') + s = sprintf('%s: %s', x.kind(), x.variable_name()); +else + p = ndi.v2.props(x); + s = sprintf('%s %s', p.document_class.class_name, p.base.id); +end +end diff --git a/tools/v2_query_plan.m b/tools/v2_query_plan.m new file mode 100644 index 0000000000..46314783cb --- /dev/null +++ b/tools/v2_query_plan.m @@ -0,0 +1,150 @@ +function T = v2_query_plan(datasetPath, options) +%V2_QUERY_PLAN Why a search through an ndi.dataset is slower than through its session. +% +% T = v2_query_plan(DATASETPATH) takes one worm from a session of the V2 +% dataset at DATASETPATH and runs the query its statements() makes (isa +% subject_statement AND depends_on subject_id = the worm) three ways: +% +% dataset the query as the ndi.dataset sends it +% session the same AND base.session_id = the session (what an +% ndi.session adds to every search) +% by edge hand-written SQL that starts from the depends_on index +% (the edge) and only then checks the class: what the plan +% would be if SQLite chose the edge first +% +% For each it prints the SQL, SQLite's EXPLAIN QUERY PLAN, the number of +% hits and the time of the SQL alone (median of 'Repeats' runs); and the +% time of the same search through ndi.dataset/database_search and +% ndi.session/database_search, which adds reading each document. T is a +% table of the timings. +% +% It opens a second, read-only use of the dataset's database file and +% writes nothing. Needs did2.database.sqlitedb's testHookExplain +% (DID-matlab PR #218). +% +% Options: +% 'Session' the session reference (default: the first concentration one) +% 'Repeats' timing runs per query (default 3) +% +% See also v2_object_layer_tryout, did2.database.compileQuery. + +arguments + datasetPath (1,:) char {mustBeFolder} + options.Session (1,:) char = '' + options.Repeats (1,1) double {mustBePositive, mustBeInteger} = 3 +end + +D = 'ndi.database.implementations.database.did2sqlite'; +dbFile = fullfile(datasetPath, '.ndi', feval([D '.DEFAULTFILENAME'])); + +% ---- the dataset, a session, a worm --------------------------------------------- +ds = ndi.dataset.dir(datasetPath); +[refs, ids] = ds.session_list(); +ref = options.Session; +if isempty(ref), ref = refs{find(startsWith(refs, 'concentration'), 1)}; end +S = ds.open_session(ids{strcmp(refs, ref)}); +sp = S.database_search(ndi.query('', 'isa', 'velocity_calculation', '')); +wormId = ndi.v2.edgeIds(ndi.v2.props(sp{1}), 'subject_id'); +wormId = wormId{1}; +fprintf('dataset %s\nsession %s (%s)\nworm %s\n', datasetPath, ref, S.id(), wormId); + +qBase = ndi.query('', 'isa', 'subject_statement', '') & ... + ndi.query('', 'depends_on', 'subject_id', wormId); +qSession = qBase & ndi.query('base.session_id', 'exact_string', S.id(), ''); + +db = did2.database.sqlitedb(dbFile); +closer = onCleanup(@() db.close()); +% the connection number is re-read before each use: ndi.dataset closes +% connections by number, and the database reopens on a new one (#218) +dbid = @() db.testHookDbId(); + +% ---- the size of each part -------------------------------------------------------- +n = @(sql, varargin) double(firstRow(mksqlite(dbid(), sql, varargin{:}))); +fprintf('\n== the parts ==\n'); +fprintf('DENOMINATOR: %d document(s) in the file\n', n('SELECT COUNT(*) AS n FROM documents')); +fprintf(' isa subject_statement: %d\n', ... + n('SELECT COUNT(*) AS n FROM superclasses WHERE classname = ?', 'subject_statement')); +fprintf(' depends_on subject_id = worm: %d\n', ... + n('SELECT COUNT(*) AS n FROM depends_on WHERE name = ? AND document_id = ?', 'subject_id', wormId)); +fprintf(' base.session_id = session: %d\n', ... + n('SELECT COUNT(*) AS n FROM documents WHERE session_id = ?', S.id())); +idx = mksqlite(dbid(), 'SELECT name, tbl_name FROM sqlite_master WHERE type = ''index'' ORDER BY tbl_name, name'); +fprintf(' indexes: %s\n', strjoin(arrayfun(@(r) sprintf('%s(%s)', r.name, r.tbl_name), idx, ... + 'UniformOutput', false), ', ')); +st = mksqlite(dbid(), 'SELECT name FROM sqlite_master WHERE name = ''sqlite_stat1'''); +fprintf(' ANALYZE statistics (sqlite_stat1): %s\n', yesno(~isempty(st))); + +% ---- each query --------------------------------------------------------------------- +rows = {}; +for form = ["dataset", "session"] + if form == "dataset", q = qBase; else, q = qSession; end + q2 = feval([D '.toDid2Query'], q); + r = db.testHookExplain(q2); + fprintf('\n== %s ==\nSQL:\n %s\nparams: %s\nplan:\n', form, r.sql, ... + strjoin(cellfun(@toText, r.params, 'UniformOutput', false), ' | ')); + printPlan(r.plan); + [t, hits] = timeIt(@() numel(db.searchIds(q2)), options.Repeats); + fprintf('SQL + did2 post-filter: %d hit(s), %.3f s (median of %d)\n', hits, t, options.Repeats); + rows(end+1, :) = {form, "did2 searchIds", hits, t}; %#ok +end + +% the same question, starting from the edge +sqlEdge = ['SELECT documents.id FROM depends_on d JOIN documents ON documents.id = d.doc_id ' ... + 'WHERE d.name = ? AND d.document_id = ? AND EXISTS (SELECT 1 FROM superclasses sc ' ... + 'WHERE sc.doc_id = documents.id AND sc.classname = ?) ORDER BY documents.rowid ASC']; +pEdge = {'subject_id', wormId, 'subject_statement'}; +fprintf('\n== by edge (hand-written) ==\nSQL:\n %s\nplan:\n', sqlEdge); +printPlan(mksqlite(dbid(), ['EXPLAIN QUERY PLAN ' sqlEdge], pEdge{:})); +[t, hits] = timeIt(@() numel(mksqlite(dbid(), sqlEdge, pEdge{:})), options.Repeats); +fprintf('SQL alone: %d hit(s), %.3f s (median of %d)\n', hits, t, options.Repeats); +rows(end+1, :) = {"by edge", "SQL alone", hits, t}; + +% ---- through NDI ---------------------------------------------------------------------- +fprintf('\n== through NDI (search + reading each document) ==\n'); +[t, hits] = timeIt(@() numel(ds.database_search(qBase)), options.Repeats); +fprintf('ndi.dataset/database_search: %d hit(s), %.3f s\n', hits, t); +rows(end+1, :) = {"dataset", "ndi database_search", hits, t}; +[t, hits] = timeIt(@() numel(S.database_search(qBase)), options.Repeats); +fprintf('ndi.session/database_search: %d hit(s), %.3f s\n', hits, t); +rows(end+1, :) = {"session", "ndi database_search", hits, t}; + +T = cell2table(rows, 'VariableNames', {'query', 'through', 'hits', 'seconds'}); +fprintf('\n== summary ==\n'); +disp(T); +end + +% ----------------------------------------------------------------------------------- +function printPlan(plan) +% EXPLAIN QUERY PLAN rows as a tree (indented by parent) +depth = containers.Map('KeyType', 'double', 'ValueType', 'double'); +depth(0) = 0; +for k = 1:numel(plan) + p = double(plan(k).parent); + d = 1; + if isKey(depth, p), d = depth(p) + 1; end + depth(double(plan(k).id)) = d; + fprintf(' %s%s\n', repmat(' ', 1, d - 1), plan(k).detail); +end +end + +function [t, out] = timeIt(fn, repeats) +times = zeros(1, repeats); +for k = 1:repeats + t0 = tic; + out = fn(); + times(k) = toc(t0); +end +t = median(times); +end + +function v = firstRow(rows) +v = rows(1).n; +end + +function s = toText(x) +if ischar(x) || isstring(x), s = char(x); else, s = mat2str(x); end +end + +function s = yesno(tf) +if tf, s = 'yes'; else, s = 'no (the planner has no table sizes)'; end +end diff --git a/tools/v2_subject_listing_profile.m b/tools/v2_subject_listing_profile.m new file mode 100644 index 0000000000..419c388143 --- /dev/null +++ b/tools/v2_subject_listing_profile.m @@ -0,0 +1,103 @@ +function T = v2_subject_listing_profile(datasetPath, options) +%V2_SUBJECT_LISTING_PROFILE Where the time goes when every subject of a dataset is listed. +% +% T = v2_subject_listing_profile(DATASETPATH) runs ndi.subject.search on the +% V2 dataset at DATASETPATH, then times each layer it goes through, one +% at a time, over the same subjects: +% +% sql the SELECT (id, body) for every `isa subject` document +% jsondecode decoding those bodies, nothing else +% fromJSON did2.document.fromJSON (decode + schema rehydration) +% matches did2's in-memory re-check of each document (q.matches) +% search did2 sqlitedb.search (the three above together) +% get by id did2 sqlitedb.get once per id (what NDI's backend does +% AFTER search, reading every document a second time) +% normalize ndi.database.internal.applyReadNormalization (the +% v1->V2 read pass NDI applies to every document read) +% ndi.document ndi.document(struct), the object NDI hands back +% database_search ndi.dataset/database_search, all of the above +% unique_id did.ido.unique_id (each new ndi.subject makes one, then +% ndi.subject.fromDocument overwrites it) +% fromDocument ndi.subject.fromDocument over the documents +% find ndi.subject.search, end to end +% +% It prints each layer's time, per document and as a share of `find`. T +% is the table. It writes nothing. +% +% Options: +% 'Limit' time the per-document layers over the first N subjects +% only and scale up (default: all of them) +% +% See also v2_query_plan, v2_object_layer_tryout. + +arguments + datasetPath (1,:) char {mustBeFolder} + options.Limit (1,1) double = Inf +end + +D = 'ndi.database.implementations.database.did2sqlite'; +dbFile = fullfile(datasetPath, '.ndi', feval([D '.DEFAULTFILENAME'])); +rows = {}; + function add(layer, seconds, n, note) + rows(end+1, :) = {string(layer), n, seconds, seconds / max(n, 1) * 1000, string(note)}; + fprintf(' %-17s %8.3f s %6d doc(s) %7.3f ms/doc %s\n', layer, seconds, n, ... + seconds / max(n, 1) * 1000, note); + end + +ds = ndi.dataset.dir(datasetPath); +fprintf('dataset %s\n\n', datasetPath); + +% ---- end to end first, cold ------------------------------------------------------- +t0 = tic; s = ndi.subject.search(ds); tFind = toc(t0); +N = numel(s); +fprintf('ndi.subject.search: %d subject(s), %.3f s\n\nDENOMINATOR: %d subject document(s)', N, tFind, N); +M = min(N, options.Limit); +scale = N / M; +if M < N, fprintf('; per-document layers timed over the first %d and scaled x%.1f', M, scale); end +fprintf('\n\n'); + +db = did2.database.sqlitedb(dbFile); +closer = onCleanup(@() db.close()); +cache = did2.schema.cache.shared(); +q2 = did2.query('', 'isa', 'subject'); + +% ---- did2 layers ------------------------------------------------------------------------ +fprintf('== did2 ==\n'); +sql = ['SELECT id, body FROM documents WHERE EXISTS (SELECT 1 FROM superclasses sc ' ... + 'WHERE sc.doc_id = documents.id AND sc.classname = ?) ORDER BY rowid ASC']; +t0 = tic; r = mksqlite(db.testHookDbId(), sql, 'subject'); add('sql', toc(t0), numel(r), 'one query'); +r = r(1:M); +t0 = tic; for k = 1:M, jsondecode(r(k).body); end; add('jsondecode', toc(t0) * scale, N, ''); +t0 = tic; docs = cell(1, M); +for k = 1:M, docs{k} = did2.document.fromJSON(r(k).body, 'SchemaCache', cache); end +add('fromJSON', toc(t0) * scale, N, 'decode + schema rehydration'); +t0 = tic; for k = 1:M, q2.matches(docs{k}); end; add('matches', toc(t0) * scale, N, 'in-memory re-check'); +t0 = tic; found = db.search(q2); add('search', toc(t0), numel(found), 'sql + fromJSON + matches'); +ids = cellfun(@(d) d.get('base.id'), found, 'UniformOutput', false); +t0 = tic; again = cell(1, M); for k = 1:M, again{k} = db.get(ids{k}); end +add('get by id', toc(t0) * scale, N, 'NDI reads every document a second time'); + +% ---- NDI layers --------------------------------------------------------------------------- +fprintf('== NDI ==\n'); +t0 = tic; nd = cell(1, M); +for k = 1:M, nd{k} = ndi.database.internal.applyReadNormalization(again{k}); end +add('normalize', toc(t0) * scale, N, 'v1->V2 read pass + ndi.document'); +structs = cellfun(@(d) d.toStruct(), again, 'UniformOutput', false); +t0 = tic; for k = 1:M, ndi.document(structs{k}); end +add('ndi.document', toc(t0) * scale, N, 'construction alone (inside normalize)'); +t0 = tic; sd = ds.database_search(ndi.query('', 'isa', 'subject', '')); +add('database_search', toc(t0), numel(sd), 'all of the above'); + +% ---- the object layer ---------------------------------------------------------------------- +fprintf('== object layer ==\n'); +t0 = tic; for k = 1:M, did.ido.unique_id(); end +add('unique_id', toc(t0) * scale, N, 'made, then overwritten'); +t0 = tic; for k = 1:M, ndi.subject.fromDocument(ds, sd{k}); end +add('fromDocument', toc(t0) * scale, N, ''); +t0 = tic; s = ndi.subject.search(ds); add('find', toc(t0), numel(s), 'end to end (warm)'); + +T = cell2table(rows, 'VariableNames', {'layer', 'documents', 'seconds', 'ms_per_doc', 'note'}); +T.share_of_find = T.seconds / T.seconds(end); +fprintf('\n== summary (share = seconds / warm find) ==\n'); +disp(T(:, {'layer', 'documents', 'seconds', 'ms_per_doc', 'share_of_find'})); +end