Skip to content

Commit e0ac92c

Browse files
claudeclaude
committed
Series ingest reconstruction: number members in the document's own base
reconstructSeriesIngestLocations rebuilt ingest_locations with index = manifest slot (1-based), i.e. the v1 numbering. DID names an ingested member sprintf('%s_%d', name, index) (sqlitedb/do_add_doc) and addFileSeries writes index as the member NUMBER in the document's base, so for a V_eta document (base 0, DID-schema #73 item 17, DID-matlab 1a6bd9f) every rebuilt entry named the next member up. The helper takes 'MemberBase' (default 1, v1 unchanged); both callers pass ndi.vintage.seriesMemberBase(document). Tests add the V_eta case (members 0,2,4 for slots 1,3,5, same uids) beside an explicit v1 case, and test-import-v2.yml now runs ReconstructSeriesIngestLocationsTest. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FxKLA8Tbf9cpvDe7fuVeKf
1 parent b6ea5a9 commit e0ac92c

5 files changed

Lines changed: 102 additions & 11 deletions

File tree

‎.github/workflows/test-import-v2.yml‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,10 @@ on:
5454
- 'tests/+ndi/+unittest/+gui/+app/TestLightsheetZarrManager.m'
5555
- 'src/ndi/+ndi/+database/+implementations/+database/didsqlite.m'
5656
- 'tests/+ndi/+unittest/+database/TestV1SubjectSearch.m'
57+
- 'src/ndi/+ndi/+cloud/+sync/+internal/reconstructSeriesIngestLocations.m'
58+
- 'src/ndi/+ndi/+cloud/+sync/+internal/updateFileInfoForLocalFiles.m'
59+
- 'src/ndi/+ndi/+cloud/+sync/+internal/updateFileInfoForRemoteFiles.m'
60+
- 'tests/+ndi/+unittest/ReconstructSeriesIngestLocationsTest.m'
5761
- '.github/workflows/test-import-v2.yml'
5862
pull_request:
5963
branches: [main, V2]
@@ -96,6 +100,10 @@ on:
96100
- 'tests/+ndi/+unittest/+gui/+app/TestLightsheetZarrManager.m'
97101
- 'src/ndi/+ndi/+database/+implementations/+database/didsqlite.m'
98102
- 'tests/+ndi/+unittest/+database/TestV1SubjectSearch.m'
103+
- 'src/ndi/+ndi/+cloud/+sync/+internal/reconstructSeriesIngestLocations.m'
104+
- 'src/ndi/+ndi/+cloud/+sync/+internal/updateFileInfoForLocalFiles.m'
105+
- 'src/ndi/+ndi/+cloud/+sync/+internal/updateFileInfoForRemoteFiles.m'
106+
- 'tests/+ndi/+unittest/ReconstructSeriesIngestLocationsTest.m'
99107
- '.github/workflows/test-import-v2.yml'
100108
workflow_dispatch:
101109

@@ -233,7 +241,8 @@ jobs:
233241
TestSuite.fromClass(?ndi.unittest.fun.doc.lightsheet.TestMakePyramid), ...
234242
TestSuite.fromClass(?ndi.unittest.fun.doc.lightsheet.TestFromSPIM), ...
235243
TestSuite.fromClass(?ndi.unittest.gui.app.TestLightsheetZarrManager), ...
236-
TestSuite.fromClass(?ndi.unittest.database.TestV1SubjectSearch)];
244+
TestSuite.fromClass(?ndi.unittest.database.TestV1SubjectSearch), ...
245+
TestSuite.fromClass(?ndi.unittest.ReconstructSeriesIngestLocationsTest)];
237246
results = run(suite);
238247
disp(table(results));
239248
nRun = numel(results);

‎src/ndi/+ndi/+cloud/+sync/+internal/reconstructSeriesIngestLocations.m‎

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
1-
function seriesInfo = reconstructSeriesIngestLocations(seriesInfo, fileInfo, fileDirectory, cloudDatasetId)
1+
function seriesInfo = reconstructSeriesIngestLocations(seriesInfo, fileInfo, fileDirectory, cloudDatasetId, options)
22
%RECONSTRUCTSERIESINGESTLOCATIONS Rebuild series ingest_locations from a downloaded manifest.
33
%
44
% SERIESINFO = ndi.cloud.sync.internal.reconstructSeriesIngestLocations( ...
55
% SERIESINFO, FILEINFO, FILEDIRECTORY, CLOUDDATASETID)
6+
% SERIESINFO = ndi.cloud.sync.internal.reconstructSeriesIngestLocations( ...
7+
% ..., 'MemberBase', BASE)
68
%
79
% For every series in SERIESINFO that declares `n_present > 0` and
810
% carries no `ingest_locations`, this reads the downloaded manifest file
@@ -42,6 +44,17 @@
4244
% `location`, matching the URI scheme
4345
% `download_file_from_cloud` understands.
4446
%
47+
% Options:
48+
% MemberBase - The number of the series' FIRST member, read off
49+
% the DOCUMENT'S VINTAGE with
50+
% ndi.vintage.seriesMemberBase(document): 1 for a v1
51+
% document (NAME_1 is manifest slot 0), 0 for a
52+
% V_eta document (NAME_0 is slot 0; DID-schema #73
53+
% item 17, DID-matlab 1a6bd9f). Default 1, the v1
54+
% behaviour. Callers holding the document must pass
55+
% it: the default names the wrong members of a V_eta
56+
% document.
57+
%
4558
% Outputs:
4659
% seriesInfo - The updated struct array.
4760
%
@@ -65,6 +78,7 @@
6578
fileInfo
6679
fileDirectory (1,1) string
6780
cloudDatasetId (1,1) string
81+
options.MemberBase (1,1) double {mustBeMember(options.MemberBase, [0 1])} = 1
6882
end
6983

7084
if isempty(seriesInfo), return, end
@@ -92,7 +106,7 @@
92106
continue
93107
end
94108

95-
ingestLocations = buildIngestLocations(manifest, cloudDatasetId);
109+
ingestLocations = buildIngestLocations(manifest, cloudDatasetId, options.MemberBase);
96110
if isempty(ingestLocations), continue, end
97111

98112
seriesInfo(k).ingest_locations = ingestLocations;
@@ -112,10 +126,14 @@
112126
end
113127
end
114128

115-
function ingestLocations = buildIngestLocations(manifest, cloudDatasetId)
129+
function ingestLocations = buildIngestLocations(manifest, cloudDatasetId, memberBase)
116130
% One ingest_locations entry per present slot of the manifest.
117-
% Order matches DID's authoring shape and its `sprintf('%s_%d', name, index)`
118-
% naming (index is 1-based, same as manifest.uids{index}).
131+
% `index` is the member NUMBER as named, not the manifest position: DID
132+
% names the member `sprintf('%s_%d', name, index)` when it ingests it
133+
% (did.implementations.sqlitedb/do_add_doc), and did.document/addFileSeries
134+
% writes `index` as the number in the document's own base. So manifest
135+
% position `slot` (1-based, manifest.uids{slot}) is member number
136+
% slot - 1 + memberBase: slot for a v1 document, slot - 1 for a V_eta one.
119137
%
120138
% Preallocated + trimmed rather than grown: a lightsheet pyramid level's
121139
% manifest names tens of thousands of members, and growing the struct
@@ -137,7 +155,7 @@
137155
memberUid = manifest.uids{slot};
138156
if isempty(memberUid), continue, end
139157
kept = kept + 1;
140-
entries(kept).index = slot;
158+
entries(kept).index = slot - 1 + memberBase;
141159
entries(kept).uid = memberUid;
142160
entries(kept).location = sprintf('ndic://%s/%s', cloudDatasetIdChar, memberUid);
143161
entries(kept).location_type = 'ndicloud';

‎src/ndi/+ndi/+cloud/+sync/+internal/updateFileInfoForLocalFiles.m‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,8 @@
108108
if hasSeriesInfo
109109
if strlength(cloudDatasetId) > 0
110110
originalSeriesInfo = ndi.cloud.sync.internal.reconstructSeriesIngestLocations( ...
111-
originalSeriesInfo, originalFileInfo, fileDirectory, cloudDatasetId);
111+
originalSeriesInfo, originalFileInfo, fileDirectory, cloudDatasetId, ...
112+
'MemberBase', ndi.vintage.seriesMemberBase(document));
112113
end
113114
document = document.setproperties('files.series_info', originalSeriesInfo);
114115
end

‎src/ndi/+ndi/+cloud/+sync/+internal/updateFileInfoForRemoteFiles.m‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@
126126
end
127127
seriesInfo = reconstructFromCloud(seriesInfo, updatedFileInfo, ...
128128
cloudDatasetId, documentId, options.customFileHandler, ...
129-
options.uploadedUidMap);
129+
options.uploadedUidMap, ndi.vintage.seriesMemberBase(document));
130130
document = document.setproperties('files.series_info', seriesInfo);
131131
end
132132
end
@@ -155,7 +155,7 @@
155155

156156

157157
function seriesInfo = reconstructFromCloud(seriesInfo, fileInfo, ...
158-
cloudDatasetId, documentId, customFileHandler, uploadedUidMap)
158+
cloudDatasetId, documentId, customFileHandler, uploadedUidMap, memberBase)
159159
% Fetch each qualifying series' manifest bytes into a scratch dir,
160160
% reconstruct ingest_locations from them via reconstructSeriesIngestLocations,
161161
% then delete the scratch dir. Manifests do NOT land in the DID file
@@ -241,7 +241,7 @@
241241
end
242242

243243
seriesInfo = ndi.cloud.sync.internal.reconstructSeriesIngestLocations( ...
244-
seriesInfo, fileInfo, tmpDir, cloudDatasetId);
244+
seriesInfo, fileInfo, tmpDir, cloudDatasetId, 'MemberBase', memberBase);
245245
end
246246

247247

‎tests/+ndi/+unittest/ReconstructSeriesIngestLocationsTest.m‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,69 @@ function testReconstructsOneEntryPerPresentSlot(testCase)
8787
end
8888
end
8989

90+
function testV1DocumentNumbersMembersFromOne(testCase)
91+
% A v1 document (MemberBase 1, the default): member NAME_1 is
92+
% manifest slot 0, so `index` equals the 1-based manifest
93+
% position. Passing the base explicitly gives exactly what the
94+
% default gives -- v1 behaviour is unchanged.
95+
[manifestUid, uids] = testCase.writeManifest(5, [1 3 5]);
96+
fi = testCase.fileInfoFor('chunkdata.bin', manifestUid);
97+
si = testCase.seriesInfoFor('chunkdata.bin', 5, 3);
98+
99+
outDefault = ndi.cloud.sync.internal.reconstructSeriesIngestLocations( ...
100+
si, fi, testCase.Dir, "ds");
101+
outV1 = ndi.cloud.sync.internal.reconstructSeriesIngestLocations( ...
102+
si, fi, testCase.Dir, "ds", 'MemberBase', 1);
103+
104+
testCase.verifyEqual([outV1(1).ingest_locations.index], [1 3 5]);
105+
testCase.verifyEqual({outV1(1).ingest_locations.uid}, uids([1 3 5]));
106+
testCase.verifyEqual(outV1, outDefault);
107+
end
108+
109+
function testVetaDocumentNumbersMembersFromZero(testCase)
110+
% A V_eta document (MemberBase 0; DID-schema #73 item 17,
111+
% DID-matlab 1a6bd9f): member NAME_0 is manifest slot 0. `index`
112+
% is the number DID puts in the member's name
113+
% (sprintf('%s_%d', name, index) in sqlitedb/do_add_doc), so the
114+
% same manifest slots 1,3,5 (1-based) are members 0,2,4 -- with
115+
% the SAME uids, because the manifest does not depend on vintage.
116+
[manifestUid, uids] = testCase.writeManifest(5, [1 3 5]);
117+
fi = testCase.fileInfoFor('chunkdata.bin', manifestUid);
118+
si = testCase.seriesInfoFor('chunkdata.bin', 5, 3);
119+
120+
out = ndi.cloud.sync.internal.reconstructSeriesIngestLocations( ...
121+
si, fi, testCase.Dir, "ds", 'MemberBase', 0);
122+
123+
il = out(1).ingest_locations;
124+
testCase.assertNumElements(il, 3);
125+
testCase.verifyEqual([il.index], [0 2 4]);
126+
testCase.verifyEqual({il.uid}, uids([1 3 5]));
127+
end
128+
129+
function testMemberBaseReadFromDocumentVintage(testCase)
130+
% The callers read the base with ndi.vintage.seriesMemberBase
131+
% from the document itself. A V_eta schema_version gives 0, a
132+
% document without one gives 1, and feeding that into the
133+
% reconstruction names the first present member accordingly.
134+
[manifestUid, ~] = testCase.writeManifest(2, [1 2]);
135+
fi = testCase.fileInfoFor('body_data', manifestUid);
136+
si = testCase.seriesInfoFor('body_data', 2, 2);
137+
138+
veta = struct('document_class', struct('schema_version', 'V_eta'));
139+
v1 = struct('document_class', struct('class_name', 'x'));
140+
testCase.verifyEqual(ndi.vintage.seriesMemberBase(veta), 0);
141+
testCase.verifyEqual(ndi.vintage.seriesMemberBase(v1), 1);
142+
143+
outVeta = ndi.cloud.sync.internal.reconstructSeriesIngestLocations( ...
144+
si, fi, testCase.Dir, "ds", ...
145+
'MemberBase', ndi.vintage.seriesMemberBase(veta));
146+
outV1 = ndi.cloud.sync.internal.reconstructSeriesIngestLocations( ...
147+
si, fi, testCase.Dir, "ds", ...
148+
'MemberBase', ndi.vintage.seriesMemberBase(v1));
149+
testCase.verifyEqual([outVeta(1).ingest_locations.index], [0 1]);
150+
testCase.verifyEqual([outV1(1).ingest_locations.index], [1 2]);
151+
end
152+
90153
function testLeavesExistingIngestLocationsAlone(testCase)
91154
% A series that already carries ingest_locations (a doc built
92155
% locally and never round-tripped, say) is not touched. The

0 commit comments

Comments
 (0)