Skip to content

Commit 05067dd

Browse files
committed
V2 import and readers work on one entity class and composed statements
did-schema V_eta_entity_composition_plan.md (a proposal, PR #88) merges 13 entity classes into one `entity` with a `type` and makes a statement's value kind a mixin. The import and the readers now handle both schemas. Writers - ndi.setup.V2.entityDocuments: an entity as the schema in use wants it -- the old class, or an `entity` plus term/text/date assertions for its other fields and relations for its other edges (vendor_id -> sold_by, background_strain_id and product_id -> derived_from). Unknown fields are an error, not dropped. - ndi.setup.V2.curie: spec identifiers -> CURIEs (orcid:, ror:, doi:, rrid:, wormbase:, ...) or IRIs; an award number becomes the funding's local_identifier. - datasetMetadata, createSession, createDataset, sessionDocuments, densityDocuments and import_V2 build through it. A web resource the dataset is stored_at becomes the dataset's `ndicloud:` identifier; the others are entities of the type the spec now gives them (code -> software, WormBook -> publication). - A strain's WormBase id is `wormbase:WBStrain...` on the strain entity, and a cohort is `instance_of` its strain instead of asserting it. - schemaHasField falls back to `entity` for merged classes; schemaHasLeaf understands composed leaves (time_calculation). Readers - ndi.v2.isaQuery also matches entities by type ('subject' -> the physical types, 'session' -> type session) and composed leaves ('temperature_observation' -> observation + temperature); blockOf falls back to the `entity` block; ndi.v2.kindOf names an entity by its type. - ndi.session.dir, ndi.dataset.dir, ndi.dataset, the bodies navigator, ndi.entity and ndi.subject use them. CI: test-import-v2 runs a second leg on did-schema and DID-matlab claude/ecstatic-pascal-md91fw, so both shapes are tested until those merge. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FxKLA8Tbf9cpvDe7fuVeKf
1 parent f3c36b0 commit 05067dd

27 files changed

Lines changed: 597 additions & 75 deletions

‎.github/workflows/test-import-v2.yml‎

Lines changed: 30 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -83,11 +83,25 @@ concurrency:
8383

8484
jobs:
8585
import-v2:
86-
name: V2 import stages
86+
name: V2 import stages (${{ matrix.schema }})
8787
runs-on: ubuntu-latest
88+
strategy:
89+
fail-fast: false
90+
matrix:
91+
# current: did-schema main with DID-matlab V2 (from requirements.txt).
92+
# next: the one-entity-class / composition schema and the did2.build that
93+
# reads it, both still on review branches (did-schema PR #88, DID-matlab
94+
# PR #218). The import and the readers build and read both shapes, so both
95+
# legs must pass. Drop `next` once both merge.
96+
include:
97+
- schema: current
98+
did_schema_ref: main
99+
did_matlab_ref: ''
100+
- schema: next
101+
did_schema_ref: claude/ecstatic-pascal-md91fw
102+
did_matlab_ref: claude/ecstatic-pascal-md91fw
88103
env:
89-
# DID-matlab comes from requirements.txt, which already pins @V2 (did2.build).
90-
DID_SCHEMA_REF: main
104+
DID_SCHEMA_REF: ${{ matrix.did_schema_ref }}
91105
steps:
92106
- name: Check out NDI-matlab
93107
uses: actions/checkout@v4
@@ -108,6 +122,14 @@ jobs:
108122
echo "DENOMINATOR: $(ls "${GITHUB_WORKSPACE}/eta-schemas" | wc -l) schema file(s) from did-schema@$(git -C did-schema rev-parse --short HEAD)"
109123
echo "DID_SCHEMA_PATH=${GITHUB_WORKSPACE}/eta-schemas" >> "$GITHUB_ENV"
110124
125+
- name: Check out DID-matlab (next leg only)
126+
if: ${{ matrix.did_matlab_ref != '' }}
127+
uses: actions/checkout@v4
128+
with:
129+
repository: VH-Lab/DID-matlab
130+
ref: ${{ matrix.did_matlab_ref }}
131+
path: did-matlab-next
132+
111133
- name: Set up MATLAB
112134
uses: matlab-actions/setup-matlab@v2
113135
with:
@@ -140,6 +162,11 @@ jobs:
140162
command: |
141163
addpath(genpath("src"));
142164
addpath(genpath("tests"));
165+
if isfolder("did-matlab-next/src")
166+
% ahead of the installed DID-matlab V2 on the path
167+
addpath(genpath(fullfile(pwd, "did-matlab-next", "src")));
168+
disp("DID-matlab from did-matlab-next (" + string(which("did2.build.document")) + ")");
169+
end
143170
import matlab.unittest.TestSuite;
144171
suite = [TestSuite.fromPackage("ndi.unittest.setup.V2", "IncludingSubpackages", true), ...
145172
TestSuite.fromClass(?ndi.unittest.vintage.TestVintageMap)];

‎src/ndi/+ndi/+dataset/dir.m‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -61,13 +61,13 @@
6161
% document (a class v1 never had). Read unscoped: the
6262
% folder's unique_reference.txt names whichever session was
6363
% opened last, not necessarily the dataset.
64-
datasetDocs = ndi_dataset_dir_obj.session.database.search(ndi.query('','isa','dataset'));
65-
datasetDocs = datasetDocs(cellfun(@(d) strcmp(d.document_properties.document_class.class_name, ...
66-
'dataset'), datasetDocs));
64+
datasetDocs = ndi_dataset_dir_obj.session.database.search(ndi.v2.isaQuery('dataset'));
65+
datasetDocs = datasetDocs(cellfun(@(d) strcmp(ndi.v2.kindOf(d.document_properties), ...
66+
'dataset'), datasetDocs)); % an entity of type dataset since 2026-10-08
6767
if isscalar(datasetDocs)
6868
correctSessionId = datasetDocs{1}.document_properties.base.session_id;
6969
else
70-
q_session = ndi.query('','isa','session');
70+
q_session = ndi.v2.isaQuery('session');
7171
candidate_session_doc = ndi_dataset_dir_obj.database_search(q_session);
7272
if isscalar(candidate_session_doc)
7373
correctSessionId = candidate_session_doc{1}.document_properties.base.session_id;
@@ -77,7 +77,7 @@
7777
end
7878

7979
if ~isempty(correctSessionId)
80-
q_session = ndi.query('','isa','session') & ndi.query('base.session_id','exact_string',correctSessionId);
80+
q_session = ndi.v2.isaQuery('session') & ndi.query('base.session_id','exact_string',correctSessionId);
8181
candidate_session_doc = ndi_dataset_dir_obj.database_search(q_session);
8282
if isscalar(candidate_session_doc)
8383
% BOTH VINTAGES. V_eta renamed session.reference ->

‎src/ndi/+ndi/+file/+navigator/bodies.m‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -195,7 +195,7 @@
195195
end
196196
referent = d{1}.dependency_value('referent_id', 'ErrorIfNotFound', 0);
197197
e = S.database_search(ndi.query('base.id', 'exact_string', referent, '') & ...
198-
ndi.query('', 'isa', 'epoch', ''));
198+
ndi.v2.isaQuery('epoch'));
199199
if numel(e) == 1
200200
id = e{1}.document_properties.epoch.local_identifier;
201201
return;

‎src/ndi/+ndi/+session/dir.m‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@
104104
read_from_database = 0;
105105

106106
if should_we_try_to_read_from_database
107-
session_doc = ndi_session_dir_obj.database_search(ndi.query('','isa','session'));
107+
session_doc = ndi_session_dir_obj.database_search(ndi.v2.isaQuery('session'));
108108
if ~isempty(session_doc)
109109
% use the oldest
110110
time_diff_max = 0;
@@ -140,7 +140,10 @@
140140
% 2026-08-13). The OBJECT property stays `reference`: that is
141141
% ndi.session's own public API, read at 20 call sites, and it
142142
% is not what the migration renamed.
143-
session_blk = session_doc.document_properties.session;
143+
% and since 2026-10-08 the session may be an `entity` of type
144+
% session, whose block is `entity` (ndi.v2.kindOf)
145+
[~, session_blk_name] = ndi.v2.kindOf(session_doc.document_properties);
146+
session_blk = session_doc.document_properties.(session_blk_name);
144147
if isfield(session_blk,'local_identifier')
145148
ndi_session_dir_obj.reference = session_blk.local_identifier;
146149
else

‎src/ndi/+ndi/+setup/+V2/createDataset.m‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,8 @@
7272

7373
f = struct('local_identifier', reference);
7474
if ~isempty(options.Name), f.name = options.Name; end
75-
own = did2.build.document('session', f, 'SessionId', datasetSessionId);
75+
own = ndi.setup.V2.entityDocuments('session', f, struct(), 'SessionId', datasetSessionId);
76+
own = own{1};
7677
for k = 1:numel(datasetDocs)
7778
if ~strcmp(datasetDocs{k}.base.session_id, datasetSessionId)
7879
error('ndi:setup:V2:wrongSession', ...
@@ -121,7 +122,7 @@
121122
function report = check(docs, dsid)
122123
n = numel(docs);
123124
ids = cellfun(@(d) d.base.id, docs, 'UniformOutput', false);
124-
cls = cellfun(@(d) d.document_class.class_name, docs, 'UniformOutput', false);
125+
cls = cellfun(@ndi.setup.V2.kindOf, docs, 'UniformOutput', false); % an entity's type since 2026-10-08
125126
sids = cellfun(@(d) d.base.session_id, docs, 'UniformOutput', false);
126127
[u, ~, j] = unique(ids);
127128
report.duplicateIds = u(accumarray(j(:), 1) > 1);
@@ -176,7 +177,8 @@
176177
end
177178
end
178179
if ~ok
179-
report.sessions.notInDataset{end+1} = docs{i}.session.local_identifier;
180+
[~, blk] = ndi.v2.kindOf(docs{i});
181+
report.sessions.notInDataset{end+1} = docs{i}.(blk).local_identifier;
180182
end
181183
end
182184
end

‎src/ndi/+ndi/+setup/+V2/createSession.m‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -89,14 +89,12 @@
8989
if ~isempty(options.Description)
9090
fields.description = options.Description;
9191
end
92-
args = {'SessionId', sid};
93-
if ~isempty(options.SessionDocId)
94-
args = [args, {'Id', options.SessionDocId}];
95-
end
92+
edges = struct();
9693
if ~isempty(options.TimeReferenceId)
97-
args = [args, {'Edges', struct('time_reference_id', options.TimeReferenceId)}];
94+
edges.time_reference_id = options.TimeReferenceId;
9895
end
99-
docs = {did2.build.document('session', fields, args{:})};
96+
docs = ndi.setup.V2.entityDocuments('session', fields, edges, 'SessionId', sid, ...
97+
'Id', options.SessionDocId);
10098
studies = options.StudyIds;
10199
if ischar(studies) || isstring(studies)
102100
studies = cellstr(studies);

‎src/ndi/+ndi/+setup/+V2/curie.m‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
function [out, local] = curie(identifiers)
2+
%CURIE Spec identifiers as V_eta global identifiers (CURIE or IRI strings).
3+
%
4+
% [OUT, LOCAL] = ndi.setup.V2.curie(IDENTIFIERS) turns a spec's
5+
% `identifiers`, a list of {scheme, value}, into the strings
6+
% `entity.global_identifier` holds on a schema built on or after 2026-10-08:
7+
% a CURIE whose prefix is registered in CURIE_lookups_meta.json
8+
% ('orcid:0000-0001-6282-7124', 'doi:10.7554/eLife...', 'ror:03xez1567',
9+
% 'rrid:SCR_015773'), or the full IRI for a URL. A value given as its
10+
% resolver's address ('https://orcid.org/...') is reduced to the local id.
11+
%
12+
% An award number is not a global identifier -- it is unique only within its
13+
% funder -- so it is returned as LOCAL ('' if none), for the funding entity's
14+
% local_identifier (V_eta_entity_composition_plan.md sec. 6).
15+
%
16+
% Errors ndi:setup:V2:unknownScheme for a scheme with no prefix.
17+
18+
out = {};
19+
local = '';
20+
if isempty(identifiers)
21+
return;
22+
end
23+
if isstruct(identifiers)
24+
identifiers = num2cell(identifiers);
25+
end
26+
resolvers = {'orcid', 'https://orcid.org/'; 'ror', 'https://ror.org/'; ...
27+
'doi', 'https://doi.org/'; 'doi', 'http://dx.doi.org/'; ...
28+
'pubmed', 'https://pubmed.ncbi.nlm.nih.gov/'; ...
29+
'wikidata', 'https://www.wikidata.org/wiki/'};
30+
for k = 1:numel(identifiers)
31+
id = identifiers{k};
32+
scheme = id.scheme;
33+
if isstruct(scheme)
34+
scheme = scheme.name;
35+
end
36+
scheme = lower(char(scheme));
37+
value = strtrim(char(id.value));
38+
switch scheme
39+
case 'url'
40+
out{end+1} = value; %#ok<AGROW>
41+
continue;
42+
case 'awardnumber'
43+
local = value;
44+
continue;
45+
case {'orcid', 'ror', 'doi', 'rrid', 'wikidata', 'swh', 'ndicloud', 'wormbase'}
46+
prefix = scheme;
47+
case 'pmid'
48+
prefix = 'pubmed';
49+
case 'pmcid'
50+
prefix = 'pmc';
51+
case 'swhid'
52+
prefix = 'swh';
53+
otherwise
54+
error('ndi:setup:V2:unknownScheme', ...
55+
'Identifier scheme "%s" has no registered prefix.', scheme);
56+
end
57+
for r = 1:size(resolvers, 1)
58+
if strcmp(resolvers{r, 1}, prefix) && startsWith(value, resolvers{r, 2}, 'IgnoreCase', true)
59+
value = value(numel(resolvers{r, 2}) + 1:end);
60+
end
61+
end
62+
if startsWith(value, [prefix ':'], 'IgnoreCase', true)
63+
value = value(numel(prefix) + 2:end);
64+
end
65+
out{end+1} = [prefix ':' value]; %#ok<AGROW>
66+
end
67+
end

‎src/ndi/+ndi/+setup/+V2/datasetMetadata.m‎

Lines changed: 89 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -111,12 +111,38 @@
111111
state = add(state, 'publication', p.key, f, struct());
112112
end
113113

114+
% On a schema with one entity class (2026-10-08) there is no web_resource:
115+
% a URL is an address for the thing it names. A resource the dataset is
116+
% `stored_at` becomes one of the dataset's own identifiers; any other one is
117+
% an entity of the `type` the spec gives it (software, publication, ...).
118+
merged = ndi.setup.V2.mergedEntities();
119+
storedAt = asCell(getOr(getOr(spec, 'dataset', struct()), 'stored_at', {}));
120+
state.storedIds = {};
114121
for e = entries(spec, 'web_resources')
115122
w = e{1};
116-
state = checkKnown(state, 'web_resource', w, {'name', 'identifiers', 'host'});
117-
state = add(state, 'web_resource', w.key, putIds(struct('name', w.name), w), struct());
118-
if isfield(w, 'host')
119-
state = relate(state, w.key, w.host, 'hosted_by', {'web_resource'}, {'organization'});
123+
if ~merged
124+
state = checkKnown(state, 'web_resource', w, {'name', 'identifiers', 'host'}, ...
125+
struct('type', ''));
126+
state = add(state, 'web_resource', w.key, putIds(struct('name', w.name), w), struct());
127+
if isfield(w, 'host')
128+
state = relate(state, w.key, w.host, 'hosted_by', {'web_resource'}, {'organization'});
129+
end
130+
elseif any(strcmp(w.key, storedAt))
131+
state = checkKnown(state, 'web_resource', w, {'name', 'identifiers'}, ...
132+
struct('host', 'a stored copy is an identifier of the dataset; its host is not recorded', ...
133+
'type', ''));
134+
state.storedIds = [state.storedIds, storedIdentifiers(w)];
135+
state.ids(w.key) = '';
136+
state.kinds(w.key) = 'web_resource';
137+
else
138+
state = checkKnown(state, 'web_resource', w, {'name', 'identifiers', 'type'}, ...
139+
struct('host', 'the schema in use has no relation from a resource to its host'));
140+
if ~isfield(w, 'type')
141+
error('ndi:setup:V2:noType', ['web resource `%s` needs a `type` ' ...
142+
'(software, publication, protocol, ...) on a schema with one entity class.'], w.key);
143+
end
144+
state = add(state, 'web_resource', w.key, putIds(struct('name', w.name), w), ...
145+
struct(), '', char(w.type));
120146
end
121147
end
122148

@@ -158,6 +184,14 @@
158184
if ~isempty(desc)
159185
f.description = desc;
160186
end
187+
if ndi.setup.V2.mergedEntities() && isfield(s, 'node') ...
188+
&& startsWith(char(s.node), 'WBStrain:')
189+
% the strain's WormBase id is one of its identifiers (one entity class;
190+
% `wormbase:WBStrain00000001`, V_eta_entity_composition_plan.md sec. 6),
191+
% since a subject is `instance_of` the strain rather than asserting it
192+
wb = struct('scheme', 'WormBase', 'value', strrep(char(s.node), ':', ''));
193+
s.identifiers = [asCell(getOr(s, 'identifiers', {})), {wb}];
194+
end
161195
f = putIds(f, s);
162196
edges = struct();
163197
if isfield(s, 'source')
@@ -253,7 +287,7 @@
253287
if subjectHasName
254288
f = putIf(f, 'name', i, 'name');
255289
end
256-
state = add(state, 'subject', i.key, f, struct());
290+
state = add(state, 'subject', i.key, f, struct(), '', 'device');
257291
if isfield(i, 'product')
258292
state = relate(state, i.key, i.product, 'instance_of', {'subject'}, {'product'});
259293
end
@@ -276,6 +310,15 @@
276310
f = putIf(f, fn{1}, d, fn{1});
277311
end
278312
f = putIds(f, d);
313+
if isfield(state, 'storedIds') && ~isempty(state.storedIds)
314+
% a stored copy is one of the dataset's addresses (one entity class only)
315+
extra = cellfun(@(v) struct('scheme', 'URL', 'value', v), state.storedIds, ...
316+
'UniformOutput', false);
317+
if isfield(f, 'global_identifier')
318+
extra = [num2cell(f.global_identifier(:)'), extra];
319+
end
320+
f.global_identifier = did2.build.list(extra{:});
321+
end
279322
key = '#dataset';
280323
state = add(state, 'dataset', key, f, struct(), options.DatasetId);
281324

@@ -288,8 +331,12 @@
288331
end
289332
state = relate(state, key, a.person, 'has_author', {'dataset'}, {'person'}, extra);
290333
end
291-
for pair = {'funding', 'funded_by', {'funding'}; 'cites', 'cites', {'publication'}; ...
292-
'stored_at', 'stored_at', {'web_resource'}; 'documented_by', 'documented_by', {'web_resource'}}'
334+
pairs = {'funding', 'funded_by', {'funding'}; 'cites', 'cites', {'publication'}; ...
335+
'stored_at', 'stored_at', {'web_resource'}; 'documented_by', 'documented_by', {'web_resource'}}';
336+
if ndi.setup.V2.mergedEntities()
337+
pairs = pairs(:, ~strcmp(pairs(1, :), 'stored_at')); % now the dataset's identifiers
338+
end
339+
for pair = pairs
293340
for t = asCell(getOr(d, pair{1}, {}))
294341
state = relate(state, key, t{1}, pair{2}, {'dataset'}, pair{3});
295342
end
@@ -436,20 +483,50 @@
436483
id = state.ids(key);
437484
end
438485

439-
function state = add(state, className, key, fields, edges, id)
486+
function state = add(state, className, key, fields, edges, id, type)
487+
% TYPE: the entity type, for a subject (or web resource) on a schema with one
488+
% entity class; ndi.setup.V2.entityDocuments.
440489
if nargin < 6
441490
id = '';
442491
end
492+
if nargin < 7
493+
type = '';
494+
end
443495
if isKey(state.ids, key)
444496
error('ndi:setup:V2:duplicateKey', 'The spec defines `%s` twice.', key);
445497
end
446-
doc = did2.build.document(className, fields, 'SessionId', state.sid, ...
447-
'Edges', edges, 'Id', id, 'Validate', state.validate);
448-
state.docs{end+1} = doc;
449-
state.ids(key) = doc.base.id;
498+
entityKinds = {'organization', 'person', 'funding', 'publication', 'web_resource', ...
499+
'software', 'product', 'strain', 'subject', 'dataset', 'study'};
500+
if any(strcmp(className, entityKinds))
501+
buildAs = className;
502+
if strcmp(className, 'web_resource') && ~isempty(type)
503+
buildAs = type; % on a schema with one entity class, what the resource is
504+
end
505+
docs = ndi.setup.V2.entityDocuments(buildAs, fields, edges, 'SessionId', state.sid, ...
506+
'Id', id, 'Validate', state.validate, 'Type', type);
507+
else
508+
docs = {did2.build.document(className, fields, 'SessionId', state.sid, ...
509+
'Edges', edges, 'Id', id, 'Validate', state.validate)};
510+
end
511+
state.docs = [state.docs, docs];
512+
state.ids(key) = docs{1}.base.id;
450513
state.kinds(key) = className;
451514
end
452515

516+
function ids = storedIdentifiers(w)
517+
% A stored copy's addresses: an NDI Cloud dataset page is `ndicloud:<id>`.
518+
ids = {};
519+
for x = asCell(getOr(w, 'identifiers', {}))
520+
v = char(x{1}.value);
521+
tok = regexp(v, 'ndi-cloud\.com/datasets/([0-9a-f]{24})', 'tokens', 'once');
522+
if ~isempty(tok)
523+
ids{end+1} = ['ndicloud:' tok{1}]; %#ok<AGROW>
524+
else
525+
ids{end+1} = v; %#ok<AGROW>
526+
end
527+
end
528+
end
529+
453530
function state = relate(state, childKey, parentKey, relation, childKinds, parentKinds, extra)
454531
if nargin < 7
455532
extra = struct();

0 commit comments

Comments
 (0)