|
| 1 | +name: V_eta migrate Dab (on-demand) |
| 2 | + |
| 3 | +# The Dab corpus (~27,561 v1 documents, ~35 MB compressed) through the WHOLE |
| 4 | +# V_eta pipeline: ndi.migrate.local over the real corpus, then open the result |
| 5 | +# through ndi.session.dir and read it back through the object API. |
| 6 | +# |
| 7 | +# WHY DAB HAS ITS OWN, DISPATCH-ONLY WORKFLOW rather than a line in |
| 8 | +# test-eta-migrate-e2e.yml. Dab is the WIDEST corpus (26 v1 classes) and the |
| 9 | +# only one that holds `ontologyTableRow` at scale (~6,205 docs). Its |
| 10 | +# decomposition -- ndi.migrate.internal.ontologyRowSubjects, the two-tier |
| 11 | +# subject fan-out (#53) -- is proven only by a fixture (TestOntologyRowSubjects) |
| 12 | +# and has never run over a real corpus. This workflow is the first full-corpus |
| 13 | +# exercise of that pass, plus the widest object read-back surface (ingestion |
| 14 | +# ~9,419, openMINDS ~2,344, syncrule_mapping ~2,484). Run alone, with a generous |
| 15 | +# timeout, and never on every push: it does not gate the fast e2e check, and a |
| 16 | +# Dab timeout/OOM here cannot turn that check red. Dab's MIGRATION (0 quarantine |
| 17 | +# / 0 orphans) is also gated independently on the DID-matlab side |
| 18 | +# (test-corpus.yml); what this workflow uniquely adds is the NDI second passes + |
| 19 | +# object-layer read-back on Dab's real documents. |
| 20 | +# |
| 21 | +# The DID-matlab, did-schema, and NDI-matlab V_eta branches share a name, so the |
| 22 | +# dep refs default to this run's branch (github.ref_name); override via the |
| 23 | +# workflow_dispatch input if needed. |
| 24 | + |
| 25 | +on: |
| 26 | + # Runs ONLY when Dab's own files change -- the test or this workflow -- so a |
| 27 | + # long Dab run never rides on an unrelated push and never gates the fast e2e |
| 28 | + # check. workflow_dispatch below only becomes available once this file reaches |
| 29 | + # the default branch (a GitHub constraint), so the scoped push is what runs |
| 30 | + # Dab from a feature branch until then. |
| 31 | + # |
| 32 | + # 2026-09-23: touched to re-run Dab on the ndi_document2ndi_object read-path |
| 33 | + # fix (V2-merge break) on the fixed head, validating against did-schema main. |
| 34 | + push: |
| 35 | + paths: |
| 36 | + - '.github/workflows/test-eta-migrate-dab.yml' |
| 37 | + - 'tests/*ndi/*unittest/*migrate/TestMigrateLocalEtaDab.m' |
| 38 | + workflow_dispatch: |
| 39 | + inputs: |
| 40 | + ref: |
| 41 | + description: 'did-matlab / did-schema branch (V_eta line)' |
| 42 | + required: false |
| 43 | + default: '' |
| 44 | + type: string |
| 45 | + |
| 46 | +concurrency: |
| 47 | + group: ${{ github.workflow }}-${{ github.ref }} |
| 48 | + cancel-in-progress: true |
| 49 | + |
| 50 | +jobs: |
| 51 | + eta_migrate_dab: |
| 52 | + name: V_eta Dab end-to-end (on-demand) |
| 53 | + runs-on: ubuntu-latest |
| 54 | + # Generous headroom. Dab is ~1/4 of Soph by document count and migrates fast |
| 55 | + # on the DID side; the unknown is the ontologyRowSubjects fan-out at scale, |
| 56 | + # so the ceiling is set well above the expected wall clock. |
| 57 | + timeout-minutes: 180 |
| 58 | + env: |
| 59 | + # DID-matlab's V_eta migrators merged to V2 (#151, 2026-09-23); validate |
| 60 | + # the cross-repo wiring against V2 so this resolves after the shared topic |
| 61 | + # branch is deleted. A workflow_dispatch input still overrides. |
| 62 | + DID_MATLAB_REF: ${{ inputs.ref || 'V2' }} |
| 63 | + # V_eta lives on did-schema `main` since #64 merged (2026-09-23) and the |
| 64 | + # shared topic branch was deleted; validate against main. |
| 65 | + DID_SCHEMA_REF: main |
| 66 | + steps: |
| 67 | + - name: Check out NDI-matlab |
| 68 | + uses: actions/checkout@v4 |
| 69 | + |
| 70 | + - name: Check out did-schema (for V_eta schemas) |
| 71 | + uses: actions/checkout@v4 |
| 72 | + with: |
| 73 | + repository: Waltham-Data-Science/did-schema |
| 74 | + ref: ${{ env.DID_SCHEMA_REF }} |
| 75 | + path: did-schema |
| 76 | + |
| 77 | + - name: Assemble V_eta schema set (stable + draft) |
| 78 | + run: | |
| 79 | + mkdir -p "${GITHUB_WORKSPACE}/eta-schemas" |
| 80 | + cp did-schema/schemas/V_eta/stable/*.json "${GITHUB_WORKSPACE}/eta-schemas/" |
| 81 | + cp did-schema/schemas/V_eta/draft/*.json "${GITHUB_WORKSPACE}/eta-schemas/" |
| 82 | +
|
| 83 | + - name: Export env (schema path + eta test gate) |
| 84 | + run: | |
| 85 | + echo "DID_SCHEMA_PATH=${GITHUB_WORKSPACE}/eta-schemas" >> "$GITHUB_ENV" |
| 86 | + echo "NDI_TEST_ETA=1" >> "$GITHUB_ENV" |
| 87 | +
|
| 88 | + - name: Start virtual display server |
| 89 | + run: | |
| 90 | + sudo apt-get install -y xvfb |
| 91 | + Xvfb :99 & |
| 92 | + echo "DISPLAY=:99" >> "$GITHUB_ENV" |
| 93 | +
|
| 94 | + - name: Set up MATLAB |
| 95 | + uses: matlab-actions/setup-matlab@v2 |
| 96 | + with: |
| 97 | + release: latest |
| 98 | + cache: true |
| 99 | + products: | |
| 100 | + Statistics_and_Machine_Learning_Toolbox |
| 101 | +
|
| 102 | + - name: Install MatBox |
| 103 | + env: |
| 104 | + # MatBox resolves its dependencies through the GitHub API. |
| 105 | + # UNAUTHENTICATED that is 60 requests/hour per runner IP, so this step |
| 106 | + # fails on `rate limit exceeded` (403) in a way indistinguishable from |
| 107 | + # a real dependency break (the Soph run 32545423486 died here on |
| 108 | + # vhlab-library-matlab's getDefaultBranch). secrets.GITHUB_TOKEN is |
| 109 | + # minted per run, needs no setup, and raises the limit to 1000/hour. |
| 110 | + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
| 111 | + uses: ehennestad/matbox-actions/install-matbox@v1 |
| 112 | + |
| 113 | + - name: Point requirements at the DID-matlab V_eta branch |
| 114 | + # requirements.txt pins a DID-matlab that predates +migrators_j / |
| 115 | + # TargetVersion='V_eta'. Rewrite it to the V_eta branch so v1_to_v2 |
| 116 | + # accepts TargetVersion='V_eta'. |
| 117 | + run: | |
| 118 | + set -eux |
| 119 | + python3 - <<'PY' |
| 120 | + import os, pathlib |
| 121 | + ref = os.environ["DID_MATLAB_REF"] |
| 122 | + for rel in ("requirements.txt", "tests/requirements.txt"): |
| 123 | + p = pathlib.Path(rel) |
| 124 | + if not p.exists(): |
| 125 | + continue |
| 126 | + out = [] |
| 127 | + for line in p.read_text().splitlines(): |
| 128 | + if "DID-matlab" in line: |
| 129 | + out.append(line.split("@", 1)[0] + "@" + ref) |
| 130 | + else: |
| 131 | + out.append(line) |
| 132 | + p.write_text("\n".join(out) + "\n") |
| 133 | + PY |
| 134 | + grep -i did-matlab requirements.txt tests/requirements.txt || true |
| 135 | +
|
| 136 | + - name: Install repo dependencies (DID-matlab@V_eta, mksqlite, vhlab-toolbox) |
| 137 | + env: |
| 138 | + # Same rate-limit fix as Install MatBox above: installRequirements |
| 139 | + # resolves every dependency (DID-matlab, mksqlite, vhlab-toolbox, |
| 140 | + # vhlab-library) through the GitHub API, so it needs the token to stay |
| 141 | + # under the 1000/hour authenticated limit rather than the 60/hour IP one. |
| 142 | + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
| 143 | + uses: matlab-actions/run-command@v2 |
| 144 | + with: |
| 145 | + command: | |
| 146 | + addpath(genpath("src")); |
| 147 | + addpath(genpath("tools")); |
| 148 | + matbox.installRequirements(nditools.projectdir()); |
| 149 | + matbox.installRequirements(fullfile(pwd, "tests")); |
| 150 | +
|
| 151 | + - name: Run the Dab V_eta migrate end-to-end test |
| 152 | + uses: matlab-actions/run-command@v2 |
| 153 | + with: |
| 154 | + command: | |
| 155 | + addpath(genpath("src")); |
| 156 | + addpath(genpath("tests")); |
| 157 | + % Dab ALONE. Hard-asserts corpus size, 0 quarantine / 0 orphans / |
| 158 | + % 0 fragments, that the session opens on the did2sqlite backend, and |
| 159 | + % that the ontologyRowSubjects fan-out (#53) leaves no unfolded |
| 160 | + % `ontologyTableRow` in the output. The pass-1 census, final survivor |
| 161 | + % set, per-session read-back counts, and fan-out OUTPUT volume are |
| 162 | + % report-only on this first landing (measured before pinned). SKIPS |
| 163 | + % (never fails) when the corpus download is impossible. |
| 164 | + results = runtests('ndi.unittest.migrate.TestMigrateLocalEtaDab'); |
| 165 | + disp(results); |
| 166 | + assertSuccess(results); |
0 commit comments