NDI Cloud file CRC-32: lookupFiles, checksummed uploads, crc32 in file listings

src/ndi/+ndi/+cloud/+api/+files/getFileUploadURL.m

@@ -1,7 +1,8 @@
-function [b, answer, apiResponse, apiURL] = getFileUploadURL(cloudDatasetID, cloudFileUID)
+function [b, answer, apiResponse, apiURL] = getFileUploadURL(cloudDatasetID, cloudFileUID, options)
 %GETFILEUPLOADURL Get a pre-signed URL for uploading a single file.
 %
 %   [B, ANSWER, APIRESPONSE, APIURL] = ndi.cloud.api.files.getFileUploadURL(CLOUDDATASETID, CLOUDFILEUID)
+%   [...] = ndi.cloud.api.files.getFileUploadURL(..., 'checksumCRC32', CRC)
 %
 %   Retrieves a URL that can be used to PUT the contents of a single file
 %   for a dataset.
@@ -10,6 +11,17 @@
 %       cloudDatasetID      - The ID of the dataset.
 %       cloudFileUID        - The unique identifier that will be used for the file.
 %
+%   Name-Value Inputs:
+%       checksumCRC32       - (Optional) The file's CRC-32 as base64 of its
+%                             4-byte big-endian value (ndi.fun.file.crc32).
+%                             When given, the URL signs the
+%                             x-amz-checksum-crc32 header, so the upload must
+%                             send it: pass the same value to putFiles as
+%                             'checksumCRC32'. S3 then rejects content that
+%                             does not match, and the cloud records the value
+%                             as the file's crc32.
+%                             Default "" (no checksum).
+%
 %   Outputs:
 %       b            - True if the call succeeded, false otherwise.
 %       answer       - The pre-signed upload URL (string) on success, or an error struct on failure.
@@ -19,17 +31,26 @@
 %   Example:
 %       [success, upload_url] = ndi.cloud.api.files.getFileUploadURL('d-12345', 'f-newfile');
 %
-%   See also: ndi.cloud.api.implementation.files.GetFileUploadURL
+%       % With a checksum, so a corrupted upload is refused:
+%       crc = ndi.fun.file.crc32('myfile.dat');
+%       [success, upload_url] = ndi.cloud.api.files.getFileUploadURL('d-12345', 'f-newfile', ...
+%           'checksumCRC32', crc);
+%       ndi.cloud.api.files.putFiles(upload_url, 'myfile.dat', 'checksumCRC32', crc);
+%
+%   See also: ndi.cloud.api.implementation.files.GetFileUploadURL,
+%             ndi.fun.file.crc32
 
     arguments
         cloudDatasetID (1,1) string
         cloudFileUID (1,1) string
+        options.checksumCRC32 (1,1) string = ""
     end
 
     % 1. Create an instance of the implementation class.
     api_call = ndi.cloud.api.implementation.files.GetFileUploadURL(...
         'cloudDatasetID', cloudDatasetID, ...
-        'cloudFileUID', cloudFileUID);
+        'cloudFileUID', cloudFileUID, ...
+        'checksumCRC32', options.checksumCRC32);
 
     % 2. Call the execute method and return its outputs directly.
     [b, answer, apiResponse, apiURL] = api_call.execute();

src/ndi/+ndi/+cloud/+api/+files/getSignedURLSet.m

@@ -27,6 +27,9 @@
 %                      (or the wrapped struct form MATLAB unpacks the
 %                      server's map into) and whose values are pre-signed
 %                      GET URLs.
+%                      `fileInfo` is an Nx1 struct array (uid, uploaded,
+%                      size, crc32) for this page's files; crc32 is '' when
+%                      none was recorded. Empty if the server predates it.
 %                      On failure, the server error payload.
 %       apiResponse  - The full matlab.net.http.ResponseMessage object.
 %       apiURL       - The URL that was called.

src/ndi/+ndi/+cloud/+api/+files/getSignedURLSetAll.m

@@ -29,6 +29,9 @@
 %                        pages, expiresAt, files.
 %                      `files` is a struct whose field names are file UIDs
 %                      and whose values are pre-signed GET URLs.
+%                      `fileInfo` is an Nx1 struct array (uid, uploaded,
+%                      size, crc32) across all pages; crc32 is '' when none
+%                      was recorded.
 %       apiResponse  - The matlab.net.http.ResponseMessage from the last page.
 %       apiURL       - The URL of the last page.
 %

src/ndi/+ndi/+cloud/+api/+files/listFiles.m

@@ -29,6 +29,9 @@
 %       uploaded            - A 0/1 flag indicating if the file has been uploaded.
 %       sourceDatasetId     - The cloudDatasetId of the file's parent dataset.
 %       size                - The file size in bytes.
+%       crc32               - The base64 big-endian CRC-32 of the content,
+%                             recorded at upload ('' when none was recorded).
+%                             Compare with ndi.fun.file.crc32 of a local copy.
 %
 %   Example:
 %       % Get the first page of files

src/ndi/+ndi/+cloud/+api/+files/listFilesAll.m

@@ -30,6 +30,8 @@
 %                          uploaded        - 0/1 flag: whether the file is uploaded.
 %                          sourceDatasetId - The parent dataset's cloud id.
 %                          size            - The file size in bytes.
+%                          crc32           - Base64 big-endian CRC-32 recorded at
+%                                            upload ('' when none was recorded).
 %       apiResponse  - The matlab.net.http.ResponseMessage object from the LAST
 %                      page call.
 %       apiURL       - The URL that was called for the LAST page.

src/ndi/+ndi/+cloud/+api/+files/lookupFiles.m

@@ -0,0 +1,80 @@
+function [b, answer, apiResponse, apiURL] = lookupFiles(cloudDatasetID, cloudFileUIDs)
+%LOOKUPFILES Look up the upload state, size and CRC-32 of files by uid.
+%
+%   [B, ANSWER, APIRESPONSE, APIURL] = ndi.cloud.api.files.lookupFiles(CLOUDDATASETID, CLOUDFILEUIDS)
+%
+%   Asks NDI Cloud, for each uid, whether the dataset has the file, whether it
+%   finished uploading, its size, and the CRC-32 recorded at upload. Use it to
+%   decide which files of a document or import batch still need uploading
+%   (for example to resume an interrupted upload) without listing the whole
+%   dataset. No download URLs are signed.
+%
+%   The server takes at most 5000 uids per call; longer lists are split into
+%   several calls and the results merged. Duplicate uids are looked up once.
+%
+%   Inputs:
+%       cloudDatasetID  - The ID of the dataset.
+%       cloudFileUIDs   - String array (or cellstr) of file uids.
+%
+%   Outputs:
+%       b            - True if every call succeeded, false otherwise.
+%       answer       - On success, a struct with fields:
+%                        files   - Nx1 struct array, one per uid the dataset
+%                                  has a record for, with fields
+%                                    uid      - char
+%                                    uploaded - logical
+%                                    size     - double, bytes
+%                                    crc32    - char, base64 big-endian CRC-32
+%                                               of the content, '' when none
+%                                               was recorded
+%                        missing - 1xM string array of uids with no record
+%                      On failure, the error body from the failing call.
+%       apiResponse  - The matlab.net.http.ResponseMessage of the last call
+%                      ([] when CLOUDFILEUIDS is empty).
+%       apiURL       - The URL of the last call ([] when CLOUDFILEUIDS is empty).
+%
+%   A cloud copy matches a local file when it is uploaded and its crc32 equals
+%   ndi.fun.file.crc32 of the local file. When crc32 is '' (files uploaded
+%   without a checksum), compare size instead.
+%
+%   Example:
+%       [ok, r] = ndi.cloud.api.files.lookupFiles(datasetId, uids);
+%       for i = 1:numel(r.files)
+%           f = r.files(i);
+%           localCrc = ndi.fun.file.crc32(fullfile(folder, f.uid));
+%           needsUpload = ~f.uploaded || ~strcmp(f.crc32, localCrc);
+%       end
+%       % r.missing lists uids never uploaded at all.
+%
+%   See also: ndi.cloud.api.implementation.files.LookupFiles,
+%             ndi.fun.file.crc32, ndi.cloud.api.files.getFileUploadURL
+
+    arguments
+        cloudDatasetID (1,1) string
+        cloudFileUIDs string
+    end
+
+    uids = unique(reshape(cloudFileUIDs, 1, []), 'stable');
+
+    b = true;
+    answer = struct('files', ndi.cloud.api.implementation.files.fileMetadataArray([]), ...
+        'missing', strings(1, 0));
+    apiResponse = [];
+    apiURL = [];
+
+    maxUids = ndi.cloud.api.implementation.files.LookupFiles.MaxUids;
+    for first = 1:maxUids:numel(uids)
+        batch = uids(first:min(first + maxUids - 1, numel(uids)));
+        api_call = ndi.cloud.api.implementation.files.LookupFiles(...
+            'cloudDatasetID', cloudDatasetID, ...
+            'cloudFileUIDs', batch);
+        [ok, batchAnswer, apiResponse, apiURL] = api_call.execute();
+        if ~ok
+            b = false;
+            answer = batchAnswer;
+            return
+        end
+        answer.files = [answer.files; batchAnswer.files];
+        answer.missing = [answer.missing batchAnswer.missing];
+    end
+end

src/ndi/+ndi/+cloud/+api/+files/putFiles.m

@@ -37,6 +37,14 @@
 %       'timeout' (double)  - Overall wait-for-completion deadline, in
 %                             seconds. Default 60.
 %
+%       'checksumCRC32' (string) - The file's CRC-32 (ndi.fun.file.crc32).
+%                             Required when the URL came from
+%                             getFileUploadURL(..., 'checksumCRC32', CRC):
+%                             that URL signs the x-amz-checksum-crc32
+%                             header, so it must be sent, with the same
+%                             value. S3 then rejects content that does not
+%                             match. Default "" (no header).
+%
 %   Outputs:
 %       b            - True if the upload succeeded (and, when
 %                      waitForCompletion is true, the server-side bulk
@@ -57,6 +65,13 @@
 %           [s2] = ndi.cloud.api.files.putFiles(url, 'localfile.dat');
 %       end
 %
+%       % Single-file upload with a checksum
+%       crc = ndi.fun.file.crc32('localfile.dat');
+%       [s, url] = ndi.cloud.api.files.getFileUploadURL('d-123', 'f-abc', 'checksumCRC32', crc);
+%       if s
+%           [s2] = ndi.cloud.api.files.putFiles(url, 'localfile.dat', 'checksumCRC32', crc);
+%       end
+%
 %       % Bulk (zip) upload, waiting for server-side extraction to finish
 %       [s, info] = ndi.cloud.api.files.getFileCollectionUploadURL('d-123');
 %       if s
@@ -76,6 +91,7 @@
         options.jobId (1,1) string = ""
         options.waitForCompletion (1,1) logical = false
         options.timeout (1,1) double {mustBePositive} = 60
+        options.checksumCRC32 (1,1) string = ""
     end
     % 1. Create an instance of the implementation class, passing the options.
     api_call = ndi.cloud.api.implementation.files.PutFiles(...
@@ -84,7 +100,8 @@
         'useCurl', options.useCurl, ...
         'jobId', options.jobId, ...
         'waitForCompletion', options.waitForCompletion, ...
-        'timeout', options.timeout);
+        'timeout', options.timeout, ...
+        'checksumCRC32', options.checksumCRC32);
 
     % 2. Call the execute method and return its outputs directly.
     [b, answer, apiResponse, apiURL] = api_call.execute();

src/ndi/+ndi/+cloud/+api/+implementation/+files/GetFileUploadURL.m

@@ -3,6 +3,7 @@
 
     properties
         cloudFileUID
+        checksumCRC32 (1,1) string = "" % base64 big-endian CRC-32, or "" for none
     end
 
     methods
@@ -12,17 +13,40 @@
             %   Inputs:
             %       'cloudDatasetID' - The ID of the dataset.
             %       'cloudFileUID'   - The unique identifier for the file.
+            %       'checksumCRC32'  - (Optional) The file's CRC-32, base64 of
+            %                          its 4-byte big-endian value. Sent as the
+            %                          checksumCRC32 query parameter.
             %
             arguments
                 args.cloudDatasetID (1,1) string
                 args.cloudFileUID (1,1) string
+                args.checksumCRC32 (1,1) string = ""
             end
 
             this.cloudDatasetID = args.cloudDatasetID;
             this.cloudFileUID = args.cloudFileUID;
+            this.checksumCRC32 = args.checksumCRC32;
             this.endpointName = 'get_file_upload_url';
         end
 
+        function apiURL = buildURL(this, organizationId)
+            %BUILDURL The upload URL request for ORGANIZATIONID, with the
+            %   checksumCRC32 query parameter when a checksum was given.
+            apiURL = ndi.cloud.api.url('get_file_upload_url', ...
+                'dataset_id', this.cloudDatasetID, ...
+                'organization_id', organizationId, ...
+                'file_uid', this.cloudFileUID);
+            if strlength(this.checksumCRC32) > 0
+                % QueryParameter percent-encodes the base64 '+', '/' and '='.
+                q = matlab.net.QueryParameter('checksumCRC32', char(this.checksumCRC32));
+                if isempty(apiURL.Query)
+                    apiURL.Query = q;
+                else
+                    apiURL.Query = [apiURL.Query q];
+                end
+            end
+        end
+
         function [b, answer, apiResponse, apiURL] = execute(this)
             %EXECUTE Performs the API call.
 
@@ -41,10 +65,7 @@
                 return;
             end
 
-            apiURL = ndi.cloud.api.url('get_file_upload_url', ...
-                'dataset_id', this.cloudDatasetID, ...
-                'organization_id', dsetinfo.organizationId, ...
-                'file_uid', this.cloudFileUID);
+            apiURL = this.buildURL(dsetinfo.organizationId);
 
             method = matlab.net.http.RequestMethod.GET;
 

src/ndi/+ndi/+cloud/+api/+implementation/+files/GetSignedURLSetAll.m

@@ -52,6 +52,7 @@
             % A containers.Map, not a struct: the keys are DID file uids and
             % a struct would carry the JSONDECODE-renamed versions of them.
             merged.files      = containers.Map('KeyType','char','ValueType','any');
+            merged.fileInfo   = ndi.cloud.api.implementation.files.fileMetadataArray([]);
             merged.pageCount  = 0;
             merged.totalCount = 0;
             merged.pages      = 0;
@@ -70,6 +71,9 @@
                     end
                     merged.pageCount = merged.pageCount + numel(pageKeys);
                 end
+                if isstruct(page) && isfield(page, 'fileInfo') && isstruct(page.fileInfo)
+                    merged.fileInfo = [merged.fileInfo; page.fileInfo(:)];
+                end
                 if isstruct(page) && isfield(page, 'totalCount')
                     merged.totalCount = page.totalCount;
                 end

src/ndi/+ndi/+cloud/+api/+implementation/+files/ListFiles.m

@@ -82,7 +82,7 @@
                 raw_answer = apiResponse.Body.Data;
 
                 % Standardize the output format.
-                answer = struct('uid', {}, 'uploaded', {}, 'sourceDatasetId', {}, 'size', {});
+                answer = struct('uid', {}, 'uploaded', {}, 'sourceDatasetId', {}, 'size', {}, 'crc32', {});
 
                 if ~isempty(raw_answer) && isfield(raw_answer, 'files') && ~isempty(raw_answer.files)
                     files_from_api = raw_answer.files;
@@ -107,7 +107,8 @@
                             'uid', file_in.uid, ...
                             'uploaded', [], ...
                             'sourceDatasetId', '', ...
-                            'size', []);
+                            'size', [], ...
+                            'crc32', '');
 
                         if isfield(file_in, 'uploaded')
                             file_out.uploaded = file_in.uploaded;
@@ -118,6 +119,11 @@
                         if isfield(file_in, 'size')
                             file_out.size = file_in.size;
                         end
+                        % Recorded at upload (base64 big-endian CRC-32);
+                        % absent for files uploaded without a checksum.
+                        if isfield(file_in, 'crc32') && ~isempty(file_in.crc32)
+                            file_out.crc32 = char(file_in.crc32);
+                        end
 
                         answer(end+1) = file_out;
                     end

src/ndi/+ndi/+cloud/+api/+implementation/+files/ListFilesAll.m

@@ -58,13 +58,13 @@
             %   Outputs:
             %       b            - True if all pages were read successfully, false otherwise.
             %       answer       - A struct array of file summaries (uid, uploaded,
-            %                      sourceDatasetId, size) on success.
+            %                      sourceDatasetId, size, crc32) on success.
             %       apiResponse  - The matlab.net.http.ResponseMessage object from the LAST page call.
             %       apiURL       - The URL that was called for the LAST page.
 
             % Initialize outputs
             b = true;
-            answer = struct('uid', {}, 'uploaded', {}, 'sourceDatasetId', {}, 'size', {});
+            answer = struct('uid', {}, 'uploaded', {}, 'sourceDatasetId', {}, 'size', {}, 'crc32', {});
             apiResponse = matlab.net.http.ResponseMessage.empty;
             apiURL = matlab.net.URI.empty;