diff --git a/src/ndi/+ndi/+cloud/+api/+files/getFileUploadURL.m b/src/ndi/+ndi/+cloud/+api/+files/getFileUploadURL.m index cb8824056..4d781a0ae 100644 --- a/src/ndi/+ndi/+cloud/+api/+files/getFileUploadURL.m +++ b/src/ndi/+ndi/+cloud/+api/+files/getFileUploadURL.m @@ -1,7 +1,8 @@ -function [b, answer, apiResponse, apiURL] = getFileUploadURL(cloudDatasetID, cloudFileUID) +function [b, answer, apiResponse, apiURL] = getFileUploadURL(cloudDatasetID, cloudFileUID, options) %GETFILEUPLOADURL Get a pre-signed URL for uploading a single file. % % [B, ANSWER, APIRESPONSE, APIURL] = ndi.cloud.api.files.getFileUploadURL(CLOUDDATASETID, CLOUDFILEUID) +% [...] = ndi.cloud.api.files.getFileUploadURL(..., 'checksumCRC32', CRC) % % Retrieves a URL that can be used to PUT the contents of a single file % for a dataset. @@ -10,6 +11,17 @@ % cloudDatasetID - The ID of the dataset. % cloudFileUID - The unique identifier that will be used for the file. % +% Name-Value Inputs: +% checksumCRC32 - (Optional) The file's CRC-32 as base64 of its +% 4-byte big-endian value (ndi.fun.file.crc32). +% When given, the URL signs the +% x-amz-checksum-crc32 header, so the upload must +% send it: pass the same value to putFiles as +% 'checksumCRC32'. S3 then rejects content that +% does not match, and the cloud records the value +% as the file's crc32. +% Default "" (no checksum). +% % Outputs: % b - True if the call succeeded, false otherwise. % answer - The pre-signed upload URL (string) on success, or an error struct on failure. @@ -19,17 +31,26 @@ % Example: % [success, upload_url] = ndi.cloud.api.files.getFileUploadURL('d-12345', 'f-newfile'); % -% See also: ndi.cloud.api.implementation.files.GetFileUploadURL +% % With a checksum, so a corrupted upload is refused: +% crc = ndi.fun.file.crc32('myfile.dat'); +% [success, upload_url] = ndi.cloud.api.files.getFileUploadURL('d-12345', 'f-newfile', ... +% 'checksumCRC32', crc); +% ndi.cloud.api.files.putFiles(upload_url, 'myfile.dat', 'checksumCRC32', crc); +% +% See also: ndi.cloud.api.implementation.files.GetFileUploadURL, +% ndi.fun.file.crc32 arguments cloudDatasetID (1,1) string cloudFileUID (1,1) string + options.checksumCRC32 (1,1) string = "" end % 1. Create an instance of the implementation class. api_call = ndi.cloud.api.implementation.files.GetFileUploadURL(... 'cloudDatasetID', cloudDatasetID, ... - 'cloudFileUID', cloudFileUID); + 'cloudFileUID', cloudFileUID, ... + 'checksumCRC32', options.checksumCRC32); % 2. Call the execute method and return its outputs directly. [b, answer, apiResponse, apiURL] = api_call.execute(); diff --git a/src/ndi/+ndi/+cloud/+api/+files/getSignedURLSet.m b/src/ndi/+ndi/+cloud/+api/+files/getSignedURLSet.m index 0a83d0f75..628cd7622 100644 --- a/src/ndi/+ndi/+cloud/+api/+files/getSignedURLSet.m +++ b/src/ndi/+ndi/+cloud/+api/+files/getSignedURLSet.m @@ -27,6 +27,9 @@ % (or the wrapped struct form MATLAB unpacks the % server's map into) and whose values are pre-signed % GET URLs. +% `fileInfo` is an Nx1 struct array (uid, uploaded, +% size, crc32) for this page's files; crc32 is '' when +% none was recorded. Empty if the server predates it. % On failure, the server error payload. % apiResponse - The full matlab.net.http.ResponseMessage object. % apiURL - The URL that was called. diff --git a/src/ndi/+ndi/+cloud/+api/+files/getSignedURLSetAll.m b/src/ndi/+ndi/+cloud/+api/+files/getSignedURLSetAll.m index 2caa3455f..f9ad256be 100644 --- a/src/ndi/+ndi/+cloud/+api/+files/getSignedURLSetAll.m +++ b/src/ndi/+ndi/+cloud/+api/+files/getSignedURLSetAll.m @@ -29,6 +29,9 @@ % pages, expiresAt, files. % `files` is a struct whose field names are file UIDs % and whose values are pre-signed GET URLs. +% `fileInfo` is an Nx1 struct array (uid, uploaded, +% size, crc32) across all pages; crc32 is '' when none +% was recorded. % apiResponse - The matlab.net.http.ResponseMessage from the last page. % apiURL - The URL of the last page. % diff --git a/src/ndi/+ndi/+cloud/+api/+files/listFiles.m b/src/ndi/+ndi/+cloud/+api/+files/listFiles.m index f452f81cd..33755d2c0 100644 --- a/src/ndi/+ndi/+cloud/+api/+files/listFiles.m +++ b/src/ndi/+ndi/+cloud/+api/+files/listFiles.m @@ -29,6 +29,9 @@ % uploaded - A 0/1 flag indicating if the file has been uploaded. % sourceDatasetId - The cloudDatasetId of the file's parent dataset. % size - The file size in bytes. +% crc32 - The base64 big-endian CRC-32 of the content, +% recorded at upload ('' when none was recorded). +% Compare with ndi.fun.file.crc32 of a local copy. % % Example: % % Get the first page of files diff --git a/src/ndi/+ndi/+cloud/+api/+files/listFilesAll.m b/src/ndi/+ndi/+cloud/+api/+files/listFilesAll.m index 30f2e1619..97313b90d 100644 --- a/src/ndi/+ndi/+cloud/+api/+files/listFilesAll.m +++ b/src/ndi/+ndi/+cloud/+api/+files/listFilesAll.m @@ -30,6 +30,8 @@ % uploaded - 0/1 flag: whether the file is uploaded. % sourceDatasetId - The parent dataset's cloud id. % size - The file size in bytes. +% crc32 - Base64 big-endian CRC-32 recorded at +% upload ('' when none was recorded). % apiResponse - The matlab.net.http.ResponseMessage object from the LAST % page call. % apiURL - The URL that was called for the LAST page. diff --git a/src/ndi/+ndi/+cloud/+api/+files/lookupFiles.m b/src/ndi/+ndi/+cloud/+api/+files/lookupFiles.m new file mode 100644 index 000000000..2e86a55b3 --- /dev/null +++ b/src/ndi/+ndi/+cloud/+api/+files/lookupFiles.m @@ -0,0 +1,80 @@ +function [b, answer, apiResponse, apiURL] = lookupFiles(cloudDatasetID, cloudFileUIDs) +%LOOKUPFILES Look up the upload state, size and CRC-32 of files by uid. +% +% [B, ANSWER, APIRESPONSE, APIURL] = ndi.cloud.api.files.lookupFiles(CLOUDDATASETID, CLOUDFILEUIDS) +% +% Asks NDI Cloud, for each uid, whether the dataset has the file, whether it +% finished uploading, its size, and the CRC-32 recorded at upload. Use it to +% decide which files of a document or import batch still need uploading +% (for example to resume an interrupted upload) without listing the whole +% dataset. No download URLs are signed. +% +% The server takes at most 5000 uids per call; longer lists are split into +% several calls and the results merged. Duplicate uids are looked up once. +% +% Inputs: +% cloudDatasetID - The ID of the dataset. +% cloudFileUIDs - String array (or cellstr) of file uids. +% +% Outputs: +% b - True if every call succeeded, false otherwise. +% answer - On success, a struct with fields: +% files - Nx1 struct array, one per uid the dataset +% has a record for, with fields +% uid - char +% uploaded - logical +% size - double, bytes +% crc32 - char, base64 big-endian CRC-32 +% of the content, '' when none +% was recorded +% missing - 1xM string array of uids with no record +% On failure, the error body from the failing call. +% apiResponse - The matlab.net.http.ResponseMessage of the last call +% ([] when CLOUDFILEUIDS is empty). +% apiURL - The URL of the last call ([] when CLOUDFILEUIDS is empty). +% +% A cloud copy matches a local file when it is uploaded and its crc32 equals +% ndi.fun.file.crc32 of the local file. When crc32 is '' (files uploaded +% without a checksum), compare size instead. +% +% Example: +% [ok, r] = ndi.cloud.api.files.lookupFiles(datasetId, uids); +% for i = 1:numel(r.files) +% f = r.files(i); +% localCrc = ndi.fun.file.crc32(fullfile(folder, f.uid)); +% needsUpload = ~f.uploaded || ~strcmp(f.crc32, localCrc); +% end +% % r.missing lists uids never uploaded at all. +% +% See also: ndi.cloud.api.implementation.files.LookupFiles, +% ndi.fun.file.crc32, ndi.cloud.api.files.getFileUploadURL + + arguments + cloudDatasetID (1,1) string + cloudFileUIDs string + end + + uids = unique(reshape(cloudFileUIDs, 1, []), 'stable'); + + b = true; + answer = struct('files', ndi.cloud.api.implementation.files.fileMetadataArray([]), ... + 'missing', strings(1, 0)); + apiResponse = []; + apiURL = []; + + maxUids = ndi.cloud.api.implementation.files.LookupFiles.MaxUids; + for first = 1:maxUids:numel(uids) + batch = uids(first:min(first + maxUids - 1, numel(uids))); + api_call = ndi.cloud.api.implementation.files.LookupFiles(... + 'cloudDatasetID', cloudDatasetID, ... + 'cloudFileUIDs', batch); + [ok, batchAnswer, apiResponse, apiURL] = api_call.execute(); + if ~ok + b = false; + answer = batchAnswer; + return + end + answer.files = [answer.files; batchAnswer.files]; + answer.missing = [answer.missing batchAnswer.missing]; + end +end diff --git a/src/ndi/+ndi/+cloud/+api/+files/putFiles.m b/src/ndi/+ndi/+cloud/+api/+files/putFiles.m index c0dc892dc..dcb6fc200 100644 --- a/src/ndi/+ndi/+cloud/+api/+files/putFiles.m +++ b/src/ndi/+ndi/+cloud/+api/+files/putFiles.m @@ -37,6 +37,14 @@ % 'timeout' (double) - Overall wait-for-completion deadline, in % seconds. Default 60. % +% 'checksumCRC32' (string) - The file's CRC-32 (ndi.fun.file.crc32). +% Required when the URL came from +% getFileUploadURL(..., 'checksumCRC32', CRC): +% that URL signs the x-amz-checksum-crc32 +% header, so it must be sent, with the same +% value. S3 then rejects content that does not +% match. Default "" (no header). +% % Outputs: % b - True if the upload succeeded (and, when % waitForCompletion is true, the server-side bulk @@ -57,6 +65,13 @@ % [s2] = ndi.cloud.api.files.putFiles(url, 'localfile.dat'); % end % +% % Single-file upload with a checksum +% crc = ndi.fun.file.crc32('localfile.dat'); +% [s, url] = ndi.cloud.api.files.getFileUploadURL('d-123', 'f-abc', 'checksumCRC32', crc); +% if s +% [s2] = ndi.cloud.api.files.putFiles(url, 'localfile.dat', 'checksumCRC32', crc); +% end +% % % Bulk (zip) upload, waiting for server-side extraction to finish % [s, info] = ndi.cloud.api.files.getFileCollectionUploadURL('d-123'); % if s @@ -76,6 +91,7 @@ options.jobId (1,1) string = "" options.waitForCompletion (1,1) logical = false options.timeout (1,1) double {mustBePositive} = 60 + options.checksumCRC32 (1,1) string = "" end % 1. Create an instance of the implementation class, passing the options. api_call = ndi.cloud.api.implementation.files.PutFiles(... @@ -84,7 +100,8 @@ 'useCurl', options.useCurl, ... 'jobId', options.jobId, ... 'waitForCompletion', options.waitForCompletion, ... - 'timeout', options.timeout); + 'timeout', options.timeout, ... + 'checksumCRC32', options.checksumCRC32); % 2. Call the execute method and return its outputs directly. [b, answer, apiResponse, apiURL] = api_call.execute(); diff --git a/src/ndi/+ndi/+cloud/+api/+implementation/+files/GetFileUploadURL.m b/src/ndi/+ndi/+cloud/+api/+implementation/+files/GetFileUploadURL.m index eeff441d6..72d458985 100644 --- a/src/ndi/+ndi/+cloud/+api/+implementation/+files/GetFileUploadURL.m +++ b/src/ndi/+ndi/+cloud/+api/+implementation/+files/GetFileUploadURL.m @@ -3,6 +3,7 @@ properties cloudFileUID + checksumCRC32 (1,1) string = "" % base64 big-endian CRC-32, or "" for none end methods @@ -12,17 +13,40 @@ % Inputs: % 'cloudDatasetID' - The ID of the dataset. % 'cloudFileUID' - The unique identifier for the file. + % 'checksumCRC32' - (Optional) The file's CRC-32, base64 of + % its 4-byte big-endian value. Sent as the + % checksumCRC32 query parameter. % arguments args.cloudDatasetID (1,1) string args.cloudFileUID (1,1) string + args.checksumCRC32 (1,1) string = "" end this.cloudDatasetID = args.cloudDatasetID; this.cloudFileUID = args.cloudFileUID; + this.checksumCRC32 = args.checksumCRC32; this.endpointName = 'get_file_upload_url'; end + function apiURL = buildURL(this, organizationId) + %BUILDURL The upload URL request for ORGANIZATIONID, with the + % checksumCRC32 query parameter when a checksum was given. + apiURL = ndi.cloud.api.url('get_file_upload_url', ... + 'dataset_id', this.cloudDatasetID, ... + 'organization_id', organizationId, ... + 'file_uid', this.cloudFileUID); + if strlength(this.checksumCRC32) > 0 + % QueryParameter percent-encodes the base64 '+', '/' and '='. + q = matlab.net.QueryParameter('checksumCRC32', char(this.checksumCRC32)); + if isempty(apiURL.Query) + apiURL.Query = q; + else + apiURL.Query = [apiURL.Query q]; + end + end + end + function [b, answer, apiResponse, apiURL] = execute(this) %EXECUTE Performs the API call. @@ -41,10 +65,7 @@ return; end - apiURL = ndi.cloud.api.url('get_file_upload_url', ... - 'dataset_id', this.cloudDatasetID, ... - 'organization_id', dsetinfo.organizationId, ... - 'file_uid', this.cloudFileUID); + apiURL = this.buildURL(dsetinfo.organizationId); method = matlab.net.http.RequestMethod.GET; diff --git a/src/ndi/+ndi/+cloud/+api/+implementation/+files/GetSignedURLSetAll.m b/src/ndi/+ndi/+cloud/+api/+implementation/+files/GetSignedURLSetAll.m index 6d0245337..b16576ff7 100644 --- a/src/ndi/+ndi/+cloud/+api/+implementation/+files/GetSignedURLSetAll.m +++ b/src/ndi/+ndi/+cloud/+api/+implementation/+files/GetSignedURLSetAll.m @@ -52,6 +52,7 @@ % A containers.Map, not a struct: the keys are DID file uids and % a struct would carry the JSONDECODE-renamed versions of them. merged.files = containers.Map('KeyType','char','ValueType','any'); + merged.fileInfo = ndi.cloud.api.implementation.files.fileMetadataArray([]); merged.pageCount = 0; merged.totalCount = 0; merged.pages = 0; @@ -70,6 +71,9 @@ end merged.pageCount = merged.pageCount + numel(pageKeys); end + if isstruct(page) && isfield(page, 'fileInfo') && isstruct(page.fileInfo) + merged.fileInfo = [merged.fileInfo; page.fileInfo(:)]; + end if isstruct(page) && isfield(page, 'totalCount') merged.totalCount = page.totalCount; end diff --git a/src/ndi/+ndi/+cloud/+api/+implementation/+files/ListFiles.m b/src/ndi/+ndi/+cloud/+api/+implementation/+files/ListFiles.m index 0bd3a084d..f9c8197bb 100644 --- a/src/ndi/+ndi/+cloud/+api/+implementation/+files/ListFiles.m +++ b/src/ndi/+ndi/+cloud/+api/+implementation/+files/ListFiles.m @@ -82,7 +82,7 @@ raw_answer = apiResponse.Body.Data; % Standardize the output format. - answer = struct('uid', {}, 'uploaded', {}, 'sourceDatasetId', {}, 'size', {}); + answer = struct('uid', {}, 'uploaded', {}, 'sourceDatasetId', {}, 'size', {}, 'crc32', {}); if ~isempty(raw_answer) && isfield(raw_answer, 'files') && ~isempty(raw_answer.files) files_from_api = raw_answer.files; @@ -107,7 +107,8 @@ 'uid', file_in.uid, ... 'uploaded', [], ... 'sourceDatasetId', '', ... - 'size', []); + 'size', [], ... + 'crc32', ''); if isfield(file_in, 'uploaded') file_out.uploaded = file_in.uploaded; @@ -118,6 +119,11 @@ if isfield(file_in, 'size') file_out.size = file_in.size; end + % Recorded at upload (base64 big-endian CRC-32); + % absent for files uploaded without a checksum. + if isfield(file_in, 'crc32') && ~isempty(file_in.crc32) + file_out.crc32 = char(file_in.crc32); + end answer(end+1) = file_out; end diff --git a/src/ndi/+ndi/+cloud/+api/+implementation/+files/ListFilesAll.m b/src/ndi/+ndi/+cloud/+api/+implementation/+files/ListFilesAll.m index b34b73e4f..e077bcd66 100644 --- a/src/ndi/+ndi/+cloud/+api/+implementation/+files/ListFilesAll.m +++ b/src/ndi/+ndi/+cloud/+api/+implementation/+files/ListFilesAll.m @@ -58,13 +58,13 @@ % Outputs: % b - True if all pages were read successfully, false otherwise. % answer - A struct array of file summaries (uid, uploaded, - % sourceDatasetId, size) on success. + % sourceDatasetId, size, crc32) on success. % apiResponse - The matlab.net.http.ResponseMessage object from the LAST page call. % apiURL - The URL that was called for the LAST page. % Initialize outputs b = true; - answer = struct('uid', {}, 'uploaded', {}, 'sourceDatasetId', {}, 'size', {}); + answer = struct('uid', {}, 'uploaded', {}, 'sourceDatasetId', {}, 'size', {}, 'crc32', {}); apiResponse = matlab.net.http.ResponseMessage.empty; apiURL = matlab.net.URI.empty; diff --git a/src/ndi/+ndi/+cloud/+api/+implementation/+files/LookupFiles.m b/src/ndi/+ndi/+cloud/+api/+implementation/+files/LookupFiles.m new file mode 100644 index 000000000..c54f019ed --- /dev/null +++ b/src/ndi/+ndi/+cloud/+api/+implementation/+files/LookupFiles.m @@ -0,0 +1,102 @@ +classdef LookupFiles < ndi.cloud.api.call +%LOOKUPFILES Implementation class for looking up file records by uid. +% +% One call to POST /datasets/{datasetId}/files/lookup, for at most +% MaxUids uids. ndi.cloud.api.files.lookupFiles splits longer lists into +% several calls. +% +% See also: ndi.cloud.api.files.lookupFiles + + properties (Constant) + MaxUids = 5000 % server limit per call + end + + properties + cloudFileUIDs (1,:) string + end + + methods + function this = LookupFiles(args) + %LOOKUPFILES Creates a new LookupFiles API call object. + % + % THIS = ndi.cloud.api.implementation.files.LookupFiles( ... + % 'cloudDatasetID', ID, 'cloudFileUIDs', UIDS) + % + % Inputs: + % 'cloudDatasetID' - The ID of the dataset. + % 'cloudFileUIDs' - String array of file uids, 1 to MaxUids + % entries. + arguments + args.cloudDatasetID (1,1) string + args.cloudFileUIDs (1,:) string + end + if isempty(args.cloudFileUIDs) + error('NDI:CloudApi:LookupFiles:NoUids', ... + 'At least one file uid is required.'); + end + if numel(args.cloudFileUIDs) > ndi.cloud.api.implementation.files.LookupFiles.MaxUids + error('NDI:CloudApi:LookupFiles:TooManyUids', ... + 'At most %d uids per call (got %d); use ndi.cloud.api.files.lookupFiles, which splits the list.', ... + ndi.cloud.api.implementation.files.LookupFiles.MaxUids, numel(args.cloudFileUIDs)); + end + this.cloudDatasetID = args.cloudDatasetID; + this.cloudFileUIDs = args.cloudFileUIDs; + this.endpointName = 'lookup_dataset_files'; + end + + function body = requestBody(this) + %REQUESTBODY The JSON body sent to the server, as a struct. + % + % When MATLAB encodes a scalar string to JSON it produces a bare + % string, not an array, so a single uid is sent twice (the server + % collapses duplicates), as BulkFetch does. + uids = this.cloudFileUIDs; + if isscalar(uids) + uids = [uids uids]; + end + body = struct('uids', uids); + end + + function [b, answer, apiResponse, apiURL] = execute(this) + %EXECUTE Performs the API call. + % + % Outputs: + % b - True if the call succeeded, false otherwise. + % answer - On success, a struct with fields `files` + % (Nx1 struct array: uid, uploaded, size, + % crc32) and `missing` (string array). On + % failure, the error body from the server. + % apiResponse - The matlab.net.http.ResponseMessage object. + % apiURL - The URL that was called. + + b = false; + + token = ndi.cloud.authenticate(); + + apiURL = ndi.cloud.api.url(this.endpointName, 'dataset_id', this.cloudDatasetID); + + method = matlab.net.http.RequestMethod.POST; + body = matlab.net.http.MessageBody(this.requestBody()); + + acceptField = matlab.net.http.HeaderField('accept','application/json'); + contentTypeField = matlab.net.http.field.ContentTypeField(matlab.net.http.MediaType('application/json')); + authorizationField = matlab.net.http.HeaderField('Authorization', ['Bearer ' token]); + headers = [acceptField contentTypeField authorizationField]; + + request = matlab.net.http.RequestMessage(method, headers, body); + + apiResponse = send(request, apiURL); + + if (apiResponse.StatusCode == 200) + b = true; + answer = ndi.cloud.api.implementation.files.lookupFilesResult(apiResponse.Body.Data); + else + if isprop(apiResponse.Body, 'Data') + answer = apiResponse.Body.Data; + else + answer = apiResponse.Body; + end + end + end + end +end diff --git a/src/ndi/+ndi/+cloud/+api/+implementation/+files/PutFiles.m b/src/ndi/+ndi/+cloud/+api/+implementation/+files/PutFiles.m index 02d791bb7..b3c62884a 100644 --- a/src/ndi/+ndi/+cloud/+api/+implementation/+files/PutFiles.m +++ b/src/ndi/+ndi/+cloud/+api/+implementation/+files/PutFiles.m @@ -16,6 +16,7 @@ jobId (1,1) string waitForCompletion (1,1) logical timeout (1,1) double + checksumCRC32 (1,1) string % base64 big-endian CRC-32, or "" for none end methods @@ -28,7 +29,17 @@ args.jobId (1,1) string = "" args.waitForCompletion (1,1) logical = false args.timeout (1,1) double {mustBePositive} = 60 + args.checksumCRC32 (1,1) string = "" end + % A CRC-32 is 4 bytes: six base64 characters and '=='. Checking + % the shape also keeps the value safe to put in a curl command. + if strlength(args.checksumCRC32) > 0 && ... + isempty(regexp(char(args.checksumCRC32), '^[A-Za-z0-9+/]{6}==$', 'once')) + error('NDI:CloudApi:PutFiles:InvalidChecksum', ... + 'checksumCRC32 must be the base64 of a 4-byte CRC-32 (e.g. ''NhCmhg==''), got ''%s''.', ... + args.checksumCRC32); + end + this.checksumCRC32 = args.checksumCRC32; this.preSignedURL = args.preSignedURL; this.filePath = args.filePath; this.useCurl = args.useCurl; @@ -77,8 +88,12 @@ apiURL = matlab.net.URI(this.preSignedURL); method = matlab.net.http.RequestMethod.PUT; provider = matlab.net.http.io.FileProvider(this.filePath); - contentTypeField = matlab.net.http.HeaderField('Content-Type', 'application/octet-stream'); - request = matlab.net.http.RequestMessage(method, contentTypeField, provider); + headers = matlab.net.http.HeaderField('Content-Type', 'application/octet-stream'); + if strlength(this.checksumCRC32) > 0 + % The upload URL signs this header (NDI Cloud #162). + headers(end+1) = matlab.net.http.HeaderField('x-amz-checksum-crc32', char(this.checksumCRC32)); + end + request = matlab.net.http.RequestMessage(method, headers, provider); try apiResponse = send(request, apiURL); @@ -92,6 +107,21 @@ end end + function command = curlArguments(this) + %CURLARGUMENTS The curl command (without the environment prefix). + % When checksumCRC32 is set, the x-amz-checksum-crc32 header is + % sent: the upload URL signs it (NDI Cloud #162), so S3 refuses + % the PUT without it and rejects content that does not match. + checksumHeader = ''; + if strlength(this.checksumCRC32) > 0 + checksumHeader = sprintf('-H "x-amz-checksum-crc32: %s" ', char(this.checksumCRC32)); + end + command = sprintf(['curl -fsSL -X PUT --upload-file "%s" ' ... + '-H "Content-Type: application/octet-stream" ' ... + '-H "Accept-Encoding: identity" ' ... + '%s"%s"'], this.filePath, checksumHeader, this.preSignedURL); + end + function [b, answer, apiResponse, apiURL] = executeWithCurl(this) % Implementation using a system call to curl b = false; @@ -115,10 +145,7 @@ % sprintf so a '%' in the library path cannot corrupt the format % string. envPrefix = ndi.common.systemCurlEnvPrefix(); - command = [envPrefix sprintf(['curl -fsSL -X PUT --upload-file "%s" ' ... - '-H "Content-Type: application/octet-stream" ' ... - '-H "Accept-Encoding: identity" ' ... - '"%s"'], this.filePath, this.preSignedURL)]; + command = [envPrefix this.curlArguments()]; [status, result] = system(command); diff --git a/src/ndi/+ndi/+cloud/+api/+implementation/+files/fileMetadataArray.m b/src/ndi/+ndi/+cloud/+api/+implementation/+files/fileMetadataArray.m new file mode 100644 index 000000000..20d71ffbd --- /dev/null +++ b/src/ndi/+ndi/+cloud/+api/+implementation/+files/fileMetadataArray.m @@ -0,0 +1,62 @@ +function files = fileMetadataArray(rawFiles) +%FILEMETADATAARRAY Normalize decoded {uid, uploaded, size, crc32} records. +% +% FILES = ndi.cloud.api.implementation.files.FILEMETADATAARRAY(RAWFILES) +% +% NDI Cloud returns per-file metadata as a JSON array of objects with fields +% uid, uploaded, size and crc32 (POST /files/lookup `files`, and the +% signed-url-set `fileInfo`). JSONDECODE turns that into a struct array, a +% cell array (when the objects differ in fields), or [] (when empty), and +% turns a null crc32 into []. This returns one consistent shape. +% +% Inputs: +% rawFiles - The decoded array: struct array, cell array of structs, +% or empty. +% +% Outputs: +% files - Nx1 struct array with fields: +% uid - char +% uploaded - logical +% size - double, bytes +% crc32 - char, base64 of the big-endian CRC-32 of the +% content; '' when the cloud has none (compare +% size instead) +% Entries without a uid are skipped. +% +% See also: ndi.cloud.api.files.lookupFiles, ndi.fun.file.crc32 + + arguments + rawFiles + end + + files = struct('uid', {}, 'uploaded', {}, 'size', {}, 'crc32', {}); + if isempty(rawFiles) + files = reshape(files, 0, 1); + return + end + + n = numel(rawFiles); + for i = 1:n + if iscell(rawFiles) + f = rawFiles{i}; + else + f = rawFiles(i); + end + if ~isstruct(f) || ~isfield(f, 'uid') || isempty(f.uid) + continue + end + + entry = struct('uid', char(f.uid), 'uploaded', false, 'size', 0, 'crc32', ''); + if isfield(f, 'uploaded') && ~isempty(f.uploaded) + entry.uploaded = logical(f.uploaded); + end + if isfield(f, 'size') && ~isempty(f.size) + entry.size = double(f.size); + end + if isfield(f, 'crc32') && ~isempty(f.crc32) + entry.crc32 = char(f.crc32); + end + files(end+1, 1) = entry; %#ok + end + files = reshape(files, [], 1); +end diff --git a/src/ndi/+ndi/+cloud/+api/+implementation/+files/lookupFilesResult.m b/src/ndi/+ndi/+cloud/+api/+implementation/+files/lookupFilesResult.m new file mode 100644 index 000000000..189110ee0 --- /dev/null +++ b/src/ndi/+ndi/+cloud/+api/+implementation/+files/lookupFilesResult.m @@ -0,0 +1,43 @@ +function result = lookupFilesResult(decodedData) +%LOOKUPFILESRESULT Turn a POST /files/lookup response body into a result struct. +% +% RESULT = ndi.cloud.api.implementation.files.LOOKUPFILESRESULT(DECODEDDATA) +% +% Takes the decoded body rather than the ResponseMessage, so a test can +% exercise it without a server. +% +% Inputs: +% decodedData - The decoded response body (apiResponse.Body.Data): a +% struct with fields `files` and `missing`. +% +% Outputs: +% result - struct with fields: +% files - Nx1 struct array (uid, uploaded, size, crc32); +% see ndi.cloud.api.implementation.files.fileMetadataArray +% missing - 1xM string array of requested uids that have no +% file record in the dataset +% +% See also: ndi.cloud.api.files.lookupFiles + + arguments + decodedData + end + + rawFiles = []; + rawMissing = []; + if isstruct(decodedData) + if isfield(decodedData, 'files') + rawFiles = decodedData.files; + end + if isfield(decodedData, 'missing') + rawMissing = decodedData.missing; + end + end + + result.files = ndi.cloud.api.implementation.files.fileMetadataArray(rawFiles); + if isempty(rawMissing) + result.missing = strings(1, 0); + else + result.missing = reshape(string(rawMissing), 1, []); + end +end diff --git a/src/ndi/+ndi/+cloud/+api/+implementation/+files/signedURLSetPage.m b/src/ndi/+ndi/+cloud/+api/+implementation/+files/signedURLSetPage.m index e6935ccf3..f054a94c5 100644 --- a/src/ndi/+ndi/+cloud/+api/+implementation/+files/signedURLSetPage.m +++ b/src/ndi/+ndi/+cloud/+api/+implementation/+files/signedURLSetPage.m @@ -24,6 +24,10 @@ % the last page % expiresAt - char timestamp when the signed URLs expire, '' % if the server did not supply one +% fileInfo - Nx1 struct array (uid, uploaded, size, crc32) +% for this page's files; empty when the server +% did not send it. See +% ndi.cloud.api.implementation.files.fileMetadataArray % % See also: ndi.cloud.api.implementation.files.signedURLFileMap @@ -52,6 +56,14 @@ % Normalize the two fields whose absence is meaningful: jsondecode turns % a JSON null into [], and callers test these with isempty. + % fileInfo is an array of objects carrying the uid as a value, so + % JSONDECODE's key renaming does not touch it. + rawFileInfo = []; + if isstruct(decodedData) && isfield(decodedData, 'fileInfo') + rawFileInfo = decodedData.fileInfo; + end + page.fileInfo = ndi.cloud.api.implementation.files.fileMetadataArray(rawFileInfo); + page.nextCursor = localCharField(decodedData, 'nextCursor'); page.expiresAt = localCharField(decodedData, 'expiresAt'); end diff --git a/src/ndi/+ndi/+cloud/+api/url.m b/src/ndi/+ndi/+cloud/+api/url.m index ef145fad6..099d01a78 100644 --- a/src/ndi/+ndi/+cloud/+api/url.m +++ b/src/ndi/+ndi/+cloud/+api/url.m @@ -70,6 +70,7 @@ endpointMap("get_file_upload_url") = "/datasets/{organizationId}/{datasetId}/files/{file_uid}"; endpointMap("get_file_collection_upload_url") = "/datasets/{organizationId}/{datasetId}/files/bulk"; endpointMap("get_file_details") = "/datasets/{datasetId}/files/{file_uid}/detail"; + endpointMap("lookup_dataset_files") = "/datasets/{datasetId}/files/lookup"; endpointMap("get_bulk_upload_status") = "/bulk-uploads/{jobId}"; endpointMap("list_dataset_bulk_uploads") = "/datasets/{datasetId}/bulk-uploads?state={state}"; % Signed download URLs for the files a document references. The diff --git a/src/ndi/+ndi/+cloud/uploadSingleFile.m b/src/ndi/+ndi/+cloud/uploadSingleFile.m index 3699525fc..c6b37ad62 100644 --- a/src/ndi/+ndi/+cloud/uploadSingleFile.m +++ b/src/ndi/+ndi/+cloud/uploadSingleFile.m @@ -13,6 +13,12 @@ % Name-Value Pairs: % 'useBulkUpload' (logical) - If true, the file will be zipped and uploaded % using the bulk upload mechanism. Defaults to false. +% 'checksum' (logical) - If true (default), compute the file's CRC-32 +% (ndi.fun.file.crc32) and send it with the +% single-file upload URL request, so S3 rejects a +% corrupted upload and NDI Cloud records the +% file's crc32. Ignored for bulk uploads, whose +% zip entries carry their own CRC-32. % % Outputs: % B - true if the upload was successful, false otherwise. @@ -24,6 +30,7 @@ filePath (1,1) string {mustBeFile} options.useBulkUpload (1,1) logical = false options.useCurl (1,1) logical = true + options.checksum (1,1) logical = true end errormsg = ''; @@ -56,7 +63,12 @@ if isfile(zip_file), delete(zip_file); end else % Get the pre-signed URL for single file upload - [b_url, url_or_error] = ndi.cloud.api.files.getFileUploadURL(cloudDatasetID, cloudFileUID); + checksumCRC32 = ""; + if options.checksum + checksumCRC32 = string(ndi.fun.file.crc32(filePath)); + end + [b_url, url_or_error] = ndi.cloud.api.files.getFileUploadURL(cloudDatasetID, cloudFileUID, ... + 'checksumCRC32', checksumCRC32); if ~b_url b = false; @@ -65,7 +77,8 @@ end % Upload the file - [b_put, put_or_error] = ndi.cloud.api.files.putFiles(url_or_error, filePath, 'useCurl', options.useCurl); + [b_put, put_or_error] = ndi.cloud.api.files.putFiles(url_or_error, filePath, 'useCurl', options.useCurl, ... + 'checksumCRC32', checksumCRC32); if ~b_put b = false; diff --git a/src/ndi/+ndi/+fun/+file/crc32.m b/src/ndi/+ndi/+fun/+file/crc32.m new file mode 100644 index 000000000..421f424ed --- /dev/null +++ b/src/ndi/+ndi/+fun/+file/crc32.m @@ -0,0 +1,54 @@ +function checksum = crc32(fileName, options) +%CRC32 CRC-32 of a file, base64-encoded the way NDI Cloud and S3 store it. +% +% SYNTAX: +% checksum = ndi.fun.file.crc32(fileName) +% checksum = ndi.fun.file.crc32(fileName, 'chunkSize', BYTES) +% +% DESCRIPTION: +% Computes the CRC-32 (IEEE, the zlib / java.util.zip.CRC32 polynomial) of +% the file's bytes and returns the base64 of its 4-byte big-endian value. +% That is the encoding S3 uses for x-amz-checksum-crc32, the value +% ndi.cloud.api.files.getFileUploadURL sends as checksumCRC32, and the +% value NDI Cloud returns as a file's crc32 (from getFileDetails, listFiles, +% lookupFiles and getSignedURLSet). Comparing this to the cloud's crc32 +% tells whether the cloud copy has the same content as the local file. +% +% The file is read in chunks, so large files do not need to fit in memory. +% +% INPUTS: +% fileName - Path to the file. +% +% NAME-VALUE INPUTS: +% chunkSize - Bytes read per chunk (default 8 MiB). +% +% OUTPUTS: +% checksum - 1x8 char, e.g. 'NhCmhg==' for a file containing 'hello'. +% +% EXAMPLE: +% c = ndi.fun.file.crc32('chunk.bin_1'); +% +% See also: ndi.cloud.api.files.lookupFiles, ndi.cloud.api.files.getFileUploadURL + + arguments + fileName (1,1) string {mustBeFile} + options.chunkSize (1,1) double {mustBePositive, mustBeInteger} = 8*1024*1024 + end + + fid = fopen(fileName, 'r'); + if fid < 0 + error('NDI:fun:file:crc32:CannotOpen', 'Could not open file: %s', fileName); + end + closer = onCleanup(@() fclose(fid)); + + crc = java.util.zip.CRC32(); + while true + data = fread(fid, options.chunkSize, '*int8'); + if isempty(data) + break; + end + crc.update(data); + end + + checksum = ndi.fun.file.crc32ToBase64(crc.getValue()); +end diff --git a/src/ndi/+ndi/+fun/+file/crc32ToBase64.m b/src/ndi/+ndi/+fun/+file/crc32ToBase64.m new file mode 100644 index 000000000..c16ba7824 --- /dev/null +++ b/src/ndi/+ndi/+fun/+file/crc32ToBase64.m @@ -0,0 +1,28 @@ +function checksum = crc32ToBase64(value) +%CRC32TOBASE64 Encode a numeric CRC-32 as base64 of its 4-byte big-endian value. +% +% SYNTAX: +% checksum = ndi.fun.file.crc32ToBase64(value) +% +% DESCRIPTION: +% NDI Cloud and S3 represent a CRC-32 as the base64 of its four bytes in +% big-endian order. This converts a numeric CRC-32 (for example from +% java.util.zip.CRC32.getValue) to that form. +% +% INPUTS: +% value - The CRC-32 as a number in [0, 4294967295]. +% +% OUTPUTS: +% checksum - 1x8 char base64 string, e.g. 'NhCmhg==' for 0x3610A686. +% +% See also: ndi.fun.file.crc32 + + arguments + value (1,1) {mustBeNumeric, mustBeInteger, mustBeNonnegative, ... + mustBeLessThanOrEqual(value, 4294967295)} + end + + v = uint32(value); + bytes = uint8(bitand(bitshift(v, [-24 -16 -8 0]), uint32(255))); + checksum = char(matlab.net.base64encode(bytes)); +end diff --git a/src/ndi/docs/NDI-matlab/manual/developer/cloud.md b/src/ndi/docs/NDI-matlab/manual/developer/cloud.md index 4e732b0dc..0bdc1fb12 100644 --- a/src/ndi/docs/NDI-matlab/manual/developer/cloud.md +++ b/src/ndi/docs/NDI-matlab/manual/developer/cloud.md @@ -453,13 +453,24 @@ pyramid level is an O(1) uid lookup rather than another round trip. | Function | Signature | Description | |---|---|---| -| `ndi.cloud.api.files.getFileDetails` | `getFileDetails(cloudDatasetID, cloudFileUID)` | Retrieves metadata for a single file, including a pre-signed download URL. | +| `ndi.cloud.api.files.getFileDetails` | `getFileDetails(cloudDatasetID, cloudFileUID)` | Retrieves metadata for a single file, including a pre-signed download URL and `crc32` (see below). | | `ndi.cloud.api.files.getFile` | `getFile(downloadURL, downloadedFile, 'useCurl', true)` | Downloads a file from a pre-signed URL and saves it to a local path. `useCurl` defaults to `true` (curl requests identity encoding so already-compressed archives are not corrupted); set it to `false` to use MATLAB's native `websave`. | -| `ndi.cloud.api.files.getFileUploadURL` | `getFileUploadURL(cloudDatasetID, cloudFileUID)` | Returns a pre-signed URL for uploading a single file. | +| `ndi.cloud.api.files.getFileUploadURL` | `getFileUploadURL(cloudDatasetID, cloudFileUID, 'checksumCRC32', crc)` | Returns a pre-signed URL for uploading a single file. With `checksumCRC32` (from `ndi.fun.file.crc32`) the URL signs the `x-amz-checksum-crc32` header, so pass the same value to `putFiles(..., 'checksumCRC32', crc)`; S3 then rejects an upload whose content does not match, and the cloud records the value as the file's `crc32`. `ndi.cloud.uploadSingleFile` does both by default. | | `ndi.cloud.api.files.getFileCollectionUploadURL` | `getFileCollectionUploadURL(cloudDatasetID)` | Returns a pre-signed URL for uploading a ZIP archive of multiple files. | | `ndi.cloud.api.files.putFiles` | `putFiles(preSignedURL, filePath, 'useCurl', true)` | Uploads a local file to a pre-signed URL via HTTP PUT. `useCurl` defaults to `true` (consistent S3 object headers); set it to `false` to use MATLAB's native HTTP client. | | `ndi.cloud.api.files.listFiles` | `listFiles(cloudDatasetId, 'limit', L, 'after', cursor)` | Lists a single keyset page of a dataset's files. Returns the page plus a `cursor`/`hasMore` envelope; pass `cursor` back as `after` for the next page. | | `ndi.cloud.api.files.listFilesAll` | `listFilesAll(cloudDatasetId, 'limit', L)` | Lists **all** of a dataset's files by following the keyset cursor across pages, with optional polling for newly uploaded files. The whole-dataset counterpart to `listFiles`. | +| `ndi.cloud.api.files.lookupFiles` | `lookupFiles(cloudDatasetID, uids)` | Returns `uploaded`, `size` and `crc32` for the given file uids (`files`), and the uids the dataset has no record of (`missing`). No URLs are signed. Lists longer than the server's 5000-uid limit are split into several calls. Use it to check one document's or import batch's files, e.g. to resume an interrupted upload. | + +#### File checksums (`crc32`) + +NDI Cloud records a CRC-32 for each uploaded file: the value sent as +`checksumCRC32` for single-file uploads, or the zip entry's CRC-32 for bulk +uploads. It is the base64 of the 4-byte big-endian CRC-32 (IEEE), the same +encoding S3 uses, and `ndi.fun.file.crc32(filePath)` computes it for a local +file. `getFileDetails`, `listFiles`/`listFilesAll`, `lookupFiles` and the +`fileInfo` array of `getSignedURLSet`/`getSignedURLSetAll` return it. Files +uploaded without a checksum have `crc32` empty (`''`); compare `size` instead. ### Compute (`ndi.cloud.api.compute.*`) diff --git a/tests/+ndi/+unittest/+cloud/FilesTest.m b/tests/+ndi/+unittest/+cloud/FilesTest.m index f28989944..0de3fe254 100644 --- a/tests/+ndi/+unittest/+cloud/FilesTest.m +++ b/tests/+ndi/+unittest/+cloud/FilesTest.m @@ -48,6 +48,17 @@ function setupNewDataset(testCase) end end methods (Access = private) + function [localFilePath, fileUID] = makeLocalTestFile(testCase) + % A temporary file named by a fresh uid, containing TestFileContent. + tempFolder = testCase.applyFixture(matlab.unittest.fixtures.TemporaryFolderFixture); + fileUID = string(did.ido.unique_id()); + localFilePath = fullfile(tempFolder.Folder, fileUID); + fid = fopen(localFilePath, 'w'); + testCase.assertGreaterThan(fid, 0, "Could not create local test file."); + fprintf(fid, '%s', testCase.TestFileContent); + fclose(fid); + end + % Private helper for teardown, called by addTeardown. function deleteDatasetAfterTest(testCase) if testCase.KeepDataset @@ -839,6 +850,155 @@ function testBulkUploadWithWaitForCompletion(testCase) testCase.Narrative = narrative; end + function testUploadWithChecksumRoundTrip(testCase) + % NDI Cloud #162: a checksum sent with the upload URL request is + % recorded as the file's crc32 and returned by getFileDetails, + % listFiles and lookupFiles. + testCase.Narrative = "Begin testUploadWithChecksumRoundTrip"; + narrative = testCase.Narrative; + + [localFilePath, fileUID] = testCase.makeLocalTestFile(); + localCrc = ndi.fun.file.crc32(localFilePath); + narrative(end+1) = "Local file " + fileUID + " has crc32 " + localCrc; + + [b_url, ans_url, resp_url, url_url] = ndi.cloud.api.files.getFileUploadURL(... + testCase.DatasetID, fileUID, 'checksumCRC32', localCrc); + msg_url = ndi.unittest.cloud.APIMessage(narrative, b_url, ans_url, resp_url, url_url); + testCase.verifyTrue(b_url, "Failed to get a checksummed upload URL. " + msg_url); + if ~b_url, testCase.Narrative = narrative; return; end + + [b_put, ans_put, resp_put, url_put] = ndi.cloud.api.files.putFiles(ans_url, localFilePath, ... + "useCurl", true, "checksumCRC32", localCrc); + msg_put = ndi.unittest.cloud.APIMessage(narrative, b_put, ans_put, resp_put, url_put); + testCase.verifyTrue(b_put, "Upload with a matching checksum failed. " + msg_put); + if ~b_put, testCase.Narrative = narrative; return; end + narrative(end+1) = "Uploaded with checksumCRC32."; + pause(10); % Give server time to process the file + + % getFileDetails + [b_d, ans_d, resp_d, url_d] = ndi.cloud.api.files.getFileDetails(testCase.DatasetID, fileUID); + msg_d = ndi.unittest.cloud.APIMessage(narrative, b_d, ans_d, resp_d, url_d); + testCase.verifyTrue(b_d, "getFileDetails failed. " + msg_d); + if b_d + testCase.verifyTrue(isfield(ans_d, 'crc32'), "getFileDetails has no crc32. " + msg_d); + if isfield(ans_d, 'crc32') + testCase.verifyEqual(char(ans_d.crc32), localCrc, "getFileDetails crc32 differs from the local file. " + msg_d); + end + end + + % listFiles (keyset page) + [b_l, ans_l, resp_l, url_l] = ndi.cloud.api.files.listFiles(testCase.DatasetID, 'limit', 10); + msg_l = ndi.unittest.cloud.APIMessage(narrative, b_l, ans_l, resp_l, url_l); + testCase.verifyTrue(b_l, "listFiles failed. " + msg_l); + if b_l + row = ans_l(strcmp({ans_l.uid}, char(fileUID))); + testCase.verifyNumElements(row, 1, "listFiles does not list the file. " + msg_l); + if isscalar(row) + testCase.verifyEqual(row.crc32, localCrc, "listFiles crc32 differs from the local file. " + msg_l); + end + end + + % lookupFiles, with one uid the dataset does not have + missingUID = string(did.ido.unique_id()); + [b_k, ans_k, resp_k, url_k] = ndi.cloud.api.files.lookupFiles(testCase.DatasetID, [fileUID missingUID]); + msg_k = ndi.unittest.cloud.APIMessage(narrative, b_k, ans_k, resp_k, url_k); + testCase.verifyTrue(b_k, "lookupFiles failed. " + msg_k); + if b_k + testCase.verifyNumElements(ans_k.files, 1, "lookupFiles should find exactly the uploaded file. " + msg_k); + if isscalar(ans_k.files) + testCase.verifyEqual(ans_k.files(1).uid, char(fileUID), msg_k); + testCase.verifyTrue(ans_k.files(1).uploaded, "lookupFiles: file not marked uploaded. " + msg_k); + testCase.verifyEqual(ans_k.files(1).size, double(strlength(testCase.TestFileContent)), ... + "lookupFiles size differs from the local file. " + msg_k); + testCase.verifyEqual(ans_k.files(1).crc32, localCrc, "lookupFiles crc32 differs from the local file. " + msg_k); + end + testCase.verifyEqual(ans_k.missing, missingUID, "lookupFiles should report the unknown uid as missing. " + msg_k); + end + + testCase.Narrative = narrative; + end + + function testUploadWithWrongChecksumIsRejected(testCase) + % NDI Cloud #162: the upload URL is signed with the checksum, so + % S3 refuses content that does not match it. + testCase.Narrative = "Begin testUploadWithWrongChecksumIsRejected"; + narrative = testCase.Narrative; + + [localFilePath, fileUID] = testCase.makeLocalTestFile(); + wrongCrc = ndi.fun.file.crc32ToBase64(0); + testCase.assertNotEqual(ndi.fun.file.crc32(localFilePath), wrongCrc); + + [b_url, ans_url, resp_url, url_url] = ndi.cloud.api.files.getFileUploadURL(... + testCase.DatasetID, fileUID, 'checksumCRC32', wrongCrc); + msg_url = ndi.unittest.cloud.APIMessage(narrative, b_url, ans_url, resp_url, url_url); + testCase.verifyTrue(b_url, "Failed to get a checksummed upload URL. " + msg_url); + if ~b_url, testCase.Narrative = narrative; return; end + + [b_put, ans_put, resp_put, url_put] = ndi.cloud.api.files.putFiles(ans_url, localFilePath, ... + "useCurl", true, "checksumCRC32", wrongCrc); + msg_put = ndi.unittest.cloud.APIMessage(narrative, b_put, ans_put, resp_put, url_put); + testCase.verifyFalse(b_put, "S3 accepted content whose CRC-32 does not match the signed checksum. " + msg_put); + + testCase.Narrative = narrative; + end + + function testUploadSingleFileRecordsChecksum(testCase) + % ndi.cloud.uploadSingleFile sends the checksum by default. + testCase.Narrative = "Begin testUploadSingleFileRecordsChecksum"; + narrative = testCase.Narrative; + + [localFilePath, fileUID] = testCase.makeLocalTestFile(); + [b_up, err_up] = ndi.cloud.uploadSingleFile(testCase.DatasetID, fileUID, localFilePath); + testCase.verifyTrue(b_up, "uploadSingleFile failed: " + string(err_up)); + if ~b_up, testCase.Narrative = narrative; return; end + pause(10); + + [b_k, ans_k, resp_k, url_k] = ndi.cloud.api.files.lookupFiles(testCase.DatasetID, fileUID); + msg_k = ndi.unittest.cloud.APIMessage(narrative, b_k, ans_k, resp_k, url_k); + testCase.verifyTrue(b_k, "lookupFiles failed. " + msg_k); + if b_k + testCase.verifyNumElements(ans_k.files, 1, msg_k); + if isscalar(ans_k.files) + testCase.verifyEqual(ans_k.files(1).crc32, ndi.fun.file.crc32(localFilePath), ... + "uploadSingleFile did not record the file's crc32. " + msg_k); + end + end + testCase.Narrative = narrative; + end + + function testLookupFilesRejectsAnInvalidBody(testCase) + % The server refuses more than 5000 uids; the client splits longer + % lists, so drive the server check with a raw request. + testCase.Narrative = "Begin testLookupFilesRejectsAnInvalidBody"; + + token = ndi.cloud.authenticate(); + apiURL = ndi.cloud.api.url('lookup_dataset_files', 'dataset_id', testCase.DatasetID); + headers = [matlab.net.http.HeaderField('accept','application/json'), ... + matlab.net.http.field.ContentTypeField(matlab.net.http.MediaType('application/json')), ... + matlab.net.http.HeaderField('Authorization', ['Bearer ' token])]; + body = matlab.net.http.MessageBody(struct('uids', "u" + string(1:5001))); + resp = send(matlab.net.http.RequestMessage(matlab.net.http.RequestMethod.POST, headers, body), apiURL); + testCase.verifyEqual(double(resp.StatusCode), 400, "5001 uids should be refused with 400."); + + body = matlab.net.http.MessageBody(struct('notUids', 1)); + resp = send(matlab.net.http.RequestMessage(matlab.net.http.RequestMethod.POST, headers, body), apiURL); + testCase.verifyEqual(double(resp.StatusCode), 400, "A body without uids should be refused with 400."); + end + + function testLookupFilesSplitsLongLists(testCase) + % More than one server call's worth of uids, none uploaded. + testCase.Narrative = "Begin testLookupFilesSplitsLongLists"; + + uids = "nouid" + string(1:5003); + [b, answer, resp, url] = ndi.cloud.api.files.lookupFiles(testCase.DatasetID, uids); + msg = ndi.unittest.cloud.APIMessage(testCase.Narrative, b, answer, resp, url); + testCase.verifyTrue(b, "lookupFiles over two calls failed. " + msg); + if b + testCase.verifyEmpty(answer.files, msg); + testCase.verifyEqual(sort(answer.missing), sort(uids), msg); + end + end + function testListFilesWithOptions(testCase) testCase.Narrative = "Begin testListFilesWithOptions"; narrative = testCase.Narrative; diff --git a/tests/+ndi/+unittest/+cloud/SignedURLSetTest.m b/tests/+ndi/+unittest/+cloud/SignedURLSetTest.m index 7f38a5a5b..371d1fa62 100644 --- a/tests/+ndi/+unittest/+cloud/SignedURLSetTest.m +++ b/tests/+ndi/+unittest/+cloud/SignedURLSetTest.m @@ -274,6 +274,37 @@ function testGetSignedURLSetContentRoundTrip(testCase) testCase.Narrative = narrative; end + % ------------------------------------------------------------------ + % Sync path: per-file metadata beside the URL map (NDI Cloud #162) + % ------------------------------------------------------------------ + function testGetSignedURLSetFileInfo(testCase) + testCase.Narrative = "Begin testGetSignedURLSetFileInfo"; + narrative = testCase.Narrative; + + [b, page, resp, url] = ndi.cloud.api.files.getSignedURLSet(... + testCase.DatasetID, testCase.CloudDocumentID); + msg = ndi.unittest.cloud.APIMessage(narrative, b, page, resp, url); + testCase.verifyTrue(b, "getSignedURLSet returned failure. " + msg); + if ~b, testCase.Narrative = narrative; return; end + + % An environment without fileInfo (NDI Cloud #162) fails here: the + % empty list does not match the URL map. + urlUIDs = sort(ndi.unittest.cloud.SignedURLSetTest.returnedUIDs(page.files)); + infoUIDs = sort(string({page.fileInfo.uid})); + testCase.verifyEqual(infoUIDs, urlUIDs, ... + "fileInfo should describe exactly the files on the page. " + msg); + for i = 1:numel(page.fileInfo) + f = page.fileInfo(i); + testCase.verifyTrue(f.uploaded, "fileInfo " + f.uid + " not marked uploaded. " + msg); + testCase.verifyGreaterThan(f.size, 0, "fileInfo " + f.uid + " has no size. " + msg); + % The files were uploaded after #162, through the bulk path, + % which records each zip entry's CRC-32. + testCase.verifyNotEmpty(f.crc32, "fileInfo " + f.uid + " has no crc32. " + msg); + end + + testCase.Narrative = narrative; + end + % ------------------------------------------------------------------ % Sync path: oversized limit is server-clamped % ------------------------------------------------------------------ @@ -355,18 +386,10 @@ function testCreateAndWaitForSignedURLSetJob(testCase) testCase.DatasetID, testCase.CloudDocumentID); msg = ndi.unittest.cloud.APIMessage(narrative, b, job, resp, url); - % Environments where the worker isn't provisioned return - % 500 CONFIGURATION_ERROR / ENQUEUE_ERROR / JOB_CREATION_ERROR. - % Skip rather than fail so this test stays meaningful on - % partial deployments. + % An environment without the worker returns 500 + % CONFIGURATION_ERROR / ENQUEUE_ERROR / JOB_CREATION_ERROR, + % which fails here. if ~b - if isstruct(job) && isfield(job, 'code') && ... - any(strcmp(job.code, {'CONFIGURATION_ERROR','ENQUEUE_ERROR','JOB_CREATION_ERROR'})) - testCase.assumeFail(... - "Signed URL set worker is not configured on this environment; skipping async path. " + msg); - testCase.Narrative = narrative; - return; - end testCase.verifyTrue(b, "createSignedURLSetJob returned failure. " + msg); testCase.Narrative = narrative; return; diff --git a/tests/+ndi/+unittest/FileCrc32OfflineTest.m b/tests/+ndi/+unittest/FileCrc32OfflineTest.m new file mode 100644 index 000000000..3eb79bec0 --- /dev/null +++ b/tests/+ndi/+unittest/FileCrc32OfflineTest.m @@ -0,0 +1,232 @@ +classdef FileCrc32OfflineTest < matlab.unittest.TestCase +% FILECRC32OFFLINETEST - offline tests for file CRC-32 support (NDI Cloud #162). +% +% Covers the parts that need no live cloud: computing a local file's CRC-32 +% in the cloud's encoding, the lookupFiles request and response handling, +% the checksumCRC32 query parameter on the upload URL request, and the +% signed-url-set fileInfo field. +% +% The live round trip (upload with a checksum, then read it back from +% getFileDetails, listFiles and lookupFiles) is in +% tests/+ndi/+unittest/+cloud/FilesTest.m. +% +% Expected CRC values were computed with Python's zlib.crc32, the same +% CRC-32 (IEEE) S3 uses. +% +% Authored without a local MATLAB runtime; needs MATLAB to validate/run. + + methods (Static, Access = private) + function filePath = writeTempFile(testCase, bytes) + folder = testCase.applyFixture(matlab.unittest.fixtures.TemporaryFolderFixture).Folder; + filePath = fullfile(folder, 'payload.bin'); + fid = fopen(filePath, 'w'); + fwrite(fid, uint8(bytes), 'uint8'); + fclose(fid); + end + end + + methods (Test) + + % ---- encoding ----------------------------------------------------- + + function testCrc32ToBase64IsBigEndian(testCase) + testCase.verifyEqual(ndi.fun.file.crc32ToBase64(hex2dec('3610A686')), 'NhCmhg=='); + testCase.verifyEqual(ndi.fun.file.crc32ToBase64(0), 'AAAAAA=='); + testCase.verifyEqual(ndi.fun.file.crc32ToBase64(4294967295), '/////w=='); + end + + function testCrc32ToBase64RejectsOutOfRange(testCase) + testCase.verifyError(@() ndi.fun.file.crc32ToBase64(-1), ?MException); + testCase.verifyError(@() ndi.fun.file.crc32ToBase64(2^32), ?MException); + end + + % ---- local file CRC ---------------------------------------------- + + function testCrc32OfKnownText(testCase) + p = ndi.unittest.FileCrc32OfflineTest.writeTempFile(testCase, double('hello')); + testCase.verifyEqual(ndi.fun.file.crc32(p), 'NhCmhg=='); + end + + function testCrc32OfEmptyFile(testCase) + p = ndi.unittest.FileCrc32OfflineTest.writeTempFile(testCase, []); + testCase.verifyEqual(ndi.fun.file.crc32(p), 'AAAAAA=='); + end + + function testCrc32OfBytesWithTheHighBitSet(testCase) + % Bytes 128-255 are read as negative int8 and handed to Java as + % byte[]; the CRC must still be of the unsigned bytes. + p = ndi.unittest.FileCrc32OfflineTest.writeTempFile(testCase, 0:255); + testCase.verifyEqual(ndi.fun.file.crc32(p), 'KQWMcw=='); + end + + function testCrc32DoesNotDependOnChunkSize(testCase) + p = ndi.unittest.FileCrc32OfflineTest.writeTempFile(testCase, ... + double('The quick brown fox jumps over the lazy dog')); + testCase.verifyEqual(ndi.fun.file.crc32(p), 'QU+jOQ=='); + testCase.verifyEqual(ndi.fun.file.crc32(p, 'chunkSize', 1), 'QU+jOQ=='); + testCase.verifyEqual(ndi.fun.file.crc32(p, 'chunkSize', 7), 'QU+jOQ=='); + end + + function testCrc32AcrossManyChunks(testCase) + p = ndi.unittest.FileCrc32OfflineTest.writeTempFile(testCase, repmat(0:255, 1, 1000)); + testCase.verifyEqual(ndi.fun.file.crc32(p, 'chunkSize', 4096), '/HCvGg=='); + end + + % ---- upload URL request ------------------------------------------ + + function testUploadUrlCarriesAnEncodedChecksum(testCase) + call = ndi.cloud.api.implementation.files.GetFileUploadURL(... + 'cloudDatasetID', "ds1", 'cloudFileUID', "uid1", ... + 'checksumCRC32', "QU+jOQ=="); + u = string(call.buildURL("org1")); + testCase.verifyTrue(contains(u, "/datasets/org1/ds1/files/uid1")); + testCase.verifyTrue(contains(u, "checksumCRC32=")); + % An unencoded '+' would reach the server as a space. + testCase.verifyFalse(contains(u, "QU+jOQ"), ... + 'the base64 checksum must be percent-encoded'); + end + + function testUploadUrlWithoutChecksumHasNoQuery(testCase) + call = ndi.cloud.api.implementation.files.GetFileUploadURL(... + 'cloudDatasetID', "ds1", 'cloudFileUID', "uid1"); + u = string(call.buildURL("org1")); + testCase.verifyFalse(contains(u, "checksumCRC32")); + end + + % ---- PUT with a checksum header ---------------------------------- + + function testPutSendsTheChecksumHeader(testCase) + p = ndi.unittest.FileCrc32OfflineTest.writeTempFile(testCase, double('hello')); + call = ndi.cloud.api.implementation.files.PutFiles(... + 'preSignedURL', "https://bucket.s3.amazonaws.com/k?X-Amz-Signature=abc", ... + 'filePath', p, 'checksumCRC32', "NhCmhg=="); + cmd = string(call.curlArguments()); + testCase.verifyTrue(contains(cmd, '-H "x-amz-checksum-crc32: NhCmhg=="'), ... + 'the signed x-amz-checksum-crc32 header must be sent'); + end + + function testPutWithoutChecksumSendsNoHeader(testCase) + p = ndi.unittest.FileCrc32OfflineTest.writeTempFile(testCase, double('hello')); + call = ndi.cloud.api.implementation.files.PutFiles(... + 'preSignedURL', "https://bucket.s3.amazonaws.com/k?X-Amz-Signature=abc", ... + 'filePath', p); + testCase.verifyFalse(contains(string(call.curlArguments()), "x-amz-checksum-crc32")); + end + + function testPutRefusesAMalformedChecksum(testCase) + % The value goes into a shell command, so only a real base64 + % CRC-32 is accepted. + p = ndi.unittest.FileCrc32OfflineTest.writeTempFile(testCase, double('hello')); + testCase.verifyError(@() ndi.cloud.api.implementation.files.PutFiles(... + 'preSignedURL', "https://bucket.s3.amazonaws.com/k", ... + 'filePath', p, 'checksumCRC32', "abc"" && rm -rf x"), ... + 'NDI:CloudApi:PutFiles:InvalidChecksum'); + end + + % ---- lookupFiles request ----------------------------------------- + + function testLookupEndpoint(testCase) + u = ndi.cloud.api.url('lookup_dataset_files', 'dataset_id', "ds1"); + testCase.verifyTrue(endsWith(string(u), "/datasets/ds1/files/lookup")); + end + + function testLookupBodyIsAJsonArrayEvenForOneUid(testCase) + call = ndi.cloud.api.implementation.files.LookupFiles(... + 'cloudDatasetID', "ds1", 'cloudFileUIDs', "4192a3c0dd1b4e00_3fe8a1b2c3d4e5f6"); + txt = jsonencode(call.requestBody()); + decoded = jsondecode(txt); + testCase.verifyTrue(iscell(decoded.uids) || numel(string(decoded.uids)) > 1, ... + 'uids must encode as a JSON array, not a bare string'); + testCase.verifyEqual(unique(string(decoded.uids)), "4192a3c0dd1b4e00_3fe8a1b2c3d4e5f6"); + end + + function testLookupBodyKeepsEveryUid(testCase) + uids = ["a1", "b2", "c3"]; + call = ndi.cloud.api.implementation.files.LookupFiles(... + 'cloudDatasetID', "ds1", 'cloudFileUIDs', uids); + decoded = jsondecode(jsonencode(call.requestBody())); + testCase.verifyEqual(reshape(string(decoded.uids), 1, []), uids); + end + + function testLookupRefusesMoreThanTheServerLimit(testCase) + uids = "u" + string(1:5001); + testCase.verifyError(@() ndi.cloud.api.implementation.files.LookupFiles(... + 'cloudDatasetID', "ds1", 'cloudFileUIDs', uids), ... + 'NDI:CloudApi:LookupFiles:TooManyUids'); + end + + function testLookupRefusesNoUids(testCase) + testCase.verifyError(@() ndi.cloud.api.implementation.files.LookupFiles(... + 'cloudDatasetID', "ds1", 'cloudFileUIDs', strings(1, 0)), ... + 'NDI:CloudApi:LookupFiles:NoUids'); + end + + function testLookupFilesWithNoUidsMakesNoCall(testCase) + % No request is sent, so this needs no credentials. + [b, answer, resp, url] = ndi.cloud.api.files.lookupFiles("ds1", strings(1, 0)); + testCase.verifyTrue(b); + testCase.verifyEmpty(answer.files); + testCase.verifyEmpty(answer.missing); + testCase.verifyEmpty(resp); + testCase.verifyEmpty(url); + end + + % ---- lookupFiles response ---------------------------------------- + + function testLookupResultNormalizesFilesAndMissing(testCase) + txt = ['{"files":[' ... + '{"uid":"4192a3c0dd1b4e00_3fe8a1b2c3d4e5f6","uploaded":true,"size":5,"crc32":"NhCmhg=="},' ... + '{"uid":"aa","uploaded":false,"size":0,"crc32":null}],' ... + '"missing":["zz"]}']; + r = ndi.cloud.api.implementation.files.lookupFilesResult(jsondecode(txt)); + + testCase.verifySize(r.files, [2 1]); + testCase.verifyEqual(r.files(1).uid, '4192a3c0dd1b4e00_3fe8a1b2c3d4e5f6'); + testCase.verifyEqual(r.files(1).uploaded, true); + testCase.verifyEqual(r.files(1).size, 5); + testCase.verifyEqual(r.files(1).crc32, 'NhCmhg=='); + testCase.verifyEqual(r.files(2).uploaded, false); + testCase.verifyEqual(r.files(2).crc32, '', 'a null crc32 becomes '''''); + testCase.verifyEqual(r.missing, "zz"); + end + + function testLookupResultWithNothingFound(testCase) + r = ndi.cloud.api.implementation.files.lookupFilesResult(... + jsondecode('{"files":[],"missing":["a","b"]}')); + testCase.verifyEmpty(r.files); + testCase.verifyTrue(all(isfield(r.files, {'uid','uploaded','size','crc32'}))); + testCase.verifyEqual(r.missing, ["a" "b"]); + end + + function testLookupResultWithNothingMissing(testCase) + r = ndi.cloud.api.implementation.files.lookupFilesResult(... + jsondecode('{"files":[{"uid":"a","uploaded":true,"size":1,"crc32":"AAAAAA=="}],"missing":[]}')); + testCase.verifySize(r.files, [1 1]); + testCase.verifyEqual(r.missing, strings(1, 0)); + end + + % ---- signed-url-set fileInfo ------------------------------------- + + function testSignedURLSetPageCarriesFileInfo(testCase) + uid = '4192a3c0dd1b4e00_3fe8a1b2c3d4e5f6'; + txt = ['{"files":{"' uid '":"https://s3/x"},' ... + '"fileInfo":[{"uid":"' uid '","uploaded":true,"size":5,"crc32":"NhCmhg=="}],' ... + '"nextCursor":null}']; + page = ndi.cloud.api.implementation.files.signedURLSetPage(jsondecode(txt), txt); + + testCase.verifySize(page.fileInfo, [1 1]); + testCase.verifyEqual(page.fileInfo(1).uid, uid, ... + 'a uid starting with a digit must survive (it is a value, not a key)'); + testCase.verifyEqual(page.fileInfo(1).crc32, 'NhCmhg=='); + testCase.verifyEqual(page.files(uid), 'https://s3/x'); + end + + function testSignedURLSetPageWithoutFileInfo(testCase) + % A server that predates fileInfo: the field is present and empty. + txt = '{"files":{"9a":"u1"},"nextCursor":null}'; + page = ndi.cloud.api.implementation.files.signedURLSetPage(jsondecode(txt), txt); + testCase.verifyTrue(isfield(page, 'fileInfo')); + testCase.verifyEmpty(page.fileInfo); + end + end +end