From 9baf2d52f48b875e8691e4b8325289049ac11080 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 18 Apr 2025 14:52:11 -0400 Subject: [PATCH 01/16] File Metadata Update --- .../11392-edit-file-metadata-empty-values.md | 11 ++ doc/sphinx-guides/source/api/changelog.rst | 1 + doc/sphinx-guides/source/api/native-api.rst | 31 +++++ .../edu/harvard/iq/dataverse/api/Files.java | 67 ++++++++++ .../datasetutility/OptionalFileParams.java | 68 +++------- .../edu/harvard/iq/dataverse/api/FilesIT.java | 123 ++++++++++++++++++ .../edu/harvard/iq/dataverse/api/UtilIT.java | 11 +- .../OptionalFileParamsTest.java | 7 +- 8 files changed, 265 insertions(+), 54 deletions(-) create mode 100644 doc/release-notes/11392-edit-file-metadata-empty-values.md diff --git a/doc/release-notes/11392-edit-file-metadata-empty-values.md b/doc/release-notes/11392-edit-file-metadata-empty-values.md new file mode 100644 index 00000000000..66059444e4f --- /dev/null +++ b/doc/release-notes/11392-edit-file-metadata-empty-values.md @@ -0,0 +1,11 @@ +### Edit File Metadata empty values should clear data + +Previously the API POST /files/{id}/metadata would ignore fields with empty values. Now the API updates the fields with the empty values essentially clearing the data. Missing fields will still be ignored. + +This feature also adds a new version of the POST endpoint (/files/{id}/metadata/version/{datasetVersion}) to specify the dataset version to make the file metadata change to. + +datasetVersion can either be the actual ID (12345) or the friendly version (1.0) + +Note that certain fields (i.e. dataFileTags) are not versioned and changes to these will update the published as well as draft versions of the file. + +See also [the guides](https://dataverse-guide--11359.org.readthedocs.build/en/11359/api/native-api.html#updating-file-metadata), #11392, and #11359. diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst index 9bcf0cbfaa6..ddef58cd9fa 100644 --- a/doc/sphinx-guides/source/api/changelog.rst +++ b/doc/sphinx-guides/source/api/changelog.rst @@ -11,6 +11,7 @@ v6.7 ---- - An undocumented :doc:`search` parameter called "show_my_data" has been removed. It was never exercised by tests and is believed to be unused. API users should use the :ref:`api-mydata` API instead. +- For POST /api/files/{id}/metadata passing an empty string (“description”:””) or array (“categories”:[]) will no longer be ignored. Empty fields will now clear out the values in the file's metadata. To ignore the fields simply do not include them in the Json string. v6.6 ---- diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 95041ab5304..aba31670aa1 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -4612,6 +4612,10 @@ Updating File Metadata Updates the file metadata for an existing file where ``ID`` is the database id of the file to update or ``PERSISTENT_ID`` is the persistent id (DOI or Handle) of the file. Requires a ``jsonString`` expressing the new metadata. No metadata from the previous version of this file will be persisted, so if you want to update a specific field first get the json with the above command and alter the fields you want. +An extended version of this API will allow for the Dataset Version ID to be specified in order to modify fields of a Datafile in an already published version of the Dataset. + +Note: As of Dataverse 6.7 passing in an empty value for a string field ("description":"") or an empty array for a list ("categories":[]) will clear the data for that field. In prior versions these fields would be ignored. To ignore the fields simply leave them out of the ``jsonString``. + A curl example using an ``ID`` .. code-block:: bash @@ -4656,6 +4660,33 @@ Note: To update the 'tabularTags' property of file metadata, use the 'dataFileTa Also note that dataFileTags are not versioned and changes to these will update the published version of the file. +Extended version of the API: + +This extended version of the API will allow the user to modify fields of a specific version of the file's metadata. + +The Dataset version id can be either the ID of the Dataset version (i.e. 12345) or the Friendly version (i.e. 1.0). + +As noted above the dataFileTags are not versioned and any change to them for this version will also change them in the draft version. It is not recommended to change them with this API. + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=24 + export DATASET_VERSION_ID=1.0 + + curl -H "X-Dataverse-key:$API_TOKEN" -X POST \ + -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"]}' \ + "$SERVER_URL/api/files/$ID/metadata/version/$DATASET_VERSION_ID" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \ + -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"]}' \ + "https://demo.dataverse.org/api/files/:persistentId/metadata/version/1.0?persistentId=doi:10.5072/FK2/AAA000" + .. _EditingVariableMetadata: Updating File Metadata Categories diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java index 61a69236f57..e7459c1b325 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java @@ -45,6 +45,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Level; import java.util.logging.Logger; @@ -531,7 +532,73 @@ public Response updateFileMetadata(@Context ContainerRequestContext crc, @FormDa .type(MediaType.TEXT_PLAIN) //Our plain text string is already json .build(); } + @POST + @AuthRequired + @Path("{id}/metadata/version/{datasetVersionId}") + public Response updateFileMetadata(@Context ContainerRequestContext crc, @FormDataParam("jsonData") String jsonData, + @PathParam("id") String fileIdOrPersistentId, + @PathParam("datasetVersionId") String datasetVersionId) { + DataverseRequest req; + try { + req = createDataverseRequest(getRequestUser(crc)); + } catch (Exception e) { + return error(BAD_REQUEST, "Error attempting to request information. Maybe a bad API token?"); + } + final DataFile df; + FileMetadata fm = null; + try { + df = execCommand(new GetDataFileCommand(req, findDataFileOrDie(fileIdOrPersistentId))); + fm = df.getFileMetadata(); + for (FileMetadata md : df.getFileMetadatas()) { + if (datasetVersionId.equals(String.valueOf(md.getDatasetVersion().getId())) || + datasetVersionId.equals(md.getDatasetVersion().getFriendlyVersionNumber())) { + fm = md; + } + } + } catch (Exception e) { + return error(BAD_REQUEST, "Error attempting get the requested data file."); + } + + try { + OptionalFileParams optionalFileParams = null; + if (jsonData != null) { + // Load optional params via JSON + optionalFileParams = new OptionalFileParams(jsonData); + } else { + return error(BAD_REQUEST, "Missing Form Data!"); + } + List fmdListMinusCurrentFile = new ArrayList<>(); + for (FileMetadata fileMetadata : df.getFileMetadatas()) { + if (!fileMetadata.getDataFile().equals(df)) { + fmdListMinusCurrentFile.add(fileMetadata); + } + } + jakarta.json.JsonObject jsonObject = JsonUtil.getJsonObject(jsonData); + String incomingLabel = jsonObject.getString("label", null); + String incomingDirectoryLabel = jsonObject.getString("directoryLabel", null); + String existingLabel = fm.getLabel(); + String existingDirectoryLabel = fm.getDirectoryLabel(); + String pathPlusFilename = IngestUtil.getPathAndFileNameToCheck(incomingLabel, incomingDirectoryLabel, existingLabel, existingDirectoryLabel); + if (IngestUtil.conflictsWithExistingFilenames(pathPlusFilename, fmdListMinusCurrentFile)) { + return error(BAD_REQUEST, BundleUtil.getStringFromBundle("files.api.metadata.update.duplicateFile", Arrays.asList(pathPlusFilename))); + } + + optionalFileParams.addOptionalParams(fm); + execCommand(new UpdateDatasetVersionCommand(fm.getDataFile().getOwner(), req, fm)); + + } catch (DataFileTagException ex) { + return error(BAD_REQUEST, ex.getMessage()); + } catch (Exception e) { + return error(Response.Status.INTERNAL_SERVER_ERROR, "Error updating metadata to DataFile: " + e); + } + String jsonString = fm.asGsonObject(true).toString(); + return Response + .status(Response.Status.OK) + .entity("File Metadata update has been completed: " + jsonString) + .type(MediaType.TEXT_PLAIN) //Our plain text string is already json + .build(); + } @GET @AuthRequired @Path("{id}") diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java index 54844160163..f7df81b6386 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java @@ -194,46 +194,28 @@ public boolean getTabIngest() { return this.tabIngest; } - public boolean hasCategories(){ - if ((categories == null)||(this.categories.isEmpty())){ - return false; - } - return true; + public boolean hasCategories() { + return categories != null; } - public boolean hasFileDataTags(){ - if ((dataFileTags == null)||(this.dataFileTags.isEmpty())){ - return false; - } - return true; + public boolean hasFileDataTags() { + return dataFileTags != null; } public boolean hasDescription(){ - if ((description == null)||(this.description.isEmpty())){ - return false; - } - return true; + return description != null; } - public boolean hasDirectoryLabel(){ - if ((directoryLabel == null)||(this.directoryLabel.isEmpty())){ - return false; - } - return true; + public boolean hasDirectoryLabel() { + return directoryLabel != null; } - public boolean hasLabel(){ - if ((label == null)||(this.label.isEmpty())){ - return false; - } - return true; + public boolean hasLabel() { + return label != null; } - public boolean hasProvFreeform(){ - if ((provFreeForm == null)||(this.provFreeForm.isEmpty())){ - return false; - } - return true; + public boolean hasProvFreeform() { + return provFreeForm != null; } public boolean hasStorageIdentifier() { @@ -245,7 +227,7 @@ public String getStorageIdentifier() { } public boolean hasFileName() { - return ((fileName!=null)&&(!fileName.isEmpty())); + return fileName != null; } public String getFileName() { @@ -253,7 +235,7 @@ public String getFileName() { } public boolean hasMimetype() { - return ((mimeType!=null)&&(!mimeType.isEmpty())); + return mimeType != null; } public String getMimeType() { @@ -266,7 +248,7 @@ public void setCheckSum(String checkSum, ChecksumType type) { } public boolean hasCheckSum() { - return ((checkSumValue!=null)&&(!checkSumValue.isEmpty())); + return checkSumValue != null; } public String getCheckSum() { @@ -294,15 +276,10 @@ public void setFileSize(long fileSize) { * @param tags */ public void setCategories(List newCategories) { - if (newCategories != null) { newCategories = Util.removeDuplicatesNullsEmptyStrings(newCategories); - if (newCategories.isEmpty()) { - newCategories = null; - } + this.categories = newCategories; } - - this.categories = newCategories; } /** @@ -495,27 +472,20 @@ private void addFileDataTags(List potentialTags) throws DataFileTagExcep } potentialTags = Util.removeDuplicatesNullsEmptyStrings(potentialTags); - - if (potentialTags.isEmpty()){ - return; - } - + // Make a new list - this.dataFileTags = new ArrayList<>(); + List newList = new ArrayList<>(); // Add valid potential tags to the list for (String tagToCheck : potentialTags){ if (DataFileTag.isDataFileTag(tagToCheck)){ - this.dataFileTags.add(tagToCheck); + newList.add(tagToCheck); }else{ String errMsg = BundleUtil.getStringFromBundle("file.addreplace.error.invalid_datafile_tag"); throw new DataFileTagException(errMsg + " [" + tagToCheck + "]. Please use one of the following: " + DataFileTag.getListofLabelsAsString()); } } - // Shouldn't happen.... - if (dataFileTags.isEmpty()){ - dataFileTags = null; - } + this.dataFileTags = newList; } private void msg(String s){ diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java index d7cce94a0b8..de940c30e17 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java @@ -1,6 +1,7 @@ package edu.harvard.iq.dataverse.api; import edu.harvard.iq.dataverse.authorization.DataverseRole; +import edu.harvard.iq.dataverse.datasetutility.OptionalFileParams; import io.restassured.RestAssured; import io.restassured.response.Response; @@ -3189,4 +3190,126 @@ public void testFileCitationByVersion() throws IOException { } + // This test handles both updating with empty fields to clear the field and + // update a specific file metadata for dataset version + // Both APIs will be tested /api/files/{id}/metadata and /api/files/{id}/metadata/version/{datasetVersionId} + @Test + public void testUpdateSpecificMetadataVersionAndTestUpdateWithEmptyFields() { + // Create User, Dataverse, and Dataset + Response createUser = UtilIT.createRandomUser(); + createUser.then().assertThat().statusCode(OK.getStatusCode()); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode()); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + createDatasetResponse.prettyPrint(); + createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode()); + Integer datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id"); + + // Upload a tab file + JsonObjectBuilder json = Json.createObjectBuilder() + .add(OptionalFileParams.DESCRIPTION_ATTR_NAME, "my description") + .add(OptionalFileParams.DIRECTORY_LABEL_ATTR_NAME, "data/subdir1") + .add(OptionalFileParams.PROVENANCE_FREEFORM_ATTR_NAME, "prov Free Form") + .add(OptionalFileParams.CATEGORIES_ATTR_NAME, Json.createArrayBuilder().add("Data")); + String pathToTestFile = "src/test/resources/tab/test.tab"; + Response uploadFile = UtilIT.uploadFileViaNative(datasetId.toString(), pathToTestFile, json.build(), apiToken); + uploadFile.prettyPrint(); + uploadFile.then().assertThat().statusCode(OK.getStatusCode()); + Long fileId = JsonPath.from(uploadFile.body().asString()).getLong("data.files[0].dataFile.id"); + assertTrue(UtilIT.sleepForLock(datasetId, "Ingest", apiToken, UtilIT.MAXIMUM_INGEST_LOCK_DURATION), "Failed test if Ingest Lock exceeds max duration " + pathToTestFile); + + // Can't add tags until after the file is ingested and determined to be a tabular file + JsonObjectBuilder updateFileJson = Json.createObjectBuilder() + .add(OptionalFileParams.FILE_DATA_TAGS_ATTR_NAME, Json.createArrayBuilder().add("Survey")); + Response updateFileResponse = UtilIT.updateFileMetadata(String.valueOf(fileId), updateFileJson.build().toString(), apiToken); + updateFileResponse.prettyPrint(); + + // Get and verify the FileData + Response getFile = UtilIT.getFileData(String.valueOf(fileId), apiToken); + getFile.prettyPrint(); + getFile.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.description", equalTo("my description")) + .body("data.dataFile.description", equalTo("my description")) + .body("data.directoryLabel", equalTo("data/subdir1")) + .body("data.categories", hasItem("Data")) + .body("data.dataFile.tabularTags", hasItem("Survey")); + + // Publish the Dataverse and Dataset + Response publishResponse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken); + publishResponse.then().assertThat().statusCode(OK.getStatusCode()); + publishResponse = UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken); + publishResponse.then().assertThat().statusCode(OK.getStatusCode()); + + // Create a new DRAFT version + json = Json.createObjectBuilder() + .add(OptionalFileParams.DESCRIPTION_ATTR_NAME, "my 1.1 description"); + Response updateResponse = UtilIT.updateFileMetadata(String.valueOf(fileId), json.build().toString(), apiToken); + updateResponse.then().assertThat().statusCode(OK.getStatusCode()); + + // Publish the draft version as 1.1 + publishResponse = UtilIT.publishDatasetViaNativeApi(datasetId, "minor", apiToken); + publishResponse.then().assertThat().statusCode(OK.getStatusCode()); + publishResponse.prettyPrint(); + + // We now have a 1.0 version and 1.1 version. Now we modify the 1.0 version + String version = "1.0"; + json = Json.createObjectBuilder() + .add(OptionalFileParams.DESCRIPTION_ATTR_NAME, "") + .add(OptionalFileParams.LABEL_ATTR_NAME, "test.tab") + .add(OptionalFileParams.DIRECTORY_LABEL_ATTR_NAME, "") + .add(OptionalFileParams.PROVENANCE_FREEFORM_ATTR_NAME, "") + .add(OptionalFileParams.CATEGORIES_ATTR_NAME, Json.createArrayBuilder()) + .add(OptionalFileParams.FILE_DATA_TAGS_ATTR_NAME, Json.createArrayBuilder()); + + Response updateFile = UtilIT.updateFileMetadata(String.valueOf(fileId), json.build().toString(), apiToken, version); + updateFile.prettyPrint(); + + // Get version 1.0 and see the changes + getFile = UtilIT.getFileData(String.valueOf(fileId), apiToken, version); + getFile.prettyPrint(); + getFile.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.description", equalTo("")) + .body("data.dataFile.description", equalTo("")) + .body("data.directoryLabel", nullValue()) + .body("data.provFreeForm", nullValue()) + .body("data.categories", nullValue()) + .body("data.dataFile.tabularTags", nullValue()); + + // Get the latest version and see the original data unchanged (except description which is why the draft version was created) + getFile = UtilIT.getFileData(String.valueOf(fileId), apiToken); + getFile.prettyPrint(); + getFile.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.description", equalTo("my 1.1 description")) + .body("data.dataFile.description", equalTo("my 1.1 description")) + .body("data.directoryLabel", equalTo("data/subdir1")) + .body("data.categories", hasItem("Data")); + + // Update metadata (creating a draft version) without the version in the path to show that this endpoint also clears the fields + json = Json.createObjectBuilder() + .add(OptionalFileParams.DESCRIPTION_ATTR_NAME, "") + .add(OptionalFileParams.LABEL_ATTR_NAME, "test.tab") + .add(OptionalFileParams.DIRECTORY_LABEL_ATTR_NAME, "") + .add(OptionalFileParams.PROVENANCE_FREEFORM_ATTR_NAME, "") + .add(OptionalFileParams.CATEGORIES_ATTR_NAME, Json.createArrayBuilder()) + .add(OptionalFileParams.FILE_DATA_TAGS_ATTR_NAME, Json.createArrayBuilder()); + updateFile = UtilIT.updateFileMetadata(String.valueOf(fileId), json.build().toString(), apiToken); + updateFile.prettyPrint(); + getFile = UtilIT.getFileData(String.valueOf(fileId), apiToken); + getFile.prettyPrint(); + getFile.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.description", equalTo("")) + .body("data.dataFile.description", equalTo("")) + .body("data.directoryLabel", nullValue()) + .body("data.provFreeForm", nullValue()) + .body("data.categories", nullValue()) + .body("data.dataFile.tabularTags", nullValue()); + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index b9d403c528c..e28ece85a23 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -1088,21 +1088,28 @@ static Response deleteFileApi(Integer fileId, String apiToken) { .header(API_TOKEN_HTTP_HEADER, apiToken) .delete("/api/files/" + fileId); } - + static Response updateFileMetadata(String fileIdOrPersistentId, String jsonAsString, String apiToken) { + return updateFileMetadata(fileIdOrPersistentId, jsonAsString,apiToken, null); + } + static Response updateFileMetadata(String fileIdOrPersistentId, String jsonAsString, String apiToken, String datasetVersionId) { String idInPath = fileIdOrPersistentId; // Assume it's a number. + String versionInPath = ""; String optionalQueryParam = ""; // If idOrPersistentId is a number we'll just put it in the path. if (!NumberUtils.isCreatable(fileIdOrPersistentId)) { idInPath = ":persistentId"; optionalQueryParam = "?persistentId=" + fileIdOrPersistentId; } + if (datasetVersionId != null) { + versionInPath = "/version/" + datasetVersionId; + } RequestSpecification requestSpecification = given() .header(API_TOKEN_HTTP_HEADER, apiToken); if (jsonAsString != null) { requestSpecification.multiPart("jsonData", jsonAsString); } return requestSpecification - .post("/api/files/" + idInPath + "/metadata" + optionalQueryParam); + .post("/api/files/" + idInPath + "/metadata" + versionInPath + optionalQueryParam); } static Response downloadFile(Integer fileId) { diff --git a/src/test/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParamsTest.java b/src/test/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParamsTest.java index c9f251f7e77..cbca33f409d 100644 --- a/src/test/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParamsTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParamsTest.java @@ -195,8 +195,9 @@ public void test_09_unusedParamsGood() throws DataFileTagException { assertNull(instance.getDescription()); assertFalse(instance.hasDescription()); - assertNull(instance.getCategories()); - assertFalse(instance.hasCategories()); + assertNotNull(instance.getCategories()); + assertTrue(instance.hasCategories()); + assertTrue(instance.getCategories().isEmpty()); assertNull(instance.getDataFileTags()); assertFalse(instance.hasFileDataTags()); @@ -292,4 +293,4 @@ private void msgt(String s){ print json.dumps(json.dumps(d)) -*/ \ No newline at end of file +*/ From 6d871aabb8c0d5514bdba794384c0ce26df989b4 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 18 Apr 2025 17:05:23 -0400 Subject: [PATCH 02/16] new version checking --- .../11392-edit-file-metadata-empty-values.md | 6 +- doc/sphinx-guides/source/api/native-api.rst | 36 +-------- .../edu/harvard/iq/dataverse/api/Files.java | 75 ++----------------- src/main/java/propertyFiles/Bundle.properties | 1 + .../edu/harvard/iq/dataverse/api/FilesIT.java | 69 +++++------------ .../edu/harvard/iq/dataverse/api/UtilIT.java | 4 +- 6 files changed, 33 insertions(+), 158 deletions(-) diff --git a/doc/release-notes/11392-edit-file-metadata-empty-values.md b/doc/release-notes/11392-edit-file-metadata-empty-values.md index 66059444e4f..e42bef13aec 100644 --- a/doc/release-notes/11392-edit-file-metadata-empty-values.md +++ b/doc/release-notes/11392-edit-file-metadata-empty-values.md @@ -2,10 +2,6 @@ Previously the API POST /files/{id}/metadata would ignore fields with empty values. Now the API updates the fields with the empty values essentially clearing the data. Missing fields will still be ignored. -This feature also adds a new version of the POST endpoint (/files/{id}/metadata/version/{datasetVersion}) to specify the dataset version to make the file metadata change to. - -datasetVersion can either be the actual ID (12345) or the friendly version (1.0) - -Note that certain fields (i.e. dataFileTags) are not versioned and changes to these will update the published as well as draft versions of the file. +An optional query parameter was added to ensure the metadata update doesn't overwrite stale data. See also [the guides](https://dataverse-guide--11359.org.readthedocs.build/en/11359/api/native-api.html#updating-file-metadata), #11392, and #11359. diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index aba31670aa1..9bf7b3679bf 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -4612,9 +4612,7 @@ Updating File Metadata Updates the file metadata for an existing file where ``ID`` is the database id of the file to update or ``PERSISTENT_ID`` is the persistent id (DOI or Handle) of the file. Requires a ``jsonString`` expressing the new metadata. No metadata from the previous version of this file will be persisted, so if you want to update a specific field first get the json with the above command and alter the fields you want. -An extended version of this API will allow for the Dataset Version ID to be specified in order to modify fields of a Datafile in an already published version of the Dataset. - -Note: As of Dataverse 6.7 passing in an empty value for a string field ("description":"") or an empty array for a list ("categories":[]) will clear the data for that field. In prior versions these fields would be ignored. To ignore the fields simply leave them out of the ``jsonString``. +Optional Parameter for verifying that the Dataset Version being edited is the latest version can be added &datasetVersionId=12345. This is to prevent stale data from being edited. A curl example using an ``ID`` @@ -4643,10 +4641,11 @@ A curl example using a ``PERSISTENT_ID`` export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx export SERVER_URL=https://demo.dataverse.org export PERSISTENT_ID=doi:10.5072/FK2/AAA000 + export VERSION_ID=12345 curl -H "X-Dataverse-key:$API_TOKEN" -X POST \ -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"dataFileTags":["Survey"],"restrict":false}' \ - "$SERVER_URL/api/files/:persistentId/metadata?persistentId=$PERSISTENT_ID" + "$SERVER_URL/api/files/:persistentId/metadata?persistentId=$PERSISTENT_ID&datasetVersionId=$VERSION_ID" The fully expanded example above (without environment variables) looks like this: @@ -4654,39 +4653,12 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \ -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"dataFileTags":["Survey"],"restrict":false}' \ - "https://demo.dataverse.org/api/files/:persistentId/metadata?persistentId=doi:10.5072/FK2/AAA000" + "https://demo.dataverse.org/api/files/:persistentId/metadata?persistentId=doi:10.5072/FK2/AAA000&datasetVersionId=12345" Note: To update the 'tabularTags' property of file metadata, use the 'dataFileTags' key when making API requests. This property is used to update the 'tabularTags' of the file metadata. Also note that dataFileTags are not versioned and changes to these will update the published version of the file. -Extended version of the API: - -This extended version of the API will allow the user to modify fields of a specific version of the file's metadata. - -The Dataset version id can be either the ID of the Dataset version (i.e. 12345) or the Friendly version (i.e. 1.0). - -As noted above the dataFileTags are not versioned and any change to them for this version will also change them in the draft version. It is not recommended to change them with this API. - -.. code-block:: bash - - export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - export SERVER_URL=https://demo.dataverse.org - export ID=24 - export DATASET_VERSION_ID=1.0 - - curl -H "X-Dataverse-key:$API_TOKEN" -X POST \ - -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"]}' \ - "$SERVER_URL/api/files/$ID/metadata/version/$DATASET_VERSION_ID" - -The fully expanded example above (without environment variables) looks like this: - -.. code-block:: bash - - curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \ - -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"]}' \ - "https://demo.dataverse.org/api/files/:persistentId/metadata/version/1.0?persistentId=doi:10.5072/FK2/AAA000" - .. _EditingVariableMetadata: Updating File Metadata Categories diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java index e7459c1b325..cd106d728b1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java @@ -45,7 +45,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Level; import java.util.logging.Logger; @@ -411,8 +410,8 @@ public Response deleteFileInDataset(@Context ContainerRequestContext crc, @PathP @AuthRequired @Path("{id}/metadata") public Response updateFileMetadata(@Context ContainerRequestContext crc, @FormDataParam("jsonData") String jsonData, - @PathParam("id") String fileIdOrPersistentId - ) throws DataFileTagException, CommandException { + @PathParam("id") String fileIdOrPersistentId, @QueryParam("datasetVersionId") String datasetVersionId + ) throws CommandException { FileMetadata upFmd = null; @@ -430,6 +429,10 @@ public Response updateFileMetadata(@Context ContainerRequestContext crc, @FormDa return error(BAD_REQUEST, "Error attempting get the requested data file."); } + Long latestDatasetVersion = df.getFileMetadata().getDatasetVersion().getId(); + if (datasetVersionId != null && latestDatasetVersion != Long.valueOf(datasetVersionId)) { + return error(Response.Status.BAD_REQUEST, BundleUtil.getStringFromBundle("file.metadata.message.parallelUpdateError")); + } //You shouldn't be trying to edit a datafile that has been replaced List result = em.createNamedQuery("DataFile.findDataFileThatReplacedId", Long.class) @@ -532,73 +535,7 @@ public Response updateFileMetadata(@Context ContainerRequestContext crc, @FormDa .type(MediaType.TEXT_PLAIN) //Our plain text string is already json .build(); } - @POST - @AuthRequired - @Path("{id}/metadata/version/{datasetVersionId}") - public Response updateFileMetadata(@Context ContainerRequestContext crc, @FormDataParam("jsonData") String jsonData, - @PathParam("id") String fileIdOrPersistentId, - @PathParam("datasetVersionId") String datasetVersionId) { - DataverseRequest req; - try { - req = createDataverseRequest(getRequestUser(crc)); - } catch (Exception e) { - return error(BAD_REQUEST, "Error attempting to request information. Maybe a bad API token?"); - } - final DataFile df; - FileMetadata fm = null; - try { - df = execCommand(new GetDataFileCommand(req, findDataFileOrDie(fileIdOrPersistentId))); - fm = df.getFileMetadata(); - for (FileMetadata md : df.getFileMetadatas()) { - if (datasetVersionId.equals(String.valueOf(md.getDatasetVersion().getId())) || - datasetVersionId.equals(md.getDatasetVersion().getFriendlyVersionNumber())) { - fm = md; - } - } - } catch (Exception e) { - return error(BAD_REQUEST, "Error attempting get the requested data file."); - } - try { - OptionalFileParams optionalFileParams = null; - if (jsonData != null) { - // Load optional params via JSON - optionalFileParams = new OptionalFileParams(jsonData); - } else { - return error(BAD_REQUEST, "Missing Form Data!"); - } - List fmdListMinusCurrentFile = new ArrayList<>(); - for (FileMetadata fileMetadata : df.getFileMetadatas()) { - if (!fileMetadata.getDataFile().equals(df)) { - fmdListMinusCurrentFile.add(fileMetadata); - } - } - jakarta.json.JsonObject jsonObject = JsonUtil.getJsonObject(jsonData); - String incomingLabel = jsonObject.getString("label", null); - String incomingDirectoryLabel = jsonObject.getString("directoryLabel", null); - String existingLabel = fm.getLabel(); - String existingDirectoryLabel = fm.getDirectoryLabel(); - String pathPlusFilename = IngestUtil.getPathAndFileNameToCheck(incomingLabel, incomingDirectoryLabel, existingLabel, existingDirectoryLabel); - if (IngestUtil.conflictsWithExistingFilenames(pathPlusFilename, fmdListMinusCurrentFile)) { - return error(BAD_REQUEST, BundleUtil.getStringFromBundle("files.api.metadata.update.duplicateFile", Arrays.asList(pathPlusFilename))); - } - - optionalFileParams.addOptionalParams(fm); - execCommand(new UpdateDatasetVersionCommand(fm.getDataFile().getOwner(), req, fm)); - - } catch (DataFileTagException ex) { - return error(BAD_REQUEST, ex.getMessage()); - } catch (Exception e) { - return error(Response.Status.INTERNAL_SERVER_ERROR, "Error updating metadata to DataFile: " + e); - } - - String jsonString = fm.asGsonObject(true).toString(); - return Response - .status(Response.Status.OK) - .entity("File Metadata update has been completed: " + jsonString) - .type(MediaType.TEXT_PLAIN) //Our plain text string is already json - .build(); - } @GET @AuthRequired @Path("{id}") diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index d7fd94bf65c..18e2406c2a9 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -1945,6 +1945,7 @@ file.metaData.dataFile.dataTab.unf=UNF file.metaData.dataFile.dataTab.variables=Variables file.metaData.dataFile.dataTab.observations=Observations file.metaData.fileAccess=File Access: +file.metadata.message.parallelUpdateError=Changes cannot be saved. This metadata has been edited since this page was opened. To continue, copy your changes, refresh the page to see the recent updates, and re-enter any changes you want to save. file.addDescription=Add file description... file.tags=Tags file.editTags=Edit Tags diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java index de940c30e17..91246552886 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java @@ -3190,11 +3190,8 @@ public void testFileCitationByVersion() throws IOException { } - // This test handles both updating with empty fields to clear the field and - // update a specific file metadata for dataset version - // Both APIs will be tested /api/files/{id}/metadata and /api/files/{id}/metadata/version/{datasetVersionId} @Test - public void testUpdateSpecificMetadataVersionAndTestUpdateWithEmptyFields() { + public void testUpdateWithEmptyFieldsAndVersionCheck() { // Create User, Dataverse, and Dataset Response createUser = UtilIT.createRandomUser(); createUser.then().assertThat().statusCode(OK.getStatusCode()); @@ -3245,53 +3242,12 @@ public void testUpdateSpecificMetadataVersionAndTestUpdateWithEmptyFields() { publishResponse = UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken); publishResponse.then().assertThat().statusCode(OK.getStatusCode()); - // Create a new DRAFT version - json = Json.createObjectBuilder() - .add(OptionalFileParams.DESCRIPTION_ATTR_NAME, "my 1.1 description"); - Response updateResponse = UtilIT.updateFileMetadata(String.valueOf(fileId), json.build().toString(), apiToken); - updateResponse.then().assertThat().statusCode(OK.getStatusCode()); - - // Publish the draft version as 1.1 - publishResponse = UtilIT.publishDatasetViaNativeApi(datasetId, "minor", apiToken); - publishResponse.then().assertThat().statusCode(OK.getStatusCode()); - publishResponse.prettyPrint(); - - // We now have a 1.0 version and 1.1 version. Now we modify the 1.0 version - String version = "1.0"; - json = Json.createObjectBuilder() - .add(OptionalFileParams.DESCRIPTION_ATTR_NAME, "") - .add(OptionalFileParams.LABEL_ATTR_NAME, "test.tab") - .add(OptionalFileParams.DIRECTORY_LABEL_ATTR_NAME, "") - .add(OptionalFileParams.PROVENANCE_FREEFORM_ATTR_NAME, "") - .add(OptionalFileParams.CATEGORIES_ATTR_NAME, Json.createArrayBuilder()) - .add(OptionalFileParams.FILE_DATA_TAGS_ATTR_NAME, Json.createArrayBuilder()); - - Response updateFile = UtilIT.updateFileMetadata(String.valueOf(fileId), json.build().toString(), apiToken, version); - updateFile.prettyPrint(); - - // Get version 1.0 and see the changes - getFile = UtilIT.getFileData(String.valueOf(fileId), apiToken, version); - getFile.prettyPrint(); - getFile.then().assertThat() - .statusCode(OK.getStatusCode()) - .body("data.description", equalTo("")) - .body("data.dataFile.description", equalTo("")) - .body("data.directoryLabel", nullValue()) - .body("data.provFreeForm", nullValue()) - .body("data.categories", nullValue()) - .body("data.dataFile.tabularTags", nullValue()); - - // Get the latest version and see the original data unchanged (except description which is why the draft version was created) + // Get the base version getFile = UtilIT.getFileData(String.valueOf(fileId), apiToken); getFile.prettyPrint(); - getFile.then().assertThat() - .statusCode(OK.getStatusCode()) - .body("data.description", equalTo("my 1.1 description")) - .body("data.dataFile.description", equalTo("my 1.1 description")) - .body("data.directoryLabel", equalTo("data/subdir1")) - .body("data.categories", hasItem("Data")); + String datasetVersionId = String.valueOf(JsonPath.from(getFile.body().asString()).getInt("data.datasetVersionId")); - // Update metadata (creating a draft version) without the version in the path to show that this endpoint also clears the fields + // first user updates which creates a new DRAFT version json = Json.createObjectBuilder() .add(OptionalFileParams.DESCRIPTION_ATTR_NAME, "") .add(OptionalFileParams.LABEL_ATTR_NAME, "test.tab") @@ -3299,8 +3255,11 @@ public void testUpdateSpecificMetadataVersionAndTestUpdateWithEmptyFields() { .add(OptionalFileParams.PROVENANCE_FREEFORM_ATTR_NAME, "") .add(OptionalFileParams.CATEGORIES_ATTR_NAME, Json.createArrayBuilder()) .add(OptionalFileParams.FILE_DATA_TAGS_ATTR_NAME, Json.createArrayBuilder()); - updateFile = UtilIT.updateFileMetadata(String.valueOf(fileId), json.build().toString(), apiToken); - updateFile.prettyPrint(); + Response updateResponse = UtilIT.updateFileMetadata(String.valueOf(fileId), json.build().toString(), apiToken, datasetVersionId); + updateResponse.prettyPrint(); + updateResponse.then().assertThat().statusCode(OK.getStatusCode()); + + // Get the latest version getFile = UtilIT.getFileData(String.valueOf(fileId), apiToken); getFile.prettyPrint(); getFile.then().assertThat() @@ -3311,5 +3270,15 @@ public void testUpdateSpecificMetadataVersionAndTestUpdateWithEmptyFields() { .body("data.provFreeForm", nullValue()) .body("data.categories", nullValue()) .body("data.dataFile.tabularTags", nullValue()); + + // Second user updates the base version which should fail since it's already been updated + json = Json.createObjectBuilder() + .add(OptionalFileParams.DESCRIPTION_ATTR_NAME, "my new description"); + updateResponse = UtilIT.updateFileMetadata(String.valueOf(fileId), json.build().toString(), apiToken, datasetVersionId); + updateResponse.prettyPrint(); + updateResponse.then().assertThat() + .body("status", equalTo(ApiConstants.STATUS_ERROR)) + .body("message", equalTo(BundleUtil.getStringFromBundle("file.metadata.message.parallelUpdateError"))) + .statusCode(BAD_REQUEST.getStatusCode()); } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index e28ece85a23..935cc91a82e 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -1101,7 +1101,7 @@ static Response updateFileMetadata(String fileIdOrPersistentId, String jsonAsStr optionalQueryParam = "?persistentId=" + fileIdOrPersistentId; } if (datasetVersionId != null) { - versionInPath = "/version/" + datasetVersionId; + optionalQueryParam = (optionalQueryParam.isEmpty() ? "?" : "&") + "datasetVersionId=" + datasetVersionId; } RequestSpecification requestSpecification = given() .header(API_TOKEN_HTTP_HEADER, apiToken); @@ -1109,7 +1109,7 @@ static Response updateFileMetadata(String fileIdOrPersistentId, String jsonAsStr requestSpecification.multiPart("jsonData", jsonAsString); } return requestSpecification - .post("/api/files/" + idInPath + "/metadata" + versionInPath + optionalQueryParam); + .post("/api/files/" + idInPath + "/metadata" + optionalQueryParam); } static Response downloadFile(Integer fileId) { From fd800fc95d8d57a0192d8bd6ac49d62531dcaa92 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 18 Apr 2025 17:08:55 -0400 Subject: [PATCH 03/16] fix test --- src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 935cc91a82e..9a09a5edd9f 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -1094,14 +1094,13 @@ static Response updateFileMetadata(String fileIdOrPersistentId, String jsonAsStr } static Response updateFileMetadata(String fileIdOrPersistentId, String jsonAsString, String apiToken, String datasetVersionId) { String idInPath = fileIdOrPersistentId; // Assume it's a number. - String versionInPath = ""; String optionalQueryParam = ""; // If idOrPersistentId is a number we'll just put it in the path. if (!NumberUtils.isCreatable(fileIdOrPersistentId)) { idInPath = ":persistentId"; optionalQueryParam = "?persistentId=" + fileIdOrPersistentId; } if (datasetVersionId != null) { - optionalQueryParam = (optionalQueryParam.isEmpty() ? "?" : "&") + "datasetVersionId=" + datasetVersionId; + optionalQueryParam = optionalQueryParam + (optionalQueryParam.isEmpty() ? "?" : "&") + "datasetVersionId=" + datasetVersionId; } RequestSpecification requestSpecification = given() .header(API_TOKEN_HTTP_HEADER, apiToken); From 8dcb065ebd04f64c2516ca05184502b52f8231a4 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Tue, 22 Apr 2025 09:44:41 -0400 Subject: [PATCH 04/16] fix test --- src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java index 91246552886..81acfe67b6e 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java @@ -3255,7 +3255,7 @@ public void testUpdateWithEmptyFieldsAndVersionCheck() { .add(OptionalFileParams.PROVENANCE_FREEFORM_ATTR_NAME, "") .add(OptionalFileParams.CATEGORIES_ATTR_NAME, Json.createArrayBuilder()) .add(OptionalFileParams.FILE_DATA_TAGS_ATTR_NAME, Json.createArrayBuilder()); - Response updateResponse = UtilIT.updateFileMetadata(String.valueOf(fileId), json.build().toString(), apiToken, datasetVersionId); + Response updateResponse = UtilIT.updateFileMetadata(String.valueOf(fileId), json.build().toString(), apiToken); updateResponse.prettyPrint(); updateResponse.then().assertThat().statusCode(OK.getStatusCode()); From 2fce5fdc7498f83945dff07961d152913b96b2dd Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Tue, 22 Apr 2025 13:49:04 -0400 Subject: [PATCH 05/16] add to test --- .../java/edu/harvard/iq/dataverse/api/FilesIT.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java index 81acfe67b6e..60a65418a7a 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java @@ -3280,5 +3280,16 @@ public void testUpdateWithEmptyFieldsAndVersionCheck() { .body("status", equalTo(ApiConstants.STATUS_ERROR)) .body("message", equalTo(BundleUtil.getStringFromBundle("file.metadata.message.parallelUpdateError"))) .statusCode(BAD_REQUEST.getStatusCode()); + + // Second user refreshes and updates. Should pass now + getFile = UtilIT.getFileData(String.valueOf(fileId), apiToken); + getFile.prettyPrint(); + getFile.then().assertThat() + .statusCode(OK.getStatusCode()); + datasetVersionId = String.valueOf(JsonPath.from(getFile.body().asString()).getInt("data.datasetVersionId")); + updateResponse = UtilIT.updateFileMetadata(String.valueOf(fileId), json.build().toString(), apiToken, datasetVersionId); + updateResponse.prettyPrint(); + updateResponse.then().assertThat() + .statusCode(OK.getStatusCode()); } } From 26f07b726c2ccd48e395b8d4f9cd0e3b288e4427 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Tue, 22 Apr 2025 17:13:17 -0400 Subject: [PATCH 06/16] adding info for debugging jenkins test failure --- src/main/java/edu/harvard/iq/dataverse/api/Files.java | 7 +++++-- src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java index cd106d728b1..4fd9a8eaf2d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java @@ -430,8 +430,11 @@ public Response updateFileMetadata(@Context ContainerRequestContext crc, @FormDa } Long latestDatasetVersion = df.getFileMetadata().getDatasetVersion().getId(); - if (datasetVersionId != null && latestDatasetVersion != Long.valueOf(datasetVersionId)) { - return error(Response.Status.BAD_REQUEST, BundleUtil.getStringFromBundle("file.metadata.message.parallelUpdateError")); + if (datasetVersionId != null && latestDatasetVersion != null && + latestDatasetVersion != Long.valueOf(datasetVersionId)) { + return error(Response.Status.BAD_REQUEST, BundleUtil.getStringFromBundle("file.metadata.message.parallelUpdateError") + + " latestDatasetVersion:" + (latestDatasetVersion != null ? String.valueOf(latestDatasetVersion):"null") + + " datasetVersionId:" + datasetVersionId); } //You shouldn't be trying to edit a datafile that has been replaced diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java index 60a65418a7a..aca90f95bfe 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java @@ -3278,7 +3278,7 @@ public void testUpdateWithEmptyFieldsAndVersionCheck() { updateResponse.prettyPrint(); updateResponse.then().assertThat() .body("status", equalTo(ApiConstants.STATUS_ERROR)) - .body("message", equalTo(BundleUtil.getStringFromBundle("file.metadata.message.parallelUpdateError"))) + .body("message", startsWith(BundleUtil.getStringFromBundle("file.metadata.message.parallelUpdateError"))) .statusCode(BAD_REQUEST.getStatusCode()); // Second user refreshes and updates. Should pass now From 10939ce17d3501d91632efe3ad8b6a5f80b88772 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Wed, 23 Apr 2025 09:08:09 -0400 Subject: [PATCH 07/16] remove jenkins debug --- src/main/java/edu/harvard/iq/dataverse/api/Files.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java index 4fd9a8eaf2d..fcbf6ef21e0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java @@ -429,12 +429,10 @@ public Response updateFileMetadata(@Context ContainerRequestContext crc, @FormDa return error(BAD_REQUEST, "Error attempting get the requested data file."); } - Long latestDatasetVersion = df.getFileMetadata().getDatasetVersion().getId(); + String latestDatasetVersion = String.valueOf(df.getFileMetadata().getDatasetVersion().getId()); if (datasetVersionId != null && latestDatasetVersion != null && - latestDatasetVersion != Long.valueOf(datasetVersionId)) { - return error(Response.Status.BAD_REQUEST, BundleUtil.getStringFromBundle("file.metadata.message.parallelUpdateError") + - " latestDatasetVersion:" + (latestDatasetVersion != null ? String.valueOf(latestDatasetVersion):"null") + - " datasetVersionId:" + datasetVersionId); + !latestDatasetVersion.equals(datasetVersionId)) { + return error(Response.Status.BAD_REQUEST, BundleUtil.getStringFromBundle("file.metadata.message.parallelUpdateError")); } //You shouldn't be trying to edit a datafile that has been replaced From f5ddcff8a466122e2e2a11c7c521e18a21cd9209 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Thu, 24 Apr 2025 10:43:08 -0400 Subject: [PATCH 08/16] per review comments --- doc/sphinx-guides/source/api/native-api.rst | 6 +++--- .../harvard/iq/dataverse/api/AbstractApiBean.java | 9 +++++++++ .../java/edu/harvard/iq/dataverse/api/Files.java | 12 +++++++----- src/main/java/propertyFiles/Bundle.properties | 2 +- .../java/edu/harvard/iq/dataverse/api/FilesIT.java | 2 +- .../java/edu/harvard/iq/dataverse/api/UtilIT.java | 2 +- 6 files changed, 22 insertions(+), 11 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 9bf7b3679bf..c397b967be3 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -4612,7 +4612,7 @@ Updating File Metadata Updates the file metadata for an existing file where ``ID`` is the database id of the file to update or ``PERSISTENT_ID`` is the persistent id (DOI or Handle) of the file. Requires a ``jsonString`` expressing the new metadata. No metadata from the previous version of this file will be persisted, so if you want to update a specific field first get the json with the above command and alter the fields you want. -Optional Parameter for verifying that the Dataset Version being edited is the latest version can be added &datasetVersionId=12345. This is to prevent stale data from being edited. +Optional Parameter for verifying that the Dataset Version being edited is the latest version can be added &sourceInternalVersionNumber=12345. This is to prevent stale data from being edited. The value for sourceInternalVersionNumber comes from ``datasetVersionId`` in the response to get $SERVER_URL/api/files/$ID API call A curl example using an ``ID`` @@ -4645,7 +4645,7 @@ A curl example using a ``PERSISTENT_ID`` curl -H "X-Dataverse-key:$API_TOKEN" -X POST \ -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"dataFileTags":["Survey"],"restrict":false}' \ - "$SERVER_URL/api/files/:persistentId/metadata?persistentId=$PERSISTENT_ID&datasetVersionId=$VERSION_ID" + "$SERVER_URL/api/files/:persistentId/metadata?persistentId=$PERSISTENT_ID&sourceInternalVersionNumber=$VERSION_ID" The fully expanded example above (without environment variables) looks like this: @@ -4653,7 +4653,7 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \ -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"dataFileTags":["Survey"],"restrict":false}' \ - "https://demo.dataverse.org/api/files/:persistentId/metadata?persistentId=doi:10.5072/FK2/AAA000&datasetVersionId=12345" + "https://demo.dataverse.org/api/files/:persistentId/metadata?persistentId=doi:10.5072/FK2/AAA000&sourceInternalVersionNumber=12345" Note: To update the 'tabularTags' property of file metadata, use the 'dataFileTags' key when making API requests. This property is used to update the 'tabularTags' of the file metadata. diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index 8841ce3563b..d6391133240 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -451,6 +451,15 @@ protected void validateInternalVersionNumberIsNotOutdated(Dataset dataset, int i } } + protected void validateInternalVersionNumberIsNotOutdated(DataFile dataFile, int internalVersion) throws WrappedResponse { + logger.severe(">>>> internalVersion:"+internalVersion+" dataFile.getFileMetadata().getDatasetVersion().getId():"+dataFile.getFileMetadata().getDatasetVersion().getId()); + if (dataFile.getFileMetadata().getDatasetVersion().getId() > internalVersion) { + throw new WrappedResponse( + badRequest(BundleUtil.getStringFromBundle("abstractApiBean.error.datafileInternalVersionNumberIsOutdated", Collections.singletonList(Integer.toString(internalVersion)))) + ); + } + } + protected DataFile findDataFileOrDie(String id) throws WrappedResponse { DataFile datafile; if (id.equals(PERSISTENT_ID_KEY)) { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java index fcbf6ef21e0..33b380df865 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java @@ -410,7 +410,7 @@ public Response deleteFileInDataset(@Context ContainerRequestContext crc, @PathP @AuthRequired @Path("{id}/metadata") public Response updateFileMetadata(@Context ContainerRequestContext crc, @FormDataParam("jsonData") String jsonData, - @PathParam("id") String fileIdOrPersistentId, @QueryParam("datasetVersionId") String datasetVersionId + @PathParam("id") String fileIdOrPersistentId, @QueryParam("sourceInternalVersionNumber") Integer sourceInternalVersionNumber ) throws CommandException { FileMetadata upFmd = null; @@ -429,10 +429,12 @@ public Response updateFileMetadata(@Context ContainerRequestContext crc, @FormDa return error(BAD_REQUEST, "Error attempting get the requested data file."); } - String latestDatasetVersion = String.valueOf(df.getFileMetadata().getDatasetVersion().getId()); - if (datasetVersionId != null && latestDatasetVersion != null && - !latestDatasetVersion.equals(datasetVersionId)) { - return error(Response.Status.BAD_REQUEST, BundleUtil.getStringFromBundle("file.metadata.message.parallelUpdateError")); + if (sourceInternalVersionNumber != null) { + try { + validateInternalVersionNumberIsNotOutdated(df, sourceInternalVersionNumber); + } catch (WrappedResponse wr) { + return wr.getResponse(); + } } //You shouldn't be trying to edit a datafile that has been replaced diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 18e2406c2a9..09ea8d2e378 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -1945,7 +1945,6 @@ file.metaData.dataFile.dataTab.unf=UNF file.metaData.dataFile.dataTab.variables=Variables file.metaData.dataFile.dataTab.observations=Observations file.metaData.fileAccess=File Access: -file.metadata.message.parallelUpdateError=Changes cannot be saved. This metadata has been edited since this page was opened. To continue, copy your changes, refresh the page to see the recent updates, and re-enter any changes you want to save. file.addDescription=Add file description... file.tags=Tags file.editTags=Edit Tags @@ -3206,3 +3205,4 @@ updateDatasetFieldsCommand.api.processDatasetUpdate.parseError=Error parsing dat #AbstractApiBean.java abstractApiBean.error.datasetInternalVersionNumberIsOutdated=Dataset internal version number {0} is outdated +abstractApiBean.error.datafileInternalVersionNumberIsOutdated=File Metadata internal version number {0} is outdated diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java index aca90f95bfe..d8d4cf80535 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java @@ -3278,7 +3278,7 @@ public void testUpdateWithEmptyFieldsAndVersionCheck() { updateResponse.prettyPrint(); updateResponse.then().assertThat() .body("status", equalTo(ApiConstants.STATUS_ERROR)) - .body("message", startsWith(BundleUtil.getStringFromBundle("file.metadata.message.parallelUpdateError"))) + .body("message", equalTo(BundleUtil.getStringFromBundle("abstractApiBean.error.datafileInternalVersionNumberIsOutdated",Collections.singletonList(datasetVersionId)))) .statusCode(BAD_REQUEST.getStatusCode()); // Second user refreshes and updates. Should pass now diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 9a09a5edd9f..f9e97022d2e 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -1100,7 +1100,7 @@ static Response updateFileMetadata(String fileIdOrPersistentId, String jsonAsStr optionalQueryParam = "?persistentId=" + fileIdOrPersistentId; } if (datasetVersionId != null) { - optionalQueryParam = optionalQueryParam + (optionalQueryParam.isEmpty() ? "?" : "&") + "datasetVersionId=" + datasetVersionId; + optionalQueryParam = optionalQueryParam + (optionalQueryParam.isEmpty() ? "?" : "&") + "sourceInternalVersionNumber=" + datasetVersionId; } RequestSpecification requestSpecification = given() .header(API_TOKEN_HTTP_HEADER, apiToken); From 1615fb91238a08194cf7ea9a760c177cfd2e5ddc Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Thu, 24 Apr 2025 10:44:18 -0400 Subject: [PATCH 09/16] per review comments --- src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index d6391133240..7811eb68244 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -452,7 +452,6 @@ protected void validateInternalVersionNumberIsNotOutdated(Dataset dataset, int i } protected void validateInternalVersionNumberIsNotOutdated(DataFile dataFile, int internalVersion) throws WrappedResponse { - logger.severe(">>>> internalVersion:"+internalVersion+" dataFile.getFileMetadata().getDatasetVersion().getId():"+dataFile.getFileMetadata().getDatasetVersion().getId()); if (dataFile.getFileMetadata().getDatasetVersion().getId() > internalVersion) { throw new WrappedResponse( badRequest(BundleUtil.getStringFromBundle("abstractApiBean.error.datafileInternalVersionNumberIsOutdated", Collections.singletonList(Integer.toString(internalVersion)))) From 8a57fc8d9d5b7a346ea59bd471f42c3322d8acc6 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 25 Apr 2025 11:02:11 -0400 Subject: [PATCH 10/16] refactor to use last update timestamp instead of version number --- .../11392-edit-file-metadata-empty-values.md | 2 +- doc/sphinx-guides/source/api/native-api.rst | 8 ++--- .../iq/dataverse/api/AbstractApiBean.java | 15 +++++++-- .../edu/harvard/iq/dataverse/api/Files.java | 6 ++-- .../iq/dataverse/util/json/JsonPrinter.java | 3 +- src/main/java/propertyFiles/Bundle.properties | 2 +- .../edu/harvard/iq/dataverse/api/FilesIT.java | 32 ++++++++++++------- .../edu/harvard/iq/dataverse/api/UtilIT.java | 6 ++-- 8 files changed, 47 insertions(+), 27 deletions(-) diff --git a/doc/release-notes/11392-edit-file-metadata-empty-values.md b/doc/release-notes/11392-edit-file-metadata-empty-values.md index e42bef13aec..652aea63aed 100644 --- a/doc/release-notes/11392-edit-file-metadata-empty-values.md +++ b/doc/release-notes/11392-edit-file-metadata-empty-values.md @@ -2,6 +2,6 @@ Previously the API POST /files/{id}/metadata would ignore fields with empty values. Now the API updates the fields with the empty values essentially clearing the data. Missing fields will still be ignored. -An optional query parameter was added to ensure the metadata update doesn't overwrite stale data. +An optional query parameter (sourceInternalVersionTimestamp) was added to ensure the metadata update doesn't overwrite stale data. See also [the guides](https://dataverse-guide--11359.org.readthedocs.build/en/11359/api/native-api.html#updating-file-metadata), #11392, and #11359. diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index c397b967be3..a27b7ae3269 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -4612,7 +4612,7 @@ Updating File Metadata Updates the file metadata for an existing file where ``ID`` is the database id of the file to update or ``PERSISTENT_ID`` is the persistent id (DOI or Handle) of the file. Requires a ``jsonString`` expressing the new metadata. No metadata from the previous version of this file will be persisted, so if you want to update a specific field first get the json with the above command and alter the fields you want. -Optional Parameter for verifying that the Dataset Version being edited is the latest version can be added &sourceInternalVersionNumber=12345. This is to prevent stale data from being edited. The value for sourceInternalVersionNumber comes from ``datasetVersionId`` in the response to get $SERVER_URL/api/files/$ID API call +Optional Parameter for verifying that the Dataset Version being edited is the latest version can be added &sourceInternalVersionTimestamp=datetime(in format: "yyyy-MM-dd'T'HH:mm:ss'Z'"). This is to prevent stale data from being edited. The value for sourceInternalVersionTimestamp comes from ``lastUpdateTime`` in the response to get $SERVER_URL/api/files/$ID API call A curl example using an ``ID`` @@ -4641,11 +4641,11 @@ A curl example using a ``PERSISTENT_ID`` export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx export SERVER_URL=https://demo.dataverse.org export PERSISTENT_ID=doi:10.5072/FK2/AAA000 - export VERSION_ID=12345 + export UPDATE_TIME=2025-04-25T13:58:28Z curl -H "X-Dataverse-key:$API_TOKEN" -X POST \ -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"dataFileTags":["Survey"],"restrict":false}' \ - "$SERVER_URL/api/files/:persistentId/metadata?persistentId=$PERSISTENT_ID&sourceInternalVersionNumber=$VERSION_ID" + "$SERVER_URL/api/files/:persistentId/metadata?persistentId=$PERSISTENT_ID&sourceInternalVersionTimestamp=$UPDATE_TIME" The fully expanded example above (without environment variables) looks like this: @@ -4653,7 +4653,7 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \ -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"dataFileTags":["Survey"],"restrict":false}' \ - "https://demo.dataverse.org/api/files/:persistentId/metadata?persistentId=doi:10.5072/FK2/AAA000&sourceInternalVersionNumber=12345" + "https://demo.dataverse.org/api/files/:persistentId/metadata?persistentId=doi:10.5072/FK2/AAA000&sourceInternalVersionTimestamp=2025-04-25T13:58:28Z" Note: To update the 'tabularTags' property of file metadata, use the 'dataFileTags' key when making API requests. This property is used to update the 'tabularTags' of the file metadata. diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index 7811eb68244..0a7200662db 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -27,6 +27,7 @@ import edu.harvard.iq.dataverse.search.savedsearch.SavedSearchServiceBean; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; +import edu.harvard.iq.dataverse.util.DateUtil; import edu.harvard.iq.dataverse.util.FileUtil; import edu.harvard.iq.dataverse.util.SystemConfig; import edu.harvard.iq.dataverse.util.json.JsonParser; @@ -51,6 +52,7 @@ import java.io.InputStream; import java.net.URI; +import java.time.Instant; import java.util.*; import java.util.concurrent.Callable; import java.util.logging.Level; @@ -451,10 +453,17 @@ protected void validateInternalVersionNumberIsNotOutdated(Dataset dataset, int i } } - protected void validateInternalVersionNumberIsNotOutdated(DataFile dataFile, int internalVersion) throws WrappedResponse { - if (dataFile.getFileMetadata().getDatasetVersion().getId() > internalVersion) { + protected void validateInternalTimestampIsNotOutdated(DataFile dataFile, String sourceInternalVersionTimestamp) throws WrappedResponse { + Date date = sourceInternalVersionTimestamp != null ? DateUtil.parseDate(sourceInternalVersionTimestamp, "yyyy-MM-dd'T'HH:mm:ss'Z'") : null; + if (date == null) { throw new WrappedResponse( - badRequest(BundleUtil.getStringFromBundle("abstractApiBean.error.datafileInternalVersionNumberIsOutdated", Collections.singletonList(Integer.toString(internalVersion)))) + badRequest(BundleUtil.getStringFromBundle("jsonparser.error.parsing.date", Collections.singletonList(sourceInternalVersionTimestamp))) + ); + } + Instant instant = date.toInstant(); + if (dataFile.getFileMetadata().getDatasetVersion().getLastUpdateTime().toInstant().getEpochSecond() != instant.getEpochSecond()) { + throw new WrappedResponse( + badRequest(BundleUtil.getStringFromBundle("abstractApiBean.error.datafileInternalVersionTimestampIsOutdated", Collections.singletonList(sourceInternalVersionTimestamp))) ); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java index 33b380df865..b38d5375f66 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java @@ -410,7 +410,7 @@ public Response deleteFileInDataset(@Context ContainerRequestContext crc, @PathP @AuthRequired @Path("{id}/metadata") public Response updateFileMetadata(@Context ContainerRequestContext crc, @FormDataParam("jsonData") String jsonData, - @PathParam("id") String fileIdOrPersistentId, @QueryParam("sourceInternalVersionNumber") Integer sourceInternalVersionNumber + @PathParam("id") String fileIdOrPersistentId, @QueryParam("sourceInternalVersionTimestamp") String sourceInternalVersionTimestamp ) throws CommandException { FileMetadata upFmd = null; @@ -429,9 +429,9 @@ public Response updateFileMetadata(@Context ContainerRequestContext crc, @FormDa return error(BAD_REQUEST, "Error attempting get the requested data file."); } - if (sourceInternalVersionNumber != null) { + if (sourceInternalVersionTimestamp != null) { try { - validateInternalVersionNumberIsNotOutdated(df, sourceInternalVersionNumber); + validateInternalTimestampIsNotOutdated(df, sourceInternalVersionTimestamp); } catch (WrappedResponse wr) { return wr.getResponse(); } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index 0b5803d75d1..cc5bef0f4a3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -873,7 +873,8 @@ public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata, boo .add("tabularData", df.isTabularData()) .add("tabularTags", getTabularFileTags(df)) .add("creationDate", df.getCreateDateFormattedYYYYMMDD()) - .add("publicationDate", df.getPublicationDateFormattedYYYYMMDD()); + .add("publicationDate", df.getPublicationDateFormattedYYYYMMDD()) + .add("lastUpdateTime", format(fileMetadata.getDatasetVersion().getLastUpdateTime())); Dataset dfOwner = df.getOwner(); if (dfOwner != null) { builder.add("fileAccessRequest", dfOwner.isFileAccessRequest()); diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 09ea8d2e378..1036e1a836c 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -3205,4 +3205,4 @@ updateDatasetFieldsCommand.api.processDatasetUpdate.parseError=Error parsing dat #AbstractApiBean.java abstractApiBean.error.datasetInternalVersionNumberIsOutdated=Dataset internal version number {0} is outdated -abstractApiBean.error.datafileInternalVersionNumberIsOutdated=File Metadata internal version number {0} is outdated +abstractApiBean.error.datafileInternalVersionTimestampIsOutdated=File Metadata internal version timestamp {0} is outdated diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java index d8d4cf80535..de45362ce98 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java @@ -5,7 +5,9 @@ import io.restassured.RestAssured; import io.restassured.response.Response; -import java.util.List; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.*; import java.util.logging.Logger; import edu.harvard.iq.dataverse.api.auth.ApiKeyAuthMechanism; @@ -29,9 +31,6 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.text.MessageFormat; -import java.util.Arrays; -import java.util.Collections; -import java.util.Map; import jakarta.json.Json; import jakarta.json.JsonObjectBuilder; @@ -3191,7 +3190,7 @@ public void testFileCitationByVersion() throws IOException { } @Test - public void testUpdateWithEmptyFieldsAndVersionCheck() { + public void testUpdateWithEmptyFieldsAndVersionCheck() throws InterruptedException { // Create User, Dataverse, and Dataset Response createUser = UtilIT.createRandomUser(); createUser.then().assertThat().statusCode(OK.getStatusCode()); @@ -3245,7 +3244,7 @@ public void testUpdateWithEmptyFieldsAndVersionCheck() { // Get the base version getFile = UtilIT.getFileData(String.valueOf(fileId), apiToken); getFile.prettyPrint(); - String datasetVersionId = String.valueOf(JsonPath.from(getFile.body().asString()).getInt("data.datasetVersionId")); + String lastUpdateTime = String.valueOf(JsonPath.from(getFile.body().asString()).getString("data.dataFile.lastUpdateTime")); // first user updates which creates a new DRAFT version json = Json.createObjectBuilder() @@ -3255,9 +3254,10 @@ public void testUpdateWithEmptyFieldsAndVersionCheck() { .add(OptionalFileParams.PROVENANCE_FREEFORM_ATTR_NAME, "") .add(OptionalFileParams.CATEGORIES_ATTR_NAME, Json.createArrayBuilder()) .add(OptionalFileParams.FILE_DATA_TAGS_ATTR_NAME, Json.createArrayBuilder()); - Response updateResponse = UtilIT.updateFileMetadata(String.valueOf(fileId), json.build().toString(), apiToken); + Response updateResponse = UtilIT.updateFileMetadata(String.valueOf(fileId), json.build().toString(), apiToken, lastUpdateTime); updateResponse.prettyPrint(); updateResponse.then().assertThat().statusCode(OK.getStatusCode()); + Thread.sleep(1500); // Get the latest version getFile = UtilIT.getFileData(String.valueOf(fileId), apiToken); @@ -3270,15 +3270,17 @@ public void testUpdateWithEmptyFieldsAndVersionCheck() { .body("data.provFreeForm", nullValue()) .body("data.categories", nullValue()) .body("data.dataFile.tabularTags", nullValue()); + String latestUpdateTime = String.valueOf(JsonPath.from(getFile.body().asString()).getString("data.dataFile.lastUpdateTime")); + assertTrue(!latestUpdateTime.equalsIgnoreCase(lastUpdateTime)); // Second user updates the base version which should fail since it's already been updated json = Json.createObjectBuilder() .add(OptionalFileParams.DESCRIPTION_ATTR_NAME, "my new description"); - updateResponse = UtilIT.updateFileMetadata(String.valueOf(fileId), json.build().toString(), apiToken, datasetVersionId); + updateResponse = UtilIT.updateFileMetadata(String.valueOf(fileId), json.build().toString(), apiToken, lastUpdateTime); updateResponse.prettyPrint(); updateResponse.then().assertThat() .body("status", equalTo(ApiConstants.STATUS_ERROR)) - .body("message", equalTo(BundleUtil.getStringFromBundle("abstractApiBean.error.datafileInternalVersionNumberIsOutdated",Collections.singletonList(datasetVersionId)))) + .body("message", equalTo(BundleUtil.getStringFromBundle("abstractApiBean.error.datafileInternalVersionTimestampIsOutdated",Collections.singletonList(lastUpdateTime)))) .statusCode(BAD_REQUEST.getStatusCode()); // Second user refreshes and updates. Should pass now @@ -3286,10 +3288,18 @@ public void testUpdateWithEmptyFieldsAndVersionCheck() { getFile.prettyPrint(); getFile.then().assertThat() .statusCode(OK.getStatusCode()); - datasetVersionId = String.valueOf(JsonPath.from(getFile.body().asString()).getInt("data.datasetVersionId")); - updateResponse = UtilIT.updateFileMetadata(String.valueOf(fileId), json.build().toString(), apiToken, datasetVersionId); + lastUpdateTime = String.valueOf(JsonPath.from(getFile.body().asString()).getString("data.dataFile.lastUpdateTime")); + updateResponse = UtilIT.updateFileMetadata(String.valueOf(fileId), json.build().toString(), apiToken, lastUpdateTime); updateResponse.prettyPrint(); updateResponse.then().assertThat() .statusCode(OK.getStatusCode()); + + // Test invalid date + updateResponse = UtilIT.updateFileMetadata(String.valueOf(fileId), json.build().toString(), apiToken, "bad-date"); + updateResponse.prettyPrint(); + updateResponse.then().assertThat() + .body("status", equalTo(ApiConstants.STATUS_ERROR)) + .body("message", equalTo(BundleUtil.getStringFromBundle("jsonparser.error.parsing.date",Collections.singletonList("bad-date")))) + .statusCode(BAD_REQUEST.getStatusCode()); } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index f9e97022d2e..04d0b27d63c 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -1092,15 +1092,15 @@ static Response deleteFileApi(Integer fileId, String apiToken) { static Response updateFileMetadata(String fileIdOrPersistentId, String jsonAsString, String apiToken) { return updateFileMetadata(fileIdOrPersistentId, jsonAsString,apiToken, null); } - static Response updateFileMetadata(String fileIdOrPersistentId, String jsonAsString, String apiToken, String datasetVersionId) { + static Response updateFileMetadata(String fileIdOrPersistentId, String jsonAsString, String apiToken, String sourceInternalVersionTimestamp) { String idInPath = fileIdOrPersistentId; // Assume it's a number. String optionalQueryParam = ""; // If idOrPersistentId is a number we'll just put it in the path. if (!NumberUtils.isCreatable(fileIdOrPersistentId)) { idInPath = ":persistentId"; optionalQueryParam = "?persistentId=" + fileIdOrPersistentId; } - if (datasetVersionId != null) { - optionalQueryParam = optionalQueryParam + (optionalQueryParam.isEmpty() ? "?" : "&") + "sourceInternalVersionNumber=" + datasetVersionId; + if (sourceInternalVersionTimestamp != null) { + optionalQueryParam = optionalQueryParam + (optionalQueryParam.isEmpty() ? "?" : "&") + "sourceInternalVersionTimestamp=" + sourceInternalVersionTimestamp; } RequestSpecification requestSpecification = given() .header(API_TOKEN_HTTP_HEADER, apiToken); From 6fedf6e887d09b71618124119e8bac597a9deddd Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 25 Apr 2025 14:26:05 -0400 Subject: [PATCH 11/16] comment on data/timestamp compare --- src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index 0a7200662db..00dadb1cae4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -461,6 +461,7 @@ protected void validateInternalTimestampIsNotOutdated(DataFile dataFile, String ); } Instant instant = date.toInstant(); + // granularity is to the second since the json output only returns dates in this format to the second if (dataFile.getFileMetadata().getDatasetVersion().getLastUpdateTime().toInstant().getEpochSecond() != instant.getEpochSecond()) { throw new WrappedResponse( badRequest(BundleUtil.getStringFromBundle("abstractApiBean.error.datafileInternalVersionTimestampIsOutdated", Collections.singletonList(sourceInternalVersionTimestamp))) From eaf49a99bd234cddffe280090781133bbfca50ba Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Tue, 29 Apr 2025 14:50:11 -0400 Subject: [PATCH 12/16] refactor so both datafiles and datasets validate update timestamp the same way --- .../11243-editmetadata-api-extension.md | 4 +++- doc/sphinx-guides/source/api/native-api.rst | 12 ++++++------ .../iq/dataverse/api/AbstractApiBean.java | 18 +++++++----------- .../edu/harvard/iq/dataverse/api/Datasets.java | 8 +++++--- .../edu/harvard/iq/dataverse/api/Files.java | 5 ++--- src/main/java/propertyFiles/Bundle.properties | 3 +-- .../harvard/iq/dataverse/api/DatasetsIT.java | 15 ++++++++------- .../edu/harvard/iq/dataverse/api/FilesIT.java | 2 +- .../edu/harvard/iq/dataverse/api/UtilIT.java | 4 ++-- 9 files changed, 35 insertions(+), 36 deletions(-) diff --git a/doc/release-notes/11243-editmetadata-api-extension.md b/doc/release-notes/11243-editmetadata-api-extension.md index 6f5a2af283b..4aaf9320074 100644 --- a/doc/release-notes/11243-editmetadata-api-extension.md +++ b/doc/release-notes/11243-editmetadata-api-extension.md @@ -1,5 +1,7 @@ ### Edit Dataset Metadata API extension - This endpoint now allows removing fields (by sending empty values), as long as they are not required by the dataset. -- New ``sourceInternalVersionNumber`` optional query parameter, which prevents inconsistencies by managing updates that +- New ``sourceInternalVersionTimestamp`` optional query parameter, which prevents inconsistencies by managing updates that may occur from other users while a dataset is being edited. + +NOTE: This release note was updated to conform to the refactoring of the validation as part of issue #11392 diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index a27b7ae3269..352234d844a 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -2137,26 +2137,26 @@ For these edits your JSON file need only include those dataset fields which you This endpoint also allows removing fields, as long as they are not required by the dataset. To remove a field, send an empty value (``""``) for individual fields. For multiple fields, send an empty array (``[]``). A sample JSON file for removing fields may be downloaded here: :download:`dataset-edit-metadata-delete-fields-sample.json <../_static/api/dataset-edit-metadata-delete-fields-sample.json>` -If another user updates the dataset version metadata before you send the update request, data inconsistencies may occur. To prevent this, you can use the optional ``sourceInternalVersionNumber`` query parameter. This parameter must include the internal version number corresponding to the dataset version being updated. Note that internal version numbers increase sequentially with each version update. +If another user updates the dataset version metadata before you send the update request, data inconsistencies may occur. To prevent this, you can use the optional ``sourceInternalVersionTimestamp`` query parameter. This parameter must include the ``lastUpdateTime`` corresponding to the dataset version being updated. The date must be in this format "yyyy-MM-dd'T'HH:mm:ss'Z'" -If this parameter is provided, the update will proceed only if the internal version number remains unchanged. Otherwise, the request will fail with an error. +If this parameter is provided, the update will proceed only if the ``lastUpdateTime`` remains unchanged. Otherwise, the request will fail with an error. -Example using ``sourceInternalVersionNumber``: +Example using ``sourceInternalVersionTimestamp``: .. code-block:: bash export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx export SERVER_URL=https://demo.dataverse.org export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/BCCP9Z - export SOURCE_INTERNAL_VERSION_NUMBER=5 + export SOURCE_INTERNAL_VERSION_TIMESTAMP=2025-04-25T13:58:28Z - curl -H "X-Dataverse-key: $API_TOKEN" -X PUT "$SERVER_URL/api/datasets/:persistentId/editMetadata?persistentId=$PERSISTENT_IDENTIFIER&replace=true&sourceInternalVersionNumber=$SOURCE_INTERNAL_VERSION_NUMBER" --upload-file dataset-update-metadata.json + curl -H "X-Dataverse-key: $API_TOKEN" -X PUT "$SERVER_URL/api/datasets/:persistentId/editMetadata?persistentId=$PERSISTENT_IDENTIFIER&replace=true&sourceInternalVersionTimestamp=$SOURCE_INTERNAL_VERSION_TIMESTAMP" --upload-file dataset-update-metadata.json The fully expanded example above (without environment variables) looks like this: .. code-block:: bash - curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT "https://demo.dataverse.org/api/datasets/:persistentId/editMetadata/?persistentId=doi:10.5072/FK2/BCCP9Z&replace=true&sourceInternalVersionNumber=5" --upload-file dataset-update-metadata.json + curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT "https://demo.dataverse.org/api/datasets/:persistentId/editMetadata/?persistentId=doi:10.5072/FK2/BCCP9Z&replace=true&sourceInternalVersionTimestamp=2025-04-25T13:58:28Z" --upload-file dataset-update-metadata.json Delete Dataset Metadata diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index 00dadb1cae4..c53f9f8119c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -445,15 +445,7 @@ public Command handleLatestPublished() { return dsv; } - protected void validateInternalVersionNumberIsNotOutdated(Dataset dataset, int internalVersion) throws WrappedResponse { - if (dataset.getLatestVersion().getVersion() > internalVersion) { - throw new WrappedResponse( - badRequest(BundleUtil.getStringFromBundle("abstractApiBean.error.datasetInternalVersionNumberIsOutdated", Collections.singletonList(Integer.toString(internalVersion)))) - ); - } - } - - protected void validateInternalTimestampIsNotOutdated(DataFile dataFile, String sourceInternalVersionTimestamp) throws WrappedResponse { + protected void validateInternalTimestampIsNotOutdated(DvObject dvObject, String sourceInternalVersionTimestamp) throws WrappedResponse { Date date = sourceInternalVersionTimestamp != null ? DateUtil.parseDate(sourceInternalVersionTimestamp, "yyyy-MM-dd'T'HH:mm:ss'Z'") : null; if (date == null) { throw new WrappedResponse( @@ -461,10 +453,14 @@ protected void validateInternalTimestampIsNotOutdated(DataFile dataFile, String ); } Instant instant = date.toInstant(); + Instant updateTimestamp = + (dvObject instanceof DataFile) ? ((DataFile) dvObject).getFileMetadata().getDatasetVersion().getLastUpdateTime().toInstant() : + (dvObject instanceof Dataset) ? ((Dataset) dvObject).getLatestVersion().getLastUpdateTime().toInstant() : + instant; // granularity is to the second since the json output only returns dates in this format to the second - if (dataFile.getFileMetadata().getDatasetVersion().getLastUpdateTime().toInstant().getEpochSecond() != instant.getEpochSecond()) { + if (updateTimestamp.getEpochSecond() != instant.getEpochSecond()) { throw new WrappedResponse( - badRequest(BundleUtil.getStringFromBundle("abstractApiBean.error.datafileInternalVersionTimestampIsOutdated", Collections.singletonList(sourceInternalVersionTimestamp))) + badRequest(BundleUtil.getStringFromBundle("abstractApiBean.error.internalVersionTimestampIsOutdated", Collections.singletonList(sourceInternalVersionTimestamp))) ); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index bf04cdd6226..e36b50ad0e0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -1093,12 +1093,14 @@ private String getCompoundDisplayValue (DatasetFieldCompoundValue dscv){ @PUT @AuthRequired @Path("{id}/editMetadata") - public Response editVersionMetadata(@Context ContainerRequestContext crc, String jsonBody, @PathParam("id") String id, @QueryParam("replace") boolean replaceData, @QueryParam("sourceInternalVersionNumber") Integer sourceInternalVersionNumber) { + public Response editVersionMetadata(@Context ContainerRequestContext crc, String jsonBody, @PathParam("id") String id, + @QueryParam("replace") boolean replaceData, + @QueryParam("sourceInternalVersionTimestamp") String sourceInternalVersionTimestamp) { try { Dataset dataset = findDatasetOrDie(id); - if (sourceInternalVersionNumber != null) { - validateInternalVersionNumberIsNotOutdated(dataset, sourceInternalVersionNumber); + if (sourceInternalVersionTimestamp != null) { + validateInternalTimestampIsNotOutdated(dataset, sourceInternalVersionTimestamp); } JsonObject json = JsonUtil.getJsonObject(jsonBody); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java index b38d5375f66..1b809894e17 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java @@ -410,8 +410,7 @@ public Response deleteFileInDataset(@Context ContainerRequestContext crc, @PathP @AuthRequired @Path("{id}/metadata") public Response updateFileMetadata(@Context ContainerRequestContext crc, @FormDataParam("jsonData") String jsonData, - @PathParam("id") String fileIdOrPersistentId, @QueryParam("sourceInternalVersionTimestamp") String sourceInternalVersionTimestamp - ) throws CommandException { + @PathParam("id") String fileIdOrPersistentId, @QueryParam("sourceInternalVersionTimestamp") String sourceInternalVersionTimestamp) { FileMetadata upFmd = null; @@ -526,7 +525,7 @@ public Response updateFileMetadata(@Context ContainerRequestContext crc, @FormDa return error(Response.Status.INTERNAL_SERVER_ERROR, "Error adding metadata to DataFile: " + e); } - } catch (WrappedResponse wr) { + } catch (CommandException | WrappedResponse ex) { return error(BAD_REQUEST, "An error has occurred attempting to update the requested DataFile, likely related to permissions."); } diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 1036e1a836c..3dae5a8ed15 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -3204,5 +3204,4 @@ datasetFieldValidator.error.emptyRequiredSingleValueForField=Empty required valu updateDatasetFieldsCommand.api.processDatasetUpdate.parseError=Error parsing dataset update: {0} #AbstractApiBean.java -abstractApiBean.error.datasetInternalVersionNumberIsOutdated=Dataset internal version number {0} is outdated -abstractApiBean.error.datafileInternalVersionTimestampIsOutdated=File Metadata internal version timestamp {0} is outdated +abstractApiBean.error.internalVersionTimestampIsOutdated=Internal version timestamp {0} is outdated diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index b6ff3d9b401..13837bed859 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -671,25 +671,26 @@ public void testAddUpdateDatasetViaNativeAPI() { """; Response updateMetadataRemoveAlternativeTitles = UtilIT.editVersionMetadataFromJsonStr(datasetPersistentId, jsonString, apiToken); + updateMetadataRemoveAlternativeTitles.prettyPrint(); updateMetadataRemoveAlternativeTitles.then().assertThat() .body("data.metadataBlocks.citation.fields[2].typeName", not(equalTo("alternativeTitle"))) .statusCode(OK.getStatusCode()); - // Test sourceInternalVersionNumber optional query parameter - - Integer internalVersionNumber = updateMetadataRemoveAlternativeTitles.then().extract().path("data.internalVersionNumber"); - assertNotNull(internalVersionNumber); + // Test sourceInternalVersionTimestamp optional query parameter + String sourceInternalVersionTimestamp = updateMetadataRemoveAlternativeTitles.then().extract().path("data.lastUpdateTime"); + assertNotNull(sourceInternalVersionTimestamp); + String oldTimestamp = "2025-04-25T13:58:28Z"; // Case 1 - Pass outdated internal version number - Response updateMetadataWithOutdatedInternalVersionNumber = UtilIT.editVersionMetadataFromJsonStr(datasetPersistentId, jsonString, apiToken, internalVersionNumber - 1); + Response updateMetadataWithOutdatedInternalVersionNumber = UtilIT.editVersionMetadataFromJsonStr(datasetPersistentId, jsonString, apiToken, oldTimestamp); updateMetadataWithOutdatedInternalVersionNumber.then().assertThat() - .body("message", equalTo(BundleUtil.getStringFromBundle("abstractApiBean.error.datasetInternalVersionNumberIsOutdated", Collections.singletonList(Integer.toString(internalVersionNumber - 1))))) + .body("message", equalTo(BundleUtil.getStringFromBundle("abstractApiBean.error.internalVersionTimestampIsOutdated", Collections.singletonList(oldTimestamp)))) .statusCode(BAD_REQUEST.getStatusCode()); // Case 2 - Pass latest internal version number - Response updateMetadataWithLatestInternalVersionNumber = UtilIT.editVersionMetadataFromJsonStr(datasetPersistentId, jsonString, apiToken, internalVersionNumber); + Response updateMetadataWithLatestInternalVersionNumber = UtilIT.editVersionMetadataFromJsonStr(datasetPersistentId, jsonString, apiToken, sourceInternalVersionTimestamp); updateMetadataWithLatestInternalVersionNumber.then().assertThat() .statusCode(OK.getStatusCode()); } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java index de45362ce98..f38ca34a283 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java @@ -3280,7 +3280,7 @@ public void testUpdateWithEmptyFieldsAndVersionCheck() throws InterruptedExcepti updateResponse.prettyPrint(); updateResponse.then().assertThat() .body("status", equalTo(ApiConstants.STATUS_ERROR)) - .body("message", equalTo(BundleUtil.getStringFromBundle("abstractApiBean.error.datafileInternalVersionTimestampIsOutdated",Collections.singletonList(lastUpdateTime)))) + .body("message", equalTo(BundleUtil.getStringFromBundle("abstractApiBean.error.internalVersionTimestampIsOutdated",Collections.singletonList(lastUpdateTime)))) .statusCode(BAD_REQUEST.getStatusCode()); // Second user refreshes and updates. Should pass now diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 04d0b27d63c..9d85903dde3 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -771,7 +771,7 @@ static Response editVersionMetadataFromJsonStr(String persistentId, String jsonS return editVersionMetadataFromJsonStr(persistentId, jsonString, apiToken, null); } - static Response editVersionMetadataFromJsonStr(String persistentId, String jsonString, String apiToken, Integer sourceInternalVersionNumber) { + static Response editVersionMetadataFromJsonStr(String persistentId, String jsonString, String apiToken, String sourceInternalVersionTimestamp) { return given() .header(API_TOKEN_HTTP_HEADER, apiToken) .body(jsonString) @@ -779,7 +779,7 @@ static Response editVersionMetadataFromJsonStr(String persistentId, String jsonS .put("/api/datasets/:persistentId/editMetadata/?persistentId=" + persistentId + "&replace=true" - + (sourceInternalVersionNumber != null ? "&sourceInternalVersionNumber=" + sourceInternalVersionNumber : "")); + + (sourceInternalVersionTimestamp != null ? "&sourceInternalVersionTimestamp=" + sourceInternalVersionTimestamp : "")); } static Response updateDatasetPIDMetadata(String persistentId, String apiToken) { From 1320d515c98ac8e2013259a198ec92dfee0832c9 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Wed, 30 Apr 2025 09:19:58 -0400 Subject: [PATCH 13/16] refactor optional qp name from sourceInternalVersionTimestamp to sourceLastUpdateTime --- .../11243-editmetadata-api-extension.md | 2 +- .../11392-edit-file-metadata-empty-values.md | 2 +- doc/sphinx-guides/source/api/native-api.rst | 16 ++++++++-------- .../iq/dataverse/api/AbstractApiBean.java | 8 ++++---- .../edu/harvard/iq/dataverse/api/Datasets.java | 6 +++--- .../java/edu/harvard/iq/dataverse/api/Files.java | 6 +++--- .../edu/harvard/iq/dataverse/api/DatasetsIT.java | 8 ++++---- .../edu/harvard/iq/dataverse/api/UtilIT.java | 10 +++++----- 8 files changed, 29 insertions(+), 29 deletions(-) diff --git a/doc/release-notes/11243-editmetadata-api-extension.md b/doc/release-notes/11243-editmetadata-api-extension.md index 4aaf9320074..3666d8bc30a 100644 --- a/doc/release-notes/11243-editmetadata-api-extension.md +++ b/doc/release-notes/11243-editmetadata-api-extension.md @@ -1,7 +1,7 @@ ### Edit Dataset Metadata API extension - This endpoint now allows removing fields (by sending empty values), as long as they are not required by the dataset. -- New ``sourceInternalVersionTimestamp`` optional query parameter, which prevents inconsistencies by managing updates that +- New ``sourceLastUpdateTime`` optional query parameter, which prevents inconsistencies by managing updates that may occur from other users while a dataset is being edited. NOTE: This release note was updated to conform to the refactoring of the validation as part of issue #11392 diff --git a/doc/release-notes/11392-edit-file-metadata-empty-values.md b/doc/release-notes/11392-edit-file-metadata-empty-values.md index 652aea63aed..5839fa100af 100644 --- a/doc/release-notes/11392-edit-file-metadata-empty-values.md +++ b/doc/release-notes/11392-edit-file-metadata-empty-values.md @@ -2,6 +2,6 @@ Previously the API POST /files/{id}/metadata would ignore fields with empty values. Now the API updates the fields with the empty values essentially clearing the data. Missing fields will still be ignored. -An optional query parameter (sourceInternalVersionTimestamp) was added to ensure the metadata update doesn't overwrite stale data. +An optional query parameter (sourceLastUpdateTime) was added to ensure the metadata update doesn't overwrite stale data. See also [the guides](https://dataverse-guide--11359.org.readthedocs.build/en/11359/api/native-api.html#updating-file-metadata), #11392, and #11359. diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 352234d844a..0f6b49ba079 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -2137,26 +2137,26 @@ For these edits your JSON file need only include those dataset fields which you This endpoint also allows removing fields, as long as they are not required by the dataset. To remove a field, send an empty value (``""``) for individual fields. For multiple fields, send an empty array (``[]``). A sample JSON file for removing fields may be downloaded here: :download:`dataset-edit-metadata-delete-fields-sample.json <../_static/api/dataset-edit-metadata-delete-fields-sample.json>` -If another user updates the dataset version metadata before you send the update request, data inconsistencies may occur. To prevent this, you can use the optional ``sourceInternalVersionTimestamp`` query parameter. This parameter must include the ``lastUpdateTime`` corresponding to the dataset version being updated. The date must be in this format "yyyy-MM-dd'T'HH:mm:ss'Z'" +If another user updates the dataset version metadata before you send the update request, data inconsistencies may occur. To prevent this, you can use the optional ``sourceLastUpdateTime`` query parameter. This parameter must include the ``lastUpdateTime`` corresponding to the dataset version being updated. The date must be in this format "yyyy-MM-dd'T'HH:mm:ss'Z'" If this parameter is provided, the update will proceed only if the ``lastUpdateTime`` remains unchanged. Otherwise, the request will fail with an error. -Example using ``sourceInternalVersionTimestamp``: +Example using ``sourceLastUpdateTime``: .. code-block:: bash export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx export SERVER_URL=https://demo.dataverse.org export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/BCCP9Z - export SOURCE_INTERNAL_VERSION_TIMESTAMP=2025-04-25T13:58:28Z + export SOURCE_LAST_UPDATE_TIME=2025-04-25T13:58:28Z - curl -H "X-Dataverse-key: $API_TOKEN" -X PUT "$SERVER_URL/api/datasets/:persistentId/editMetadata?persistentId=$PERSISTENT_IDENTIFIER&replace=true&sourceInternalVersionTimestamp=$SOURCE_INTERNAL_VERSION_TIMESTAMP" --upload-file dataset-update-metadata.json + curl -H "X-Dataverse-key: $API_TOKEN" -X PUT "$SERVER_URL/api/datasets/:persistentId/editMetadata?persistentId=$PERSISTENT_IDENTIFIER&replace=true&sourceLastUpdateTime=SOURCE_LAST_UPDATE_TIME" --upload-file dataset-update-metadata.json The fully expanded example above (without environment variables) looks like this: .. code-block:: bash - curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT "https://demo.dataverse.org/api/datasets/:persistentId/editMetadata/?persistentId=doi:10.5072/FK2/BCCP9Z&replace=true&sourceInternalVersionTimestamp=2025-04-25T13:58:28Z" --upload-file dataset-update-metadata.json + curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT "https://demo.dataverse.org/api/datasets/:persistentId/editMetadata/?persistentId=doi:10.5072/FK2/BCCP9Z&replace=true&sourceLastUpdateTime=2025-04-25T13:58:28Z" --upload-file dataset-update-metadata.json Delete Dataset Metadata @@ -4612,7 +4612,7 @@ Updating File Metadata Updates the file metadata for an existing file where ``ID`` is the database id of the file to update or ``PERSISTENT_ID`` is the persistent id (DOI or Handle) of the file. Requires a ``jsonString`` expressing the new metadata. No metadata from the previous version of this file will be persisted, so if you want to update a specific field first get the json with the above command and alter the fields you want. -Optional Parameter for verifying that the Dataset Version being edited is the latest version can be added &sourceInternalVersionTimestamp=datetime(in format: "yyyy-MM-dd'T'HH:mm:ss'Z'"). This is to prevent stale data from being edited. The value for sourceInternalVersionTimestamp comes from ``lastUpdateTime`` in the response to get $SERVER_URL/api/files/$ID API call +Optional Parameter for verifying that the Dataset Version being edited is the latest version can be added &sourceLastUpdateTime=datetime(in format: "yyyy-MM-dd'T'HH:mm:ss'Z'"). This is to prevent stale data from being edited. The value for sourceLastUpdateTime comes from ``lastUpdateTime`` in the response to get $SERVER_URL/api/files/$ID API call A curl example using an ``ID`` @@ -4645,7 +4645,7 @@ A curl example using a ``PERSISTENT_ID`` curl -H "X-Dataverse-key:$API_TOKEN" -X POST \ -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"dataFileTags":["Survey"],"restrict":false}' \ - "$SERVER_URL/api/files/:persistentId/metadata?persistentId=$PERSISTENT_ID&sourceInternalVersionTimestamp=$UPDATE_TIME" + "$SERVER_URL/api/files/:persistentId/metadata?persistentId=$PERSISTENT_ID&sourceLastUpdateTime=$UPDATE_TIME" The fully expanded example above (without environment variables) looks like this: @@ -4653,7 +4653,7 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \ -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"dataFileTags":["Survey"],"restrict":false}' \ - "https://demo.dataverse.org/api/files/:persistentId/metadata?persistentId=doi:10.5072/FK2/AAA000&sourceInternalVersionTimestamp=2025-04-25T13:58:28Z" + "https://demo.dataverse.org/api/files/:persistentId/metadata?persistentId=doi:10.5072/FK2/AAA000&sourceLastUpdateTime=2025-04-25T13:58:28Z" Note: To update the 'tabularTags' property of file metadata, use the 'dataFileTags' key when making API requests. This property is used to update the 'tabularTags' of the file metadata. diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index c53f9f8119c..77c003ed3cb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -445,11 +445,11 @@ public Command handleLatestPublished() { return dsv; } - protected void validateInternalTimestampIsNotOutdated(DvObject dvObject, String sourceInternalVersionTimestamp) throws WrappedResponse { - Date date = sourceInternalVersionTimestamp != null ? DateUtil.parseDate(sourceInternalVersionTimestamp, "yyyy-MM-dd'T'HH:mm:ss'Z'") : null; + protected void validateInternalTimestampIsNotOutdated(DvObject dvObject, String sourceLastUpdateTime) throws WrappedResponse { + Date date = sourceLastUpdateTime != null ? DateUtil.parseDate(sourceLastUpdateTime, "yyyy-MM-dd'T'HH:mm:ss'Z'") : null; if (date == null) { throw new WrappedResponse( - badRequest(BundleUtil.getStringFromBundle("jsonparser.error.parsing.date", Collections.singletonList(sourceInternalVersionTimestamp))) + badRequest(BundleUtil.getStringFromBundle("jsonparser.error.parsing.date", Collections.singletonList(sourceLastUpdateTime))) ); } Instant instant = date.toInstant(); @@ -460,7 +460,7 @@ protected void validateInternalTimestampIsNotOutdated(DvObject dvObject, String // granularity is to the second since the json output only returns dates in this format to the second if (updateTimestamp.getEpochSecond() != instant.getEpochSecond()) { throw new WrappedResponse( - badRequest(BundleUtil.getStringFromBundle("abstractApiBean.error.internalVersionTimestampIsOutdated", Collections.singletonList(sourceInternalVersionTimestamp))) + badRequest(BundleUtil.getStringFromBundle("abstractApiBean.error.internalVersionTimestampIsOutdated", Collections.singletonList(sourceLastUpdateTime))) ); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index e36b50ad0e0..bfcf530b57e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -1095,12 +1095,12 @@ private String getCompoundDisplayValue (DatasetFieldCompoundValue dscv){ @Path("{id}/editMetadata") public Response editVersionMetadata(@Context ContainerRequestContext crc, String jsonBody, @PathParam("id") String id, @QueryParam("replace") boolean replaceData, - @QueryParam("sourceInternalVersionTimestamp") String sourceInternalVersionTimestamp) { + @QueryParam("sourceLastUpdateTime") String sourceLastUpdateTime) { try { Dataset dataset = findDatasetOrDie(id); - if (sourceInternalVersionTimestamp != null) { - validateInternalTimestampIsNotOutdated(dataset, sourceInternalVersionTimestamp); + if (sourceLastUpdateTime != null) { + validateInternalTimestampIsNotOutdated(dataset, sourceLastUpdateTime); } JsonObject json = JsonUtil.getJsonObject(jsonBody); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java index 1b809894e17..5834e7e0008 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java @@ -410,7 +410,7 @@ public Response deleteFileInDataset(@Context ContainerRequestContext crc, @PathP @AuthRequired @Path("{id}/metadata") public Response updateFileMetadata(@Context ContainerRequestContext crc, @FormDataParam("jsonData") String jsonData, - @PathParam("id") String fileIdOrPersistentId, @QueryParam("sourceInternalVersionTimestamp") String sourceInternalVersionTimestamp) { + @PathParam("id") String fileIdOrPersistentId, @QueryParam("sourceLastUpdateTime") String sourceLastUpdateTime) { FileMetadata upFmd = null; @@ -428,9 +428,9 @@ public Response updateFileMetadata(@Context ContainerRequestContext crc, @FormDa return error(BAD_REQUEST, "Error attempting get the requested data file."); } - if (sourceInternalVersionTimestamp != null) { + if (sourceLastUpdateTime != null) { try { - validateInternalTimestampIsNotOutdated(df, sourceInternalVersionTimestamp); + validateInternalTimestampIsNotOutdated(df, sourceLastUpdateTime); } catch (WrappedResponse wr) { return wr.getResponse(); } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 13837bed859..162c4b0e125 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -676,9 +676,9 @@ public void testAddUpdateDatasetViaNativeAPI() { .body("data.metadataBlocks.citation.fields[2].typeName", not(equalTo("alternativeTitle"))) .statusCode(OK.getStatusCode()); - // Test sourceInternalVersionTimestamp optional query parameter - String sourceInternalVersionTimestamp = updateMetadataRemoveAlternativeTitles.then().extract().path("data.lastUpdateTime"); - assertNotNull(sourceInternalVersionTimestamp); + // Test sourceLastUpdateTime optional query parameter + String sourceLastUpdateTime = updateMetadataRemoveAlternativeTitles.then().extract().path("data.lastUpdateTime"); + assertNotNull(sourceLastUpdateTime); String oldTimestamp = "2025-04-25T13:58:28Z"; // Case 1 - Pass outdated internal version number @@ -690,7 +690,7 @@ public void testAddUpdateDatasetViaNativeAPI() { // Case 2 - Pass latest internal version number - Response updateMetadataWithLatestInternalVersionNumber = UtilIT.editVersionMetadataFromJsonStr(datasetPersistentId, jsonString, apiToken, sourceInternalVersionTimestamp); + Response updateMetadataWithLatestInternalVersionNumber = UtilIT.editVersionMetadataFromJsonStr(datasetPersistentId, jsonString, apiToken, sourceLastUpdateTime); updateMetadataWithLatestInternalVersionNumber.then().assertThat() .statusCode(OK.getStatusCode()); } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 9d85903dde3..672b43c11fa 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -771,7 +771,7 @@ static Response editVersionMetadataFromJsonStr(String persistentId, String jsonS return editVersionMetadataFromJsonStr(persistentId, jsonString, apiToken, null); } - static Response editVersionMetadataFromJsonStr(String persistentId, String jsonString, String apiToken, String sourceInternalVersionTimestamp) { + static Response editVersionMetadataFromJsonStr(String persistentId, String jsonString, String apiToken, String sourceLastUpdateTime) { return given() .header(API_TOKEN_HTTP_HEADER, apiToken) .body(jsonString) @@ -779,7 +779,7 @@ static Response editVersionMetadataFromJsonStr(String persistentId, String jsonS .put("/api/datasets/:persistentId/editMetadata/?persistentId=" + persistentId + "&replace=true" - + (sourceInternalVersionTimestamp != null ? "&sourceInternalVersionTimestamp=" + sourceInternalVersionTimestamp : "")); + + (sourceLastUpdateTime != null ? "&sourceLastUpdateTime=" + sourceLastUpdateTime : "")); } static Response updateDatasetPIDMetadata(String persistentId, String apiToken) { @@ -1092,15 +1092,15 @@ static Response deleteFileApi(Integer fileId, String apiToken) { static Response updateFileMetadata(String fileIdOrPersistentId, String jsonAsString, String apiToken) { return updateFileMetadata(fileIdOrPersistentId, jsonAsString,apiToken, null); } - static Response updateFileMetadata(String fileIdOrPersistentId, String jsonAsString, String apiToken, String sourceInternalVersionTimestamp) { + static Response updateFileMetadata(String fileIdOrPersistentId, String jsonAsString, String apiToken, String sourceLastUpdateTime) { String idInPath = fileIdOrPersistentId; // Assume it's a number. String optionalQueryParam = ""; // If idOrPersistentId is a number we'll just put it in the path. if (!NumberUtils.isCreatable(fileIdOrPersistentId)) { idInPath = ":persistentId"; optionalQueryParam = "?persistentId=" + fileIdOrPersistentId; } - if (sourceInternalVersionTimestamp != null) { - optionalQueryParam = optionalQueryParam + (optionalQueryParam.isEmpty() ? "?" : "&") + "sourceInternalVersionTimestamp=" + sourceInternalVersionTimestamp; + if (sourceLastUpdateTime != null) { + optionalQueryParam = optionalQueryParam + (optionalQueryParam.isEmpty() ? "?" : "&") + "sourceLastUpdateTime=" + sourceLastUpdateTime; } RequestSpecification requestSpecification = given() .header(API_TOKEN_HTTP_HEADER, apiToken); From 7563cf80d94943ad0f98d8d8d648e88f651a5409 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 24 Jun 2025 09:13:31 -0500 Subject: [PATCH 14/16] Suggested doc edits (#11590) * Suggested doc edits * Update doc/sphinx-guides/source/api/native-api.rst Co-authored-by: Philip Durbin * Update doc/sphinx-guides/source/api/native-api.rst Co-authored-by: Philip Durbin * Update doc/sphinx-guides/source/api/native-api.rst Co-authored-by: Philip Durbin --------- Co-authored-by: Philip Durbin Co-authored-by: Steven Winship <39765413+stevenwinship@users.noreply.github.com> --- doc/sphinx-guides/source/api/native-api.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 159439f62b8..c9fc558c1af 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -2153,9 +2153,9 @@ For these edits your JSON file need only include those dataset fields which you This endpoint also allows removing fields, as long as they are not required by the dataset. To remove a field, send an empty value (``""``) for individual fields. For multiple fields, send an empty array (``[]``). A sample JSON file for removing fields may be downloaded here: :download:`dataset-edit-metadata-delete-fields-sample.json <../_static/api/dataset-edit-metadata-delete-fields-sample.json>` -If another user updates the dataset version metadata before you send the update request, data inconsistencies may occur. To prevent this, you can use the optional ``sourceLastUpdateTime`` query parameter. This parameter must include the ``lastUpdateTime`` corresponding to the dataset version being updated. The date must be in this format "yyyy-MM-dd'T'HH:mm:ss'Z'" +If another user updates the dataset version metadata before you send the update request, metadata inconsistencies may occur. To prevent this, you can use the optional ``sourceLastUpdateTime`` query parameter. This parameter must include the ``lastUpdateTime`` corresponding to the dataset version being updated. The date must be in the format ``yyyy-MM-dd'T'HH:mm:ss'Z'``. -If this parameter is provided, the update will proceed only if the ``lastUpdateTime`` remains unchanged. Otherwise, the request will fail with an error. +If this parameter is provided, the update will proceed only if the ``lastUpdateTime`` remains unchanged (meaning no one has updated the dataset metadata since you retrieved it). Otherwise, the request will fail with an error. Example using ``sourceLastUpdateTime``: @@ -4675,7 +4675,7 @@ Updating File Metadata Updates the file metadata for an existing file where ``ID`` is the database id of the file to update or ``PERSISTENT_ID`` is the persistent id (DOI or Handle) of the file. Requires a ``jsonString`` expressing the new metadata. No metadata from the previous version of this file will be persisted, so if you want to update a specific field first get the json with the above command and alter the fields you want. -Optional Parameter for verifying that the Dataset Version being edited is the latest version can be added &sourceLastUpdateTime=datetime(in format: "yyyy-MM-dd'T'HH:mm:ss'Z'"). This is to prevent stale data from being edited. The value for sourceLastUpdateTime comes from ``lastUpdateTime`` in the response to get $SERVER_URL/api/files/$ID API call +An optional parameter, sourceLastUpdateTime=datetime (in format: ``yyyy-MM-dd'T'HH:mm:ss'Z'``), can be used to verify that the file metadata being edited has not been changed since you last retrieved it, thereby avoiding potential lost metadata updates. The value for sourceLastUpdateTime can be taken from ``lastUpdateTime`` in the response to get $SERVER_URL/api/files/$ID API call. A curl example using an ``ID`` @@ -4697,7 +4697,7 @@ The fully expanded example above (without environment variables) looks like this -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"dataFileTags":["Survey"],"restrict":false}' \ "https://demo.dataverse.org/api/files/24/metadata" -A curl example using a ``PERSISTENT_ID`` +A curl example using a ``PERSISTENT_ID`` and the sourceLastUpdateTime parameter: .. code-block:: bash From 0c72a89e28e450bca933488b1413ab922b64980a Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Mon, 14 Jul 2025 11:27:50 -0400 Subject: [PATCH 15/16] remove unused bundle entry --- src/main/java/propertyFiles/Bundle.properties | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 0530e9ee52b..3fdbfa17f64 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -3220,7 +3220,6 @@ updateDatasetFieldsCommand.api.processDatasetUpdate.parseError=Error parsing dat #AbstractApiBean.java abstractApiBean.error.internalVersionTimestampIsOutdated=Internal version timestamp {0} is outdated -abstractApiBean.error.datasetInternalVersionNumberIsOutdated=Dataset internal version number {0} is outdated #RoleAssigneeServiceBean.java roleAssigneeServiceBean.error.dataverseRequestCannotBeNull=DataverseRequest cannot be null. From 509e55a9866ee4ff50e7201bef7c8c55d0e0b914 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Mon, 14 Jul 2025 11:42:37 -0400 Subject: [PATCH 16/16] update changelog to move this PR to 6.8 --- doc/sphinx-guides/source/api/changelog.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst index fdc0e7f1879..506f1e6baae 100644 --- a/doc/sphinx-guides/source/api/changelog.rst +++ b/doc/sphinx-guides/source/api/changelog.rst @@ -7,11 +7,15 @@ This API changelog is experimental and we would love feedback on its usefulness. :local: :depth: 1 +v6.8 +---- +- For POST /api/files/{id}/metadata passing an empty string ("description":"") or array ("categories":[]) will no longer be ignored. Empty fields will now clear out the values in the file's metadata. To ignore the fields simply do not include them in the JSON string. +- For PUT /api/datasets/{id}/editMetadata the query parameter "sourceInternalVersionNumber" has been removed and replaced with "sourceLastUpdateTime" to verify that the data being edited hasn't been modified and isn't stale. + v6.7 ---- - An undocumented :doc:`search` parameter called "show_my_data" has been removed. It was never exercised by tests and is believed to be unused. API users should use the :ref:`api-mydata` API instead. -- For POST /api/files/{id}/metadata passing an empty string (“description”:””) or array (“categories”:[]) will no longer be ignored. Empty fields will now clear out the values in the file's metadata. To ignore the fields simply do not include them in the Json string. - /api/datasets/{id}/curationStatus API now includes a JSON object with curation label, createtime, and assigner rather than a string 'label' and it supports a new boolean includeHistory parameter (default false) that returns a JSON array of statuses - /api/datasets/{id}/listCurationStates includes new columns "Status Set Time" and "Status Set By" columns listing the time the current status was applied and by whom. It also supports the boolean includeHistory parameter. - Due to updates in libraries used by Dataverse, XML serialization may have changed slightly with respect to whether self-closing tags are used for empty elements. This primiarily affects XML-based metadata exports. The XML structure of the export itself has not changed, so this is only an incompatibility if you are not using an XML parser.