Skip to content

Commit

Permalink
#1593 support removing existing fields from a JSON object in a merge …
Browse files Browse the repository at this point in the history 
…patch using a regular expression

Signed-off-by: Thomas Jäckle <thomas.jaeckle@beyonnex.io>
  • Loading branch information
@thjaeckle
thjaeckle authored and Stanchev Aleksandar committed Apr 18, 2023
1 parent 73af3f7 commit 75e7280
Show file tree
Hide file tree
Showing 3 changed files with 183 additions and 1 deletion.
76 changes: 76 additions & 0 deletions documentation/src/main/resources/pages/ditto/httpapi-concepts.md
View file
Expand Up @@ -393,6 +393,82 @@ interpreted as delete in contrast to `PUT` requests where `null` values have no
Like `PUT` requests, `PATCH` requests can be applied at any level of the JSON structure of a thing, e.g. patching a
complete thing at root level or patching a single property value at property level.

### Removing fields in a merge update with a regex

{% include note.html content="This is an addition to the JSON merge patch (RFC-7396), enhancing using `null` values
for deleting certain parts of JSON objects specified with a regular expression before applying new fields to it." %}

The merge patch functionality in Ditto solves a common problem to the JSON merge patch (RFC-7396): whenever a JSON object
shall be patched, all the old json fields are merged with all the new json fields, unless the exact field names are
specified in the patch with `"<field>": null`.
This would however require to know all existing fields upfront, which for a merge patch can not be assumed.

The solution is a little enhancement to Ditto's merge patch functionality: The ability to delete arbitrary parts from
JSON objects using a regular expression **before** applying all other patch values.

The syntax for this function is rather specific (so that no "normally" occurring JSON keys match the same syntax):
```json
{
"{%raw%}{{ /.*/ }}{%endraw%}": null
}
```

When such a `{%raw%}{{ /<regex>/ }}{%endraw%}` with the value `null` is detected in the merge patch, the content between the 2 `/` is
interpreted as regular expression to apply for finding keys to delete from the target object.
As a result, using `"{%raw%}{{ /.*/ }}{%endraw%}": null` would delete all the values inside a JSON object before applying the new
values provided in the patch.

Example:
Assuming that inside a JSON object every month some aggregated data is stored with the year and month:
```json
{
"thingId": "{thingId}",
"policyId": "{policyId}",
"features": {
"aggregated-history": {
"properties": {
"2022-11": 42.3,
"2022-12": 54.3,
"2023-01": 80.2,
"2023-02": 99.9
}
}
}
}
```

Then the data from "last year" could be purged with the following patch, while adding a new value to the existing ones
of this year:
```json
{
"features": {
"aggregated-history": {
"properties": {
"{%raw%}{{ /2022-.*/ }}{%endraw%}": null,
"2023-03": 105.21
}
}
}
}
```

The resulting Thing JSON after applying the patch would then look like:
```json
{
"thingId": "{thingId}",
"policyId": "{policyId}",
"features": {
"aggregated-history": {
"properties": {
"2023-01": 80.2,
"2023-02": 99.9,
"2023-03": 105.21
}
}
}
}
```

### Permissions required for merge update

To successfully execute merge update the authorized subject needs to have *WRITE* permission on *all* resources
Expand Down
31 changes: 30 additions & 1 deletion json/src/main/java/org/eclipse/ditto/json/JsonMergePatch.java
View file
Expand Up @@ -12,9 +12,11 @@
*/
package org.eclipse.ditto.json;

import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

import javax.annotation.Nullable;
Expand Down Expand Up @@ -165,16 +167,43 @@ private static JsonObject mergeJsonObjects(final JsonObject jsonObject1, final J
}
});

final List<JsonKey> toBeNulledKeysByRegex = determineToBeNulledKeysByRegex(jsonObject1, jsonObject2);

// add fields of jsonObject2 not present in jsonObject1
jsonObject2.forEach(jsonField -> {
if (!jsonObject1.contains(jsonField.getKey())) {
if (!jsonObject1.contains(jsonField.getKey()) && !toBeNulledKeysByRegex.contains(jsonField.getKey())) {
builder.set(jsonField);
}
});

return builder.build();
}

private static List<JsonKey> determineToBeNulledKeysByRegex(
final JsonObject jsonObject1,
final JsonObject jsonObject2) {

final List<JsonKey> toBeNulledKeysByRegex = new ArrayList<>();
final List<JsonKey> keyRegexes = jsonObject1.getKeys().stream()
.filter(key -> key.toString().startsWith("{{") && key.toString().endsWith("}}"))
.collect(Collectors.toList());
keyRegexes.forEach(keyRegex -> {
final String keyRegexWithoutCurly = keyRegex.toString().substring(2, keyRegex.length() - 2).trim();
if (keyRegexWithoutCurly.startsWith("/") && keyRegexWithoutCurly.endsWith("/")) {
final String regexStr = keyRegexWithoutCurly.substring(1, keyRegexWithoutCurly.length() - 1);
final Pattern pattern = Pattern.compile(regexStr);
jsonObject1.getValue(keyRegex)
.filter(JsonValue::isNull) // only support deletion via regex, so only support "null" values
.ifPresent(keyRegexValue ->
jsonObject2.getKeys().stream()
.filter(key -> pattern.matcher(key).matches())
.forEach(toBeNulledKeysByRegex::add)
);
}
});
return toBeNulledKeysByRegex;
}

/**
* Applies this merge patch on the given json value.
*
Expand Down
77 changes: 77 additions & 0 deletions json/src/test/java/org/eclipse/ditto/json/JsonMergePatchTest.java
View file
Expand Up @@ -447,4 +447,81 @@ public void mergeFieldsFromBothObjectsRFC7396_TestCase13() {
Assertions.assertThat(mergedObject).isEqualTo(expectedObject);
}

@Test
public void removeFieldsUsingRegexWithNullValue() {
final JsonObject originalObject = JsonFactory.newObjectBuilder()
.set("a", JsonFactory.newObjectBuilder()
.set("2023-04-01", JsonValue.of(true))
.set("2023-04-02", JsonValue.of("hello"))
.set("2023-04-03", JsonValue.of("darkness"))
.set("2023-04-04", JsonValue.of("my"))
.set("2023-04-05", JsonValue.of("old"))
.set("2023-04-06", JsonValue.of("friend"))
.build())
.set("b", JsonFactory.newObjectBuilder()
.set("2023-04-01", JsonValue.of(true))
.set("2023-04-02", JsonValue.of("hello"))
.set("2023-04-03", JsonValue.of("darkness"))
.set("2023-04-04", JsonValue.of("my"))
.set("2023-04-05", JsonValue.of("old"))
.set("2023-04-06", JsonValue.of("friend"))
.build())
.set("c", JsonFactory.newObjectBuilder()
.set("some", JsonValue.of(true))
.set("2023-04-02", JsonValue.of("hello"))
.set("totally-other", JsonValue.of("darkness"))
.set("foo", JsonValue.of("my"))
.build())
.build();

final JsonObject objectToPatch = JsonFactory.newObjectBuilder()
.set("a", JsonFactory.newObjectBuilder()
.set("{{ /2023-04-.*/ }}", JsonValue.nullLiteral())
.set("2023-05-01", JsonValue.of("new"))
.set("2023-05-02", JsonValue.of("catch"))
.set("2023-05-03", JsonValue.of("phrase"))
.build())
.set("b", JsonFactory.newObjectBuilder()
.set("{{ /2023-04-01/ }}", JsonValue.nullLiteral())
.set("{{ /^2023-04-03$/ }}", JsonValue.nullLiteral())
.set("{{ /[0-9]{4}-04-.+4/ }}", JsonValue.nullLiteral())
.set("2023-05-01", JsonValue.of("new"))
.set("2023-05-02", JsonValue.of("catch"))
.set("2023-05-03", JsonValue.of("phrase"))
.build())
.set("c", JsonFactory.newObjectBuilder()
.set("{{ /.*/ }}", JsonValue.nullLiteral())
.set("2023-05-01", JsonValue.of("new"))
.set("2023-05-02", JsonValue.of("catch"))
.set("2023-05-03", JsonValue.of("phrase"))
.build())
.build();

final JsonValue expectedObject = JsonFactory.newObjectBuilder()
.set("a", JsonFactory.newObjectBuilder()
.set("2023-05-01", JsonValue.of("new"))
.set("2023-05-02", JsonValue.of("catch"))
.set("2023-05-03", JsonValue.of("phrase"))
.build())
.set("b", JsonFactory.newObjectBuilder()
.set("2023-04-02", JsonValue.of("hello"))
.set("2023-04-05", JsonValue.of("old"))
.set("2023-04-06", JsonValue.of("friend"))
.set("2023-05-01", JsonValue.of("new"))
.set("2023-05-02", JsonValue.of("catch"))
.set("2023-05-03", JsonValue.of("phrase"))
.build())
.set("c", JsonFactory.newObjectBuilder()
.set("2023-05-01", JsonValue.of("new"))
.set("2023-05-02", JsonValue.of("catch"))
.set("2023-05-03", JsonValue.of("phrase"))
.build())
.build();

final JsonValue mergedObject = JsonMergePatch.of(objectToPatch).applyOn(originalObject);

Assertions.assertThat(mergedObject).isEqualTo(expectedObject);
}


}

0 comments on commit 75e7280

Please sign in to comment.