From 1260f70290c65dde373131375dc95d048e7e8832 Mon Sep 17 00:00:00 2001 From: Mahesh Subramanian Date: Fri, 28 Feb 2020 15:38:57 +0000 Subject: [PATCH 1/5] Gson library now available for transformation operation. Locking version to 2.8.5 as the next version 2.8.6 is causing issues with swarm start up (#48) --- CHANGELOG.md | 2 +- event-tool/pom.xml | 6 ++++++ pom.xml | 3 ++- stream-transformation-tool-api/pom.xml | 5 +++++ 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 50af093..572f2ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ on [Keep a CHANGELOG](http://keepachangelog.com/). This project adheres to [Semantic Versioning](http://semver.org/). ## [Unreleased] - +- Gson library available for easier manipulation of json objects for transformation ## [5.3.5] - 2020-02-25 ### Added diff --git a/event-tool/pom.xml b/event-tool/pom.xml index 623eba0..bb24556 100644 --- a/event-tool/pom.xml +++ b/event-tool/pom.xml @@ -93,6 +93,12 @@ logging + + com.google.code.gson + gson + ${gson.version} + + org.slf4j diff --git a/pom.xml b/pom.xml index 56f0211..596bab4 100644 --- a/pom.xml +++ b/pom.xml @@ -31,7 +31,8 @@ 2017.11.0 77 1.13.1 - + + 2.8.5 diff --git a/stream-transformation-tool-api/pom.xml b/stream-transformation-tool-api/pom.xml index e2a01f7..e22ab94 100644 --- a/stream-transformation-tool-api/pom.xml +++ b/stream-transformation-tool-api/pom.xml @@ -22,6 +22,11 @@ uk.gov.justice.services messaging-core + + com.google.code.gson + gson + ${gson.version} + From 54a19c61aa846aed767ab961c1278f0f1ec3f079 Mon Sep 17 00:00:00 2001 From: Mahesh Subramanian Date: Wed, 4 Mar 2020 10:28:12 +0000 Subject: [PATCH 2/5] Surfacing all exceptions for visibility in the logs. (#49) (cherry picked from commit 722b654b81b71635c20507d803f206092217c994) --- .../service/EventStreamTransformationService.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/stream-transformation-tool-service/src/main/java/uk/gov/justice/tools/eventsourcing/transformation/service/EventStreamTransformationService.java b/stream-transformation-tool-service/src/main/java/uk/gov/justice/tools/eventsourcing/transformation/service/EventStreamTransformationService.java index 83edeb7..28df859 100644 --- a/stream-transformation-tool-service/src/main/java/uk/gov/justice/tools/eventsourcing/transformation/service/EventStreamTransformationService.java +++ b/stream-transformation-tool-service/src/main/java/uk/gov/justice/tools/eventsourcing/transformation/service/EventStreamTransformationService.java @@ -5,7 +5,6 @@ import uk.gov.justice.services.eventsourcing.source.core.EventSourceTransformation; import uk.gov.justice.services.eventsourcing.source.core.exception.EventStreamException; -import uk.gov.justice.services.jdbc.persistence.JdbcRepositoryException; import uk.gov.justice.services.messaging.JsonEnvelope; import uk.gov.justice.tools.eventsourcing.transformation.EventStreamReader; import uk.gov.justice.tools.eventsourcing.transformation.EventTransformationRegistry; @@ -62,8 +61,11 @@ public class EventStreamTransformationService { @Inject private RetryStreamOperator retryStreamOperator; - - @Transactional(value = REQUIRES_NEW, rollbackOn = {EventStreamException.class, JdbcRepositoryException.class}) + /* + * The transactional annotation will always rollback for RuntimeException and its subclasses. So, no need to handle that explicitly. + * Only checked exceptions need to be specified + */ + @Transactional(value = REQUIRES_NEW, rollbackOn = {EventStreamException.class}) public UUID transformEventStream(final UUID originalStreamId, final int pass) throws EventStreamException { try { @@ -88,8 +90,8 @@ public UUID transformEventStream(final UUID originalStreamId, final int pass) th if (action.isDeactivate()) { streamRepository.deactivateStream(originalStreamId); } - } catch (final EventStreamException | JdbcRepositoryException e) { - logger.error(format("Unknown error while transforming events on stream %s", originalStreamId), e); + } catch (final Exception e) { + logger.error(format("Error while transforming events on stream %s", originalStreamId), e); throw e; } From 60e3a67df61b3dee369ffb23ee47c77179fb1e6a Mon Sep 17 00:00:00 2001 From: Mahesh Subramanian Date: Thu, 5 Mar 2020 14:57:38 +0000 Subject: [PATCH 3/5] Adding json-path library (cherry picked from commit cd57fadfe88c7c89bcabaa6d90cf7762763376c7) --- event-tool/pom.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/event-tool/pom.xml b/event-tool/pom.xml index bb24556..e7ab763 100644 --- a/event-tool/pom.xml +++ b/event-tool/pom.xml @@ -98,6 +98,10 @@ gson ${gson.version} + + com.jayway.jsonpath + json-path + From de59d6e2ed20d35379c20f18d57fdb347e916ebf Mon Sep 17 00:00:00 2001 From: Mahesh Subramanian Date: Sun, 8 Mar 2020 00:34:41 +0000 Subject: [PATCH 4/5] Adding ability to whitelist attributes at a global level. The global attributes are not fully qualified json path and rely on the json path for an attribute (not object or array) ending with it (#51) (cherry picked from commit 269ef5107769a27384a98e27a1e50735926ffa65) --- .../anonymization/model/Events.java | 6 +++-- .../service/EventAnonymiserService.java | 18 ++++++++++---- .../schema/event-anonymisation-schema.json | 15 ++++++++++-- .../service/EventAnonymiserServiceTest.java | 17 +++++++++++++ .../resources/events-anonymisation-rule.json | 5 ++++ .../src/test/resources/test-object-data.json | 24 +++++++++++++++++-- 6 files changed, 75 insertions(+), 10 deletions(-) diff --git a/stream-transformation-tool-anonymise/src/main/java/uk/gov/justice/tools/eventsourcing/anonymization/model/Events.java b/stream-transformation-tool-anonymise/src/main/java/uk/gov/justice/tools/eventsourcing/anonymization/model/Events.java index 2d772df..e9d6c7a 100644 --- a/stream-transformation-tool-anonymise/src/main/java/uk/gov/justice/tools/eventsourcing/anonymization/model/Events.java +++ b/stream-transformation-tool-anonymise/src/main/java/uk/gov/justice/tools/eventsourcing/anonymization/model/Events.java @@ -5,13 +5,15 @@ public class Events { + private List globalAttributes; + private List events; public Events() { } - public Events(final List events) { - this.events = events; + public List getGlobalAttributes() { + return globalAttributes; } public List getEvents() { diff --git a/stream-transformation-tool-anonymise/src/main/java/uk/gov/justice/tools/eventsourcing/anonymization/service/EventAnonymiserService.java b/stream-transformation-tool-anonymise/src/main/java/uk/gov/justice/tools/eventsourcing/anonymization/service/EventAnonymiserService.java index 2b85afd..9ba263b 100644 --- a/stream-transformation-tool-anonymise/src/main/java/uk/gov/justice/tools/eventsourcing/anonymization/service/EventAnonymiserService.java +++ b/stream-transformation-tool-anonymise/src/main/java/uk/gov/justice/tools/eventsourcing/anonymization/service/EventAnonymiserService.java @@ -29,6 +29,7 @@ import javax.json.JsonValue; import javax.json.JsonValue.ValueType; +import org.apache.commons.collections.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -89,7 +90,8 @@ private JsonObjectBuilder processObjectPayload(final JsonObject payload, final J } else { final String pathForField = getPathForField(jsonPath, fieldName); - final boolean anonymise = !fieldsToIgnore.contains(pathForField); + final boolean attributePathWhitelistedGlobally = isAttributePathEndingWithGloballyWhitelistedAttributeValues(pathForField); + final boolean anonymise = !attributePathWhitelistedGlobally && !fieldsToIgnore.contains(pathForField); setFieldValueInObject(fieldName, jsonValue, builder, anonymise); } } @@ -100,8 +102,7 @@ private JsonObjectBuilder processObjectPayload(final JsonObject payload, final J private JsonArrayBuilder processArrayPayload(final JsonArray payload, final JsonArrayBuilder builder, final String jsonPath, final List fieldsToIgnore) { - for (int counter = 0; counter < payload.size(); counter++) { - final JsonValue value = payload.get(counter); + for (final JsonValue value : payload) { final ValueType fieldValueType = value.getValueType(); final String pathForField = getPathForField(jsonPath, "[*]"); @@ -118,7 +119,8 @@ private JsonArrayBuilder processArrayPayload(final JsonArray payload, final Json builder.add(value); } } else { - final boolean anonymise = !fieldsToIgnore.contains(pathForField); + final boolean attributePathWhitelistedGlobally = isAttributePathEndingWithGloballyWhitelistedAttributeValues(pathForField); + final boolean anonymise = !attributePathWhitelistedGlobally && !fieldsToIgnore.contains(pathForField); setFieldValueInArray(value, builder, anonymise); } } @@ -152,6 +154,14 @@ private String getTransformedStringValue(final JsonValue value) { return (String) stringPatternGeneratorFactory.getGenerator(valueAsString).convert(valueAsString); } + private boolean isAttributePathEndingWithGloballyWhitelistedAttributeValues(final String jsonPathForAttribute) { + return getAttributesWhitelistedForAllEvents().stream().anyMatch(jsonPathForAttribute::endsWith); + } + + private List getAttributesWhitelistedForAllEvents() { + return CollectionUtils.isNotEmpty(EVENTS.getGlobalAttributes()) ? EVENTS.getGlobalAttributes() : newArrayList(); + } + private List getFieldsToIgnoreForEvent(final String eventName) { final Optional optionalEvent = EVENTS.getEvents().stream().filter(e -> e.getEventName().equals(eventName)).findFirst(); return optionalEvent.isPresent() ? optionalEvent.get().getFieldsToBeIgnored() : emptyList(); diff --git a/stream-transformation-tool-anonymise/src/main/resources/schema/event-anonymisation-schema.json b/stream-transformation-tool-anonymise/src/main/resources/schema/event-anonymisation-schema.json index 9dd4bce..dfc80c8 100644 --- a/stream-transformation-tool-anonymise/src/main/resources/schema/event-anonymisation-schema.json +++ b/stream-transformation-tool-anonymise/src/main/resources/schema/event-anonymisation-schema.json @@ -2,6 +2,15 @@ "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "properties": { + "globalAttributes": { + "type": "array", + "description": "Fields that are considered for all events. This field does not denote a complete JSON path. It only identifies the ending of a json path (partial entry)", + "items": [ + { + "type": "string" + } + ] + }, "events": { "type": "array", "items": [ @@ -23,12 +32,14 @@ "required": [ "eventName", "fieldsToBeIgnored" - ] + ], + "additionalProperties": false } ] } }, "required": [ "events" - ] + ], + "additionalProperties": false } \ No newline at end of file diff --git a/stream-transformation-tool-anonymise/src/test/java/uk/gov/justice/tools/eventsourcing/anonymization/service/EventAnonymiserServiceTest.java b/stream-transformation-tool-anonymise/src/test/java/uk/gov/justice/tools/eventsourcing/anonymization/service/EventAnonymiserServiceTest.java index 3d0e94a..b08dea2 100644 --- a/stream-transformation-tool-anonymise/src/test/java/uk/gov/justice/tools/eventsourcing/anonymization/service/EventAnonymiserServiceTest.java +++ b/stream-transformation-tool-anonymise/src/test/java/uk/gov/justice/tools/eventsourcing/anonymization/service/EventAnonymiserServiceTest.java @@ -100,6 +100,23 @@ public void shouldNotAnonymiseJsonArrayPayload() { assertThat(anonymisedJsonArray.getString(6), is("SC208979B")); } + @Test + public void shouldNotAnonymiseGloballyWhitelistedAttributes() { + final JsonObject anonymisedPayload = service.anonymiseObjectPayload(buildObjectPayload("test-object-data.json"), "test-object-event"); + final JsonArray exampleArray = anonymisedPayload.getJsonArray("exampleArray"); + assertThat(exampleArray.getJsonObject(0).getJsonArray("repeatingParentArrayAttribute").getJsonObject(0).getString("repeatingChildAttribute"), is("child1")); + assertThat(exampleArray.getJsonObject(0).getJsonArray("repeatingParentArrayAttribute").getJsonObject(0).getString("repeatingAttribute"), is("test1")); + assertThat(exampleArray.getJsonObject(0).getJsonArray("repeatingParentArrayAttribute").getJsonObject(0).getJsonArray("repeatingParentArrayAttribute").getJsonObject(0).getString("repeatingChildAttribute"), is("child2")); + assertThat(exampleArray.getJsonObject(0).getJsonArray("repeatingParentArrayAttribute").getJsonObject(0).getJsonArray("repeatingParentArrayAttribute").getJsonObject(0).getString("repeatingAttribute"), is("test2")); + + final JsonObject exampleObject = anonymisedPayload.getJsonObject("example"); + final JsonObject repeatingObject = exampleObject.getJsonObject("repeatingParentObjectAttribute"); + assertThat(repeatingObject.getString("repeatingChildAttribute"), is("child3")); + assertThat(repeatingObject.getString("repeatingAttribute"), is("test3")); + assertThat(repeatingObject.getJsonObject("repeatingParentObjectAttribute").getString("repeatingChildAttribute"), is("child4")); + assertThat(repeatingObject.getJsonObject("repeatingParentObjectAttribute").getString("repeatingAttribute"), is("test4")); + } + private JsonObject buildObjectPayload(final String payloadFileName) { final JsonReader jsonReader = createReader(new StringReader(getFileContentsAsString(payloadFileName))); diff --git a/stream-transformation-tool-anonymise/src/test/resources/events-anonymisation-rule.json b/stream-transformation-tool-anonymise/src/test/resources/events-anonymisation-rule.json index 61c02f3..6960ab5 100644 --- a/stream-transformation-tool-anonymise/src/test/resources/events-anonymisation-rule.json +++ b/stream-transformation-tool-anonymise/src/test/resources/events-anonymisation-rule.json @@ -1,4 +1,9 @@ { + "globalAttributes": [ + "repeatingParentArrayAttribute[*].repeatingChildAttribute", + "repeatingParentObjectAttribute.repeatingChildAttribute", + "repeatingAttribute" + ], "events": [ { "eventName": "test-object-event", diff --git a/stream-transformation-tool-anonymise/src/test/resources/test-object-data.json b/stream-transformation-tool-anonymise/src/test/resources/test-object-data.json index e38f976..8e822e3 100644 --- a/stream-transformation-tool-anonymise/src/test/resources/test-object-data.json +++ b/stream-transformation-tool-anonymise/src/test/resources/test-object-data.json @@ -2,7 +2,19 @@ "exampleArray": [ { "attributeIntAsString": "001", - "attributeIntAsStringAnonymise": "001" + "attributeIntAsStringAnonymise": "001", + "repeatingParentArrayAttribute": [ + { + "repeatingChildAttribute": "child1", + "repeatingAttribute" : "test1", + "repeatingParentArrayAttribute": [ + { + "repeatingChildAttribute": "child2", + "repeatingAttribute" : "test2" + } + ] + } + ] } ], "example": { @@ -27,6 +39,14 @@ }, "test234@mail.com", "SC208979B" - ] + ], + "repeatingParentObjectAttribute": { + "repeatingChildAttribute": "child3", + "repeatingAttribute" : "test3", + "repeatingParentObjectAttribute": { + "repeatingChildAttribute": "child4", + "repeatingAttribute" : "test4" + } + } } } \ No newline at end of file From 094803c4ab0bae5b9b5a6c566f63575f2b7d4a82 Mon Sep 17 00:00:00 2001 From: Mahesh Subramanian Date: Wed, 11 Mar 2020 17:22:17 +0000 Subject: [PATCH 5/5] Updating changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 572f2ec..abb53e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ on [Keep a CHANGELOG](http://keepachangelog.com/). This project adheres to ## [Unreleased] - Gson library available for easier manipulation of json objects for transformation +- Surfacing all exception when processing a stream +- Json path library added +- Added global whitelisting of attributes when using the anonymisation cartridge. This will now make it easier to +ignore attributes which occur in a repeating or nested manner. This approach is limited to just data types other +than an object or array. The global attributes are not fully qualified path and just denote the ending of the path + ## [5.3.5] - 2020-02-25 ### Added