Merge pull request #52 from CJSCommonPlatform/dev/backport-changes

Backporting changes from framework 6 version of the tool
CJSCommonPlatform · Mar 12, 2020 · b9b09c4 · b9b09c4
2 parents 2d219af + 094803c
commit b9b09c4
Show file tree

Hide file tree

Showing 11 changed files with 105 additions and 16 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,12 @@ on [Keep a CHANGELOG](http://keepachangelog.com/). This project adheres to
 [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased]
+- Gson library available for easier manipulation of json objects for transformation
+- Surfacing all exception when processing a stream
+- Json path library added
+- Added global whitelisting of attributes when using the anonymisation cartridge.  This will now make it easier to 
+ignore attributes which occur in a repeating or nested manner.  This approach is limited to just data types other 
+than an object or array.  The global attributes are not fully qualified path and just denote the ending of the path
 
 
 ## [5.3.5] - 2020-02-25

diff --git a/event-tool/pom.xml b/event-tool/pom.xml
@@ -93,6 +93,16 @@
             <artifactId>logging</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>com.google.code.gson</groupId>
+            <artifactId>gson</artifactId>
+            <version>${gson.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.jayway.jsonpath</groupId>
+            <artifactId>json-path</artifactId>
+        </dependency>
+
         <!--External-->
         <dependency>
             <groupId>org.slf4j</groupId>

diff --git a/pom.xml b/pom.xml
@@ -31,7 +31,8 @@
         <wildfly.swarm.version>2017.11.0</wildfly.swarm.version>
         <version.swarm.fraction-plugin>77</version.swarm.fraction-plugin>
         <junit-dataprovider.version>1.13.1</junit-dataprovider.version>
-
+        <!-- Locking version of gson library as the next version 2.8.6 is bring up error on swarm starting up-->
+        <gson.version>2.8.5</gson.version>
     </properties>
 
     <modules>

diff --git a/...nonymise/src/main/java/uk/gov/justice/tools/eventsourcing/anonymization/model/Events.java b/...nonymise/src/main/java/uk/gov/justice/tools/eventsourcing/anonymization/model/Events.java
@@ -5,13 +5,15 @@
 
 public class Events {
 
+    private List<String> globalAttributes;
+
     private List<Event> events;
 
     public Events() {
     }
 
-    public Events(final List<Event> events) {
-        this.events = events;
+    public List<String> getGlobalAttributes() {
+        return globalAttributes;
     }
 
     public List<Event> getEvents() {

diff --git a/...java/uk/gov/justice/tools/eventsourcing/anonymization/service/EventAnonymiserService.java b/...java/uk/gov/justice/tools/eventsourcing/anonymization/service/EventAnonymiserService.java
@@ -29,6 +29,7 @@
 import javax.json.JsonValue;
 import javax.json.JsonValue.ValueType;
 
+import org.apache.commons.collections.CollectionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -89,7 +90,8 @@ private JsonObjectBuilder processObjectPayload(final JsonObject payload, final J
             } else {
                 final String pathForField = getPathForField(jsonPath, fieldName);
 
-                final boolean anonymise = !fieldsToIgnore.contains(pathForField);
+                final boolean attributePathWhitelistedGlobally = isAttributePathEndingWithGloballyWhitelistedAttributeValues(pathForField);
+                final boolean anonymise = !attributePathWhitelistedGlobally && !fieldsToIgnore.contains(pathForField);
                 setFieldValueInObject(fieldName, jsonValue, builder, anonymise);
             }
         }
@@ -100,8 +102,7 @@ private JsonObjectBuilder processObjectPayload(final JsonObject payload, final J
     private JsonArrayBuilder processArrayPayload(final JsonArray payload, final JsonArrayBuilder builder, final String jsonPath, final List<String> fieldsToIgnore) {
 
 
-        for (int counter = 0; counter < payload.size(); counter++) {
-            final JsonValue value = payload.get(counter);
+        for (final JsonValue value : payload) {
             final ValueType fieldValueType = value.getValueType();
             final String pathForField = getPathForField(jsonPath, "[*]");
 
@@ -118,7 +119,8 @@ private JsonArrayBuilder processArrayPayload(final JsonArray payload, final Json
                     builder.add(value);
                 }
             } else {
-                final boolean anonymise = !fieldsToIgnore.contains(pathForField);
+                final boolean attributePathWhitelistedGlobally = isAttributePathEndingWithGloballyWhitelistedAttributeValues(pathForField);
+                final boolean anonymise = !attributePathWhitelistedGlobally && !fieldsToIgnore.contains(pathForField);
                 setFieldValueInArray(value, builder, anonymise);
             }
         }
@@ -152,6 +154,14 @@ private String getTransformedStringValue(final JsonValue value) {
         return (String) stringPatternGeneratorFactory.getGenerator(valueAsString).convert(valueAsString);
     }
 
+    private boolean isAttributePathEndingWithGloballyWhitelistedAttributeValues(final String jsonPathForAttribute) {
+        return getAttributesWhitelistedForAllEvents().stream().anyMatch(jsonPathForAttribute::endsWith);
+    }
+
+    private List<String> getAttributesWhitelistedForAllEvents() {
+        return CollectionUtils.isNotEmpty(EVENTS.getGlobalAttributes()) ? EVENTS.getGlobalAttributes() : newArrayList();
+    }
+
     private List<String> getFieldsToIgnoreForEvent(final String eventName) {
         final Optional<Event> optionalEvent = EVENTS.getEvents().stream().filter(e -> e.getEventName().equals(eventName)).findFirst();
         return optionalEvent.isPresent() ? optionalEvent.get().getFieldsToBeIgnored() : emptyList();

diff --git a/...m-transformation-tool-anonymise/src/main/resources/schema/event-anonymisation-schema.json b/...m-transformation-tool-anonymise/src/main/resources/schema/event-anonymisation-schema.json
@@ -2,6 +2,15 @@
   "$schema": "http://json-schema.org/draft-04/schema#",
   "type": "object",
   "properties": {
+    "globalAttributes": {
+      "type": "array",
+      "description": "Fields that are considered for all events.  This field does not denote a complete JSON path.  It only identifies the ending of a json path (partial entry)",
+      "items": [
+        {
+          "type": "string"
+        }
+      ]
+    },
     "events": {
       "type": "array",
       "items": [
@@ -23,12 +32,14 @@
           "required": [
             "eventName",
             "fieldsToBeIgnored"
-          ]
+          ],
+          "additionalProperties": false
         }
       ]
     }
   },
   "required": [
     "events"
-  ]
+  ],
+  "additionalProperties": false
 }
diff --git a/.../uk/gov/justice/tools/eventsourcing/anonymization/service/EventAnonymiserServiceTest.java b/.../uk/gov/justice/tools/eventsourcing/anonymization/service/EventAnonymiserServiceTest.java
@@ -100,6 +100,23 @@ public void shouldNotAnonymiseJsonArrayPayload() {
         assertThat(anonymisedJsonArray.getString(6), is("SC208979B"));
     }
 
+    @Test
+    public void shouldNotAnonymiseGloballyWhitelistedAttributes() {
+        final JsonObject anonymisedPayload = service.anonymiseObjectPayload(buildObjectPayload("test-object-data.json"), "test-object-event");
+        final JsonArray exampleArray = anonymisedPayload.getJsonArray("exampleArray");
+        assertThat(exampleArray.getJsonObject(0).getJsonArray("repeatingParentArrayAttribute").getJsonObject(0).getString("repeatingChildAttribute"), is("child1"));
+        assertThat(exampleArray.getJsonObject(0).getJsonArray("repeatingParentArrayAttribute").getJsonObject(0).getString("repeatingAttribute"), is("test1"));
+        assertThat(exampleArray.getJsonObject(0).getJsonArray("repeatingParentArrayAttribute").getJsonObject(0).getJsonArray("repeatingParentArrayAttribute").getJsonObject(0).getString("repeatingChildAttribute"), is("child2"));
+        assertThat(exampleArray.getJsonObject(0).getJsonArray("repeatingParentArrayAttribute").getJsonObject(0).getJsonArray("repeatingParentArrayAttribute").getJsonObject(0).getString("repeatingAttribute"), is("test2"));
+
+        final JsonObject exampleObject = anonymisedPayload.getJsonObject("example");
+        final JsonObject repeatingObject = exampleObject.getJsonObject("repeatingParentObjectAttribute");
+        assertThat(repeatingObject.getString("repeatingChildAttribute"), is("child3"));
+        assertThat(repeatingObject.getString("repeatingAttribute"), is("test3"));
+        assertThat(repeatingObject.getJsonObject("repeatingParentObjectAttribute").getString("repeatingChildAttribute"), is("child4"));
+        assertThat(repeatingObject.getJsonObject("repeatingParentObjectAttribute").getString("repeatingAttribute"), is("test4"));
+    }
+
 
     private JsonObject buildObjectPayload(final String payloadFileName) {
         final JsonReader jsonReader = createReader(new StringReader(getFileContentsAsString(payloadFileName)));

diff --git a/stream-transformation-tool-anonymise/src/test/resources/events-anonymisation-rule.json b/stream-transformation-tool-anonymise/src/test/resources/events-anonymisation-rule.json
@@ -1,4 +1,9 @@
 {
+  "globalAttributes": [
+    "repeatingParentArrayAttribute[*].repeatingChildAttribute",
+    "repeatingParentObjectAttribute.repeatingChildAttribute",
+    "repeatingAttribute"
+  ],
   "events": [
     {
       "eventName": "test-object-event",

diff --git a/stream-transformation-tool-anonymise/src/test/resources/test-object-data.json b/stream-transformation-tool-anonymise/src/test/resources/test-object-data.json
@@ -2,7 +2,19 @@
   "exampleArray": [
     {
       "attributeIntAsString": "001",
-      "attributeIntAsStringAnonymise": "001"
+      "attributeIntAsStringAnonymise": "001",
+      "repeatingParentArrayAttribute": [
+        {
+          "repeatingChildAttribute": "child1",
+          "repeatingAttribute" : "test1",
+          "repeatingParentArrayAttribute": [
+            {
+              "repeatingChildAttribute": "child2",
+              "repeatingAttribute" : "test2"
+            }
+          ]
+        }
+      ]
     }
   ],
   "example": {
@@ -27,6 +39,14 @@
       },
       "test234@mail.com",
       "SC208979B"
-    ]
+    ],
+    "repeatingParentObjectAttribute": {
+      "repeatingChildAttribute": "child3",
+      "repeatingAttribute" : "test3",
+      "repeatingParentObjectAttribute": {
+        "repeatingChildAttribute": "child4",
+        "repeatingAttribute" : "test4"
+      }
+    }
   }
 }
diff --git a/stream-transformation-tool-api/pom.xml b/stream-transformation-tool-api/pom.xml
@@ -22,6 +22,11 @@
             <groupId>uk.gov.justice.services</groupId>
             <artifactId>messaging-core</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.google.code.gson</groupId>
+            <artifactId>gson</artifactId>
+            <version>${gson.version}</version>
+        </dependency>
     </dependencies>
 
 </project>
diff --git a/.../justice/tools/eventsourcing/transformation/service/EventStreamTransformationService.java b/.../justice/tools/eventsourcing/transformation/service/EventStreamTransformationService.java
@@ -5,7 +5,6 @@
 
 import uk.gov.justice.services.eventsourcing.source.core.EventSourceTransformation;
 import uk.gov.justice.services.eventsourcing.source.core.exception.EventStreamException;
-import uk.gov.justice.services.jdbc.persistence.JdbcRepositoryException;
 import uk.gov.justice.services.messaging.JsonEnvelope;
 import uk.gov.justice.tools.eventsourcing.transformation.EventStreamReader;
 import uk.gov.justice.tools.eventsourcing.transformation.EventTransformationRegistry;
@@ -62,8 +61,11 @@ public class EventStreamTransformationService {
     @Inject
     private RetryStreamOperator retryStreamOperator;
 
-
-    @Transactional(value = REQUIRES_NEW, rollbackOn = {EventStreamException.class, JdbcRepositoryException.class})
+    /*
+     * The transactional annotation will always rollback for RuntimeException and its subclasses.  So, no need to handle that explicitly.
+     * Only checked exceptions need to be specified
+     */
+    @Transactional(value = REQUIRES_NEW, rollbackOn = {EventStreamException.class})
     public UUID transformEventStream(final UUID originalStreamId, final int pass) throws EventStreamException {
 
         try {
@@ -88,8 +90,8 @@ public UUID transformEventStream(final UUID originalStreamId, final int pass) th
             if (action.isDeactivate()) {
                 streamRepository.deactivateStream(originalStreamId);
             }
-        } catch (final EventStreamException | JdbcRepositoryException e) {
-            logger.error(format("Unknown error while transforming events on stream %s", originalStreamId), e);
+        } catch (final Exception e) {
+            logger.error(format("Error while transforming events on stream %s", originalStreamId), e);
             throw e;
         }