Scala Common Enrich: extend PII Enrichment to include identification …

…events in EnrichedEvent (closes #3580)
snowplow · Jan 31, 2018 · 6c87d3a · 6c87d3a
1 parent 95c9e16
commit 6c87d3a
Show file tree

Hide file tree

Showing 3 changed files with 28 additions and 24 deletions.
diff --git a/...lowanalytics.snowplow.enrich/common/enrichments/registry/PiiPseudonymizerEnrichment.scala b/...lowanalytics.snowplow.enrich/common/enrichments/registry/PiiPseudonymizerEnrichment.scala
@@ -177,17 +177,18 @@ object PiiPseudonymizerEnrichment extends ParseableEnrichment {
   implicit val json4sFormats = DefaultFormats
 
   override val supportedSchema =
-    SchemaCriterion("com.snowplowanalytics.snowplow.enrichments", "pii_enrichment_config", "jsonschema", 1, 0, 0)
+    SchemaCriterion("com.snowplowanalytics.snowplow.enrichments", "pii_enrichment_config", "jsonschema", 2, 0, 0)
 
   def parse(config: JValue, schemaKey: SchemaKey): ValidatedNelMessage[PiiPseudonymizerEnrichment] = {
     for {
       conf <- matchesSchema(config, schemaKey)
-      enabled = ScalazJson4sUtils.extract[Boolean](conf, "enabled").toOption.getOrElse(false)
+      enabled                 = ScalazJson4sUtils.extract[Boolean](conf, "enabled").toOption.getOrElse(false)
+      emitIdentificationEvent = ScalazJson4sUtils.extract[Boolean](conf, "emitIdentificationEvent").toOption.getOrElse(false)
       piiFields        <- ScalazJson4sUtils.extract[List[JObject]](conf, "parameters", "pii").leftMap(_.getMessage)
       strategyFunction <- extractStrategyFunction(config)
       hashFunction     <- getHashFunction(strategyFunction)
       piiFieldList     <- extractFields(piiFields, PiiStrategyPseudonymize(hashFunction))
-    } yield if (enabled) PiiPseudonymizerEnrichment(piiFieldList) else PiiPseudonymizerEnrichment(List())
+    } yield if (enabled) PiiPseudonymizerEnrichment(piiFieldList, emitIdentificationEvent) else PiiPseudonymizerEnrichment(List(), false)
   }.leftMap(_.toProcessingMessageNel)
 
   private def getHashFunction(strategyFunction: String): Validation[String, MessageDigest] =
@@ -256,8 +257,9 @@ object PiiPseudonymizerEnrichment extends ParseableEnrichment {
  * unstruct_event or an array in the case of derived_events and contexts
  *
  * @param fieldList a list of configured PiiFields
+ * @param emitIdentificationEvent whether to emit an identification event
  */
-case class PiiPseudonymizerEnrichment(fieldList: List[PiiField]) extends Enrichment {
+case class PiiPseudonymizerEnrichment(fieldList: List[PiiField], emitIdentificationEvent: Boolean) extends Enrichment {
   def transformer(event: EnrichedEvent): Unit = fieldList.foreach(_.transform(event))
 }
 

diff --git a/...snowplowanalytics.snowplow.enrich.common/enrichments/registry/EnrichmentConfigsSpec.scala b/...snowplowanalytics.snowplow.enrich.common/enrichments/registry/EnrichmentConfigsSpec.scala
@@ -294,6 +294,7 @@ class EnrichmentConfigsSpec extends Specification with ValidationMatchers {
       import PiiConstants._
       val piiPseudonymizerEnrichmentJson = parse("""{
           |  "enabled": true,
+          |  "emitIdentificationEvent": true,
           |  "parameters": {
           |    "pii": [
           |      {
@@ -317,21 +318,9 @@ class EnrichmentConfigsSpec extends Specification with ValidationMatchers {
           |  }
           |}""".stripMargin)
 
-      val schemaKey = SchemaKey("com.snowplowanalytics.snowplow.enrichments", "pii_enrichment_config", "jsonschema", "1-0-0")
+      val schemaKey = SchemaKey("com.snowplowanalytics.snowplow.enrichments", "pii_enrichment_config", "jsonschema", "2-0-0")
 
       val result = PiiPseudonymizerEnrichment.parse(piiPseudonymizerEnrichmentJson, schemaKey)
-      val expected = PiiPseudonymizerEnrichment(
-        fieldList = List(
-          PiiScalar(strategy     = PiiStrategyPseudonymize(hashFunction = java.security.MessageDigest.getInstance("SHA-256")),
-                    fieldMutator = ScalarMutators.get("user_id").get),
-          PiiJson(
-            strategy        = PiiStrategyPseudonymize(hashFunction = java.security.MessageDigest.getInstance("SHA-256")),
-            fieldMutator    = JsonMutators.get("contexts").get,
-            schemaCriterion = SchemaCriterion.parse("iglu:com.acme/email_sent/jsonschema/1-*-*").toOption.get,
-            jsonPath        = "$.emailAddress"
-          )
-        )
-      )
       result must beSuccessful.like {
         case piiRes: PiiPseudonymizerEnrichment => {
           (piiRes.fieldList.size must_== 2) and
@@ -345,7 +334,8 @@ class EnrichmentConfigsSpec extends Specification with ValidationMatchers {
             (piiRes.fieldList(1).asInstanceOf[PiiJson].schemaCriterion.toString must_== "iglu:com.acme/email_sent/jsonschema/1-*-*") and
             (piiRes.fieldList(1).asInstanceOf[PiiJson].jsonPath must_== "$.emailAddress") and
             (piiRes.fieldList(1).asInstanceOf[PiiJson].strategy.asInstanceOf[PiiStrategyPseudonymize].hashFunction.toString must contain(
-              "SHA-256"))
+              "SHA-256") and
+              (piiRes.emitIdentificationEvent mustEqual (true)))
         }
       }
     }

diff --git a/...nalytics.snowplow.enrich.common/enrichments/registry/PiiPseudonymizerEnrichmentSpec.scala b/...nalytics.snowplow.enrich.common/enrichments/registry/PiiPseudonymizerEnrichmentSpec.scala
@@ -154,7 +154,9 @@ class PiiPseudonymizerEnrichmentSpec extends Specification with ValidationMatche
                     fieldMutator = ScalarMutators.get("ip_domain").get),
           PiiScalar(strategy     = PiiStrategyPseudonymize(hashFunction = MessageDigest.getInstance("SHA-256")),
                     fieldMutator = ScalarMutators.get("user_fingerprint").get)
-        )))
+        ),
+        false
+      ))
     )
     val output   = commonSetup(enrichmentMap = enrichmentMap)
     val expected = new EnrichedEvent()
@@ -205,7 +207,9 @@ class PiiPseudonymizerEnrichmentSpec extends Specification with ValidationMatche
             schemaCriterion = SchemaCriterion.parse("iglu:com.mailgun/message_clicked/jsonschema/1-0-0").toOption.get,
             jsonPath        = "$.ip"
           )
-        )))
+        ),
+        false
+      ))
     )
 
     val output   = commonSetup(enrichmentMap = enrichmentMap)
@@ -255,7 +259,9 @@ class PiiPseudonymizerEnrichmentSpec extends Specification with ValidationMatche
             schemaCriterion = SchemaCriterion.parse("iglu:com.acme/email_sent/jsonschema/1-*-*").toOption.get,
             jsonPath        = "$.field.that.does.not.exist.in.this.instance"
           )
-        )))
+        ),
+        false
+      ))
     )
 
     val output   = commonSetup(enrichmentMap = enrichmentMap)
@@ -293,7 +299,9 @@ class PiiPseudonymizerEnrichmentSpec extends Specification with ValidationMatche
             schemaCriterion = SchemaCriterion.parse("iglu:com.acme/email_sent/jsonschema/1-0-*").toOption.get,
             jsonPath        = "$.['emailAddress', 'emailAddress2']"
           )
-        )))
+        ),
+        false
+      ))
     )
 
     val output   = commonSetup(enrichmentMap = enrichmentMap)
@@ -333,7 +341,9 @@ class PiiPseudonymizerEnrichmentSpec extends Specification with ValidationMatche
             schemaCriterion = SchemaCriterion.parse("iglu:com.acme/email_sent/jsonschema/1-*-0").toOption.get,
             jsonPath        = "$.emailAddress"
           )
-        )))
+        ),
+        false
+      ))
     )
     val output   = commonSetup(enrichmentMap = enrichmentMap)
     val expected = new EnrichedEvent()
@@ -373,7 +383,9 @@ class PiiPseudonymizerEnrichmentSpec extends Specification with ValidationMatche
             schemaCriterion = SchemaCriterion.parse("iglu:com.acme/email_sent/jsonschema/1-*-*").toOption.get,
             jsonPath        = "$.someInt"
           )
-        )))
+        ),
+        false
+      ))
     )
     val output   = commonSetup(enrichmentMap = enrichmentMap)
     val expected = new EnrichedEvent()