Refactoring including improved docstrings and imports

3-enrich/scala-common-enrich/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentManager.scala

@@ -10,7 +10,10 @@
  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
  */
-package com.snowplowanalytics.snowplow.enrich.common
+package com.snowplowanalytics
+package snowplow
+package enrich
+package common
 package enrichments
 
 // Scala
@@ -21,10 +24,10 @@ import scalaz._
 import Scalaz._
 
 // SnowPlow Utils
-import com.snowplowanalytics.util.Tap._
+import util.Tap._
 
 // Scala MaxMind GeoIP
-import com.snowplowanalytics.maxmind.geoip.IpGeo
+import maxmind.geoip.IpGeo
 
 // This project
 import inputs.{CanonicalInput, NvGetPayload}
@@ -51,7 +54,11 @@ object EnrichmentManager {
   /**
    * Runs our enrichment process.
    *
-   * @param input Our canonical input
+   * @param registry Contains configuration
+   *        for all enrichments to apply
+   * @param hostEtlVersion ETL version
+   * @param etlTstamp ETL timestamp
+   * @param raw Our canonical input
    *        to enrich
    * @return a MaybeCanonicalOutput - i.e.
    *         a ValidationNel containing

3-enrich/scala-common-enrich/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentRegistry.scala

@@ -16,8 +16,6 @@ package enrich
 package common
 package enrichments
 
-import utils.ScalazJson4sUtils
-
 // Scalaz
 import scalaz._
 import Scalaz._
@@ -29,12 +27,21 @@ import org.json4s.JsonDSL._
 import org.json4s.jackson.JsonMethods._
 
 // Iglu
-import iglu.client._
+import iglu.client.{
+  SchemaKey, 
+  Resolver
+}
 import iglu.client.validation.ValidatableJsonMethods._
 import iglu.client.validation.ProcessingMessageMethods._
 import iglu.client.validation.ValidatableJsonNode
 
-import registry._
+// This project
+import registry.{
+  Enrichment,
+  AnonIpEnrichment,
+  IpToGeoEnrichment
+}
+import utils.ScalazJson4sUtils
 
 /**
  * Companion which holds a constructor
@@ -48,7 +55,13 @@ object EnrichmentConfigRegistry {
    * Constructs our EnrichmentConfigRegistry
    * from the supplied JSON JValue.
    *
-   * TODO: rest of docstring 
+   * @param node A JValue representing an array of enrichment JSONs
+   * @param localMode Whether to use the local MaxMind data file
+   *        Enabled for tests
+   * @param resolver (implicit) The Iglu resolver used for
+   *        schema lookup and validation
+   * @return Validation boxing an EnrichmentConfigRegistry object
+   *         containing enrichments configured from node
    * @todo remove all the JsonNode round-tripping when
    *       we have ValidatableJValue
    */
@@ -62,7 +75,7 @@ object EnrichmentConfigRegistry {
       } yield arr).flatten
 
     // Check each enrichment validates against its own schema
-    val configs: ValidatedNelMessage[Map[String, Enrichment]] = (for {
+    val configs: ValidatedNelMessage[EnrichmentMap] = (for {
         jsons <- enrichments
       } yield for {    
         json  <- jsons
@@ -85,12 +98,14 @@ object EnrichmentConfigRegistry {
    *
    * @param enrichmentConfig JValue with enrichment information
    * @param schemaKey SchemaKey for the JValue
+   * @param localMode Whether to use the local MaxMind data file
+   *        Enabled for tests
    * @return ValidatedNelMessage boxing Option boxing Tuple2 containing
    *         the Enrichment object and the schemaKey
    */
   private def buildEnrichmentConfig(schemaKey: SchemaKey, enrichmentConfig: JValue, localMode: Boolean): ValidatedNelMessage[Option[Tuple2[String, Enrichment]]] = {
 
-    val name: ValidatedNelMessage[String] = ScalazJson4sUtils.extractString(enrichmentConfig, NonEmptyList("name")).toValidationNel
+    val name = ScalazJson4sUtils.extractString(enrichmentConfig, NonEmptyList("name")).toValidationNel
     name.flatMap( nm => {
 
       if (nm == "ip_to_geo") {
@@ -111,8 +126,12 @@ object EnrichmentConfigRegistry {
  *
  * In the future this may evolve to holding
  * all of our enrichments themselves.
+ *
+ * @param configs Map whose keys are enrichment
+ *        names and whose values are the
+ *        corresponding enrichment objects
  */
-case class EnrichmentConfigRegistry(private val configs: Map[String, Enrichment]) {
+case class EnrichmentConfigRegistry(private val configs: EnrichmentMap) {
 
   /**
    * Returns an Option boxing the AnonIpEnrichment
@@ -143,7 +162,15 @@ case class EnrichmentConfigRegistry(private val configs: Map[String, Enrichment]
   private def getEnrichment[A <: Enrichment: Manifest](name: String): Option[A] =
     configs.get(name).map(cast[A](_))
 
-  // Adapted from http://stackoverflow.com/questions/6686992/scala-asinstanceof-with-parameterized-types
+  /**
+   * Adapted from http://stackoverflow.com/questions/6686992/scala-asinstanceof-with-parameterized-types
+   * Used to convert an Enrichment to a
+   * specific subtype of Enrichment
+   * 
+   * @tparam A Type to cast to
+   * @param a The object to cast to type A
+   * @return a, converted to type A
+   */
   private def cast[A <: AnyRef : Manifest](a : Any) : A 
     = manifest.runtimeClass.cast(a).asInstanceOf[A]
 }

3-enrich/scala-common-enrich/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/AnonIpEnrichment.scala

@@ -17,22 +17,19 @@ package common
 package enrichments
 package registry
 
-import utils.ScalazJson4sUtils
-
 // Scalaz
 import scalaz._
 import Scalaz._
 
 // json4s
-import org.json4s.scalaz.JsonScalaz._
-import org.json4s._
-import org.json4s.JsonDSL._
-import org.json4s.jackson.JsonMethods._
+import org.json4s.JValue
 
 // Iglu
-import iglu.client._
+import iglu.client.SchemaKey
 import iglu.client.validation.ProcessingMessageMethods._
 
+// This project
+import utils.ScalazJson4sUtils
 
 /**
 * Companion object. Lets us create a AnonIpEnrichment
@@ -46,6 +43,8 @@ object AnonIpEnrichment extends ParseableEnrichment {
    * Creates an AnonIpEnrichment instance from a JValue.
    * 
    * @param config The anon_ip enrichment JSON
+   * @param schemaKey The SchemaKey provided for the enrichment
+   *        Must be a supported SchemaKey for this enrichment   
    * @return a configured AnonIpEnrichment instance
    */
   def parse(config: JValue, schemaKey: SchemaKey): ValidatedNelMessage[AnonIpEnrichment] = {
@@ -96,6 +95,8 @@ object AnonOctets extends Enumeration {
 
 /**
  * Config for an anon_ip enrichment
+ *
+ * @param octets The number of octets to anonymize
  */
 case class AnonIpEnrichment(
   octets: AnonOctets.AnonOctets

3-enrich/scala-common-enrich/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/IpToGeoEnrichment.scala

@@ -17,9 +17,6 @@ package common
 package enrichments
 package registry
 
-// This project but has to be defined before Scalaz
-import utils.ScalazJson4sUtils
-
 // Java
 import java.net.URI
 
@@ -28,13 +25,10 @@ import scalaz._
 import Scalaz._
 
 // json4s
-import org.json4s.scalaz.JsonScalaz._
-import org.json4s._
-import org.json4s.JsonDSL._
-import org.json4s.jackson.JsonMethods._
+import org.json4s.JValue
 
 // Iglu
-import iglu.client._
+import iglu.client.SchemaKey
 import iglu.client.validation.ProcessingMessageMethods._
 
 // Scala MaxMind GeoIP
@@ -45,6 +39,7 @@ import maxmind.geoip.{
 
 // This project
 import common.utils.ConversionUtils
+import utils.ScalazJson4sUtils
 
 /**
 * Companion object. Lets us create a IpToGeoEnrichment
@@ -58,6 +53,10 @@ object IpToGeoEnrichment extends ParseableEnrichment {
    * Creates an IpToGeoEnrichment instance from a JValue.
    * 
    * @param config The ip_to_geo enrichment JSON
+   * @param schemaKey The SchemaKey provided for the enrichment
+   *        Must be a supported SchemaKey for this enrichment
+   * @param localMode Whether to use the local MaxMind data file
+   *        Enabled for tests
    * @return a configured IpToGeoEnrichment instance
    */
   def parse(config: JValue, schemaKey: SchemaKey, localMode: Boolean): ValidatedNelMessage[IpToGeoEnrichment] = {
@@ -80,6 +79,8 @@ object IpToGeoEnrichment extends ParseableEnrichment {
    *
    * @param maxmindFile A String holding the
    *        URI to the hosted MaxMind file
+   * @param database Name of the MaxMind
+   *        database
    * @return a Validation-boxed URI
    */
   private def getMaxmindUri(uri: String, database: String): ValidatedMessage[URI] =
@@ -93,7 +94,10 @@ object IpToGeoEnrichment extends ParseableEnrichment {
 /**
  * Contains enrichments related to geo-location.
  *
- * TODO: add params to make clear uri is now full URI
+ * @param uri Full URI to the MaxMind data file
+ * @param database Name of the MaxMind database
+ * @param localMode Whether to use the local
+ *        MaxMind data file. Enabled for tests.
  */
 case class IpToGeoEnrichment(
   uri: URI,

3-enrich/scala-common-enrich/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/enrichments.scala

@@ -17,22 +17,20 @@ package common
 package enrichments
 package registry
 
-import utils.ScalazJson4sUtils
-
 // Scalaz
 import scalaz._
 import Scalaz._
 
 // json4s
-import org.json4s.scalaz.JsonScalaz._
-import org.json4s._
-import org.json4s.JsonDSL._
-import org.json4s.jackson.JsonMethods._
+import org.json4s.JValue
 
 // Iglu
-import iglu.client._
+import iglu.client.SchemaKey
 import iglu.client.validation.ProcessingMessageMethods._
 
+// This project
+import utils.ScalazJson4sUtils
+
 /**
  * Trait inherited by every enrichment config case class
  */
@@ -58,7 +56,7 @@ trait ParseableEnrichment {
     if (schemaKey == supportedSchemaKey) {
       config.success
     } else {
-      ("Wrong type of JSON for an enrichment of type %").format(schemaKey.name).toProcessingMessage.fail.toValidationNel
+      ("Wrong type of JSON for an enrichment of type %s").format(schemaKey.name).toProcessingMessage.fail.toValidationNel
     }
   }
 

3-enrich/scala-common-enrich/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/ScalazJson4sUtils.scala

@@ -21,10 +21,12 @@ import scalaz._
 import Scalaz._
 
 // json4s
-import org.json4s.scalaz.JsonScalaz._
-import org.json4s._
+import org.json4s.{
+  DefaultFormats,
+  JValue,
+  MappingException
+}
 import org.json4s.JsonDSL._
-import org.json4s.jackson.JsonMethods._
 
 // Iglu
 import iglu.client.validation.ProcessingMessageMethods._

3-enrich/scala-common-enrich/src/main/scala/com.snowplowanalytics.snowplow.enrich/package.scala

@@ -25,12 +25,13 @@ import org.apache.http.NameValuePair
 // Scala MaxMind GeoIP
 import com.snowplowanalytics.maxmind.geoip.IpLocation
 
+// JSON Schema Validator
+import com.github.fge.jsonschema.core.report.ProcessingMessage
+
 // This project
 import common.inputs.CanonicalInput
 import common.outputs.CanonicalOutput
-
-// JSON Schema Validator
-import com.github.fge.jsonschema.core.report.ProcessingMessage
+import common.enrichments.registry.Enrichment
 
 /**
  * Scala package object to hold types,
@@ -52,6 +53,13 @@ package object common {
    */
   type HttpHeaders = List[String]
 
+  /**
+   * Type alias for a map whose
+   * keys are enrichment names and
+   * whose values are enrichments
+   */
+  type EnrichmentMap = Map[String, Enrichment]
+
   /**
    * Type alias for a non-empty
    * set of name-value pairs
@@ -67,10 +75,6 @@ package object common {
    */
   type Validated[A] = ValidationNel[String, A]
 
-  type ValidatedMessage[A] = Validation[ProcessingMessage, A]
-
-  type ValidatedNelMessage[A] = ValidationNel[ProcessingMessage, A]
-
   /**
    * Type alias for a `Validation`
    * containing either an error
@@ -128,4 +132,23 @@ package object common {
    * or a MaybeCanonicalOutput for `Success`.
    */
   type ValidatedMaybeCanonicalOutput = Validated[MaybeCanonicalOutput]
+
+  /**
+   * Type alias for a `Validation`
+   * containing ProcessingMessages
+   * for `Failure` or any type for
+   * `Success`
+   *
+   * @tparam A the type of `Success`
+   */
+  type ValidatedMessage[A] = Validation[ProcessingMessage, A]
+
+  /**
+   * Type alias for a `ValidationNel`
+   * containing ProcessingMessage
+   * for `Failure` or any type for
+   * `Success`
+   */
+  type ValidatedNelMessage[A] = ValidationNel[ProcessingMessage, A]
+
 }

3-enrich/scala-common-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/ExtractGeoLocationTest.scala

@@ -39,9 +39,6 @@ class ExtractGeoLocationTest extends Specification with DataTables with Validati
   "extractGeoLocation should correctly extract location data from IP addresses where possible"      ! e2^
                                                                                                     end
 
-  //val dbFile = getClass.getResource("/maxmind/GeoLiteCity.dat").toURI.getPath
-  //val ipGeo = IpGeo(dbFile, memCache = true, lruCache = 20000)
-
   val config = IpToGeoEnrichment(new URI("/not-used/"), "GeoLiteCity.dat", true)
 
   // Impossible to make extractIpLocation throw a validation error

3-enrich/scala-common-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/privacyEnrichmentTests.scala

@@ -18,8 +18,6 @@ package registry
 import org.specs2.Specification
 import org.specs2.matcher.DataTables
 
-import AnonIpEnrichment._
-
 /**
  * Tests the anonymzeIp function
  */