Merge branch 'feature/core-2015-refresh'

snowplow · Apr 2, 2015 · 80236a1 · 80236a1
2 parents 7689624 + b724ed2
commit 80236a1
Show file tree

Hide file tree

Showing 115 changed files with 3,545 additions and 395 deletions.
diff --git a/3-enrich/emr-etl-runner/config/config.yml.sample b/3-enrich/emr-etl-runner/config/config.yml.sample
@@ -41,8 +41,8 @@
 :etl:
   :job_name: Snowplow ETL # Give your job a name
   :versions:
-    :hadoop_enrich: 0.13.0 # Version of the Hadoop Enrichment process
-    :hadoop_shred: 0.3.0 # Version of the Hadoop Shredding process
+    :hadoop_enrich: 0.14.0 # Version of the Hadoop Enrichment process
+    :hadoop_shred: 0.4.0 # Version of the Hadoop Shredding process
   :collector_format: cloudfront # Or 'clj-tomcat' for the Clojure Collector, or 'thrift' for Thrift records, or 'tsv/com.amazon.aws.cloudfront/wd_access_log' for Cloudfront access logs
   :continue_on_unexpected_error: false # Set to 'true' (and set :out_errors: above) if you don't want any exceptions thrown from ETL
 :iglu:

diff --git a/3-enrich/emr-etl-runner/config/enrichments/campaign_attribution.json b/3-enrich/emr-etl-runner/config/enrichments/campaign_attribution.json
@@ -1,5 +1,5 @@
 {
-	"schema": "iglu:com.snowplowanalytics.snowplow/campaign_attribution/jsonschema/1-0-0",
+	"schema": "iglu:com.snowplowanalytics.snowplow/campaign_attribution/jsonschema/1-0-1",
 
 	"data": {
 

diff --git a/3-enrich/emr-etl-runner/config/enrichments/currency_conversion_config.json b/3-enrich/emr-etl-runner/config/enrichments/currency_conversion_config.json
@@ -0,0 +1,16 @@
+{
+	"schema": "iglu:com.snowplowanalytics.snowplow/currency_conversion_config/jsonschema/1-0-0",
+
+	"data": {
+
+		"enabled": true,
+		"vendor": "com.snowplowanalytics.snowplow",
+		"name": "currency_conversion_config",
+		"parameters": {
+			"accountType": "DEVELOPER",
+			"apiKey": "{{KEY}}",
+			"baseCurrency": "USD",
+			"rateAt": "EOD_PRIOR"
+		}
+	}
+}
diff --git a/3-enrich/emr-etl-runner/config/enrichments/ua_parser_config.json b/3-enrich/emr-etl-runner/config/enrichments/ua_parser_config.json
@@ -0,0 +1,12 @@
+{
+	"schema": "iglu:com.snowplowanalytics.snowplow/ua_parser_config/jsonschema/1-0-0",
+
+	"data": {
+
+		"enabled": true,
+		"vendor": "com.snowplowanalytics.snowplow",
+		"name": "ua_parser_config",
+		"enabled": true,
+		"parameters": {}
+	}
+}
diff --git a/3-enrich/emr-etl-runner/config/enrichments/user_agent_utils_config.json b/3-enrich/emr-etl-runner/config/enrichments/user_agent_utils_config.json
@@ -0,0 +1,11 @@
+{
+	"schema": "iglu:com.snowplowanalytics.snowplow/user_agent_utils_config/jsonschema/1-0-0",
+
+	"data": {
+
+		"vendor": "com.snowplowanalytics.snowplow",
+		"name": "user_agent_utils_config",
+		"enabled": true,
+		"parameters": {}
+	}
+}
diff --git a/3-enrich/scala-common-enrich/project/BuildSettings.scala b/3-enrich/scala-common-enrich/project/BuildSettings.scala
@@ -21,7 +21,7 @@ object BuildSettings {
   // Basic settings for our app
   lazy val basicSettings = Seq[Setting[_]](
     organization          :=  "com.snowplowanalytics",
-    version               :=  "0.12.0",
+    version               :=  "0.13.0",
     description           :=  "Common functionality for enriching raw Snowplow events",
     scalaVersion          :=  "2.10.1",
     scalacOptions         :=  Seq("-deprecation", "-encoding", "utf8",

diff --git a/3-enrich/scala-common-enrich/project/Dependencies.scala b/3-enrich/scala-common-enrich/project/Dependencies.scala
@@ -25,7 +25,9 @@ object Dependencies {
     "Snowplow Analytics Maven repo" at "http://maven.snplow.com/releases/",
     "Snowplow Analytics Maven snapshot repo" at "http://maven.snplow.com/snapshots/",
     // For user-agent-utils
-    "user-agent-utils repo" at "https://raw.github.com/HaraldWalker/user-agent-utils/mvn-repo/"
+    "user-agent-utils repo" at "https://raw.github.com/HaraldWalker/user-agent-utils/mvn-repo/",
+    // For uaParser utils
+    "user-agent-parser repo" at "https://clojars.org/repo/"
   )
 
   object V {
@@ -38,17 +40,19 @@ object Dependencies {
     val useragent        = "1.14"
     val jacksonDatabind  = "2.2.3"
     val jsonValidator    = "2.2.3"
-    val mavenArtifact    = "3.2.2" 
+    val mavenArtifact    = "3.2.2"
+    val uaParser         = "1.3.0"
     // Scala
     val scalaz7          = "7.0.0"
     val snowplowRawEvent = "0.1.0"
     val collectorPayload = "0.0.0"
     val schemaSniffer    = "0.0.0"
     val scalaUtil        = "0.1.0"
-    val refererParser    = "0.2.2"
+    val refererParser    = "0.2.3"
     val maxmindIplookups = "0.2.0"
     val json4s           = "3.2.11"
     val igluClient       = "0.2.0"
+    val scalaForex       = "0.2.0"
     // Scala (test only)
     val specs2           = "1.14"
     val scalazSpecs2     = "0.1.2"
@@ -68,7 +72,9 @@ object Dependencies {
     val jacksonDatabind  = "com.fasterxml.jackson.core" %  "jackson-databind"          % V.jacksonDatabind
     val jsonValidator    = "com.github.fge"             %  "json-schema-validator"     % V.jsonValidator    
     val mavenArtifact    = "org.apache.maven"           %  "maven-artifact"            % V.mavenArtifact
+    val uaParser         = "org.clojars.timewarrior"    %  "ua-parser"                 % V.uaParser
     // Scala
+    val scalaForex       = "com.snowplowanalytics"      %% "scala-forex"               % V.scalaForex
     val scalaz7          = "org.scalaz"                 %% "scalaz-core"               % V.scalaz7
     val snowplowRawEvent = "com.snowplowanalytics"      %  "snowplow-thrift-raw-event" % V.snowplowRawEvent
     val collectorPayload = "com.snowplowanalytics"      %  "collector-payload-1"       % V.collectorPayload

diff --git a/3-enrich/scala-common-enrich/project/SnowplowCommonEnrichBuild.scala b/3-enrich/scala-common-enrich/project/SnowplowCommonEnrichBuild.scala
@@ -39,8 +39,9 @@ object SnowplowCommonEnrichBuild extends Build {
         Libraries.commonsIo,
         Libraries.useragent,
         Libraries.jacksonDatabind,
-        Libraries.jsonValidator,  
-        Libraries.mavenArtifact,      
+        Libraries.jsonValidator,
+        Libraries.mavenArtifact,
+        Libraries.uaParser,
         // Scala
         Libraries.scalaz7,
         Libraries.snowplowRawEvent,
@@ -51,8 +52,9 @@ object SnowplowCommonEnrichBuild extends Build {
         Libraries.maxmindIplookups,
         Libraries.json4sJackson,
         Libraries.json4sScalaz,
-        Libraries.igluClient,        
+        Libraries.igluClient,
         Libraries.scalaUri,
+        Libraries.scalaForex,
         // Scala (test only)
         Libraries.specs2,
         Libraries.scalazSpecs2,

diff --git a/...in/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/ClientEnrichments.scala b/...in/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/ClientEnrichments.scala
@@ -20,17 +20,13 @@ import java.lang.{Integer => JInteger}
 import scalaz._
 import Scalaz._
 
-// UserAgentUtils
-import eu.bitwalker.useragentutils._
-
 /**
  * Contains enrichments related to the
  * client - where the client is the
  * software which is using the SnowPlow
  * tracker.
  *
- * Enrichments relate to the useragent,
- * browser resolution, etc.
+ * Enrichments relate to browser resolution
  */
 object ClientEnrichments {
 
@@ -43,30 +39,6 @@ object ClientEnrichments {
    */
   private val ResRegex = """(\d+)x(\d+)""".r
 
-  /**
-   * Case class to wrap everything we
-   * can extract from the useragent
-   * using UserAgentUtils.
-   *
-   * TODO: update this definition when
-   * we swap out UserAgentUtils for
-   * ua-parser
-   */
-  case class ClientAttributes(
-      // Browser
-      browserName: String,
-      browserFamily: String,
-      browserVersion: Option[String],
-      browserType: String,
-      browserRenderEngine: String,
-      // OS the browser is running on
-      osName: String,
-      osFamily: String,
-      osManufacturer: String,
-      // Hardware the OS is running on
-      deviceType: String,
-      deviceIsMobile: Boolean)
-
   /**
    * Extracts view dimensions (e.g. screen resolution,
    * browser/app viewport) stored as per the Tracker
@@ -93,43 +65,4 @@ object ClientEnrichments {
       case _ => "Field [%s]: [%s] does not contain valid view dimensions".format(field, res).fail
     }
 
-  /**
-   * Extracts the client attributes
-   * from a useragent string, using
-   * UserAgentUtils.
-   *
-   * TODO: rewrite this when we swap
-   * out UserAgentUtils for ua-parser
-   *
-   * @param useragent The useragent
-   *        String to extract from.
-   *        Should be encoded (i.e.
-   *        not previously decoded).
-   * @return the ClientAttributes or
-   *         the message of the
-   *         exception, boxed in a
-   *         Scalaz Validation
-   */
-  def extractClientAttributes(useragent: String): Validation[String, ClientAttributes] = 
-
-    try {
-      val ua = UserAgent.parseUserAgentString(useragent)
-      val b  = ua.getBrowser
-      val v  = Option(ua.getBrowserVersion)
-      val os = ua.getOperatingSystem
-
-      ClientAttributes(
-        browserName = b.getName,
-        browserFamily = b.getGroup.getName,
-        browserVersion = v map { _.getVersion },
-        browserType = b.getBrowserType.getName,
-        browserRenderEngine = b.getRenderingEngine.toString,
-        osName = os.getName,
-        osFamily = os.getGroup.getName,
-        osManufacturer = os.getManufacturer.getName,
-        deviceType = os.getDeviceType.getName,
-        deviceIsMobile = os.isMobileDevice).success
-    } catch {
-      case e => "Exception parsing useragent [%s]: [%s]".format(useragent, e.getMessage).fail
-    }
-}
+}