Converting transactions from given currency to "home currency" (closes …

…snowplow/snowplow#370)

Converting transactions from given currency to home currency

Converting transactions from given currency to home currency

Converting transactions from given currency to home currency

Converting transactions from given currency to home currency (closes snowplow/snowplow#370)

Converting transactions from given currency to home currency (closes snowplow/snowplow#370)

Converting transactions from given currency to home currency (closes snowplow/snowplow#370)

Converting transactions from given currency to home currency (closes snowplow/snowplow#370)

Converting transactions from given currency to home currency (closes snowplow/snowplow#370)

Converting transactions from given currency to home currency (closes snowplow/snowplow#370)

Converting transactions from given currency to home currency (closes snowplow/snowplow#370)

Converting transactions from given currency to home currency (closes snowplow/snowplow#370)

Converting transactions from given currency to home currency (closes snowplow/snowplow#370)

Converting transactions from given currency to home currency (closes snowplow/snowplow#370)

Converting transactions from given currency to home currency (closes snowplow/snowplow#370)

project/Dependencies.scala

@@ -47,6 +47,7 @@ object Dependencies {
     val maxmindIplookups = "0.2.0"
     val json4s           = "3.2.11"
     val igluClient       = "0.2.0"
+    val scalaForex       = "0.2.0"
     // Scala (test only)
     val specs2           = "1.14"
     val scalazSpecs2     = "0.1.2"
@@ -67,6 +68,7 @@ object Dependencies {
     val jsonValidator    = "com.github.fge"             %  "json-schema-validator"     % V.jsonValidator    
     val mavenArtifact    = "org.apache.maven"           %  "maven-artifact"            % V.mavenArtifact
     // Scala
+    val scalaForex       = "com.snowplowanalytics"      %% "scala-forex"               % V.scalaForex
     val scalaz7          = "org.scalaz"                 %% "scalaz-core"               % V.scalaz7
     val snowplowRawEvent = "com.snowplowanalytics"      %  "snowplow-thrift-raw-event" % V.snowplowRawEvent
     val scalaUtil        = "com.snowplowanalytics"      %  "scala-util"                % V.scalaUtil

project/SnowplowCommonEnrichBuild.scala

@@ -51,6 +51,7 @@ object SnowplowCommonEnrichBuild extends Build {
         Libraries.json4sScalaz,
         Libraries.igluClient,        
         Libraries.scalaUri,
+        Libraries.scalaForex,
         // Scala (test only)
         Libraries.specs2,
         Libraries.scalazSpecs2,

src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentManager.scala

@@ -224,12 +224,12 @@ object EnrichmentManager {
 
       // Set the URL components
       val components = CU.explodeUri(u)
-      event.page_urlscheme = components.scheme
-      event.page_urlhost = components.host
-      event.page_urlport = components.port
-      event.page_urlpath = components.path.orNull
-      event.page_urlquery = components.query.orNull
-      event.page_urlfragment = components.fragment.orNull
+        event.page_urlscheme = components.scheme
+        event.page_urlhost = components.host
+        event.page_urlport = components.port
+        event.page_urlpath = components.path.orNull
+        event.page_urlquery = components.query.orNull
+        event.page_urlfragment = components.fragment.orNull
     }
 
     // If our IpToGeo enrichment is enabled,
@@ -264,12 +264,44 @@ object EnrichmentManager {
       }
     }
 
-    // Finally anonymize the IP address
+    // To anonymize the IP address
     Option(event.user_ipaddress).map(ip => event.user_ipaddress = registry.getAnonIpEnrichment match {
       case Some(anon) => anon.anonymizeIp(ip)
       case None => ip
     })
 
+    // For Currency Conversion Enrichment
+    val trTax      = CU.stringToMaybeDouble("tr_tx", event.tr_tax )
+    val tiPrice    = CU.stringToMaybeDouble("ti_pr", event.ti_price)
+    val trTotal    = CU.stringToMaybeDouble("tr_tt", event.tr_total )
+    val trShipping = CU.stringToMaybeDouble("tr_sh", event.tr_shipping)
+
+    // Finalize the currency conversion
+    val currency = {
+      registry.getCurrencyConversionEnrichment match {
+        case Some(currency) => {
+          event.currency_base = currency.baseCurrency
+          var convertedCu = (trTotal |@| trTax |@| trShipping |@| tiPrice) { 
+            currency.convertCurrencies(Option(event.tr_currency), _, _, _, Option(event.ti_currency), _, Option(raw.context.timestamp))
+          }  
+          convertedCu match {
+            // TODO: clean this pattern match up
+            case Success(len) =>  {
+              for(arr <- len){
+                val (total, tax, shipping, price) = arr
+                event.tr_total_base = total.orNull
+                event.tr_tax_base = tax.orNull
+                event.tr_shipping_base = shipping.orNull
+                event.ti_price_base = price.orNull
+              }                    
+            }    
+          }
+          convertedCu
+        } 
+        case None => unitSuccess                              
+      }
+    }
+
     // Potentially set the referrer details and URL components
     val refererUri = CU.stringToUri(event.page_referrer)
     for (uri <- refererUri; u <- uri) {
@@ -330,15 +362,16 @@ object EnrichmentManager {
     event.se_label = CU.truncate(event.se_label, 255)
 
     // Collect our errors on Failure, or return our event on Success
-    (useragent.toValidationNel    |@|
-      client.toValidationNel      |@|
-      pageUri.toValidationNel     |@|
-      geoLocation.toValidationNel |@|
-      refererUri.toValidationNel  |@|
-      transform                   |@|
-      secondPassTransform         |@|
+    (useragent.toValidationNel                |@|
+      client.toValidationNel                  |@|
+      pageUri.toValidationNel                 |@|
+      geoLocation.toValidationNel             |@|
+      refererUri.toValidationNel              |@|
+      transform                               |@|
+      currency.toValidationNel               |@|
+      secondPassTransform                     |@|
       campaign) {
-      (_,_,_,_,_,_,_,_) => event
+      (_,_,_,_,_,_,_,_,_) => event
     }
   }
 }

src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentRegistry.scala

@@ -42,10 +42,16 @@ import registry.{
   AnonIpEnrichment,
   IpLookupsEnrichment,
   RefererParserEnrichment,
-  CampaignAttributionEnrichment
+  CampaignAttributionEnrichment,
+  CurrencyConversionEnrichment
 }
+
+// CurrencyConversionEnrichmentConfig
+import registry.CurrencyConversionEnrichmentConfig
+
 import utils.ScalazJson4sUtils
 
+
 /**
  * Companion which holds a constructor
  * for the EnrichmentRegistry.
@@ -125,7 +131,9 @@ object EnrichmentRegistry {
             RefererParserEnrichment.parse(enrichmentConfig, schemaKey).map((nm, _).some)            
           } else if (nm == "campaign_attribution") {
             CampaignAttributionEnrichment.parse(enrichmentConfig, schemaKey).map((nm, _).some)            
-          } else {
+          } else if (nm == "currency_conversion_config") {
+            CurrencyConversionEnrichmentConfig.parse(enrichmentConfig, schemaKey).map((nm, _).some)
+          }else {
             None.success // Enrichment is not recognized yet
           }
         })
@@ -184,6 +192,15 @@ case class EnrichmentRegistry(private val configs: EnrichmentMap) {
    */
   def getCampaignAttributionEnrichment: Option[CampaignAttributionEnrichment] = 
     getEnrichment[CampaignAttributionEnrichment]("campaign_attribution")
+
+  /**
+   * Returns an Option boxing the CurrencyConversionEnrichment
+   * config value if present, or None if not
+   *
+   * @return Option boxing the CurrencyConversionEnrichment instance
+   */
+  def getCurrencyConversionEnrichment: Option[CurrencyConversionEnrichment] = 
+    getEnrichment[CurrencyConversionEnrichment]("currency_conversion")
 
   /**
    * Returns an Option boxing an Enrichment

src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CurrencyConversionEnrichment.scala

@@ -0,0 +1,155 @@
+/*
+ * Copyright (c) 2012-2015 Snowplow Analytics Ltd. All rights reserved.
+ *
+ * This program is licensed to you under the Apache License Version 2.0,
+ * and you may not use this file except in compliance with the Apache License Version 2.0.
+ * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the Apache License Version 2.0 is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
+*/
+package com.snowplowanalytics
+package snowplow
+package enrich
+package common
+package enrichments
+package registry
+
+// This project
+import utils.MapTransformer._
+
+// Java
+import java.lang.{Integer => JInteger}
+import java.math.{BigDecimal => JBigDecimal}
+import java.lang.{Byte => JByte}
+import java.net.URI
+import java.net._
+
+// Maven Artifact
+import org.apache.maven.artifact.versioning.DefaultArtifactVersion
+
+// Scalaz
+import scalaz._
+import Scalaz._
+
+// json4s
+import org.json4s.JValue
+
+// Iglu
+import iglu.client.SchemaKey
+import iglu.client.validation.ProcessingMessageMethods._
+
+// Joda-Money
+import org.joda.money.{Money}
+import org.joda.time.{DateTime, DateTimeZone}
+
+// Scala-Forex
+import com.snowplowanalytics.forex._
+import com.snowplowanalytics.forex.oerclient.{OerClientConfig, DeveloperAccount, AccountType, OerResponseError}
+import com.snowplowanalytics.forex.oerclient.OerResponseError._
+import com.snowplowanalytics.forex.{Forex, ForexConfig}
+import com.snowplowanalytics.forex.ForexLookupWhen._
+import com.snowplowanalytics.forex.oerclient._
+
+// This project
+import common.utils.ConversionUtils
+import utils.ScalazJson4sUtils
+
+/**
+ * Companion object. Lets us create an CurrencyConversionEnrichment
+ * instance from a JValue.
+ */
+object CurrencyConversionEnrichmentConfig extends ParseableEnrichment {
+
+  val supportedSchemaKey = SchemaKey("com.snowplowanalytics.snowplow", "currency_conversion_config", "jsonschema", "1-0-0")
+
+  //Creates an CurrencyConversionEnrichment instance from a JValue
+  def parse(config: JValue, schemaKey: SchemaKey): ValidatedNelMessage[CurrencyConversionEnrichment] = {
+    isParseable(config, schemaKey).flatMap( conf => {
+      (for {
+        apiKey        <- ScalazJson4sUtils.extract[String](config, "parameters", "apiKey")
+        baseCurrency  <- ScalazJson4sUtils.extract[String](config, "parameters", "baseCurrency")
+        rateAt        <- ScalazJson4sUtils.extract[String](config, "parameters", "rateAt")
+        enrich        =  CurrencyConversionEnrichment(apiKey, baseCurrency, rateAt)
+      } yield enrich).toValidationNel
+    })
+  }
+}
+
+// Object and a case object with the same name
+case class CurrencyConversionEnrichment(
+  apiKey: String,
+  baseCurrency: String, 
+  rateAt: String) extends Enrichment {
+
+  val version = new DefaultArtifactVersion("0.1.0")
+
+  // To provide Validation for org.joda.money.Money variable
+  def eitherToValidation(input:Either[OerResponseError, Money]): Validation[String, Option[String]]= {
+    input match {
+      case Right(l) => (l.getAmount().toPlainString()).some.success
+      case Left(l) => (s"Open Exchange Rates error, message: ${l.errorMessage}").failure
+    }
+  }
+
+  def getEitherValidation(fx: Forex, trCurrency: Option[String], trTotal: Option[Double], tstamp: DateTime): Validation[String, Option[String]] = {
+      trCurrency match {
+        case Some(trCurr) =>{ 
+          trTotal match{
+            case Some(trC) => {
+              eitherToValidation(fx.convert(trC, trCurr).to(baseCurrency).at(tstamp))
+            } 
+            case None => None.success                                            
+          }
+        }
+        case None => None.success
+      } 
+  }
+  /**
+  * Convert's currency for a given
+  * set of currency, using
+  * Scala-Forex.
+  *
+  * @param trCurrency The desired
+  *        currency for a given 
+  *        amount
+  * @param trAmounts Contains
+  *        total amount, tax, shipping
+  *        amount's
+  * @param tiCurrency Trasaction Item
+  *        Currency
+   * @param tiPrice Trasaction Item
+  *         Price
+  * @return the converted currency
+  *         in TransacrionAmounts
+  *         format and Ttansaction
+  *         Item Price 
+  */
+  def convertCurrencies(trCurrency: Option[String], trTotal: Option[Double], trAmountsTax: Option[Double], trAmountsShipping: Option[Double], tiCurrency: Option[String], tiPrice: Option[Double], collectorTstamp: Option[DateTime]): ValidationNel[String, (Option[String], Option[String], Option[String], Option[String])] = {
+    val check = Double.NaN
+    try{
+        val fx = Forex(ForexConfig( nowishCacheSize = 0, nowishSecs = 0, eodCacheSize= 0), OerClientConfig(apiKey, DeveloperAccount))  
+                collectorTstamp match {
+                  case Some(tstamp) =>{
+                    val newCurrencyTr  = getEitherValidation(fx, trCurrency, trTotal, tstamp)  
+                    val newCurrencyTi = getEitherValidation(fx, tiCurrency, tiPrice, tstamp)
+                    val newTrAmountsTax = getEitherValidation(fx, trCurrency, trAmountsTax, tstamp) 
+                    val newTrAmountsShipping = getEitherValidation(fx, trCurrency, trAmountsShipping, tstamp) 
+                    (newCurrencyTr.toValidationNel  |@| newTrAmountsTax.toValidationNel  |@| newTrAmountsShipping.toValidationNel |@| newCurrencyTi.toValidationNel) {
+                      (_, _, _, _)
+                    }       
+
+                  }
+                  case None => "DateTime Missing".failNel 
+                }
+    } catch {
+      case e : NoSuchElementException =>"Provided Currency not supported : %s".format(e).failNel
+      case f : UnknownHostException => "Could not extract Convert Currencies from OER Service :%s".format(f).failNel
+      case g => "Exception Converting Currency :%s".format(g).failNel
+    }
+
+  }
+}
+

src/main/scala/com.snowplowanalytics.snowplow.enrich/common/outputs/EnrichedEvent.scala

@@ -133,9 +133,14 @@ class EnrichedEvent {
   @BeanProperty var tr_total: String = _
   @BeanProperty var tr_tax: String = _
   @BeanProperty var tr_shipping: String = _
+  @BeanProperty var tr_currency: String = _
   @BeanProperty var tr_city: String = _
   @BeanProperty var tr_state: String = _
   @BeanProperty var tr_country: String = _
+  @BeanProperty var tr_total_base: String = _
+  @BeanProperty var tr_tax_base: String = _
+  @BeanProperty var tr_shipping_base: String = _
+  @BeanProperty var currency_base: String = _
 
   // Ecommerce transaction item (from querystring)
   @BeanProperty var ti_orderid: String = _
@@ -144,6 +149,8 @@ class EnrichedEvent {
   @BeanProperty var ti_category: String = _
   @BeanProperty var ti_price: String = _
   @BeanProperty var ti_quantity: String = _
+  @BeanProperty var ti_currency: String = _
+  @BeanProperty var ti_price_base: String = _
 
   // Page Pings
   @BeanProperty var pp_xoffset_min: JInteger = _

src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/ConversionUtils.scala

@@ -338,6 +338,31 @@ object ConversionUtils {
         "Field [%s]: cannot convert [%s] to Double-like String".format(field, str).fail
     }
 
+  /**
+   * Convert a String to a String containing a
+   * Redshift-compatible Double.
+   *
+   * @param str The String which we hope contains
+   *        a Double
+   * @param field The name of the field we are
+   *        validating. To use in our error message
+   * @return a Scalaz Validation, being either
+   *         a Failure String or a Success Double
+   */
+  def stringToMaybeDouble(field: String,str: String):Validation[String, Option[Double]] = {
+    try {
+      if (Option(str).isEmpty || str == "null") { // "null" String check is LEGACY to handle a bug in the JavaScript tracker
+        None.success
+      } else {
+        val jbigdec = new JBigDecimal(str)
+        jbigdec.doubleValue().some.success
+      }
+    } catch {
+     case nfe: NumberFormatException =>
+        "Field [%s]: cannot convert [%s] to Double-like String".format(field, str).fail
+    }  
+  }
+
   /**
    * Extract a Java Byte representing
    * 1 or 0 only from a String, or error.
@@ -425,3 +450,4 @@ object ConversionUtils {
     else
       "Cannot convert byte [%s] to boolean, only 1 or 0.".format(b).fail   
 }
+