Skip to content

Commit

Permalink
Added ability to disable user-agent-utils enrichment (closes #792)
Browse files Browse the repository at this point in the history
Added ability to disable user-agent-utils enrichment (closes #792)

Added ability to disable user-agent-utils enrichment (closes #792)

Added ability to disable user-agent-utils enrichment (closes #792)

Added ability to disable user-agent-utils enrichment (closes #792)

Added ability to disable user-agent-utils enrichment (closes #792)

Added ability to disable user-agent-utils enrichment (closes #792)

Added ability to disable user-agent-utils enrichment (closes #792)

Added ability to disable user-agent-utils enrichment (closes #792)
  • Loading branch information
AALEKH authored and fblundun committed Mar 2, 2015
1 parent f5878ed commit 9005b62
Show file tree
Hide file tree
Showing 7 changed files with 224 additions and 108 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -20,17 +20,13 @@ import java.lang.{Integer => JInteger}
import scalaz._
import Scalaz._

// UserAgentUtils
import eu.bitwalker.useragentutils._

/**
* Contains enrichments related to the
* client - where the client is the
* software which is using the SnowPlow
* tracker.
*
* Enrichments relate to the useragent,
* browser resolution, etc.
* Enrichments relate to browser resolution
*/
object ClientEnrichments {

Expand All @@ -43,30 +39,6 @@ object ClientEnrichments {
*/
private val ResRegex = """(\d+)x(\d+)""".r

/**
* Case class to wrap everything we
* can extract from the useragent
* using UserAgentUtils.
*
* TODO: update this definition when
* we swap out UserAgentUtils for
* ua-parser
*/
case class ClientAttributes(
// Browser
browserName: String,
browserFamily: String,
browserVersion: Option[String],
browserType: String,
browserRenderEngine: String,
// OS the browser is running on
osName: String,
osFamily: String,
osManufacturer: String,
// Hardware the OS is running on
deviceType: String,
deviceIsMobile: Boolean)

/**
* Extracts view dimensions (e.g. screen resolution,
* browser/app viewport) stored as per the Tracker
Expand All @@ -93,43 +65,4 @@ object ClientEnrichments {
case _ => "Field [%s]: [%s] does not contain valid view dimensions".format(field, res).fail
}

/**
* Extracts the client attributes
* from a useragent string, using
* UserAgentUtils.
*
* TODO: rewrite this when we swap
* out UserAgentUtils for ua-parser
*
* @param useragent The useragent
* String to extract from.
* Should be encoded (i.e.
* not previously decoded).
* @return the ClientAttributes or
* the message of the
* exception, boxed in a
* Scalaz Validation
*/
def extractClientAttributes(useragent: String): Validation[String, ClientAttributes] =

try {
val ua = UserAgent.parseUserAgentString(useragent)
val b = ua.getBrowser
val v = Option(ua.getBrowserVersion)
val os = ua.getOperatingSystem

ClientAttributes(
browserName = b.getName,
browserFamily = b.getGroup.getName,
browserVersion = v map { _.getVersion },
browserType = b.getBrowserType.getName,
browserRenderEngine = b.getRenderingEngine.toString,
osName = os.getName,
osFamily = os.getGroup.getName,
osManufacturer = os.getManufacturer.getName,
deviceType = os.getDeviceType.getName,
deviceIsMobile = os.isMobileDevice).success
} catch {
case e => "Exception parsing useragent [%s]: [%s]".format(useragent, e.getMessage).fail
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -196,27 +196,6 @@ object EnrichmentManager {

val secondPassTransform = event.transform(sourceMap, secondPassTransformMap)

// Parse the useragent
val client = Option(event.useragent) match {
case Some(ua) =>
val ca = CE.extractClientAttributes(ua)
ca.flatMap(c => {
event.br_name = c.browserName
event.br_family = c.browserFamily
c.browserVersion.map(bv => event.br_version = bv)
event.br_type = c.browserType
event.br_renderengine = c.browserRenderEngine
event.os_name = c.osName
event.os_family = c.osFamily
event.os_manufacturer = c.osManufacturer
event.dvce_type = c.deviceType
event.dvce_ismobile = CU.booleanToJByte(c.deviceIsMobile)
c.success
})
ca
case None => unitSuccess // No fields updated
}

// Potentially update the page_url and set the page URL components
val pageUri = WPE.extractPageUri(raw.context.refererUri, Option(event.page_url))
for (uri <- pageUri; u <- uri) {
Expand Down Expand Up @@ -265,11 +244,39 @@ object EnrichmentManager {
}
}

// Finally anonymize the IP address
// Anonymize the IP address
Option(event.user_ipaddress).map(ip => event.user_ipaddress = registry.getAnonIpEnrichment match {
case Some(anon) => anon.anonymizeIp(ip)
case None => ip
})

// Parse the useragent using user-agent-utils
val client = {
registry.getUserAgentUtilsEnrichment match {
case Some(uap) => {
Option(event.useragent) match {
case Some(ua) =>
val ca = uap.extractClientAttributes(ua)
ca.flatMap(c => {
event.br_name = c.browserName
event.br_family = c.browserFamily
c.browserVersion.map(bv => event.br_version = bv)
event.br_type = c.browserType
event.br_renderengine = c.browserRenderEngine
event.os_name = c.osName
event.os_family = c.osFamily
event.os_manufacturer = c.osManufacturer
event.dvce_type = c.deviceType
event.dvce_ismobile = CU.booleanToJByte(c.deviceIsMobile)
c.success
})
ca
case None => unitSuccess // No fields updated
}
}
case None => unitSuccess
}
}

// Potentially set the referrer details and URL components
val refererUri = CU.stringToUri(event.page_referrer)
Expand Down Expand Up @@ -343,3 +350,4 @@ object EnrichmentManager {
}
}
}

Original file line number Diff line number Diff line change
Expand Up @@ -42,10 +42,16 @@ import registry.{
AnonIpEnrichment,
IpLookupsEnrichment,
RefererParserEnrichment,
CampaignAttributionEnrichment
CampaignAttributionEnrichment,
UserAgentUtilsEnrichment
}

// UserAgentUtilsEnrichmentConfig
import registry.UserAgentUtilsEnrichmentConfig

import utils.ScalazJson4sUtils


/**
* Companion which holds a constructor
* for the EnrichmentRegistry.
Expand Down Expand Up @@ -125,6 +131,8 @@ object EnrichmentRegistry {
RefererParserEnrichment.parse(enrichmentConfig, schemaKey).map((nm, _).some)
} else if (nm == "campaign_attribution") {
CampaignAttributionEnrichment.parse(enrichmentConfig, schemaKey).map((nm, _).some)
} else if (nm == "user_agent_utils_config") {
UserAgentUtilsEnrichmentConfig.parse(enrichmentConfig, schemaKey).map((nm, _).some)
} else {
None.success // Enrichment is not recognized yet
}
Expand Down Expand Up @@ -184,6 +192,15 @@ case class EnrichmentRegistry(private val configs: EnrichmentMap) {
*/
def getCampaignAttributionEnrichment: Option[CampaignAttributionEnrichment] =
getEnrichment[CampaignAttributionEnrichment]("campaign_attribution")

/**
* Returns an Option boxing the UserAgentUtilsEnrichment
* config value if present, or None if not
*
* @return Option boxing the UserAgentUtilsEnrichment instance
*/
def getUserAgentUtilsEnrichment: Option[UserAgentUtilsEnrichment.type] =
getEnrichment[UserAgentUtilsEnrichment.type]("user_agent_utils")

/**
* Returns an Option boxing an Enrichment
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
/*Copyright <Notice>
*
* This program is licensed to you under the Apache License Version 2.0,
* and you may not use this file except in compliance with the Apache License Version 2.0.
* You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the Apache License Version 2.0 is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
*/
package com.snowplowanalytics
package snowplow
package enrich
package common
package enrichments
package registry

// Maven Artifact
import org.apache.maven.artifact.versioning.DefaultArtifactVersion

// Scalaz
import scalaz._
import Scalaz._

// UserAgentUtils
import eu.bitwalker.useragentutils._

// json4s
import org.json4s.JValue

// Iglu
import iglu.client.SchemaKey
import iglu.client.validation.ProcessingMessageMethods._

// This project
import utils.ScalazJson4sUtils

object UserAgentUtilsEnrichmentConfig extends ParseableEnrichment {

val supportedSchemaKey = SchemaKey("com.snowplowanalytics.snowplow", "user_agent_utils_config", "jsonschema", "1-0-0")

//Creates an UserAgentUtilsEnrichment instance from a JValue
def parse(config: JValue, schemaKey: SchemaKey): ValidatedNelMessage[UserAgentUtilsEnrichment.type] =
isParseable(config, schemaKey).map(_ => UserAgentUtilsEnrichment)
}

/**
* Case class to wrap everything we
* can extract from the useragent
* using UserAgentUtils.
*
* Not to be declared inside a class Object
* http://stackoverflow.com/questions/17270003/why-are-classes-inside-scala-package-objects-dispreferred
*/
case class ClientAttributes(
// Browser
browserName: String,
browserFamily: String,
browserVersion: Option[String],
browserType: String,
browserRenderEngine: String,
// OS the browser is running on
osName: String,
osFamily: String,
osManufacturer: String,
// Hardware the OS is running on
deviceType: String,
deviceIsMobile: Boolean)

// Object and a case object with the same name

case object UserAgentUtilsEnrichment extends Enrichment {

val version = new DefaultArtifactVersion("0.1.0")

/**
* Extracts the client attributes
* from a useragent string, using
* UserAgentUtils.
*
* TODO: rewrite this when we swap
* out UserAgentUtils for ua-parser
*
* @param useragent The useragent
* String to extract from.
* Should be encoded (i.e.
* not previously decoded).
* @return the ClientAttributes or
* the message of the
* exception, boxed in a
* Scalaz Validation
*/
def extractClientAttributes(useragent: String): Validation[String, ClientAttributes] = {

try {
val ua = UserAgent.parseUserAgentString(useragent)
val b = ua.getBrowser
val v = Option(ua.getBrowserVersion)
val os = ua.getOperatingSystem
ClientAttributes(
browserName = b.getName,
browserFamily = b.getGroup.getName,
browserVersion = v map { _.getVersion },
browserType = b.getBrowserType.getName,
browserRenderEngine = b.getRenderingEngine.toString,
osName = os.getName,
osFamily = os.getGroup.getName,
osManufacturer = os.getManufacturer.getName,
deviceType = os.getDeviceType.getName,
deviceIsMobile = os.isMobileDevice).success
} catch {
case e => "Exception parsing useragent [%s]: [%s]".format(useragent, e.getMessage).fail
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -49,19 +49,3 @@ class ExtractViewDimensionsSpec extends Specification with DataTables {
(_, input, expected) => ClientEnrichments.extractViewDimensions(FieldName, input) must_== expected
}
}

class UserAgentParseSpec extends org.specs2.mutable.Specification with ValidationMatchers with DataTables {
import ClientEnrichments._

"useragent parser" should {
"parse useragent" in {
"Input UserAgent" | "Parsed UserAgent" |
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.152 Safari/537.36" !! ClientAttributes(browserName = "Chrome 33", browserFamily="Chrome", browserVersion = Some("33.0.1750.152"), browserType = "Browser", browserRenderEngine = "WEBKIT", osName = "Mac OS X", osFamily = "Mac OS X", osManufacturer = "Apple Inc.", deviceType = "Computer", deviceIsMobile = false) |
"Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0" !! ClientAttributes(browserName = "Internet Explorer 11", browserFamily="Internet Explorer", browserVersion = Some("11.0"), browserType = "Browser", browserRenderEngine = "TRIDENT", osName = "Windows 7", osFamily = "Windows", osManufacturer = "Microsoft Corporation", deviceType = "Computer", deviceIsMobile = false) |> {
(input, expected) => {
ClientEnrichments.extractClientAttributes(input) must beSuccessful.like { case a => a must_== expected }
}
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -143,4 +143,21 @@ class EnrichmentConfigsSpec extends Specification with ValidationMatchers {
}
}

"Parsing a valid user_agent_utils_config enrichment JSON" should {
"successfully construct a UserAgentUtilsEnrichment case object" in {

val userAgentUtilsEnrichmentJson = parse("""{
"enabled": true,
"parameters": {
}
}""")

val schemaKey = SchemaKey("com.snowplowanalytics.snowplow", "user_agent_utils_config", "jsonschema", "1-0-0")

val result = UserAgentUtilsEnrichmentConfig.parse(userAgentUtilsEnrichmentJson, schemaKey)
result must beSuccessful(UserAgentUtilsEnrichment)

}
}

}

0 comments on commit 9005b62

Please sign in to comment.