From a4d2e29a55c205bad5dbf3d91968fa077f3f6d46 Mon Sep 17 00:00:00 2001 From: Konstantinos Servis Date: Mon, 5 Mar 2018 17:45:20 +0200 Subject: [PATCH] Spark Enrich: apply automated code formatting (closes #3655) --- 3-enrich/spark-enrich/build.sbt | 11 +- .../EnrichJob.scala | 70 ++++++----- .../EnrichJobConfig.scala | 53 +++++--- .../SparkJob.scala | 3 +- .../singleton.scala | 20 +-- .../BeforeAfterAll.scala | 2 +- .../EnrichJobSpec.scala | 114 +++++++++++------- .../MasterCfSpec.scala | 44 +++---- .../MasterCljTomcatSpec.scala | 10 +- .../PiiFilteringSpec.scala | 8 +- .../SparkSpec.scala | 10 +- .../bad/CorruptedCfLinesSpec.scala | 3 +- .../bad/CorruptedThriftLinesSpec.scala | 15 ++- .../bad/NullNumericFieldsSpec.scala | 3 +- .../bad/UnsupportedPayloadCfLinesSpec.scala | 3 +- .../good/ApiRequestEnrichmentCfLineSpec.scala | 23 ++-- .../good/Aug2013CfLineSpec.scala | 2 +- .../good/CampaignAttributionCfLineSpec.scala | 2 +- .../good/CljTomcatCallrailEventSpec.scala | 2 +- .../good/CljTomcatMailchimpEventSpec.scala | 2 +- .../good/CljTomcatMailgunEventSpec.scala | 21 ++-- .../good/CljTomcatMandrillEventSpec.scala | 2 +- .../good/CljTomcatOlarkEventSpec.scala | 8 +- .../good/CljTomcatPagerdutyEventSpec.scala | 2 +- .../good/CljTomcatPingdomEventSpec.scala | 2 +- .../good/CljTomcatSendgridEventSpec.scala | 2 +- .../good/CljTomcatStatusGatorEventSpec.scala | 5 +- .../good/CljTomcatTp1SingleEventSpec.scala | 2 +- .../good/CljTomcatTp1TnuidSpec.scala | 2 +- .../good/CljTomcatTp2MegaEventsSpec.scala | 6 +- .../good/CljTomcatTp2MultiEventsSpec.scala | 25 ++-- .../good/CljTomcatUnbounceEventSpec.scala | 7 +- .../good/CollectorPayload1LzoSpec.scala | 36 +++--- .../good/Core2015RefreshSpec.scala | 2 +- ...urrencyConversionTransactionItemSpec.scala | 26 ++-- .../CurrencyConversionTransactionSpec.scala | 38 +++--- .../good/DerivedTstampSpec.scala | 2 +- .../good/ForwardCompatibleSchemaverSpec.scala | 25 ++-- .../good/LzoGoogleAnalyticsEventSpec.scala | 33 ++--- .../good/NdjsonUrbanAirshipSpec.scala | 59 ++++----- .../good/PagePingCfLineSpec.scala | 2 +- .../good/RefererParserCfLineSpec.scala | 2 +- .../good/SnowplowRawEventLzoSpec.scala | 23 ++-- .../good/SqlQueryEnrichmentCfLineSpec.scala | 22 ++-- .../good/StructEventCfLineSpec.scala | 8 +- .../good/TransactionCfLineSpec.scala | 16 +-- .../good/TransactionItemCfLineSpec.scala | 12 +- .../good/UnstructEventBase64CfLineSpec.scala | 8 +- .../good/UnstructEventCfLineSpec.scala | 2 +- .../good/WebDistributionLoader23Spec.scala | 8 +- .../good/WebDistributionLoaderSpec.scala | 8 +- 51 files changed, 482 insertions(+), 334 deletions(-) diff --git a/3-enrich/spark-enrich/build.sbt b/3-enrich/spark-enrich/build.sbt index b30b411e93..e548abb29c 100644 --- a/3-enrich/spark-enrich/build.sbt +++ b/3-enrich/spark-enrich/build.sbt @@ -12,10 +12,11 @@ * See the Apache License Version 2.0 for the specific language governing permissions and * limitations there under. */ -lazy val root = project.in(file(".")) +lazy val root = project + .in(file(".")) .settings( - name := "snowplow-spark-enrich", - version := "1.12.0", + name := "snowplow-spark-enrich", + version := "1.12.0", description := "The Snowplow Spark Enrichment process" ) .settings(BuildSettings.formatting) @@ -41,4 +42,6 @@ lazy val root = project.in(file(".")) ) ) -shellPrompt := { _ => "spark-enrich> " } +shellPrompt := { _ => + "spark-enrich> " +} diff --git a/3-enrich/spark-enrich/src/main/scala/com.snowplowanalytics.snowplow.enrich.spark/EnrichJob.scala b/3-enrich/spark-enrich/src/main/scala/com.snowplowanalytics.snowplow.enrich.spark/EnrichJob.scala index a063ffb0c7..ecb7a9856a 100644 --- a/3-enrich/spark-enrich/src/main/scala/com.snowplowanalytics.snowplow.enrich.spark/EnrichJob.scala +++ b/3-enrich/spark-enrich/src/main/scala/com.snowplowanalytics.snowplow.enrich.spark/EnrichJob.scala @@ -58,8 +58,10 @@ object EnrichJob extends SparkJob { classOf[Array[EnrichedEvent]], classOf[com.snowplowanalytics.snowplow.CollectorPayload.thrift.model1.CollectorPayload], classOf[Array[com.snowplowanalytics.snowplow.CollectorPayload.thrift.model1.CollectorPayload]], - Class.forName("com.snowplowanalytics.snowplow.CollectorPayload.thrift.model1.CollectorPayload$_Fields"), - Class.forName("[Lcom.snowplowanalytics.snowplow.CollectorPayload.thrift.model1.CollectorPayload$_Fields;"), + Class.forName( + "com.snowplowanalytics.snowplow.CollectorPayload.thrift.model1.CollectorPayload$_Fields"), + Class.forName( + "[Lcom.snowplowanalytics.snowplow.CollectorPayload.thrift.model1.CollectorPayload$_Fields;"), classOf[com.snowplowanalytics.snowplow.collectors.thrift.SnowplowRawEvent], classOf[Array[com.snowplowanalytics.snowplow.collectors.thrift.SnowplowRawEvent]], Class.forName("com.snowplowanalytics.snowplow.collectors.thrift.SnowplowRawEvent$_Fields"), @@ -74,11 +76,12 @@ object EnrichJob extends SparkJob { classOf[org.apache.spark.internal.io.FileCommitProtocol$TaskCommitMessage], classOf[org.apache.spark.sql.execution.datasources.FileFormatWriter$WriteTaskResult] ) - override def sparkConfig(): SparkConf = new SparkConf() - .setAppName(getClass().getSimpleName()) - .setIfMissing("spark.master", "local[*]") - .set("spark.serializer", classOf[KryoSerializer].getName()) - .registerKryoClasses(classesToRegister) + override def sparkConfig(): SparkConf = + new SparkConf() + .setAppName(getClass().getSimpleName()) + .setIfMissing("spark.master", "local[*]") + .set("spark.serializer", classOf[KryoSerializer].getName()) + .registerKryoClasses(classesToRegister) override def run(spark: SparkSession, args: Array[String]): Unit = { val job = EnrichJob(spark, args) @@ -114,8 +117,11 @@ object EnrichJob extends SparkJob { def enrich(line: Any, config: ParsedEnrichJobConfig): (Any, List[ValidatedEnrichedEvent]) = { import singleton._ val registry = RegistrySingleton.get(config.igluConfig, config.enrichments, config.local) - val loader = LoaderSingleton.get(config.inFormat).asInstanceOf[Loader[Any]] - val event = EtlPipeline.processEvents(registry, etlVersion, config.etlTstamp, + val loader = LoaderSingleton.get(config.inFormat).asInstanceOf[Loader[Any]] + val event = EtlPipeline.processEvents( + registry, + etlVersion, + config.etlTstamp, loader.toCollectorPayload(line))(ResolverSingleton.get(config.igluConfig)) (line, event) } @@ -156,10 +162,12 @@ class EnrichJob(@transient val spark: SparkSession, args: Array[String]) extends hadoopConfig.set("io.compression.codec.lzo.class", classOf[LzoCodec].getName()) // Job configuration - private val enrichConfig = EnrichJobConfig.loadConfigFrom(args).fold( - e => throw FatalEtlError(e.map(_.toString)), - identity - ) + private val enrichConfig = EnrichJobConfig + .loadConfigFrom(args) + .fold( + e => throw FatalEtlError(e.map(_.toString)), + identity + ) /** * Run the enrich job by: @@ -189,14 +197,16 @@ class EnrichJob(@transient val spark: SparkSession, args: Array[String]) extends // Handling of malformed rows val bad = common .map { case (line, enriched) => (line, projectBads(enriched)) } - .flatMap { case (line, errors) => - val originalLine = line match { - case bytes: Array[Byte] => new String(Base64.encodeBase64(bytes), "UTF-8") - case other => other.toString - } - errors.map(e => Row(BadRow(originalLine, e).toCompactJson)) + .flatMap { + case (line, errors) => + val originalLine = line match { + case bytes: Array[Byte] => new String(Base64.encodeBase64(bytes), "UTF-8") + case other => other.toString + } + errors.map(e => Row(BadRow(originalLine, e).toCompactJson)) } - spark.createDataFrame(bad, StructType(StructField("_", StringType, true) :: Nil)) + spark + .createDataFrame(bad, StructType(StructField("_", StringType, true) :: Nil)) .write .mode(SaveMode.Overwrite) .text(enrichConfig.badFolder) @@ -204,10 +214,15 @@ class EnrichJob(@transient val spark: SparkSession, args: Array[String]) extends // Handling of properly-formed rows val good = common .flatMap { case (_, enriched) => projectGoods(enriched) } - spark.createDataset(good)(Encoders.bean(classOf[EnrichedEvent])) + spark + .createDataset(good)(Encoders.bean(classOf[EnrichedEvent])) .toDF() // hack to preserve the order of the fields in the csv, otherwise it's alphabetical - .select(classOf[EnrichedEvent].getDeclaredFields().filterNot(_.getName.equals("pii")).map(f => col(f.getName())): _*) + .select( + classOf[EnrichedEvent] + .getDeclaredFields() + .filterNot(_.getName.equals("pii")) + .map(f => col(f.getName())): _*) .write .option("sep", "\t") .option("escape", "") @@ -221,17 +236,16 @@ class EnrichJob(@transient val spark: SparkSession, args: Array[String]) extends * @param inFormat Collector format in which the data is coming in * @return A RDD containing strings or byte arrays */ - private def getInputRDD(inFormat: String, path: String): RDD[_] = { + private def getInputRDD(inFormat: String, path: String): RDD[_] = inFormat match { case "thrift" => MultiInputFormat.setClassConf(classOf[Array[Byte]], hadoopConfig) sc.newAPIHadoopFile[ - LongWritable, - BinaryWritable[Array[Byte]], - MultiInputFormat[Array[Byte]] - ](path) + LongWritable, + BinaryWritable[Array[Byte]], + MultiInputFormat[Array[Byte]] + ](path) .map(_._2.get()) case _ => sc.textFile(path) } - } } diff --git a/3-enrich/spark-enrich/src/main/scala/com.snowplowanalytics.snowplow.enrich.spark/EnrichJobConfig.scala b/3-enrich/spark-enrich/src/main/scala/com.snowplowanalytics.snowplow.enrich.spark/EnrichJobConfig.scala index c45f586d32..07222e20d0 100644 --- a/3-enrich/spark-enrich/src/main/scala/com.snowplowanalytics.snowplow.enrich.spark/EnrichJobConfig.scala +++ b/3-enrich/spark-enrich/src/main/scala/com.snowplowanalytics.snowplow.enrich.spark/EnrichJobConfig.scala @@ -82,28 +82,43 @@ case class ParsedEnrichJobConfig( object EnrichJobConfig { private val parser = new scopt.OptionParser[RawEnrichJobConfig]("EnrichJob") { head("EnrichJob") - opt[String]("input-folder").required().valueName("") + opt[String]("input-folder") + .required() + .valueName("") .action((f, c) => c.copy(inFolder = f)) .text("Folder where the input events are located") - opt[String]("input-format").required().valueName("") + opt[String]("input-format") + .required() + .valueName("") .action((f, c) => c.copy(inFormat = f)) .text("The format in which the collector is saving data") - opt[String]("output-folder").required().valueName("") + opt[String]("output-folder") + .required() + .valueName("") .action((f, c) => c.copy(outFolder = f)) .text("Output folder where the enriched events will be stored") - opt[String]("bad-folder").required().valueName("") + opt[String]("bad-folder") + .required() + .valueName("") .action((f, c) => c.copy(badFolder = f)) .text("Output folder where the malformed events will be stored") - opt[String]("enrichments").required().valueName("") + opt[String]("enrichments") + .required() + .valueName("") .action((e, c) => c.copy(enrichments = e)) .text("Directory where the JSONs describing the enrichments are stored") - opt[String]("iglu-config").required().valueName("") + opt[String]("iglu-config") + .required() + .valueName("") .action((i, c) => c.copy(igluConfig = i)) .text("Iglu resolver configuration") - opt[Long]("etl-timestamp").required().valueName("") + opt[Long]("etl-timestamp") + .required() + .valueName("") .action((t, c) => c.copy(etlTstamp = t)) .text("Timestamp at which the job was launched, in milliseconds") - opt[Unit]("local").hidden() + opt[Unit]("local") + .hidden() .action((_, c) => c.copy(local = true)) .text("Whether to build a local enrichment registry") help("help").text("Prints this usage text") @@ -118,11 +133,20 @@ object EnrichJobConfig { val resolver = ResolverSingleton.getIgluResolver(c.igluConfig) val registry = resolver .flatMap(RegistrySingleton.getEnrichmentRegistry(c.enrichments, c.local)(_)) - val loader = Loader.getLoader(c.inFormat) + val loader = Loader + .getLoader(c.inFormat) .fold(_.toProcessingMessage.failureNel, _.successNel) (resolver |@| registry |@| loader) { (_, reg, _) => - ParsedEnrichJobConfig(c.inFolder, c.inFormat, c.outFolder, c.badFolder, - c.enrichments, c.igluConfig, c.local, new DateTime(c.etlTstamp), filesToCache(reg)) + ParsedEnrichJobConfig( + c.inFolder, + c.inFormat, + c.outFolder, + c.badFolder, + c.enrichments, + c.igluConfig, + c.local, + new DateTime(c.etlTstamp), + filesToCache(reg)) } } @@ -133,12 +157,11 @@ object EnrichJobConfig { */ def loadConfigFrom( args: Array[String] - ): ValidatedNelMessage[ParsedEnrichJobConfig] = { + ): ValidatedNelMessage[ParsedEnrichJobConfig] = parser.parse(args, RawEnrichJobConfig()).map(transform) match { case Some(c) => c - case _ => "Parsing of the configuration failed".toProcessingMessage.failureNel + case _ => "Parsing of the configuration failed".toProcessingMessage.failureNel } - } /** * Build the list of enrichment files to cache. @@ -148,6 +171,6 @@ object EnrichJobConfig { private def filesToCache(registry: EnrichmentRegistry): List[(URI, String)] = registry.getIpLookupsEnrichment match { case Some(ipLookups) => ipLookups.dbsToCache - case None => Nil + case None => Nil } } diff --git a/3-enrich/spark-enrich/src/main/scala/com.snowplowanalytics.snowplow.enrich.spark/SparkJob.scala b/3-enrich/spark-enrich/src/main/scala/com.snowplowanalytics.snowplow.enrich.spark/SparkJob.scala index dec7eb518d..af179d6ee4 100644 --- a/3-enrich/spark-enrich/src/main/scala/com.snowplowanalytics.snowplow.enrich.spark/SparkJob.scala +++ b/3-enrich/spark-enrich/src/main/scala/com.snowplowanalytics.snowplow.enrich.spark/SparkJob.scala @@ -26,7 +26,8 @@ trait SparkJob { def main(args: Array[String]): Unit = { val config = sparkConfig() - val spark = SparkSession.builder() + val spark = SparkSession + .builder() .config(config) .getOrCreate() run(spark, args) diff --git a/3-enrich/spark-enrich/src/main/scala/com.snowplowanalytics.snowplow.enrich.spark/singleton.scala b/3-enrich/spark-enrich/src/main/scala/com.snowplowanalytics.snowplow.enrich.spark/singleton.scala index ea6bbed10f..03f2f1ad03 100644 --- a/3-enrich/spark-enrich/src/main/scala/com.snowplowanalytics.snowplow.enrich.spark/singleton.scala +++ b/3-enrich/spark-enrich/src/main/scala/com.snowplowanalytics.snowplow.enrich.spark/singleton.scala @@ -33,9 +33,11 @@ import iglu.client.validation.ProcessingMessageMethods._ /** Singletons needed for unserializable classes. */ object singleton { + /** Singleton for Iglu's Resolver to maintain one Resolver per node. */ object ResolverSingleton { @volatile private var instance: Resolver = _ + /** * Retrieve or build an instance of Iglu's Resolver. * @param igluConfig JSON representing the Iglu configuration @@ -67,7 +69,8 @@ object singleton { /** Singleton for EnrichmentRegistry. */ object RegistrySingleton { @volatile private var instance: EnrichmentRegistry = _ - @volatile private var enrichments: String = _ + @volatile private var enrichments: String = _ + /** * Retrieve or build an instance of EnrichmentRegistry. * @param igluConfig JSON representing the Iglu configuration @@ -95,21 +98,21 @@ object singleton { * @param resolver (implicit) The Iglu resolver used for schema lookup and validation * @return An EnrichmentRegistry or one or more error messages boxed in a Scalaz ValidationNel */ - private[spark] def getEnrichmentRegistry(enrichments: String, local: Boolean - )(implicit resolver: Resolver): ValidatedNelMessage[EnrichmentRegistry] = { + private[spark] def getEnrichmentRegistry(enrichments: String, local: Boolean)( + implicit resolver: Resolver): ValidatedNelMessage[EnrichmentRegistry] = for { - node <- base64ToJsonNode(enrichments, "enrichments") - .toValidationNel: ValidatedNelMessage[JsonNode] + node <- base64ToJsonNode(enrichments, "enrichments").toValidationNel: ValidatedNelMessage[ + JsonNode] reg <- EnrichmentRegistry.parse(fromJsonNode(node), local) } yield reg - } } /** Singleton for Loader. */ object LoaderSingleton { import common.loaders.Loader @volatile private var instance: Loader[_] = _ - @volatile private var inFormat: String = _ + @volatile private var inFormat: String = _ + /** * Retrieve or build an instance of EnrichmentRegistry. * @param inFormat Collector format in which the data is coming in @@ -118,7 +121,8 @@ object singleton { if (instance == null || this.inFormat != inFormat) { synchronized { if (instance == null || this.inFormat != inFormat) { - instance = Loader.getLoader(inFormat) + instance = Loader + .getLoader(inFormat) .valueOr(e => throw new FatalEtlError(e.toString)) this.inFormat = inFormat } diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/BeforeAfterAll.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/BeforeAfterAll.scala index 97bba46f9a..139c7d4452 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/BeforeAfterAll.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/BeforeAfterAll.scala @@ -24,7 +24,7 @@ import org.specs2.specification.Step * TODO: To remove once specs2 has been updated. */ trait BeforeAfterAll extends SpecificationLike { - override def map(fragments: =>Fragments) = + override def map(fragments: => Fragments) = Step(beforeAll) ^ fragments ^ Step(afterAll) def beforeAll(): Unit diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/EnrichJobSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/EnrichJobSpec.scala index dae46ad7ac..0da2c2b2d0 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/EnrichJobSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/EnrichJobSpec.scala @@ -38,8 +38,8 @@ import org.specs2.execute.{AsResult, ResultExecution} import org.specs2.matcher.{Expectable, Matcher} import org.specs2.matcher.Matchers._ - object EnrichJobSpec { + /** Case class representing the input lines written in a file. */ case class Lines(l: String*) { val lines = l.toList @@ -59,18 +59,20 @@ object EnrichJobSpec { /** Case class representing the directories where the output of the job has been written. */ case class OutputDirs(output: File, badRows: File) { + /** Delete recursively the output and bad rows directories. */ def deleteAll(): Unit = List(badRows, output).foreach(deleteRecursively) } - val etlVersion = s"spark-${generated.ProjectSettings.version}-common-${generated.ProjectSettings.commonEnrichVersion}" + val etlVersion = + s"spark-${generated.ProjectSettings.version}-common-${generated.ProjectSettings.commonEnrichVersion}" val etlTimestamp = "2001-09-09 01:46:40.000" - private val outputFields = classOf[common.outputs.EnrichedEvent] - .getDeclaredFields + private val outputFields = classOf[common.outputs.EnrichedEvent].getDeclaredFields .map(_.getName) private val unmatchableFields = List("event_id") + /** * Is lzo available? */ @@ -84,7 +86,7 @@ object EnrichJobSpec { */ case class BeFieldEqualTo(expected: String, index: Int) extends Matcher[String] { - private val field = outputFields(index) + private val field = outputFields(index) private val unmatcheable = isUnmatchable(field) @@ -107,10 +109,14 @@ object EnrichJobSpec { /** A Specs2 matcher to check if a directory on disk is empty or not. */ val beEmptyDir: Matcher[File] = - ((f: File) => - !f.isDirectory || - f.list().length == 0 || - f.listFiles().filter(f => f.getName != "_SUCCESS" && !f.getName.endsWith(".crc")).map(_.length).sum == 0, + ( + (f: File) => + !f.isDirectory || + f.list().length == 0 || + f.listFiles() + .filter(f => f.getName != "_SUCCESS" && !f.getName.endsWith(".crc")) + .map(_.length) + .sum == 0, "is populated dir") /** @@ -131,9 +137,7 @@ object EnrichJobSpec { .filter(s => s.contains("part-") && !s.contains("crc")) files.headOption match { case Some(f) => - val list = Source.fromFile(new File(f)) - .getLines - .toList + val list = Source.fromFile(new File(f)).getLines.toList Some(list) case None => None } @@ -157,7 +161,7 @@ object EnrichJobSpec { * Throws an exception if deletion is unsuccessful. */ def deleteRecursively(file: File): Unit = { - def listFilesSafely(file: File): Seq[File] = { + def listFilesSafely(file: File): Seq[File] = if (file.exists()) { val files = file.listFiles() if (files == null) throw new IOException(s"Failed to list files for dir: $file") @@ -165,7 +169,6 @@ object EnrichJobSpec { } else { Seq.empty[File] } - } try { if (file.isDirectory) { @@ -207,7 +210,8 @@ object EnrichJobSpec { * @return the created file */ def randomFile(tag: String): File = - new File(System.getProperty("java.io.tmpdir"), + new File( + System.getProperty("java.io.tmpdir"), s"snowplow-enrich-job-${tag}-${Random.nextInt(Int.MaxValue)}") /** Remove the timestamp from bad rows so that what remains is deterministic */ @@ -224,8 +228,8 @@ object EnrichJobSpec { */ private val igluCentralConfig = { val encoder = new Base64(true) - new String(encoder.encode( - """|{ + new String( + encoder.encode("""|{ |"schema": "iglu:com.snowplowanalytics.iglu/resolver-config/jsonschema/1-0-0", |"data": { |"cacheSize": 500, @@ -271,18 +275,20 @@ object EnrichJobSpec { * @param lookup One of the lookup types * @return JSON fragment containing the lookup's database and URI */ - def getLookupJson(lookup: String): String = { + def getLookupJson(lookup: String): String = """|"%s": { |"database": "%s", |"uri": "http://snowplow-hosted-assets.s3.amazonaws.com/third-party/maxmind" - |}""".format(lookup, lookup match { - case "geo" => "GeoIPCity.dat" - case "isp" => "GeoIPISP.dat" - case "organization" => "GeoIPOrg.dat" - case "domain" => "GeoIPDomain.dat" - case "netspeed" => "GeoIPNetSpeedCell.dat" - }) - } + |}""".format( + lookup, + lookup match { + case "geo" => "GeoIPCity.dat" + case "isp" => "GeoIPISP.dat" + case "organization" => "GeoIPOrg.dat" + case "domain" => "GeoIPDomain.dat" + case "netspeed" => "GeoIPNetSpeedCell.dat" + } + ) /** Converts the JavaScript script to Base64. */ def getJavascriptScript(): String = { @@ -407,8 +413,7 @@ object EnrichJobSpec { |}""".stripMargin val encoder = new Base64(true) // true means "url safe" - new String(encoder.encode( - s"""|{ + new String(encoder.encode(s"""|{ |"schema": "iglu:com.snowplowanalytics.snowplow/enrichments/jsonschema/1-0-0", |"data": [ |{ @@ -605,8 +610,16 @@ trait EnrichJobSpec extends SparkSpec { apiRequestEnabled: Boolean = false, sqlQueryEnabled: Boolean = false): Unit = { val input = mkTmpFile("input", lines) - runEnrichJob(input.toString(), collector, anonOctets, anonOctetsEnabled, lookups, - currencyConversionEnabled, javascriptScriptEnabled, apiRequestEnabled, sqlQueryEnabled) + runEnrichJob( + input.toString(), + collector, + anonOctets, + anonOctetsEnabled, + lookups, + currencyConversionEnabled, + javascriptScriptEnabled, + apiRequestEnabled, + sqlQueryEnabled) deleteRecursively(input) } @@ -626,15 +639,29 @@ trait EnrichJobSpec extends SparkSpec { apiRequestEnabled: Boolean, sqlQueryEnabled: Boolean): Unit = { val config = Array( - "--input-folder", inputFile, - "--input-format", collector, - "--output-folder", dirs.output.toString(), - "--bad-folder", dirs.badRows.toString(), - "--enrichments", getEnrichments(anonOctets, anonOctetsEnabled, lookups, - currencyConversionEnabled, javascriptScriptEnabled, apiRequestEnabled, sqlQueryEnabled), - "--iglu-config", igluConfig, - "--etl-timestamp", 1000000000000L.toString, - "--local") + "--input-folder", + inputFile, + "--input-format", + collector, + "--output-folder", + dirs.output.toString(), + "--bad-folder", + dirs.badRows.toString(), + "--enrichments", + getEnrichments( + anonOctets, + anonOctetsEnabled, + lookups, + currencyConversionEnabled, + javascriptScriptEnabled, + apiRequestEnabled, + sqlQueryEnabled), + "--iglu-config", + igluConfig, + "--etl-timestamp", + 1000000000000L.toString, + "--local" + ) val job = EnrichJob(spark, config) job.run() @@ -650,9 +677,11 @@ trait EnrichJobSpec extends SparkSpec { import com.twitter.elephantbird.mapreduce.io.ThriftWritable import com.twitter.elephantbird.mapreduce.output.LzoThriftBlockOutputFormat import org.apache.hadoop.io.LongWritable - val f = new File(System.getProperty("java.io.tmpdir"), + val f = new File( + System.getProperty("java.io.tmpdir"), s"snowplow-enrich-job-${tag}-${scala.util.Random.nextInt(Int.MaxValue)}") - val rdd = spark.sparkContext.parallelize(Seq(payload)) + val rdd = spark.sparkContext + .parallelize(Seq(payload)) .map { e => val writable = ThriftWritable.newInstance(classOf[CollectorPayload]) writable.set(e) @@ -664,7 +693,8 @@ trait EnrichJobSpec extends SparkSpec { classOf[org.apache.hadoop.io.LongWritable], classOf[ThriftWritable[CollectorPayload]], classOf[LzoThriftBlockOutputFormat[ThriftWritable[CollectorPayload]]], - hadoopConfig) + hadoopConfig + ) f } diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/MasterCfSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/MasterCfSpec.scala index ca2482fc71..c664f3f2ef 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/MasterCfSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/MasterCfSpec.scala @@ -18,29 +18,29 @@ import org.specs2.mutable.Specification object MasterCfSpec { // Concatenate ALL lines from ALL other jobs - val lines = bad.BadTrackerCfLinesSpec.lines.lines ++ // 3 bad - bad.CorruptedCfLinesSpec.lines.lines ++ // 1 bad - bad.InvalidCfLinesSpec.lines.lines ++ // 3 bad - bad.UnsupportedPayloadCfLinesSpec.lines.lines ++ // 1 bad = 8 BAD - good.Aug2013CfLineSpec.lines.lines ++ // 1 good - good.Sep2013CfLineSpec.lines.lines ++ // 1 good - good.Oct2013CfLineSpec.lines.lines ++ // 1 good - good.LateOct2013CfLineSpec.lines.lines ++ // 1 good - good.Apr2014CfLineSpec.lines.lines ++ // 1 good - good.FutureCfLineSpec.lines.lines ++ // 1 good - good.PagePingCfLineSpec.lines.lines ++ // 1 good - good.PageViewCfLineSpec.lines.lines ++ // 1 good - good.RefererParserCfLineSpec.lines.lines ++ // 1 good - good.CampaignAttributionCfLineSpec.lines.lines ++ // 1 good - good.StructEventCfLineSpec.lines.lines ++ // 1 good - good.UnstructEventCfLineSpec.lines.lines ++ // 1 good - good.UnstructEventCfLineSpec.lines.lines ++ // 1 good - good.TransactionCfLineSpec.lines.lines ++ // 1 good - good.TransactionItemCfLineSpec.lines.lines ++ // 1 good = 15 GOOD - misc.DiscardableCfLinesSpec.lines.lines // 2 discarded + val lines = bad.BadTrackerCfLinesSpec.lines.lines ++ // 3 bad + bad.CorruptedCfLinesSpec.lines.lines ++ // 1 bad + bad.InvalidCfLinesSpec.lines.lines ++ // 3 bad + bad.UnsupportedPayloadCfLinesSpec.lines.lines ++ // 1 bad = 8 BAD + good.Aug2013CfLineSpec.lines.lines ++ // 1 good + good.Sep2013CfLineSpec.lines.lines ++ // 1 good + good.Oct2013CfLineSpec.lines.lines ++ // 1 good + good.LateOct2013CfLineSpec.lines.lines ++ // 1 good + good.Apr2014CfLineSpec.lines.lines ++ // 1 good + good.FutureCfLineSpec.lines.lines ++ // 1 good + good.PagePingCfLineSpec.lines.lines ++ // 1 good + good.PageViewCfLineSpec.lines.lines ++ // 1 good + good.RefererParserCfLineSpec.lines.lines ++ // 1 good + good.CampaignAttributionCfLineSpec.lines.lines ++ // 1 good + good.StructEventCfLineSpec.lines.lines ++ // 1 good + good.UnstructEventCfLineSpec.lines.lines ++ // 1 good + good.UnstructEventCfLineSpec.lines.lines ++ // 1 good + good.TransactionCfLineSpec.lines.lines ++ // 1 good + good.TransactionItemCfLineSpec.lines.lines ++ // 1 good = 15 GOOD + misc.DiscardableCfLinesSpec.lines.lines // 2 discarded object expected { val goodCount = 15 - val badCount = 8 + val badCount = 8 } } @@ -50,7 +50,7 @@ class MasterCfSpec extends Specification with EnrichJobSpec { override def appName = "master-cf" sequential "A job which processes a CloudFront file containing 15 valid events, 6 bad lines and 3 " + - "discardable lines" should { + "discardable lines" should { runEnrichJob(Lines(MasterCfSpec.lines: _*), "cloudfront", "1", false, List("geo")) "write 15 events" in { diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/MasterCljTomcatSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/MasterCljTomcatSpec.scala index 6304ae7ca9..7b6ab1a23e 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/MasterCljTomcatSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/MasterCljTomcatSpec.scala @@ -18,10 +18,10 @@ import org.specs2.mutable.Specification object MasterCljTomcatSpec { // Concatenate ALL lines from ALL other jobs - val lines = good.CljTomcatTp1SingleEventSpec.lines.lines ++ // 1 good - good.CljTomcatCallrailEventSpec.lines.lines ++ // 1 good - good.CljTomcatTp2MultiEventsSpec.lines.lines ++ // 3 good - good.CljTomcatTp2MegaEventsSpec.lines.lines // 7,500 good = 7,505 GOOD + val lines = good.CljTomcatTp1SingleEventSpec.lines.lines ++ // 1 good + good.CljTomcatCallrailEventSpec.lines.lines ++ // 1 good + good.CljTomcatTp2MultiEventsSpec.lines.lines ++ // 3 good + good.CljTomcatTp2MegaEventsSpec.lines.lines // 7,500 good = 7,505 GOOD object expected { val goodCount = 7505 } @@ -33,7 +33,7 @@ class MasterCljTomcatSpec extends Specification with EnrichJobSpec { override def appName = "master-clj-tomcat" sequential "A job which processes a Clojure-Tomcat file containing 7,505 valid events, 0 bad lines and " + - "3 discardable lines" should { + "3 discardable lines" should { runEnrichJob(Lines(MasterCljTomcatSpec.lines: _*), "clj-tomcat", "1", false, List("geo")) "write 7,505 events" in { diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/PiiFilteringSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/PiiFilteringSpec.scala index 3bf54011d3..549e53dee2 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/PiiFilteringSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/PiiFilteringSpec.scala @@ -21,13 +21,11 @@ class PiiFilteringSpec extends Specification with EnrichJobSpec { import EnrichJobSpec._ override def appName = "pii-filter-cf" sequential - "A job which processes a CloudFront file" should{ - "filter out pii events" in - { + "A job which processes a CloudFront file" should { + "filter out pii events" in { runEnrichJob(Lines(MasterCfSpec.lines: _*), "cloudfront", "1", false, List("geo")) val Some(goods) = readPartFile(dirs.output) - - val Some(bads) = readPartFile(dirs.badRows) + val Some(bads) = readPartFile(dirs.badRows) goods must not(contain(matching(".*pii.*"))) bads must not(contain(matching(".*pii.*"))) } diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/SparkSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/SparkSpec.scala index 34fb21de4f..d9e5b364cd 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/SparkSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/SparkSpec.scala @@ -1,4 +1,3 @@ - /* * Copyright (c) 2012-2018 Snowplow Analytics Ltd. All rights reserved. * @@ -35,17 +34,20 @@ trait SparkSpec extends BeforeAfterAll { .set("spark.kryo.registrationRequired", "true") .registerKryoClasses(EnrichJob.classesToRegister) var spark: SparkSession = - SparkSession.builder() + SparkSession + .builder() .config(conf) .getOrCreate() val hadoopConfig = spark.sparkContext.hadoopConfiguration hadoopConfig.set("io.compression.codecs", classOf[com.hadoop.compression.lzo.LzopCodec].getName()) - hadoopConfig.set("io.compression.codec.lzo.class", + hadoopConfig.set( + "io.compression.codec.lzo.class", classOf[com.hadoop.compression.lzo.LzoCodec].getName()) override def beforeAll(): Unit = - SparkSession.builder() + SparkSession + .builder() .config(conf) .getOrCreate() diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/bad/CorruptedCfLinesSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/bad/CorruptedCfLinesSpec.scala index 249a899ea6..c2f1c45dc2 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/bad/CorruptedCfLinesSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/bad/CorruptedCfLinesSpec.scala @@ -21,7 +21,8 @@ object CorruptedCfLinesSpec { val lines = EnrichJobSpec.Lines( "20yy-05-24 00:08:40 LHR5 3397 74.125.17.210 GET d3gs014xn8p70.cloudfront.net /ice.png 200 http://www.psychicbazaar.com/oracles/119-psycards-book-and-deck-starter-pack.html Mozilla/5.0%20(Linux;%20U;%20Android%202.3.4;%20generic)%20AppleWebKit/535.1%20(KHTML,%20like%20Gecko;%20Google%20Web%20Preview)%20Version/4.0%20Mobile%20Safari/535.1 -" ) - val expected = """{"line":"20yy-05-24 00:08:40 LHR5 3397 74.125.17.210 GET d3gs014xn8p70.cloudfront.net /ice.png 200 http://www.psychicbazaar.com/oracles/119-psycards-book-and-deck-starter-pack.html Mozilla/5.0%20(Linux;%20U;%20Android%202.3.4;%20generic)%20AppleWebKit/535.1%20(KHTML,%20like%20Gecko;%20Google%20Web%20Preview)%20Version/4.0%20Mobile%20Safari/535.1 -","errors":[{"level":"error","message":"Unexpected exception converting date [20yy-05-24] and time [00:08:40] to timestamp: [Invalid format: \"20yy-05-24T00:08:40+00:00\" is malformed at \"yy-05-24T00:08:40+00:00\"]"}]}""" + val expected = + """{"line":"20yy-05-24 00:08:40 LHR5 3397 74.125.17.210 GET d3gs014xn8p70.cloudfront.net /ice.png 200 http://www.psychicbazaar.com/oracles/119-psycards-book-and-deck-starter-pack.html Mozilla/5.0%20(Linux;%20U;%20Android%202.3.4;%20generic)%20AppleWebKit/535.1%20(KHTML,%20like%20Gecko;%20Google%20Web%20Preview)%20Version/4.0%20Mobile%20Safari/535.1 -","errors":[{"level":"error","message":"Unexpected exception converting date [20yy-05-24] and time [00:08:40] to timestamp: [Invalid format: \"20yy-05-24T00:08:40+00:00\" is malformed at \"yy-05-24T00:08:40+00:00\"]"}]}""" } /** Input data _is_ in the CloudFront access log format, but the fields are somehow corrupted. */ diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/bad/CorruptedThriftLinesSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/bad/CorruptedThriftLinesSpec.scala index 0bf7685968..6060c3e60f 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/bad/CorruptedThriftLinesSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/bad/CorruptedThriftLinesSpec.scala @@ -18,7 +18,8 @@ package bad import org.specs2.mutable.Specification object CorruptedThriftLinesSpec { - val expected = """{"line":"bac=","errors":[{"level":"error","message":"Error deserializing raw event: Cannot read. Remote side has closed. Tried to read 2 bytes, but only got 1 bytes. (This is often indicative of an internal error on the server side. Please check your server logs.)"}]}""" + val expected = + """{"line":"bac=","errors":[{"level":"error","message":"Error deserializing raw event: Cannot read. Remote side has closed. Tried to read 2 bytes, but only got 1 bytes. (This is often indicative of an internal error on the server side. Please check your server logs.)"}]}""" } /** Input Thrift data cannot be decoded so should be base 64 encoded in the resulting bad row. */ @@ -29,8 +30,16 @@ class CorruptedThriftLinesSpec extends Specification with EnrichJobSpec { "A job which processes a corrupted input line" should { if (!isLzoSupported) "native-lzo not supported" in skipped else { - runEnrichJob(getClass().getResource("CorruptedThriftLinesSpec.line.lzo").toString(), "thrift", - "1", false, List("geo"), false, false, false, false) + runEnrichJob( + getClass().getResource("CorruptedThriftLinesSpec.line.lzo").toString(), + "thrift", + "1", + false, + List("geo"), + false, + false, + false, + false) "write a bad row JSON containing the input line and all errors" in { val Some(bads) = readPartFile(dirs.badRows) diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/bad/NullNumericFieldsSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/bad/NullNumericFieldsSpec.scala index b1f06772e0..a93653990b 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/bad/NullNumericFieldsSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/bad/NullNumericFieldsSpec.scala @@ -21,7 +21,8 @@ object NullNumericFieldsSpec { val lines = EnrichJobSpec.Lines( "2014-10-11 14:01:05 - 37 172.31.38.31 GET 24.209.95.109 /i 200 http://www.myvideowebsite.com/embed/ab123456789?auto_start=e9&rf=cb Mozilla%2F5.0+%28Macintosh%3B+Intel+Mac+OS+X+10.6%3B+rv%3A32.0%29+Gecko%2F20100101+Firefox%2F32.0 e=se&se_ca=video-player%3Anewformat&se_ac=play-time&se_la=efba3ef384&se_va=&tid=" ) - val expected = """{"line":"2014-10-11 14:01:05 - 37 172.31.38.31 GET 24.209.95.109 /i 200 http://www.myvideowebsite.com/embed/ab123456789?auto_start=e9&rf=cb Mozilla%2F5.0+%28Macintosh%3B+Intel+Mac+OS+X+10.6%3B+rv%3A32.0%29+Gecko%2F20100101+Firefox%2F32.0 e=se&se_ca=video-player%3Anewformat&se_ac=play-time&se_la=efba3ef384&se_va=&tid=","errors":[{"level":"error","message":"Field [se_va]: cannot convert [] to Double-like String"},{"level":"error","message":"Field [tid]: [] is not a valid integer"}]}""" + val expected = + """{"line":"2014-10-11 14:01:05 - 37 172.31.38.31 GET 24.209.95.109 /i 200 http://www.myvideowebsite.com/embed/ab123456789?auto_start=e9&rf=cb Mozilla%2F5.0+%28Macintosh%3B+Intel+Mac+OS+X+10.6%3B+rv%3A32.0%29+Gecko%2F20100101+Firefox%2F32.0 e=se&se_ca=video-player%3Anewformat&se_ac=play-time&se_la=efba3ef384&se_va=&tid=","errors":[{"level":"error","message":"Field [se_va]: cannot convert [] to Double-like String"},{"level":"error","message":"Field [tid]: [] is not a valid integer"}]}""" } /** diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/bad/UnsupportedPayloadCfLinesSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/bad/UnsupportedPayloadCfLinesSpec.scala index 521113052a..c9dd859501 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/bad/UnsupportedPayloadCfLinesSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/bad/UnsupportedPayloadCfLinesSpec.scala @@ -21,7 +21,8 @@ object UnsupportedPayloadCfLinesSpec { val lines = EnrichJobSpec.Lines( "2012-05-24 11:35:53 DFW3 3343 99.116.172.58 GET d3gs014xn8p70.cloudfront.net /not-ice.png 200 http://www.psychicbazaar.com/2-tarot-cards/genre/all/type/all?p=5 Mozilla/5.0%20(Windows%20NT%206.1;%20WOW64;%20rv:12.0)%20Gecko/20100101%20Firefox/12.0 e=pv&page=Tarot%2520cards%2520-%2520Psychic%2520Bazaar&tid=344260&uid=288112e0a5003be2&vid=1&lang=en-US&refr=http%253A%252F%252Fwww.psychicbazaar.com%252F2-tarot-cards%252Fgenre%252Fall%252Ftype%252Fall%253Fp%253D4&f_pdf=1&f_qt=0&f_realp=0&f_wma=0&f_dir=0&f_fla=1&f_java=1&f_gears=0&f_ag=1&res=1366x768&cookie=1" ) - val expected = """{"line":"2012-05-24 11:35:53 DFW3 3343 99.116.172.58 GET d3gs014xn8p70.cloudfront.net /not-ice.png 200 http://www.psychicbazaar.com/2-tarot-cards/genre/all/type/all?p=5 Mozilla/5.0%20(Windows%20NT%206.1;%20WOW64;%20rv:12.0)%20Gecko/20100101%20Firefox/12.0 e=pv&page=Tarot%2520cards%2520-%2520Psychic%2520Bazaar&tid=344260&uid=288112e0a5003be2&vid=1&lang=en-US&refr=http%253A%252F%252Fwww.psychicbazaar.com%252F2-tarot-cards%252Fgenre%252Fall%252Ftype%252Fall%253Fp%253D4&f_pdf=1&f_qt=0&f_realp=0&f_wma=0&f_dir=0&f_fla=1&f_java=1&f_gears=0&f_ag=1&res=1366x768&cookie=1","errors":[{"level":"error","message":"Request path /not-ice.png does not match (/)vendor/version(/) pattern nor is a legacy /i(ce.png) request"}]}""" + val expected = + """{"line":"2012-05-24 11:35:53 DFW3 3343 99.116.172.58 GET d3gs014xn8p70.cloudfront.net /not-ice.png 200 http://www.psychicbazaar.com/2-tarot-cards/genre/all/type/all?p=5 Mozilla/5.0%20(Windows%20NT%206.1;%20WOW64;%20rv:12.0)%20Gecko/20100101%20Firefox/12.0 e=pv&page=Tarot%2520cards%2520-%2520Psychic%2520Bazaar&tid=344260&uid=288112e0a5003be2&vid=1&lang=en-US&refr=http%253A%252F%252Fwww.psychicbazaar.com%252F2-tarot-cards%252Fgenre%252Fall%252Ftype%252Fall%253Fp%253D4&f_pdf=1&f_qt=0&f_realp=0&f_wma=0&f_dir=0&f_fla=1&f_java=1&f_gears=0&f_ag=1&res=1366x768&cookie=1","errors":[{"level":"error","message":"Request path /not-ice.png does not match (/)vendor/version(/) pattern nor is a legacy /i(ce.png) request"}]}""" } /** Input data _is_ in the CloudFront access log format, but the fields are somehow corrupted. */ diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/ApiRequestEnrichmentCfLineSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/ApiRequestEnrichmentCfLineSpec.scala index cb8321fb85..64b0f86947 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/ApiRequestEnrichmentCfLineSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/ApiRequestEnrichmentCfLineSpec.scala @@ -19,7 +19,8 @@ import org.specs2.mutable.Specification object ApiRequestEnrichmentCfLineSpec { import EnrichJobSpec._ - val contexts = """eyJkYXRhIjpbeyJkYXRhIjp7Im9zVHlwZSI6Ik9TWCIsImFwcGxlSWRmdiI6InNvbWVfYXBwbGVJZGZ2Iiwib3BlbklkZmEiOiJzb21lX0lkZmEiLCJjYXJyaWVyIjoic29tZV9jYXJyaWVyIiwiZGV2aWNlTW9kZWwiOiJsYXJnZSIsIm9zVmVyc2lvbiI6IjMuMC4wIiwiYXBwbGVJZGZhIjoic29tZV9hcHBsZUlkZmEiLCJhbmRyb2lkSWRmYSI6InNvbWVfYW5kcm9pZElkZmEiLCJkZXZpY2VNYW51ZmFjdHVyZXIiOiJBbXN0cmFkIn0sInNjaGVtYSI6ImlnbHU6Y29tLnNub3dwbG93YW5hbHl0aWNzLnNub3dwbG93L21vYmlsZV9jb250ZXh0L2pzb25zY2hlbWEvMS0wLTAifSx7ImRhdGEiOnsibG9uZ2l0dWRlIjoxMCwiYmVhcmluZyI6NTAsInNwZWVkIjoxNiwiYWx0aXR1ZGUiOjIwLCJhbHRpdHVkZUFjY3VyYWN5IjowLjMsImxhdGl0dWRlTG9uZ2l0dWRlQWNjdXJhY3kiOjAuNSwibGF0aXR1ZGUiOjd9LCJzY2hlbWEiOiJpZ2x1OmNvbS5zbm93cGxvd2FuYWx5dGljcy5zbm93cGxvdy9nZW9sb2NhdGlvbl9jb250ZXh0L2pzb25zY2hlbWEvMS0wLTAifV0sInNjaGVtYSI6ImlnbHU6Y29tLnNub3dwbG93YW5hbHl0aWNzLnNub3dwbG93L2NvbnRleHRzL2pzb25zY2hlbWEvMS0wLTAifQ==""" + val contexts = + """eyJkYXRhIjpbeyJkYXRhIjp7Im9zVHlwZSI6Ik9TWCIsImFwcGxlSWRmdiI6InNvbWVfYXBwbGVJZGZ2Iiwib3BlbklkZmEiOiJzb21lX0lkZmEiLCJjYXJyaWVyIjoic29tZV9jYXJyaWVyIiwiZGV2aWNlTW9kZWwiOiJsYXJnZSIsIm9zVmVyc2lvbiI6IjMuMC4wIiwiYXBwbGVJZGZhIjoic29tZV9hcHBsZUlkZmEiLCJhbmRyb2lkSWRmYSI6InNvbWVfYW5kcm9pZElkZmEiLCJkZXZpY2VNYW51ZmFjdHVyZXIiOiJBbXN0cmFkIn0sInNjaGVtYSI6ImlnbHU6Y29tLnNub3dwbG93YW5hbHl0aWNzLnNub3dwbG93L21vYmlsZV9jb250ZXh0L2pzb25zY2hlbWEvMS0wLTAifSx7ImRhdGEiOnsibG9uZ2l0dWRlIjoxMCwiYmVhcmluZyI6NTAsInNwZWVkIjoxNiwiYWx0aXR1ZGUiOjIwLCJhbHRpdHVkZUFjY3VyYWN5IjowLjMsImxhdGl0dWRlTG9uZ2l0dWRlQWNjdXJhY3kiOjAuNSwibGF0aXR1ZGUiOjd9LCJzY2hlbWEiOiJpZ2x1OmNvbS5zbm93cGxvd2FuYWx5dGljcy5zbm93cGxvdy9nZW9sb2NhdGlvbl9jb250ZXh0L2pzb25zY2hlbWEvMS0wLTAifV0sInNjaGVtYSI6ImlnbHU6Y29tLnNub3dwbG93YW5hbHl0aWNzLnNub3dwbG93L2NvbnRleHRzL2pzb25zY2hlbWEvMS0wLTAifQ==""" val lines = Lines( s"2012-05-27 11:35:53 DFW3 3343 70.46.123.145 GET d3gs014xn8p70.cloudfront.net /ice.png 200 http://www.psychicbazaar.com/oracles/119-psycards-book-and-deck-starter-pack.html?view=print#detail Mozilla/5.0%20(Windows%20NT%206.1;%20WOW64;%20rv:12.0)%20Gecko/20100101%20Firefox/12.0 &e=ue&cx=$contexts&ue_pr=%7B%22schema%22%3A%22iglu%3Acom.snowplowanalytics.snowplow%2Funstruct_event%2Fjsonschema%2F1-0-0%22%2C%22data%22%3A%7B%22schema%22%3A%22iglu%3Acom.snowplowanalytics.snowplow-website%2Fsignup_form_submitted%2Fjsonschema%2F1-0-0%22%2C%22data%22%3A%7B%22name%22%3A%22Bob%C2%AE%22%2C%22email%22%3A%22alex%2Btest%40snowplowanalytics.com%22%2C%22company%22%3A%22SP%22%2C%22eventsPerMonth%22%3A%22%3C%201%20million%22%2C%22serviceType%22%3A%22unsure%22%7D%7D%7D&dtm=1364230969450&evn=com.acme&tid=598951&vp=2560x934&ds=2543x1420&vid=43&duid=9795bd0203804cd1&p=web&tv=js-0.11.1&fp=2876815413&aid=pbzsite&lang=en-GB&cs=UTF-8&tz=Europe%2FLondon&refr=http%3A%2F%2Fwww.psychicbazaar.com%2F&f_pdf=1&f_qt=0&f_realp=0&f_wma=0&f_dir=0&f_fla=1&f_java=1&f_gears=0&f_ag=1&res=2560x1440&cd=32&cookie=1&url=http%3A%2F%2Fwww.psychicbazaar.com%2F2-tarot-cards" ) @@ -51,7 +52,7 @@ object ApiRequestEnrichmentCfLineSpec { "Florida", null, null, - "nuvox.net", // Using the MaxMind domain lookup service + "nuvox.net", // Using the MaxMind domain lookup service null, "http://www.psychicbazaar.com/2-tarot-cards", null, // No page title for events @@ -159,7 +160,7 @@ object ApiRequestEnrichmentCfLineSpec { /** Check if test running in CI environment */ def continuousIntegration: Boolean = sys.env.get("CI") match { case Some("true") => true - case _ => false + case _ => false } } @@ -171,9 +172,14 @@ class ApiRequestEnrichmentCfLineSpec extends Specification with EnrichJobSpec { // Only run API Request Enrichment test if the its started on CI and api-request-test.py is running if (ApiRequestEnrichmentCfLineSpec.continuousIntegration) { "A job which processes a CloudFront file containing 1 valid custom unstructured event, " + - "derived context and custom contexts" should { - runEnrichJob(ApiRequestEnrichmentCfLineSpec.lines, "cloudfront", "1", false, - List("geo", "domain"), apiRequestEnabled = true) + "derived context and custom contexts" should { + runEnrichJob( + ApiRequestEnrichmentCfLineSpec.lines, + "cloudfront", + "1", + false, + List("geo", "domain"), + apiRequestEnabled = true) "correctly attach derived context fetched from REST server" in { val Some(goods) = readPartFile(dirs.output) @@ -190,7 +196,8 @@ class ApiRequestEnrichmentCfLineSpec extends Specification with EnrichJobSpec { } } } else { - println("WARNING: Skipping APIRequestEnrichmentCfLineSpec as no CI=true environment variable " + - "was found and integration-tests/api-lookup-test.py probably not running") + println( + "WARNING: Skipping APIRequestEnrichmentCfLineSpec as no CI=true environment variable " + + "was found and integration-tests/api-lookup-test.py probably not running") } } diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/Aug2013CfLineSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/Aug2013CfLineSpec.scala index 9de6cee47a..d89c35ffdf 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/Aug2013CfLineSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/Aug2013CfLineSpec.scala @@ -133,7 +133,7 @@ object Aug2013CfLineSpec { "UTF-8", "1024", "635" - ) + ) } /** diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CampaignAttributionCfLineSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CampaignAttributionCfLineSpec.scala index 40e347a478..90ce236511 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CampaignAttributionCfLineSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CampaignAttributionCfLineSpec.scala @@ -144,7 +144,7 @@ class CampaignAttributionCfLineSpec extends Specification with EnrichJobSpec { override def appName = "campaign-attribution-cf-lines" sequential "A job which processes a CloudFront file containing 1 valid page ping with campaign attribution" + - " fields" should { + " fields" should { runEnrichJob(CampaignAttributionCfLineSpec.lines, "cloudfront", "1", false, List("geo")) "correctly output 1 page ping" in { diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatCallrailEventSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatCallrailEventSpec.scala index 544e7541f8..74bd5e0e7d 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatCallrailEventSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatCallrailEventSpec.scala @@ -145,7 +145,7 @@ class CljTomcatCallrailEventSpec extends Specification with EnrichJobSpec { override def appName = "clj-tomcat-callrail-event" sequential "A job which processes a Clojure-Tomcat file containing a GET raw event representing 1 valid " + - "completed call" should { + "completed call" should { runEnrichJob(CljTomcatCallrailEventSpec.lines, "clj-tomcat", "2", true, List("geo")) "correctly output 1 completed call" in { diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatMailchimpEventSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatMailchimpEventSpec.scala index 80db02512b..3cbda94942 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatMailchimpEventSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatMailchimpEventSpec.scala @@ -139,7 +139,7 @@ class CljTomcatMailchimpEventSpec extends Specification with EnrichJobSpec { override def appName = "clj-tomcat-mailchimp-event" sequential "A job which processes a Clojure-Tomcat file containing a POST raw event representing 1 valid " + - "completed call" should { + "completed call" should { runEnrichJob(CljTomcatMailchimpEventSpec.lines, "clj-tomcat", "2", true, List("geo")) "correctly output 1 completed call" in { diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatMailgunEventSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatMailgunEventSpec.scala index bf08ac799e..a2853e6fdf 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatMailgunEventSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatMailgunEventSpec.scala @@ -22,9 +22,9 @@ import org.specs2.mutable.Specification object CljTomcatMailgunEventSpec { import EnrichJobSpec._ val lines = Lines( - "2017-11-08 17:23:47 - - 52.205.206.237 POST 52.205.206.237 /com.mailgun/v1 200 - - &cv=clj-1.1.0-tom-0.2.0&nuid=856c2ed4-addc-45f5-bfe3-027256cf0057 - - - application%2Fx-www-form-urlencoded ZG9tYWluPXNhbmRib3g1NzA3MDA3MjA3NWQ0Y2ZkOTAwOGQ0MzMyMTA4NzM0Yy5tYWlsZ3VuLm9yZyZteV92YXJfMT1NYWlsZ3VuK1ZhcmlhYmxlKyUyMzEmbXktdmFyLTI9YXdlc29tZSZtZXNzYWdlLWhlYWRlcnM9JTVCJTVCJTIyUmVjZWl2ZWQlMjIlMkMrJTIyYnkrbHVuYS5tYWlsZ3VuLm5ldCt3aXRoK1NNVFArbWdydCs4NzM0NjYzMzExNzMzJTNCK0ZyaSUyQyswMytNYXkrMjAxMysxOCUzQTI2JTNBMjcrJTJCMDAwMCUyMiU1RCUyQyslNUIlMjJDb250ZW50LVR5cGUlMjIlMkMrJTVCJTIybXVsdGlwYXJ0JTJGYWx0ZXJuYXRpdmUlMjIlMkMrJTdCJTIyYm91bmRhcnklMjIlM0ErJTIyZWI2NjNkNzNhZTBhNGQ2YzkxNTNjYzBhZWM4Yjc1MjAlMjIlN0QlNUQlNUQlMkMrJTVCJTIyTWltZS1WZXJzaW9uJTIyJTJDKyUyMjEuMCUyMiU1RCUyQyslNUIlMjJTdWJqZWN0JTIyJTJDKyUyMlRlc3QrZGVsaXZlcit3ZWJob29rJTIyJTVEJTJDKyU1QiUyMkZyb20lMjIlMkMrJTIyQm9iKyUzQ2JvYiU0MHNhbmRib3g1NzA3MDA3MjA3NWQ0Y2ZkOTAwOGQ0MzMyMTA4NzM0Yy5tYWlsZ3VuLm9yZyUzRSUyMiU1RCUyQyslNUIlMjJUbyUyMiUyQyslMjJBbGljZSslM0NhbGljZSU0MGV4YW1wbGUuY29tJTNFJTIyJTVEJTJDKyU1QiUyMk1lc3NhZ2UtSWQlMjIlMkMrJTIyJTNDMjAxMzA1MDMxODI2MjYuMTg2NjYuMTY1NDAlNDBzYW5kYm94NTcwNzAwNzIwNzVkNGNmZDkwMDhkNDMzMjEwODczNGMubWFpbGd1bi5vcmclM0UlMjIlNUQlMkMrJTVCJTIyWC1NYWlsZ3VuLVZhcmlhYmxlcyUyMiUyQyslMjIlN0IlNUMlMjJteV92YXJfMSU1QyUyMiUzQSslNUMlMjJNYWlsZ3VuK1ZhcmlhYmxlKyUyMzElNUMlMjIlMkMrJTVDJTIybXktdmFyLTIlNUMlMjIlM0ErJTVDJTIyYXdlc29tZSU1QyUyMiU3RCUyMiU1RCUyQyslNUIlMjJEYXRlJTIyJTJDKyUyMkZyaSUyQyswMytNYXkrMjAxMysxOCUzQTI2JTNBMjcrJTJCMDAwMCUyMiU1RCUyQyslNUIlMjJTZW5kZXIlMjIlMkMrJTIyYm9iJTQwc2FuZGJveDU3MDcwMDcyMDc1ZDRjZmQ5MDA4ZDQzMzIxMDg3MzRjLm1haWxndW4ub3JnJTIyJTVEJTVEJk1lc3NhZ2UtSWQ9JTNDMjAxMzA1MDMxODI2MjYuMTg2NjYuMTY1NDAlNDBzYW5kYm94NTcwNzAwNzIwNzVkNGNmZDkwMDhkNDMzMjEwODczNGMubWFpbGd1bi5vcmclM0UmcmVjaXBpZW50PWFsaWNlJTQwZXhhbXBsZS5jb20mZXZlbnQ9ZGVsaXZlcmVkJnRpbWVzdGFtcD0xNTEwMTYxODI3JnRva2VuPWNkODdmNWEzMDAwMjc5NGUzN2FhNDllNjdmYjQ2OTkwZTU3OGIxZTkxOTc3NzNkODE3JnNpZ25hdHVyZT1jOTAyZmY5ZTNkZWE1NGMyZGJlMTg3MWY5MDQxNjUzMjkyZWE5Njg5ZDNkMmIyZDJlY2ZhOTk2ZjAyNWI5NjY5JmJvZHktcGxhaW49", - "2017-11-08 17:24:22 - - 52.205.206.237 POST 52.205.206.237 /com.mailgun/v1 200 - - &cv=clj-1.1.0-tom-0.2.0&nuid=8bc04475-779b-4ed4-bec3-555e737fac74 - - - multipart%2Fform-data%3B+boundary%3D353d603f-eede-4b49-97ac-724fbc54ea3c LS0zNTNkNjAzZi1lZWRlLTRiNDktOTdhYy03MjRmYmM1NGVhM2MNCkNvbnRlbnQtRGlzcG9zaXRpb246IGZvcm0tZGF0YTsgbmFtZT0iTWVzc2FnZS1JZCINCg0KPDIwMTMwNTAzMTkyNjU5LjEzNjUxLjIwMjg3QHNhbmRib3g1NzA3MDA3MjA3NWQ0Y2ZkOTAwOGQ0MzMyMTA4NzM0Yy5tYWlsZ3VuLm9yZz4NCi0tMzUzZDYwM2YtZWVkZS00YjQ5LTk3YWMtNzI0ZmJjNTRlYTNjDQpDb250ZW50LURpc3Bvc2l0aW9uOiBmb3JtLWRhdGE7IG5hbWU9IlgtTWFpbGd1bi1TaWQiDQoNCld5SXdOekk1TUNJc0lDSnBaRzkxWW5SMGFHbHpiMjVsWlhocGMzUnpRR2R0WVdsc0xtTnZiU0lzSUNJMklsMD0NCi0tMzUzZDYwM2YtZWVkZS00YjQ5LTk3YWMtNzI0ZmJjNTRlYTNjDQpDb250ZW50LURpc3Bvc2l0aW9uOiBmb3JtLWRhdGE7IG5hbWU9ImF0dGFjaG1lbnQtY291bnQiDQoNCjENCi0tMzUzZDYwM2YtZWVkZS00YjQ5LTk3YWMtNzI0ZmJjNTRlYTNjDQpDb250ZW50LURpc3Bvc2l0aW9uOiBmb3JtLWRhdGE7IG5hbWU9ImJvZHktcGxhaW4iDQoNCg0KLS0zNTNkNjAzZi1lZWRlLTRiNDktOTdhYy03MjRmYmM1NGVhM2MNCkNvbnRlbnQtRGlzcG9zaXRpb246IGZvcm0tZGF0YTsgbmFtZT0iY29kZSINCg0KNjA1DQotLTM1M2Q2MDNmLWVlZGUtNGI0OS05N2FjLTcyNGZiYzU0ZWEzYw0KQ29udGVudC1EaXNwb3NpdGlvbjogZm9ybS1kYXRhOyBuYW1lPSJkZXNjcmlwdGlvbiINCg0KTm90IGRlbGl2ZXJpbmcgdG8gcHJldmlvdXNseSBib3VuY2VkIGFkZHJlc3MNCi0tMzUzZDYwM2YtZWVkZS00YjQ5LTk3YWMtNzI0ZmJjNTRlYTNjDQpDb250ZW50LURpc3Bvc2l0aW9uOiBmb3JtLWRhdGE7IG5hbWU9ImRvbWFpbiINCg0Kc2FuZGJveDU3MDcwMDcyMDc1ZDRjZmQ5MDA4ZDQzMzIxMDg3MzRjLm1haWxndW4ub3JnDQotLTM1M2Q2MDNmLWVlZGUtNGI0OS05N2FjLTcyNGZiYzU0ZWEzYw0KQ29udGVudC1EaXNwb3NpdGlvbjogZm9ybS1kYXRhOyBuYW1lPSJldmVudCINCg0KZHJvcHBlZA0KLS0zNTNkNjAzZi1lZWRlLTRiNDktOTdhYy03MjRmYmM1NGVhM2MNCkNvbnRlbnQtRGlzcG9zaXRpb246IGZvcm0tZGF0YTsgbmFtZT0ibWVzc2FnZS1oZWFkZXJzIg0KDQpbWyJSZWNlaXZlZCIsICJieSBsdW5hLm1haWxndW4ubmV0IHdpdGggU01UUCBtZ3J0IDg3NTU1NDY3NTE0MDU7IEZyaSwgMDMgTWF5IDIwMTMgMTk6MjY6NTkgKzAwMDAiXSwgWyJDb250ZW50LVR5cGUiLCBbIm11bHRpcGFydC9hbHRlcm5hdGl2ZSIsIHsiYm91bmRhcnkiOiAiMjMwNDFiY2RmYWU1NGFhZmI4MDFhOGRhMDI4M2FmODUifV1dLCBbIk1pbWUtVmVyc2lvbiIsICIxLjAiXSwgWyJTdWJqZWN0IiwgIlRlc3QgZHJvcCB3ZWJob29rIl0sIFsiRnJvbSIsICJCb2IgPGJvYkBzYW5kYm94NTcwNzAwNzIwNzVkNGNmZDkwMDhkNDMzMjEwODczNGMubWFpbGd1bi5vcmc-Il0sIFsiVG8iLCAiQWxpY2UgPGFsaWNlQGV4YW1wbGUuY29tPiJdLCBbIk1lc3NhZ2UtSWQiLCAiPDIwMTMwNTAzMTkyNjU5LjEzNjUxLjIwMjg3QHNhbmRib3g1NzA3MDA3MjA3NWQ0Y2ZkOTAwOGQ0MzMyMTA4NzM0Yy5tYWlsZ3VuLm9yZz4iXSwgWyJMaXN0LVVuc3Vic2NyaWJlIiwgIjxtYWlsdG86dStuYTZ0bXkzZWdlNHRnbmxkbXl5dHFvanFtZnNkZW1ieW1lM3RteTNjaGE0d2NuZGJnYXlkcXlyZ29pNndzemRwb3ZyaGk1ZGluZnp3NjN0Zm12NGdzNDN1b21zdGltZGhudnF3czNib21ueHcyanR1aHVzdGVxamdtcTZ0bUBzYW5kYm94NTcwNzAwNzIwNzVkNGNmZDkwMDhkNDMzMjEwODczNGMubWFpbGd1bi5vcmc-Il0sIFsiWC1NYWlsZ3VuLVNpZCIsICJXeUl3TnpJNU1DSXNJQ0pwWkc5MVluUjBhR2x6YjI1bFpYaHBjM1J6UUdkdFlXbHNMbU52YlNJc0lDSTJJbDA9Il0sIFsiWC1NYWlsZ3VuLVZhcmlhYmxlcyIsICJ7XCJteV92YXJfMVwiOiBcIk1haWxndW4gVmFyaWFibGUgIzFcIiwgXCJteS12YXItMlwiOiBcImF3ZXNvbWVcIn0iXSwgWyJEYXRlIiwgIkZyaSwgMDMgTWF5IDIwMTMgMTk6MjY6NTkgKzAwMDAiXSwgWyJTZW5kZXIiLCAiYm9iQHNhbmRib3g1NzA3MDA3MjA3NWQ0Y2ZkOTAwOGQ0MzMyMTA4NzM0Yy5tYWlsZ3VuLm9yZyJdXQ0KLS0zNTNkNjAzZi1lZWRlLTRiNDktOTdhYy03MjRmYmM1NGVhM2MNCkNvbnRlbnQtRGlzcG9zaXRpb246IGZvcm0tZGF0YTsgbmFtZT0ibXktdmFyLTIiDQoNCmF3ZXNvbWUNCi0tMzUzZDYwM2YtZWVkZS00YjQ5LTk3YWMtNzI0ZmJjNTRlYTNjDQpDb250ZW50LURpc3Bvc2l0aW9uOiBmb3JtLWRhdGE7IG5hbWU9Im15X3Zhcl8xIg0KDQpNYWlsZ3VuIFZhcmlhYmxlICMxDQotLTM1M2Q2MDNmLWVlZGUtNGI0OS05N2FjLTcyNGZiYzU0ZWEzYw0KQ29udGVudC1EaXNwb3NpdGlvbjogZm9ybS1kYXRhOyBuYW1lPSJyZWFzb24iDQoNCmhhcmRmYWlsDQotLTM1M2Q2MDNmLWVlZGUtNGI0OS05N2FjLTcyNGZiYzU0ZWEzYw0KQ29udGVudC1EaXNwb3NpdGlvbjogZm9ybS1kYXRhOyBuYW1lPSJyZWNpcGllbnQiDQoNCmFsaWNlQGV4YW1wbGUuY29tDQotLTM1M2Q2MDNmLWVlZGUtNGI0OS05N2FjLTcyNGZiYzU0ZWEzYw0KQ29udGVudC1EaXNwb3NpdGlvbjogZm9ybS1kYXRhOyBuYW1lPSJzaWduYXR1cmUiDQoNCjcxZjgxMjQ4NWFlM2ZiMzk4ZGU4ZDFhODZiMTM5ZjI0MzkxZDYwNGZkOTRkYWI1OWU3Yzk5Y2ZjZDUwNjg4NWMNCi0tMzUzZDYwM2YtZWVkZS00YjQ5LTk3YWMtNzI0ZmJjNTRlYTNjDQpDb250ZW50LURpc3Bvc2l0aW9uOiBmb3JtLWRhdGE7IG5hbWU9InRpbWVzdGFtcCINCg0KMTUxMDE2MTg2Mg0KLS0zNTNkNjAzZi1lZWRlLTRiNDktOTdhYy03MjRmYmM1NGVhM2MNCkNvbnRlbnQtRGlzcG9zaXRpb246IGZvcm0tZGF0YTsgbmFtZT0idG9rZW4iDQoNCjllM2ZmZmM3ZWJhNTdlMjgyZTg5ZjdhZmNmMjQzNTYzODY4ZTlkZTRlY2ZlYTc4YzA5DQotLTM1M2Q2MDNmLWVlZGUtNGI0OS05N2FjLTcyNGZiYzU0ZWEzYw0KQ29udGVudC1EaXNwb3NpdGlvbjogZm9ybS1kYXRhOyBuYW1lPSJhdHRhY2htZW50LTEiOyBmaWxlbmFtZT0ibWVzc2FnZS5taW1lIg0KQ29udGVudC1UeXBlOiBhcHBsaWNhdGlvbi9vY3RldC1zdHJlYW0NCkNvbnRlbnQtTGVuZ3RoOiAxMzg2DQoNClJlY2VpdmVkOiBieSBsdW5hLm1haWxndW4ubmV0IHdpdGggU01UUCBtZ3J0IDg3NTU1NDY3NTE0MDU7IEZyaSwgMDMgTWF5IDIwMTMKIDE5OjI2OjU5ICswMDAwCkNvbnRlbnQtVHlwZTogbXVsdGlwYXJ0L2FsdGVybmF0aXZlOyBib3VuZGFyeT0iMjMwNDFiY2RmYWU1NGFhZmI4MDFhOGRhMDI4M2FmODUiCk1pbWUtVmVyc2lvbjogMS4wClN1YmplY3Q6IFRlc3QgZHJvcCB3ZWJob29rCkZyb206IEJvYiA8Ym9iQHNhbmRib3g1NzA3MDA3MjA3NWQ0Y2ZkOTAwOGQ0MzMyMTA4NzM0Yy5tYWlsZ3VuLm9yZz4KVG86IEFsaWNlIDxhbGljZUBleGFtcGxlLmNvbT4KTWVzc2FnZS1JZDogPDIwMTMwNTAzMTkyNjU5LjEzNjUxLjIwMjg3QHNhbmRib3g1NzA3MDA3MjA3NWQ0Y2ZkOTAwOGQ0MzMyMTA4NzM0Yy5tYWlsZ3VuLm9yZz4KTGlzdC1VbnN1YnNjcmliZTogPG1haWx0bzp1K25hNnRteTNlZ2U0dGdubGRteXl0cW9qcW1mc2RlbWJ5bWUzdG15M2NoYTR3Y25kYmdheWRxeXJnb2k2d3N6ZHBvdnJoaTVkaW5menc2M3RmbXY0Z3M0M3VvbXN0aW1kaG52cXdzM2JvbW54dzJqdHVodXN0ZXFqZ21xNnRtQHNhbmRib3g1NzA3MDA3MjA3NWQ0Y2ZkOTAwOGQ0MzMyMTA4NzM0Yy5tYWlsZ3VuLm9yZz4KWC1NYWlsZ3VuLVNpZDogV3lJd056STVNQ0lzSUNKcFpHOTFZblIwYUdsemIyNWxaWGhwYzNSelFHZHRZV2xzTG1OdmJTSXNJQ0kySWwwPQpYLU1haWxndW4tVmFyaWFibGVzOiB7Im15X3Zhcl8xIjogIk1haWxndW4gVmFyaWFibGUgIzEiLCAibXktdmFyLTIiOiAiYXdlc29tZSJ9CkRhdGU6IEZyaSwgMDMgTWF5IDIwMTMgMTk6MjY6NTkgKzAwMDAKU2VuZGVyOiBib2JAc2FuZGJveDU3MDcwMDcyMDc1ZDRjZmQ5MDA4ZDQzMzIxMDg3MzRjLm1haWxndW4ub3JnCgotLTIzMDQxYmNkZmFlNTRhYWZiODAxYThkYTAyODNhZjg1Ck1pbWUtVmVyc2lvbjogMS4wCkNvbnRlbnQtVHlwZTogdGV4dC9wbGFpbjsgY2hhcnNldD0iYXNjaWkiCkNvbnRlbnQtVHJhbnNmZXItRW5jb2Rpbmc6IDdiaXQKCkhpIEFsaWNlLCBJIHNlbnQgYW4gZW1haWwgdG8gdGhpcyBhZGRyZXNzIGJ1dCBpdCB3YXMgYm91bmNlZC4KCi0tMjMwNDFiY2RmYWU1NGFhZmI4MDFhOGRhMDI4M2FmODUKTWltZS1WZXJzaW9uOiAxLjAKQ29udGVudC1UeXBlOiB0ZXh0L2h0bWw7IGNoYXJzZXQ9ImFzY2lpIgpDb250ZW50LVRyYW5zZmVyLUVuY29kaW5nOiA3Yml0Cgo8aHRtbD4KICAgICAgICAgICAgICAgICAgICAgICAgICAgIDxib2R5PkhpIEFsaWNlLCBJIHNlbnQgYW4gZW1haWwgdG8gdGhpcyBhZGRyZXNzIGJ1dCBpdCB3YXMgYm91bmNlZC4KICAgICAgICAgICAgICAgICAgICAgICAgICAgIDxicj4KPC9ib2R5PjwvaHRtbD4KLS0yMzA0MWJjZGZhZTU0YWFmYjgwMWE4ZGEwMjgzYWY4NS0tCg0KLS0zNTNkNjAzZi1lZWRlLTRiNDktOTdhYy03MjRmYmM1NGVhM2MtLQ0K" - ) + "2017-11-08 17:23:47 - - 52.205.206.237 POST 52.205.206.237 /com.mailgun/v1 200 - - &cv=clj-1.1.0-tom-0.2.0&nuid=856c2ed4-addc-45f5-bfe3-027256cf0057 - - - application%2Fx-www-form-urlencoded ZG9tYWluPXNhbmRib3g1NzA3MDA3MjA3NWQ0Y2ZkOTAwOGQ0MzMyMTA4NzM0Yy5tYWlsZ3VuLm9yZyZteV92YXJfMT1NYWlsZ3VuK1ZhcmlhYmxlKyUyMzEmbXktdmFyLTI9YXdlc29tZSZtZXNzYWdlLWhlYWRlcnM9JTVCJTVCJTIyUmVjZWl2ZWQlMjIlMkMrJTIyYnkrbHVuYS5tYWlsZ3VuLm5ldCt3aXRoK1NNVFArbWdydCs4NzM0NjYzMzExNzMzJTNCK0ZyaSUyQyswMytNYXkrMjAxMysxOCUzQTI2JTNBMjcrJTJCMDAwMCUyMiU1RCUyQyslNUIlMjJDb250ZW50LVR5cGUlMjIlMkMrJTVCJTIybXVsdGlwYXJ0JTJGYWx0ZXJuYXRpdmUlMjIlMkMrJTdCJTIyYm91bmRhcnklMjIlM0ErJTIyZWI2NjNkNzNhZTBhNGQ2YzkxNTNjYzBhZWM4Yjc1MjAlMjIlN0QlNUQlNUQlMkMrJTVCJTIyTWltZS1WZXJzaW9uJTIyJTJDKyUyMjEuMCUyMiU1RCUyQyslNUIlMjJTdWJqZWN0JTIyJTJDKyUyMlRlc3QrZGVsaXZlcit3ZWJob29rJTIyJTVEJTJDKyU1QiUyMkZyb20lMjIlMkMrJTIyQm9iKyUzQ2JvYiU0MHNhbmRib3g1NzA3MDA3MjA3NWQ0Y2ZkOTAwOGQ0MzMyMTA4NzM0Yy5tYWlsZ3VuLm9yZyUzRSUyMiU1RCUyQyslNUIlMjJUbyUyMiUyQyslMjJBbGljZSslM0NhbGljZSU0MGV4YW1wbGUuY29tJTNFJTIyJTVEJTJDKyU1QiUyMk1lc3NhZ2UtSWQlMjIlMkMrJTIyJTNDMjAxMzA1MDMxODI2MjYuMTg2NjYuMTY1NDAlNDBzYW5kYm94NTcwNzAwNzIwNzVkNGNmZDkwMDhkNDMzMjEwODczNGMubWFpbGd1bi5vcmclM0UlMjIlNUQlMkMrJTVCJTIyWC1NYWlsZ3VuLVZhcmlhYmxlcyUyMiUyQyslMjIlN0IlNUMlMjJteV92YXJfMSU1QyUyMiUzQSslNUMlMjJNYWlsZ3VuK1ZhcmlhYmxlKyUyMzElNUMlMjIlMkMrJTVDJTIybXktdmFyLTIlNUMlMjIlM0ErJTVDJTIyYXdlc29tZSU1QyUyMiU3RCUyMiU1RCUyQyslNUIlMjJEYXRlJTIyJTJDKyUyMkZyaSUyQyswMytNYXkrMjAxMysxOCUzQTI2JTNBMjcrJTJCMDAwMCUyMiU1RCUyQyslNUIlMjJTZW5kZXIlMjIlMkMrJTIyYm9iJTQwc2FuZGJveDU3MDcwMDcyMDc1ZDRjZmQ5MDA4ZDQzMzIxMDg3MzRjLm1haWxndW4ub3JnJTIyJTVEJTVEJk1lc3NhZ2UtSWQ9JTNDMjAxMzA1MDMxODI2MjYuMTg2NjYuMTY1NDAlNDBzYW5kYm94NTcwNzAwNzIwNzVkNGNmZDkwMDhkNDMzMjEwODczNGMubWFpbGd1bi5vcmclM0UmcmVjaXBpZW50PWFsaWNlJTQwZXhhbXBsZS5jb20mZXZlbnQ9ZGVsaXZlcmVkJnRpbWVzdGFtcD0xNTEwMTYxODI3JnRva2VuPWNkODdmNWEzMDAwMjc5NGUzN2FhNDllNjdmYjQ2OTkwZTU3OGIxZTkxOTc3NzNkODE3JnNpZ25hdHVyZT1jOTAyZmY5ZTNkZWE1NGMyZGJlMTg3MWY5MDQxNjUzMjkyZWE5Njg5ZDNkMmIyZDJlY2ZhOTk2ZjAyNWI5NjY5JmJvZHktcGxhaW49", + "2017-11-08 17:24:22 - - 52.205.206.237 POST 52.205.206.237 /com.mailgun/v1 200 - - &cv=clj-1.1.0-tom-0.2.0&nuid=8bc04475-779b-4ed4-bec3-555e737fac74 - - - multipart%2Fform-data%3B+boundary%3D353d603f-eede-4b49-97ac-724fbc54ea3c LS0zNTNkNjAzZi1lZWRlLTRiNDktOTdhYy03MjRmYmM1NGVhM2MNCkNvbnRlbnQtRGlzcG9zaXRpb246IGZvcm0tZGF0YTsgbmFtZT0iTWVzc2FnZS1JZCINCg0KPDIwMTMwNTAzMTkyNjU5LjEzNjUxLjIwMjg3QHNhbmRib3g1NzA3MDA3MjA3NWQ0Y2ZkOTAwOGQ0MzMyMTA4NzM0Yy5tYWlsZ3VuLm9yZz4NCi0tMzUzZDYwM2YtZWVkZS00YjQ5LTk3YWMtNzI0ZmJjNTRlYTNjDQpDb250ZW50LURpc3Bvc2l0aW9uOiBmb3JtLWRhdGE7IG5hbWU9IlgtTWFpbGd1bi1TaWQiDQoNCld5SXdOekk1TUNJc0lDSnBaRzkxWW5SMGFHbHpiMjVsWlhocGMzUnpRR2R0WVdsc0xtTnZiU0lzSUNJMklsMD0NCi0tMzUzZDYwM2YtZWVkZS00YjQ5LTk3YWMtNzI0ZmJjNTRlYTNjDQpDb250ZW50LURpc3Bvc2l0aW9uOiBmb3JtLWRhdGE7IG5hbWU9ImF0dGFjaG1lbnQtY291bnQiDQoNCjENCi0tMzUzZDYwM2YtZWVkZS00YjQ5LTk3YWMtNzI0ZmJjNTRlYTNjDQpDb250ZW50LURpc3Bvc2l0aW9uOiBmb3JtLWRhdGE7IG5hbWU9ImJvZHktcGxhaW4iDQoNCg0KLS0zNTNkNjAzZi1lZWRlLTRiNDktOTdhYy03MjRmYmM1NGVhM2MNCkNvbnRlbnQtRGlzcG9zaXRpb246IGZvcm0tZGF0YTsgbmFtZT0iY29kZSINCg0KNjA1DQotLTM1M2Q2MDNmLWVlZGUtNGI0OS05N2FjLTcyNGZiYzU0ZWEzYw0KQ29udGVudC1EaXNwb3NpdGlvbjogZm9ybS1kYXRhOyBuYW1lPSJkZXNjcmlwdGlvbiINCg0KTm90IGRlbGl2ZXJpbmcgdG8gcHJldmlvdXNseSBib3VuY2VkIGFkZHJlc3MNCi0tMzUzZDYwM2YtZWVkZS00YjQ5LTk3YWMtNzI0ZmJjNTRlYTNjDQpDb250ZW50LURpc3Bvc2l0aW9uOiBmb3JtLWRhdGE7IG5hbWU9ImRvbWFpbiINCg0Kc2FuZGJveDU3MDcwMDcyMDc1ZDRjZmQ5MDA4ZDQzMzIxMDg3MzRjLm1haWxndW4ub3JnDQotLTM1M2Q2MDNmLWVlZGUtNGI0OS05N2FjLTcyNGZiYzU0ZWEzYw0KQ29udGVudC1EaXNwb3NpdGlvbjogZm9ybS1kYXRhOyBuYW1lPSJldmVudCINCg0KZHJvcHBlZA0KLS0zNTNkNjAzZi1lZWRlLTRiNDktOTdhYy03MjRmYmM1NGVhM2MNCkNvbnRlbnQtRGlzcG9zaXRpb246IGZvcm0tZGF0YTsgbmFtZT0ibWVzc2FnZS1oZWFkZXJzIg0KDQpbWyJSZWNlaXZlZCIsICJieSBsdW5hLm1haWxndW4ubmV0IHdpdGggU01UUCBtZ3J0IDg3NTU1NDY3NTE0MDU7IEZyaSwgMDMgTWF5IDIwMTMgMTk6MjY6NTkgKzAwMDAiXSwgWyJDb250ZW50LVR5cGUiLCBbIm11bHRpcGFydC9hbHRlcm5hdGl2ZSIsIHsiYm91bmRhcnkiOiAiMjMwNDFiY2RmYWU1NGFhZmI4MDFhOGRhMDI4M2FmODUifV1dLCBbIk1pbWUtVmVyc2lvbiIsICIxLjAiXSwgWyJTdWJqZWN0IiwgIlRlc3QgZHJvcCB3ZWJob29rIl0sIFsiRnJvbSIsICJCb2IgPGJvYkBzYW5kYm94NTcwNzAwNzIwNzVkNGNmZDkwMDhkNDMzMjEwODczNGMubWFpbGd1bi5vcmc-Il0sIFsiVG8iLCAiQWxpY2UgPGFsaWNlQGV4YW1wbGUuY29tPiJdLCBbIk1lc3NhZ2UtSWQiLCAiPDIwMTMwNTAzMTkyNjU5LjEzNjUxLjIwMjg3QHNhbmRib3g1NzA3MDA3MjA3NWQ0Y2ZkOTAwOGQ0MzMyMTA4NzM0Yy5tYWlsZ3VuLm9yZz4iXSwgWyJMaXN0LVVuc3Vic2NyaWJlIiwgIjxtYWlsdG86dStuYTZ0bXkzZWdlNHRnbmxkbXl5dHFvanFtZnNkZW1ieW1lM3RteTNjaGE0d2NuZGJnYXlkcXlyZ29pNndzemRwb3ZyaGk1ZGluZnp3NjN0Zm12NGdzNDN1b21zdGltZGhudnF3czNib21ueHcyanR1aHVzdGVxamdtcTZ0bUBzYW5kYm94NTcwNzAwNzIwNzVkNGNmZDkwMDhkNDMzMjEwODczNGMubWFpbGd1bi5vcmc-Il0sIFsiWC1NYWlsZ3VuLVNpZCIsICJXeUl3TnpJNU1DSXNJQ0pwWkc5MVluUjBhR2x6YjI1bFpYaHBjM1J6UUdkdFlXbHNMbU52YlNJc0lDSTJJbDA9Il0sIFsiWC1NYWlsZ3VuLVZhcmlhYmxlcyIsICJ7XCJteV92YXJfMVwiOiBcIk1haWxndW4gVmFyaWFibGUgIzFcIiwgXCJteS12YXItMlwiOiBcImF3ZXNvbWVcIn0iXSwgWyJEYXRlIiwgIkZyaSwgMDMgTWF5IDIwMTMgMTk6MjY6NTkgKzAwMDAiXSwgWyJTZW5kZXIiLCAiYm9iQHNhbmRib3g1NzA3MDA3MjA3NWQ0Y2ZkOTAwOGQ0MzMyMTA4NzM0Yy5tYWlsZ3VuLm9yZyJdXQ0KLS0zNTNkNjAzZi1lZWRlLTRiNDktOTdhYy03MjRmYmM1NGVhM2MNCkNvbnRlbnQtRGlzcG9zaXRpb246IGZvcm0tZGF0YTsgbmFtZT0ibXktdmFyLTIiDQoNCmF3ZXNvbWUNCi0tMzUzZDYwM2YtZWVkZS00YjQ5LTk3YWMtNzI0ZmJjNTRlYTNjDQpDb250ZW50LURpc3Bvc2l0aW9uOiBmb3JtLWRhdGE7IG5hbWU9Im15X3Zhcl8xIg0KDQpNYWlsZ3VuIFZhcmlhYmxlICMxDQotLTM1M2Q2MDNmLWVlZGUtNGI0OS05N2FjLTcyNGZiYzU0ZWEzYw0KQ29udGVudC1EaXNwb3NpdGlvbjogZm9ybS1kYXRhOyBuYW1lPSJyZWFzb24iDQoNCmhhcmRmYWlsDQotLTM1M2Q2MDNmLWVlZGUtNGI0OS05N2FjLTcyNGZiYzU0ZWEzYw0KQ29udGVudC1EaXNwb3NpdGlvbjogZm9ybS1kYXRhOyBuYW1lPSJyZWNpcGllbnQiDQoNCmFsaWNlQGV4YW1wbGUuY29tDQotLTM1M2Q2MDNmLWVlZGUtNGI0OS05N2FjLTcyNGZiYzU0ZWEzYw0KQ29udGVudC1EaXNwb3NpdGlvbjogZm9ybS1kYXRhOyBuYW1lPSJzaWduYXR1cmUiDQoNCjcxZjgxMjQ4NWFlM2ZiMzk4ZGU4ZDFhODZiMTM5ZjI0MzkxZDYwNGZkOTRkYWI1OWU3Yzk5Y2ZjZDUwNjg4NWMNCi0tMzUzZDYwM2YtZWVkZS00YjQ5LTk3YWMtNzI0ZmJjNTRlYTNjDQpDb250ZW50LURpc3Bvc2l0aW9uOiBmb3JtLWRhdGE7IG5hbWU9InRpbWVzdGFtcCINCg0KMTUxMDE2MTg2Mg0KLS0zNTNkNjAzZi1lZWRlLTRiNDktOTdhYy03MjRmYmM1NGVhM2MNCkNvbnRlbnQtRGlzcG9zaXRpb246IGZvcm0tZGF0YTsgbmFtZT0idG9rZW4iDQoNCjllM2ZmZmM3ZWJhNTdlMjgyZTg5ZjdhZmNmMjQzNTYzODY4ZTlkZTRlY2ZlYTc4YzA5DQotLTM1M2Q2MDNmLWVlZGUtNGI0OS05N2FjLTcyNGZiYzU0ZWEzYw0KQ29udGVudC1EaXNwb3NpdGlvbjogZm9ybS1kYXRhOyBuYW1lPSJhdHRhY2htZW50LTEiOyBmaWxlbmFtZT0ibWVzc2FnZS5taW1lIg0KQ29udGVudC1UeXBlOiBhcHBsaWNhdGlvbi9vY3RldC1zdHJlYW0NCkNvbnRlbnQtTGVuZ3RoOiAxMzg2DQoNClJlY2VpdmVkOiBieSBsdW5hLm1haWxndW4ubmV0IHdpdGggU01UUCBtZ3J0IDg3NTU1NDY3NTE0MDU7IEZyaSwgMDMgTWF5IDIwMTMKIDE5OjI2OjU5ICswMDAwCkNvbnRlbnQtVHlwZTogbXVsdGlwYXJ0L2FsdGVybmF0aXZlOyBib3VuZGFyeT0iMjMwNDFiY2RmYWU1NGFhZmI4MDFhOGRhMDI4M2FmODUiCk1pbWUtVmVyc2lvbjogMS4wClN1YmplY3Q6IFRlc3QgZHJvcCB3ZWJob29rCkZyb206IEJvYiA8Ym9iQHNhbmRib3g1NzA3MDA3MjA3NWQ0Y2ZkOTAwOGQ0MzMyMTA4NzM0Yy5tYWlsZ3VuLm9yZz4KVG86IEFsaWNlIDxhbGljZUBleGFtcGxlLmNvbT4KTWVzc2FnZS1JZDogPDIwMTMwNTAzMTkyNjU5LjEzNjUxLjIwMjg3QHNhbmRib3g1NzA3MDA3MjA3NWQ0Y2ZkOTAwOGQ0MzMyMTA4NzM0Yy5tYWlsZ3VuLm9yZz4KTGlzdC1VbnN1YnNjcmliZTogPG1haWx0bzp1K25hNnRteTNlZ2U0dGdubGRteXl0cW9qcW1mc2RlbWJ5bWUzdG15M2NoYTR3Y25kYmdheWRxeXJnb2k2d3N6ZHBvdnJoaTVkaW5menc2M3RmbXY0Z3M0M3VvbXN0aW1kaG52cXdzM2JvbW54dzJqdHVodXN0ZXFqZ21xNnRtQHNhbmRib3g1NzA3MDA3MjA3NWQ0Y2ZkOTAwOGQ0MzMyMTA4NzM0Yy5tYWlsZ3VuLm9yZz4KWC1NYWlsZ3VuLVNpZDogV3lJd056STVNQ0lzSUNKcFpHOTFZblIwYUdsemIyNWxaWGhwYzNSelFHZHRZV2xzTG1OdmJTSXNJQ0kySWwwPQpYLU1haWxndW4tVmFyaWFibGVzOiB7Im15X3Zhcl8xIjogIk1haWxndW4gVmFyaWFibGUgIzEiLCAibXktdmFyLTIiOiAiYXdlc29tZSJ9CkRhdGU6IEZyaSwgMDMgTWF5IDIwMTMgMTk6MjY6NTkgKzAwMDAKU2VuZGVyOiBib2JAc2FuZGJveDU3MDcwMDcyMDc1ZDRjZmQ5MDA4ZDQzMzIxMDg3MzRjLm1haWxndW4ub3JnCgotLTIzMDQxYmNkZmFlNTRhYWZiODAxYThkYTAyODNhZjg1Ck1pbWUtVmVyc2lvbjogMS4wCkNvbnRlbnQtVHlwZTogdGV4dC9wbGFpbjsgY2hhcnNldD0iYXNjaWkiCkNvbnRlbnQtVHJhbnNmZXItRW5jb2Rpbmc6IDdiaXQKCkhpIEFsaWNlLCBJIHNlbnQgYW4gZW1haWwgdG8gdGhpcyBhZGRyZXNzIGJ1dCBpdCB3YXMgYm91bmNlZC4KCi0tMjMwNDFiY2RmYWU1NGFhZmI4MDFhOGRhMDI4M2FmODUKTWltZS1WZXJzaW9uOiAxLjAKQ29udGVudC1UeXBlOiB0ZXh0L2h0bWw7IGNoYXJzZXQ9ImFzY2lpIgpDb250ZW50LVRyYW5zZmVyLUVuY29kaW5nOiA3Yml0Cgo8aHRtbD4KICAgICAgICAgICAgICAgICAgICAgICAgICAgIDxib2R5PkhpIEFsaWNlLCBJIHNlbnQgYW4gZW1haWwgdG8gdGhpcyBhZGRyZXNzIGJ1dCBpdCB3YXMgYm91bmNlZC4KICAgICAgICAgICAgICAgICAgICAgICAgICAgIDxicj4KPC9ib2R5PjwvaHRtbD4KLS0yMzA0MWJjZGZhZTU0YWFmYjgwMWE4ZGEwMjgzYWY4NS0tCg0KLS0zNTNkNjAzZi1lZWRlLTRiNDktOTdhYy03MjRmYmM1NGVhM2MtLQ0K" + ) val expected1 = List( null, @@ -135,9 +135,15 @@ object CljTomcatMailgunEventSpec { null, null, null - ) + ) - val expected2 = expected1.updated(3, "2017-11-08 17:24:22.000").updated(17, "8bc04475-779b-4ed4-bec3-555e737fac74").updated(58, """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.mailgun/message_dropped/jsonschema/1-0-0","data":{"attachmentCount":1,"recipient":"alice@example.com","messageHeaders":"[[\"Received\", \"by luna.mailgun.net with SMTP mgrt 8755546751405; Fri, 03 May 2013 19:26:59 +0000\"], [\"Content-Type\", [\"multipart/alternative\", {\"boundary\": \"23041bcdfae54aafb801a8da0283af85\"}]], [\"Mime-Version\", \"1.0\"], [\"Subject\", \"Test drop webhook\"], [\"From\", \"Bob \"], [\"To\", \"Alice \"], [\"Message-Id\", \"<20130503192659.13651.20287@sandbox57070072075d4cfd9008d4332108734c.mailgun.org>\"], [\"List-Unsubscribe\", \"\"], [\"X-Mailgun-Sid\", \"WyIwNzI5MCIsICJpZG91YnR0aGlzb25lZXhpc3RzQGdtYWlsLmNvbSIsICI2Il0=\"], [\"X-Mailgun-Variables\", \"{\\\"my_var_1\\\": \\\"Mailgun Variable #1\\\", \\\"my-var-2\\\": \\\"awesome\\\"}\"], [\"Date\", \"Fri, 03 May 2013 19:26:59 +0000\"], [\"Sender\", \"bob@sandbox57070072075d4cfd9008d4332108734c.mailgun.org\"]]","timestamp":"2017-11-08T17:24:22.000Z","description":"Not delivering to previously bounced address","domain":"sandbox57070072075d4cfd9008d4332108734c.mailgun.org","signature":"71f812485ae3fb398de8d1a86b139f24391d604fd94dab59e7c99cfcd506885c","reason":"hardfail","code":"605","token":"9e3fffc7eba57e282e89f7afcf243563868e9de4ecfea78c09","messageId":"<20130503192659.13651.20287@sandbox57070072075d4cfd9008d4332108734c.mailgun.org>","xMailgunSid":"WyIwNzI5MCIsICJpZG91YnR0aGlzb25lZXhpc3RzQGdtYWlsLmNvbSIsICI2Il0=","myVar1":"Mailgun Variable #1","myVar2":"awesome","attachment1":"Received: by luna.mailgun.net with SMTP mgrt 8755546751405; Fri, 03 May 2013\n 19:26:59 +0000\nContent-Type: multipart/alternative; boundary=\"23041bcdfae54aafb801a8da0283af85\"\nMime-Version: 1.0\nSubject: Test drop webhook\nFrom: Bob \nTo: Alice \nMessage-Id: <20130503192659.13651.20287@sandbox57070072075d4cfd9008d4332108734c.mailgun.org>\nList-Unsubscribe: \nX-Mailgun-Sid: WyIwNzI5MCIsICJpZG91YnR0aGlzb25lZXhpc3RzQGdtYWlsLmNvbSIsICI2Il0=\nX-Mailgun-Variables: {\"my_var_1\": \"Mailgun Variable #1\", \"my-var-2\": \"awesome\"}\nDate: Fri, 03 May 2013 19:26:59 +0000\nSender: bob@sandbox57070072075d4cfd9008d4332108734c.mailgun.org\n\n--23041bcdfae54aafb801a8da0283af85\nMime-Version: 1.0\nContent-Type: text/plain; charset=\"ascii\"\nContent-Transfer-Encoding: 7bit\n\nHi Alice, I sent an email to this address but it was bounced.\n\n--23041bcdfae54aafb801a8da0283af85\nMime-Version: 1.0\nContent-Type: text/html; charset=\"ascii\"\nContent-Transfer-Encoding: 7bit\n\n\n Hi Alice, I sent an email to this address but it was bounced.\n
\n\n--23041bcdfae54aafb801a8da0283af85--"}}}""") + val expected2 = expected1 + .updated(3, "2017-11-08 17:24:22.000") + .updated(17, "8bc04475-779b-4ed4-bec3-555e737fac74") + .updated( + 58, + """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.mailgun/message_dropped/jsonschema/1-0-0","data":{"attachmentCount":1,"recipient":"alice@example.com","messageHeaders":"[[\"Received\", \"by luna.mailgun.net with SMTP mgrt 8755546751405; Fri, 03 May 2013 19:26:59 +0000\"], [\"Content-Type\", [\"multipart/alternative\", {\"boundary\": \"23041bcdfae54aafb801a8da0283af85\"}]], [\"Mime-Version\", \"1.0\"], [\"Subject\", \"Test drop webhook\"], [\"From\", \"Bob \"], [\"To\", \"Alice \"], [\"Message-Id\", \"<20130503192659.13651.20287@sandbox57070072075d4cfd9008d4332108734c.mailgun.org>\"], [\"List-Unsubscribe\", \"\"], [\"X-Mailgun-Sid\", \"WyIwNzI5MCIsICJpZG91YnR0aGlzb25lZXhpc3RzQGdtYWlsLmNvbSIsICI2Il0=\"], [\"X-Mailgun-Variables\", \"{\\\"my_var_1\\\": \\\"Mailgun Variable #1\\\", \\\"my-var-2\\\": \\\"awesome\\\"}\"], [\"Date\", \"Fri, 03 May 2013 19:26:59 +0000\"], [\"Sender\", \"bob@sandbox57070072075d4cfd9008d4332108734c.mailgun.org\"]]","timestamp":"2017-11-08T17:24:22.000Z","description":"Not delivering to previously bounced address","domain":"sandbox57070072075d4cfd9008d4332108734c.mailgun.org","signature":"71f812485ae3fb398de8d1a86b139f24391d604fd94dab59e7c99cfcd506885c","reason":"hardfail","code":"605","token":"9e3fffc7eba57e282e89f7afcf243563868e9de4ecfea78c09","messageId":"<20130503192659.13651.20287@sandbox57070072075d4cfd9008d4332108734c.mailgun.org>","xMailgunSid":"WyIwNzI5MCIsICJpZG91YnR0aGlzb25lZXhpc3RzQGdtYWlsLmNvbSIsICI2Il0=","myVar1":"Mailgun Variable #1","myVar2":"awesome","attachment1":"Received: by luna.mailgun.net with SMTP mgrt 8755546751405; Fri, 03 May 2013\n 19:26:59 +0000\nContent-Type: multipart/alternative; boundary=\"23041bcdfae54aafb801a8da0283af85\"\nMime-Version: 1.0\nSubject: Test drop webhook\nFrom: Bob \nTo: Alice \nMessage-Id: <20130503192659.13651.20287@sandbox57070072075d4cfd9008d4332108734c.mailgun.org>\nList-Unsubscribe: \nX-Mailgun-Sid: WyIwNzI5MCIsICJpZG91YnR0aGlzb25lZXhpc3RzQGdtYWlsLmNvbSIsICI2Il0=\nX-Mailgun-Variables: {\"my_var_1\": \"Mailgun Variable #1\", \"my-var-2\": \"awesome\"}\nDate: Fri, 03 May 2013 19:26:59 +0000\nSender: bob@sandbox57070072075d4cfd9008d4332108734c.mailgun.org\n\n--23041bcdfae54aafb801a8da0283af85\nMime-Version: 1.0\nContent-Type: text/plain; charset=\"ascii\"\nContent-Transfer-Encoding: 7bit\n\nHi Alice, I sent an email to this address but it was bounced.\n\n--23041bcdfae54aafb801a8da0283af85\nMime-Version: 1.0\nContent-Type: text/html; charset=\"ascii\"\nContent-Transfer-Encoding: 7bit\n\n\n Hi Alice, I sent an email to this address but it was bounced.\n
\n\n--23041bcdfae54aafb801a8da0283af85--"}}}""" + ) val expected = List(expected1, expected2) } @@ -146,14 +152,13 @@ class CljTomcatMailgunEventSpec extends Specification with EnrichJobSpec { override def appName = "clj-tomcat-mailgun-event" sequential "A job which processes a Clojure-Tomcat file containing a POST raw event representing 2 valid " + - "completed call" should { + "completed call" should { runEnrichJob(CljTomcatMailgunEventSpec.lines, "clj-tomcat", "2", true, List("geo")) "correctly output 2 completed calls" in { val Some(goods) = readPartFile(dirs.output) goods.size must_== CljTomcatMailgunEventSpec.expected.length - for(line_idx <- goods.indices) - { + for (line_idx <- goods.indices) { val actual = goods(line_idx).split("\t").map(s => if (s.isEmpty()) null else s) for (idx <- CljTomcatMailgunEventSpec.expected(line_idx).indices) { actual(idx) must BeFieldEqualTo(CljTomcatMailgunEventSpec.expected(line_idx)(idx), idx) diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatMandrillEventSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatMandrillEventSpec.scala index d6718e85f6..9cae20933d 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatMandrillEventSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatMandrillEventSpec.scala @@ -139,7 +139,7 @@ class CljTomcatMandrillEventSpec extends Specification with EnrichJobSpec { override def appName = "clj-tomcat-mandrill-event" sequential "A job which processes a Clojure-Tomcat file containing a Mandrill POST raw event representing " + - "1 valid completed call" should { + "1 valid completed call" should { runEnrichJob(CljTomcatMandrillEventSpec.lines, "clj-tomcat", "2", true, List("geo")) "correctly output 1 completed call" in { diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatOlarkEventSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatOlarkEventSpec.scala index d01a47c674..fcb0ab8e80 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatOlarkEventSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatOlarkEventSpec.scala @@ -13,7 +13,6 @@ package com.snowplowanalytics.snowplow.enrich.spark package good - // Specs2 import org.specs2.mutable.Specification @@ -27,7 +26,7 @@ object CljTomcatOlarkEventSpec { import EnrichJobSpec._ val lines = Lines( "2014-10-09 16:28:31 - 13 255.255.255.255 POST 255.255.255.255 /com.olark/v1 404 - - aid=email&cv=clj-0.6.0-tom-0.0.4&nuid=- - - - application%2Fx-www-form-urlencoded ZGF0YT0lN0IlMjJraW5kJTIyJTNBKyUyMkNvbnZlcnNhdGlvbiUyMiUyQyslMjJ0YWdzJTIyJTNBKyU1QiUyMnRlc3RfZXhhbXBsZSUyMiU1RCUyQyslMjJpdGVtcyUyMiUzQSslNUIlN0IlMjJib2R5JTIyJTNBKyUyMkhpK2Zyb20rYW4rb3BlcmF0b3IlMjIlMkMrJTIydGltZXN0YW1wJTIyJTNBKyUyMjE0NzM3NzQ4MTkuMjYzMDgzJTIyJTJDKyUyMmtpbmQlMjIlM0ErJTIyTWVzc2FnZVRvVmlzaXRvciUyMiUyQyslMjJuaWNrbmFtZSUyMiUzQSslMjJPbGFyaytvcGVyYXRvciUyMiUyQyslMjJvcGVyYXRvcklkJTIyJTNBKyUyMjY0NzU2MyUyMiU3RCUyQyslN0IlMjJib2R5JTIyJTNBKyUyMkhpK2Zyb20rYSt2aXNpdG9yJTIyJTJDKyUyMnRpbWVzdGFtcCUyMiUzQSslMjIxNDczNzc0ODIxLjQxMTE1NCUyMiUyQyslMjJraW5kJTIyJTNBKyUyMk1lc3NhZ2VUb09wZXJhdG9yJTIyJTJDKyUyMm5pY2tuYW1lJTIyJTNBKyUyMlJldHVybmluZytWaXNpdG9yKyU3QytVU0ErJTI4U2FuK0ZyYW5jaXNjbyUyQytDQSUyOSslMjM3NjE3JTIyJTJDKyUyMnZpc2l0b3Jfbmlja25hbWUlMjIlM0ErJTIyT2xhcmsrVmlzaXRvciUyMiU3RCU1RCUyQyslMjJvcGVyYXRvcnMlMjIlM0ErJTdCJTIyNjQ3NTYzJTIyJTNBKyU3QiUyMnVzZXJuYW1lJTIyJTNBKyUyMnlhbGklMjIlMkMrJTIyZW1haWxBZGRyZXNzJTIyJTNBKyUyMnlhbGklNDBzbm93cGxvd2FuYWx5dGljcy5jb20lMjIlMkMrJTIya2luZCUyMiUzQSslMjJPcGVyYXRvciUyMiUyQyslMjJuaWNrbmFtZSUyMiUzQSslMjJZYWxpJTIyJTJDKyUyMmlkJTIyJTNBKyUyMjY0NzU2MyUyMiU3RCU3RCUyQyslMjJ2aXNpdG9yJTIyJTNBKyU3QiUyMmNpdHklMjIlM0ErJTIyU2FuK0ZyYW5jaXNjbyUyMiUyQyslMjJraW5kJTIyJTNBKyUyMlZpc2l0b3IlMjIlMkMrJTIyb3JnYW5pemF0aW9uJTIyJTNBKyUyMlZpc2l0b3IrT3JnYW5pemF0aW9uJTIyJTJDKyUyMmNvbnZlcnNhdGlvbkJlZ2luUGFnZSUyMiUzQSslMjJodHRwJTNBJTJGJTJGd3d3Lm9sYXJrLmNvbSUyMiUyQyslMjJjb3VudHJ5Q29kZSUyMiUzQSslMjJVUyUyMiUyQyslMjJyZWZlcnJlciUyMiUzQSslMjJodHRwJTNBJTJGJTJGd3d3Lm9sYXJrLmNvbSUyMiUyQyslMjJpcCUyMiUzQSslMjIxMjcuMC4wLjElMjIlMkMrJTIycmVnaW9uJTIyJTNBKyUyMkNBJTIyJTJDKyUyMmNoYXRfZmVlZGJhY2slMjIlM0ErJTdCJTIyb3ZlcmFsbF9jaGF0JTIyJTNBKzQlMkMrJTIycmVzcG9uc2l2ZW5lc3MlMjIlM0ErNSUyQyslMjJmcmllbmRsaW5lc3MlMjIlM0ErNSUyQyslMjJrbm93bGVkZ2UlMjIlM0ErNCU3RCUyQyslMjJvcGVyYXRpbmdTeXN0ZW0lMjIlM0ErJTIyV2luZG93cyUyMiUyQyslMjJlbWFpbEFkZHJlc3MlMjIlM0ErJTIyc3VwcG9ydCUyQmludGVncmF0aW9udGVzdCU0MG9sYXJrLmNvbSUyMiUyQyslMjJjb3VudHJ5JTIyJTNBKyUyMlVuaXRlZCtTdGF0ZXMlMjIlMkMrJTIycGhvbmVOdW1iZXIlMjIlM0ErJTIyNTU1NTU1NTU1NSUyMiUyQyslMjJmdWxsTmFtZSUyMiUzQSslMjJPbGFyayUyMiUyQyslMjJpZCUyMiUzQSslMjJOT1RBUkVBTFZJU0lUT1JJRFM1TEdsNlFVcksyT2FQUCUyMiUyQyslMjJicm93c2VyJTIyJTNBKyUyMkludGVybmV0K0V4cGxvcmVyKzExJTIyJTdEJTJDKyUyMmlkJTIyJTNBKyUyMk5PVEFSRUFMVFJBTlNDUklQVDVMR2NiVlRhM2hLQlJCJTIyJTJDKyUyMm1hbnVhbGx5U3VibWl0dGVkJTIyJTNBK2ZhbHNlJTdE" - ) + ) val expected = List( "email", @@ -138,7 +137,7 @@ object CljTomcatOlarkEventSpec { null, null, null - ) + ) } class CljTomcatOlarkEventSpec extends Specification with EnrichJobSpec { @@ -154,7 +153,8 @@ class CljTomcatOlarkEventSpec extends Specification with EnrichJobSpec { for (idx <- CljTomcatOlarkEventSpec.expected.indices) { Try(parse(CljTomcatOlarkEventSpec.expected(idx))) match { case Success(parsedJSON) => parse(actual(idx)) must beEqualTo(parsedJSON) - case Failure(msg) => actual(idx) must BeFieldEqualTo(CljTomcatOlarkEventSpec.expected(idx), idx) + case Failure(msg) => + actual(idx) must BeFieldEqualTo(CljTomcatOlarkEventSpec.expected(idx), idx) } } } diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatPagerdutyEventSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatPagerdutyEventSpec.scala index 1687b94a52..fb28fa3ed9 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatPagerdutyEventSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatPagerdutyEventSpec.scala @@ -139,7 +139,7 @@ class CljTomcatPagerdutyEventSpec extends Specification with EnrichJobSpec { override def appName = "clj-tomcat-pagerduty-event" sequential "A job which processes a Clojure-Tomcat file containing a PagerDuty POST raw event representing" + - " 1 valid completed call" should { + " 1 valid completed call" should { runEnrichJob(CljTomcatPagerdutyEventSpec.lines, "clj-tomcat", "2", true, List("geo")) "correctly output 1 completed call" in { diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatPingdomEventSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatPingdomEventSpec.scala index 44251b6033..4e73045a48 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatPingdomEventSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatPingdomEventSpec.scala @@ -139,7 +139,7 @@ class CljTomcatPingdomEventSpec extends Specification with EnrichJobSpec { override def appName = "clj-tomcat-pingdom-event" sequential "A job which processes a Clojure-Tomcat file containing a Pingdom GET raw event representing 1 " + - "valid completed call" should { + "valid completed call" should { runEnrichJob(CljTomcatPingdomEventSpec.lines, "clj-tomcat", "2", true, List("geo")) "correctly output 1 completed call" in { diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatSendgridEventSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatSendgridEventSpec.scala index 7bad223ad1..62d243d845 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatSendgridEventSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatSendgridEventSpec.scala @@ -139,7 +139,7 @@ class CljTomcatSendgridEventSpec extends Specification with EnrichJobSpec { override def appName = "clj-tomcat-sendgrid-event" sequential "A job which processes a Clojure-Tomcat file containing a Sendgrid POST raw event representing " + - "1 valid completed call" should { + "1 valid completed call" should { runEnrichJob(CljTomcatSendgridEventSpec.lines, "clj-tomcat", "2", true, List("geo")) "correctly output 1 completed call" in { diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatStatusGatorEventSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatStatusGatorEventSpec.scala index d37d00322c..03847c04ac 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatStatusGatorEventSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatStatusGatorEventSpec.scala @@ -13,7 +13,6 @@ package com.snowplowanalytics.snowplow.enrich.spark package good - // Specs2 import org.specs2.mutable.Specification @@ -25,7 +24,7 @@ object CljTomcatStatusGatorEventSpec { import EnrichJobSpec._ val lines = Lines( "2017-11-11 15:37:08 - - 54.205.110.234 POST 54.205.110.234 /com.statusgator/v1 200 - Ruby &cv=clj-1.1.0-tom-0.2.0&nuid=61d4eae4-f00f-4a87-8d53-8a396a1a77a6 - - - application%2Fx-www-form-urlencoded c2VydmljZV9uYW1lPUFtYXpvbitXZWIrU2VydmljZXMmZmF2aWNvbl91cmw9aHR0cHMlM0ElMkYlMkZkd3hqZDljZDZyd25vLmNsb3VkZnJvbnQubmV0JTJGZmF2aWNvbnMlMkZhbWF6b24td2ViLXNlcnZpY2VzLmljbyZzdGF0dXNfcGFnZV91cmw9aHR0cCUzQSUyRiUyRnN0YXR1cy5hd3MuYW1hem9uLmNvbSUyRiZob21lX3BhZ2VfdXJsPWh0dHAlM0ElMkYlMkZhd3MuYW1hem9uLmNvbSUyRiZjdXJyZW50X3N0YXR1cz13YXJuJmxhc3Rfc3RhdHVzPXVwJm9jY3VycmVkX2F0PTIwMTctMTEtMTFUMTUlM0EzNiUzQTE4JTJCMDAlM0EwMA" - ) + ) val expected = List( null, @@ -136,7 +135,7 @@ object CljTomcatStatusGatorEventSpec { null, null, null - ) + ) } class CljTomcatStatusGatorEventSpec extends Specification with EnrichJobSpec { diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatTp1SingleEventSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatTp1SingleEventSpec.scala index 056a80f449..28560776aa 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatTp1SingleEventSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatTp1SingleEventSpec.scala @@ -139,7 +139,7 @@ class CljTomcatTp1SingleEventSpec extends Specification with EnrichJobSpec { override def appName = "clj-tomcat-tp1-single-event" sequential "A job which processes a Clojure-Tomcat file containing a GET raw event representing 1 valid " + - "page view" should { + "page view" should { runEnrichJob(CljTomcatTp1SingleEventSpec.lines, "clj-tomcat", "2", true, List("geo")) "correctly output 1 page ping" in { diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatTp1TnuidSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatTp1TnuidSpec.scala index 40c124c378..0518df487d 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatTp1TnuidSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatTp1TnuidSpec.scala @@ -140,7 +140,7 @@ class CljTomcatTp1TnuidSpec extends Specification with EnrichJobSpec { override def appName = "clj-tomcat-tp1-nuid-event" sequential "A job which processes a Clojure-Tomcat file containing a GET raw event representing 1 valid " + - "page view" should { + "page view" should { runEnrichJob(CljTomcatTp1TnuidSpec.lines, "clj-tomcat", "2", true, List("geo")) "correctly output 1 page ping" in { diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatTp2MegaEventsSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatTp2MegaEventsSpec.scala index 4b7efce252..afd55d352b 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatTp2MegaEventsSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatTp2MegaEventsSpec.scala @@ -21,9 +21,9 @@ import org.specs2.mutable.Specification object CljTomcatTp2MegaEventsSpec { val lines = { - val file = getClass.getResource("CljTomcatTp2MegaEventsSpec.line").getFile + val file = getClass.getResource("CljTomcatTp2MegaEventsSpec.line").getFile val source = Source.fromFile(file) - val line = source.mkString + val line = source.mkString source.close() EnrichJobSpec.Lines(line) } @@ -34,7 +34,7 @@ class CljTomcatTp2MegaEventsSpec extends Specification with EnrichJobSpec { override def appName = "clj-tomcat-tp2-mega-events" sequential "A job which processes a Clojure-Tomcat file containing a POST raw event representing 7,500 " + - "valid events" should { + "valid events" should { runEnrichJob(CljTomcatTp2MegaEventsSpec.lines, "clj-tomcat", "2", true, List("geo")) "correctly output 7,500 events" in { diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatTp2MultiEventsSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatTp2MultiEventsSpec.scala index 1742fc295c..60358b7e81 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatTp2MultiEventsSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatTp2MultiEventsSpec.scala @@ -29,17 +29,18 @@ object CljTomcatTp2MultiEventsSpec { val v_tracker = "py-0.5.0" val v_collector = "clj-0.7.0-tom-0.1.0" val user_ipaddress = "9d08dd610683994e914d753f2ae06a1962c63825" - val contexts = """{"data":[{"data":{"osType":"OSX","appleIdfv":"some_appleIdfv","openIdfa":"some_Idfa","carrier":"some_carrier","deviceModel":"large","osVersion":"3.0.0","appleIdfa":"some_appleIdfa","androidIdfa":"some_androidIdfa","deviceManufacturer":"Amstrad"},"schema":"iglu:com.snowplowanalytics.snowplow/mobile_context/jsonschema/1-0-0"},{"data":{"longitude":10,"bearing":50,"speed":16,"altitude":20,"altitudeAccuracy":0.3,"latitudeLongitudeAccuracy":0.5,"latitude":7},"schema":"iglu:com.snowplowanalytics.snowplow/geolocation_context/jsonschema/1-0-0"}],"schema":"iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-0"}""" - val useragent = "python-requests/2.2.1 CPython/3.3.5 Linux/3.2.0-61-generic" - val br_name = "Unknown" - val br_family = "Unknown" - val br_type = "unknown" - val br_renderengine = "OTHER" - val os_name = "Linux" - val os_family = "Linux" - val os_manufacturer = "Other" - val dvce_type = "Computer" - val dvce_ismobile = "0" + val contexts = + """{"data":[{"data":{"osType":"OSX","appleIdfv":"some_appleIdfv","openIdfa":"some_Idfa","carrier":"some_carrier","deviceModel":"large","osVersion":"3.0.0","appleIdfa":"some_appleIdfa","androidIdfa":"some_androidIdfa","deviceManufacturer":"Amstrad"},"schema":"iglu:com.snowplowanalytics.snowplow/mobile_context/jsonschema/1-0-0"},{"data":{"longitude":10,"bearing":50,"speed":16,"altitude":20,"altitudeAccuracy":0.3,"latitudeLongitudeAccuracy":0.5,"latitude":7},"schema":"iglu:com.snowplowanalytics.snowplow/geolocation_context/jsonschema/1-0-0"}],"schema":"iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-0"}""" + val useragent = "python-requests/2.2.1 CPython/3.3.5 Linux/3.2.0-61-generic" + val br_name = "Unknown" + val br_family = "Unknown" + val br_type = "unknown" + val br_renderengine = "OTHER" + val os_name = "Linux" + val os_family = "Linux" + val os_manufacturer = "Other" + val dvce_type = "Computer" + val dvce_ismobile = "0" // 3 events List( @@ -385,7 +386,7 @@ class CljTomcatTp2MultiEventsSpec extends Specification with EnrichJobSpec { override def appName = "clj-tomcat-tp2-multi-events" sequential "A job which processes a Clojure-Tomcat file containing a POST raw event representing " + - "3 events" should { + "3 events" should { runEnrichJob(CljTomcatTp2MultiEventsSpec.lines, "clj-tomcat", "2", true, List("geo")) "correctly output 1 page view and 2 structured events" in { diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatUnbounceEventSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatUnbounceEventSpec.scala index 5791e5f4ab..74e722021c 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatUnbounceEventSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CljTomcatUnbounceEventSpec.scala @@ -30,7 +30,7 @@ object CljTomcatUnbounceEventSpec { import EnrichJobSpec._ val lines = Lines( "2017-11-15 12:12:36 - - 52.91.101.141 POST 50.19.99.184 /com.unbounce/v1 200 - Apache-HttpClient%2F4.5.1+%28Java%2F1.8.0_101%29 &cv=clj-1.1.0-tom-0.2.0&nuid=9965b812-9030-42de-8c18-41213abcf73b - - - application%2Fx-www-form-urlencoded cGFnZV91cmw9aHR0cCUzQSUyRiUyRnVuYm91bmNlcGFnZXMuY29tJTJGd2F5ZmFyaW5nLTE0NyUyRiZwYWdlX25hbWU9V2F5ZmFyaW5nJnBhZ2VfaWQ9NzY0ODE3N2QtNzMyMy00MzMwLWI0ZjktOTk1MWE1MjEzOGI2JnZhcmlhbnQ9YSZkYXRhLmpzb249JTdCJTIydXNlcmZpZWxkMSUyMiUzQSU1QiUyMmFzZGZhc2RmYWQlMjIlNUQlMkMlMjJpcF9hZGRyZXNzJTIyJTNBJTVCJTIyODUuNzMuMzkuMTYzJTIyJTVEJTJDJTIycGFnZV91dWlkJTIyJTNBJTVCJTIyNzY0ODE3N2QtNzMyMy00MzMwLWI0ZjktOTk1MWE1MjEzOGI2JTIyJTVEJTJDJTIydmFyaWFudCUyMiUzQSU1QiUyMmElMjIlNUQlMkMlMjJ0aW1lX3N1Ym1pdHRlZCUyMiUzQSU1QiUyMjEyJTNBMTIrUE0rVVRDJTIyJTVEJTJDJTIyZGF0ZV9zdWJtaXR0ZWQlMjIlM0ElNUIlMjIyMDE3LTExLTE1JTIyJTVEJTJDJTIycGFnZV91cmwlMjIlM0ElNUIlMjJodHRwJTNBJTJGJTJGdW5ib3VuY2VwYWdlcy5jb20lMkZ3YXlmYXJpbmctMTQ3JTJGJTIyJTVEJTJDJTIycGFnZV9uYW1lJTIyJTNBJTVCJTIyV2F5ZmFyaW5nJTIyJTVEJTdEJmRhdGEueG1sPSUzQyUzRnhtbCt2ZXJzaW9uJTNEJTIyMS4wJTIyK2VuY29kaW5nJTNEJTIyVVRGLTglMjIlM0YlM0UlM0Nmb3JtX2RhdGElM0UlM0N1c2VyZmllbGQxJTNFYXNkZmFzZGZhZCUzQyUyRnVzZXJmaWVsZDElM0UlM0NpcF9hZGRyZXNzJTNFODUuNzMuMzkuMTYzJTNDJTJGaXBfYWRkcmVzcyUzRSUzQ3BhZ2VfdXVpZCUzRTc2NDgxNzdkLTczMjMtNDMzMC1iNGY5LTk5NTFhNTIxMzhiNiUzQyUyRnBhZ2VfdXVpZCUzRSUzQ3ZhcmlhbnQlM0VhJTNDJTJGdmFyaWFudCUzRSUzQ3RpbWVfc3VibWl0dGVkJTNFMTIlM0ExMitQTStVVEMlM0MlMkZ0aW1lX3N1Ym1pdHRlZCUzRSUzQ2RhdGVfc3VibWl0dGVkJTNFMjAxNy0xMS0xNSUzQyUyRmRhdGVfc3VibWl0dGVkJTNFJTNDcGFnZV91cmwlM0VodHRwJTNBJTJGJTJGdW5ib3VuY2VwYWdlcy5jb20lMkZ3YXlmYXJpbmctMTQ3JTJGJTNDJTJGcGFnZV91cmwlM0UlM0NwYWdlX25hbWUlM0VXYXlmYXJpbmclM0MlMkZwYWdlX25hbWUlM0UlM0MlMkZmb3JtX2RhdGElM0U" - ) + ) val expected = List( null, @@ -141,7 +141,7 @@ object CljTomcatUnbounceEventSpec { null, null, null - ) + ) } class CljTomcatUnbounceEventSpec extends Specification with EnrichJobSpec { @@ -157,7 +157,8 @@ class CljTomcatUnbounceEventSpec extends Specification with EnrichJobSpec { for (idx <- CljTomcatUnbounceEventSpec.expected.indices) { Try(parse(CljTomcatUnbounceEventSpec.expected(idx))) match { case Success(parsedJSON) => parse(actual(idx)) must beEqualTo(parsedJSON) - case Failure(msg) => actual(idx) must BeFieldEqualTo(CljTomcatUnbounceEventSpec.expected(idx), idx) + case Failure(msg) => + actual(idx) must BeFieldEqualTo(CljTomcatUnbounceEventSpec.expected(idx), idx) } } } diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CollectorPayload1LzoSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CollectorPayload1LzoSpec.scala index 415d2426ce..8baebaeec5 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CollectorPayload1LzoSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CollectorPayload1LzoSpec.scala @@ -27,7 +27,8 @@ import CollectorPayload.thrift.model1.CollectorPayload object CollectorPayload1LzoSpec { import EnrichJobSpec._ - val payloadData = "e=pp&page=Loading%20JSON%20data%20into%20Redshift%20-%20the%20challenges%20of%20quering%20JSON%20data%2C%20and%20how%20Snowplow%20can%20be%20used%20to%20meet%20those%20challenges&pp_mix=0&pp_max=1&pp_miy=64&pp_may=935&cx=eyJkYXRhIjpbeyJzY2hlbWEiOiJpZ2x1OmNvbS5zbm93cGxvd2FuYWx5dGljcy5zbm93cGxvdy91cmlfcmVkaXJlY3QvanNvbnNjaGVtYS8xLTAtMCIsImRhdGEiOnsidXJpIjoiaHR0cDovL3Nub3dwbG93YW5hbHl0aWNzLmNvbS8ifX1dLCJzY2hlbWEiOiJpZ2x1OmNvbS5zbm93cGxvd2FuYWx5dGljcy5zbm93cGxvdy9jb250ZXh0cy9qc29uc2NoZW1hLzEtMC0wIn0=&dtm=1398762054889&tid=612876&vp=1279x610&ds=1279x5614&vid=2&duid=44082d3af0e30126&p=web&tv=js-2.0.0&fp=2071613637&aid=snowplowweb&lang=fr&cs=UTF-8&tz=Europe%2FBerlin&tna=cloudfront&evn=com.snowplowanalytics&refr=http%3A%2F%2Fsnowplowanalytics.com%2Fservices%2Fpipelines.html&f_pdf=1&f_qt=1&f_realp=0&f_wma=0&f_dir=0&f_fla=1&f_java=1&f_gears=0&f_ag=0&res=1280x800&cd=24&cookie=1&url=http%3A%2F%2Fsnowplowanalytics.com%2Fblog%2F2013%2F11%2F20%2Floading-json-data-into-redshift%2F%23weaknesses" + val payloadData = + "e=pp&page=Loading%20JSON%20data%20into%20Redshift%20-%20the%20challenges%20of%20quering%20JSON%20data%2C%20and%20how%20Snowplow%20can%20be%20used%20to%20meet%20those%20challenges&pp_mix=0&pp_max=1&pp_miy=64&pp_may=935&cx=eyJkYXRhIjpbeyJzY2hlbWEiOiJpZ2x1OmNvbS5zbm93cGxvd2FuYWx5dGljcy5zbm93cGxvdy91cmlfcmVkaXJlY3QvanNvbnNjaGVtYS8xLTAtMCIsImRhdGEiOnsidXJpIjoiaHR0cDovL3Nub3dwbG93YW5hbHl0aWNzLmNvbS8ifX1dLCJzY2hlbWEiOiJpZ2x1OmNvbS5zbm93cGxvd2FuYWx5dGljcy5zbm93cGxvdy9jb250ZXh0cy9qc29uc2NoZW1hLzEtMC0wIn0=&dtm=1398762054889&tid=612876&vp=1279x610&ds=1279x5614&vid=2&duid=44082d3af0e30126&p=web&tv=js-2.0.0&fp=2071613637&aid=snowplowweb&lang=fr&cs=UTF-8&tz=Europe%2FBerlin&tna=cloudfront&evn=com.snowplowanalytics&refr=http%3A%2F%2Fsnowplowanalytics.com%2Fservices%2Fpipelines.html&f_pdf=1&f_qt=1&f_realp=0&f_wma=0&f_dir=0&f_fla=1&f_java=1&f_gears=0&f_ag=0&res=1280x800&cd=24&cookie=1&url=http%3A%2F%2Fsnowplowanalytics.com%2Fblog%2F2013%2F11%2F20%2Floading-json-data-into-redshift%2F%23weaknesses" val collectorPayload = new CollectorPayload( "iglu:com.snowplowanalytics.snowplow/CollectorPayload/thrift/1-0-0", "255.255.255.255", @@ -39,7 +40,8 @@ object CollectorPayload1LzoSpec { collectorPayload.setPath("/i") collectorPayload.setHostname("localhost") collectorPayload.setQuerystring(payloadData) - collectorPayload.setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.8 Safari/537.36") + collectorPayload.setUserAgent( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.8 Safari/537.36") collectorPayload.setNetworkUserId("8712a379-4bcb-46ee-815d-85f26540577f") val expected = List( @@ -123,7 +125,7 @@ object CollectorPayload1LzoSpec { "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.8 Safari/537.36", // previously "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.131 Safari/537.36", "Chrome 31", // previously "Chrome" "Chrome", - "31.0.1650.8",// previously "34.0.1847.131" + "31.0.1650.8", // previously "34.0.1847.131" "Browser", "WEBKIT", "fr", @@ -165,24 +167,24 @@ class CollectorPayload1LzoSpec extends Specification with EnrichJobSpec { sequential "A job which processes a RawThrift file containing 1 valid page view" should { - if (!isLzoSupported) "native-lzo not supported" in skipped - else { - val f = writeLzo("input", CollectorPayload1LzoSpec.collectorPayload) - runEnrichJob(f.toString(), "thrift", "1", true, List("geo"), false, false, false, false) + if (!isLzoSupported) "native-lzo not supported" in skipped + else { + val f = writeLzo("input", CollectorPayload1LzoSpec.collectorPayload) + runEnrichJob(f.toString(), "thrift", "1", true, List("geo"), false, false, false, false) - "correctly output 1 page view" in { - val Some(goods) = readPartFile(dirs.output) - goods.size must_== 1 - val actual = goods.head.split("\t").map(s => if (s.isEmpty()) null else s) - for (idx <- CollectorPayload1LzoSpec.expected.indices) { - actual(idx) must BeFieldEqualTo(CollectorPayload1LzoSpec.expected(idx), idx) - } + "correctly output 1 page view" in { + val Some(goods) = readPartFile(dirs.output) + goods.size must_== 1 + val actual = goods.head.split("\t").map(s => if (s.isEmpty()) null else s) + for (idx <- CollectorPayload1LzoSpec.expected.indices) { + actual(idx) must BeFieldEqualTo(CollectorPayload1LzoSpec.expected(idx), idx) } + } - "not write any bad rows" in { - dirs.badRows must beEmptyDir - } + "not write any bad rows" in { + dirs.badRows must beEmptyDir } + } } } diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/Core2015RefreshSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/Core2015RefreshSpec.scala index 2e35d41b7b..fe4efe8226 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/Core2015RefreshSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/Core2015RefreshSpec.scala @@ -160,7 +160,7 @@ class Core2015RefreshSpec extends Specification with EnrichJobSpec { override def appName = "core-2015-refresh" sequential "A job which processes a CloudFront file containing 1 valid page ping with campaign attribution" + - " fields" should { + " fields" should { runEnrichJob(Core2015RefreshSpec.lines, "cloudfront", "1", false, List("geo"), false, true) "correctly output 1 page ping" in { diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CurrencyConversionTransactionItemSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CurrencyConversionTransactionItemSpec.scala index 4c51fb65a3..3c55adeb0b 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CurrencyConversionTransactionItemSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CurrencyConversionTransactionItemSpec.scala @@ -90,12 +90,12 @@ object CurrencyConversionTransactionItemSpec { null, // null, // null, // - "order-123", // Transaction item fields are set - "PBZ1001", // + "order-123", // Transaction item fields are set + "PBZ1001", // "Blue t-shirt", // - "APPAREL", // - "2000", // - "2", // + "APPAREL", // + "2000", // + "2", // null, // Page ping fields are empty null, // null, // @@ -135,9 +135,9 @@ object CurrencyConversionTransactionItemSpec { null, null, null, - "GBP", // ti_currency + "GBP", // ti_currency "2501.12", // ti_price_base - "EUR", // base_currency + "EUR", // base_currency null, null, null, @@ -155,8 +155,13 @@ class CurrencyConversionTransactionItemSpec extends Specification with EnrichJob // Only run currency enrichment tests if the credentials exist in an environment variable if (sys.env.get("OER_KEY").isDefined) { "A job which processes a CloudFront file containing 1 valid transaction item" should { - runEnrichJob(CurrencyConversionTransactionItemSpec.lines, - "cloudfront", "1", false, List("geo"), true) + runEnrichJob( + CurrencyConversionTransactionItemSpec.lines, + "cloudfront", + "1", + false, + List("geo"), + true) "correctly output 1 transaction item" in { val Some(goods) = readPartFile(dirs.output) @@ -172,6 +177,7 @@ class CurrencyConversionTransactionItemSpec extends Specification with EnrichJob } } } else { - println("WARNING: Skipping CurrencyConversionTransactionItemSpec as no OER_KEY environment variable was found for authentication") + println( + "WARNING: Skipping CurrencyConversionTransactionItemSpec as no OER_KEY environment variable was found for authentication") } } diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CurrencyConversionTransactionSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CurrencyConversionTransactionSpec.scala index ccd4422bcf..cb41ef825f 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CurrencyConversionTransactionSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/CurrencyConversionTransactionSpec.scala @@ -83,14 +83,14 @@ object CurrencyConversionTransactionSpec { null, // null, // null, // Unstructured event field empty - "order-123", // Transaction fields are set + "order-123", // Transaction fields are set "psychicbazaar", // - "8000", // - "200", // - "50", // - "London", // - "England", // - "UK", // + "8000", // + "200", // + "50", // + "London", // + "England", // + "UK", // null, // Transaction item fields empty null, // null, // @@ -132,13 +132,13 @@ object CurrencyConversionTransactionSpec { null, // Not set (legacy input lines) null, null, - "USD", // tr_currency + "USD", // tr_currency "6384.46", // tr_total_base - "159.61", // tr_tax_base - "39.90", // tr_shipping_base - null, // ti_currency - null, // ti_price_base - "EUR", // base_currency + "159.61", // tr_tax_base + "39.90", // tr_shipping_base + null, // ti_currency + null, // ti_price_base + "EUR", // base_currency "America/New_York", null, null, @@ -165,8 +165,13 @@ class CurrencyConversionTransactionSpec extends Specification with EnrichJobSpec // Only run currency enrichment tests if the credentials exist in an environment variable if (sys.env.get("OER_KEY").isDefined) { "A job which processes a CloudFront file containing 1 valid transaction" should { - runEnrichJob(CurrencyConversionTransactionSpec.lines, "cloudfront", "4", true, - List("geo", "netspeed"), true) + runEnrichJob( + CurrencyConversionTransactionSpec.lines, + "cloudfront", + "4", + true, + List("geo", "netspeed"), + true) "correctly output 1 transaction" in { val Some(goods) = readPartFile(dirs.output) @@ -182,6 +187,7 @@ class CurrencyConversionTransactionSpec extends Specification with EnrichJobSpec } } } else { - println("WARNING: Skipping CurrencyConversionTransactionSpec as no OER_KEY environment variable was found for authentication") + println( + "WARNING: Skipping CurrencyConversionTransactionSpec as no OER_KEY environment variable was found for authentication") } } diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/DerivedTstampSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/DerivedTstampSpec.scala index fce97f04cb..f5b42cb6ae 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/DerivedTstampSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/DerivedTstampSpec.scala @@ -163,7 +163,7 @@ class DerivedTstampSpec extends Specification with EnrichJobSpec { override def appName = "derive-tstamp" sequential "A job which processes a CloudFront file containing 1 valid page ping with campaign attribution" + - " fields" should { + " fields" should { runEnrichJob(DerivedTstampSpec.lines, "cloudfront", "1", false, List("geo"), false, true) "correctly output 1 page ping" in { diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/ForwardCompatibleSchemaverSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/ForwardCompatibleSchemaverSpec.scala index ce65a638b3..22756f26d1 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/ForwardCompatibleSchemaverSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/ForwardCompatibleSchemaverSpec.scala @@ -29,17 +29,18 @@ object ForwardCompatibleSchemaverSpec { val v_tracker = "py-0.5.0" val v_collector = "clj-0.7.0-tom-0.1.0" val user_ipaddress = "9d08dd610683994e914d753f2ae06a1962c63825" - val contexts = """{"data":[{"data":{"osType":"OSX","appleIdfv":"some_appleIdfv","openIdfa":"some_Idfa","carrier":"some_carrier","deviceModel":"large","osVersion":"3.0.0","appleIdfa":"some_appleIdfa","androidIdfa":"some_androidIdfa","deviceManufacturer":"Amstrad"},"schema":"iglu:com.snowplowanalytics.snowplow/mobile_context/jsonschema/1-0-0"},{"data":{"longitude":10,"bearing":50,"speed":16,"altitude":20,"altitudeAccuracy":0.3,"latitudeLongitudeAccuracy":0.5,"latitude":7},"schema":"iglu:com.snowplowanalytics.snowplow/geolocation_context/jsonschema/1-0-0"}],"schema":"iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-0"}""" - val useragent = "python-requests/2.2.1 CPython/3.3.5 Linux/3.2.0-61-generic" - val br_name = "Unknown" - val br_family = "Unknown" - val br_type = "unknown" - val br_renderengine = "OTHER" - val os_name = "Linux" - val os_family = "Linux" - val os_manufacturer = "Other" - val dvce_type = "Computer" - val dvce_ismobile = "0" + val contexts = + """{"data":[{"data":{"osType":"OSX","appleIdfv":"some_appleIdfv","openIdfa":"some_Idfa","carrier":"some_carrier","deviceModel":"large","osVersion":"3.0.0","appleIdfa":"some_appleIdfa","androidIdfa":"some_androidIdfa","deviceManufacturer":"Amstrad"},"schema":"iglu:com.snowplowanalytics.snowplow/mobile_context/jsonschema/1-0-0"},{"data":{"longitude":10,"bearing":50,"speed":16,"altitude":20,"altitudeAccuracy":0.3,"latitudeLongitudeAccuracy":0.5,"latitude":7},"schema":"iglu:com.snowplowanalytics.snowplow/geolocation_context/jsonschema/1-0-0"}],"schema":"iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-0"}""" + val useragent = "python-requests/2.2.1 CPython/3.3.5 Linux/3.2.0-61-generic" + val br_name = "Unknown" + val br_family = "Unknown" + val br_type = "unknown" + val br_renderengine = "OTHER" + val os_name = "Linux" + val os_family = "Linux" + val os_manufacturer = "Other" + val dvce_type = "Computer" + val dvce_ismobile = "0" // 3 events List( @@ -389,7 +390,7 @@ class ForwardCompatibleSchemaverSpec extends Specification with EnrichJobSpec { override def appName = "forwarrd-compatbile-schemaver" sequential "A job which processes a Clojure-Tomcat file containing a POST raw event representing " + - "3 events" should { + "3 events" should { runEnrichJob(ForwardCompatibleSchemaverSpec.lines, "clj-tomcat", "2", true, List("geo")) "correctly output 1 page view and 2 structured events" in { diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/LzoGoogleAnalyticsEventSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/LzoGoogleAnalyticsEventSpec.scala index 6f4b89e59d..1455aee570 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/LzoGoogleAnalyticsEventSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/LzoGoogleAnalyticsEventSpec.scala @@ -39,7 +39,8 @@ object LzoGoogleAnalyticsEventSpec { collectorPayload.setPath("/com.google.analytics/v1") collectorPayload.setHostname("localhost") collectorPayload.setBody(payloadData) - collectorPayload.setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.8 Safari/537.36") + collectorPayload.setUserAgent( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.8 Safari/537.36") collectorPayload.setNetworkUserId("8712a379-4bcb-46ee-815d-85f26540577f") val expected = List( @@ -123,7 +124,7 @@ object LzoGoogleAnalyticsEventSpec { "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.8 Safari/537.36", // previously "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.131 Safari/537.36", "Chrome 31", // previously "Chrome" "Chrome", - "31.0.1650.8",// previously "34.0.1847.131" + "31.0.1650.8", // previously "34.0.1847.131" "Browser", "WEBKIT", null, @@ -165,23 +166,23 @@ class LzoGoogleAnalyticsEventSpec extends Specification with EnrichJobSpec { sequential "A job which processes a RawThrift file containing 1 valid google analytics page view" should { - if (!isLzoSupported) "native-lzo not supported" in skipped - else { - val f = writeLzo("google-analytics", LzoGoogleAnalyticsEventSpec.collectorPayload) - runEnrichJob(f.toString(), "thrift", "1", true, List("geo"), false, false, false, false) + if (!isLzoSupported) "native-lzo not supported" in skipped + else { + val f = writeLzo("google-analytics", LzoGoogleAnalyticsEventSpec.collectorPayload) + runEnrichJob(f.toString(), "thrift", "1", true, List("geo"), false, false, false, false) - "correctly output 1 google analytics page view" in { - val Some(goods) = readPartFile(dirs.output) - goods.size must_== 1 - val actual = goods.head.split("\t").map(s => if (s.isEmpty()) null else s) - for (idx <- LzoGoogleAnalyticsEventSpec.expected.indices) { - actual(idx) must BeFieldEqualTo(LzoGoogleAnalyticsEventSpec.expected(idx), idx) - } + "correctly output 1 google analytics page view" in { + val Some(goods) = readPartFile(dirs.output) + goods.size must_== 1 + val actual = goods.head.split("\t").map(s => if (s.isEmpty()) null else s) + for (idx <- LzoGoogleAnalyticsEventSpec.expected.indices) { + actual(idx) must BeFieldEqualTo(LzoGoogleAnalyticsEventSpec.expected(idx), idx) } + } - "not write any bad rows" in { - dirs.badRows must beEmptyDir - } + "not write any bad rows" in { + dirs.badRows must beEmptyDir } + } } } diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/NdjsonUrbanAirshipSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/NdjsonUrbanAirshipSpec.scala index ae2b935c91..7016faf655 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/NdjsonUrbanAirshipSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/NdjsonUrbanAirshipSpec.scala @@ -15,7 +15,7 @@ package com.snowplowanalytics.snowplow.enrich.spark package good -import scala.collection.mutable.{ArrayBuffer, ListBuffer, Buffer} +import scala.collection.mutable.{ArrayBuffer, Buffer, ListBuffer} import org.specs2.mutable.Specification @@ -24,8 +24,9 @@ import org.json4s.jackson.JsonMethods._ object NdjsonUrbanAirshipSingleEvent { import EnrichJobSpec._ - val lines = Lines(compact( - parse(""" + val lines = Lines( + compact( + parse(""" |{ | "id": "e3314efb-9058-dbaf-c4bb-b754fca73613", | "offset": "1", @@ -40,8 +41,7 @@ object NdjsonUrbanAirshipSingleEvent { | "type": "CLOSE" |} |""".stripMargin) - ) - ) + )) val expected = List( null, "srv", @@ -169,7 +169,7 @@ object NdjsonUrbanAirshipSingleEvent { null, // dvce_screenheight null, // doc_charset null, // doc_width - null // doc_height + null // doc_height ) } @@ -240,7 +240,7 @@ object NdjsonUrbanAirshipMultiEvent { |}""".stripMargin ) ) - val sampleInAppResolutionEventResponse = compact( + val sampleInAppResolutionEventResponse = compact( parse( """{ | "schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0", @@ -266,31 +266,28 @@ object NdjsonUrbanAirshipMultiEvent { |}""".stripMargin ) ) - val eventSource = "srv" + val eventSource = "srv" val collectorTstamp = "2015-11-13 16:31:52.393" - val eventType = "unstruct" - val adapter = "com.urbanairship.connect-v1" - val loaderType = "ndjson" + val eventType = "unstruct" + val adapter = "com.urbanairship.connect-v1" + val loaderType = "ndjson" val expectedBase = { - val r = ArrayBuffer.fill(NdjsonUrbanAirshipSingleEvent.expected.size)(null:String) - r(1) = eventSource - r(2) = etlTimestamp - r(3) = collectorTstamp - r(5) = eventType - r(9) = adapter + val r = ArrayBuffer.fill(NdjsonUrbanAirshipSingleEvent.expected.size)(null: String) + r(1) = eventSource + r(2) = etlTimestamp + r(3) = collectorTstamp + r(5) = eventType + r(9) = adapter r(10) = loaderType r(11) = etlVersion r.toList } - val lines = Lines(sampleLine, - sampleBlank, - sampleInAppResolutionEvent, - sampleBlank, - sampleBlank) // the blanks should be ignored + val lines = Lines(sampleLine, sampleBlank, sampleInAppResolutionEvent, sampleBlank, sampleBlank) // the blanks should be ignored val expectedJsonOutputIdx = 58 // position of unstruct event json in list - val expected = List(expectedBase.updated(expectedJsonOutputIdx, sampleLineResponse), + val expected = List( + expectedBase.updated(expectedJsonOutputIdx, sampleLineResponse), expectedBase.updated(expectedJsonOutputIdx, sampleInAppResolutionEventResponse)) } @@ -300,8 +297,12 @@ class NdjsonUrbanAirshipSingleSpec extends Specification with EnrichJobSpec { override def appName = "ndjson-urban-airship-single" sequential "A job which processes a NDJSON file with one event" should { - runEnrichJob(NdjsonUrbanAirshipSingleEvent.lines, "ndjson/com.urbanairship.connect/v1", - "2", true, List("geo")) + runEnrichJob( + NdjsonUrbanAirshipSingleEvent.lines, + "ndjson/com.urbanairship.connect/v1", + "2", + true, + List("geo")) "correctly output 1 event" in { val Some(goods) = readPartFile(dirs.output) @@ -323,8 +324,12 @@ class NdjsonUrbanAirshipMultiSpec extends Specification with EnrichJobSpec { override def appName = "ndjson-urban-airship-multi" sequential "A job which processes a NDJSON file with more than one event (but two valid ones)" should { - runEnrichJob(NdjsonUrbanAirshipMultiEvent.lines, "ndjson/com.urbanairship.connect/v1", - "2", true, List("geo")) + runEnrichJob( + NdjsonUrbanAirshipMultiEvent.lines, + "ndjson/com.urbanairship.connect/v1", + "2", + true, + List("geo")) "correctly output 2 events" in { val Some(goods) = readPartFile(dirs.output) diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/PagePingCfLineSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/PagePingCfLineSpec.scala index b0138d1b9c..0143e56613 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/PagePingCfLineSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/PagePingCfLineSpec.scala @@ -96,7 +96,7 @@ object PagePingCfLineSpec { null, // null, // null, // - "21", // Page ping fields are set + "21", // Page ping fields are set "214", // "251", // "517", // diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/RefererParserCfLineSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/RefererParserCfLineSpec.scala index b23c15ccad..d18c212163 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/RefererParserCfLineSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/RefererParserCfLineSpec.scala @@ -143,7 +143,7 @@ class RefererParserCfLineSpec extends Specification with EnrichJobSpec { override def appName = "referer-parser-cf-lines" sequential "A job which processes a CloudFront file containing 1 valid page ping with an internal " + - "subdomain referer" should { + "subdomain referer" should { runEnrichJob(RefererParserCfLineSpec.lines, "cloudfront", "1", false, List("geo")) "correctly output 1 page ping" in { diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/SnowplowRawEventLzoSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/SnowplowRawEventLzoSpec.scala index 44fbd5e19c..083bea3b0b 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/SnowplowRawEventLzoSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/SnowplowRawEventLzoSpec.scala @@ -20,19 +20,24 @@ import java.io.File import org.specs2.mutable.Specification -import collectors.thrift.{SnowplowRawEvent, TrackerPayload, PayloadProtocol, PayloadFormat} +import collectors.thrift.{PayloadFormat, PayloadProtocol, SnowplowRawEvent, TrackerPayload} object SnowplowRawEventLzoSpec { import EnrichJobSpec._ - val payloadData = "e=pp&page=Loading%20JSON%20data%20into%20Redshift%20-%20the%20challenges%20of%20quering%20JSON%20data%2C%20and%20how%20Snowplow%20can%20be%20used%20to%20meet%20those%20challenges&pp_mix=0&pp_max=1&pp_miy=64&pp_may=935&cx=eyJkYXRhIjpbeyJzY2hlbWEiOiJpZ2x1OmNvbS5zbm93cGxvd2FuYWx5dGljcy5zbm93cGxvdy91cmlfcmVkaXJlY3QvanNvbnNjaGVtYS8xLTAtMCIsImRhdGEiOnsidXJpIjoiaHR0cDovL3Nub3dwbG93YW5hbHl0aWNzLmNvbS8ifX1dLCJzY2hlbWEiOiJpZ2x1OmNvbS5zbm93cGxvd2FuYWx5dGljcy5zbm93cGxvdy9jb250ZXh0cy9qc29uc2NoZW1hLzEtMC0wIn0=&dtm=1398762054889&tid=612876&vp=1279x610&ds=1279x5614&vid=2&duid=44082d3af0e30126&p=web&tv=js-2.0.0&fp=2071613637&aid=snowplowweb&lang=fr&cs=UTF-8&tz=Europe%2FBerlin&tna=cloudfront&evn=com.snowplowanalytics&refr=http%3A%2F%2Fsnowplowanalytics.com%2Fservices%2Fpipelines.html&f_pdf=1&f_qt=1&f_realp=0&f_wma=0&f_dir=0&f_fla=1&f_java=1&f_gears=0&f_ag=0&res=1280x800&cd=24&cookie=1&url=http%3A%2F%2Fsnowplowanalytics.com%2Fblog%2F2013%2F11%2F20%2Floading-json-data-into-redshift%2F%23weaknesses" + val payloadData = + "e=pp&page=Loading%20JSON%20data%20into%20Redshift%20-%20the%20challenges%20of%20quering%20JSON%20data%2C%20and%20how%20Snowplow%20can%20be%20used%20to%20meet%20those%20challenges&pp_mix=0&pp_max=1&pp_miy=64&pp_may=935&cx=eyJkYXRhIjpbeyJzY2hlbWEiOiJpZ2x1OmNvbS5zbm93cGxvd2FuYWx5dGljcy5zbm93cGxvdy91cmlfcmVkaXJlY3QvanNvbnNjaGVtYS8xLTAtMCIsImRhdGEiOnsidXJpIjoiaHR0cDovL3Nub3dwbG93YW5hbHl0aWNzLmNvbS8ifX1dLCJzY2hlbWEiOiJpZ2x1OmNvbS5zbm93cGxvd2FuYWx5dGljcy5zbm93cGxvdy9jb250ZXh0cy9qc29uc2NoZW1hLzEtMC0wIn0=&dtm=1398762054889&tid=612876&vp=1279x610&ds=1279x5614&vid=2&duid=44082d3af0e30126&p=web&tv=js-2.0.0&fp=2071613637&aid=snowplowweb&lang=fr&cs=UTF-8&tz=Europe%2FBerlin&tna=cloudfront&evn=com.snowplowanalytics&refr=http%3A%2F%2Fsnowplowanalytics.com%2Fservices%2Fpipelines.html&f_pdf=1&f_qt=1&f_realp=0&f_wma=0&f_dir=0&f_fla=1&f_java=1&f_gears=0&f_ag=0&res=1280x800&cd=24&cookie=1&url=http%3A%2F%2Fsnowplowanalytics.com%2Fblog%2F2013%2F11%2F20%2Floading-json-data-into-redshift%2F%23weaknesses" val payload = new TrackerPayload( - PayloadProtocol.Http, PayloadFormat.HttpGet, payloadData + PayloadProtocol.Http, + PayloadFormat.HttpGet, + payloadData ) - val snowplowRawEvent = new SnowplowRawEvent(1381175274000L, "collector", "UTF-8", "255.255.255.255"); + val snowplowRawEvent = + new SnowplowRawEvent(1381175274000L, "collector", "UTF-8", "255.255.255.255"); snowplowRawEvent.setPayload(payload); snowplowRawEvent.setNetworkUserId("8712a379-4bcb-46ee-815d-85f26540577f") - snowplowRawEvent.setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.8 Safari/537.36"); + snowplowRawEvent.setUserAgent( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.8 Safari/537.36"); val expected = List( "snowplowweb", @@ -115,7 +120,7 @@ object SnowplowRawEventLzoSpec { "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.8 Safari/537.36", // previously "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.131 Safari/537.36", "Chrome 31", // previously "Chrome" "Chrome", - "31.0.1650.8",// previously "34.0.1847.131" + "31.0.1650.8", // previously "34.0.1847.131" "Browser", "WEBKIT", "fr", @@ -180,9 +185,11 @@ class SnowplowRawEventLzoSpec extends Specification with EnrichJobSpec { import com.twitter.elephantbird.mapreduce.io.ThriftWritable import com.twitter.elephantbird.mapreduce.output.LzoThriftBlockOutputFormat import org.apache.hadoop.io.LongWritable - val f = new File(System.getProperty("java.io.tmpdir"), + val f = new File( + System.getProperty("java.io.tmpdir"), s"snowplow-enrich-job-${tag}-${scala.util.Random.nextInt(Int.MaxValue)}") - val rdd = spark.sparkContext.parallelize(Seq(event)) + val rdd = spark.sparkContext + .parallelize(Seq(event)) .map { e => val writable = ThriftWritable.newInstance(classOf[SnowplowRawEvent]) writable.set(e) diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/SqlQueryEnrichmentCfLineSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/SqlQueryEnrichmentCfLineSpec.scala index b1b4b32082..648b6ea379 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/SqlQueryEnrichmentCfLineSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/SqlQueryEnrichmentCfLineSpec.scala @@ -19,7 +19,8 @@ import org.specs2.mutable.Specification object SqlQueryEnrichmentCfLineSpec { import EnrichJobSpec._ - val contexts = """eyJkYXRhIjpbeyJkYXRhIjp7Im9zVHlwZSI6Ik9TWCIsImFwcGxlSWRmdiI6InNvbWVfYXBwbGVJZGZ2Iiwib3BlbklkZmEiOiJzb21lX0lkZmEiLCJjYXJyaWVyIjoiYm9yaXMiLCJkZXZpY2VNb2RlbCI6ImxhcmdlIiwib3NWZXJzaW9uIjoiMy4wLjAiLCJhcHBsZUlkZmEiOiJzb21lX2FwcGxlSWRmYSIsImFuZHJvaWRJZGZhIjoic29tZV9hbmRyb2lkSWRmYSIsImRldmljZU1hbnVmYWN0dXJlciI6IkFtc3RyYWQifSwic2NoZW1hIjoiaWdsdTpjb20uc25vd3Bsb3dhbmFseXRpY3Muc25vd3Bsb3cvbW9iaWxlX2NvbnRleHQvanNvbnNjaGVtYS8xLTAtMCJ9LHsiZGF0YSI6eyJsb25naXR1ZGUiOjEwLCJiZWFyaW5nIjo1MCwic3BlZWQiOjI1LjAsImFsdGl0dWRlIjoyMCwiYWx0aXR1ZGVBY2N1cmFjeSI6MC4zLCJsYXRpdHVkZUxvbmdpdHVkZUFjY3VyYWN5IjowLjUsImxhdGl0dWRlIjo3fSwic2NoZW1hIjoiaWdsdTpjb20uc25vd3Bsb3dhbmFseXRpY3Muc25vd3Bsb3cvZ2VvbG9jYXRpb25fY29udGV4dC9qc29uc2NoZW1hLzEtMC0wIn1dLCJzY2hlbWEiOiJpZ2x1OmNvbS5zbm93cGxvd2FuYWx5dGljcy5zbm93cGxvdy9jb250ZXh0cy9qc29uc2NoZW1hLzEtMC0wIn0=""" + val contexts = + """eyJkYXRhIjpbeyJkYXRhIjp7Im9zVHlwZSI6Ik9TWCIsImFwcGxlSWRmdiI6InNvbWVfYXBwbGVJZGZ2Iiwib3BlbklkZmEiOiJzb21lX0lkZmEiLCJjYXJyaWVyIjoiYm9yaXMiLCJkZXZpY2VNb2RlbCI6ImxhcmdlIiwib3NWZXJzaW9uIjoiMy4wLjAiLCJhcHBsZUlkZmEiOiJzb21lX2FwcGxlSWRmYSIsImFuZHJvaWRJZGZhIjoic29tZV9hbmRyb2lkSWRmYSIsImRldmljZU1hbnVmYWN0dXJlciI6IkFtc3RyYWQifSwic2NoZW1hIjoiaWdsdTpjb20uc25vd3Bsb3dhbmFseXRpY3Muc25vd3Bsb3cvbW9iaWxlX2NvbnRleHQvanNvbnNjaGVtYS8xLTAtMCJ9LHsiZGF0YSI6eyJsb25naXR1ZGUiOjEwLCJiZWFyaW5nIjo1MCwic3BlZWQiOjI1LjAsImFsdGl0dWRlIjoyMCwiYWx0aXR1ZGVBY2N1cmFjeSI6MC4zLCJsYXRpdHVkZUxvbmdpdHVkZUFjY3VyYWN5IjowLjUsImxhdGl0dWRlIjo3fSwic2NoZW1hIjoiaWdsdTpjb20uc25vd3Bsb3dhbmFseXRpY3Muc25vd3Bsb3cvZ2VvbG9jYXRpb25fY29udGV4dC9qc29uc2NoZW1hLzEtMC0wIn1dLCJzY2hlbWEiOiJpZ2x1OmNvbS5zbm93cGxvd2FuYWx5dGljcy5zbm93cGxvdy9jb250ZXh0cy9qc29uc2NoZW1hLzEtMC0wIn0=""" val lines = Lines( s"2012-05-27 11:35:53 DFW3 3343 70.46.123.145 GET d3gs014xn8p70.cloudfront.net /ice.png 200 http://www.psychicbazaar.com/oracles/119-psycards-book-and-deck-starter-pack.html?view=print#detail Mozilla/5.0%20(Windows%20NT%206.1;%20WOW64;%20rv:12.0)%20Gecko/20100101%20Firefox/12.0 &e=ue&cx=$contexts&ue_pr=%7B%22schema%22%3A%22iglu%3Acom.snowplowanalytics.snowplow%2Funstruct_event%2Fjsonschema%2F1-0-0%22%2C%22data%22%3A%7B%22schema%22%3A%22iglu%3Acom.snowplowanalytics.snowplow-website%2Fsignup_form_submitted%2Fjsonschema%2F1-0-0%22%2C%22data%22%3A%7B%22name%22%3A%22Bob%C2%AE%22%2C%22email%22%3A%22alex%2Btest%40snowplowanalytics.com%22%2C%22company%22%3A%22SP%22%2C%22eventsPerMonth%22%3A%22%3C%201%20million%22%2C%22serviceType%22%3A%22unsure%22%7D%7D%7D&dtm=1364230969450&evn=com.acme&tid=598951&vp=2560x934&ds=2543x1420&vid=43&duid=9795bd0203804cd1&p=web&tv=js-0.11.1&fp=2876815413&aid=pbzsite&lang=en-GB&cs=UTF-8&tz=Europe%2FLondon&refr=http%3A%2F%2Fwww.psychicbazaar.com%2F&f_pdf=1&f_qt=0&f_realp=0&f_wma=0&f_dir=0&f_fla=1&f_java=1&f_gears=0&f_ag=1&res=2560x1440&cd=32&cookie=1&url=http%3A%2F%2Fwww.psychicbazaar.com%2F2-tarot-cards" ) @@ -51,7 +52,7 @@ object SqlQueryEnrichmentCfLineSpec { "Florida", null, null, - "nuvox.net", // Using the MaxMind domain lookup service + "nuvox.net", // Using the MaxMind domain lookup service null, "http://www.psychicbazaar.com/2-tarot-cards", null, // No page title for events @@ -159,7 +160,7 @@ object SqlQueryEnrichmentCfLineSpec { /** Check if test running in CI environment */ def continuousIntegration: Boolean = sys.env.get("CI") match { case Some("true") => true - case _ => false + case _ => false } } @@ -171,9 +172,15 @@ class SqlQueryEnrichmentCfLineSpec extends Specification with EnrichJobSpec { // Only run SQL Query Enrichment test if the its started on CI and test Postgres DB has been deployed if (SqlQueryEnrichmentCfLineSpec.continuousIntegration) { "A job which processes a CloudFront file containing 1 valid custom unstructured event, " + - "derived context and custom contexts" should { - runEnrichJob(SqlQueryEnrichmentCfLineSpec.lines, "cloudfront", "1", false, - List("geo", "domain"), apiRequestEnabled = false, sqlQueryEnabled = true) + "derived context and custom contexts" should { + runEnrichJob( + SqlQueryEnrichmentCfLineSpec.lines, + "cloudfront", + "1", + false, + List("geo", "domain"), + apiRequestEnabled = false, + sqlQueryEnabled = true) "correctly attach derived context fetched from SQL DataBase" in { val Some(goods) = readPartFile(dirs.output) @@ -189,6 +196,7 @@ class SqlQueryEnrichmentCfLineSpec extends Specification with EnrichJobSpec { } } } else { - println("WARNING: Skipping SqlQueryEnrichmentCfLineSpec as no CI=true environment variable was found") + println( + "WARNING: Skipping SqlQueryEnrichmentCfLineSpec as no CI=true environment variable was found") } } diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/StructEventCfLineSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/StructEventCfLineSpec.scala index a09867add9..ff4a681e35 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/StructEventCfLineSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/StructEventCfLineSpec.scala @@ -76,11 +76,11 @@ object StructEventCfLineSpec { null, // null, // null, // No custom contexts - "ecomm", // Structured event fields are set + "ecomm", // Structured event fields are set "add-to-basket", // - "Χαριτίνη", // Check Unicode handling - "1", // - "35708.23", // + "Χαριτίνη", // Check Unicode handling + "1", // + "35708.23", // null, // Unstructured event field empty null, // Transaction fields empty null, // diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/TransactionCfLineSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/TransactionCfLineSpec.scala index 9f4672b54a..b259b235aa 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/TransactionCfLineSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/TransactionCfLineSpec.scala @@ -82,14 +82,14 @@ object TransactionCfLineSpec { null, // null, // null, // Unstructured event field empty - "order-123", // Transaction fields are set + "order-123", // Transaction fields are set "psychicbazaar", // - "8000", // - "200", // - "50", // - "London", // - "England", // - "UK", // + "8000", // + "200", // + "50", // + "London", // + "England", // + "UK", // null, // Transaction item fields empty null, // null, // @@ -130,7 +130,7 @@ object TransactionCfLineSpec { "1080", null, // Not set (legacy input lines) null, // - null // + null // ) } diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/TransactionItemCfLineSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/TransactionItemCfLineSpec.scala index 28d7aef3d0..106c9d8515 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/TransactionItemCfLineSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/TransactionItemCfLineSpec.scala @@ -90,12 +90,12 @@ object TransactionItemCfLineSpec { null, // null, // null, // - "order-123", // Transaction item fields are set - "PBZ1001", // + "order-123", // Transaction item fields are set + "PBZ1001", // "Blue t-shirt", // - "APPAREL", // - "2000", // - "2", // + "APPAREL", // + "2000", // + "2", // null, // Page ping fields are empty null, // null, // @@ -130,7 +130,7 @@ object TransactionItemCfLineSpec { "1080", null, // Not set (legacy input lines) null, // - null // + null // ) } diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/UnstructEventBase64CfLineSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/UnstructEventBase64CfLineSpec.scala index 4adc89e2f9..9187c4b322 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/UnstructEventBase64CfLineSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/UnstructEventBase64CfLineSpec.scala @@ -50,7 +50,7 @@ object UnstructEventBase64CfLineSpec { "Florida", null, null, - "nuvox.net", // Using the MaxMind domain lookup service + "nuvox.net", // Using the MaxMind domain lookup service null, "http://www.psychicbazaar.com/2-tarot-cards", null, // No page title for events @@ -143,7 +143,11 @@ class UnstructEventBase64CfLineSpec extends Specification with EnrichJobSpec { override def appName = "unstruct-event-base64-cf-lines" sequential "A job which processes a CloudFront file containing 1 valid custom unstructured event" should { - runEnrichJob(UnstructEventBase64CfLineSpec.lines, "cloudfront", "1", false, + runEnrichJob( + UnstructEventBase64CfLineSpec.lines, + "cloudfront", + "1", + false, List("geo", "domain")) "correctly output 1 custom unstructured event" in { diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/UnstructEventCfLineSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/UnstructEventCfLineSpec.scala index 6ed6902168..53df0007e8 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/UnstructEventCfLineSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/UnstructEventCfLineSpec.scala @@ -50,7 +50,7 @@ object UnstructEventCfLineSpec { "Florida", null, null, - "nuvox.net", // Using the MaxMind domain lookup service + "nuvox.net", // Using the MaxMind domain lookup service null, "http://www.psychicbazaar.com/2-tarot-cards", null, // No page title for events diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/WebDistributionLoader23Spec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/WebDistributionLoader23Spec.scala index 31147964db..3d47a9de48 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/WebDistributionLoader23Spec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/WebDistributionLoader23Spec.scala @@ -144,8 +144,12 @@ class WebDistributionLoader23Spec extends Specification with EnrichJobSpec { override def appName = "web-distribution-loader-23" sequential "A job which processes a CloudFront web distribution log file" should { - runEnrichJob(WebDistributionLoader23Spec.lines, "tsv/com.amazon.aws.cloudfront/wd_access_log", - "1", false, List("geo")) + runEnrichJob( + WebDistributionLoader23Spec.lines, + "tsv/com.amazon.aws.cloudfront/wd_access_log", + "1", + false, + List("geo")) "correctly output 1 page ping" in { val Some(goods) = readPartFile(dirs.output) diff --git a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/WebDistributionLoaderSpec.scala b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/WebDistributionLoaderSpec.scala index bbc79cdb2b..ce61ba6c92 100644 --- a/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/WebDistributionLoaderSpec.scala +++ b/3-enrich/spark-enrich/src/test/scala/com.snowplowanalytics.snowplow.enrich.spark/good/WebDistributionLoaderSpec.scala @@ -144,8 +144,12 @@ class WebDistributionLoaderSpec extends Specification with EnrichJobSpec { override def appName = "web-distribution-loader" sequential "A job which processes a CloudFront web distribution log file" should { - runEnrichJob(WebDistributionLoaderSpec.lines, "tsv/com.amazon.aws.cloudfront/wd_access_log", - "1", false, List("geo")) + runEnrichJob( + WebDistributionLoaderSpec.lines, + "tsv/com.amazon.aws.cloudfront/wd_access_log", + "1", + false, + List("geo")) "correctly output 1 page ping" in { val Some(goods) = readPartFile(dirs.output) goods.size must_== 1