Merge pull request #4254 from snowplow/release/r118-morgantina

Release R118 Morgantina - Bad rows
snowplow · Jan 16, 2020 · 3b8d9cc · 3b8d9cc
2 parents 95f938f + 44a2769
commit 3b8d9cc
Show file tree

Hide file tree

Showing 409 changed files with 27,005 additions and 20,450 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -1,7 +1,7 @@
 dist: trusty
 language: scala
 scala:
-  - 2.11.12
+  - 2.12.10
 jdk:
   - oraclejdk8
 before_script:
@@ -70,8 +70,6 @@ before_install:
   - gem uninstall -i /home/travis/.rvm/gems/jruby-9.1.6.0 bundler -a -x
   - gem install bundler -v 1.15.4
   - docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD
-addons:
-  postgresql: '9.3'
 services:
   - docker
   - postgresql
@@ -111,12 +109,6 @@ deploy:
     on:
       tags: true
       condition: '"$(.travis/is_release_tag.sh beam_enrich $TRAVIS_TAG)" == "" && $? == 0 && $TEST_DIR = 3-enrich/beam-enrich/'
-  - provider: script
-    script: ./.travis/deploy.sh hadoop_event_recovery $TRAVIS_TAG
-    skip_cleanup: true
-    on:
-      tags: true
-      condition: '$([[ "$TEST_DIR" == "3-enrich/scala-common-enrich/" ]] && .travis/is_release_tag.sh hadoop_event_recovery $TRAVIS_TAG && [ $? -eq 0 ] && echo "Deploying")'
   - provider: script
     script: ./.travis/deploy.sh emr $TRAVIS_TAG
     skip_cleanup: true

diff --git a/.travis/release_hadoop_event_recovery.yml b/.travis/release_hadoop_event_recovery.yml
diff --git a/.travis/release_scala_stream_collector.yml b/.travis/release_scala_stream_collector.yml
@@ -32,19 +32,19 @@ packages :
         prefix : "snowplow_scala_stream_collector_kinesis_"
         suffix : ""
         binary_paths :
-          - "2-collectors/scala-stream-collector/kinesis/target/scala-2.11/snowplow-stream-collector-kinesis-{{ packages.0.build_version }}.jar"
+          - "2-collectors/scala-stream-collector/kinesis/target/scala-2.12/snowplow-stream-collector-kinesis-{{ packages.0.build_version }}.jar"
       - type : "zip"
         prefix : "snowplow_scala_stream_collector_google_pubsub_"
         suffix : ""
         binary_paths :
-          - "2-collectors/scala-stream-collector/pubsub/target/scala-2.11/snowplow-stream-collector-google-pubsub-{{ packages.0.build_version }}.jar"
+          - "2-collectors/scala-stream-collector/pubsub/target/scala-2.12/snowplow-stream-collector-google-pubsub-{{ packages.0.build_version }}.jar"
       - type : "zip"
         prefix : "snowplow_scala_stream_collector_kafka_"
         suffix : ""
         binary_paths :
-          - "2-collectors/scala-stream-collector/kafka/target/scala-2.11/snowplow-stream-collector-kafka-{{ packages.0.build_version }}.jar"
+          - "2-collectors/scala-stream-collector/kafka/target/scala-2.12/snowplow-stream-collector-kafka-{{ packages.0.build_version }}.jar"
       - type : "zip"
         prefix : "snowplow_scala_stream_collector_nsq_"
         suffix : ""
         binary_paths :
-          - "2-collectors/scala-stream-collector/nsq/target/scala-2.11/snowplow-stream-collector-nsq-{{ packages.0.build_version }}.jar"
+          - "2-collectors/scala-stream-collector/nsq/target/scala-2.12/snowplow-stream-collector-nsq-{{ packages.0.build_version }}.jar"
diff --git a/.travis/release_spark_enrich.yml b/.travis/release_spark_enrich.yml
@@ -77,4 +77,4 @@ packages:
         prefix : "snowplow-spark-enrich-"
         suffix : ".jar"
         binary_paths:
-          - "3-enrich/spark-enrich/target/scala-2.11/snowplow-spark-enrich-{{ packages.0.build_version }}.jar"
+          - "3-enrich/spark-enrich/target/scala-2.12/snowplow-spark-enrich-{{ packages.0.build_version }}.jar"
diff --git a/.travis/release_stream_enrich.yml b/.travis/release_stream_enrich.yml
@@ -31,14 +31,14 @@ packages :
         prefix : "snowplow_stream_enrich_kinesis_"
         suffix : ""
         binary_paths :
-          - "3-enrich/stream-enrich/kinesis/target/scala-2.11/snowplow-stream-enrich-kinesis-{{ packages.0.build_version }}.jar"
+          - "3-enrich/stream-enrich/kinesis/target/scala-2.12/snowplow-stream-enrich-kinesis-{{ packages.0.build_version }}.jar"
       - type : "zip"
         prefix : "snowplow_stream_enrich_kafka_"
         suffix : ""
         binary_paths :
-          - "3-enrich/stream-enrich/kafka/target/scala-2.11/snowplow-stream-enrich-kafka-{{ packages.0.build_version }}.jar"
+          - "3-enrich/stream-enrich/kafka/target/scala-2.12/snowplow-stream-enrich-kafka-{{ packages.0.build_version }}.jar"
       - type : "zip"
         prefix : "snowplow_stream_enrich_nsq_"
         suffix : ""
         binary_paths :
-          - "3-enrich/stream-enrich/nsq/target/scala-2.11/snowplow-stream-enrich-nsq-{{ packages.0.build_version }}.jar"
+          - "3-enrich/stream-enrich/nsq/target/scala-2.12/snowplow-stream-enrich-nsq-{{ packages.0.build_version }}.jar"
diff --git a/2-collectors/scala-stream-collector/.scalafmt.conf b/2-collectors/scala-stream-collector/.scalafmt.conf
@@ -0,0 +1,21 @@
+version = "2.0.0-RC6"
+style = default
+align = none
+maxColumn = 100
+docstrings = JavaDoc
+optIn.breakChainOnFirstMethodDot = true
+spaces.afterKeywordBeforeParen = true
+continuationIndent.callSite = 2
+continuationIndent.defnSite = 2
+verticalMultiline.atDefnSite = true
+verticalMultiline.arityThreshold = 3
+verticalMultiline.newlineAfterOpenParen = true
+verticalMultiline.newlineBeforeImplicitKW = true
+verticalMultiline.excludeDanglingParens = []
+importSelectors = noBinPack
+rewrite.rules = [
+  AsciiSortImports,
+  RedundantBraces,
+  RedundantParens,
+  PreferCurlyFors
+]
diff --git a/2-collectors/scala-stream-collector/build.sbt b/2-collectors/scala-stream-collector/build.sbt
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013-2019 Snowplow Analytics Ltd. All rights reserved.
+ * Copyright (c) 2013-2020 Snowplow Analytics Ltd. All rights reserved.
  *
  * This program is licensed to you under the Apache License Version 2.0, and
  * you may not use this file except in compliance with the Apache License
@@ -16,7 +16,7 @@ import com.typesafe.sbt.packager.docker._
 
 lazy val commonDependencies = Seq(
   // Java
-  Dependencies.Libraries.yodaTime,
+  Dependencies.Libraries.jodaTime,
   Dependencies.Libraries.slf4j,
   Dependencies.Libraries.log4jOverSlf4j,
   Dependencies.Libraries.config,
@@ -27,27 +27,24 @@ lazy val commonDependencies = Seq(
   Dependencies.Libraries.scalaz7,
   Dependencies.Libraries.akkaStream,
   Dependencies.Libraries.akkaHttp,
+  Dependencies.Libraries.akkaStream,
   Dependencies.Libraries.akkaSlf4j,
-  Dependencies.Libraries.json4sJackson,
-  Dependencies.Libraries.snowplowCommonEnrich,
+  Dependencies.Libraries.badRows,
   Dependencies.Libraries.collectorPayload,
   Dependencies.Libraries.pureconfig,
   // Scala (test)
   Dependencies.Libraries.akkaTestkit,
   Dependencies.Libraries.akkaHttpTestkit,
+  Dependencies.Libraries.akkaStreamTestkit,
   Dependencies.Libraries.specs2
 )
 
 lazy val buildSettings = Seq(
   organization  :=  "com.snowplowanalytics",
   name          :=  "snowplow-stream-collector",
-  version       :=  "0.17.0",
+  version       :=  "1.0.0",
   description   :=  "Scala Stream Collector for Snowplow raw events",
-  scalaVersion  :=  "2.11.12",
-  scalacOptions :=  BuildSettings.compilerOptions,
-  scalacOptions in (Compile, console) ~= { _.filterNot(Set("-Ywarn-unused-import")) },
-  scalacOptions in (Test, console)    := (scalacOptions in (Compile, console)).value,
-  javacOptions  :=  BuildSettings.javaCompilerOptions,
+  scalaVersion  :=  "2.12.10",
   resolvers     ++= Dependencies.resolutionRepos
 )
 
@@ -60,6 +57,7 @@ lazy val dockerSettings = Seq(
 
 lazy val allSettings = buildSettings ++
   BuildSettings.sbtAssemblySettings ++
+  BuildSettings.formatting ++
   Seq(libraryDependencies ++= commonDependencies) ++
   dockerSettings
 

diff --git a/...core/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/Collector.scala b/...core/src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/Collector.scala
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013-2019 Snowplow Analytics Ltd. All rights reserved.
+ * Copyright (c) 2013-2020 Snowplow Analytics Ltd. All rights reserved.
  *
  * This program is licensed to you under the Apache License Version 2.0, and
  * you may not use this file except in compliance with the Apache License
@@ -12,9 +12,7 @@
  * implied.  See the Apache License Version 2.0 for the specific language
  * governing permissions and limitations there under.
  */
-package com.snowplowanalytics.snowplow
-package collectors
-package scalastream
+package com.snowplowanalytics.snowplow.collectors.scalastream
 
 import java.io.File
 
@@ -24,18 +22,24 @@ import akka.http.scaladsl.model.StatusCodes
 import akka.http.scaladsl.server.Route
 import akka.http.scaladsl.server.Directives._
 import akka.stream.ActorMaterializer
-import com.snowplowanalytics.snowplow.collectors.scalastream.metrics._
-import com.snowplowanalytics.snowplow.collectors.scalastream.model._
 import com.typesafe.config.{Config, ConfigFactory}
 import com.typesafe.sslconfig.akka.AkkaSSLConfig
 import org.slf4j.LoggerFactory
 import pureconfig._
+import pureconfig.generic.{FieldCoproductHint, ProductHint}
+import pureconfig.generic.auto._
+
+import metrics._
+import model._
 
 // Main entry point of the Scala collector.
 trait Collector {
 
   lazy val log = LoggerFactory.getLogger(getClass())
 
+  implicit def hint[T] = ProductHint[T](ConfigFieldMapping(CamelCase, CamelCase))
+  implicit val _ = new FieldCoproductHint[SinkConfig]("enabled")
+
   def parseConfig(args: Array[String]): (CollectorConfig, Config) = {
     case class FileConfig(config: File = new File("."))
     val parser = new scopt.OptionParser[FileConfig](generated.BuildInfo.name) {
@@ -60,8 +64,6 @@ trait Collector {
       System.exit(1)
     }
 
-    implicit def hint[T] = ProductHint[T](ConfigFieldMapping(CamelCase, CamelCase))
-    implicit val sinkConfigHint = new FieldCoproductHint[SinkConfig]("enabled")
     (loadConfigOrThrow[CollectorConfig](conf.getConfig("collector")), conf)
   }
 
@@ -125,14 +127,17 @@ trait Collector {
       bind(routes, collectorConf.interface, collectorConf.port)
 
     collectorConf.ssl match {
-      case SSLConfig(true, true, port) =>
+      case SSLConfig(true, true, _) =>
         unsecureEndpoint(redirectRoutes)
         secureEndpoint
-      case SSLConfig(true, false, port) =>
+        ()
+      case SSLConfig(true, false, _) =>
         unsecureEndpoint(routes)
         secureEndpoint
+        ()
       case _ =>
         unsecureEndpoint(routes)
+        ()
     }
   }
 }
diff --git a/...src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/CollectorRoute.scala b/...src/main/scala/com.snowplowanalytics.snowplow.collectors.scalastream/CollectorRoute.scala
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013-2019 Snowplow Analytics Ltd. All rights reserved.
+ * Copyright (c) 2013-2020 Snowplow Analytics Ltd. All rights reserved.
  *
  * This program is licensed to you under the Apache License Version 2.0, and
  * you may not use this file except in compliance with the Apache License
@@ -18,7 +18,8 @@ import akka.http.scaladsl.model._
 import akka.http.scaladsl.model.headers.HttpCookiePair
 import akka.http.scaladsl.server.{Directive1, Route}
 import akka.http.scaladsl.server.Directives._
-import com.snowplowanalytics.snowplow.collectors.scalastream.model.DntCookieMatcher
+
+import model.DntCookieMatcher
 import monitoring.BeanRegistry
 
 trait CollectorRoute {
@@ -91,7 +92,7 @@ trait CollectorRoute {
             } ~
             path("""ice\.png""".r | "i".r) { path =>
               (get | head) {
-                val (r,l) = collectorService.cookie(
+                val (r, _) = collectorService.cookie(
                   qs,
                   None,
                   "/" + path,
@@ -134,7 +135,7 @@ trait CollectorRoute {
     */
   def cookieIfWanted(name: Option[String]): Directive1[Option[HttpCookiePair]] = name match {
     case Some(n) => optionalCookie(n)
-    case None => optionalHeaderValue(x => None)
+    case None => optionalHeaderValue(_ => None)
   }
 
   /**
@@ -150,13 +151,13 @@ trait CollectorRoute {
     }
 
   private def crossDomainRoute: Route = get {
-    path("""crossdomain\.xml""".r) { path =>
+    path("""crossdomain\.xml""".r) { _ =>
       complete(collectorService.flashCrossDomainPolicy)
     }
   }
 
   private def healthRoute: Route = get {
-    path("health".r) { path =>
+    path("health".r) { _ =>
       complete(HttpResponse(200, entity = "OK"))
     }
   }