Merge branch 'release/r11'

snowplow · Jan 16, 2019 · 116f529 · 116f529
2 parents adc6775 + 697de5d
commit 116f529
Show file tree

Hide file tree

Showing 172 changed files with 6,285 additions and 6,337 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -4,33 +4,41 @@ scala:
 jdk:
 - oraclejdk8
 script:
+- .travis/publish_local.sh
 - cd 2-repositories/iglu-server
-- ./scripts/create-test-user.bash
+- "./scripts/create-test-user.bash"
+- sbt test
 - cd ../..
 before_deploy:
-  - pip install --user release-manager==0.3.0
+- pip install --user release-manager==0.3.0
 deploy:
 - provider: script
   skip_cleanup: true
-  script: ./.travis/deploy_scala_core.sh $TRAVIS_TAG
+  script: "./.travis/deploy_scala_core.sh $TRAVIS_TAG"
   on:
     condition: '"$(.travis/is_core_release_tag.sh $TRAVIS_TAG)" == "" && $? == 0'
     tags: true
 - provider: script
   skip_cleanup: true
-  script: ./.travis/deploy_schema_ddl.sh $TRAVIS_TAG
+  script: "./.travis/deploy_schema_ddl.sh $TRAVIS_TAG"
   on:
     condition: '"$(.travis/is_schemaddl_release_tag.sh $TRAVIS_TAG)" == "" && $? == 0'
     tags: true
 - provider: script
   skip_cleanup: true
-  script: ./.travis/deploy_igluctl.sh $TRAVIS_TAG
+  script: "./.travis/deploy_igluctl.sh $TRAVIS_TAG"
   on:
     condition: '"$(.travis/is_igluctl_release_tag.sh $TRAVIS_TAG)" == "" && $? == 0'
     tags: true
 - provider: script
   skip_cleanup: true
-  script: ./.travis/deploy_iglu_server.sh $TRAVIS_TAG
+  script: "./.travis/deploy_iglu_server.sh $TRAVIS_TAG"
+  on:
+    condition: '"$(.travis/is_iglu_server_release_tag.sh $TRAVIS_TAG)" == "" && $? == 0'
+    tags: true
+- provider: script
+  skip_cleanup: true
+  script: "./.travis/deploy_iglu_server_docker.sh $TRAVIS_TAG"
   on:
     condition: '"$(.travis/is_iglu_server_release_tag.sh $TRAVIS_TAG)" == "" && $? == 0'
     tags: true
@@ -42,3 +50,5 @@ env:
   - secure: JycPiwXieJVIgpN1O1tJrnK3UPeq0JICe2MYVylMSVK4/VxxpvA8a9Op1DOb8yYg5Jws1t48nmJaiW+vseiIU2AD/9I2YKZ7dAuU8MgGe644QvCXiy/H3KNw7vO5G0zHJuRmVGvzIrCqv34OKp+V9jk3zXclJEB7csqlt1PSkcE=
   - secure: J4P9lwDqK0vcKOSsLodMOH2ob8lRySQCyK3gBZ4dQ23lMw3AvnuHO4OlZoBgO6l5We2dSJMduDq8cLvSvV7sZwmSEesz2VUgnaT87ciDG4yi+TWqebbvGkV/gkvoVhe6AnIOx9sM8WLg5XdwFpZoV6Ss6QLImG+Tg4xrxJr7nCU=
   - secure: PlkOwsrnw7u1LmLsdOabxCTmHWoziFy3rQCMbbykOyhKE9QutobONYSVJinQ0LnkCorXd+asr8EfyC+UCAUqyPVWnzF+OLUrcHGgcsIlxuNH6sBkuqpufH0jc526uYDIpR0tSvCrQLRc4ZtP456cBBflPeb2Qy3BYt5BVapGusg=
+  - secure: YdDMiFhKvtNlsVlNbDdD9Yh/iam24mAcGZ7erpjlMhvnbp0kp6r7fC1QjUtsmEu9rVmUqPqhzdhw36Qz4DgHY320PdOjFuUpmw4uW0SFFS+h6nmk6KwOVdjWgWvFqubtiqQRKE0Z1uhmaTbaa/uHV/AbFTza04YO2Ce01eMDsg4=
+  - secure: e0jUVwwpiONjT1sywAbi5u9ERkErInz0T940s5OnVBQW+KWc71b61GExFBDQ+bkcCGjhUc/e9mxxh5ryUZUdY0X8aGHHvmztUSceHLss4mu8OoiG2gAcHOGsBUkBMsAsgTE08Bzzss4BwolfL8rODo4uuSzR/Snz9m9+ynEtydk=
diff --git a/.travis/deploy_iglu_server_docker.sh b/.travis/deploy_iglu_server_docker.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -e
+
+echo "Deploying Iglu Server image"
+
+tag=$1
+
+file="${HOME}/.dockercfg"
+docker_repo="snowplow-docker-registry.bintray.io"
+curl -X GET \
+    -u${BINTRAY_SNOWPLOW_DOCKER_USER}:${BINTRAY_SNOWPLOW_DOCKER_API_KEY} \
+    https://${docker_repo}/v2/auth > $file
+
+cd $TRAVIS_BUILD_DIR/2-repositories/iglu-server
+
+project_version=$(sbt version -Dsbt.log.noformat=true | perl -ne 'print "$1\n" if /info.*(\d+\.\d+\.\d+[^\r\n]*)/' | tail -n 1 | tr -d '\n')
+if [[ "${tag}" = *"${project_version}" ]]; then
+    sbt docker:publishLocal
+    formatted_tag="${tag////:}"
+    docker push "${docker_repo}/snowplow/${formatted_tag//_/-}"
+else
+    echo "Tag version '${tag}' doesn't match version in scala project ('${project_version}'). Aborting!"
+    exit 1
+fi
+
diff --git a/.travis/deploy_igluctl.sh b/.travis/deploy_igluctl.sh
@@ -25,11 +25,15 @@ project_version=$(sbt version -Dsbt.log.noformat=true | tail -n 1 | perl -ne 'pr
 if [ "${project_version}" == "${release}" ]; then
     # local publish only dependency, scala-core
     cd "${TRAVIS_BUILD_DIR}/0-common/scala-core"
+    echo "DEPLOY: localPublish iglu-core (for igluctl)..."
     sbt +publishLocal
+    echo "DEPLOY: localPublish iglu-core-circe (for igluctl)..."
     sbt "project igluCoreCirce" +publishLocal --warn
+    echo "DEPLOY: localPublish iglu-core-json4s (for igluctl)..."
     sbt "project igluCoreJson4s" +publishLocal --warn
     # universal publish schema-ddl
     cd "${TRAVIS_BUILD_DIR}/0-common/schema-ddl"
+    echo "DEPLOY: localPublish schema-ddl (for igluctl)..."
     sbt +publishLocal --warn
 else
     echo "Tag version '${release}' doesn't match version in scala project ('${project_version}'). Aborting!"
@@ -44,3 +48,5 @@ release-manager \
     --make-version \
     --make-artifact \
     --upload-artifact
+
+echo "DEPLOY: igluctl deployed..."
diff --git a/.travis/deploy_scala_core.sh b/.travis/deploy_scala_core.sh
@@ -1,7 +1,6 @@
 #!/bin/bash
 
-set -e
-
+set -e 
 tag=$1
 
 project="scala-core/"
@@ -10,6 +9,8 @@ project_len=${#project}
 cicd=${tag:0:${project_len}}
 release=${tag:${project_len}}
 
+limit=30    # Timeout to extend Travis build time (from 10 mins)
+
 if [ "${cicd}" == "${project}" ]; then
     if [ "${release}" == "" ]; then
         echo "WARNING! No release specified! Ignoring."
@@ -33,19 +34,49 @@ cd "${TRAVIS_BUILD_DIR}/0-common/scala-core"
 
 project_version=$(sbt version -Dsbt.log.noformat=true | tail -n 1 | perl -ne 'print $1 if /(\d+\.\d+[^\r\n]*)/')
 
+function travis_wait {
+  minutes=0
+  while kill -0 $! >/dev/null 2>&1; do
+    echo -n -e " \b" # never leave evidences!
+
+    if [ $minutes == $limit ]; then
+      break;
+    fi
+
+    minutes=$((minutes+1))
+
+    sleep 60
+  done
+}
+
 if [ "${project_version}" == "${release}" ]; then
     # igluCore
+    echo "DEPLOY: testing iglu-core..."
     sbt +test --warn
+    echo "DEPLOY: publishing iglu-core..."
     sbt +publish
-    sbt +bintraySyncMavenCentral
+    echo "DEPLOY: publishing iglu-core to Maven Central..."
+    sbt +bintraySyncMavenCentral &
+    travis_wait
+
     # igluCoreCirce
+    echo "DEPLOY: testing iglu-core-circe..."
     sbt "project igluCoreCirce" +test --warn
+    echo "DEPLOY: publishing iglu-core-circe..."
     sbt "project igluCoreCirce" +publish
-    sbt "project igluCoreCirce" +bintraySyncMavenCentral
+    echo "DEPLOY: publishing iglu-core-circe to Maven Central..."
+    sbt "project igluCoreCirce" +bintraySyncMavenCentral &
+    travis_wait
+
     # igluCoreJson4s
+    echo "DEPLOY: testing iglu-core-json4s..."
     sbt "project igluCoreJson4s" +test --warn
+    echo "DEPLOY: publishing iglu-core-json4s..."
     sbt "project igluCoreJson4s" +publish
-    sbt "project igluCoreJson4s" +bintraySyncMavenCentral
+    echo "DEPLOY: publishing iglu-core-json4s to Maven Central..."
+    sbt "project igluCoreJson4s" +bintraySyncMavenCentral &
+    travis_wait
+    echo "DEPLOY: Iglu Core deployed..."
 
 
 else

diff --git a/.travis/deploy_schema_ddl.sh b/.travis/deploy_schema_ddl.sh
@@ -10,6 +10,8 @@ project_len=${#project}
 cicd=${tag:0:${project_len}}
 release=${tag:${project_len}}
 
+limit=30    # Timeout to extend Travis build time (from 10 mins)
+
 if [ "${cicd}" == "${project}" ]; then
     if [ "${release}" == "" ]; then
         echo "WARNING! No release specified! Ignoring."
@@ -33,17 +35,40 @@ cd "${TRAVIS_BUILD_DIR}/0-common/schema-ddl"
 
 project_version=$(sbt version -Dsbt.log.noformat=true | tail -n 1 | perl -ne 'print $1 if /(\d+\.\d+[^\r\n]*)/')
 
+function travis_wait {
+  minutes=0
+  while kill -0 $! >/dev/null 2>&1; do
+    echo -n -e " \b" # never leave evidences!
+
+    if [ $minutes == $limit ]; then
+      break;
+    fi
+
+    minutes=$((minutes+1))
+
+    sleep 60
+  done
+}
+
 if [ "${project_version}" == "${release}" ]; then
     # local publish only dependency, scala-core
     cd "${TRAVIS_BUILD_DIR}/0-common/scala-core"
+    echo "DEPLOY: localPublish iglu-core (for schema-ddl)..."
     sbt +publishLocal
+    echo "DEPLOY: localPublish iglu-core-circe (for schema-ddl)..."
     sbt "project igluCoreCirce" +publishLocal --warn
+    echo "DEPLOY: localPublish iglu-core-json4s (for schema-ddl)..."
     sbt "project igluCoreJson4s" +publishLocal --warn
     # universal publish schema-ddl
     cd "${TRAVIS_BUILD_DIR}/0-common/schema-ddl"
+    echo "DEPLOY: testing schema-ddl..."
     sbt +test --warn
+    echo "DEPLOY: publishing schema-ddl..."
     sbt +publish
-    sbt +bintraySyncMavenCentral
+    echo "DEPLOY: publishing schema-ddl to Maven Central..."
+    sbt +bintraySyncMavenCentral &
+    travis_wait
+    echo "DEPLOY: Schema DDL deployed..."
 
 else
     echo "Tag version '${release}' doesn't match version in scala project ('${project_version}'). Aborting!"

diff --git a/.travis/publish_local.sh b/.travis/publish_local.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -e
+
+cd "${TRAVIS_BUILD_DIR}/0-common/scala-core"
+sbt +publishLocal
+sbt "project igluCoreCirce" +publishLocal --warn
+sbt "project igluCoreJson4s" +publishLocal --warn
+
+cd "${TRAVIS_BUILD_DIR}/0-common/schema-ddl"
+sbt +publishLocal --warn
diff --git a/0-common/igluctl/build.sbt b/0-common/igluctl/build.sbt
@@ -18,9 +18,9 @@ lazy val root = project.in(file("."))
   .settings(
     name                  :=  "igluctl",
     organization          :=  "com.snowplowanalytics",
-    version               :=  "0.5.0",
+    version               :=  "0.6.0",
     description           :=  "Iglu Command Line Interface",
-    scalaVersion          :=  "2.11.12",
+    scalaVersion          :=  "2.11.12"
   )
   .settings(buildSettings: _*)
   .settings(
@@ -34,11 +34,13 @@ lazy val root = project.in(file("."))
       Libraries.jaxbImpl,
       Libraries.activation,
       // Scala
+      Libraries.catsEffect,
       Libraries.schemaddl,
       Libraries.igluClient,
-      Libraries.scopt,
+      Libraries.decline,
       Libraries.scalajHttp,
-      Libraries.awscala,
+      Libraries.fs2,
+      Libraries.fs2Io,
       // Scala (test only)
       Libraries.specs2,
       Libraries.scalaCheck

diff --git a/0-common/igluctl/project/BuildSettings.scala b/0-common/igluctl/project/BuildSettings.scala
@@ -46,7 +46,9 @@ object BuildSettings {
       "-source", "1.8",
       "-target", "1.8"
     ),
-    scalacOptions in Test := Seq("-Yrangepos")
+    scalacOptions in Test := Seq("-Yrangepos"),
+
+    addCompilerPlugin("org.spire-math" %% "kind-projector" % "0.9.7")
   )
 
   lazy val scalifySettings = Seq(
@@ -72,7 +74,14 @@ object BuildSettings {
     assemblyJarName in assembly := { name.value },
 
     // Make this executable
-    mainClass in assembly := Some("com.snowplowanalytics.iglu.ctl.Main")
+    mainClass in assembly := Some("com.snowplowanalytics.iglu.ctl.Main"),
+
+    assemblyMergeStrategy in assembly := {
+      case PathList("com", "github", "fge", tail@_*) => MergeStrategy.first
+      case x =>
+        val oldStrategy = (assemblyMergeStrategy in assembly).value
+        oldStrategy(x)
+    }
   )
 
   lazy val buildSettings = basicSettings ++ scalifySettings ++ sbtAssemblySettings

diff --git a/0-common/igluctl/project/Dependencies.scala b/0-common/igluctl/project/Dependencies.scala
@@ -16,7 +16,7 @@ object Dependencies {
 
   object V {
     // Java
-    val jsonValidator    = "2.2.6"
+    val jsonValidator    = "2.2.10"
     val awsJava          = "1.11.250"
     // JAXB APIs aren't resolved by default as of Java 9
     // https://docs.oracle.com/javase/9/migrate/#GUID-F640FA9D-FB66-4D85-AD2B-D931174C09A3
@@ -25,11 +25,12 @@ object Dependencies {
     val jaxbImpl         = "2.3.0"
     val activation       = "1.1.1"
     // Scala
-    val schemaddl        = "0.8.0"
+    val catsEffect       = "1.1.0"
+    val schemaddl        = "0.9.0"
     val igluClient       = "0.5.0"
-    val scopt            = "3.5.0"
+    val decline          = "0.6.0"
     val scalajHttp       = "2.3.0"
-    val awscala          = "0.5.9"
+    val fs2              = "1.0.2"
     // Scala (test only)
     val specs2           = "4.0.1"
     val scalaCheck       = "1.13.5"
@@ -38,19 +39,21 @@ object Dependencies {
 
   object Libraries {
     // Java
-    val jsonValidator    = "com.github.fge"             %  "json-schema-validator"     % V.jsonValidator
+    val jsonValidator    = "com.github.java-json-tools" % "json-schema-validator"      % V.jsonValidator
     val awsJava          = "com.amazonaws"              %  "aws-java-sdk-s3"           % V.awsJava
     // JAXB APIs
     val javaxXmlBind     = "javax.xml.bind"             %  "jaxb-api"                  % V.javaxXmlBind
     val jaxbCore         = "com.sun.xml.bind"           %  "jaxb-core"                 % V.jaxbCore
     val jaxbImpl         = "com.sun.xml.bind"           %  "jaxb-impl"                 % V.jaxbImpl
     val activation       = "javax.activation"           %  "activation"                % V.activation
     // Scala
+    val catsEffect       = "org.typelevel"              %% "cats-effect"               % V.catsEffect
     val igluClient       = "com.snowplowanalytics"      %% "iglu-scala-client"         % V.igluClient
     val schemaddl        = "com.snowplowanalytics"      %% "schema-ddl"                % V.schemaddl
-    val scopt            = "com.github.scopt"           %% "scopt"                     % V.scopt
+    val decline          = "com.monovore"               %% "decline"                   % V.decline
     val scalajHttp       = "org.scalaj"                 %% "scalaj-http"               % V.scalajHttp
-    val awscala          = "com.github.seratch"         %% "awscala"                   % V.awscala
+    val fs2              = "co.fs2"                     %% "fs2-core"                  % V.fs2
+    val fs2Io            = "co.fs2"                     %% "fs2-io"                    % V.fs2
     // Scala (test only)
     val specs2           = "org.specs2"                 %% "specs2-core"               % V.specs2         % "test"
     val scalaCheck       = "org.scalacheck"             %% "scalacheck"                % V.scalaCheck     % "test"

diff --git a/0-common/igluctl/project/build.properties b/0-common/igluctl/project/build.properties
@@ -1 +1 @@
-sbt.version=1.2.1
+sbt.version=1.2.8