Merge branch 'release/r83-bald-eagle'

.gitmodules

@@ -43,3 +43,6 @@
 [submodule "1-trackers/golang-tracker"]
 	path = 1-trackers/golang-tracker
 	url = https://github.com/snowplow/snowplow-golang-tracker.git
+[submodule "4-storage/kinesis-s3"]
+	path = 4-storage/kinesis-s3
+	url = https://github.com/snowplow/kinesis-s3.git

.travis.yml

@@ -3,6 +3,11 @@ scala:
 - 2.10.1
 jdk:
 - oraclejdk7
+before_script:
+  - psql -U postgres -c "CREATE USER enricher WITH password 'supersecret1';"
+  - psql -U postgres -c "CREATE DATABASE sql_enrichment_test WITH OWNER = enricher;"
+  - psql -U postgres -d sql_enrichment_test < integration-tests/sql-enrichment-test.sql
+  - psql -U postgres -d sql_enrichment_test -c "GRANT ALL ON enrichment_test TO enricher;"
 script:
   - python integration-tests/api-lookup-test.py &
   - cd 3-enrich/scala-common-enrich
@@ -25,6 +30,9 @@ before_install:
   - export PATH=$PATH:$HOME/.local/bin
 addons:
   hostname: short-hostname
+  postgresql: "9.3"
+services:
+  - postgresql
 deploy:
   skip_cleanup: true
   provider: script

3-enrich/config/enrichments/sql_query_enrichment_config.json

@@ -0,0 +1,59 @@
+{
+	"schema": "iglu:com.snowplowanalytics.snowplow.enrichments/sql_query_enrichment_config/jsonschema/1-0-0",
+
+	"data": {
+
+		"vendor": "com.snowplowanalytics.snowplow.enrichments",
+		"name": "sql_query_enrichment_config",
+		"enabled": false,
+		"parameters": {
+			"inputs": [
+				{
+					"placeholder": 1,
+					"pojo": {
+						"field": "user_id"
+					}
+				},
+				{
+					"placeholder": 1,
+					"json": {
+						"field": "contexts",
+						"schemaCriterion": "iglu:com.snowplowanalytics.snowplow/client_session/jsonschema/1-*-*",
+						"jsonPath": "$.userId"
+					}
+				},
+				{
+					"placeholder": 2,
+					"pojo": {
+						"field": "app_id"
+					}
+				}
+			],
+			"database": {
+				"postgresql": {
+					"host": "cluster01.redshift.acme.com",
+					"port": 5439,
+					"sslMode": true,
+					"username": "snowplow_enrich_ro",
+					"password": "1asIkJed",
+					"database": "crm"
+				}
+			},
+			"query": {
+				"sql": "SELECT username, email_address, date_of_birth FROM tbl_users WHERE user = ? AND client = ? LIMIT 1"
+			},
+			"output": {
+				"expectedRows": "AT_MOST_ONE",
+				"json": {
+					"schema": "iglu:com.acme/user/jsonschema/1-0-0",
+					"describes": "ALL_ROWS",
+					"propertyNames": "CAMEL_CASE"
+				}
+			},
+			"cache": {
+				"size": 3000,
+				"ttl": 60
+			}
+		}
+    }
+}

3-enrich/emr-etl-runner/Gemfile

@@ -14,14 +14,14 @@
 # License::   Apache License Version 2.0
 
 source "https://rubygems.org"
-ruby "1.9.3"
+ruby "2.2.3"
 
 # ErmEtlRunner is a Ruby app (not a RubyGem)
 # built with Bundler, so we add in the
 # RubyGems it requires here.
-gem "contracts", "= 0.7"
+gem "contracts", "~> 0.9", "<= 0.11"
 gem "elasticity", "~> 6.0.7"
-gem "sluice", "~> 0.2.2"
+gem "sluice", "~> 0.4.0"
 gem "awrence", "~> 0.1.0"
 gem "snowplow-tracker", "~> 0.5.2"
 

3-enrich/emr-etl-runner/Gemfile.lock

@@ -1,80 +1,102 @@
 GEM
   remote: https://rubygems.org/
   specs:
+    CFPropertyList (2.3.2)
     awrence (0.1.0)
     builder (3.2.2)
-    contracts (0.7)
-    coveralls (0.8.1)
-      json (~> 1.8)
-      rest-client (>= 1.6.8, < 2)
-      simplecov (~> 0.10.0)
+    contracts (0.11.0)
+    coveralls (0.8.15)
+      json (>= 1.8, < 3)
+      simplecov (~> 0.12.0)
       term-ansicolor (~> 1.3)
       thor (~> 0.19.1)
+      tins (>= 1.6.0, < 2)
     diff-lcs (1.2.5)
     docile (1.1.5)
-    domain_name (0.5.24)
+    domain_name (0.5.20160826)
       unf (>= 0.0.5, < 1.0.0)
-    elasticity (6.0.7)
+    elasticity (6.0.8)
       fog (~> 1.0)
       rest-client (~> 1.0)
       unf (~> 0.1)
-    excon (0.45.3)
-    fog (1.24.0)
-      fog-brightbox
-      fog-core (~> 1.23)
+    excon (0.52.0)
+    fission (0.5.0)
+      CFPropertyList (~> 2.2)
+    fog (1.25.0)
+      fog-brightbox (~> 0.4)
+      fog-core (~> 1.25)
       fog-json
+      fog-profitbricks
       fog-radosgw (>= 0.0.2)
       fog-sakuracloud (>= 0.0.4)
       fog-softlayer
+      fog-terremark
+      fog-vmfusion
+      fog-voxel
+      fog-xml (~> 0.1.1)
       ipaddress (~> 0.5)
       nokogiri (~> 1.5, >= 1.5.11)
-    fog-brightbox (0.7.1)
+      opennebula
+    fog-brightbox (0.11.0)
       fog-core (~> 1.22)
       fog-json
       inflecto (~> 0.0.2)
-    fog-core (1.31.0)
+    fog-core (1.42.0)
       builder
-      excon (~> 0.45)
+      excon (~> 0.49)
       formatador (~> 0.2)
-      mime-types
-      net-scp (~> 1.1)
-      net-ssh (>= 2.1.3)
     fog-json (1.0.2)
       fog-core (~> 1.0)
       multi_json (~> 1.10)
-    fog-radosgw (0.0.4)
+    fog-profitbricks (0.0.5)
+      fog-core
+      fog-xml
+      nokogiri
+    fog-radosgw (0.0.5)
       fog-core (>= 1.21.0)
       fog-json
       fog-xml (>= 0.0.1)
-    fog-sakuracloud (1.0.1)
+    fog-sakuracloud (1.7.5)
       fog-core
       fog-json
-    fog-softlayer (0.4.6)
+    fog-softlayer (1.1.4)
       fog-core
       fog-json
+    fog-terremark (0.1.0)
+      fog-core
+      fog-xml
+    fog-vmfusion (0.1.0)
+      fission
+      fog-core
+    fog-voxel (0.1.0)
+      fog-core
+      fog-xml
     fog-xml (0.1.2)
       fog-core
       nokogiri (~> 1.5, >= 1.5.11)
     formatador (0.2.5)
     http-cookie (1.0.2)
       domain_name (~> 0.5)
     inflecto (0.0.2)
-    ipaddress (0.8.0)
-    jruby-jars (1.7.20.1)
-    jruby-rack (1.1.18)
-    json (1.8.3)
-    json (1.8.3-java)
-    mime-types (2.6.1)
-    mini_portile (0.6.2)
-    multi_json (1.11.1)
-    net-scp (1.2.1)
-      net-ssh (>= 2.6.5)
-    net-ssh (2.9.2)
-    netrc (0.10.3)
-    nokogiri (1.6.6.2)
-      mini_portile (~> 0.6.0)
-    nokogiri (1.6.6.2-java)
-    rake (10.4.2)
+    ipaddress (0.8.3)
+    jruby-jars (9.1.4.0)
+    jruby-rack (1.1.20)
+    json (2.0.2-java)
+    mime-types (2.99.2)
+    multi_json (1.12.1)
+    net-ssh (2.9.4)
+    netrc (0.11.0)
+    nokogiri (1.6.8-java)
+    opennebula (5.0.2)
+      json
+      nokogiri
+      rbvmomi
+    rake (11.2.2)
+    rbvmomi (1.9.2)
+      builder (~> 3.2)
+      json (>= 1.8)
+      nokogiri (~> 1.5)
+      trollop (~> 2.1)
     rest-client (1.8.0)
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 3.0)
@@ -86,45 +108,46 @@ GEM
     rspec-core (2.99.2)
     rspec-expectations (2.99.2)
       diff-lcs (>= 1.1.3, < 2.0)
-    rspec-mocks (2.99.3)
-    rubyzip (1.1.7)
-    simplecov (0.10.0)
+    rspec-mocks (2.99.4)
+    rubyzip (1.2.0)
+    simplecov (0.12.0)
       docile (~> 1.1.0)
-      json (~> 1.8)
+      json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.0)
-    sluice (0.2.2)
-      contracts (~> 0.4)
-      fog (= 1.24)
+    sluice (0.4.0)
+      contracts (~> 0.9)
+      fog (= 1.25)
+      net-ssh (~> 2.9.2)
     snowplow-tracker (0.5.2)
       contracts (~> 0.7, <= 0.11)
     term-ansicolor (1.3.2)
       tins (~> 1.0)
     thor (0.19.1)
-    tins (1.5.2)
-    unf (0.1.4)
-      unf_ext
+    tins (1.12.0)
+    trollop (2.1.2)
     unf (0.1.4-java)
-    unf_ext (0.0.7.1)
-    warbler (1.4.8)
-      jruby-jars (>= 1.5.6, < 2.0)
+    warbler (2.0.3)
+      jruby-jars (>= 9.0.0.0)
       jruby-rack (>= 1.1.1, < 1.3)
-      rake (>= 0.9.6)
-      rubyzip (>= 0.9, < 1.2)
+      rake (>= 10.1.0)
+      rubyzip (~> 1.0, < 1.4)
 
 PLATFORMS
   java
-  ruby
 
 DEPENDENCIES
   awrence (~> 0.1.0)
-  contracts (= 0.7)
+  contracts (~> 0.9, <= 0.11)
   coveralls
   elasticity (~> 6.0.7)
   rspec (~> 2.14, >= 2.14.1)
-  sluice (~> 0.2.2)
+  sluice (~> 0.4.0)
   snowplow-tracker (~> 0.5.2)
   warbler
 
+RUBY VERSION
+   ruby 2.2.3p0 (jruby 9.0.5.0)
+
 BUNDLED WITH
-   1.11.2
+   1.12.5

3-enrich/emr-etl-runner/bin/snowplow-ami4-bootstrap.sh

@@ -15,21 +15,25 @@
 # Copyright::     Copyright (c) 2016 Snowplow Analytics Ltd
 # License::       Apache License Version 2.0
 
-# Version::       0.1.0
+# Version::       0.2.0
 # Compatibility:: AMI 4.1.0
 
 # Fast fail
 set -e
 
+# Get commons-codec version as optional first argument
+if [ -z $1 ]; then CC_VERSION="1.5"; else CC_VERSION=$1; fi
+
 # Because AMI 4.1.0 has commons-codecs which are binary
-# incompatible with 1.5, we download 1.5 to the classpath.
+# incompatible with shipped with Scala Hadoop Enrich,
+# we download appropriate version to the classpath.
 # See https://forums.aws.amazon.com/thread.jspa?threadID=173258
 function fix_commons_codec() {
-    wget 'http://central.maven.org/maven2/commons-codec/commons-codec/1.5/commons-codec-1.5.jar'
+    wget "http://central.maven.org/maven2/commons-codec/commons-codec/$CC_VERSION/commons-codec-$CC_VERSION.jar"
     sudo mkdir -p /usr/lib/hadoop/lib
-    sudo cp commons-codec-1.5.jar /usr/lib/hadoop/lib/remedial-commons-codec-1.5.jar
-    rm commons-codec-1.5.jar
-    echo "Fixed commons-codec-1.5.jar"
+    sudo cp "commons-codec-$CC_VERSION.jar" "/usr/lib/hadoop/lib/remedial-commons-codec-$CC_VERSION.jar"
+    rm "commons-codec-$CC_VERSION.jar"
+    echo "Fixed commons-codec-$CC_VERSION.jar"
 }
 
 # Because the AMIs have Scala 2.11 installed, and this

3-enrich/emr-etl-runner/config/config.yml.sample

@@ -51,7 +51,7 @@ collectors:
 enrich:
   job_name: Snowplow ETL # Give your job a name
   versions:
-    hadoop_enrich: 1.7.0 # Version of the Hadoop Enrichment process
+    hadoop_enrich: 1.8.0 # Version of the Hadoop Enrichment process
     hadoop_shred: 0.9.0 # Version of the Hadoop Shredding process
     hadoop_elasticsearch: 0.1.0 # Version of the Hadoop to Elasticsearch copying process
   continue_on_unexpected_error: false # Set to 'true' (and set :out_errors: above) if you don't want any exceptions thrown from ETL

3-enrich/emr-etl-runner/lib/snowplow-emr-etl-runner.rb

@@ -28,6 +28,6 @@
 module Snowplow
   module EmrEtlRunner
     NAME          = "snowplow-emr-etl-runner"
-    VERSION       = "0.21.0"
+    VERSION       = "0.22.0"
   end
 end

3-enrich/emr-etl-runner/lib/snowplow-emr-etl-runner/cli.rb

@@ -158,7 +158,7 @@ def self.load_enrichments(enrichments_dir, optparse)
         return [] if enrichments_dir.nil?
 
         # Check the enrichments directory exists and is a directory
-        unless Dir.exists?(enrichments_dir)
+        unless Dir.exist?(enrichments_dir)
           raise ConfigError, "Enrichments directory '#{enrichments_dir}' does not exist, or is not a directory\n#{optparse}"
         end