Merge pull request #1500 from snowplow/feature/r62-fixes

Feature/r62 fixes
snowplow · Mar 17, 2015 · 7689624 · 7689624
2 parents f5878ed + dc9e86d
commit 7689624
Show file tree

Hide file tree

Showing 25 changed files with 260 additions and 118 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -19,9 +19,9 @@
 [submodule "1-trackers/nodejs-tracker"]
 	path = 1-trackers/nodejs-tracker
 	url = https://github.com/snowplow/snowplow-nodejs-tracker.git
-[submodule "1-trackers/ios-tracker"]
-	path = 1-trackers/ios-tracker
-	url = https://github.com/snowplow/snowplow-ios-tracker
+[submodule "1-trackers/objc-tracker"]
+	path = 1-trackers/objc-tracker
+	url = https://github.com/snowplow/snowplow-objc-tracker
 [submodule "1-trackers/android-tracker"]
 	path = 1-trackers/android-tracker
 	url = https://github.com/snowplow/snowplow-android-tracker

diff --git a/1-trackers/android-tracker b/1-trackers/android-tracker
diff --git a/1-trackers/ios-tracker b/1-trackers/ios-tracker
diff --git a/1-trackers/java-tracker b/1-trackers/java-tracker
diff --git a/1-trackers/javascript-tracker b/1-trackers/javascript-tracker
diff --git a/1-trackers/objc-tracker b/1-trackers/objc-tracker
diff --git a/1-trackers/php-tracker b/1-trackers/php-tracker
diff --git a/1-trackers/python-tracker b/1-trackers/python-tracker
diff --git a/1-trackers/ruby-tracker b/1-trackers/ruby-tracker
diff --git a/3-enrich/emr-etl-runner/Gemfile b/3-enrich/emr-etl-runner/Gemfile
@@ -19,11 +19,10 @@ ruby "1.9.3"
 # ErmEtlRunner is a Ruby app (not a RubyGem)
 # built with Bundler, so we add in the
 # RubyGems it requires here.
-gem "contracts", "~> 0.4"
+gem "contracts", "~> 0.7"
 gem "elasticity", "~> 3.0.4"
 gem "sluice", "~> 0.2.1"
 gem "awrence", "~> 0.1.0"
-gem "time_diff", "~> 0.3.0"
 
 group :development do
   gem "rspec", "~> 2.14", ">= 2.14.1"

diff --git a/3-enrich/emr-etl-runner/Gemfile.lock b/3-enrich/emr-etl-runner/Gemfile.lock
@@ -1,90 +1,141 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (4.1.8)
-      i18n (~> 0.6, >= 0.6.9)
-      json (~> 1.7, >= 1.7.7)
-      minitest (~> 5.1)
-      thread_safe (~> 0.1)
-      tzinfo (~> 1.1)
+    CFPropertyList (2.3.0)
     awrence (0.1.0)
     builder (3.2.2)
-    contracts (0.4)
-    coveralls (0.7.0)
-      multi_json (~> 1.3)
-      rest-client
-      simplecov (>= 0.7)
-      term-ansicolor
-      thor
+    contracts (0.7)
+    coveralls (0.7.11)
+      multi_json (~> 1.10)
+      rest-client (>= 1.6.8, < 2)
+      simplecov (~> 0.9.1)
+      term-ansicolor (~> 1.3)
+      thor (~> 0.19.1)
     diff-lcs (1.2.5)
     docile (1.1.5)
     elasticity (3.0.4)
       fog (~> 1.0)
       nokogiri (~> 1.0)
       rest-client (~> 1.0)
       unf (~> 0.1)
-    excon (0.37.0)
-    fog (1.22.1)
-      fog-brightbox
-      fog-core (~> 1.22)
+    excon (0.44.4)
+    fission (0.5.0)
+      CFPropertyList (~> 2.2)
+    fog (1.28.0)
+      fog-atmos
+      fog-aws (~> 0.0)
+      fog-brightbox (~> 0.4)
+      fog-core (~> 1.27, >= 1.27.3)
+      fog-ecloud
       fog-json
+      fog-profitbricks
+      fog-radosgw (>= 0.0.2)
+      fog-riakcs
+      fog-sakuracloud (>= 0.0.4)
+      fog-serverlove
+      fog-softlayer
+      fog-storm_on_demand
+      fog-terremark
+      fog-vmfusion
+      fog-voxel
+      fog-xml (~> 0.1.1)
       ipaddress (~> 0.5)
       nokogiri (~> 1.5, >= 1.5.11)
-    fog-brightbox (0.1.1)
+    fog-atmos (0.1.0)
+      fog-core
+      fog-xml
+    fog-aws (0.1.1)
+      fog-core (~> 1.27)
+      fog-json (~> 1.0)
+      fog-xml (~> 0.1)
+      ipaddress (~> 0.8)
+    fog-brightbox (0.7.1)
       fog-core (~> 1.22)
       fog-json
-      inflecto
-    fog-core (1.22.0)
+      inflecto (~> 0.0.2)
+    fog-core (1.29.0)
       builder
-      excon (~> 0.33)
+      excon (~> 0.38)
       formatador (~> 0.2)
       mime-types
       net-scp (~> 1.1)
       net-ssh (>= 2.1.3)
+    fog-ecloud (0.0.2)
+      fog-core
+      fog-xml
     fog-json (1.0.0)
       multi_json (~> 1.0)
+    fog-profitbricks (0.0.1)
+      fog-core
+      fog-xml
+      nokogiri
+    fog-radosgw (0.0.3)
+      fog-core (>= 1.21.0)
+      fog-json
+      fog-xml (>= 0.0.1)
+    fog-riakcs (0.1.0)
+      fog-core
+      fog-json
+      fog-xml
+    fog-sakuracloud (1.0.0)
+      fog-core
+      fog-json
+    fog-serverlove (0.1.1)
+      fog-core
+      fog-json
+    fog-softlayer (0.4.1)
+      fog-core
+      fog-json
+    fog-storm_on_demand (0.1.0)
+      fog-core
+      fog-json
+    fog-terremark (0.0.4)
+      fog-core
+      fog-xml
+    fog-vmfusion (0.0.1)
+      fission
+      fog-core
+    fog-voxel (0.0.2)
+      fog-core
+      fog-xml
+    fog-xml (0.1.1)
+      fog-core
+      nokogiri (~> 1.5, >= 1.5.11)
     formatador (0.2.5)
-    i18n (0.6.11)
     inflecto (0.0.2)
     ipaddress (0.8.0)
-    json (1.8.1)
-    mime-types (2.3)
-    mini_portile (0.6.0)
-    minitest (5.4.3)
-    multi_json (1.10.1)
+    mime-types (2.4.3)
+    mini_portile (0.6.2)
+    multi_json (1.11.0)
     net-scp (1.2.1)
       net-ssh (>= 2.6.5)
-    net-ssh (2.9.1)
-    nokogiri (1.6.2.1)
-      mini_portile (= 0.6.0)
-    rest-client (1.6.7)
-      mime-types (>= 1.16)
+    net-ssh (2.9.2)
+    netrc (0.10.3)
+    nokogiri (1.6.6.2)
+      mini_portile (~> 0.6.0)
+    rest-client (1.7.3)
+      mime-types (>= 1.16, < 3.0)
+      netrc (~> 0.7)
     rspec (2.99.0)
       rspec-core (~> 2.99.0)
       rspec-expectations (~> 2.99.0)
       rspec-mocks (~> 2.99.0)
-    rspec-core (2.99.1)
-    rspec-expectations (2.99.1)
+    rspec-core (2.99.2)
+    rspec-expectations (2.99.2)
       diff-lcs (>= 1.1.3, < 2.0)
-    rspec-mocks (2.99.1)
-    simplecov (0.8.2)
+    rspec-mocks (2.99.3)
+    simplecov (0.9.2)
       docile (~> 1.1.0)
-      multi_json
-      simplecov-html (~> 0.8.0)
-    simplecov-html (0.8.0)
+      multi_json (~> 1.0)
+      simplecov-html (~> 0.9.0)
+    simplecov-html (0.9.0)
     sluice (0.2.1)
       contracts (~> 0.4)
       fog (~> 1.22)
     term-ansicolor (1.3.0)
       tins (~> 1.0)
     thor (0.19.1)
-    thread_safe (0.3.4)
-    time_diff (0.3.0)
-      activesupport
-      i18n
-    tins (1.3.0)
-    tzinfo (1.2.2)
-      thread_safe (~> 0.1)
+    tins (1.3.5)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.6)
@@ -94,9 +145,8 @@ PLATFORMS
 
 DEPENDENCIES
   awrence (~> 0.1.0)
-  contracts (~> 0.4)
+  contracts (~> 0.7)
   coveralls
   elasticity (~> 3.0.4)
   rspec (~> 2.14, >= 2.14.1)
   sluice (~> 0.2.1)
-  time_diff (~> 0.3.0)
diff --git a/3-enrich/emr-etl-runner/config/config.yml.sample b/3-enrich/emr-etl-runner/config/config.yml.sample
@@ -26,6 +26,7 @@
   :placement: ADD HERE     # Set this if not running in VPC. Leave blank otherwise
   :ec2_subnet_id: ADD HERE # Set this if running in VPC. Leave blank otherwise
   :ec2_key_name: ADD HERE
+  :bootstrap: []           # Set this to specify custom boostrap actions. Leave empty otherwise
   :software:
     :hbase:                # To launch on cluster, provide version, "0.92.0", keep quotes
     :lingual:              # To launch on cluster, provide version, "1.1", keep quotes

diff --git a/3-enrich/emr-etl-runner/lib/snowplow-emr-etl-runner.rb b/3-enrich/emr-etl-runner/lib/snowplow-emr-etl-runner.rb
@@ -26,6 +26,6 @@
 module Snowplow
   module EmrEtlRunner
     NAME          = "snowplow-emr-etl-runner"
-    VERSION       = "0.12.0"
+    VERSION       = "0.13.0"
   end
 end
diff --git a/3-enrich/emr-etl-runner/lib/snowplow-emr-etl-runner/cli.rb b/3-enrich/emr-etl-runner/lib/snowplow-emr-etl-runner/cli.rb
@@ -15,14 +15,14 @@
 
 require 'optparse'
 require 'yaml'
-
 require 'contracts'
-include Contracts
 
 module Snowplow
   module EmrEtlRunner
     module Cli
 
+      include Contracts
+
       # Get our arguments and configuration.
       #
       # Source from parse_args (i.e. the CLI)

diff --git a/3-enrich/emr-etl-runner/lib/snowplow-emr-etl-runner/contracts.rb b/3-enrich/emr-etl-runner/lib/snowplow-emr-etl-runner/contracts.rb
@@ -14,11 +14,12 @@
 # License::   Apache License Version 2.0
 
 require 'contracts'
-include Contracts
 
 module Snowplow
   module EmrEtlRunner
 
+    include Contracts
+
     # The Hash containing assets for Hadoop.
     AssetsHash = ({
       :enrich  => String,
@@ -65,7 +66,8 @@ module EmrEtlRunner
       :log => String,
       :raw => ({
         :in => String,
-        :processing => String
+        :processing => String,
+        :archive => String
         }),
       :enriched => ({
         :good => String,
@@ -98,6 +100,7 @@ module EmrEtlRunner
         :placement => Maybe[String],
         :ec2_subnet_id => Maybe[String],
         :ec2_key_name => String,
+        :bootstrap => ArrayOf[String],
         :software => ({
           :hbase => Maybe[String],
           :lingual => Maybe[String]

diff --git a/3-enrich/emr-etl-runner/lib/snowplow-emr-etl-runner/emr_job.rb b/3-enrich/emr-etl-runner/lib/snowplow-emr-etl-runner/emr_job.rb
@@ -15,22 +15,20 @@
 
 require 'set'
 require 'elasticity'
-
 require 'sluice'
-
 require 'awrence'
 require 'json'
 require 'base64'
-
 require 'contracts'
-include Contracts
 
 # Ruby class to execute Snowplow's Hive jobs against Amazon EMR
 # using Elasticity (https://github.com/rslifka/elasticity).
 module Snowplow
   module EmrEtlRunner
     class EmrJob
 
+      include Contracts
+
       # Constants
       JAVA_PACKAGE = "com.snowplowanalytics.snowplow"
       PARTFILE_REGEXP = ".*part-.*"
@@ -88,6 +86,12 @@ def initialize(debug, enrich, shred, s3distcp, config, enrichments_array)
           end
         end
 
+        # Add custom bootstrap actions
+        bootstrap_actions = config[:emr][:bootstrap]
+        bootstrap_actions.each do |bootstrap_action|
+          @jobflow.add_bootstrap_action(Elasticity::BootstrapAction.new(bootstrap_action))
+        end
+
         # Install and launch HBase
         hbase = config[:emr][:software][:hbase]
         unless not hbase
@@ -359,6 +363,12 @@ def wait_for()
           rescue SocketError => se
             logger.warn "Got socket error #{se}, waiting 5 minutes before checking jobflow again"
             sleep(300)
+          rescue Errno::ECONNREFUSED => ref
+            logger.warn "Got connection refused #{ref}, waiting 5 minutes before checking jobflow again"
+            sleep(300)
+          rescue Errno::ECONNRESET => res
+            logger.warn "Got connection reset #{res}, waiting 5 minutes before checking jobflow again"
+            sleep(300)
           end
         end
 

diff --git a/3-enrich/emr-etl-runner/lib/snowplow-emr-etl-runner/logging.rb b/3-enrich/emr-etl-runner/lib/snowplow-emr-etl-runner/logging.rb
@@ -14,14 +14,14 @@
 # License::   Apache License Version 2.0
 
 require 'logger'
-
 require 'contracts'
-include Contracts
 
 module Snowplow
   module EmrEtlRunner
     module Logging
 
+      include Contracts
+
       $stdout.sync = true
 
       # Get the Logger

diff --git a/3-enrich/emr-etl-runner/lib/snowplow-emr-etl-runner/runner.rb b/3-enrich/emr-etl-runner/lib/snowplow-emr-etl-runner/runner.rb
@@ -14,12 +14,13 @@
 # License::   Apache License Version 2.0
 
 require 'contracts'
-include Contracts
 
 module Snowplow
   module EmrEtlRunner
     class Runner
 
+      include Contracts
+
       # Supported options
       @@collector_format_regex = /^(?:cloudfront|clj-tomcat|thrift|(?:json\/.+\/.+)|(?:tsv\/.+\/.+))$/
       @@skip_options = Set.new(%w(staging s3distcp emr enrich shred archive))