Removed enrichment-specific code from EmrEtlRunner (#811)

Replaced enrichment-specific args from Emr-EtlRunner -> Hadoop Enrich, replaced with enrichments JSON (#808)
snowplow · Jul 9, 2014 · 5b30db3 · 5b30db3
1 parent a0287d8
commit 5b30db3
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 12 deletions.
diff --git a/3-enrich/emr-etl-runner/lib/snowplow-emr-etl-runner/emr_job.rb b/3-enrich/emr-etl-runner/lib/snowplow-emr-etl-runner/emr_job.rb
@@ -162,8 +162,7 @@ def initialize(debug, shred, s3distcp, config)
           },
           { :input_format     => config[:etl][:collector_format],
             :etl_tstamp       => etl_tstamp,
-            :maxmind_file     => assets[:maxmind],
-            :anon_ip_octets   => config[:enrichments][:anon_ip]
+            :enrichments      => Base64.strict_encode(JSON.generate(build_enrichments_json(config)))
           }
         )
         @jobflow.add_step(enrich_step)
@@ -341,7 +340,6 @@ def self.build_enrichments_json(config)
           :schema => 'iglu:com.snowplowanalytics.snowplow/enrichments/jsonschema/1-0-0',
           :data   => enrichments_json_data
         }
-
       end
 
       Contract IgluConfigHash => String
@@ -351,15 +349,7 @@ def self.jsonify(iglu_hash)
 
       Contract String, String, String => AssetsHash
       def self.get_assets(assets_bucket, hadoop_enrich_version, hadoop_shred_version)
-
-        asset_host = 
-          if assets_bucket == "s3://snowplow-hosted-assets/"
-            "http://snowplow-hosted-assets.s3.amazonaws.com/" # Use the public S3 URL
-          else
-            assets_bucket
-          end
-
-        { :maxmind  => "#{asset_host}third-party/maxmind/GeoLiteCity.dat",
+        {
           :enrich   => "#{assets_bucket}3-enrich/hadoop-etl/snowplow-hadoop-etl-#{hadoop_enrich_version}.jar",
           :shred    => "#{assets_bucket}3-enrich/scala-hadoop-shred/snowplow-hadoop-shred-#{hadoop_shred_version}.jar",
         }

diff --git a/CHANGELOG b/CHANGELOG
@@ -1,7 +1,15 @@
 Version 0.9.X (2014-XX-XX)
 --------------------------
+Schemas: added schema for collection of all enrichments (#806)
+Schemas: added campaigns schema (#805)
+Schemas: added ip_anon schema (#804)
+Schemas: added ip_to_geo schema (#803)
+EmrEtlRunner: Removed enrichment-specific code (#811)
+EmrEtlRunner: Replaced enrichment-specific args to Hadoop Enrich with enrichments JSON (#808)
+Scala Common Enrich: bumped to 0.5.0                                                    - TODO
 Scala Common Enrich: stored etl_tstamp in new field in CanonicalOutput (#818)
 Scala Common Enrich: removed event_vendor and ue_name and renamed ue_properties to unstruct_event (#836)
+Scala Hadoop Enrich: bumped to Scala Common Enrich 0.5.0                                - TODO
 Scala Hadoop Enrich: passed etl_tstamp into Scala Common Enrich (#817)
 Scala Hadoop Enrich: removed event_vendor and ue_name and renamed ue_properties to unstruct_event (#835)
 Redshift: bumped table-def to 0.4.0