Skip to content

Commit

Permalink
Created function to build enrichments JSON (#811)
Browse files Browse the repository at this point in the history
  • Loading branch information
fblundun committed Jun 17, 2014
1 parent 2fef273 commit 4a8961b
Showing 1 changed file with 14 additions and 10 deletions.
24 changes: 14 additions & 10 deletions 3-enrich/emr-etl-runner/lib/snowplow-emr-etl-runner/emr_job.rb
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ def initialize(debug, shred, s3distcp, config)
{ :input_format => config[:etl][:collector_format],
:etl_tstamp => etl_tstamp,
:maxmind_file => assets[:maxmind],
:anon_ip_octets => self.class.get_anon_ip_octets(config[:enrichments][:anon_ip])
:anon_ip_octets => config[:enrichments][:anon_ip]
}
)
@jobflow.add_step(enrich_step)
Expand Down Expand Up @@ -321,20 +321,24 @@ def wait_for()
success
end

def self.build_enrichments_json(config)
enrichments_json_data = []
enrichment_files = Dir.glob(config[:enrichments] + '/*.json')
for file in enrichment_files do
enrichments_json_data.push(JSON.parse(file))
end
enrichments_json = {
:schema => 'iglu:com.snowplowanalytics.snowplow/enrichments/jsonschema/1-0-0',
:data => enrichments_json_data
}

end

Contract IgluConfigHash => String
def self.jsonify(iglu_hash)
Base64.strict_encode(iglu_hash.to_camelback_keys.to_json)
end

Contract AnonIpHash => String
def self.get_anon_ip_octets(anon_ip)
if anon_ip[:enabled]
anon_ip[:anon_octets].to_s
else
'0' # Anonymize 0 octets == anonymization disabled
end
end

Contract String, String, String => AssetsHash
def self.get_assets(assets_bucket, hadoop_enrich_version, hadoop_shred_version)

Expand Down

0 comments on commit 4a8961b

Please sign in to comment.