Skip to content

Commit

Permalink
Created function to build enrichments JSON (snowplow/snowplow#811)
Browse files Browse the repository at this point in the history
  • Loading branch information
fblundun authored and peel committed May 25, 2020
1 parent 700475e commit 286fa7d
Showing 1 changed file with 13 additions and 10 deletions.
23 changes: 13 additions & 10 deletions lib/snowplow-emr-etl-runner/emr_job.rb
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ def initialize(debug, shred, s3distcp, config)
{ :input_format => config[:etl][:collector_format],
:etl_tstamp => etl_tstamp,
:maxmind_file => assets[:maxmind],
:anon_ip_octets => self.class.get_anon_ip_octets(config[:enrichments][:anon_ip])
:anon_ip_octets => config[:enrichments][:anon_ip]
}
)
@jobflow.add_step(enrich_step)
Expand Down Expand Up @@ -330,22 +330,25 @@ def self.partition_by_run(folder, run_id, retain=true)
Contract Maybe[String] => Maybe[String]
def self.fix_equals(path)
path.gsub!('=', '%3D') if path

def self.build_enrichments_json(config)
enrichments_json_data = []
enrichment_files = Dir.glob(config[:enrichments] + '/*.json')
for file in enrichment_files do
enrichments_json_data.push(JSON.parse(file))
end
enrichments_json = {
:schema => 'iglu:com.snowplowanalytics.snowplow/enrichments/jsonschema/1-0-0',
:data => enrichments_json_data
}

end

Contract IgluConfigHash => String
def self.jsonify(iglu_hash)
Base64.strict_encode64(iglu_hash.to_camelback_keys.to_json)
end

Contract AnonIpHash => String
def self.get_anon_ip_octets(anon_ip)
if anon_ip[:enabled]
anon_ip[:anon_octets].to_s
else
'0' # Anonymize 0 octets == anonymization disabled
end
end

Contract String, String, String => AssetsHash
def self.get_assets(assets_bucket, hadoop_enrich_version, hadoop_shred_version)

Expand Down

0 comments on commit 286fa7d

Please sign in to comment.