EmrEtlRunner & StorageLoader: supported environment variables in YAML…

… config files (closes #1215)

3-enrich/emr-etl-runner/lib/snowplow-emr-etl-runner/cli.rb

@@ -16,6 +16,7 @@
 require 'optparse'
 require 'yaml'
 require 'contracts'
+require 'erb'
 
 module Snowplow
   module EmrEtlRunner
@@ -135,14 +136,16 @@ def self.load_config(config_file, optparse)
 
         # A single hyphen indicates that the config should be read from stdin
         if config_file == '-'
-          config = YAML.load($stdin.readlines.join)
+          config = $stdin.readlines.join
         elsif File.file?(config_file)
-          config = YAML.load_file(config_file)
+          config = File.new(config_file).read
         else
           raise ConfigError, "Configuration file '#{config_file}' does not exist, or is not a file\n#{optparse}"
         end
 
-        recursive_symbolize_keys(config)
+        erb_config = ERB.new(config).result(binding)
+
+        recursive_symbolize_keys(YAML.load(erb_config))
       end
 
       # Load the enrichments directory into an array

4-storage/storage-loader/lib/snowplow-storage-loader/config.rb

@@ -16,6 +16,7 @@
 require 'optparse'
 require 'date'
 require 'yaml'
+require 'erb'
 require 'sluice'
 
 # Config module to hold functions related to CLI argument parsing
@@ -36,12 +37,14 @@ def get_config()
         options = Config.parse_args()
 
         if Config.indicates_read_from_stdin?(options[:config])
-          unsymbolized_config = YAML.load($stdin.readlines.join)
+          unsymbolized_config = $stdin.readlines.join
         else
-          unsymbolized_config = YAML.load_file(options[:config])
+          unsymbolized_config = File.new(options[:config]).read
         end
 
-        config = Config.recursive_symbolize_keys(unsymbolized_config)
+        erb_config = ERB.new(unsymbolized_config).result(binding)
+
+        config = Config.recursive_symbolize_keys(YAML.load(erb_config))
 
 
         # Add in our skip and include settings