EmrEtlRunner: do not try to download non-existent RDB loader log files (

closes #3405)
snowplow · Aug 22, 2017 · fcbabdf · fcbabdf
1 parent 768e021
commit fcbabdf
Showing 1 changed file with 5 additions and 3 deletions.
diff --git a/3-enrich/emr-etl-runner/lib/snowplow-emr-etl-runner/emr_job.rb b/3-enrich/emr-etl-runner/lib/snowplow-emr-etl-runner/emr_job.rb
@@ -596,13 +596,15 @@ def run(config)
       Contract String, String, String, String => nil
       def output_rdb_loader_logs(region, aws_access_key_id, aws_secret_key, log_level)
 
-        if @rdb_loader_logs.empty?
+        s3 = Sluice::Storage::S3::new_fog_s3_from(region, aws_access_key_id, aws_secret_key)
+
+        loc = Sluice::Storage::S3::Location.new(@rdb_loader_log_base)
+
+        if @rdb_loader_logs.empty? or Sluice::Storage::S3::is_empty?(s3, loc)
           logger.info "No RDB Loader logs"
         else
           logger.info "RDB Loader logs"
 
-          s3 = Sluice::Storage::S3::new_fog_s3_from(region, aws_access_key_id, aws_secret_key)
-
           @rdb_loader_logs.each do |l|
             tmp = Tempfile.new("rdbloader")
             uri = URI.parse(l[1])