Merge 638c045 into 23d9e2a

caliban_toolbox/aws_functions.py

@@ -29,6 +29,7 @@
 import re
 
 import boto3
+import botocore
 
 from urllib.parse import urlencode
 from getpass import getpass
@@ -112,6 +113,9 @@ def aws_download_files(upload_log, output_dir):
     aws_folder = upload_log['aws_folder'][0]
     stage = upload_log['stage'][0]
 
+    # track missing files
+    missing = []
+
     # download all images
     for file in files_to_download:
 
@@ -121,4 +125,14 @@ def aws_download_files(upload_log, output_dir):
         # path to file in aws
         aws_path = os.path.join(aws_folder, stage, file)
 
-        s3.download_file(Bucket='caliban-output', Key=aws_path, Filename=local_path)
+        try:
+            s3.download_file(Bucket='caliban-output', Key=img_path, Filename=save_path)
+        except botocore.exceptions.ClientError as e:
+            error_code = e.response['Error']['Code']
+            if error_code == '404':
+                print('The file {} does not exist'.format(img))
+                missing.append(img)
+            else:
+                raise e
+
+    return missing

caliban_toolbox/figure_eight_functions.py

@@ -275,6 +275,9 @@ def download_figure_eight_output(base_dir):
 
     Args:
         base_dir: directory containing relevant job files
+
+    Returns:
+        list: file names of NPZs not found in AWS bucket
     """
 
     # get information from job creation
@@ -293,4 +296,6 @@ def download_figure_eight_output(base_dir):
     if not os.path.isdir(output_dir):
         os.makedirs(output_dir)
 
-    aws_download_files(log_file, output_dir)
+    missing = aws_download_files(log_file, output_dir)
+
+    return missing