diff --git a/caliban_toolbox/aws_functions.py b/caliban_toolbox/aws_functions.py index b193bb1..57f1594 100755 --- a/caliban_toolbox/aws_functions.py +++ b/caliban_toolbox/aws_functions.py @@ -29,6 +29,7 @@ import re import boto3 +import botocore from urllib.parse import urlencode from getpass import getpass @@ -112,6 +113,9 @@ def aws_download_files(upload_log, output_dir): aws_folder = upload_log['aws_folder'][0] stage = upload_log['stage'][0] + # track missing files + missing = [] + # download all images for file in files_to_download: @@ -121,4 +125,14 @@ def aws_download_files(upload_log, output_dir): # path to file in aws aws_path = os.path.join(aws_folder, stage, file) - s3.download_file(Bucket='caliban-output', Key=aws_path, Filename=local_path) + try: + s3.download_file(Bucket='caliban-output', Key=img_path, Filename=save_path) + except botocore.exceptions.ClientError as e: + error_code = e.response['Error']['Code'] + if error_code == '404': + print('The file {} does not exist'.format(img)) + missing.append(img) + else: + raise e + + return missing diff --git a/caliban_toolbox/figure_eight_functions.py b/caliban_toolbox/figure_eight_functions.py index c05374d..57fcaa4 100644 --- a/caliban_toolbox/figure_eight_functions.py +++ b/caliban_toolbox/figure_eight_functions.py @@ -275,6 +275,9 @@ def download_figure_eight_output(base_dir): Args: base_dir: directory containing relevant job files + + Returns: + list: file names of NPZs not found in AWS bucket """ # get information from job creation @@ -293,4 +296,6 @@ def download_figure_eight_output(base_dir): if not os.path.isdir(output_dir): os.makedirs(output_dir) - aws_download_files(log_file, output_dir) + missing = aws_download_files(log_file, output_dir) + + return missing