Skip to content

Commit

Permalink
fix issues with s3_download
Browse files Browse the repository at this point in the history
  • Loading branch information
xieyongliang committed May 4, 2023
1 parent 44029c4 commit 6c0d58e
Showing 1 changed file with 4 additions and 4 deletions.
8 changes: 4 additions & 4 deletions modules/shared.py
Original file line number Diff line number Diff line change
Expand Up @@ -702,13 +702,13 @@ def s3_download(s3uri, path):
for obj in objects:
response = s3_client.head_object(
Bucket = bucket,
Key = obj.key
Key = obj['Key']
)
obj_key = 's3://{0}/{1}'.format(bucket, obj.key)
obj_key = 's3://{0}/{1}'.format(bucket, obj['Key'])
if obj_key not in cache or cache[obj_key] != response['ETag']:
filename = obj.key[obj.key.rfind('/') + 1 : ]
filename = obj['Key'][obj['Key'].rfind('/') + 1 : ]

s3_client.download_file(bucket, obj.key, os.path.join(path, filename))
s3_client.download_file(bucket, obj['Key'], os.path.join(path, filename))
cache[obj_key] = response['ETag']

json.dump(cache, open('cache', 'w'))
Expand Down

0 comments on commit 6c0d58e

Please sign in to comment.