-
Notifications
You must be signed in to change notification settings - Fork 1.5k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[FEATURE] Implement TupleS3StoreBackend::get_all #9692
Changes from 3 commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -543,13 +543,25 @@ | |
return s3_object_key | ||
|
||
def _get(self, key): | ||
client = self._create_client() | ||
s3_object_key = self._build_s3_object_key(key) | ||
return self._get_by_s3_object_key(client, s3_object_key) | ||
|
||
s3 = self._create_client() | ||
@override | ||
def _get_all(self) -> list[Any]: | ||
"""Get all objects from the store. | ||
NOTE: This is non-performant because we download each object separately. | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Is there any reference to docs/code we can put here? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. good call. Updated. |
||
""" | ||
client = self._create_client() | ||
keys = self.list_keys() | ||
keys = [k for k in keys if k != StoreBackend.STORE_BACKEND_ID_KEY] | ||
s3_object_keys = [self._build_s3_object_key(key) for key in keys] | ||
return [self._get_by_s3_object_key(client, key) for key in s3_object_keys] | ||
|
||
def _get_by_s3_object_key(self, s3_client, s3_object_key): | ||
try: | ||
s3_response_object = s3.get_object(Bucket=self.bucket, Key=s3_object_key) | ||
except (s3.exceptions.NoSuchKey, s3.exceptions.NoSuchBucket): | ||
s3_response_object = s3_client.get_object(Bucket=self.bucket, Key=s3_object_key) | ||
except (s3_client.exceptions.NoSuchKey, s3_client.exceptions.NoSuchBucket): | ||
raise InvalidKeyError( # noqa: TRY003 | ||
f"Unable to retrieve object from TupleS3StoreBackend with the following Key: {s3_object_key!s}" # noqa: E501 | ||
) | ||
|
@@ -560,10 +572,6 @@ | |
.decode(s3_response_object.get("ContentEncoding", "utf-8")) | ||
) | ||
|
||
@override | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Moved up to be with its friend |
||
def _get_all(self) -> list[Any]: | ||
raise NotImplementedError | ||
|
||
def _set( | ||
self, | ||
key, | ||
|
@@ -636,7 +644,6 @@ | |
key = self._convert_filepath_to_key(s3_object_key) | ||
if key: | ||
key_list.append(key) | ||
|
||
return key_list | ||
|
||
def get_url_for_key(self, key, protocol=None): | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Hmm do we not create this during init?