Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[FEATURE] Implement TupleS3StoreBackend::get_all #9692

Merged
merged 4 commits into from
Apr 3, 2024
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
23 changes: 15 additions & 8 deletions great_expectations/data_context/store/tuple_store_backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -543,13 +543,25 @@
return s3_object_key

def _get(self, key):
client = self._create_client()

Check warning on line 546 in great_expectations/data_context/store/tuple_store_backend.py

View check run for this annotation

Codecov / codecov/patch

great_expectations/data_context/store/tuple_store_backend.py#L546

Added line #L546 was not covered by tests
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm do we not create this during init?

s3_object_key = self._build_s3_object_key(key)
return self._get_by_s3_object_key(client, s3_object_key)

Check warning on line 548 in great_expectations/data_context/store/tuple_store_backend.py

View check run for this annotation

Codecov / codecov/patch

great_expectations/data_context/store/tuple_store_backend.py#L548

Added line #L548 was not covered by tests

s3 = self._create_client()
@override
def _get_all(self) -> list[Any]:
"""Get all objects from the store.
NOTE: This is non-performant because we download each object separately.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there any reference to docs/code we can put here?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

good call. Updated.

"""
client = self._create_client()
keys = self.list_keys()
keys = [k for k in keys if k != StoreBackend.STORE_BACKEND_ID_KEY]
s3_object_keys = [self._build_s3_object_key(key) for key in keys]
return [self._get_by_s3_object_key(client, key) for key in s3_object_keys]

Check warning on line 559 in great_expectations/data_context/store/tuple_store_backend.py

View check run for this annotation

Codecov / codecov/patch

great_expectations/data_context/store/tuple_store_backend.py#L555-L559

Added lines #L555 - L559 were not covered by tests

def _get_by_s3_object_key(self, s3_client, s3_object_key):
try:
s3_response_object = s3.get_object(Bucket=self.bucket, Key=s3_object_key)
except (s3.exceptions.NoSuchKey, s3.exceptions.NoSuchBucket):
s3_response_object = s3_client.get_object(Bucket=self.bucket, Key=s3_object_key)
except (s3_client.exceptions.NoSuchKey, s3_client.exceptions.NoSuchBucket):

Check warning on line 564 in great_expectations/data_context/store/tuple_store_backend.py

View check run for this annotation

Codecov / codecov/patch

great_expectations/data_context/store/tuple_store_backend.py#L563-L564

Added lines #L563 - L564 were not covered by tests
raise InvalidKeyError( # noqa: TRY003
f"Unable to retrieve object from TupleS3StoreBackend with the following Key: {s3_object_key!s}" # noqa: E501
)
Expand All @@ -560,10 +572,6 @@
.decode(s3_response_object.get("ContentEncoding", "utf-8"))
)

@override
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Moved up to be with its friend _get

def _get_all(self) -> list[Any]:
raise NotImplementedError

def _set(
self,
key,
Expand Down Expand Up @@ -636,7 +644,6 @@
key = self._convert_filepath_to_key(s3_object_key)
if key:
key_list.append(key)

return key_list

def get_url_for_key(self, key, protocol=None):
Expand Down
22 changes: 22 additions & 0 deletions tests/data_context/store/test_store_backends.py
Original file line number Diff line number Diff line change
Expand Up @@ -629,6 +629,28 @@ def test_TupleS3StoreBackend_with_prefix(aws_credentials):
assert my_new_store.get_public_url_for_key(("BBB",)) == "http://www.test.com/my_file_BBB"


@mock_s3
@pytest.mark.aws_deps
def test_TupleS3StoreBackend_get_all(aws_credentials):
bucket = "leakybucket"

# create a bucket in Moto's mock AWS environment
conn = boto3.resource("s3", region_name="us-east-1")
conn.create_bucket(Bucket=bucket)

my_store = TupleS3StoreBackend(filepath_template="my_file_{0}", bucket=bucket)

val_a = "aaa"
val_b = "bbb"

my_store.set(("AAA",), val_a, content_type="text/html; charset=utf-8")
my_store.set(("BBB",), val_b, content_type="text/html; charset=utf-8")

result = my_store.get_all()

assert sorted(result) == [val_a, val_b]


@mock_s3
@pytest.mark.aws_deps
def test_tuple_s3_store_backend_slash_conditions(aws_credentials): # noqa: PLR0915
Expand Down