From b981260648aff4ace0579a01b746dd0483fa8534 Mon Sep 17 00:00:00 2001 From: Leo Singer Date: Wed, 7 Feb 2024 11:50:04 -0500 Subject: [PATCH] Don't require upstream patches for boto3 The AWS Python SDK, [boto3], has [resource] objects that provide high-level interfaces to AWS services. The [DynamoDB resource] greatly simplifies marshalling and unmarshalling data. We rely on the resource method for [TransactWriteItems] among others that are absent from boto3. We opened PR https://github.com/boto/boto3/pull/4010 to add that method. The resource methods are synthesized at runtime from a data file. Fortunately, boto3 has a [Loader] mechanism that allows the user to add extra data files, and the [loader search path] is configurable. In order to not depend upon our upstream PR for boto3, we distribute the extra data files and fix up the loader search path by putting it in a [.pth file] which Python executes automatically during startup. FIXME: The .pth file method does not currently work when we are doing editable installs. It _should_ work with editable installs if we factor the data files and the .pth file to a separate package that we add as a dependency. [boto3]: https://github.com/boto/boto3 [resource]: https://boto3.amazonaws.com/v1/documentation/api/latest/guide/resources.html [DynamoDB resource]: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#resources [TransactWriteItems]: https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_TransactWriteItems.html [Loader]: https://botocore.amazonaws.com/v1/documentation/api/latest/reference/loaders.html [loader search path]: https://botocore.amazonaws.com/v1/documentation/api/latest/reference/loaders.html#the-search-path [.pth file]: https://docs.python.org/3/library/site.html --- README.md | 5 ---- boto3_missing.pth | 1 + boto3_missing/__init__.py | 20 ++++++++++++++ .../2012-08-10/resources-1.sdk-extras.json | 23 ++++++++++++++++ dynamodb_autoincrement.py | 27 ++++++------------- pyproject.toml | 10 ++++--- test_dynamodb_autoincrement.py | 10 +++---- 7 files changed, 64 insertions(+), 32 deletions(-) create mode 100644 boto3_missing.pth create mode 100644 boto3_missing/__init__.py create mode 100644 boto3_missing/data/dynamodb/2012-08-10/resources-1.sdk-extras.json diff --git a/README.md b/README.md index f05e33a..77a2868 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,3 @@ Use optimistic locking to put DynamoDB records with auto-incrementing attributes - https://aws.amazon.com/blogs/aws/new-amazon-dynamodb-transactions/ - https://bitesizedserverless.com/bite/reliable-auto-increments-in-dynamodb/ - -## FIXME - -This package currently depends on code that is in a pull request for boto3 that is not yet merged or released. -See https://github.com/boto/boto3/pull/4010. diff --git a/boto3_missing.pth b/boto3_missing.pth new file mode 100644 index 0000000..deea47b --- /dev/null +++ b/boto3_missing.pth @@ -0,0 +1 @@ +import boto3_missing; boto3_missing.install() \ No newline at end of file diff --git a/boto3_missing/__init__.py b/boto3_missing/__init__.py new file mode 100644 index 0000000..c0cbd78 --- /dev/null +++ b/boto3_missing/__init__.py @@ -0,0 +1,20 @@ +# Copyright © 2023 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +"""Add missing boto3 SDK data. + +See +https://botocore.amazonaws.com/v1/documentation/api/latest/reference/loaders.html, +https://github.com/boto/boto3/pull/4010 +""" + +from os import environ, pathsep + +from . import data + + +def install(): + new_path = [*data.__path__] + if orig_path := environ.get("AWS_DATA_PATH"): + new_path.extend(orig_path.split(pathsep)) + environ["AWS_DATA_PATH"] = pathsep.join(new_path) diff --git a/boto3_missing/data/dynamodb/2012-08-10/resources-1.sdk-extras.json b/boto3_missing/data/dynamodb/2012-08-10/resources-1.sdk-extras.json new file mode 100644 index 0000000..1913383 --- /dev/null +++ b/boto3_missing/data/dynamodb/2012-08-10/resources-1.sdk-extras.json @@ -0,0 +1,23 @@ +{ + "merge": { + "service": { + "actions": { + "GetItem": { + "request": { "operation": "GetItem" } + }, + "PutItem": { + "request": { "operation": "PutItem" } + }, + "Query": { + "request": { "operation": "Query" } + }, + "Scan": { + "request": { "operation": "Scan" } + }, + "TransactWriteItems": { + "request": { "operation": "TransactWriteItems" } + } + } + } + } +} diff --git a/dynamodb_autoincrement.py b/dynamodb_autoincrement.py index 252c89c..d514ed4 100644 --- a/dynamodb_autoincrement.py +++ b/dynamodb_autoincrement.py @@ -9,6 +9,9 @@ from mypy_boto3_dynamodb.service_resource import DynamoDBServiceResource +# FIXME: remove instances of 'type: ignore[attr-defined]' below once +# boto3-missing becomes unnecessary. + PrimitiveDynamoDBValues = Optional[Union[str, int, float, Decimal, bool]] DynamoDBValues = Union[ @@ -33,18 +36,6 @@ class BaseDynamoDBAutoIncrement(ABC): def next(self, item: DynamoDBItem) -> tuple[Iterable[dict[str, Any]], str]: raise NotImplementedError - def _put_item(self, *, TableName, **kwargs): - # FIXME: DynamoDB resource does not have put_item method; emulate it - self.dynamodb.Table(TableName).put_item(**kwargs) - - def _get_item(self, *, TableName, **kwargs): - # FIXME: DynamoDB resource does not have get_item method; emulate it - return self.dynamodb.Table(TableName).get_item(**kwargs) - - def _query(self, *, TableName, **kwargs): - # FIXME: DynamoDB resource does not have put_item method; emulate it - return self.dynamodb.Table(TableName).query(**kwargs) - def put(self, item: DynamoDBItem): TransactionCanceledException = ( self.dynamodb.meta.client.exceptions.TransactionCanceledException @@ -53,11 +44,9 @@ def put(self, item: DynamoDBItem): puts, next_counter = self.next(item) if self.dangerously: for put in puts: - self._put_item(**put) + self.dynamodb.put_item(**put) # type: ignore[attr-defined] else: try: - # FIXME: depends on an unmerged PR for boto3. - # See https://github.com/boto/boto3/pull/4010 self.dynamodb.transact_write_items( # type: ignore[attr-defined] TransactItems=[{"Put": put} for put in puts] ) @@ -69,7 +58,7 @@ def put(self, item: DynamoDBItem): class DynamoDBAutoIncrement(BaseDynamoDBAutoIncrement): def next(self, item): counter = ( - self._get_item( + self.dynamodb.get_item( AttributesToGet=[self.attribute_name], Key=self.counter_table_key, TableName=self.counter_table_name, @@ -117,7 +106,7 @@ def next(self, item): class DynamoDBHistoryAutoIncrement(BaseDynamoDBAutoIncrement): def list(self) -> list[int]: - result = self._query( + result = self.dynamodb.query( # type: ignore[attr-defined] TableName=self.table_name, ExpressionAttributeNames={ **{f"#{i}": key for i, key in enumerate(self.counter_table_key.keys())}, @@ -145,10 +134,10 @@ def get(self, version: Optional[int] = None) -> DynamoDBItem: "TableName": self.table_name, "Key": {**self.counter_table_key, self.attribute_name: version}, } - return self._get_item(**kwargs).get("Item") + return self.dynamodb.get_item(**kwargs).get("Item") # type: ignore[attr-defined] def next(self, item): - existing_item = self._get_item( + existing_item = self.dynamodb.get_item( TableName=self.counter_table_name, Key=self.counter_table_key, ).get("Item") diff --git a/pyproject.toml b/pyproject.toml index bdd637e..be58e8c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ classifiers = [ "Topic :: Database", ] dependencies = [ - "boto3 @ git+https://github.com/lpsinger/boto3@dynamodb-resource-transact-write-items", + "boto3", "boto3-stubs[dynamodb]", ] requires-python = ">=3.9" @@ -41,9 +41,13 @@ exclude_also = [ "@abstractmethod" ] -[tool.setuptools] -py-modules = [ +[tool.setuptools.packages.find] +include = [ "dynamodb_autoincrement", + "boto3_missing", ] +[tool.setuptools.package-data] +boto3_missing = ["**/*.json", "../*.pth"] + [tool.setuptools_scm] diff --git a/test_dynamodb_autoincrement.py b/test_dynamodb_autoincrement.py index f773e46..04db3bf 100644 --- a/test_dynamodb_autoincrement.py +++ b/test_dynamodb_autoincrement.py @@ -91,7 +91,7 @@ def test_autoincrement_safely(autoincrement_safely, dynamodb, last_id): if last_id is None: next_id = 1 else: - dynamodb.Table("autoincrement").put_item( + dynamodb.put_item(TableName="autoincrement", Item={"tableName": "widgets", "widgetID": last_id} ) next_id = last_id + 1 @@ -99,11 +99,11 @@ def test_autoincrement_safely(autoincrement_safely, dynamodb, last_id): result = autoincrement_safely.put({"widgetName": "runcible spoon"}) assert result == next_id - assert dynamodb.Table("widgets").scan()["Items"] == [ + assert dynamodb.scan(TableName="widgets")["Items"] == [ {"widgetID": next_id, "widgetName": "runcible spoon"}, ] - assert dynamodb.Table("autoincrement").scan()["Items"] == [ + assert dynamodb.scan(TableName="autoincrement")["Items"] == [ { "tableName": "widgets", "widgetID": next_id, @@ -152,7 +152,7 @@ def test_autoincrement_dangerously_fails_on_many_parallel_puts( @pytest.fixture(params=[None, {"widgetID": 1}, {"widgetID": 1, "version": 1}]) def initial_item(request, create_tables, dynamodb): if request.param is not None: - dynamodb.Table("widgets").put_item(Item=request.param) + dynamodb.put_item(TableName="widgets", Item=request.param) return request.param @@ -174,7 +174,7 @@ def test_autoincrement_version( ) assert new_version == 1 + has_initial_item - history_items = dynamodb.Table("widgetHistory").query( + history_items = dynamodb.query( TableName="widgetHistory", KeyConditionExpression="widgetID = :widgetID", ExpressionAttributeValues={