From b9eedad961a41ed0cb6441164263bdbdc88b80d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=A9=AC=E5=A4=A9=E9=A9=B0?= Date: Tue, 12 May 2020 11:19:47 +0800 Subject: [PATCH 01/17] dax support --- pynamodb/connection/base.py | 36 +++++++++++++++++++++++++++++++++++- pynamodb/connection/dax.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 pynamodb/connection/dax.py diff --git a/pynamodb/connection/base.py b/pynamodb/connection/base.py index 02fb02b8d..cdd3c7afc 100644 --- a/pynamodb/connection/base.py +++ b/pynamodb/connection/base.py @@ -52,6 +52,7 @@ TableError, QueryError, PutError, DeleteError, UpdateError, GetError, ScanError, TableDoesNotExist, VerboseClientError, TransactGetError, TransactWriteError) +from amazondax.DaxError import DaxClientError from pynamodb.expressions.condition import Condition from pynamodb.expressions.operand import Path from pynamodb.expressions.projection import create_projection_expression @@ -59,6 +60,7 @@ from pynamodb.settings import get_settings_value from pynamodb.signals import pre_dynamodb_send, post_dynamodb_send from pynamodb.types import HASH, RANGE +from pynamodb.connection.dax import OP_READ, OP_WRITE, DaxClient BOTOCORE_EXCEPTIONS = (BotoCoreError, ClientError) RATE_LIMITING_ERROR_CODES = ['ProvisionedThroughputExceededException', 'ThrottlingException'] @@ -243,7 +245,8 @@ class Connection(object): def __init__(self, region=None, host=None, read_timeout_seconds=None, connect_timeout_seconds=None, max_retry_attempts=None, base_backoff_ms=None, - max_pool_connections=None, extra_headers=None): + max_pool_connections=None, extra_headers=None, + dax_write_endpoints=None, dax_read_endpoints=None, fall_back_to_dynamodb=False): self._tables = {} self.host = host self._local = local() @@ -282,6 +285,11 @@ def __init__(self, region=None, host=None, self._extra_headers = extra_headers else: self._extra_headers = get_settings_value('extra_headers') + self.dax_write_endpoints = dax_write_endpoints or [] + self.dax_read_endpoints = dax_read_endpoints or [] + self._dax_write_client = None + self._dax_read_client = None + self.__fall_back_to_dynamodb = fall_back_to_dynamodb def __repr__(self): return six.u("Connection<{}>".format(self.client.meta.endpoint_url)) @@ -363,6 +371,14 @@ def _make_api_call(self, operation_name, operation_kwargs): 1. It's faster to avoid using botocore's response parsing 2. It provides a place to monkey patch HTTP requests for unit testing """ + try: + if operation_name in OP_WRITE and self.dax_write_endpoints: + return self.dax_write_client.dispatch(operation_name, operation_kwargs) + elif operation_name in OP_READ and self.dax_read_endpoints: + return self.dax_read_client.dispatch(operation_name, operation_kwargs) + except DaxClientError as err: + if not self._fall_back_to_dynamodb: + raise operation_model = self.client._service_model.operation_model(operation_name) request_dict = self.client._convert_to_request_dict( operation_kwargs, @@ -535,6 +551,24 @@ def client(self): self._client = self.session.create_client(SERVICE_NAME, self.region, endpoint_url=self.host, config=config) return self._client + @property + def dax_write_client(self): + if self._dax_write_client is None: + self._dax_write_client = DaxClient( + endpoints=self.dax_write_endpoints, + region_name=self.region + ) + return self._dax_write_client + + @property + def dax_read_client(self): + if self._dax_read_client is None: + self._dax_read_client = DaxClient( + endpoints=self.dax_read_endpoints, + region_name=self.region + ) + return self._dax_read_client + def get_meta_table(self, table_name, refresh=False): """ Returns a MetaTable diff --git a/pynamodb/connection/dax.py b/pynamodb/connection/dax.py new file mode 100644 index 000000000..2f31a8bc2 --- /dev/null +++ b/pynamodb/connection/dax.py @@ -0,0 +1,35 @@ +from amazondax import AmazonDaxClient + + +OP_WRITE = { + 'PutItem': 'put_item', + 'DeleteItem': 'delete_item', + 'UpdateItem': 'update_item', + 'BatchWriteItem': 'batch_write_item', + 'TransactWriteItems': 'transact_write_items', + +} + +OP_READ = { + 'GetItem': 'get_item', + 'Scan': 'scan', + 'BatchGetItem': 'batch_get_item', + 'Query': 'query', + 'TransactGetItems': 'transact_get_items', +} + +OP_NAME_TO_METHOD = OP_WRITE.copy() +OP_NAME_TO_METHOD.update(OP_READ) + + +class DaxClient(object): + + def __init__(self, endpoints, region_name): + self.connection = AmazonDaxClient( + endpoints=endpoints, + region_name=region_name + ) + + def dispatch(self, operation_name, kwargs): + method = getattr(self.connection, OP_NAME_TO_METHOD[operation_name]) + return method(**kwargs) From b3d052c6264ef9152846ef919fe8b928ebf27943 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=A9=AC=E5=A4=A9=E9=A9=B0?= Date: Tue, 12 May 2020 12:01:58 +0800 Subject: [PATCH 02/17] dax support --- pynamodb/connection/table.py | 15 +++++++++++++-- pynamodb/models.py | 8 +++++++- pynamodb/settings.py | 3 +++ 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/pynamodb/connection/table.py b/pynamodb/connection/table.py index a8041362a..ec7b7335f 100644 --- a/pynamodb/connection/table.py +++ b/pynamodb/connection/table.py @@ -23,10 +23,18 @@ def __init__(self, extra_headers=None, aws_access_key_id=None, aws_secret_access_key=None, - aws_session_token=None): + aws_session_token=None, + dax_write_endpoints=None, + dax_read_endpoints=None, + fall_back_to_dynamodb=False): + self._hash_keyname = None self._range_keyname = None self.table_name = table_name + if not dax_read_endpoints: + dax_read_endpoints = [] + if not dax_write_endpoints: + dax_write_endpoints = [] self.connection = Connection(region=region, host=host, connect_timeout_seconds=connect_timeout_seconds, @@ -34,7 +42,10 @@ def __init__(self, max_retry_attempts=max_retry_attempts, base_backoff_ms=base_backoff_ms, max_pool_connections=max_pool_connections, - extra_headers=extra_headers) + extra_headers=extra_headers, + dax_write_endpoints=dax_write_endpoints, + dax_read_endpoints=dax_read_endpoints, + fall_back_to_dynamodb=fall_back_to_dynamodb) if aws_access_key_id and aws_secret_access_key: self.connection.session.set_credentials(aws_access_key_id, diff --git a/pynamodb/models.py b/pynamodb/models.py index 8de8c96f7..ec613cdf3 100644 --- a/pynamodb/models.py +++ b/pynamodb/models.py @@ -220,6 +220,10 @@ def __init__(cls, name, bases, attrs): setattr(attr_obj, 'aws_secret_access_key', None) if not hasattr(attr_obj, 'aws_session_token'): setattr(attr_obj, 'aws_session_token', None) + if not hasattr(attr_obj, 'dax_write_endpoints'): + setattr(attr_obj, 'dax_write_endpoints', get_settings_value('dax_write_endpoints')) + if not hasattr(attr_obj, 'dax_read_endpoints'): + setattr(attr_obj, 'dax_read_endpoints', get_settings_value('dax_read_endpoints')) elif isinstance(attr_obj, Index): attr_obj.Meta.model = cls if not hasattr(attr_obj.Meta, "index_name"): @@ -1060,7 +1064,9 @@ def _get_connection(cls): extra_headers=cls.Meta.extra_headers, aws_access_key_id=cls.Meta.aws_access_key_id, aws_secret_access_key=cls.Meta.aws_secret_access_key, - aws_session_token=cls.Meta.aws_session_token) + aws_session_token=cls.Meta.aws_session_token, + dax_write_endpoints=cls.Meta.dax_write_endpoints, + dax_read_endpoints=cls.Meta.dax_read_endpoints) return cls._connection def _deserialize(self, attrs): diff --git a/pynamodb/settings.py b/pynamodb/settings.py index 1b649979f..718c939dd 100644 --- a/pynamodb/settings.py +++ b/pynamodb/settings.py @@ -14,6 +14,9 @@ 'region': 'us-east-1', 'max_pool_connections': 10, 'extra_headers': None, + 'dax_write_endpoints': [], + 'dax_read_endpoints': [], + 'fall_back_to_dynamodb': False } OVERRIDE_SETTINGS_PATH = getenv('PYNAMODB_CONFIG', '/etc/pynamodb/global_default_settings.py') From b0a8c13a9a03eb913dfd98de7f56de5d7d7a678f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=A9=AC=E5=A4=A9=E9=A9=B0?= Date: Tue, 12 May 2020 12:30:15 +0800 Subject: [PATCH 03/17] add session for DaxClient --- pynamodb/connection/base.py | 2 ++ pynamodb/connection/dax.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pynamodb/connection/base.py b/pynamodb/connection/base.py index cdd3c7afc..79b691fb8 100644 --- a/pynamodb/connection/base.py +++ b/pynamodb/connection/base.py @@ -555,6 +555,7 @@ def client(self): def dax_write_client(self): if self._dax_write_client is None: self._dax_write_client = DaxClient( + session=self.session, endpoints=self.dax_write_endpoints, region_name=self.region ) @@ -564,6 +565,7 @@ def dax_write_client(self): def dax_read_client(self): if self._dax_read_client is None: self._dax_read_client = DaxClient( + session=self.session, endpoints=self.dax_read_endpoints, region_name=self.region ) diff --git a/pynamodb/connection/dax.py b/pynamodb/connection/dax.py index 2f31a8bc2..6f92c51d4 100644 --- a/pynamodb/connection/dax.py +++ b/pynamodb/connection/dax.py @@ -24,8 +24,9 @@ class DaxClient(object): - def __init__(self, endpoints, region_name): + def __init__(self, session, endpoints, region_name): self.connection = AmazonDaxClient( + session=session, endpoints=endpoints, region_name=region_name ) From 47e87fae621d169f886febb10d08cc04cc639fe1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=A9=AC=E5=A4=A9=E9=A9=B0?= Date: Tue, 12 May 2020 12:49:13 +0800 Subject: [PATCH 04/17] Revert "add session for DaxClient" This reverts commit b0a8c13a9a03eb913dfd98de7f56de5d7d7a678f. --- pynamodb/connection/base.py | 2 -- pynamodb/connection/dax.py | 3 +-- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/pynamodb/connection/base.py b/pynamodb/connection/base.py index 79b691fb8..cdd3c7afc 100644 --- a/pynamodb/connection/base.py +++ b/pynamodb/connection/base.py @@ -555,7 +555,6 @@ def client(self): def dax_write_client(self): if self._dax_write_client is None: self._dax_write_client = DaxClient( - session=self.session, endpoints=self.dax_write_endpoints, region_name=self.region ) @@ -565,7 +564,6 @@ def dax_write_client(self): def dax_read_client(self): if self._dax_read_client is None: self._dax_read_client = DaxClient( - session=self.session, endpoints=self.dax_read_endpoints, region_name=self.region ) diff --git a/pynamodb/connection/dax.py b/pynamodb/connection/dax.py index 6f92c51d4..2f31a8bc2 100644 --- a/pynamodb/connection/dax.py +++ b/pynamodb/connection/dax.py @@ -24,9 +24,8 @@ class DaxClient(object): - def __init__(self, session, endpoints, region_name): + def __init__(self, endpoints, region_name): self.connection = AmazonDaxClient( - session=session, endpoints=endpoints, region_name=region_name ) From 4112978c87fba075f0863e0cee3bef607085eb99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=A9=AC=E5=A4=A9=E9=A9=B0?= Date: Tue, 12 May 2020 19:22:16 +0800 Subject: [PATCH 05/17] Update dax.py --- pynamodb/connection/dax.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pynamodb/connection/dax.py b/pynamodb/connection/dax.py index 2f31a8bc2..3d8331d39 100644 --- a/pynamodb/connection/dax.py +++ b/pynamodb/connection/dax.py @@ -1,5 +1,9 @@ from amazondax import AmazonDaxClient +import logging + +logger = logging.getLogger("debug") + OP_WRITE = { 'PutItem': 'put_item', @@ -31,5 +35,7 @@ def __init__(self, endpoints, region_name): ) def dispatch(self, operation_name, kwargs): + logger.info(operation_name) + logger.info(kwargs) method = getattr(self.connection, OP_NAME_TO_METHOD[operation_name]) return method(**kwargs) From 7cb1f9a47698b6420ace0e4ba776a1ab62a94a2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=A9=AC=E5=A4=A9=E9=A9=B0?= Date: Tue, 12 May 2020 19:37:44 +0800 Subject: [PATCH 06/17] Update dax.py --- pynamodb/connection/dax.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pynamodb/connection/dax.py b/pynamodb/connection/dax.py index 3d8331d39..00d32c316 100644 --- a/pynamodb/connection/dax.py +++ b/pynamodb/connection/dax.py @@ -18,7 +18,7 @@ 'GetItem': 'get_item', 'Scan': 'scan', 'BatchGetItem': 'batch_get_item', - 'Query': 'query', + # 'Query': 'query', 'TransactGetItems': 'transact_get_items', } From 6f243eac4789b0e1bf8fb7f52810f19f1f479ff1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=A9=AC=E5=A4=A9=E9=A9=B0?= Date: Mon, 18 May 2020 14:44:40 +0800 Subject: [PATCH 07/17] Update dax.py --- pynamodb/connection/dax.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pynamodb/connection/dax.py b/pynamodb/connection/dax.py index 00d32c316..d542c2091 100644 --- a/pynamodb/connection/dax.py +++ b/pynamodb/connection/dax.py @@ -1,9 +1,5 @@ from amazondax import AmazonDaxClient -import logging - -logger = logging.getLogger("debug") - OP_WRITE = { 'PutItem': 'put_item', @@ -35,7 +31,5 @@ def __init__(self, endpoints, region_name): ) def dispatch(self, operation_name, kwargs): - logger.info(operation_name) - logger.info(kwargs) method = getattr(self.connection, OP_NAME_TO_METHOD[operation_name]) return method(**kwargs) From 33256e3d4de5df333c5dcdc356a0fe2c14055580 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=A9=AC=E5=A4=A9=E9=A9=B0?= Date: Fri, 29 May 2020 11:58:22 +0800 Subject: [PATCH 08/17] bug fix import 'List' from typing --- pynamodb/connection/table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pynamodb/connection/table.py b/pynamodb/connection/table.py index c1108dd86..f156b62d9 100644 --- a/pynamodb/connection/table.py +++ b/pynamodb/connection/table.py @@ -3,7 +3,7 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~ """ -from typing import Any, Dict, Mapping, Optional, Sequence +from typing import Any, Dict, Mapping, Optional, Sequence, List from pynamodb.connection.base import Connection, MetaTable from pynamodb.constants import DEFAULT_BILLING_MODE, KEY From a7a044b11f7a230046f4718e85cda1c20d26a069 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=A9=AC=E5=A4=A9=E9=A9=B0?= Date: Fri, 29 May 2020 15:50:56 +0800 Subject: [PATCH 09/17] comment scan and query; add docs of DAX --- docs/dax.rst | 32 ++++++++++++++++++++++++++++++++ docs/index.rst | 2 ++ pynamodb/connection/base.py | 1 - pynamodb/connection/dax.py | 4 +++- 4 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 docs/dax.rst diff --git a/docs/dax.rst b/docs/dax.rst new file mode 100644 index 000000000..4c7e2a248 --- /dev/null +++ b/docs/dax.rst @@ -0,0 +1,32 @@ +.. _dax: + +Use DAX +==================== + +Amazon DynamoDB Accelerator (DAX) is a write-through caching service that is designed to simplify the process of adding a cache to DynamoDB tables. + + +.. note:: + + 'query' and 'scan' requests will not hit DAX due to serious consistent issues. + + Because DAX operates separately from DynamoDB, it is important that you understand the consistency models of both DAX and DynamoDB to ensure that your applications behave as you expect. + See + `the documentation for more information `__. + + +.. code-block:: python + + from pynamodb.models import Model + from pynamodb.attributes import UnicodeAttribute + + + class Thread(Model): + class Meta: + table_name = "Thread" + dax_read_endpoints = ['xxxx:8111'] + dax_write_endpoints = ['xxxx:8111'] + fall_back_to_dynamodb = False + + forum_name = UnicodeAttribute(hash_key=True) + diff --git a/docs/index.rst b/docs/index.rst index e52f36db4..377778ef6 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -19,6 +19,7 @@ Features * Batch operations with automatic pagination * Iterators for working with Query and Scan operations * `Fully tested `_ +* Dax support Topics ====== @@ -47,6 +48,7 @@ Topics contributing release_notes versioning + dax API docs ======== diff --git a/pynamodb/connection/base.py b/pynamodb/connection/base.py index ca2b42382..1f5b9a3c9 100644 --- a/pynamodb/connection/base.py +++ b/pynamodb/connection/base.py @@ -243,7 +243,6 @@ class Connection(object): A higher level abstraction over botocore """ - def __init__(self, region: Optional[str] = None, host: Optional[str] = None, diff --git a/pynamodb/connection/dax.py b/pynamodb/connection/dax.py index d542c2091..6bbf54b9e 100644 --- a/pynamodb/connection/dax.py +++ b/pynamodb/connection/dax.py @@ -12,9 +12,11 @@ OP_READ = { 'GetItem': 'get_item', - 'Scan': 'scan', 'BatchGetItem': 'batch_get_item', + # query and scan has a serious consistency issue + # https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DAX.consistency.html#DAX.consistency.query-cache # 'Query': 'query', + # 'Scan': 'scan', 'TransactGetItems': 'transact_get_items', } From 58189f09be9c597b925907b718538eb470c72826 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=A9=AC=E5=A4=A9=E9=A9=B0?= Date: Fri, 29 May 2020 20:05:35 +0800 Subject: [PATCH 10/17] Update base.py --- pynamodb/connection/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pynamodb/connection/base.py b/pynamodb/connection/base.py index 1f5b9a3c9..292dc7832 100644 --- a/pynamodb/connection/base.py +++ b/pynamodb/connection/base.py @@ -297,7 +297,7 @@ def __init__(self, self.dax_read_endpoints = dax_read_endpoints or [] self._dax_write_client = None self._dax_read_client = None - self.__fall_back_to_dynamodb = fall_back_to_dynamodb + self._fall_back_to_dynamodb = fall_back_to_dynamodb def __repr__(self) -> str: return "Connection<{}>".format(self.client.meta.endpoint_url) From f91a98d6a40eddde329cb4a020f0e1b649238f44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=A9=AC=E5=A4=A9=E9=A9=B0?= Date: Fri, 24 Jul 2020 23:54:01 +0800 Subject: [PATCH 11/17] remove fall_back_to_dynamodb --- pynamodb/connection/base.py | 9 +++------ pynamodb/connection/table.py | 6 ++---- pynamodb/settings.py | 3 +-- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/pynamodb/connection/base.py b/pynamodb/connection/base.py index 2eb760550..964e2d460 100644 --- a/pynamodb/connection/base.py +++ b/pynamodb/connection/base.py @@ -253,8 +253,7 @@ def __init__(self, max_pool_connections: Optional[int] = None, extra_headers: Optional[Mapping[str, str]] = None, dax_write_endpoints=None, - dax_read_endpoints=None, - fall_back_to_dynamodb=False): + dax_read_endpoints=None): self._tables: Dict[str, MetaTable] = {} self.host = host self._local = local() @@ -297,7 +296,6 @@ def __init__(self, self.dax_read_endpoints = dax_read_endpoints or [] self._dax_write_client = None self._dax_read_client = None - self._fall_back_to_dynamodb = fall_back_to_dynamodb def __repr__(self) -> str: return "Connection<{}>".format(self.client.meta.endpoint_url) @@ -375,9 +373,8 @@ def _make_api_call(self, operation_name, operation_kwargs): return self.dax_write_client.dispatch(operation_name, operation_kwargs) elif operation_name in OP_READ and self.dax_read_endpoints: return self.dax_read_client.dispatch(operation_name, operation_kwargs) - except DaxClientError as err: - if not self._fall_back_to_dynamodb: - raise + except DaxClientError: + raise operation_model = self.client._service_model.operation_model(operation_name) request_dict = self.client._convert_to_request_dict( operation_kwargs, diff --git a/pynamodb/connection/table.py b/pynamodb/connection/table.py index f156b62d9..638d24c4c 100644 --- a/pynamodb/connection/table.py +++ b/pynamodb/connection/table.py @@ -31,8 +31,7 @@ def __init__( aws_secret_access_key: Optional[str] = None, aws_session_token: Optional[str] = None, dax_write_endpoints: Optional[List[str]] = None, - dax_read_endpoints: Optional[List[str]] = None, - fall_back_to_dynamodb: Optional[bool] = False + dax_read_endpoints: Optional[List[str]] = None ) -> None: self._hash_keyname = None self._range_keyname = None @@ -50,8 +49,7 @@ def __init__( max_pool_connections=max_pool_connections, extra_headers=extra_headers, dax_write_endpoints=dax_write_endpoints, - dax_read_endpoints=dax_read_endpoints, - fall_back_to_dynamodb=fall_back_to_dynamodb) + dax_read_endpoints=dax_read_endpoints) if aws_access_key_id and aws_secret_access_key: self.connection.session.set_credentials(aws_access_key_id, diff --git a/pynamodb/settings.py b/pynamodb/settings.py index 18aa1ef28..c675f8662 100644 --- a/pynamodb/settings.py +++ b/pynamodb/settings.py @@ -17,8 +17,7 @@ 'max_pool_connections': 10, 'extra_headers': None, 'dax_write_endpoints': [], - 'dax_read_endpoints': [], - 'fall_back_to_dynamodb': False + 'dax_read_endpoints': [] } OVERRIDE_SETTINGS_PATH = getenv('PYNAMODB_CONFIG', '/etc/pynamodb/global_default_settings.py') From ed199395e508bec1512c8984a87bd8a9b95a0935 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=A9=AC=E5=A4=A9=E9=A9=B0?= Date: Sat, 25 Jul 2020 00:00:26 +0800 Subject: [PATCH 12/17] Update dax.rst --- docs/dax.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/dax.rst b/docs/dax.rst index 4c7e2a248..e082593f3 100644 --- a/docs/dax.rst +++ b/docs/dax.rst @@ -26,7 +26,6 @@ Amazon DynamoDB Accelerator (DAX) is a write-through caching service that is des table_name = "Thread" dax_read_endpoints = ['xxxx:8111'] dax_write_endpoints = ['xxxx:8111'] - fall_back_to_dynamodb = False forum_name = UnicodeAttribute(hash_key=True) From 2707cb8d33057c558507060bd1f9631cef0fb538 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=A9=AC=E5=A4=A9=E9=A9=B0?= Date: Sat, 25 Jul 2020 16:56:11 +0800 Subject: [PATCH 13/17] add requirements --- requirements-dev.txt | 1 + setup.py | 1 + 2 files changed, 2 insertions(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index faac6c1ed..c2b8409c6 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,6 +1,7 @@ pytest>=5 pytest-env pytest-mock +amazon-dax-client>=1.1.7 # Due to https://github.com/boto/botocore/issues/1872. Remove after botocore fixes. python-dateutil==2.8.0 diff --git a/setup.py b/setup.py index bbf7e1e65..63fda0afc 100644 --- a/setup.py +++ b/setup.py @@ -4,6 +4,7 @@ install_requires = [ 'botocore>=1.12.54', 'python-dateutil>=2.1,<3.0.0', + 'amazon-dax-client>=1.1.7' ] setup( From 7277e4fea2754e257248101981ec946a64322854 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=A9=AC=E5=A4=A9=E9=A9=B0?= Date: Sat, 25 Jul 2020 17:25:11 +0800 Subject: [PATCH 14/17] add type hints --- pynamodb/connection/base.py | 4 ++-- pynamodb/models.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pynamodb/connection/base.py b/pynamodb/connection/base.py index 964e2d460..fe8c9a942 100644 --- a/pynamodb/connection/base.py +++ b/pynamodb/connection/base.py @@ -549,7 +549,7 @@ def client(self): @property def dax_write_client(self): - if self._dax_write_client is None: + if self._dax_write_client is None and self.dax_write_endpoints: self._dax_write_client = DaxClient( endpoints=self.dax_write_endpoints, region_name=self.region @@ -558,7 +558,7 @@ def dax_write_client(self): @property def dax_read_client(self): - if self._dax_read_client is None: + if self._dax_read_client is None and self.dax_read_endpoints: self._dax_read_client = DaxClient( endpoints=self.dax_read_endpoints, region_name=self.region diff --git a/pynamodb/models.py b/pynamodb/models.py index a37ae72d4..f4ff3512f 100644 --- a/pynamodb/models.py +++ b/pynamodb/models.py @@ -192,6 +192,8 @@ class MetaModel(AttributeContainerMeta): aws_session_token: Optional[str] billing_mode: Optional[str] stream_view_type: Optional[str] + dax_write_endpoints: Optional[List[str]] + dax_read_endpoints: Optional[List[str]] """ Model meta class From 3a1f38999eadb91f055051a23c1f246e6b8094a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=A9=AC=E5=A4=A9=E9=A9=B0?= Date: Sat, 25 Jul 2020 17:34:04 +0800 Subject: [PATCH 15/17] Update base.py --- pynamodb/connection/base.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/pynamodb/connection/base.py b/pynamodb/connection/base.py index fe8c9a942..5fd3862bb 100644 --- a/pynamodb/connection/base.py +++ b/pynamodb/connection/base.py @@ -252,8 +252,8 @@ def __init__(self, base_backoff_ms: Optional[int] = None, max_pool_connections: Optional[int] = None, extra_headers: Optional[Mapping[str, str]] = None, - dax_write_endpoints=None, - dax_read_endpoints=None): + dax_write_endpoints: Optional[List[str]] = None, + dax_read_endpoints: Optional[List[str]] = None): self._tables: Dict[str, MetaTable] = {} self.host = host self._local = local() @@ -292,8 +292,14 @@ def __init__(self, self._extra_headers = extra_headers else: self._extra_headers = get_settings_value('extra_headers') - self.dax_write_endpoints = dax_write_endpoints or [] - self.dax_read_endpoints = dax_read_endpoints or [] + if dax_write_endpoints is not None: + self.dax_write_endpoints = dax_write_endpoints + else: + self.dax_write_endpoints = get_settings_value('dax_write_endpoints') + if dax_read_endpoints is not None: + self.dax_read_endpoints = dax_read_endpoints + else: + self.dax_read_endpoints = get_settings_value('dax_read_endpoints') self._dax_write_client = None self._dax_read_client = None From b9f4b6365b445e8ec00e1a21930515d3541a892b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=A9=AC=E5=A4=A9=E9=A9=B0?= Date: Sat, 25 Jul 2020 17:37:38 +0800 Subject: [PATCH 16/17] Update base.py --- pynamodb/connection/base.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pynamodb/connection/base.py b/pynamodb/connection/base.py index 5fd3862bb..6d0c73ce8 100644 --- a/pynamodb/connection/base.py +++ b/pynamodb/connection/base.py @@ -300,8 +300,8 @@ def __init__(self, self.dax_read_endpoints = dax_read_endpoints else: self.dax_read_endpoints = get_settings_value('dax_read_endpoints') - self._dax_write_client = None - self._dax_read_client = None + self._dax_write_client: DaxClient = None + self._dax_read_client: DaxClient = None def __repr__(self) -> str: return "Connection<{}>".format(self.client.meta.endpoint_url) @@ -555,7 +555,7 @@ def client(self): @property def dax_write_client(self): - if self._dax_write_client is None and self.dax_write_endpoints: + if self._dax_write_client is None: self._dax_write_client = DaxClient( endpoints=self.dax_write_endpoints, region_name=self.region @@ -564,7 +564,7 @@ def dax_write_client(self): @property def dax_read_client(self): - if self._dax_read_client is None and self.dax_read_endpoints: + if self._dax_read_client is None: self._dax_read_client = DaxClient( endpoints=self.dax_read_endpoints, region_name=self.region From 00466b37e6cf84fe5c41d9037afa881609c2ebb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=A9=AC=E5=A4=A9=E9=A9=B0?= Date: Sat, 25 Jul 2020 17:48:18 +0800 Subject: [PATCH 17/17] Update base.py --- pynamodb/connection/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pynamodb/connection/base.py b/pynamodb/connection/base.py index 6d0c73ce8..1e9769bb3 100644 --- a/pynamodb/connection/base.py +++ b/pynamodb/connection/base.py @@ -300,8 +300,8 @@ def __init__(self, self.dax_read_endpoints = dax_read_endpoints else: self.dax_read_endpoints = get_settings_value('dax_read_endpoints') - self._dax_write_client: DaxClient = None - self._dax_read_client: DaxClient = None + self._dax_write_client: Optional[DaxClient] = None + self._dax_read_client: Optional[DaxClient] = None def __repr__(self) -> str: return "Connection<{}>".format(self.client.meta.endpoint_url)