refactor(idempotent): Change UX to use a config class for non-persist…

…ence related features (#306)

* refactor(idempotent): Create a config class

* fix: Reanable test

* chore: Some refactoring

* docs: Update docs

* docs(batch): add example on how to integrate with sentry.io (#308)

* chore(docs): Update the docs

* fix(tests): Fix coverage-html and various update

* refactor: Change back to configure

* chore: Hide missing code coverage

* chore: bump ci

* chore: bump ci

Co-authored-by: Heitor Lessa <lessa@amazon.co.uk>

Makefile

@@ -21,7 +21,7 @@ test:
 	poetry run pytest --cache-clear tests/performance
 
 coverage-html:
-	poetry run pytest --cov-report html
+	poetry run pytest -m "not perf" --cov-report=html
 
 pr: lint test security-baseline complexity-baseline
 

aws_lambda_powertools/tracing/extensions.py

@@ -8,8 +8,8 @@ def aiohttp_trace_config():
     TraceConfig
         aiohttp trace config
     """
-    from aws_xray_sdk.ext.aiohttp.client import aws_xray_trace_config
+    from aws_xray_sdk.ext.aiohttp.client import aws_xray_trace_config  # pragma: no cover
 
-    aws_xray_trace_config.__doc__ = "aiohttp extension for X-Ray (aws_xray_trace_config)"
+    aws_xray_trace_config.__doc__ = "aiohttp extension for X-Ray (aws_xray_trace_config)"  # pragma: no cover
 
-    return aws_xray_trace_config()
+    return aws_xray_trace_config()  # pragma: no cover

aws_lambda_powertools/utilities/idempotency/__init__.py

@@ -5,6 +5,6 @@
 from aws_lambda_powertools.utilities.idempotency.persistence.base import BasePersistenceLayer
 from aws_lambda_powertools.utilities.idempotency.persistence.dynamodb import DynamoDBPersistenceLayer
 
-from .idempotency import idempotent
+from .idempotency import IdempotencyConfig, idempotent
 
-__all__ = ("DynamoDBPersistenceLayer", "BasePersistenceLayer", "idempotent")
+__all__ = ("DynamoDBPersistenceLayer", "BasePersistenceLayer", "idempotent", "IdempotencyConfig")

aws_lambda_powertools/utilities/idempotency/config.py

@@ -0,0 +1,43 @@
+from typing import Dict
+
+
+class IdempotencyConfig:
+    def __init__(
+        self,
+        event_key_jmespath: str = "",
+        payload_validation_jmespath: str = "",
+        jmespath_options: Dict = None,
+        raise_on_no_idempotency_key: bool = False,
+        expires_after_seconds: int = 60 * 60,  # 1 hour default
+        use_local_cache: bool = False,
+        local_cache_max_items: int = 256,
+        hash_function: str = "md5",
+    ):
+        """
+        Initialize the base persistence layer
+
+        Parameters
+        ----------
+        event_key_jmespath: str
+            A jmespath expression to extract the idempotency key from the event record
+        payload_validation_jmespath: str
+            A jmespath expression to extract the payload to be validated from the event record
+        raise_on_no_idempotency_key: bool, optional
+            Raise exception if no idempotency key was found in the request, by default False
+        expires_after_seconds: int
+            The number of seconds to wait before a record is expired
+        use_local_cache: bool, optional
+            Whether to locally cache idempotency results, by default False
+        local_cache_max_items: int, optional
+            Max number of items to store in local cache, by default 1024
+        hash_function: str, optional
+            Function to use for calculating hashes, by default md5.
+        """
+        self.event_key_jmespath = event_key_jmespath
+        self.payload_validation_jmespath = payload_validation_jmespath
+        self.jmespath_options = jmespath_options
+        self.raise_on_no_idempotency_key = raise_on_no_idempotency_key
+        self.expires_after_seconds = expires_after_seconds
+        self.use_local_cache = use_local_cache
+        self.local_cache_max_items = local_cache_max_items
+        self.hash_function = hash_function

aws_lambda_powertools/utilities/idempotency/idempotency.py

@@ -5,6 +5,7 @@
 from typing import Any, Callable, Dict, Optional
 
 from aws_lambda_powertools.middleware_factory import lambda_handler_decorator
+from aws_lambda_powertools.utilities.idempotency.config import IdempotencyConfig
 from aws_lambda_powertools.utilities.idempotency.exceptions import (
     IdempotencyAlreadyInProgressError,
     IdempotencyInconsistentStateError,
@@ -29,6 +30,7 @@ def idempotent(
     event: Dict[str, Any],
     context: LambdaContext,
     persistence_store: BasePersistenceLayer,
+    config: IdempotencyConfig = None,
 ) -> Any:
     """
     Middleware to handle idempotency
@@ -43,20 +45,25 @@ def idempotent(
         Lambda's Context
     persistence_store: BasePersistenceLayer
         Instance of BasePersistenceLayer to store data
+    config: IdempotencyConfig
+        Configutation
 
     Examples
     --------
     **Processes Lambda's event in an idempotent manner**
-        >>> from aws_lambda_powertools.utilities.idempotency import idempotent, DynamoDBPersistenceLayer
+        >>> from aws_lambda_powertools.utilities.idempotency import (
+        >>>    idempotent, DynamoDBPersistenceLayer, IdempotencyConfig
+        >>> )
         >>>
-        >>> persistence_store = DynamoDBPersistenceLayer(event_key_jmespath="body", table_name="idempotency_store")
+        >>> idem_config=IdempotencyConfig(event_key_jmespath="body")
+        >>> persistence_layer = DynamoDBPersistenceLayer(table_name="idempotency_store")
         >>>
-        >>> @idempotent(persistence_store=persistence_store)
+        >>> @idempotent(config=idem_config, persistence_store=persistence_layer)
         >>> def handler(event, context):
         >>>     return {"StatusCode": 200}
     """
 
-    idempotency_handler = IdempotencyHandler(handler, event, context, persistence_store)
+    idempotency_handler = IdempotencyHandler(handler, event, context, config or IdempotencyConfig(), persistence_store)
 
     # IdempotencyInconsistentStateError can happen under rare but expected cases when persistent state changes in the
     # small time between put & get requests. In most cases we can retry successfully on this exception.
@@ -82,6 +89,7 @@ def __init__(
         lambda_handler: Callable[[Any, LambdaContext], Any],
         event: Dict[str, Any],
         context: LambdaContext,
+        config: IdempotencyConfig,
         persistence_store: BasePersistenceLayer,
     ):
         """
@@ -98,6 +106,7 @@ def __init__(
         persistence_store : BasePersistenceLayer
             Instance of persistence layer to store idempotency records
         """
+        persistence_store.configure(config)
         self.persistence_store = persistence_store
         self.context = context
         self.event = event

aws_lambda_powertools/utilities/idempotency/persistence/base.py

@@ -9,13 +9,14 @@
 import warnings
 from abc import ABC, abstractmethod
 from types import MappingProxyType
-from typing import Any, Dict
+from typing import Any, Dict, Optional
 
 import jmespath
 
 from aws_lambda_powertools.shared.cache_dict import LRUDict
 from aws_lambda_powertools.shared.jmespath_functions import PowertoolsFunctions
 from aws_lambda_powertools.shared.json_encoder import Encoder
+from aws_lambda_powertools.utilities.idempotency.config import IdempotencyConfig
 from aws_lambda_powertools.utilities.idempotency.exceptions import (
     IdempotencyInvalidStatusError,
     IdempotencyItemAlreadyExistsError,
@@ -107,55 +108,49 @@ class BasePersistenceLayer(ABC):
     Abstract Base Class for Idempotency persistence layer.
     """
 
-    def __init__(
-        self,
-        event_key_jmespath: str = "",
-        payload_validation_jmespath: str = "",
-        expires_after_seconds: int = 60 * 60,  # 1 hour default
-        use_local_cache: bool = False,
-        local_cache_max_items: int = 256,
-        hash_function: str = "md5",
-        raise_on_no_idempotency_key: bool = False,
-        jmespath_options: Dict = None,
-    ) -> None:
+    def __init__(self):
+        """Initialize the defaults """
+        self.configured = False
+        self.event_key_jmespath: Optional[str] = None
+        self.event_key_compiled_jmespath = None
+        self.jmespath_options: Optional[dict] = None
+        self.payload_validation_enabled = False
+        self.validation_key_jmespath = None
+        self.raise_on_no_idempotency_key = False
+        self.expires_after_seconds: int = 60 * 60  # 1 hour default
+        self.use_local_cache = False
+        self._cache: Optional[LRUDict] = None
+        self.hash_function = None
+
+    def configure(self, config: IdempotencyConfig) -> None:
         """
-        Initialize the base persistence layer
+        Initialize the base persistence layer from the configuration settings
 
         Parameters
         ----------
-        event_key_jmespath: str
-            A jmespath expression to extract the idempotency key from the event record
-        payload_validation_jmespath: str
-            A jmespath expression to extract the payload to be validated from the event record
-        expires_after_seconds: int
-            The number of seconds to wait before a record is expired
-        use_local_cache: bool, optional
-            Whether to locally cache idempotency results, by default False
-        local_cache_max_items: int, optional
-            Max number of items to store in local cache, by default 1024
-        hash_function: str, optional
-            Function to use for calculating hashes, by default md5.
-        raise_on_no_idempotency_key: bool, optional
-            Raise exception if no idempotency key was found in the request, by default False
-        jmespath_options : Dict
-            Alternative JMESPath options to be included when filtering expr
-        """
-        self.event_key_jmespath = event_key_jmespath
-        if self.event_key_jmespath:
-            self.event_key_compiled_jmespath = jmespath.compile(event_key_jmespath)
-        self.expires_after_seconds = expires_after_seconds
-        self.use_local_cache = use_local_cache
-        if self.use_local_cache:
-            self._cache = LRUDict(max_items=local_cache_max_items)
-        self.payload_validation_enabled = False
-        if payload_validation_jmespath:
-            self.validation_key_jmespath = jmespath.compile(payload_validation_jmespath)
+        config: IdempotencyConfig
+            Idempotency configuration settings
+        """
+        if self.configured:
+            # Prevent being reconfigured multiple times
+            return
+        self.configured = True
+
+        self.event_key_jmespath = config.event_key_jmespath
+        if config.event_key_jmespath:
+            self.event_key_compiled_jmespath = jmespath.compile(config.event_key_jmespath)
+        self.jmespath_options = config.jmespath_options
+        if not self.jmespath_options:
+            self.jmespath_options = {"custom_functions": PowertoolsFunctions()}
+        if config.payload_validation_jmespath:
+            self.validation_key_jmespath = jmespath.compile(config.payload_validation_jmespath)
             self.payload_validation_enabled = True
-        self.hash_function = getattr(hashlib, hash_function)
-        self.raise_on_no_idempotency_key = raise_on_no_idempotency_key
-        if not jmespath_options:
-            jmespath_options = {"custom_functions": PowertoolsFunctions()}
-        self.jmespath_options = jmespath_options
+        self.raise_on_no_idempotency_key = config.raise_on_no_idempotency_key
+        self.expires_after_seconds = config.expires_after_seconds
+        self.use_local_cache = config.use_local_cache
+        if self.use_local_cache:
+            self._cache = LRUDict(max_items=config.local_cache_max_items)
+        self.hash_function = getattr(hashlib, config.hash_function)
 
     def _get_hashed_idempotency_key(self, lambda_event: Dict[str, Any]) -> str:
         """
@@ -180,9 +175,9 @@ def _get_hashed_idempotency_key(self, lambda_event: Dict[str, Any]) -> str:
             )
 
         if self.is_missing_idempotency_key(data):
-            warnings.warn(f"No value found for idempotency_key. jmespath: {self.event_key_jmespath}")
             if self.raise_on_no_idempotency_key:
                 raise IdempotencyKeyError("No data found to create a hashed idempotency_key")
+            warnings.warn(f"No value found for idempotency_key. jmespath: {self.event_key_jmespath}")
 
         return self._generate_hash(data)
 

aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py

@@ -26,8 +26,6 @@ def __init__(
         validation_key_attr: str = "validation",
         boto_config: Optional[Config] = None,
         boto3_session: Optional[boto3.session.Session] = None,
-        *args,
-        **kwargs,
     ):
         """
         Initialize the DynamoDB client
@@ -57,9 +55,9 @@ def __init__(
         **Create a DynamoDB persistence layer with custom settings**
             >>> from aws_lambda_powertools.utilities.idempotency import idempotent, DynamoDBPersistenceLayer
             >>>
-            >>> persistence_store = DynamoDBPersistenceLayer(event_key="body", table_name="idempotency_store")
+            >>> persistence_store = DynamoDBPersistenceLayer(table_name="idempotency_store")
             >>>
-            >>> @idempotent(persistence_store=persistence_store)
+            >>> @idempotent(persistence_store=persistence_store, event_key="body")
             >>> def handler(event, context):
             >>>     return {"StatusCode": 200}
         """
@@ -74,7 +72,7 @@ def __init__(
         self.status_attr = status_attr
         self.data_attr = data_attr
         self.validation_key_attr = validation_key_attr
-        super(DynamoDBPersistenceLayer, self).__init__(*args, **kwargs)
+        super(DynamoDBPersistenceLayer, self).__init__()
 
     def _item_to_data_record(self, item: Dict[str, Any]) -> DataRecord:
         """