Merge pull request release-engineering#284 from crungehottman/RHELDST…

…-7898

Consume cdn-definitions from table [RHELDST-7898]

configuration/lambda_config.json

@@ -5,6 +5,13 @@
       "us-east-1"
     ]
   },
+  "config_table": {
+    "name": "$PROJECT-config-$ENV_TYPE",
+    "available_regions": [
+      "us-east-1"
+    ]
+  },
+  "config_cache_ttl": 2,
   "headers": {
     "max_age": "600"
   },

exodus_lambda/functions/base.py

@@ -17,8 +17,20 @@ def conf(self):
             if isinstance(self._conf_file, dict):
                 self._conf = self._conf_file
             else:
-                with open(self._conf_file, "r") as json_file:
-                    self._conf = json.load(json_file)
+                for conf_file in [
+                    self._conf_file,
+                    os.path.join(
+                        os.path.dirname(
+                            os.path.dirname(os.path.dirname(__file__))
+                        ),
+                        "configuration",
+                        "lambda_config.json",
+                    ),
+                ]:
+                    if os.path.exists(conf_file):
+                        with open(conf_file, "r") as json_file:
+                            self._conf = json.load(json_file)
+                        break
         return self._conf
 
     @property

exodus_lambda/functions/origin_request.py

@@ -1,27 +1,53 @@
 import json
+import time
 import urllib
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 
 import boto3
-from cdn_definitions import load_data
+import cachetools
 
 from .base import LambdaBase
 
 
 class OriginRequest(LambdaBase):
-    def __init__(
-        self, conf_file="lambda_config.json", definitions_source=None
-    ):
+    def __init__(self, conf_file="lambda_config.json"):
         super().__init__("origin-request", conf_file)
         self._db_client = None
-        self._definitions_source = definitions_source
-        self._definitions = None
+        self._cache = cachetools.TTLCache(
+            maxsize=1,
+            ttl=timedelta(
+                minutes=self.conf.get("config_cache_ttl", 2)
+            ).total_seconds(),
+            timer=time.monotonic,
+        )
 
     @property
     def definitions(self):
-        if self._definitions is None:
-            self._definitions = load_data(source=self._definitions_source)
-        return self._definitions
+        out = self._cache.get("exodus-config")
+        if out is None:
+            table = self.conf["config_table"]["name"]
+
+            query_result = self.db_client.query(
+                TableName=table,
+                Limit=1,
+                ScanIndexForward=False,
+                KeyConditionExpression="config_id = :id and from_date <= :d",
+                ExpressionAttributeValues={
+                    ":id": {"S": "exodus-config"},
+                    ":d": {
+                        "S": str(
+                            datetime.now(timezone.utc).isoformat(
+                                timespec="milliseconds"
+                            )
+                        )
+                    },
+                },
+            )
+            if query_result["Items"]:
+                item = query_result["Items"][0]
+                out = json.loads(item["config"]["S"])
+                self._cache["exodus-config"] = out
+        return out
 
     @property
     def db_client(self):

requirements.in

@@ -4,3 +4,4 @@ docutils
 cdn-definitions
 requests
 PyYAML
+cachetools

requirements.txt

@@ -15,6 +15,10 @@ botocore==1.21.63 \
     #   -r requirements.in
     #   boto3
     #   s3transfer
+cachetools==4.2.4 \
+    --hash=sha256:89ea6f1b638d5a73a4f9226be57ac5e4f399d22770b92355f92dcb0f7f001693 \
+    --hash=sha256:92971d3cb7d2a97efff7c7bb1657f21a8f5fb309a37530537c71b1774189f2d1
+    # via -r requirements.in
 cdn-definitions==2.1.0 \
     --hash=sha256:2d4438564435e50c9a1a2db4928d622a2c09453dc3942ca8eea9d8f98ee1977a
     # via -r requirements.in

scripts/build-package

@@ -3,9 +3,6 @@ set -e
 
 pip install --require-hashes -r requirements.txt --target ./package
 pip install --no-deps --target ./package .
-git clone https://${GITHUB_TOKEN}@github.com/release-engineering/cdn-definitions-private.git
-mv ./cdn-definitions-private/data.yaml ./package/cdn_definitions/data.yaml
-cp ./configuration/exodus-lambda-deploy.yaml ./package
 envsubst < ./configuration/lambda_config.json > ./package/lambda_config.json
 aws cloudformation package \
 	--template ./package/exodus-lambda-deploy.yaml \

tests/functions/test_alias.py

@@ -3,15 +3,18 @@
 
 from exodus_lambda.functions.origin_request import OriginRequest
 
-CONF_PATH = "exodus_lambda/functions/lambda_config.json"
+from ..test_utils.utils import generate_test_config
+
+CONF_PATH = "configuration/lambda_config.json"
+TEST_CONF = generate_test_config(CONF_PATH)
 
 Alias = namedtuple("Alias", ["src", "dest"])
 
 
 def test_alias_single():
     """Each alias is only applied a single time."""
 
-    req = OriginRequest(CONF_PATH)
+    req = OriginRequest(conf_file=TEST_CONF)
 
     aliases = [
         {"src": "/foo/bar", "dest": ""},
@@ -28,7 +31,7 @@ def test_alias_single():
 def test_alias_boundary():
     """Aliases are only resolved at path boundaries."""
 
-    req = OriginRequest(CONF_PATH)
+    req = OriginRequest(conf_file=TEST_CONF)
 
     aliases = [{"src": "/foo/bar", "dest": "/"}]
 
@@ -39,7 +42,7 @@ def test_alias_boundary():
 def test_alias_equal():
     """Paths exactly matching an alias can be resolved."""
 
-    req = OriginRequest(CONF_PATH)
+    req = OriginRequest(conf_file=TEST_CONF)
 
     aliases = [{"src": "/foo/bar", "dest": "/quux"}]
 

tests/functions/test_origin_request.py

@@ -65,16 +65,19 @@
     ],
 )
 @mock.patch("boto3.client")
+@mock.patch("exodus_lambda.functions.origin_request.cachetools")
 @mock.patch("exodus_lambda.functions.origin_request.datetime")
 def test_origin_request(
     mocked_datetime,
+    mocked_cache,
     mocked_boto3_client,
     req_uri,
     real_uri,
     content_type,
     caplog,
 ):
     mocked_datetime.now().isoformat.return_value = MOCKED_DT
+    mocked_cache.TTLCache.return_value = {"exodus-config": mock_definitions()}
     mocked_boto3_client().query.return_value = {
         "Items": [
             {
@@ -90,7 +93,7 @@ def test_origin_request(
 
     with caplog.at_level(logging.DEBUG):
         request = OriginRequest(
-            conf_file=TEST_CONF, definitions_source=mock_definitions()
+            conf_file=TEST_CONF,
         ).handler(event, context=None)
 
     assert "Item found for '%s'" % real_uri in caplog.text
@@ -109,28 +112,34 @@ def test_origin_request(
 
 
 @mock.patch("boto3.client")
+@mock.patch("exodus_lambda.functions.origin_request.cachetools")
 @mock.patch("exodus_lambda.functions.origin_request.datetime")
-def test_origin_request_no_item(mocked_datetime, mocked_boto3_client, caplog):
+def test_origin_request_no_item(
+    mocked_datetime, mocked_cache, mocked_boto3_client, caplog
+):
     mocked_datetime.now().isoformat.return_value = MOCKED_DT
+    mocked_cache.TTLCache.return_value = {"exodus-config": mock_definitions()}
     mocked_boto3_client().query.return_value = {"Items": []}
 
     event = {"Records": [{"cf": {"request": {"uri": TEST_PATH}}}]}
 
     with caplog.at_level(logging.DEBUG):
-        request = OriginRequest(
-            conf_file=TEST_CONF, definitions_source=mock_definitions()
-        ).handler(event, context=None)
+        request = OriginRequest(conf_file=TEST_CONF).handler(
+            event, context=None
+        )
 
     assert request == {"status": "404", "statusDescription": "Not Found"}
     assert "No item found for '%s'" % TEST_PATH in caplog.text
 
 
 @mock.patch("boto3.client")
+@mock.patch("exodus_lambda.functions.origin_request.cachetools")
 @mock.patch("exodus_lambda.functions.origin_request.datetime")
 def test_origin_request_invalid_item(
-    mocked_datetime, mocked_boto3_client, caplog
+    mocked_datetime, mocked_cache, mocked_boto3_client, caplog
 ):
     mocked_datetime.now().isoformat.return_value = MOCKED_DT
+    mocked_cache.TTLCache.return_value = {"exodus-config": mock_definitions()}
     mocked_boto3_client().query.return_value = {
         "Items": [
             {
@@ -143,9 +152,7 @@ def test_origin_request_invalid_item(
     event = {"Records": [{"cf": {"request": {"uri": TEST_PATH}}}]}
 
     with pytest.raises(KeyError):
-        OriginRequest(
-            conf_file=TEST_CONF, definitions_source=mock_definitions()
-        ).handler(event, context=None)
+        OriginRequest(conf_file=TEST_CONF).handler(event, context=None)
 
     assert (
         "Exception occurred while processing %s"
@@ -157,3 +164,48 @@ def test_origin_request_invalid_item(
         )
         in caplog.text
     )
+
+
+@mock.patch("boto3.client")
+def test_origin_request_definitions(mocked_boto3_client):
+    mocked_defs = mock_definitions()
+    mocked_boto3_client().query.return_value = {
+        "Items": [
+            {
+                "from_date": {"S": "2020-02-17T00:00:00.000+00:00"},
+                "config_id": {"S": "exodus-config"},
+                "config": {"S": json.dumps(mocked_defs)},
+            }
+        ]
+    }
+
+    assert mocked_defs == OriginRequest(conf_file=TEST_CONF).definitions
+
+
+@pytest.mark.parametrize(
+    "cur_time, count", [(1741221.281833067, 2), (1741025.281833067, 1)]
+)
+@mock.patch("boto3.client")
+@mock.patch("exodus_lambda.functions.origin_request.time.monotonic")
+def test_origin_request_definitions_cache(
+    mocked_time, mocked_boto3_client, cur_time, count
+):
+    mocked_defs = mock_definitions()
+    mocked_time.return_value = 1741021.281833067
+    mocked_boto3_client().query.return_value = {
+        "Items": [
+            {
+                "from_date": {"S": "2020-02-17T00:00:00.000+00:00"},
+                "config_id": {"S": "exodus-config"},
+                "config": {"S": json.dumps(mocked_defs)},
+            }
+        ]
+    }
+
+    obj = OriginRequest(conf_file=TEST_CONF)
+    obj.definitions
+    assert obj._cache.currsize == 1  # pylint:disable=protected-access
+
+    mocked_time.return_value = cur_time
+    obj.definitions
+    assert mocked_boto3_client().query.call_count == count

tests/test_utils/utils.py

@@ -1,6 +1,8 @@
 import json
 import os
 
+from cdn_definitions import load_data
+
 
 def generate_test_config(conf="configuration/lambda_config.json"):
     with open(conf, "r") as json_file:
@@ -34,10 +36,17 @@ def generate_test_config(conf="configuration/lambda_config.json"):
     conf["logging"]["loggers"]["origin-request"]["level"] = "DEBUG"
     conf["logging"]["loggers"]["default"]["level"] = "DEBUG"
 
+    conf["config_table"]["name"] = "test-config-table"
+    conf["config_table"]["cache_ttl"] = 2
+
     return conf
 
 
 def mock_definitions():
-    return os.path.join(
-        os.path.dirname(os.path.dirname(__file__)), "test_data", "data.yaml"
+    return load_data(
+        os.path.join(
+            os.path.dirname(os.path.dirname(__file__)),
+            "test_data",
+            "data.yaml",
+        )
     )