From 258f3a08c23727e4e48d9eeec20f5848401f86d5 Mon Sep 17 00:00:00 2001 From: Caleigh Runge-Hottman Date: Wed, 20 Oct 2021 19:05:05 -0400 Subject: [PATCH] Consume cdn-definitions from table [RHELDST-7898] The exodus-lambda functions previously consumed the cdn definitions from the the exodus-lambda package. This workflow required the lambdas to be rebuilt and redeployed after each update to the cdn-definitions data set. Release engineers should be able to update the cdn-definitions data set without rebuilding and redeploying the lambdas. The definitions are now stored in a $project-config-$env DynamoDB table, which is decoupled from the exodus-lambda package. --- configuration/lambda_config.json | 7 +++ exodus_lambda/functions/base.py | 16 +++++- exodus_lambda/functions/origin_request.py | 46 +++++++++++---- requirements.in | 1 + requirements.txt | 4 ++ scripts/build-package | 3 - tests/functions/test_alias.py | 11 ++-- tests/functions/test_origin_request.py | 70 ++++++++++++++++++++--- tests/test_utils/utils.py | 13 ++++- 9 files changed, 141 insertions(+), 30 deletions(-) diff --git a/configuration/lambda_config.json b/configuration/lambda_config.json index 1435edc2..32060970 100755 --- a/configuration/lambda_config.json +++ b/configuration/lambda_config.json @@ -5,6 +5,13 @@ "us-east-1" ] }, + "config_table": { + "name": "$PROJECT-config-$ENV_TYPE", + "available_regions": [ + "us-east-1" + ] + }, + "config_cache_ttl": 2, "headers": { "max_age": "600" }, diff --git a/exodus_lambda/functions/base.py b/exodus_lambda/functions/base.py index cd3eac2d..fe6d0f7a 100644 --- a/exodus_lambda/functions/base.py +++ b/exodus_lambda/functions/base.py @@ -17,8 +17,20 @@ def conf(self): if isinstance(self._conf_file, dict): self._conf = self._conf_file else: - with open(self._conf_file, "r") as json_file: - self._conf = json.load(json_file) + for conf_file in [ + self._conf_file, + os.path.join( + os.path.dirname( + os.path.dirname(os.path.dirname(__file__)) + ), + "configuration", + "lambda_config.json", + ), + ]: + if os.path.exists(conf_file): + with open(conf_file, "r") as json_file: + self._conf = json.load(json_file) + break return self._conf @property diff --git a/exodus_lambda/functions/origin_request.py b/exodus_lambda/functions/origin_request.py index 53eabf5b..809a96a0 100755 --- a/exodus_lambda/functions/origin_request.py +++ b/exodus_lambda/functions/origin_request.py @@ -1,27 +1,53 @@ import json +import time import urllib -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone import boto3 -from cdn_definitions import load_data +import cachetools from .base import LambdaBase class OriginRequest(LambdaBase): - def __init__( - self, conf_file="lambda_config.json", definitions_source=None - ): + def __init__(self, conf_file="lambda_config.json"): super().__init__("origin-request", conf_file) self._db_client = None - self._definitions_source = definitions_source - self._definitions = None + self._cache = cachetools.TTLCache( + maxsize=1, + ttl=timedelta( + minutes=self.conf.get("config_cache_ttl", 2) + ).total_seconds(), + timer=time.monotonic, + ) @property def definitions(self): - if self._definitions is None: - self._definitions = load_data(source=self._definitions_source) - return self._definitions + out = self._cache.get("exodus-config") + if out is None: + table = self.conf["config_table"]["name"] + + query_result = self.db_client.query( + TableName=table, + Limit=1, + ScanIndexForward=False, + KeyConditionExpression="config_id = :id and from_date <= :d", + ExpressionAttributeValues={ + ":id": {"S": "exodus-config"}, + ":d": { + "S": str( + datetime.now(timezone.utc).isoformat( + timespec="milliseconds" + ) + ) + }, + }, + ) + if query_result["Items"]: + item = query_result["Items"][0] + out = json.loads(item["config"]["S"]) + self._cache["exodus-config"] = out + return out @property def db_client(self): diff --git a/requirements.in b/requirements.in index 125b7dcf..1ed689de 100644 --- a/requirements.in +++ b/requirements.in @@ -4,3 +4,4 @@ docutils cdn-definitions requests PyYAML +cachetools diff --git a/requirements.txt b/requirements.txt index 4d8a6c71..0b16535d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,6 +15,10 @@ botocore==1.21.17 \ # -r requirements.in # boto3 # s3transfer +cachetools==4.2.4 \ + --hash=sha256:89ea6f1b638d5a73a4f9226be57ac5e4f399d22770b92355f92dcb0f7f001693 \ + --hash=sha256:92971d3cb7d2a97efff7c7bb1657f21a8f5fb309a37530537c71b1774189f2d1 + # via -r requirements.in cdn-definitions==2.1.0 \ --hash=sha256:2d4438564435e50c9a1a2db4928d622a2c09453dc3942ca8eea9d8f98ee1977a # via -r requirements.in diff --git a/scripts/build-package b/scripts/build-package index 08af4654..87d9b5ce 100755 --- a/scripts/build-package +++ b/scripts/build-package @@ -3,9 +3,6 @@ set -e pip install --require-hashes -r requirements.txt --target ./package pip install --no-deps --target ./package . -git clone https://${GITHUB_TOKEN}@github.com/release-engineering/cdn-definitions-private.git -mv ./cdn-definitions-private/data.yaml ./package/cdn_definitions/data.yaml -cp ./configuration/exodus-lambda-deploy.yaml ./package envsubst < ./configuration/lambda_config.json > ./package/lambda_config.json aws cloudformation package \ --template ./package/exodus-lambda-deploy.yaml \ diff --git a/tests/functions/test_alias.py b/tests/functions/test_alias.py index b2517f0e..d7b73d4f 100644 --- a/tests/functions/test_alias.py +++ b/tests/functions/test_alias.py @@ -3,7 +3,10 @@ from exodus_lambda.functions.origin_request import OriginRequest -CONF_PATH = "exodus_lambda/functions/lambda_config.json" +from ..test_utils.utils import generate_test_config + +CONF_PATH = "configuration/lambda_config.json" +TEST_CONF = generate_test_config(CONF_PATH) Alias = namedtuple("Alias", ["src", "dest"]) @@ -11,7 +14,7 @@ def test_alias_single(): """Each alias is only applied a single time.""" - req = OriginRequest(CONF_PATH) + req = OriginRequest(conf_file=TEST_CONF) aliases = [ {"src": "/foo/bar", "dest": ""}, @@ -28,7 +31,7 @@ def test_alias_single(): def test_alias_boundary(): """Aliases are only resolved at path boundaries.""" - req = OriginRequest(CONF_PATH) + req = OriginRequest(conf_file=TEST_CONF) aliases = [{"src": "/foo/bar", "dest": "/"}] @@ -39,7 +42,7 @@ def test_alias_boundary(): def test_alias_equal(): """Paths exactly matching an alias can be resolved.""" - req = OriginRequest(CONF_PATH) + req = OriginRequest(conf_file=TEST_CONF) aliases = [{"src": "/foo/bar", "dest": "/quux"}] diff --git a/tests/functions/test_origin_request.py b/tests/functions/test_origin_request.py index 8575b743..5e412e84 100644 --- a/tests/functions/test_origin_request.py +++ b/tests/functions/test_origin_request.py @@ -65,9 +65,11 @@ ], ) @mock.patch("boto3.client") +@mock.patch("exodus_lambda.functions.origin_request.cachetools") @mock.patch("exodus_lambda.functions.origin_request.datetime") def test_origin_request( mocked_datetime, + mocked_cache, mocked_boto3_client, req_uri, real_uri, @@ -75,6 +77,7 @@ def test_origin_request( caplog, ): mocked_datetime.now().isoformat.return_value = MOCKED_DT + mocked_cache.TTLCache.return_value = {"exodus-config": mock_definitions()} mocked_boto3_client().query.return_value = { "Items": [ { @@ -90,7 +93,7 @@ def test_origin_request( with caplog.at_level(logging.DEBUG): request = OriginRequest( - conf_file=TEST_CONF, definitions_source=mock_definitions() + conf_file=TEST_CONF, ).handler(event, context=None) assert "Item found for '%s'" % real_uri in caplog.text @@ -109,28 +112,34 @@ def test_origin_request( @mock.patch("boto3.client") +@mock.patch("exodus_lambda.functions.origin_request.cachetools") @mock.patch("exodus_lambda.functions.origin_request.datetime") -def test_origin_request_no_item(mocked_datetime, mocked_boto3_client, caplog): +def test_origin_request_no_item( + mocked_datetime, mocked_cache, mocked_boto3_client, caplog +): mocked_datetime.now().isoformat.return_value = MOCKED_DT + mocked_cache.TTLCache.return_value = {"exodus-config": mock_definitions()} mocked_boto3_client().query.return_value = {"Items": []} event = {"Records": [{"cf": {"request": {"uri": TEST_PATH}}}]} with caplog.at_level(logging.DEBUG): - request = OriginRequest( - conf_file=TEST_CONF, definitions_source=mock_definitions() - ).handler(event, context=None) + request = OriginRequest(conf_file=TEST_CONF).handler( + event, context=None + ) assert request == {"status": "404", "statusDescription": "Not Found"} assert "No item found for '%s'" % TEST_PATH in caplog.text @mock.patch("boto3.client") +@mock.patch("exodus_lambda.functions.origin_request.cachetools") @mock.patch("exodus_lambda.functions.origin_request.datetime") def test_origin_request_invalid_item( - mocked_datetime, mocked_boto3_client, caplog + mocked_datetime, mocked_cache, mocked_boto3_client, caplog ): mocked_datetime.now().isoformat.return_value = MOCKED_DT + mocked_cache.TTLCache.return_value = {"exodus-config": mock_definitions()} mocked_boto3_client().query.return_value = { "Items": [ { @@ -143,9 +152,7 @@ def test_origin_request_invalid_item( event = {"Records": [{"cf": {"request": {"uri": TEST_PATH}}}]} with pytest.raises(KeyError): - OriginRequest( - conf_file=TEST_CONF, definitions_source=mock_definitions() - ).handler(event, context=None) + OriginRequest(conf_file=TEST_CONF).handler(event, context=None) assert ( "Exception occurred while processing %s" @@ -157,3 +164,48 @@ def test_origin_request_invalid_item( ) in caplog.text ) + + +@mock.patch("boto3.client") +def test_origin_request_definitions(mocked_boto3_client): + mocked_defs = mock_definitions() + mocked_boto3_client().query.return_value = { + "Items": [ + { + "from_date": {"S": "2020-02-17T00:00:00.000+00:00"}, + "config_id": {"S": "exodus-config"}, + "config": {"S": json.dumps(mocked_defs)}, + } + ] + } + + assert mocked_defs == OriginRequest(conf_file=TEST_CONF).definitions + + +@pytest.mark.parametrize( + "cur_time, count", [(1741221.281833067, 2), (1741025.281833067, 1)] +) +@mock.patch("boto3.client") +@mock.patch("exodus_lambda.functions.origin_request.time.monotonic") +def test_origin_request_definitions_cache( + mocked_time, mocked_boto3_client, cur_time, count +): + mocked_defs = mock_definitions() + mocked_time.return_value = 1741021.281833067 + mocked_boto3_client().query.return_value = { + "Items": [ + { + "from_date": {"S": "2020-02-17T00:00:00.000+00:00"}, + "config_id": {"S": "exodus-config"}, + "config": {"S": json.dumps(mocked_defs)}, + } + ] + } + + obj = OriginRequest(conf_file=TEST_CONF) + obj.definitions + assert obj._cache.currsize == 1 # pylint:disable=protected-access + + mocked_time.return_value = cur_time + obj.definitions + assert mocked_boto3_client().query.call_count == count diff --git a/tests/test_utils/utils.py b/tests/test_utils/utils.py index 08c58d4c..18f5e340 100644 --- a/tests/test_utils/utils.py +++ b/tests/test_utils/utils.py @@ -1,6 +1,8 @@ import json import os +from cdn_definitions import load_data + def generate_test_config(conf="configuration/lambda_config.json"): with open(conf, "r") as json_file: @@ -34,10 +36,17 @@ def generate_test_config(conf="configuration/lambda_config.json"): conf["logging"]["loggers"]["origin-request"]["level"] = "DEBUG" conf["logging"]["loggers"]["default"]["level"] = "DEBUG" + conf["config_table"]["name"] = "test-config-table" + conf["config_table"]["cache_ttl"] = 2 + return conf def mock_definitions(): - return os.path.join( - os.path.dirname(os.path.dirname(__file__)), "test_data", "data.yaml" + return load_data( + os.path.join( + os.path.dirname(os.path.dirname(__file__)), + "test_data", + "data.yaml", + ) )