From 2b92aca8143939ac554f201f555b6de94866fe95 Mon Sep 17 00:00:00 2001 From: Di Chen Date: Mon, 13 Dec 2021 12:38:22 +0800 Subject: [PATCH] exodus-cdn-playbooks should be able to configure log levels & more [RHELDST-9223] --- configuration/lambda_config.json | 8 +-- exodus_lambda/functions/base.py | 24 +++---- exodus_lambda/functions/origin_request.py | 6 +- exodus_lambda/functions/origin_response.py | 6 +- scripts/build-package | 5 ++ tests/conftest.py | 29 ++++++++ tests/functions/test_alias.py | 3 +- tests/functions/test_base.py | 9 +-- tests/functions/test_origin_request.py | 3 +- tests/functions/test_origin_response.py | 9 ++- tests/test_utils/utils.py | 81 +++++++++++++--------- 11 files changed, 114 insertions(+), 69 deletions(-) diff --git a/configuration/lambda_config.json b/configuration/lambda_config.json index 32060970..98fcd2ca 100755 --- a/configuration/lambda_config.json +++ b/configuration/lambda_config.json @@ -11,9 +11,9 @@ "us-east-1" ] }, - "config_cache_ttl": 2, + "config_cache_ttl": $EXODUS_CONFIG_CACHE_TTL, "headers": { - "max_age": "600" + "max_age": "$EXODUS_HEADERS_MAX_AGE" }, "logging": { "version": 1, @@ -27,10 +27,10 @@ }, "loggers": { "origin-response": { - "level": "WARNING" + "level": "$ORIGIN_RESPONSE_LOGGER_LEVEL" }, "origin-request": { - "level": "WARNING" + "level": "$ORIGIN_REQUEST_LOGGER_LEVEL" }, "default": { "level": "WARNING" diff --git a/exodus_lambda/functions/base.py b/exodus_lambda/functions/base.py index 9c06f240..329eade8 100644 --- a/exodus_lambda/functions/base.py +++ b/exodus_lambda/functions/base.py @@ -18,22 +18,14 @@ def conf(self): if isinstance(self._conf_file, dict): self._conf = self._conf_file else: - for conf_file in [ - self._conf_file, - os.path.join( - os.path.dirname( - os.path.dirname(os.path.dirname(__file__)) - ), - "configuration", - "lambda_config.json", - ), - ]: - if os.path.exists(conf_file): - with open( - conf_file, "r", encoding="UTF-8" - ) as json_file: - self._conf = json.load(json_file) - break + # to be polished: what if the self._conf_file not exist + if os.path.exists(self._conf_file): + with open( + self._conf_file, "r", encoding="UTF-8" + ) as json_file: + self._conf = json.load(json_file) + else: + self.logger.error("Conf file not found: '%s'", self._conf_file) return self._conf @property diff --git a/exodus_lambda/functions/origin_request.py b/exodus_lambda/functions/origin_request.py index 47411b5c..b036de8c 100755 --- a/exodus_lambda/functions/origin_request.py +++ b/exodus_lambda/functions/origin_request.py @@ -1,3 +1,4 @@ +import os import json import time import urllib @@ -9,8 +10,11 @@ from .base import LambdaBase +CONF_FILE = os.environ.get("EXODUS_LAMBDA_CONF_FILE") or "lambda_config.json" + + class OriginRequest(LambdaBase): - def __init__(self, conf_file="lambda_config.json"): + def __init__(self, conf_file=CONF_FILE): super().__init__("origin-request", conf_file) self._db_client = None self._cache = cachetools.TTLCache( diff --git a/exodus_lambda/functions/origin_response.py b/exodus_lambda/functions/origin_response.py index 6ec3c4da..24a80217 100755 --- a/exodus_lambda/functions/origin_response.py +++ b/exodus_lambda/functions/origin_response.py @@ -1,11 +1,15 @@ +import os import json from base64 import b64encode from .base import LambdaBase +CONF_FILE = os.environ.get("EXODUS_LAMBDA_CONF_FILE") or "lambda_config.json" + + class OriginResponse(LambdaBase): - def __init__(self, conf_file="lambda_config.json"): + def __init__(self, conf_file=CONF_FILE): super().__init__("origin-response", conf_file) def handler(self, event, context): diff --git a/scripts/build-package b/scripts/build-package index 01ebfa53..4d5ea886 100755 --- a/scripts/build-package +++ b/scripts/build-package @@ -1,6 +1,11 @@ #!/bin/bash set -e +export ORIGIN_RESPONSE_LOGGER_LEVEL=${ORIGIN_RESPONSE_LOGGER_LEVEL:-WARNING} +export ORIGIN_REQUEST_LOGGER_LEVEL=${ORIGIN_REQUEST_LOGGER_LEVEL:-WARNING} +export EXODUS_HEADERS_MAX_AGE=${EXODUS_HEADERS_MAX_AGE:-600} +export EXODUS_CONFIG_CACHE_TTL=${EXODUS_CONFIG_CACHE_TTL:2} + pip install --require-hashes -r requirements.txt --target ./package pip install --no-deps --target ./package . cp ./configuration/exodus-lambda-deploy.yaml ./package diff --git a/tests/conftest.py b/tests/conftest.py index 7d1bb2e5..cf19d010 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,8 @@ +import os import boto3 import pytest +import tempfile +import subprocess def get_distribution_url(stack_name): @@ -41,3 +44,29 @@ def cdn_test_url(request): else: pytest.skip("Test skipped without --cdn-test-url or --lambda-stack") return url + + +def mock_conf_file(): + temp_file = tempfile.NamedTemporaryFile(delete=False) + + os.environ["PROJECT"] = "test" + os.environ["ENV_TYPE"] = "dev" + os.environ["EXODUS_CONFIG_CACHE_TTL"] = "2" + os.environ["ORIGIN_RESPONSE_LOGGER_LEVEL"] = "DEBUG" + os.environ["ORIGIN_REQUEST_LOGGER_LEVEL"] = "DEBUG" + os.environ["EXODUS_HEADERS_MAX_AGE"] = "600" + os.environ["EXODUS_LAMBDA_CONF_FILE"] = temp_file.name + + cmd = "envsubst < ./configuration/lambda_config.json > {temp_path}" + cmd = cmd.format(temp_path=temp_file.name) + + subprocess.run( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=True, + shell=True, + ) + + +mock_conf_file() diff --git a/tests/functions/test_alias.py b/tests/functions/test_alias.py index d7b73d4f..d377e869 100644 --- a/tests/functions/test_alias.py +++ b/tests/functions/test_alias.py @@ -5,8 +5,7 @@ from ..test_utils.utils import generate_test_config -CONF_PATH = "configuration/lambda_config.json" -TEST_CONF = generate_test_config(CONF_PATH) +TEST_CONF = generate_test_config() Alias = namedtuple("Alias", ["src", "dest"]) diff --git a/tests/functions/test_base.py b/tests/functions/test_base.py index 250d7be3..063ace0a 100644 --- a/tests/functions/test_base.py +++ b/tests/functions/test_base.py @@ -1,18 +1,19 @@ import logging - +import os import pytest from exodus_lambda.functions.base import LambdaBase from ..test_utils.utils import generate_test_config -CONF_PATH = "configuration/lambda_config.json" -TEST_CONF = generate_test_config(CONF_PATH) + +CONF_FILE = os.environ.get("EXODUS_LAMBDA_CONF_FILE") +TEST_CONF = generate_test_config() def test_base_handler(): with pytest.raises(NotImplementedError): - LambdaBase(conf_file=CONF_PATH).handler(event=None, context=None) + LambdaBase(conf_file=CONF_FILE).handler(event=None, context=None) @pytest.mark.parametrize( diff --git a/tests/functions/test_origin_request.py b/tests/functions/test_origin_request.py index 3c4e865d..de983299 100644 --- a/tests/functions/test_origin_request.py +++ b/tests/functions/test_origin_request.py @@ -11,8 +11,7 @@ TEST_PATH = "/origin/rpms/repo/ver/dir/filename.ext" MOCKED_DT = "2020-02-17T15:38:05.864+00:00" -CONF_PATH = "configuration/lambda_config.json" -TEST_CONF = generate_test_config(CONF_PATH) +TEST_CONF = generate_test_config() @pytest.mark.parametrize( diff --git a/tests/functions/test_origin_response.py b/tests/functions/test_origin_response.py index fe7841cf..0c3447af 100644 --- a/tests/functions/test_origin_response.py +++ b/tests/functions/test_origin_response.py @@ -6,8 +6,7 @@ from ..test_utils.utils import generate_test_config -CONF_PATH = "configuration/lambda_config.json" -TEST_CONF = generate_test_config(CONF_PATH) +TEST_CONF = generate_test_config() MAX_AGE = TEST_CONF["headers"]["max_age"] @@ -60,7 +59,7 @@ def test_origin_response_valid_headers(original_uri, want_digest): } ] - response = OriginResponse(conf_file=CONF_PATH).handler(event, context=None) + response = OriginResponse(conf_file=TEST_CONF).handler(event, context=None) assert response["headers"] == expected_headers @@ -75,14 +74,14 @@ def test_origin_response_empty_headers(test_input): ] } - response = OriginResponse(conf_file=CONF_PATH).handler(event, context=None) + response = OriginResponse(conf_file=TEST_CONF).handler(event, context=None) assert response["headers"] == {} def test_origin_response_missing_headers(): event = {"Records": [{"cf": {"request": {}, "response": {}}}]} - response = OriginResponse(conf_file=CONF_PATH).handler(event, context=None) + response = OriginResponse(conf_file=TEST_CONF).handler(event, context=None) assert response == {} diff --git a/tests/test_utils/utils.py b/tests/test_utils/utils.py index c275a63e..f50b808e 100644 --- a/tests/test_utils/utils.py +++ b/tests/test_utils/utils.py @@ -1,42 +1,55 @@ import json import os +CONF_FILE = os.environ.get("EXODUS_LAMBDA_CONF_FILE") -def generate_test_config(conf="configuration/lambda_config.json"): - with open(conf, "r") as json_file: - conf = json.load(json_file) - - # table - conf["table"]["name"] = "test-table" - conf["table"]["available_regions"] = [ - "us-east-1", - "us-east-2", - "us-west-1", - "us-west-2", - "ca-central-1", - "eu-central-1", - "eu-west-2", - "eu-west-3", - "eu-west-1", - "sa-east-1", - "ap-south-1", - "ap-northeast-2", - "ap-southeast-1", - "ap-southeast-2", - "ap-northeast-1", - ] - - # logging - conf["logging"]["formatters"]["default"][ - "format" - ] = "[%(levelname)s] - %(message)s\n" - conf["logging"]["loggers"]["origin-response"]["level"] = "DEBUG" - conf["logging"]["loggers"]["origin-request"]["level"] = "DEBUG" - conf["logging"]["loggers"]["default"]["level"] = "DEBUG" - - conf["config_table"]["name"] = "test-config-table" - conf["config_table"]["cache_ttl"] = 2 +def generate_test_config(conf="configuration/lambda_config.json"): + conf = { + "table": { + "name": "test-table", + "available_regions": [ + "us-east-1", + "us-east-2", + "us-west-1", + "us-west-2", + "ca-central-1", + "eu-central-1", + "eu-west-2", + "eu-west-3", + "eu-west-1", + "sa-east-1", + "ap-south-1", + "ap-northeast-2", + "ap-southeast-1", + "ap-southeast-2", + "ap-northeast-1", + ], + }, + "config_table": { + "name": "test-config-table", + "available_regions": ["us-east-1"], + "cache_ttl": 2, + }, + "config_cache_ttl": 2, + "headers": {"max_age": "600"}, + "logging": { + "version": 1, + "incremental": "True", + "disable_existing_loggers": "False", + "formatters": { + "default": { + "format": "[%(levelname)s] - %(message)s\n", + "datefmt": "%Y-%m-%d %H:%M:%S", + } + }, + "loggers": { + "origin-response": {"level": "DEBUG"}, + "origin-request": {"level": "DEBUG"}, + "default": {"level": "DEBUG"}, + }, + }, + } return conf