exodus-cdn-playbooks should be able to configure log levels & more [R…

…HELDST-9223]

configuration/lambda_config.json

@@ -11,9 +11,9 @@
       "us-east-1"
     ]
   },
-  "config_cache_ttl": 2,
+  "config_cache_ttl": $EXODUS_CONFIG_CACHE_TTL,
   "headers": {
-    "max_age": "600"
+    "max_age": "$EXODUS_HEADERS_MAX_AGE"
   },
   "logging": {
     "version": 1,
@@ -27,10 +27,10 @@
     },
     "loggers": {
       "origin-response": {
-        "level": "WARNING"
+        "level": "$ORIGIN_RESPONSE_LOGGER_LEVEL"
       },
       "origin-request": {
-        "level": "WARNING"
+        "level": "$ORIGIN_REQUEST_LOGGER_LEVEL"
       },
       "default": {
         "level": "WARNING"

exodus_lambda/functions/base.py

@@ -18,22 +18,14 @@ def conf(self):
             if isinstance(self._conf_file, dict):
                 self._conf = self._conf_file
             else:
-                for conf_file in [
-                    self._conf_file,
-                    os.path.join(
-                        os.path.dirname(
-                            os.path.dirname(os.path.dirname(__file__))
-                        ),
-                        "configuration",
-                        "lambda_config.json",
-                    ),
-                ]:
-                    if os.path.exists(conf_file):
-                        with open(
-                            conf_file, "r", encoding="UTF-8"
-                        ) as json_file:
-                            self._conf = json.load(json_file)
-                        break
+                # to be polished: what if the self._conf_file not exist
+                if os.path.exists(self._conf_file):
+                    with open(
+                        self._conf_file, "r", encoding="UTF-8"
+                    ) as json_file:
+                        self._conf = json.load(json_file)
+                else:
+                    self.logger.error("Conf file not found: '%s'", self._conf_file)
         return self._conf
 
     @property

exodus_lambda/functions/origin_request.py

@@ -1,3 +1,4 @@
+import os
 import json
 import time
 import urllib
@@ -9,8 +10,11 @@
 from .base import LambdaBase
 
 
+CONF_FILE = os.environ.get("EXODUS_LAMBDA_CONF_FILE") or "lambda_config.json"
+
+
 class OriginRequest(LambdaBase):
-    def __init__(self, conf_file="lambda_config.json"):
+    def __init__(self, conf_file=CONF_FILE):
         super().__init__("origin-request", conf_file)
         self._db_client = None
         self._cache = cachetools.TTLCache(

exodus_lambda/functions/origin_response.py

@@ -1,11 +1,15 @@
+import os
 import json
 from base64 import b64encode
 
 from .base import LambdaBase
 
 
+CONF_FILE = os.environ.get("EXODUS_LAMBDA_CONF_FILE") or "lambda_config.json"
+
+
 class OriginResponse(LambdaBase):
-    def __init__(self, conf_file="lambda_config.json"):
+    def __init__(self, conf_file=CONF_FILE):
         super().__init__("origin-response", conf_file)
 
     def handler(self, event, context):

scripts/build-package

@@ -1,6 +1,11 @@
 #!/bin/bash
 set -e
 
+export ORIGIN_RESPONSE_LOGGER_LEVEL=${ORIGIN_RESPONSE_LOGGER_LEVEL:-WARNING}
+export ORIGIN_REQUEST_LOGGER_LEVEL=${ORIGIN_REQUEST_LOGGER_LEVEL:-WARNING}
+export EXODUS_HEADERS_MAX_AGE=${EXODUS_HEADERS_MAX_AGE:-600}
+export EXODUS_CONFIG_CACHE_TTL=${EXODUS_CONFIG_CACHE_TTL:2}
+
 pip install --require-hashes -r requirements.txt --target ./package
 pip install --no-deps --target ./package .
 cp ./configuration/exodus-lambda-deploy.yaml ./package

tests/conftest.py

@@ -1,5 +1,8 @@
+import os
 import boto3
 import pytest
+import tempfile
+import subprocess
 
 
 def get_distribution_url(stack_name):
@@ -41,3 +44,29 @@ def cdn_test_url(request):
     else:
         pytest.skip("Test skipped without --cdn-test-url or --lambda-stack")
     return url
+
+
+def mock_conf_file():
+    temp_file = tempfile.NamedTemporaryFile(delete=False)
+
+    os.environ["PROJECT"] = "test"
+    os.environ["ENV_TYPE"] = "dev"
+    os.environ["EXODUS_CONFIG_CACHE_TTL"] = "2"
+    os.environ["ORIGIN_RESPONSE_LOGGER_LEVEL"] = "DEBUG"
+    os.environ["ORIGIN_REQUEST_LOGGER_LEVEL"] = "DEBUG"
+    os.environ["EXODUS_HEADERS_MAX_AGE"] = "600"
+    os.environ["EXODUS_LAMBDA_CONF_FILE"] = temp_file.name
+
+    cmd = "envsubst < ./configuration/lambda_config.json > {temp_path}"
+    cmd = cmd.format(temp_path=temp_file.name)
+
+    subprocess.run(
+        cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        check=True,
+        shell=True,
+    )
+
+
+mock_conf_file()

tests/functions/test_alias.py

@@ -5,8 +5,7 @@
 
 from ..test_utils.utils import generate_test_config
 
-CONF_PATH = "configuration/lambda_config.json"
-TEST_CONF = generate_test_config(CONF_PATH)
+TEST_CONF = generate_test_config()
 
 Alias = namedtuple("Alias", ["src", "dest"])
 

tests/functions/test_base.py

@@ -1,18 +1,19 @@
 import logging
-
+import os
 import pytest
 
 from exodus_lambda.functions.base import LambdaBase
 
 from ..test_utils.utils import generate_test_config
 
-CONF_PATH = "configuration/lambda_config.json"
-TEST_CONF = generate_test_config(CONF_PATH)
+
+CONF_FILE = os.environ.get("EXODUS_LAMBDA_CONF_FILE")
+TEST_CONF = generate_test_config()
 
 
 def test_base_handler():
     with pytest.raises(NotImplementedError):
-        LambdaBase(conf_file=CONF_PATH).handler(event=None, context=None)
+        LambdaBase(conf_file=CONF_FILE).handler(event=None, context=None)
 
 
 @pytest.mark.parametrize(

tests/functions/test_origin_request.py

@@ -11,8 +11,7 @@
 
 TEST_PATH = "/origin/rpms/repo/ver/dir/filename.ext"
 MOCKED_DT = "2020-02-17T15:38:05.864+00:00"
-CONF_PATH = "configuration/lambda_config.json"
-TEST_CONF = generate_test_config(CONF_PATH)
+TEST_CONF = generate_test_config()
 
 
 @pytest.mark.parametrize(

tests/functions/test_origin_response.py

@@ -6,8 +6,7 @@
 
 from ..test_utils.utils import generate_test_config
 
-CONF_PATH = "configuration/lambda_config.json"
-TEST_CONF = generate_test_config(CONF_PATH)
+TEST_CONF = generate_test_config()
 MAX_AGE = TEST_CONF["headers"]["max_age"]
 
 
@@ -60,7 +59,7 @@ def test_origin_response_valid_headers(original_uri, want_digest):
             }
         ]
 
-    response = OriginResponse(conf_file=CONF_PATH).handler(event, context=None)
+    response = OriginResponse(conf_file=TEST_CONF).handler(event, context=None)
     assert response["headers"] == expected_headers
 
 
@@ -75,14 +74,14 @@ def test_origin_response_empty_headers(test_input):
         ]
     }
 
-    response = OriginResponse(conf_file=CONF_PATH).handler(event, context=None)
+    response = OriginResponse(conf_file=TEST_CONF).handler(event, context=None)
     assert response["headers"] == {}
 
 
 def test_origin_response_missing_headers():
     event = {"Records": [{"cf": {"request": {}, "response": {}}}]}
 
-    response = OriginResponse(conf_file=CONF_PATH).handler(event, context=None)
+    response = OriginResponse(conf_file=TEST_CONF).handler(event, context=None)
     assert response == {}
 
 

tests/test_utils/utils.py

@@ -1,42 +1,55 @@
 import json
 import os
 
+CONF_FILE = os.environ.get("EXODUS_LAMBDA_CONF_FILE")
 
-def generate_test_config(conf="configuration/lambda_config.json"):
-    with open(conf, "r") as json_file:
-        conf = json.load(json_file)
-
-    # table
-    conf["table"]["name"] = "test-table"
-    conf["table"]["available_regions"] = [
-        "us-east-1",
-        "us-east-2",
-        "us-west-1",
-        "us-west-2",
-        "ca-central-1",
-        "eu-central-1",
-        "eu-west-2",
-        "eu-west-3",
-        "eu-west-1",
-        "sa-east-1",
-        "ap-south-1",
-        "ap-northeast-2",
-        "ap-southeast-1",
-        "ap-southeast-2",
-        "ap-northeast-1",
-    ]
-
-    # logging
-    conf["logging"]["formatters"]["default"][
-        "format"
-    ] = "[%(levelname)s] - %(message)s\n"
-    conf["logging"]["loggers"]["origin-response"]["level"] = "DEBUG"
-    conf["logging"]["loggers"]["origin-request"]["level"] = "DEBUG"
-    conf["logging"]["loggers"]["default"]["level"] = "DEBUG"
-
-    conf["config_table"]["name"] = "test-config-table"
-    conf["config_table"]["cache_ttl"] = 2
 
+def generate_test_config(conf="configuration/lambda_config.json"):
+    conf = {
+        "table": {
+            "name": "test-table",
+            "available_regions": [
+                "us-east-1",
+                "us-east-2",
+                "us-west-1",
+                "us-west-2",
+                "ca-central-1",
+                "eu-central-1",
+                "eu-west-2",
+                "eu-west-3",
+                "eu-west-1",
+                "sa-east-1",
+                "ap-south-1",
+                "ap-northeast-2",
+                "ap-southeast-1",
+                "ap-southeast-2",
+                "ap-northeast-1",
+            ],
+        },
+        "config_table": {
+            "name": "test-config-table",
+            "available_regions": ["us-east-1"],
+            "cache_ttl": 2,
+        },
+        "config_cache_ttl": 2,
+        "headers": {"max_age": "600"},
+        "logging": {
+            "version": 1,
+            "incremental": "True",
+            "disable_existing_loggers": "False",
+            "formatters": {
+                "default": {
+                    "format": "[%(levelname)s] - %(message)s\n",
+                    "datefmt": "%Y-%m-%d %H:%M:%S",
+                }
+            },
+            "loggers": {
+                "origin-response": {"level": "DEBUG"},
+                "origin-request": {"level": "DEBUG"},
+                "default": {"level": "DEBUG"},
+            },
+        },
+    }
     return conf