From a8df9adc5f008b32615eac0cda0749d51263566f Mon Sep 17 00:00:00 2001 From: Rohan McGovern Date: Tue, 2 Apr 2024 10:41:02 +1000 Subject: [PATCH] Add cache flush endpoint [RHELDST-23276] This commit introduces a new endpoint, "/{env}/cdn-flush". It can be used to explicitly perform cache flushing of the CDN edge when needed, using Akamai Fast Purge API. The intent is for this API to be used only for troubleshooting and for niche scenarios. Later commits should introduce *implicit* cache flushing into the usual publish workflow, so it would generally be unnecessary to use this. --- .safety-policy.yml | 7 + docs/deployment.rst | 60 ++++++ examples/exodus-flush | 111 +++++++++++ exodus_gw/deps.py | 30 ++- exodus_gw/dramatiq/middleware/settings.py | 7 +- exodus_gw/logging.py | 2 + exodus_gw/routers/cdn.py | 68 ++++++- exodus_gw/routers/publish.py | 18 +- exodus_gw/schemas.py | 10 +- exodus_gw/settings.py | 70 +++++++ exodus_gw/worker/__init__.py | 1 + exodus_gw/worker/cache.py | 160 +++++++++++++++ requirements.in | 1 + requirements.txt | 163 ++++++++++++++- scripts/systemd/exodus-gw-sidecar.service | 3 + test-requirements.txt | 55 ++++- tests/routers/test_cdn_cache.py | 63 ++++++ tests/worker/test_cdn_cache.py | 232 ++++++++++++++++++++++ 18 files changed, 1025 insertions(+), 36 deletions(-) create mode 100755 examples/exodus-flush create mode 100644 exodus_gw/worker/cache.py create mode 100644 tests/routers/test_cdn_cache.py create mode 100644 tests/worker/test_cdn_cache.py diff --git a/.safety-policy.yml b/.safety-policy.yml index 4b006f6d..9506001f 100644 --- a/.safety-policy.yml +++ b/.safety-policy.yml @@ -10,4 +10,11 @@ security: sqlalchemy 2 at time of writing. See RHELDST-15252. expires: '2023-03-01' + 65213: + # CVE-2023-6129, pyopenssl>=22.0.0, + # POLY1305 MAC issue on PowerPC CPUs + reason: >- + Vulnerability is specific to PPC architecture, which is not + used or relevant for this service. + expires: '2025-04-04' continue-on-vulnerability-error: False diff --git a/docs/deployment.rst b/docs/deployment.rst index 6e67fd31..0c146756 100644 --- a/docs/deployment.rst +++ b/docs/deployment.rst @@ -112,3 +112,63 @@ users may specify a logger name and the level at which to set said logger. exodus-gw = INFO s3 = DEBUG ... + +CDN cache flush +............... + +exodus-gw supports flushing the cache of an Akamai CDN edge via +the `Fast Purge API `_. + +This feature is optional. If configuration is not provided, related APIs in +exodus-gw will continue to function but will skip cache flush operations. + +Enabling the feature requires the deployment of two sets of configuration. + +Firstly, in the ``exodus-gw.ini`` section for the relevant environment, +set ``cache_flush_urls`` to enable cache flush by URL and/or +``cache_flush_arl_templates`` to enable cache flushing by ARL. Both options +can be used together as needed. + +.. code-block:: ini + + [env.live] + # Root URL(s) of CDN properties for which to flush cache. + # Several can be provided. + cache_flush_urls = + https://cdn1.example.com + https://cdn2.example.com + + # Templates of ARL(s) for which to flush cache. + # Templates can use placeholders: + # - path: path of a file under CDN root + # - ttl (optional): a TTL value will be substituted + cache_flush_arl_templates = + S/=/123/22334455/{ttl}/cdn1.example.com/{path} + S/=/123/22334455/{ttl}/cdn2.example.com/{path} + +Secondly, use environment variables to deploy credentials for the +Fast Purge API, according to the below table. The fields here correspond +to those used by the `.edgerc file `_ +as found in Akamai's documentation. + +Note that "" should be replaced with the specific corresponding +environment name, e.g. ``EXODUS_GW_FASTPURGE_HOST_LIVE`` for a ``live`` +environment. + +.. list-table:: Fast Purge credentials + + * - Variable + - ``.edgerc`` field + - Example + * - ``EXODUS_GW_FASTPURGE_CLIENT_SECRET_`` + - ``client_secret`` + - ``abcdEcSnaAt123FNkBxy456z25qx9Yp5CPUxlEfQeTDkfh4QA=I`` + * - ``EXODUS_GW_FASTPURGE_HOST_`` + - ``host`` + - ``akab-lmn789n2k53w7qrs10cxy-nfkxaa4lfk3kd6ym.luna.akamaiapis.net`` + * - ``EXODUS_GW_FASTPURGE_ACCESS_TOKEN_`` + - ``access_token`` + - ``akab-zyx987xa6osbli4k-e7jf5ikib5jknes3`` + * - ``EXODUS_GW_FASTPURGE_CLIENT_TOKEN_`` + - ``client_token`` + - ``akab-nomoflavjuc4422-fa2xznerxrm3teg7`` diff --git a/examples/exodus-flush b/examples/exodus-flush new file mode 100755 index 00000000..e7347b93 --- /dev/null +++ b/examples/exodus-flush @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +# +# Flush CDN cache for certain path(s). +# +# Example: +# +# $ examples/exodus-flush /some/path /another/path ... +# + +import argparse +import logging +import os +import sys +import time +from urllib.parse import urljoin + +import requests + +LOG = logging.getLogger("exodus-publish") + +DEFAULT_URL = "https://localhost:8010" + + +def assert_success(response: requests.Response): + """Raise if 'response' was not successful. + + This is the same as response.raise_for_status(), merely wrapping it + to ensure the body is logged when possible.""" + + try: + response.raise_for_status() + except Exception as outer: + try: + body = response.json() + except: + raise outer + + LOG.error("Unsuccessful response from exodus-gw: %s", body) + raise + + +def flush_cache(args): + session = requests.Session() + if args.cert: + session.cert = (args.cert, args.key) + + url = os.path.join(args.url, f"{args.env}/cdn-flush") + r = session.post(url, json=[{"web_uri": path} for path in args.path]) + assert_success(r) + + # We have a task, now wait for it to complete. + task = r.json() + + task_id = task["id"] + task_url = urljoin(args.url, task["links"]["self"]) + task_state = task["state"] + + while task_state not in ["COMPLETE", "FAILED"]: + LOG.info("Task %s: %s", task_id, task_state) + time.sleep(5) + + r = session.get(task_url) + assert_success(r) + + task = r.json() + task_state = task["state"] + + LOG.info("Task %s: %s", task_id, task_state) + + if task_state == "COMPLETE": + LOG.info("Cache flush completed at %s", task["updated"]) + else: + LOG.error("Cache flush failed!") + sys.exit(41) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--debug", action="store_true", help="Enable verbose logging" + ) + parser.add_argument("path", nargs="+", help="Path(s) to flush") + + gw = parser.add_argument_group("exodus-gw settings") + + gw.add_argument( + "--cert", + default=os.path.expandvars("${HOME}/certs/${USER}.crt"), + help="Certificate for HTTPS authentication with exodus-gw (must match --key)", + ) + gw.add_argument( + "--key", + default=os.path.expandvars("${HOME}/certs/${USER}.key"), + help="Private key for HTTPS authentication with exodus-gw (must match --cert)", + ) + gw.add_argument("--url", default=DEFAULT_URL) + gw.add_argument("--env", default="test") + + args = parser.parse_args() + + if args.debug: + logging.basicConfig(level=logging.DEBUG) + else: + logging.basicConfig(level=logging.WARN, format="%(message)s") + LOG.setLevel(logging.INFO) + + return flush_cache(args) + + +if __name__ == "__main__": + main() diff --git a/exodus_gw/deps.py b/exodus_gw/deps.py index c861d5a0..d8a6c668 100644 --- a/exodus_gw/deps.py +++ b/exodus_gw/deps.py @@ -3,8 +3,9 @@ import logging import sys from asyncio import LifoQueue +from datetime import datetime, timedelta -from fastapi import Depends, Path, Request +from fastapi import Depends, HTTPException, Path, Query, Request from .auth import call_context as get_call_context from .aws.client import S3ClientWrapper @@ -87,6 +88,32 @@ async def get_s3_client( await queue.put(client) +async def get_deadline_from_query( + deadline: str | None = Query( + default=None, + examples=["2022-07-25T15:47:47Z"], + description=( + "A timestamp by which this task may be abandoned if not completed.\n\n" + "When omitted, a server default will apply." + ), + ), + settings: Settings = Depends(get_settings), +) -> datetime: + now = datetime.utcnow() + + if isinstance(deadline, str): + try: + deadline_obj = datetime.strptime(deadline, "%Y-%m-%dT%H:%M:%SZ") + except Exception as exc_info: + raise HTTPException( + status_code=400, detail=repr(exc_info) + ) from exc_info + else: + deadline_obj = now + timedelta(hours=settings.task_deadline) + + return deadline_obj + + # These are the preferred objects for use in endpoints, # e.g. # @@ -95,5 +122,6 @@ async def get_s3_client( db = Depends(get_db) call_context = Depends(get_call_context) env = Depends(get_environment_from_path) +deadline = Depends(get_deadline_from_query) settings = Depends(get_settings) s3_client = Depends(get_s3_client) diff --git a/exodus_gw/dramatiq/middleware/settings.py b/exodus_gw/dramatiq/middleware/settings.py index 652f9bf8..ea76498d 100644 --- a/exodus_gw/dramatiq/middleware/settings.py +++ b/exodus_gw/dramatiq/middleware/settings.py @@ -21,7 +21,12 @@ def before_declare_actor(self, broker, actor): @wraps(original_fn) def new_fn(*args, **kwargs): - kwargs["settings"] = self.__settings() + # Settings are automatically injected if there is no + # value present. + # If a value is present, it's not overwritten; this allows + # calling actors with specific settings during tests. + if not kwargs.get("settings"): + kwargs["settings"] = self.__settings() return original_fn(*args, **kwargs) actor.fn = new_fn diff --git a/exodus_gw/logging.py b/exodus_gw/logging.py index df684434..a220a370 100644 --- a/exodus_gw/logging.py +++ b/exodus_gw/logging.py @@ -64,6 +64,8 @@ def __init__(self, datefmt=None): "publish_id": "publish_id", "message_id": "message_id", "duration_ms": "duration_ms", + "url": "url", + "response": "response", } self.datefmt = datefmt diff --git a/exodus_gw/routers/cdn.py b/exodus_gw/routers/cdn.py index adf75fd3..8e5d4433 100644 --- a/exodus_gw/routers/cdn.py +++ b/exodus_gw/routers/cdn.py @@ -12,10 +12,11 @@ from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes, serialization from cryptography.hazmat.primitives.asymmetric import padding -from fastapi import APIRouter, HTTPException, Path, Query +from fastapi import APIRouter, Body, HTTPException, Path, Query from fastapi.responses import Response +from sqlalchemy.orm import Session -from exodus_gw import auth, schemas +from exodus_gw import auth, models, schemas, worker from .. import deps from ..settings import Environment, Settings @@ -283,3 +284,66 @@ def cdn_access( "expires": expires.isoformat(timespec="minutes") + "Z", "cookie": cookie_str, } + + +@router.post( + "/{env}/cdn-flush", + summary="Flush cache", + status_code=200, + dependencies=[auth.needs_role("cdn-flusher")], + response_model=schemas.Task, +) +def flush_cdn_cache( + items: list[schemas.FlushItem] = Body( + ..., + examples=[ + [ + { + "web_uri": "/some/path/i/want/to/flush", + }, + { + "web_uri": "/another/path/i/want/to/flush", + }, + ] + ], + ), + deadline: datetime = deps.deadline, + env: Environment = deps.env, + db: Session = deps.db, +) -> models.Task: + """Flush given paths from CDN cache(s) corresponding to this environment. + + This API may be used to request CDN edge servers downstream from exodus-gw + and exodus-cdn to discard cached versions of content, ensuring that + subsequent requests will receive up-to-date content. + + The API is provided for troubleshooting and for scenarios where it's + known that explicit cache flushes are needed. It's not necessary to use + this API during a typical upload and publish workflow. + + Returns a task. Successful completion of the task indicates that CDN + caches have been flushed. + + **Required roles**: `{env}-cdn-flusher` + """ + paths = sorted(set([item.web_uri for item in items])) + + msg = worker.flush_cdn_cache.send( + env=env.name, + paths=paths, + ) + + LOG.info( + "Enqueued cache flush for %s path(s) (%s, ...)", + len(paths), + paths[0] if paths else "", + ) + + task = models.Task( + id=msg.message_id, + state="NOT_STARTED", + deadline=deadline, + ) + db.add(task) + + return task diff --git a/exodus_gw/routers/publish.py b/exodus_gw/routers/publish.py index d99c668f..d862718d 100644 --- a/exodus_gw/routers/publish.py +++ b/exodus_gw/routers/publish.py @@ -112,7 +112,7 @@ import logging import os -from datetime import datetime, timedelta +from datetime import datetime from uuid import uuid4 from fastapi import APIRouter, Body, HTTPException, Query @@ -382,9 +382,7 @@ def commit_publish( env: Environment = deps.env, db: Session = deps.db, settings: Settings = deps.settings, - deadline: str | None = Query( - default=None, examples=["2022-07-25T15:47:47Z"] - ), + deadline: datetime = deps.deadline, commit_mode: models.CommitModes | None = Query( default=None, title="commit mode", @@ -435,16 +433,6 @@ def commit_publish( commit_mode_str = (commit_mode or models.CommitModes.phase2).value now = datetime.utcnow() - if isinstance(deadline, str): - try: - deadline_obj = datetime.strptime(deadline, "%Y-%m-%dT%H:%M:%SZ") - except Exception as exc_info: - raise HTTPException( - status_code=400, detail=repr(exc_info) - ) from exc_info - else: - deadline_obj = now + timedelta(hours=settings.task_deadline) - db_publish = ( db.query(models.Publish) # Publish should be locked, but if doing a phase1 commit we will only @@ -508,7 +496,7 @@ def commit_publish( id=msg.message_id, publish_id=msg.kwargs["publish_id"], state="NOT_STARTED", - deadline=deadline_obj, + deadline=deadline, commit_mode=commit_mode, ) db.add(task) diff --git a/exodus_gw/schemas.py b/exodus_gw/schemas.py index b65ab109..cf99b4b5 100644 --- a/exodus_gw/schemas.py +++ b/exodus_gw/schemas.py @@ -121,6 +121,13 @@ class Item(ItemBase): ) +class FlushItem(BaseModel): + web_uri: str = Field( + ..., + description="URI, relative to CDN root, of which to flush cache", + ) + + class PublishStates(str, Enum): pending = "PENDING" committing = "COMMITTING" @@ -176,7 +183,8 @@ def terminal(cls) -> list["TaskStates"]: class Task(BaseModel): id: UUID = Field(..., description="Unique ID of task object.") publish_id: UUID | None = Field( - None, description="Unique ID of publish object handled by this task." + None, + description="Unique ID of publish object related to this task, if any.", ) state: TaskStates = Field(..., description="Current state of this task.") updated: datetime | None = Field( diff --git a/exodus_gw/settings.py b/exodus_gw/settings.py index 9a28e3ca..bf0f469c 100644 --- a/exodus_gw/settings.py +++ b/exodus_gw/settings.py @@ -7,6 +7,25 @@ from pydantic_settings import BaseSettings, SettingsConfigDict +def split_ini_list(raw: str | None) -> list[str]: + # Given a string value from an .ini file, splits it into multiple + # strings over line boundaries, using the typical form supported + # in .ini files. + # e.g. + # + # [section] + # my-setting= + # foo + # bar + # + # => returns ["foo", "bar"] + + if not raw: + return [] + + return [elem.strip() for elem in raw.split("\n") if elem.strip()] + + class Environment(object): def __init__( self, @@ -17,6 +36,8 @@ def __init__( config_table, cdn_url, cdn_key_id, + cache_flush_urls=None, + cache_flush_arl_templates=None, ): self.name = name self.aws_profile = aws_profile @@ -25,11 +46,54 @@ def __init__( self.config_table = config_table self.cdn_url = cdn_url self.cdn_key_id = cdn_key_id + self.cache_flush_urls = split_ini_list(cache_flush_urls) + self.cache_flush_arl_templates = split_ini_list( + cache_flush_arl_templates + ) @property def cdn_private_key(self): return os.getenv("EXODUS_GW_CDN_PRIVATE_KEY_%s" % self.name.upper()) + @property + def fastpurge_enabled(self) -> bool: + """True if this environment has fastpurge-based cache flushing enabled. + + When True, it is guaranteed that all needed credentials for fastpurge + are available for this environment. + """ + return ( + # *at least one* URL or ARL template must be set... + (self.cache_flush_urls or self.cache_flush_arl_templates) + # ... and *all* fastpurge credentials must be set + and self.fastpurge_access_token + and self.fastpurge_client_secret + and self.fastpurge_client_token + and self.fastpurge_host + ) + + @property + def fastpurge_client_secret(self): + return os.getenv( + "EXODUS_GW_FASTPURGE_CLIENT_SECRET_%s" % self.name.upper() + ) + + @property + def fastpurge_host(self): + return os.getenv("EXODUS_GW_FASTPURGE_HOST_%s" % self.name.upper()) + + @property + def fastpurge_access_token(self): + return os.getenv( + "EXODUS_GW_FASTPURGE_ACCESS_TOKEN_%s" % self.name.upper() + ) + + @property + def fastpurge_client_token(self): + return os.getenv( + "EXODUS_GW_FASTPURGE_CLIENT_TOKEN_%s" % self.name.upper() + ) + class MigrationMode(str, Enum): upgrade = "upgrade" @@ -279,6 +343,10 @@ def load_settings() -> Settings: config_table = config.get(env, "config_table", fallback=None) cdn_url = config.get(env, "cdn_url", fallback=None) cdn_key_id = config.get(env, "cdn_key_id", fallback=None) + cache_flush_urls = config.get(env, "cache_flush_urls", fallback=None) + cache_flush_arl_templates = config.get( + env, "cache_flush_arl_templates", fallback=None + ) settings.environments.append( Environment( @@ -289,6 +357,8 @@ def load_settings() -> Settings: config_table=config_table, cdn_url=cdn_url, cdn_key_id=cdn_key_id, + cache_flush_urls=cache_flush_urls, + cache_flush_arl_templates=cache_flush_arl_templates, ) ) diff --git a/exodus_gw/worker/__init__.py b/exodus_gw/worker/__init__.py index 9ea3c6c8..2cef37bd 100644 --- a/exodus_gw/worker/__init__.py +++ b/exodus_gw/worker/__init__.py @@ -11,6 +11,7 @@ # pylint: disable=wrong-import-position from .autoindex import autoindex_partial # noqa +from .cache import flush_cdn_cache # noqa from .deploy import deploy_config # noqa from .publish import commit # noqa from .scheduled import cleanup # noqa diff --git a/exodus_gw/worker/cache.py b/exodus_gw/worker/cache.py new file mode 100644 index 00000000..08e81118 --- /dev/null +++ b/exodus_gw/worker/cache.py @@ -0,0 +1,160 @@ +import logging +import os +import re +from datetime import datetime + +import dramatiq +import fastpurge +from dramatiq.middleware import CurrentMessage +from sqlalchemy.orm import Session + +from exodus_gw import models +from exodus_gw.database import db_engine +from exodus_gw.schemas import TaskStates +from exodus_gw.settings import Settings + +LOG = logging.getLogger("exodus-gw") + + +class Flusher: + def __init__( + self, + paths: list[str], + settings: Settings, + env: str, + ): + self.paths = [p.removeprefix("/") for p in paths] + self.settings = settings + + for environment in settings.environments: + if environment.name == env: + self.env = environment + + assert self.env + + def arl_ttl(self, path: str): + # Return an appropriate TTL value for certain paths. + # + # Note that this logic has to match the behavior configured at + # the CDN edge. + # + # This logic was originally sourced from rhsm-akamai-cache-purge. + + ttl = "30d" # default ttl + ostree_re = r".*/ostree/repo/refs/heads/.*/(base|standard)$" + if path.endswith(("/repodata/repomd.xml", "/")): + ttl = "4h" + elif ( + path.endswith(("/PULP_MANIFEST", "/listing")) + or ("/repodata/" in path) + or re.match(ostree_re, path) + ): + ttl = "10m" + + return ttl + + @property + def urls_for_flush(self): + out: list[str] = [] + + for cdn_base_url in self.env.cache_flush_urls: + for path in self.paths: + out.append(os.path.join(cdn_base_url, path)) + + for arl_template in self.env.cache_flush_arl_templates: + for path in self.paths: + out.append( + arl_template.format( + path=path, + ttl=self.arl_ttl(path), + ) + ) + + return out + + def do_flush(self, urls: list[str]): + if not self.env.fastpurge_enabled or not urls: + LOG.info("fastpurge is not enabled for %s", self.env.name) + return + + for url in urls: + LOG.info("fastpurge: flushing", extra=dict(url=url)) + + fp = fastpurge.FastPurgeClient( + auth=dict( + host=self.env.fastpurge_host, + access_token=self.env.fastpurge_access_token, + client_token=self.env.fastpurge_client_token, + client_secret=self.env.fastpurge_client_secret, + ) + ) + + responses = fp.purge_by_url(urls).result() + + for r in responses: + LOG.info("fastpurge: response", extra=dict(response=r)) + + def run(self): + urls = self.urls_for_flush + self.do_flush(urls) + + LOG.info( + "%s flush of %s URL(s) (%s, ...)", + "Completed" if self.env.fastpurge_enabled else "Skipped", + len(urls), + urls[0] if urls else "", + ) + + +def load_task(db: Session, task_id: str): + return ( + db.query(models.Task) + .filter(models.Task.id == task_id) + .with_for_update() + .first() + ) + + +@dramatiq.actor( + time_limit=Settings().actor_time_limit, + max_backoff=Settings().actor_max_backoff, +) +def flush_cdn_cache( + paths: list[str], + env: str, + settings: Settings = Settings(), +) -> None: + db = Session(bind=db_engine(settings)) + task_id = CurrentMessage.get_current_message().message_id + + task = load_task(db, task_id) + + if task and task.state == TaskStates.not_started: + # Mark the task in progress so clients know we're working on it... + task.state = TaskStates.in_progress + db.commit() + + # The commit dropped our "for update" lock, so reload it. + task = load_task(db, task_id) + + if not task or task.state != TaskStates.in_progress: + LOG.error( + "Task in unexpected state %s", task.state if task else "" + ) + return + + if task.deadline and task.deadline < datetime.utcnow(): + LOG.error("Task exceeded deadline of %s", task.deadline) + task.state = TaskStates.failed + db.commit() + return + + flusher = Flusher( + paths=paths, + settings=settings, + env=env, + ) + flusher.run() + + task.state = TaskStates.complete + db.commit() diff --git a/requirements.in b/requirements.in index dd1fb64e..2dc55cd2 100644 --- a/requirements.in +++ b/requirements.in @@ -16,6 +16,7 @@ dramatiq[watch] pycron cryptography repo-autoindex>=1.2.0 +fastpurge # Needed to get compatible idna between requirements.txt and test-requirements.txt, # consider removing when 'requests' no longer requires <4 idna==3.4 diff --git a/requirements.txt b/requirements.txt index a4488972..15af2b0b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -140,6 +140,10 @@ botocore==1.34.34 \ # aiobotocore # boto3 # s3transfer +certifi==2024.2.2 \ + --hash=sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f \ + --hash=sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1 + # via requests cffi==1.16.0 \ --hash=sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc \ --hash=sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a \ @@ -194,6 +198,98 @@ cffi==1.16.0 \ --hash=sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956 \ --hash=sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357 # via cryptography +charset-normalizer==3.3.2 \ + --hash=sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027 \ + --hash=sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087 \ + --hash=sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786 \ + --hash=sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8 \ + --hash=sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09 \ + --hash=sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185 \ + --hash=sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574 \ + --hash=sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e \ + --hash=sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519 \ + --hash=sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898 \ + --hash=sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269 \ + --hash=sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3 \ + --hash=sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f \ + --hash=sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6 \ + --hash=sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8 \ + --hash=sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a \ + --hash=sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73 \ + --hash=sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc \ + --hash=sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714 \ + --hash=sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2 \ + --hash=sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc \ + --hash=sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce \ + --hash=sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d \ + --hash=sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e \ + --hash=sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6 \ + --hash=sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269 \ + --hash=sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96 \ + --hash=sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d \ + --hash=sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a \ + --hash=sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4 \ + --hash=sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77 \ + --hash=sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d \ + --hash=sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0 \ + --hash=sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed \ + --hash=sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068 \ + --hash=sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac \ + --hash=sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25 \ + --hash=sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8 \ + --hash=sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab \ + --hash=sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26 \ + --hash=sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2 \ + --hash=sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db \ + --hash=sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f \ + --hash=sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5 \ + --hash=sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99 \ + --hash=sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c \ + --hash=sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d \ + --hash=sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811 \ + --hash=sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa \ + --hash=sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a \ + --hash=sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03 \ + --hash=sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b \ + --hash=sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04 \ + --hash=sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c \ + --hash=sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001 \ + --hash=sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458 \ + --hash=sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389 \ + --hash=sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99 \ + --hash=sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985 \ + --hash=sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537 \ + --hash=sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238 \ + --hash=sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f \ + --hash=sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d \ + --hash=sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796 \ + --hash=sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a \ + --hash=sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143 \ + --hash=sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8 \ + --hash=sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c \ + --hash=sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5 \ + --hash=sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5 \ + --hash=sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711 \ + --hash=sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4 \ + --hash=sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6 \ + --hash=sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c \ + --hash=sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7 \ + --hash=sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4 \ + --hash=sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b \ + --hash=sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae \ + --hash=sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12 \ + --hash=sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c \ + --hash=sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae \ + --hash=sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8 \ + --hash=sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887 \ + --hash=sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b \ + --hash=sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4 \ + --hash=sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f \ + --hash=sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5 \ + --hash=sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33 \ + --hash=sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519 \ + --hash=sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561 + # via requests click==8.1.7 \ --hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \ --hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de @@ -231,7 +327,9 @@ cryptography==42.0.5 \ --hash=sha256:e807b3188f9eb0eaa7bbb579b462c5ace579f1cedb28107ce8b48a9f7ad3679e \ --hash=sha256:f12764b8fffc7a123f641d7d049d382b73f96a34117e0b637b80643169cec8ac \ --hash=sha256:f8837fe1d6ac4a8052a9a8ddab256bc006242696f03368a4009be7ee3075cdb7 - # via -r requirements.in + # via + # -r requirements.in + # pyopenssl defusedxml==0.7.1 \ --hash=sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69 \ --hash=sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61 @@ -242,9 +340,17 @@ dramatiq[watch]==1.16.0 \ --hash=sha256:00a676a96d0f47ea4ba59a82018dd3d8885fb8cec7765dc8209142f4b493870e \ --hash=sha256:650860af82a98905ee03f7cc94b7c356f89528e3008c213aee6a35e2faecde05 # via -r requirements.in -fastapi==0.110.0 \ - --hash=sha256:266775f0dcc95af9d3ef39bad55cff525329a931d5fd51930aadd4f428bf7ff3 \ - --hash=sha256:87a1f6fb632a218222c5984be540055346a8f5d8a68e8f6fb647b1dc9934de4b +edgegrid-python==1.3.1 \ + --hash=sha256:480ad3e8e4586ba3190f176ae2c2e5a3ab66e47834a01812826d01e1c2b6439a \ + --hash=sha256:c7dbfd66b4ea8e386fe5391005b2cb9bf8556f4f04ab2fad53fe27554b553b7d + # via fastpurge +fastapi==0.110.1 \ + --hash=sha256:5df913203c482f820d31f48e635e022f8cbfe7350e4830ef05a3163925b1addc \ + --hash=sha256:6feac43ec359dfe4f45b2c18ec8c94edb8dc2dfc461d417d9e626590c071baad + # via -r requirements.in +fastpurge==1.0.5 \ + --hash=sha256:0e8c082afca8d4fd289963a59b4ba022704263b271117bac18b7d8bd768b0b9f \ + --hash=sha256:74499c36be7372ee63bf694d61b0c85b6877d15830be4cfe2918f4baec2a0803 # via -r requirements.in frozenlist==1.4.1 \ --hash=sha256:04ced3e6a46b4cfffe20f9ae482818e34eba9b5fb0ce4056e4cc9b6e212d09b7 \ @@ -484,6 +590,7 @@ idna==3.4 \ # via # -r requirements.in # anyio + # requests # yarl importlib-metadata==7.1.0 \ --hash=sha256:30962b96c0c223483ed6cc7280e7f0199feb01a0e40cfae4d4450fc6fab1f570 \ @@ -579,6 +686,10 @@ markupsafe==2.1.5 \ # via # jinja2 # mako +more-executors==2.11.4 \ + --hash=sha256:a304139c6bece5be18aed7dcff4c48440412cb7cbe90f64ba4572772fcb0407f \ + --hash=sha256:f1b21d72c4c15069e891d9b96bca05f9abde149e3c11ca54630c5a1a5ee8f4b5 + # via fastpurge multidict==6.0.5 \ --hash=sha256:01265f5e40f5a17f8241d52656ed27192be03bfa8764d88e8220141d1e4b3556 \ --hash=sha256:0275e35209c27a3f7951e1ce7aaf93ce0d163b28948444bec61dd7badc6d3f8c \ @@ -673,6 +784,11 @@ multidict==6.0.5 \ # via # aiohttp # yarl +ndg-httpsclient==0.5.1 \ + --hash=sha256:d2c7225f6a1c6cf698af4ebc962da70178a99bcde24ee6d1961c4f3338130d57 \ + --hash=sha256:d72faed0376ab039736c2ba12e30695e2788c4aa569c9c3e3d72131de2592210 \ + --hash=sha256:dd174c11d971b6244a891f7be2b32ca9853d3797a72edb34fa5d7b07d8fff7d4 + # via edgegrid-python packaging==21.3 \ --hash=sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb \ --hash=sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522 @@ -698,6 +814,12 @@ psycopg2==2.9.9 \ --hash=sha256:de80739447af31525feddeb8effd640782cf5998e1a4e9192ebdf829717e3913 \ --hash=sha256:ff432630e510709564c01dafdbe996cb552e0b9f3f065eb89bdce5bd31fabf4c # via -r requirements.in +pyasn1==0.6.0 \ + --hash=sha256:3a35ab2c4b5ef98e17dfdec8ab074046fbda76e281c5a706ccd82328cfc8f64c \ + --hash=sha256:cca4bb0f2df5504f02f6f8a775b6e416ff9b0b3b16f7ee80b5a3153d9b804473 + # via + # edgegrid-python + # ndg-httpsclient pycparser==2.22 \ --hash=sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6 \ --hash=sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc @@ -797,6 +919,12 @@ pydantic-settings==2.2.1 \ --hash=sha256:00b9f6a5e95553590434c0fa01ead0b216c3e10bc54ae02e37f359948643c5ed \ --hash=sha256:0235391d26db4d2190cb9b31051c4b46882d28a51533f97440867f012d4da091 # via -r requirements.in +pyopenssl==24.1.0 \ + --hash=sha256:17ed5be5936449c5418d1cd269a1a9e9081bc54c17aed272b45856a3d3dc86ad \ + --hash=sha256:cabed4bfaa5df9f1a16c0ef64a0cb65318b5cd077a7eda7d6970131ca2f41a6f + # via + # edgegrid-python + # ndg-httpsclient pyparsing==3.1.2 \ --hash=sha256:a1bac0ce561155ecc3ed78ca94d3c9378656ad4c94c1270de543f621420f94ad \ --hash=sha256:f9db75911801ed778fe61bb643079ff86601aca99fcae6345aa67292038fb742 @@ -874,6 +1002,17 @@ repo-autoindex==1.2.1 \ --hash=sha256:4f65dc75afd3687247584719434227f25fd5e3116c845523f64955fa22fe2d4d \ --hash=sha256:6941669aa4382372296077fde0f9238232bfd72458e186e92f788e4535561a45 # via -r requirements.in +requests==2.31.0 \ + --hash=sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f \ + --hash=sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1 + # via + # edgegrid-python + # fastpurge + # requests-toolbelt +requests-toolbelt==1.0.0 \ + --hash=sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6 \ + --hash=sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06 + # via edgegrid-python rpds-py==0.18.0 \ --hash=sha256:01e36a39af54a30f28b73096dd39b6802eddd04c90dbe161c1b8dbe22353189f \ --hash=sha256:044a3e61a7c2dafacae99d1e722cc2d4c05280790ec5a05031b3876809d89a5c \ @@ -984,7 +1123,10 @@ s3transfer==0.10.1 \ six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 - # via python-dateutil + # via + # fastpurge + # more-executors + # python-dateutil sniffio==1.3.1 \ --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \ --hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc @@ -1042,9 +1184,9 @@ sqlalchemy==2.0.29 \ # via # -r requirements.in # alembic -starlette==0.36.3 \ - --hash=sha256:13d429aa93a61dc40bf503e8c801db1f1bca3dc706b10ef2434a36123568f044 \ - --hash=sha256:90a671733cfb35771d8cc605e0b679d23b992f8dcfad48cc60b38cb29aeb7080 +starlette==0.37.2 \ + --hash=sha256:6fe59f29268538e5d0d182f2791a479a0c64638e6935d1c6989e63fb2699c6ee \ + --hash=sha256:9af890290133b79fc3db55474ade20f6220a364a0402e0b556e7cd5e1e093823 # via # -r requirements.in # asgi-correlation-id @@ -1061,7 +1203,10 @@ typing-extensions==4.10.0 \ urllib3==2.0.7 \ --hash=sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84 \ --hash=sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e - # via botocore + # via + # botocore + # edgegrid-python + # requests uvicorn[standard]==0.29.0 \ --hash=sha256:2c2aac7ff4f4365c206fd773a39bf4ebd1047c238f8b8268ad996829323473de \ --hash=sha256:6a69214c0b6a087462412670b3ef21224fa48cae0e452b5883e8e8bdfdd11dd0 diff --git a/scripts/systemd/exodus-gw-sidecar.service b/scripts/systemd/exodus-gw-sidecar.service index aa53752f..61d61e19 100644 --- a/scripts/systemd/exodus-gw-sidecar.service +++ b/scripts/systemd/exodus-gw-sidecar.service @@ -43,6 +43,9 @@ com.redhat.api.platform:\n\ test-cdn-consumer:\n\ users:\n\ byInternalUsername: [${USER}]\n\ + test-cdn-flusher:\n\ + users:\n\ + byInternalUsername: [${USER}]\n\ \nEND\n\ " diff --git a/test-requirements.txt b/test-requirements.txt index 223ce0b8..1354e4f4 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -417,6 +417,7 @@ cryptography==42.0.5 \ # via # -r requirements.in # authlib + # pyopenssl defusedxml==0.7.1 \ --hash=sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69 \ --hash=sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61 @@ -444,9 +445,17 @@ dramatiq[watch]==1.16.0 \ --hash=sha256:00a676a96d0f47ea4ba59a82018dd3d8885fb8cec7765dc8209142f4b493870e \ --hash=sha256:650860af82a98905ee03f7cc94b7c356f89528e3008c213aee6a35e2faecde05 # via -r requirements.in -fastapi==0.110.0 \ - --hash=sha256:266775f0dcc95af9d3ef39bad55cff525329a931d5fd51930aadd4f428bf7ff3 \ - --hash=sha256:87a1f6fb632a218222c5984be540055346a8f5d8a68e8f6fb647b1dc9934de4b +edgegrid-python==1.3.1 \ + --hash=sha256:480ad3e8e4586ba3190f176ae2c2e5a3ab66e47834a01812826d01e1c2b6439a \ + --hash=sha256:c7dbfd66b4ea8e386fe5391005b2cb9bf8556f4f04ab2fad53fe27554b553b7d + # via fastpurge +fastapi==0.110.1 \ + --hash=sha256:5df913203c482f820d31f48e635e022f8cbfe7350e4830ef05a3163925b1addc \ + --hash=sha256:6feac43ec359dfe4f45b2c18ec8c94edb8dc2dfc461d417d9e626590c071baad + # via -r requirements.in +fastpurge==1.0.5 \ + --hash=sha256:0e8c082afca8d4fd289963a59b4ba022704263b271117bac18b7d8bd768b0b9f \ + --hash=sha256:74499c36be7372ee63bf694d61b0c85b6877d15830be4cfe2918f4baec2a0803 # via -r requirements.in freezegun==1.4.0 \ --hash=sha256:10939b0ba0ff5adaecf3b06a5c2f73071d9678e507c5eaedb23c761d56ac774b \ @@ -832,6 +841,10 @@ mock==5.1.0 \ --hash=sha256:18c694e5ae8a208cdb3d2c20a993ca1a7b0efa258c247a1e565150f477f83744 \ --hash=sha256:5e96aad5ccda4718e0a229ed94b2024df75cc2d55575ba5762d31f5767b8767d # via -r test-requirements.in +more-executors==2.11.4 \ + --hash=sha256:a304139c6bece5be18aed7dcff4c48440412cb7cbe90f64ba4572772fcb0407f \ + --hash=sha256:f1b21d72c4c15069e891d9b96bca05f9abde149e3c11ca54630c5a1a5ee8f4b5 + # via fastpurge multidict==6.0.5 \ --hash=sha256:01265f5e40f5a17f8241d52656ed27192be03bfa8764d88e8220141d1e4b3556 \ --hash=sha256:0275e35209c27a3f7951e1ce7aaf93ce0d163b28948444bec61dd7badc6d3f8c \ @@ -959,6 +972,11 @@ mypy-extensions==1.0.0 \ --hash=sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d \ --hash=sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782 # via mypy +ndg-httpsclient==0.5.1 \ + --hash=sha256:d2c7225f6a1c6cf698af4ebc962da70178a99bcde24ee6d1961c4f3338130d57 \ + --hash=sha256:d72faed0376ab039736c2ba12e30695e2788c4aa569c9c3e3d72131de2592210 \ + --hash=sha256:dd174c11d971b6244a891f7be2b32ca9853d3797a72edb34fa5d7b07d8fff7d4 + # via edgegrid-python packaging==21.3 \ --hash=sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb \ --hash=sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522 @@ -1002,6 +1020,12 @@ psycopg2==2.9.9 \ --hash=sha256:de80739447af31525feddeb8effd640782cf5998e1a4e9192ebdf829717e3913 \ --hash=sha256:ff432630e510709564c01dafdbe996cb552e0b9f3f065eb89bdce5bd31fabf4c # via -r requirements.in +pyasn1==0.6.0 \ + --hash=sha256:3a35ab2c4b5ef98e17dfdec8ab074046fbda76e281c5a706ccd82328cfc8f64c \ + --hash=sha256:cca4bb0f2df5504f02f6f8a775b6e416ff9b0b3b16f7ee80b5a3153d9b804473 + # via + # edgegrid-python + # ndg-httpsclient pycparser==2.22 \ --hash=sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6 \ --hash=sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc @@ -1113,6 +1137,12 @@ pylint==3.1.0 \ --hash=sha256:507a5b60953874766d8a366e8e8c7af63e058b26345cfcb5f91f89d987fd6b74 \ --hash=sha256:6a69beb4a6f63debebaab0a3477ecd0f559aa726af4954fc948c51f7a2549e23 # via -r test-requirements.in +pyopenssl==24.1.0 \ + --hash=sha256:17ed5be5936449c5418d1cd269a1a9e9081bc54c17aed272b45856a3d3dc86ad \ + --hash=sha256:cabed4bfaa5df9f1a16c0ef64a0cb65318b5cd077a7eda7d6970131ca2f41a6f + # via + # edgegrid-python + # ndg-httpsclient pyparsing==3.1.2 \ --hash=sha256:a1bac0ce561155ecc3ed78ca94d3c9378656ad4c94c1270de543f621420f94ad \ --hash=sha256:f9db75911801ed778fe61bb643079ff86601aca99fcae6345aa67292038fb742 @@ -1214,8 +1244,15 @@ requests==2.31.0 \ --hash=sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1 # via # coveralls + # edgegrid-python + # fastpurge + # requests-toolbelt # safety # sphinx +requests-toolbelt==1.0.0 \ + --hash=sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6 \ + --hash=sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06 + # via edgegrid-python rich==13.7.1 \ --hash=sha256:4edbae314f59eb482f54e9e30bf00d33350aaa94f4bfcd4e9e3110e64d0d7222 \ --hash=sha256:9be308cb1fe2f1f57d67ce99e95af38a1e2bc71ad9813b0e247cf7ffbcc3a432 @@ -1403,7 +1440,10 @@ shellingham==1.5.4 \ six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 - # via python-dateutil + # via + # fastpurge + # more-executors + # python-dateutil sniffio==1.3.1 \ --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \ --hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc @@ -1495,9 +1535,9 @@ sqlalchemy==2.0.29 \ # via # -r requirements.in # alembic -starlette==0.36.3 \ - --hash=sha256:13d429aa93a61dc40bf503e8c801db1f1bca3dc706b10ef2434a36123568f044 \ - --hash=sha256:90a671733cfb35771d8cc605e0b679d23b992f8dcfad48cc60b38cb29aeb7080 +starlette==0.37.2 \ + --hash=sha256:6fe59f29268538e5d0d182f2791a479a0c64638e6935d1c6989e63fb2699c6ee \ + --hash=sha256:9af890290133b79fc3db55474ade20f6220a364a0402e0b556e7cd5e1e093823 # via # -r requirements.in # asgi-correlation-id @@ -1542,6 +1582,7 @@ urllib3==2.0.7 \ --hash=sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e # via # botocore + # edgegrid-python # requests # safety uvicorn[standard]==0.29.0 \ diff --git a/tests/routers/test_cdn_cache.py b/tests/routers/test_cdn_cache.py new file mode 100644 index 00000000..3d8cdc4b --- /dev/null +++ b/tests/routers/test_cdn_cache.py @@ -0,0 +1,63 @@ +from fastapi.testclient import TestClient + +from exodus_gw.main import app +from exodus_gw.models.dramatiq import DramatiqMessage + + +def test_flush_cache_denied(auth_header, caplog): + """flush-cache denies request if user is missing role""" + with TestClient(app) as client: + response = client.post( + "/test/cdn-flush", + json=[ + {"web_uri": "/path1"}, + {"web_uri": "/path2"}, + ], + headers=auth_header(roles=["irrelevant-role"]), + ) + + # It should be forbidden + assert response.status_code == 403 + + # Should have been an "Access denied" event + assert "Access denied; path=/test/cdn-flush" in caplog.text + + +def test_flush_cache_typical(auth_header, db): + """flush-cache enqueues actor as expected in typical case""" + + with TestClient(app) as client: + response = client.post( + "/test/cdn-flush", + json=[ + {"web_uri": "/path1"}, + {"web_uri": "/path2"}, + ], + headers=auth_header(roles=["test-cdn-flusher"]), + ) + + # It should have succeeded + assert response.status_code == 200 + + # Should have given us some NOT_STARTED task + task = response.json() + assert task["state"] == "NOT_STARTED" + + # Check the enqueued messages... + messages: list[DramatiqMessage] = db.query(DramatiqMessage).all() + + # It should have enqueued one message + assert len(messages) == 1 + + message = messages[0] + + # Should be the message corresponding to the returned task + assert task["id"] == message.id + + # Should be a message for the expected actor with + # expected args + assert message.actor == "flush_cdn_cache" + + kwargs = message.body["kwargs"] + assert kwargs["env"] == "test" + assert kwargs["paths"] == ["/path1", "/path2"] diff --git a/tests/worker/test_cdn_cache.py b/tests/worker/test_cdn_cache.py new file mode 100644 index 00000000..dce024f6 --- /dev/null +++ b/tests/worker/test_cdn_cache.py @@ -0,0 +1,232 @@ +import pathlib +from datetime import datetime, timedelta + +import fastpurge +import pytest +from dramatiq.middleware import CurrentMessage +from more_executors import f_return +from sqlalchemy.orm import Session + +from exodus_gw.models.service import Task +from exodus_gw.settings import load_settings +from exodus_gw.worker import flush_cdn_cache + + +class FakeFastPurgeClient: + # Minimal fake for a fastpurge.FastPurgeClient which records + # purged URLs and always succeeds. + INSTANCE = None + + def __init__(self, **kwargs): + self._kwargs = kwargs + self._purged_urls = [] + FakeFastPurgeClient.INSTANCE = self + + def purge_by_url(self, urls): + self._purged_urls.extend(urls) + return f_return({"fake": "response"}) + + +@pytest.fixture(autouse=True) +def fake_fastpurge_client(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(fastpurge, "FastPurgeClient", FakeFastPurgeClient) + yield + FakeFastPurgeClient.INSTANCE = None + + +@pytest.fixture +def fake_message_id(monkeypatch: pytest.MonkeyPatch) -> str: + class FakeMessage: + @property + def message_id(self): + return "3ce55238-f7d7-46d1-a302-c79674108dc9" + + monkeypatch.setattr( + CurrentMessage, "get_current_message", lambda: FakeMessage() + ) + + return FakeMessage().message_id + + +def test_flush_cdn_cache_bad_task( + db: Session, + caplog: pytest.LogCaptureFixture, + fake_message_id: str, +): + """flush_cdn_cache bails out if no appropriate task exists.""" + settings = load_settings() + + # It should run to completion... + flush_cdn_cache( + paths=["/foo", "/bar"], + env="test", + settings=settings, + ) + + # ...but it should complain + assert "Task in unexpected state" in caplog.text + + +def test_flush_cdn_cache_expired_task( + db: Session, + caplog: pytest.LogCaptureFixture, + fake_message_id: str, +): + """flush_cdn_cache bails out if task has passed the deadline.""" + settings = load_settings() + + task = Task(id=fake_message_id) + task.deadline = datetime.utcnow() - timedelta(hours=3) + task.state = "NOT_STARTED" + db.add(task) + db.commit() + + # It should run to completion... + flush_cdn_cache( + paths=["/foo", "/bar"], + env="test", + settings=settings, + ) + + # ...but it should complain + assert "Task exceeded deadline" in caplog.text + + # And the task should be marked as failed + db.refresh(task) + assert task.state == "FAILED" + + +def test_flush_cdn_cache_fastpurge_disabled( + db: Session, + caplog: pytest.LogCaptureFixture, + fake_message_id: str, +): + """flush_cdn_cache succeeds but does nothing if fastpurge is not configured.""" + settings = load_settings() + + task = Task(id=fake_message_id) + task.state = "NOT_STARTED" + db.add(task) + db.commit() + + # It should run to completion... + flush_cdn_cache( + paths=["/foo", "/bar"], + env="test", + settings=settings, + ) + + # And the task should have succeeded + db.refresh(task) + assert task.state == "COMPLETE" + + # But it didn't actually touch the fastpurge API and the logs + # should tell us about this + assert "fastpurge is not enabled" in caplog.text + assert "Skipped flush" in caplog.text + + +def test_flush_cdn_cache_typical( + db: Session, + caplog: pytest.LogCaptureFixture, + fake_message_id: str, + tmp_path: pathlib.Path, + monkeypatch: pytest.MonkeyPatch, +): + """flush_cdn_cache performs expected cache flushes in + a typical usage scenario. + """ + + # Write an ini file with some fastpurge stuff under our control. + conf_path = tmp_path / "exodus-gw.ini" + conf_path.write_text( + """ + +[env.cachetest] +aws_profile = cachetest +bucket = my-bucket +table = my-table +config_table = my-config + +cdn_url = http://localhost:8049/_/cookie +cdn_key_id = XXXXXXXXXXXXXX + +cache_flush_urls = + https://cdn1.example.com + https://cdn2.example.com/root + +cache_flush_arl_templates = + S/=/123/4567/{ttl}/cdn1.example.com/{path} cid=/// + S/=/234/6677/{ttl}/cdn2.example.com/other/{path} x/y/z + +""" + ) + + # Make load_settings use our config file above. + monkeypatch.setenv("EXODUS_GW_INI_PATH", str(conf_path)) + + # Provide some fastpurge credentials + monkeypatch.setenv("EXODUS_GW_FASTPURGE_HOST_CACHETEST", "fphost") + monkeypatch.setenv("EXODUS_GW_FASTPURGE_CLIENT_TOKEN_CACHETEST", "ctok") + monkeypatch.setenv("EXODUS_GW_FASTPURGE_CLIENT_SECRET_CACHETEST", "csec") + monkeypatch.setenv("EXODUS_GW_FASTPURGE_ACCESS_TOKEN_CACHETEST", "atok") + + settings = load_settings() + + task = Task(id=fake_message_id) + task.state = "NOT_STARTED" + db.add(task) + db.commit() + + # It should run to completion... + flush_cdn_cache( + paths=[ + # Paths here are chosen to exercise: + # - different TTL values for different types of file + # - leading "/" vs no leading "/" - both should be tolerated + "/path/one/repodata/repomd.xml", + "path/two/listing", + "third/path", + ], + env="cachetest", + settings=settings, + ) + + # The task should have succeeded + db.refresh(task) + assert task.state == "COMPLETE" + + # Check how it used the fastpurge client + fp_client = FakeFastPurgeClient.INSTANCE + + # It should have created a client + assert fp_client + + # It should have provided the credentials from env vars + assert fp_client._kwargs["auth"] == { + "access_token": "atok", + "client_secret": "csec", + "client_token": "ctok", + "host": "fphost", + } + + # It should have flushed cache for all the expected URLs, + # using both the CDN root URLs and the ARL templates + assert sorted(fp_client._purged_urls) == [ + # Used the ARL templates. Note the different TTL values + # for different paths. + "S/=/123/4567/10m/cdn1.example.com/path/two/listing cid=///", + "S/=/123/4567/30d/cdn1.example.com/third/path cid=///", + "S/=/123/4567/4h/cdn1.example.com/path/one/repodata/repomd.xml cid=///", + "S/=/234/6677/10m/cdn2.example.com/other/path/two/listing x/y/z", + "S/=/234/6677/30d/cdn2.example.com/other/third/path x/y/z", + "S/=/234/6677/4h/cdn2.example.com/other/path/one/repodata/repomd.xml x/y/z", + # Used the CDN URL which didn't have a leading path. + "https://cdn1.example.com/path/one/repodata/repomd.xml", + "https://cdn1.example.com/path/two/listing", + "https://cdn1.example.com/third/path", + # Used the CDN URL which had a leading path. + "https://cdn2.example.com/root/path/one/repodata/repomd.xml", + "https://cdn2.example.com/root/path/two/listing", + "https://cdn2.example.com/root/third/path", + ]