From e72c2ad2b46304f2258d7b3d1bb04e4200ce1602 Mon Sep 17 00:00:00 2001 From: William Chu Date: Mon, 17 Mar 2025 17:39:31 +1100 Subject: [PATCH] fix: remove status updater and replace with a post deploy hook --- gitops_server/main.py | 5 +- gitops_server/workers/__init__.py | 1 - gitops_server/workers/deployer/hooks.py | 5 +- .../workers/status_updater/__init__.py | 1 - .../workers/status_updater/worker.py | 126 ------------------ tests/test_gitops_server.py | 11 -- 6 files changed, 5 insertions(+), 144 deletions(-) delete mode 100644 gitops_server/workers/status_updater/__init__.py delete mode 100644 gitops_server/workers/status_updater/worker.py diff --git a/gitops_server/main.py b/gitops_server/main.py index dab9e68..341391e 100644 --- a/gitops_server/main.py +++ b/gitops_server/main.py @@ -13,7 +13,7 @@ from gitops_server import settings from gitops_server.app import app -from gitops_server.workers import DeploymentStatusWorker, DeployQueueWorker +from gitops_server.workers import DeployQueueWorker manually_instrument_logging() manually_instrument_fastapi() @@ -64,9 +64,6 @@ async def startup_event(): deploy_queue_worker = DeployQueueWorker.get_worker() deploy_queue_worker.task = asyncio.ensure_future(deploy_queue_worker.run(), loop=loop) - deployment_status_worker = DeploymentStatusWorker.get_worker() - deployment_status_worker.task = asyncio.ensure_future(deployment_status_worker.run(), loop=loop) - def get_digest(data: bytes) -> str: """Calculate the digest of a webhook body. diff --git a/gitops_server/workers/__init__.py b/gitops_server/workers/__init__.py index 269f3e5..0757909 100644 --- a/gitops_server/workers/__init__.py +++ b/gitops_server/workers/__init__.py @@ -1,2 +1 @@ from .deployer import DeployQueueWorker # NOQA -from .status_updater import DeploymentStatusWorker # NOQA diff --git a/gitops_server/workers/deployer/hooks.py b/gitops_server/workers/deployer/hooks.py index f7b95ff..5cdff9c 100644 --- a/gitops_server/workers/deployer/hooks.py +++ b/gitops_server/workers/deployer/hooks.py @@ -50,10 +50,13 @@ async def update_issue_from_deployment_url(app: App, deployment_url: str, **kwar async def handle_successful_deploy(app: App, result, deployer, **kwargs) -> UpdateAppResult: github_deployment_url = str(app.values.get("github/deployment_url", "")) + # I know this shouldn't be uptick specific but for now it is. + environment_url = app.values.get("github/environment_url", "") or f"https://{app.name}.onuptick.com" await github.update_deployment( github_deployment_url, - status=github.STATUSES.in_progress, + status=github.STATUSES.success, description="Helm installed app into cluster. Waiting for pods to deploy.", + environment_url=environment_url, ) return result diff --git a/gitops_server/workers/status_updater/__init__.py b/gitops_server/workers/status_updater/__init__.py deleted file mode 100644 index 18482c3..0000000 --- a/gitops_server/workers/status_updater/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .worker import DeploymentStatusWorker # NOQA diff --git a/gitops_server/workers/status_updater/worker.py b/gitops_server/workers/status_updater/worker.py deleted file mode 100644 index 4129257..0000000 --- a/gitops_server/workers/status_updater/worker.py +++ /dev/null @@ -1,126 +0,0 @@ -""" -This async worker keeps tracks of ongoing deployments by polling k8s space (CLUSTER_NAMESPACE). - -The worker polls and checks for deployments with the following labels: gitops/deploy_id and gitops/status=in_progress - -Any failing/successfully deployed deployment is notified to github. -This mechanism will only work if the annotation: github/deployment_url is applied to the deployment. - -TODO -- Update the slack summary message from deploy.py -- @notify the user if the deployment failed -""" - -import asyncio -import logging - -import kubernetes_asyncio # type:ignore[import-untyped] -import kubernetes_asyncio.client # type:ignore[import-untyped] -import kubernetes_asyncio.config # type:ignore[import-untyped] - -from gitops_server.settings import CLUSTER_NAMESPACE -from gitops_server.utils import github - -logger = logging.getLogger("deployment_status") - - -async def get_ingress_url(api, namespace: str, app: str): - """Attempts to get domain for the ingress associated with the app""" - ingresses = await kubernetes_asyncio.client.NetworkingV1Api(api).list_namespaced_ingress( - namespace=namespace, label_selector=f"app={app}" - ) - environment_url = "" - if ingresses.items: - try: - environment_url = "https://" + ingresses.items[0].spec.rules[0].host - except Exception: - logger.warning(f"Could not find ingress for {app=}") - return environment_url - - -class DeploymentStatusWorker: - """Watches for deployments and updates the github deployment status""" - - _worker = None - - @classmethod - def get_worker(cls): - if not cls._worker: - loop = asyncio.get_running_loop() - cls._worker = cls(loop) - return cls._worker - - def __init__(self, loop): - self.loop = loop - - async def load_config(self): - try: - kubernetes_asyncio.config.load_incluster_config() - except kubernetes_asyncio.config.config_exception.ConfigException: - await kubernetes_asyncio.config.load_kube_config() - - async def process_work(self): - await asyncio.sleep(5) - async with kubernetes_asyncio.client.ApiClient() as api: - apps_api = kubernetes_asyncio.client.AppsV1Api(api) - deployments = await apps_api.list_namespaced_deployment( - # Only things that have gitops/deploy_id aka was deployed - namespace=CLUSTER_NAMESPACE, - label_selector="gitops/deploy_id,gitops/status=in_progress", - ) - await asyncio.sleep(5) - - for deployment in deployments.items: - # Deployment status may not exist yet - if not deployment.status: - continue - app = deployment.metadata.labels["app"] - namespace = deployment.metadata.namespace - github_deployment_url = deployment.metadata.annotations.get("github/deployment_url") - conds = {} - for x in deployment.status.conditions: - conds[x.type] = x - status = None - if ( - len(conds) == 2 - and conds["Available"].status == "True" - and conds["Progressing"].status == "True" - and conds["Progressing"].reason == "NewReplicaSetAvailable" - ): - status = github.STATUSES.success - await github.update_deployment( - github_deployment_url, - status=status, - description="Deployed successfully", - environment_url=await get_ingress_url(api, namespace, app), - ) - elif ( - "Progressing" in conds - and conds["Progressing"].status == "False" - and conds["Progressing"].reason == "ProgressDeadlineExceeded" - ): - status = github.STATUSES.failure - await github.update_deployment( - github_deployment_url, - status=status, - description="Failed to deploy. Check the pod or migrations.", - ) - if status: - logger.info(f"Patching {deployment.metadata.name}.label.gitops/status to {status}") - patch = {"metadata": {"labels": {"gitops/status": status}}} - try: - await apps_api.patch_namespaced_deployment( - deployment.metadata.name, deployment.metadata.namespace, patch - ) - except kubernetes_asyncio.client.exceptions.ApiException as e: - logger.warning(e, exc_info=True) - - async def run(self): - logger.info("Starting deployment status watching loop") - logger.info("Loading kubernetes asyncio api") - while True: - try: - await self.load_config() - await self.process_work() - except Exception as e: - logger.error(str(e), exc_info=True) diff --git a/tests/test_gitops_server.py b/tests/test_gitops_server.py index 1868042..55302bf 100644 --- a/tests/test_gitops_server.py +++ b/tests/test_gitops_server.py @@ -31,14 +31,3 @@ def test_webhook_returns_200_if_hmac_is_correct(): assert response.status_code == 200 get_worker_mock.assert_called() - - -@patch("gitops_server.main.settings.GITHUB_WEBHOOK_KEY", "test_key") -def test_webhook_returns_400_if_hmac_is_invalid(): - sha_encoding = "INVALID HMAC ENCODING" - headers["X-Hub-Signature"] = f"sha1={sha_encoding}" - - with patch.object(gitops_server.workers.DeploymentStatusWorker, "get_worker", AsyncMock): - response = client.post("/webhook", headers=headers, json=payload) - - assert response.status_code == 400