Release/2.7.3

agent_api/app.py

@@ -6,6 +6,7 @@
 from agent_api.routes import build_v1_routes
 from common.api.flask_ext.authentication import ServiceAccountAuth
 from common.api.flask_ext.config import Config
+from common.api.flask_ext.cors import CORS
 from common.api.flask_ext.database_connection import DatabaseConnection
 from common.api.flask_ext.exception_handling import ExceptionHandling
 from common.api.flask_ext.health import Health
@@ -15,6 +16,7 @@
 # Create and configure the app
 app = Flask(__name__, instance_relative_config=True)
 Config(app, config_module="agent_api.config")
+CORS(app, allowed_methods=["POST"])
 Logging(app)
 ExceptionHandling(app)
 Timing(app)

common/api/flask_ext/cors.py

@@ -1,23 +1,21 @@
 __all__ = ["CORS"]
+from conf import settings
 from http import HTTPStatus
+from fnmatch import fnmatch
 from typing import Optional
+from urllib.parse import urlparse
 
 from flask import Flask, Response, make_response, request
 from werkzeug.exceptions import NotFound
 
 from common.api.flask_ext.base_extension import BaseExtension
 
-# TODO: See PD-286 -- at some point we should not just use wildcards and actually settle on a strategy of indicating
-# which domains to accept cross-origin-resource-sharing from.
 CORS_HEADERS: dict[str, str] = {
-    # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Origin
-    "Access-Control-Allow-Origin": "*",
     # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Headers
-    "Access-Control-Allow-Headers": "*",
-    # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Methods
-    "Access-Control-Allow-Methods": "*",
+    "Access-Control-Allow-Headers": "Accept, Content-Type, Origin, Host, Authorization, X-Forwarded-For",
     # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Expose-Headers
-    "Access-Control-Expose-Headers": "*",
+    # Add this only if we want to expose custom response headers
+    # "Access-Control-Expose-Headers": "*",
     # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Credentials
     "Access-Control-Allow-Credentials": "true",
     # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Max-Age
@@ -43,12 +41,21 @@ def make_preflight_response() -> Optional[Response]:
         else:
             return None
 
-    @staticmethod
-    def set_cors_headers(response: Response) -> Response:
-        if response.headers is not None:
-            response.headers.update(CORS_HEADERS)
-            return response
+    def set_cors_headers(self, response: Response) -> Response:
+        if response.headers is not None and (origin := request.headers.get("Origin")) is not None:
+            try:
+                netloc = urlparse(origin).netloc
+                if any([fnmatch(netloc, pattern) for pattern in settings.CORS_DOMAINS]):
+                    response.headers.update(CORS_HEADERS)
+                    # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Origin
+                    response.headers["Access-Control-Allow-Origin"] = origin
+                    response.headers["Vary"] = "Origin"
+                    # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Methods
+                    response.headers["Access-Control-Allow-Methods"] = self.allowed_methods
+            except Exception:
+                pass
+        return response
 
     def init_app(self) -> None:
         self.add_before_request_func(CORS.make_preflight_response)
-        self.add_after_request_func(CORS.set_cors_headers)
+        self.add_after_request_func(self.set_cors_headers)

common/tests/integration/flask_ext/test_cors.py

@@ -23,7 +23,7 @@ def client(flask_app):
 @pytest.mark.integration
 def test_options_request(client):
     """OPTIONS request intercepted and given NO_CONTENT repsonse + CORS headers."""
-    response = client.options("/test-endpoint")
+    response = client.options("/test-endpoint", headers={"Origin": "https://xyz.com"})
     assert response.status_code == HTTPStatus.NO_CONTENT
     for header_name in CORS_HEADER_TUPLE:
         assert header_name in response.headers
@@ -32,7 +32,7 @@ def test_options_request(client):
 @pytest.mark.integration
 def test_options_request_404(client):
     """OPTIONS request returns a 404 for invalid url endpoints."""
-    response = client.options("/bad-endpoint")
+    response = client.options("/bad-endpoint", headers={"Origin": "https://xyz.com"})
     assert response.status_code == HTTPStatus.NOT_FOUND
 
 
@@ -42,7 +42,7 @@ def test_options_request_404(client):
 @pytest.mark.integration
 def test_requests_get(client):
     """Served GET requests have proper headers."""
-    response = client.get("/test-endpoint")
+    response = client.get("/test-endpoint", headers={"Origin": "https://xyz.com"})
     assert response.status_code == HTTPStatus.OK
     for header_name in CORS_HEADER_TUPLE:
         assert header_name in response.headers
@@ -51,7 +51,7 @@ def test_requests_get(client):
 @pytest.mark.integration
 def test_requests_put(client):
     """Served PUT requests have proper headers."""
-    response = client.put("/test-endpoint")
+    response = client.put("/test-endpoint", headers={"Origin": "https://xyz.com"})
     assert response.status_code == HTTPStatus.OK
     for header_name in CORS_HEADER_TUPLE:
         assert header_name in response.headers
@@ -60,7 +60,7 @@ def test_requests_put(client):
 @pytest.mark.integration
 def test_requests_post(client):
     """Served POST requests have proper headers."""
-    response = client.post("/test-endpoint")
+    response = client.post("/test-endpoint", headers={"Origin": "https://xyz.com"})
     assert response.status_code == HTTPStatus.OK
     for header_name in CORS_HEADER_TUPLE:
         assert header_name in response.headers
@@ -69,7 +69,7 @@ def test_requests_post(client):
 @pytest.mark.integration
 def test_requests_patch(client):
     """Served PATCH requests have proper headers."""
-    response = client.patch("/test-endpoint")
+    response = client.patch("/test-endpoint", headers={"Origin": "https://xyz.com"})
     assert response.status_code == HTTPStatus.OK
     for header_name in CORS_HEADER_TUPLE:
         assert header_name in response.headers
@@ -78,7 +78,7 @@ def test_requests_patch(client):
 @pytest.mark.integration
 def test_requests_delete(client):
     """Served DELETE requests have proper headers."""
-    response = client.delete("/test-endpoint")
+    response = client.delete("/test-endpoint", headers={"Origin": "https://xyz.com"})
     assert response.status_code == HTTPStatus.NO_CONTENT
     for header_name in CORS_HEADER_TUPLE:
         assert header_name in response.headers

common/tests/unit/flask_ext/test_cors.py

@@ -1,3 +1,4 @@
+import flask
 import pytest
 from flask import Response
 
@@ -7,8 +8,30 @@
 
 
 @pytest.mark.unit
-def test_headers_added():
-    response = Response()
-    CORS.set_cors_headers(response)
-    for header_name in CORS_HEADER_TUPLE:
-        assert header_name in response.headers
+def test_headers_added_matching_domain():
+    test_app = flask.Flask("test_flask_app")
+    cors = CORS()
+
+    matching_domains = ["https://xyz.com", "https://test.mydomain.com", "http://localhost:8080"]
+    for domain in matching_domains:
+        with test_app.test_request_context(headers={"Origin": domain}):
+            response = Response()
+            cors.set_cors_headers(response)
+            assert response.headers["Access-Control-Allow-Origin"] is domain
+            for header_name in CORS_HEADER_TUPLE:
+                assert header_name in response.headers
+
+
+@pytest.mark.unit
+def test_no_headers_other_domain():
+    test_app = flask.Flask("test_flask_app")
+    cors = CORS()
+
+    matching_domains = ["https://abc.com", "https://test.mydomain.com.io", "http://localhost"]
+    for domain in matching_domains:
+        with test_app.test_request_context(headers={"Origin": domain}):
+            response = Response()
+            cors.set_cors_headers(response)
+            assert "Access-Control-Allow-Origin" not in response.headers
+            for header_name in CORS_HEADER_TUPLE:
+                assert header_name not in response.headers

conf/cloud.py

@@ -9,6 +9,8 @@ class ReconnectingPooledMySQLDatabase(ReconnectMixin, PooledMySQLDatabase):
     pass
 
 
+CORS_DOMAINS: list[str] = os.environ.get("CORS_DOMAINS", "*").split(",")
+
 DATABASE: dict[str, object] = {
     "name": "datakitchen",
     "engine": ReconnectingPooledMySQLDatabase,

conf/minikube.py

@@ -8,6 +8,8 @@ class ReconnectingPooledMySQLDatabase(ReconnectMixin, PooledMySQLDatabase):
     pass
 
 
+CORS_DOMAINS: list[str] = os.environ.get("CORS_DOMAINS", "*").split(",")
+
 DATABASE: dict[str, object] = {
     "name": "datakitchen",
     "engine": ReconnectingPooledMySQLDatabase,

conf/test.py

@@ -1,5 +1,7 @@
 from peewee import SqliteDatabase
 
+CORS_DOMAINS: list[str] = ["*.mydomain.com", "xyz.com", "localhost:*"]
+
 DATABASE: dict[str, object] = {
     "name": "file:cachedb?mode=memory&cache=shared",
     "engine": SqliteDatabase,

deploy/charts/observability-app/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: dataops-observability-app
 type: application
 appVersion: "2.x.x"
-version: "2.2.4"
+version: "2.3.0"
 
 description: DataOps Observability
 home: https://datakitchen.io

deploy/charts/observability-app/templates/_environments.tpl

@@ -15,6 +15,8 @@ Environment
 {{- define "observability.environment.flask" -}}
 - name: FLASK_DEBUG
   value: {{ .Values.observability.flask_debug | quote }}
+- name: CORS_DOMAINS
+  value: {{ .Values.observability.cors_domains | quote }}
 {{- end -}}
 
 {{- define "observability.environment.database" -}}

deploy/charts/observability-app/templates/_helpers.tpl

@@ -187,3 +187,10 @@ CLI Hook
 {{- define "observability.cli_hook.image" }}
 {{- include "observability.image" (list . .Values.cli_hook) }}
 {{- end }}
+
+{{/*
+Cronjob
+*/}}
+{{- define "observability.cronjob.name" -}}
+cronjob-{{ .name }}
+{{- end -}}

deploy/charts/observability-app/templates/cronjob.yaml

@@ -0,0 +1,63 @@
+{{- range $job := (.Values.cronjob).enable }}
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: {{ include "observability.cronjob.name" . }}
+spec:
+  schedule: "{{ .schedule }}"
+  jobTemplate:
+    spec:
+      backoffLimit: {{ default 0 .backoffLimit }}
+      template:
+        metadata:
+          name: {{ include "observability.cronjob.name" . }}
+        spec:
+          restartPolicy: Never
+          {{- with $.Values.imagePullSecrets }}
+          imagePullSecrets:
+            {{ toYaml . | nindent 12 }}
+          {{- end }}
+          serviceAccountName: {{ include "observability.serviceAccountName" $ }}
+          containers:
+            - name: {{ include "observability.cronjob.name" . }}
+              image: {{ .image | quote }}
+              imagePullPolicy: {{ .imagePullPolicy }}
+              env:
+                {{- include "observability.environment.base" $ | nindent 16 }}
+                {{- include "observability.environment.database" $ | nindent 16 }}
+                {{- include "observability.environment.smtp" $ | nindent 16 }}
+              command:
+                {{- range .command }}
+                - {{ . | quote -}}
+                {{- end }}
+              args:
+                {{- range .args }}
+                - {{ . | quote -}}
+                {{- end }}
+              volumeMounts:
+              {{- range .configFiles }}
+                - mountPath: {{ .mountPath }}
+                  name: {{ include "observability.cronjob.name" $job }}-configmap-volume
+                  readOnly: true
+                  subPath: {{ .name }}
+              {{- end }}
+          volumes:
+            - name: {{ include "observability.cronjob.name" . }}-configmap-volume
+              configMap:
+                name: {{ include "observability.cronjob.name" . }}-configmap
+                items:
+                {{- range .configFiles }}
+                  - key: {{ .name }}
+                    path: {{ .name }}
+                {{- end }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "observability.cronjob.name" . }}-configmap
+data:
+  {{- range .configFiles }}
+  {{ .name }}: {{ .jsonData | toPrettyJson | quote }}
+  {{- end }}
+---
+{{- end }}

deploy/charts/observability-app/values.yaml

@@ -7,6 +7,7 @@ observability:
   mysql_secrets_name: mysql
   mysql_user: observability
   flask_debug: "false"
+  cors_domains: "localhost:*"
   services_secrets_name: external-service-keys
   keys_secrets_name: internal-keys
   pythonpath: /dk/lib/python3.12/site-packages

deploy/docker/docker-bake.json

@@ -11,7 +11,11 @@
                 "datakitchen/dataops-observability-be:v${index(regex(\"([0-9]+)\\\\.[0-9]+\\\\.[0-9]+\", OBSERVABILITY_VERSION), 0)}"
             ],
             "context": ".",
-            "platforms": ["linux/amd64", "linux/arm64"]
+            "platforms": ["linux/amd64", "linux/arm64"],
+            "attest": [
+                { "type": "provenance", "mode": "max" },
+                { "type": "sbom" }
+            ]
         },
         "ui": {
             "dockerfile": "deploy/docker/observability-ui.dockerfile",
@@ -21,7 +25,11 @@
                 "datakitchen/dataops-observability-ui:v${index(regex(\"([0-9]+)\\\\.[0-9]+\\\\.[0-9]+\", OBSERVABILITY_VERSION), 0)}"
             ],
             "context": ".",
-            "platforms": ["linux/amd64", "linux/arm64"]
+            "platforms": ["linux/amd64", "linux/arm64"],
+            "attest": [
+                { "type": "provenance", "mode": "max" },
+                { "type": "sbom" }
+            ]
         }
     }
 }

deploy/docker/observability-be.dockerfile

@@ -1,6 +1,6 @@
 # DEV NOTE:  YOU MUST RUN `docker build` FROM THE TOP-LEVEL OF `observability-be` AND POINT TO THIS FILE.
 ARG BASE_IMAGE_URL
-FROM ${BASE_IMAGE_URL}python:3.12.7-alpine3.20 AS build-image
+FROM ${BASE_IMAGE_URL}python:3.12.11-alpine3.22 AS build-image
 LABEL maintainer="DataKitchen"
 
 RUN apk update && apk upgrade && apk add --no-cache \
@@ -10,7 +10,7 @@ RUN apk update && apk upgrade && apk add --no-cache \
     make \
     cmake \
     musl-dev \
-    librdkafka-dev=2.4.0-r0
+    librdkafka-dev=2.10.0-r0
 
 COPY pyproject.toml /tmp/dk/
 #           -O: Strips asserts from the code which removes some unnecessary codepaths resulting in a small
@@ -30,9 +30,9 @@ ENV PYTHONPATH ${PYTHONPATH}:/dk/lib/python3.12/site-packages
 # --prefix=/dk: The destination installation environment folder
 RUN python3 -O -m pip install --no-deps /tmp/dk --prefix=/dk
 
-FROM ${BASE_IMAGE_URL}python:3.12.7-alpine3.20 AS runtime-image
+FROM ${BASE_IMAGE_URL}python:3.12.11-alpine3.22 AS runtime-image
 
-RUN apk update && apk upgrade && apk add --no-cache librdkafka=2.4.0-r0
+RUN apk update && apk upgrade && apk add --no-cache librdkafka=2.10.0-r0
 
 # Grab the pre-built app from the build-image. This way we don't have
 # excess laying around in the final image.
@@ -43,3 +43,11 @@ COPY --from=build-image /tmp/dk/deploy/migrations/ /dk/lib/migrations/
 
 ENV PYTHONPATH ${PYTHONPATH}:/dk/lib/python3.12/site-packages
 ENV PATH ${PATH}:/dk/bin
+
+RUN addgroup -S observability && adduser -S observability -G observability
+
+# gunicorn needs access to this folder
+RUN mkdir /dk/var
+RUN chown -R observability:observability /dk/var
+
+USER observability

deploy/docker/observability-ui.dockerfile

@@ -1,14 +1,14 @@
 ARG BASE_IMAGE_URL
 
-FROM --platform=${BUILDPLATFORM} ${BASE_IMAGE_URL}node:23.10-alpine3.21 AS build-image
+FROM --platform=${BUILDPLATFORM} ${BASE_IMAGE_URL}node:23.11.1-alpine3.22 AS build-image
 
 WORKDIR /observability_ui
 COPY observability_ui/ /observability_ui
 
 RUN yarn
 RUN yarn build:ci
 
-FROM ${BASE_IMAGE_URL}nginxinc/nginx-unprivileged:alpine3.21
+FROM ${BASE_IMAGE_URL}nginxinc/nginx-unprivileged:alpine3.22
 
 WORKDIR /observability_ui
 

observability_api/app.py

@@ -20,7 +20,7 @@
 URLConverters(app)
 Config(app, config_module="observability_api.config")
 DatabaseConnection(app)
-CORS(app, allowed_methods=["GET", "POST", "PATCH", "DELETE"])
+CORS(app, allowed_methods=["GET", "POST", "PATCH", "PUT", "DELETE"])
 Logging(app)
 ExceptionHandling(app)
 Timing(app)