From 721f60fa811f92ac8943b5a102eadc38576ebeb7 Mon Sep 17 00:00:00 2001 From: nailo2c Date: Mon, 6 Jul 2026 14:28:16 +0800 Subject: [PATCH] Reduce K8s test resource usage with single-node Kind clusters --- .../workflows/kustomize-overlays-tests.yml | 6 +- contributing-docs/testing/k8s_tests.rst | 7 +- .../airflow_breeze/utils/kubernetes_utils.py | 18 +++- dev/breeze/tests/test_kubernetes_utils.py | 87 +++++++++++++++++++ scripts/ci/kubernetes/kind-cluster-conf.yaml | 1 - 5 files changed, 108 insertions(+), 11 deletions(-) create mode 100644 dev/breeze/tests/test_kubernetes_utils.py diff --git a/.github/workflows/kustomize-overlays-tests.yml b/.github/workflows/kustomize-overlays-tests.yml index 3a4bcf7ee9f46..3fef28b18b345 100644 --- a/.github/workflows/kustomize-overlays-tests.yml +++ b/.github/workflows/kustomize-overlays-tests.yml @@ -102,13 +102,13 @@ jobs: # convention (sources come from the base # image, not the workspace). # 5. upload-k8s-image -> loads the now-built kubernetes-variant - # image into every kind node, so kubelet + # image into the kind node, so kubelet # can resolve the chart's image reference # without hitting ghcr (which is private). # 6. deploy-airflow # 7. smoke-test-overlay -> renders the overlay, auto-preloads the # overlay's own images into the kind - # nodes (no per-overlay images list + # node (no per-overlay images list # needed - discovery walks the rendered # manifest), applies, walks the # STATUS.yaml verify: block with @@ -124,7 +124,7 @@ jobs: run: breeze k8s configure-cluster - name: "Build kubernetes-variant image on top of the restored PROD image" run: breeze k8s build-k8s-image --no-copy-local-sources - - name: "Upload kubernetes-variant image to kind nodes" + - name: "Upload kubernetes-variant image to the kind node" run: breeze k8s upload-k8s-image - name: "Deploy Airflow chart" run: breeze k8s deploy-airflow --executor CeleryExecutor diff --git a/contributing-docs/testing/k8s_tests.rst b/contributing-docs/testing/k8s_tests.rst index 299e77dfb5b20..9d6e33219f3db 100644 --- a/contributing-docs/testing/k8s_tests.rst +++ b/contributing-docs/testing/k8s_tests.rst @@ -282,7 +282,6 @@ Should result in KinD creating the K8S cluster. apiServerPort: 48366 nodes: - role: control-plane - - role: worker extraPortMappings: - containerPort: 30007 hostPort: 18150 @@ -291,12 +290,11 @@ Should result in KinD creating the K8S cluster. Creating cluster "airflow-python-3.10-v1.24.2" ... ✓ Ensuring node image (kindest/node:v1.24.2) đŸ–ŧ - ✓ Preparing nodes đŸ“Ļ đŸ“Ļ + ✓ Preparing nodes đŸ“Ļ ✓ Writing configuration 📜 ✓ Starting control-plane 🕹ī¸ ✓ Installing CNI 🔌 ✓ Installing StorageClass 💾 - ✓ Joining worker nodes 🚜 Set kubectl context to "kind-airflow-python-3.10-v1.24.2" You can now use your cluster with: @@ -379,11 +377,9 @@ Should show the status of current KinD cluster. coredns-6d4b75cb6d-rwp9d 1/1 Running 0 71s coredns-6d4b75cb6d-vqnrc 1/1 Running 0 71s etcd-airflow-python-3.10-v1.24.2-control-plane 1/1 Running 0 84s - kindnet-ckc8l 1/1 Running 0 69s kindnet-qqt8k 1/1 Running 0 71s kube-apiserver-airflow-python-3.10-v1.24.2-control-plane 1/1 Running 0 84s kube-controller-manager-airflow-python-3.10-v1.24.2-control-plane 1/1 Running 0 84s - kube-proxy-6g7hn 1/1 Running 0 69s kube-proxy-dwfvp 1/1 Running 0 71s kube-scheduler-airflow-python-3.10-v1.24.2-control-plane 1/1 Running 0 84s @@ -449,7 +445,6 @@ Should show the status of current KinD cluster. Good version of helm installed: 3.9.2 in /Users/jarek/IdeaProjects/airflow/kubernetes-tests/.venv/bin Stable repo is already added Uploading Airflow image ghcr.io/apache/airflow/main/prod/python3.10-kubernetes to cluster airflow-python-3.10-v1.24.2 - Image: "ghcr.io/apache/airflow/main/prod/python3.10-kubernetes" with ID "sha256:fb6195f7c2c2ad97788a563a3fe9420bf3576c85575378d642cd7985aff97412" not yet present on node "airflow-python-3.10-v1.24.2-worker", loading... Image: "ghcr.io/apache/airflow/main/prod/python3.10-kubernetes" with ID "sha256:fb6195f7c2c2ad97788a563a3fe9420bf3576c85575378d642cd7985aff97412" not yet present on node "airflow-python-3.10-v1.24.2-control-plane", loading... NEXT STEP: You might now deploy Airflow by: diff --git a/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py b/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py index 2a0d6918b5006..1989d461847f8 100644 --- a/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py @@ -402,12 +402,28 @@ def set_random_cluster_ports(python: str, kubernetes_version: str, output: Outpu get_console(output=output).print("\n") +# Must match containerPort in scripts/ci/kubernetes/kind-cluster-conf.yaml and +# nodePort in scripts/ci/kubernetes/nodeport.yaml. +FORWARDED_NODE_PORT = 30007 + + +def _extract_forwarded_host_port(conf: dict[str, Any]) -> int: + for node in conf["nodes"]: + for port_mapping in node.get("extraPortMappings", []): + if port_mapping["containerPort"] == FORWARDED_NODE_PORT: + return port_mapping["hostPort"] + raise ValueError( + f"No node in the kind cluster config has an extraPortMappings entry for " + f"containerPort {FORWARDED_NODE_PORT}. Delete and recreate the cluster." + ) + + def get_kubernetes_port_numbers(python: str, kubernetes_version: str) -> tuple[int, int]: conf = _get_kind_cluster_config_content(python=python, kubernetes_version=kubernetes_version) if not conf: return 0, 0 k8s_api_server_port = conf["networking"]["apiServerPort"] - api_server_port = conf["nodes"][1]["extraPortMappings"][0]["hostPort"] + api_server_port = _extract_forwarded_host_port(conf) return k8s_api_server_port, api_server_port diff --git a/dev/breeze/tests/test_kubernetes_utils.py b/dev/breeze/tests/test_kubernetes_utils.py new file mode 100644 index 0000000000000..64abbb0a6740e --- /dev/null +++ b/dev/breeze/tests/test_kubernetes_utils.py @@ -0,0 +1,87 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +from unittest.mock import patch + +import pytest +import yaml + +from airflow_breeze.utils.kubernetes_utils import get_kubernetes_port_numbers + +SINGLE_NODE_CONFIG = """\ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: + ipFamily: ipv4 + apiServerAddress: "127.0.0.1" + apiServerPort: 48366 +nodes: + - role: control-plane + extraPortMappings: + - containerPort: 30007 + hostPort: 18150 + listenAddress: "127.0.0.1" + protocol: TCP +""" + +# Rendered config of a cluster created before the switch to single-node clusters — +# port extraction must keep working for clusters that already exist on disk. +LEGACY_TWO_NODE_CONFIG = """\ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: + ipFamily: ipv4 + apiServerAddress: "127.0.0.1" + apiServerPort: 48366 +nodes: + - role: control-plane + - role: worker + extraPortMappings: + - containerPort: 30007 + hostPort: 18150 + listenAddress: "127.0.0.1" + protocol: TCP +""" + +NO_FORWARDED_PORT_CONFIG = """\ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: + ipFamily: ipv4 + apiServerAddress: "127.0.0.1" + apiServerPort: 48366 +nodes: + - role: control-plane +""" + + +@pytest.mark.parametrize( + "config", [SINGLE_NODE_CONFIG, LEGACY_TWO_NODE_CONFIG], ids=["single-node", "legacy-two-node"] +) +@patch("airflow_breeze.utils.kubernetes_utils._get_kind_cluster_config_content", autospec=True) +def test_get_kubernetes_port_numbers_extracts_ports_from_any_node(mock_get_content, config): + mock_get_content.return_value = yaml.safe_load(config) + assert get_kubernetes_port_numbers(python="3.10", kubernetes_version="v1.30.13") == (48366, 18150) + + +@patch("airflow_breeze.utils.kubernetes_utils._get_kind_cluster_config_content", autospec=True) +def test_get_kubernetes_port_numbers_raises_when_forwarded_port_mapping_is_missing(mock_get_content): + mock_get_content.return_value = yaml.safe_load(NO_FORWARDED_PORT_CONFIG) + with pytest.raises(ValueError, match="extraPortMappings"): + get_kubernetes_port_numbers(python="3.10", kubernetes_version="v1.30.13") diff --git a/scripts/ci/kubernetes/kind-cluster-conf.yaml b/scripts/ci/kubernetes/kind-cluster-conf.yaml index 4e891f80ddf62..f0d9eb5799eaa 100644 --- a/scripts/ci/kubernetes/kind-cluster-conf.yaml +++ b/scripts/ci/kubernetes/kind-cluster-conf.yaml @@ -23,7 +23,6 @@ networking: apiServerPort: {{API_SERVER_PORT}} nodes: - role: control-plane - - role: worker extraPortMappings: - containerPort: 30007 hostPort: {{FORWARDED_PORT_NUMBER}}