Start testing k8s 1.29 (#486)

* Use metadata.yaml to define k8s versions. * Start testing k8s 1.29.
astronomer · Feb 6, 2024 · 5326a02 · 5326a02
1 parent 2bfe0ad
commit 5326a02
Show file tree

Hide file tree

Showing 19 changed files with 36,155 additions and 80 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -41,94 +41,118 @@ workflows:
             - build-and-release-internal
 
       - airflow-test:
-          name: test-1-25-11-CeleryExecutor
-          kube_version: 1.25.11
+          name: test-1-25-16-CeleryExecutor
+          kube_version: 1.25.16
           executor: CeleryExecutor
           requires:
             - build-and-release-internal
 
             - approve-test-all
       - airflow-test:
-          name: test-1-25-11-LocalExecutor
-          kube_version: 1.25.11
+          name: test-1-25-16-LocalExecutor
+          kube_version: 1.25.16
           executor: LocalExecutor
           requires:
             - build-and-release-internal
 
             - approve-test-all
       - airflow-test:
-          name: test-1-25-11-KubernetesExecutor
-          kube_version: 1.25.11
+          name: test-1-25-16-KubernetesExecutor
+          kube_version: 1.25.16
           executor: KubernetesExecutor
           requires:
             - build-and-release-internal
 
             - approve-test-all
       - airflow-test:
-          name: test-1-26-6-CeleryExecutor
-          kube_version: 1.26.6
+          name: test-1-26-13-CeleryExecutor
+          kube_version: 1.26.13
           executor: CeleryExecutor
           requires:
             - build-and-release-internal
 
             - approve-test-all
       - airflow-test:
-          name: test-1-26-6-LocalExecutor
-          kube_version: 1.26.6
+          name: test-1-26-13-LocalExecutor
+          kube_version: 1.26.13
           executor: LocalExecutor
           requires:
             - build-and-release-internal
 
             - approve-test-all
       - airflow-test:
-          name: test-1-26-6-KubernetesExecutor
-          kube_version: 1.26.6
+          name: test-1-26-13-KubernetesExecutor
+          kube_version: 1.26.13
           executor: KubernetesExecutor
           requires:
             - build-and-release-internal
 
             - approve-test-all
       - airflow-test:
-          name: test-1-27-3-CeleryExecutor
-          kube_version: 1.27.3
+          name: test-1-27-10-CeleryExecutor
+          kube_version: 1.27.10
           executor: CeleryExecutor
           requires:
             - build-and-release-internal
 
             - approve-test-all
       - airflow-test:
-          name: test-1-27-3-LocalExecutor
-          kube_version: 1.27.3
+          name: test-1-27-10-LocalExecutor
+          kube_version: 1.27.10
           executor: LocalExecutor
           requires:
             - build-and-release-internal
 
             - approve-test-all
       - airflow-test:
-          name: test-1-27-3-KubernetesExecutor
-          kube_version: 1.27.3
+          name: test-1-27-10-KubernetesExecutor
+          kube_version: 1.27.10
           executor: KubernetesExecutor
           requires:
             - build-and-release-internal
 
             - approve-test-all
       - airflow-test:
-          name: test-1-28-0-CeleryExecutor
-          kube_version: 1.28.0
+          name: test-1-28-6-CeleryExecutor
+          kube_version: 1.28.6
+          executor: CeleryExecutor
+          requires:
+            - build-and-release-internal
+
+            - approve-test-all
+      - airflow-test:
+          name: test-1-28-6-LocalExecutor
+          kube_version: 1.28.6
+          executor: LocalExecutor
+          requires:
+            - build-and-release-internal
+
+            - approve-test-all
+      - airflow-test:
+          name: test-1-28-6-KubernetesExecutor
+          kube_version: 1.28.6
+          executor: KubernetesExecutor
+          requires:
+            - build-and-release-internal
+
+            - approve-test-all
+      - airflow-test:
+          name: test-1-29-1-CeleryExecutor
+          kube_version: 1.29.1
           executor: CeleryExecutor
           requires:
             - build-and-release-internal
 
       - airflow-test:
-          name: test-1-28-0-LocalExecutor
-          kube_version: 1.28.0
+          name: test-1-29-1-LocalExecutor
+          kube_version: 1.29.1
           executor: LocalExecutor
           requires:
             - build-and-release-internal
 
       - airflow-test:
-          name: test-1-28-0-KubernetesExecutor
-          kube_version: 1.28.0
+          name: test-1-29-1-KubernetesExecutor
+          kube_version: 1.29.1
           executor: KubernetesExecutor
           requires:
             - build-and-release-internal
@@ -139,18 +163,21 @@ workflows:
             - "test-1-24-15-CeleryExecutor"
             - "test-1-24-15-LocalExecutor"
             - "test-1-24-15-KubernetesExecutor"
-            - "test-1-25-11-CeleryExecutor"
-            - "test-1-25-11-LocalExecutor"
-            - "test-1-25-11-KubernetesExecutor"
-            - "test-1-26-6-CeleryExecutor"
-            - "test-1-26-6-LocalExecutor"
-            - "test-1-26-6-KubernetesExecutor"
-            - "test-1-27-3-CeleryExecutor"
-            - "test-1-27-3-LocalExecutor"
-            - "test-1-27-3-KubernetesExecutor"
-            - "test-1-28-0-CeleryExecutor"
-            - "test-1-28-0-LocalExecutor"
-            - "test-1-28-0-KubernetesExecutor"
+            - "test-1-25-16-CeleryExecutor"
+            - "test-1-25-16-LocalExecutor"
+            - "test-1-25-16-KubernetesExecutor"
+            - "test-1-26-13-CeleryExecutor"
+            - "test-1-26-13-LocalExecutor"
+            - "test-1-26-13-KubernetesExecutor"
+            - "test-1-27-10-CeleryExecutor"
+            - "test-1-27-10-LocalExecutor"
+            - "test-1-27-10-KubernetesExecutor"
+            - "test-1-28-6-CeleryExecutor"
+            - "test-1-28-6-LocalExecutor"
+            - "test-1-28-6-KubernetesExecutor"
+            - "test-1-29-1-CeleryExecutor"
+            - "test-1-29-1-LocalExecutor"
+            - "test-1-29-1-KubernetesExecutor"
           filters:
             branches:
               only:
@@ -171,7 +198,7 @@ workflows:
 jobs:
   run_pre_commit:
     docker:
-      - image: quay.io/astronomer/ci-pre-commit:2024-01
+      - image: quay.io/astronomer/ci-pre-commit:2024-02
     steps:
       - checkout
       - run:
@@ -206,7 +233,7 @@ jobs:
 
   unittest-charts:
     docker:
-      - image: quay.io/astronomer/ci-helm-release:2024-01
+      - image: quay.io/astronomer/ci-helm-release:2024-02
     steps:
       - setup_remote_docker:
           docker_layer_caching: true
@@ -221,7 +248,7 @@ jobs:
 
   build-and-release-internal:
     docker:
-      - image: quay.io/astronomer/ci-helm-release:2024-01
+      - image: quay.io/astronomer/ci-helm-release:2024-02
     steps:
       - checkout
       - run:
@@ -261,7 +288,7 @@ jobs:
           path: test-results
   release-internal:
     docker:
-      - image: quay.io/astronomer/ci-helm-release:2024-01
+      - image: quay.io/astronomer/ci-helm-release:2024-02
     steps:
       - checkout
       - run:
@@ -270,7 +297,7 @@ jobs:
 
   release-public:
     docker:
-      - image: quay.io/astronomer/ci-helm-release:2024-01
+      - image: quay.io/astronomer/ci-helm-release:2024-02
     steps:
       - checkout
       - publish-github-release

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -64,9 +64,9 @@ repos:
       - id: circle-config-yaml
         name: Ensure .circleci/config.yml is up to date
         language: python
-        files: "config.yml$|config.yml.j2|generate_circleci_config.py$"
-        entry: .circleci/generate_circleci_config.py
-        additional_dependencies: ["jinja2"]
+        files: "(config.yml|config.yml.j2|generate_circleci_config.py)$"
+        entry: bin/generate_circleci_config.py
+        additional_dependencies: ["jinja2", "PyYAML"]
       - id: helm-dependency-update
         name: Ensure helm lock is up to date with requirements in Chart.yaml
         files: "Chart.yaml"

diff --git a/.circleci/generate_circleci_config.py → bin/generate_circleci_config.py b/.circleci/generate_circleci_config.py → bin/generate_circleci_config.py
@@ -1,40 +1,34 @@
 #!/usr/bin/env python3
 """This script is used to create the circle config file so that we can stay
 DRY."""
-import os
 from pathlib import Path
+import yaml
 
 from jinja2 import Template
 
-# When adding a new version, look up the most recent patch version on Dockerhub
-# https://hub.docker.com/r/kindest/node/tags
-# This should match what is in tests/__init__.py
-kube_versions = [
-    "1.24.15",
-    "1.25.11",
-    "1.26.6",
-    "1.27.3",
-    "1.28.0",
-]
+git_root_dir = next(
+    iter([x for x in Path(__file__).resolve().parents if (x / ".git").is_dir()]), None
+)
+metadata = yaml.safe_load((git_root_dir / "metadata.yaml").read_text())
+kube_versions = metadata["test_k8s_versions"]
 
 executors = ["CeleryExecutor", "LocalExecutor", "KubernetesExecutor"]
-ci_runner_version = "2024-01"
+ci_runner_version = "2024-02"
 
 
 def main():
     """Render the Jinja2 template file."""
-    circle_directory = os.path.dirname(os.path.realpath(__file__))
-    config_template_path = os.path.join(circle_directory, "config.yml.j2")
-    config_path = os.path.join(circle_directory, "config.yml")
+    config_file_template_path = git_root_dir / ".circleci" / "config.yml.j2"
+    config_file_path = git_root_dir / ".circleci" / "config.yml"
 
-    templated_file_content = Path(config_template_path).read_text()
+    templated_file_content = Path(config_file_template_path).read_text()
     template = Template(templated_file_content)
     config = template.render(
         kube_versions=kube_versions,
         executors=executors,
         ci_runner_version=ci_runner_version,
     )
-    with open(config_path, "w") as circle_ci_config_file:
+    with open(config_file_path, "w") as circle_ci_config_file:
         warning_header = (
             "# Warning: automatically generated file\n"
             + "# Please edit config.yml.j2, and use the script generate_circleci_config.py\n"

diff --git a/metadata.yaml b/metadata.yaml
@@ -0,0 +1,9 @@
+# When adding a new version, look up the most recent patch version on Dockerhub
+# https://hub.docker.com/r/kindest/node/tags
+test_k8s_versions:
+  - 1.24.15
+  - 1.25.16
+  - 1.26.13
+  - 1.27.10
+  - 1.28.6
+  - 1.29.1
diff --git a/tests/__init__.py b/tests/__init__.py
@@ -1,21 +1,14 @@
 from pathlib import Path
 
-import git
 import yaml
 
 # The top-level path of this repository
-git_repo = git.Repo(__file__, search_parent_directories=True)
-git_root_dir = Path(git_repo.git.rev_parse("--show-toplevel"))
-
-with open(f"{git_root_dir}/.circleci/config.yml") as f:
-    circleci_config = yaml.safe_load(f.read())
+git_root_dir = [x for x in Path(__file__).resolve().parents if (x / ".git").is_dir()][
+    -1
+]
 
-# This should match the major.minor version list in .circleci/generate_circleci_config.py
-# Patch version should always be 0
+metadata = yaml.safe_load((Path(git_root_dir) / "metadata.yaml").read_text())
+# replace all patch versions with 0 so we end up with ['1.26.0', '1.27.0']
 supported_k8s_versions = [
-    "1.24.0",
-    "1.25.0",
-    "1.26.0",
-    "1.27.0",
-    "1.28.0",
+    ".".join(x.split(".")[:-1] + ["0"]) for x in metadata["test_k8s_versions"]
 ]
diff --git a/tests/chart_tests/conftest.py b/tests/chart_tests/conftest.py
@@ -19,13 +19,12 @@
 from pathlib import Path
 
 import docker
-import git
 import pytest
 from filelock import FileLock
 
-# The top-level path of this repository
-git_repo = git.Repo(__file__, search_parent_directories=True)
-git_root_dir = Path(git_repo.git.rev_parse("--show-toplevel"))
+git_root_dir = [x for x in Path(__file__).resolve().parents if (x / ".git").is_dir()][
+    -1
+]
 
 
 @pytest.fixture(autouse=True, scope="session")

diff --git a/tests/chart_tests/helm_template_generator.py b/tests/chart_tests/helm_template_generator.py
@@ -33,7 +33,10 @@
 api_client = ApiClient()
 
 BASE_URL_SPEC = "https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master"
-GIT_ROOT = Path(__file__).parent.parent.parent
+# The top-level path of this repository
+git_root_dir = [x for x in Path(__file__).resolve().parents if (x / ".git").is_dir()][
+    -1
+]
 DEBUG = os.getenv("DEBUG", "").lower() in ["yes", "true", "1"]
 
 
@@ -49,7 +52,7 @@ def get_schema_k8s(api_version, kind, kube_version):
     else:
         schema_path = f"v{kube_version}-standalone/{kind}-{api_version}.json"
 
-    local_sp = Path(f"{GIT_ROOT}/tests/k8s_schema/{schema_path}")
+    local_sp = Path(f"{git_root_dir}/tests/k8s_schema/{schema_path}")
     if not local_sp.exists():
         if not local_sp.parent.is_dir():
             local_sp.parent.mkdir()