6136 improve CI (#2112)

* Update various component versions * Move generate_circleci_config.py
astronomer · Feb 6, 2024 · 0abc690 · 0abc690
1 parent d54abcb
commit 0abc690
Show file tree

Hide file tree

Showing 7 changed files with 284 additions and 284 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -110,7 +110,7 @@ jobs:
 
   run_pre_commit:
     docker:
-      - image: quay.io/astronomer/ci-pre-commit:2023-11
+      - image: quay.io/astronomer/ci-pre-commit:2024-02
     steps:
       - checkout
       - run:
@@ -145,7 +145,7 @@ jobs:
 
   unittest-charts:
     docker:
-      - image: quay.io/astronomer/ci-helm-release:2023-11
+      - image: quay.io/astronomer/ci-helm-release:2024-02
     parallelism: 8
     resource_class: xlarge
     steps:
@@ -199,7 +199,7 @@ jobs:
 
   release-to-internal:
     docker:
-      - image: quay.io/astronomer/ci-helm-release:2023-11
+      - image: quay.io/astronomer/ci-helm-release:2024-02
     steps:
       - attach_workspace:
           at: /tmp/workspace
@@ -238,7 +238,7 @@ jobs:
 
   release-to-public:
     docker:
-      - image: quay.io/astronomer/ci-helm-release:2023-11
+      - image: quay.io/astronomer/ci-helm-release:2024-02
     steps:
       - attach_workspace:
           at: /tmp/workspace
@@ -249,84 +249,84 @@ jobs:
             bin/release-helm-chart -p /tmp/workspace/astronomer-*.tgz
       - publish-github-release
 
-  platform-1-24-15:
+  platform-1-24-17:
     machine:
-      image: ubuntu-2204:2023.07.2
+      image: ubuntu-2204:2024.01.1
       resource_class: xlarge
     environment:
-      KUBE_VERSION: v1.24.15
+      KUBE_VERSION: v1.24.17
     steps:
       - helm-install:
           astronomer-tags: "platform postgresql monitoring logging"
       - run:
-          name: Check chart for k8s 1.24.15 compatibility with kubent
+          name: Check chart for k8s 1.24.17 compatibility with kubent
           command: |
             set -o pipefail
-            helm template --kube-version=v1.24.15 -f values.yaml --set global.baseDomain=example.com . |
+            helm template --kube-version=v1.24.17 -f values.yaml --set global.baseDomain=example.com . |
             kubent --cluster=false --helm3=false --filename -
           when: always
-  platform-1-25-11:
+  platform-1-25-16:
     machine:
-      image: ubuntu-2204:2023.07.2
+      image: ubuntu-2204:2024.01.1
       resource_class: xlarge
     environment:
-      KUBE_VERSION: v1.25.11
+      KUBE_VERSION: v1.25.16
     steps:
       - helm-install:
           astronomer-tags: "platform postgresql monitoring logging"
       - run:
-          name: Check chart for k8s 1.25.11 compatibility with kubent
+          name: Check chart for k8s 1.25.16 compatibility with kubent
           command: |
             set -o pipefail
-            helm template --kube-version=v1.25.11 -f values.yaml --set global.baseDomain=example.com . |
+            helm template --kube-version=v1.25.16 -f values.yaml --set global.baseDomain=example.com . |
             kubent --cluster=false --helm3=false --filename -
           when: always
-  platform-1-26-6:
+  platform-1-26-13:
     machine:
-      image: ubuntu-2204:2023.07.2
+      image: ubuntu-2204:2024.01.1
       resource_class: xlarge
     environment:
-      KUBE_VERSION: v1.26.6
+      KUBE_VERSION: v1.26.13
     steps:
       - helm-install:
           astronomer-tags: "platform postgresql monitoring logging"
       - run:
-          name: Check chart for k8s 1.26.6 compatibility with kubent
+          name: Check chart for k8s 1.26.13 compatibility with kubent
           command: |
             set -o pipefail
-            helm template --kube-version=v1.26.6 -f values.yaml --set global.baseDomain=example.com . |
+            helm template --kube-version=v1.26.13 -f values.yaml --set global.baseDomain=example.com . |
             kubent --cluster=false --helm3=false --filename -
           when: always
-  platform-1-27-3:
+  platform-1-27-10:
     machine:
-      image: ubuntu-2204:2023.07.2
+      image: ubuntu-2204:2024.01.1
       resource_class: xlarge
     environment:
-      KUBE_VERSION: v1.27.3
+      KUBE_VERSION: v1.27.10
     steps:
       - helm-install:
           astronomer-tags: "platform postgresql monitoring logging"
       - run:
-          name: Check chart for k8s 1.27.3 compatibility with kubent
+          name: Check chart for k8s 1.27.10 compatibility with kubent
           command: |
             set -o pipefail
-            helm template --kube-version=v1.27.3 -f values.yaml --set global.baseDomain=example.com . |
+            helm template --kube-version=v1.27.10 -f values.yaml --set global.baseDomain=example.com . |
             kubent --cluster=false --helm3=false --filename -
           when: always
-  platform-1-28-0:
+  platform-1-28-6:
     machine:
-      image: ubuntu-2204:2023.07.2
+      image: ubuntu-2204:2024.01.1
       resource_class: xlarge
     environment:
-      KUBE_VERSION: v1.28.0
+      KUBE_VERSION: v1.28.6
     steps:
       - helm-install:
           astronomer-tags: "platform postgresql monitoring logging"
       - run:
-          name: Check chart for k8s 1.28.0 compatibility with kubent
+          name: Check chart for k8s 1.28.6 compatibility with kubent
           command: |
             set -o pipefail
-            helm template --kube-version=v1.28.0 -f values.yaml --set global.baseDomain=example.com . |
+            helm template --kube-version=v1.28.6 -f values.yaml --set global.baseDomain=example.com . |
             kubent --cluster=false --helm3=false --filename -
           when: always
 
@@ -451,25 +451,25 @@ workflows:
       - approve-test-all-platforms:
           type: approval
 
-      - platform-1-24-15:
+      - platform-1-24-17:
           requires:
             - build-artifact
-      - platform-1-25-11:
+      - platform-1-25-16:
           requires:
             - approve-test-all-platforms
             - build-artifact
 
-      - platform-1-26-6:
+      - platform-1-26-13:
           requires:
             - approve-test-all-platforms
             - build-artifact
 
-      - platform-1-27-3:
+      - platform-1-27-10:
           requires:
             - approve-test-all-platforms
             - build-artifact
 
-      - platform-1-28-0:
+      - platform-1-28-6:
           requires:
             - build-artifact
       - approve-internal-release:
@@ -487,8 +487,8 @@ workflows:
                 - '/^release-0\.\d+$/'
           requires:
             - approve-internal-release
-            - platform-1-24-15
-            - platform-1-28-0
+            - platform-1-24-17
+            - platform-1-28-6
       - approve-upgrade-test:
           type: approval
           filters:
@@ -507,9 +507,9 @@ workflows:
           type: approval
           requires:
             - release-to-internal
-            - platform-1-25-11
-            - platform-1-26-6
-            - platform-1-27-3
+            - platform-1-25-16
+            - platform-1-26-13
+            - platform-1-27-10
           filters:
             branches:
               only:
@@ -544,20 +544,20 @@ workflows:
             - run_pre_commit
             - unittest-charts
 
-      - platform-1-24-15:
+      - platform-1-24-17:
           requires:
             - build-artifact
 
-      - platform-1-28-0:
+      - platform-1-28-6:
           requires:
             - build-artifact
 
       - release-to-internal:
           context:
             - gcp-astronomer-prod
           requires:
-            - platform-1-24-15
-            - platform-1-28-0
+            - platform-1-24-17
+            - platform-1-28-6
           filters:
             branches:
               only:

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -12,8 +12,8 @@ repos:
       - id: circle-config-yaml
         name: Checks for consistency between config.yml and config.yml.j2
         language: python
-        files: "config.yml$|config.yml.j2|generate_circleci_config.py$|values.yaml$"
-        entry: .circleci/generate_circleci_config.py
+        files: '(config.yml|config.yml.j2|generate_circleci_config.py|values.yaml|metadata.yaml)$'
+        entry: bin/generate_circleci_config.py
         additional_dependencies: ["jinja2", "PyYAML"]
   - repo: https://github.com/codespell-project/codespell
     rev: v2.2.6
@@ -26,11 +26,11 @@ repos:
       - id: remove-tabs
         exclude_types: [makefile, binary]
   - repo: https://github.com/psf/black
-    rev: 23.11.0
+    rev: 24.1.1
     hooks:
       - id: black
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: "v0.1.5"
+    rev: "v0.2.1"
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix, --ignore, E501]

diff --git a/.circleci/generate_circleci_config.py → bin/generate_circleci_config.py b/.circleci/generate_circleci_config.py → bin/generate_circleci_config.py
@@ -1,47 +1,49 @@
 #!/usr/bin/env python3
 """This script is used to create the circle config file so that we can stay
 DRY."""
-import os
 import subprocess
 from pathlib import Path
 import yaml
 
 from jinja2 import Template
 
-metadata = yaml.safe_load((Path(__file__).parents[1] / "metadata.yaml").read_text())
+git_root_dir = next(
+    iter([x for x in Path(__file__).resolve().parents if (x / ".git").is_dir()]), None
+)
+metadata = yaml.safe_load((git_root_dir / "metadata.yaml").read_text())
 kube_versions = metadata["test_k8s_versions"]
 
 # https://circleci.com/developer/machine/image/ubuntu-2204
-machine_image_version = "ubuntu-2204:2023.07.2"
-ci_runner_version = "2023-11"
+machine_image_version = "ubuntu-2204:2024.01.1"
+ci_runner_version = "2024-02"
 
 
-def list_docker_images(path):
-    command = f"cd {path} && helm template . --set forceIncompatibleKubernetes=true -f tests/enable_all_features.yaml 2>/dev/null | awk '/image: / {{print $2}}' | sed 's/\"//g' | sort -u"
+def list_docker_images():
+    command = f"{git_root_dir}/bin/show-docker-images.py --with-houston"
     docker_images_output = subprocess.check_output(command, shell=True)
-    docker_image_list = docker_images_output.decode("utf-8").strip().split("\n")
+    docker_image_list = [
+        x.split()[1] for x in docker_images_output.decode("utf-8").strip().split("\n")
+    ]
 
     return sorted(set(docker_image_list))
 
 
 def main():
     """Render the Jinja2 template file."""
-    project_directory = Path(__file__).parent.parent
-    circle_directory = os.path.dirname(os.path.realpath(__file__))
-    config_template_path = os.path.join(circle_directory, "config.yml.j2")
-    config_path = os.path.join(circle_directory, "config.yml")
+    config_file_template_path = git_root_dir / ".circleci" / "config.yml.j2"
+    config_file_path = git_root_dir / ".circleci" / "config.yml"
 
-    docker_images = list_docker_images(str(project_directory))
+    docker_images = list_docker_images()
 
-    templated_file_content = Path(config_template_path).read_text()
+    templated_file_content = config_file_template_path.read_text()
     template = Template(templated_file_content)
     config = template.render(
         kube_versions=kube_versions,
         docker_images=docker_images,
         machine_image_version=machine_image_version,
         ci_runner_version=ci_runner_version,
     )
-    with open(config_path, "w") as circle_ci_config_file:
+    with open(config_file_path, "w") as circle_ci_config_file:
         warning_header = (
             "# Warning: automatically generated file\n"
             + "# Please edit config.yml.j2, and use the script generate_circleci_config.py\n"

diff --git a/bin/install-ci-tools b/bin/install-ci-tools
@@ -1,11 +1,11 @@
 #!/usr/bin/env bash
 set -xeuo pipefail
 
-KIND_VERSION="0.20.0" # https://github.com/kubernetes-sigs/kind/releases
-HELM_VERSION="3.13.2" # https://github.com/helm/helm/releases
-GCLOUD_VERSION="447.0.0"
+KIND_VERSION="0.21.0" # https://github.com/kubernetes-sigs/kind/releases
+HELM_VERSION="3.14.0" # https://github.com/helm/helm/releases
+GCLOUD_VERSION="462.0.1"
 MKCERT_VERSION="1.4.4" # https://github.com/FiloSottile/mkcert/tags
-KUBENT_VERSION="0.7.0" # https://github.com/doitintl/kube-no-trouble
+KUBENT_VERSION="0.7.1" # https://github.com/doitintl/kube-no-trouble/releases
 
 OS=$(uname | tr '[:upper:]' '[:lower:]')
 

diff --git a/metadata.yaml b/metadata.yaml
@@ -1,8 +1,8 @@
 # When adding a new version, look up the most recent patch version on Dockerhub
 # https://hub.docker.com/r/kindest/node/tags
 test_k8s_versions:
-  - 1.24.15
-  - 1.25.11
-  - 1.26.6
-  - 1.27.3
-  - 1.28.0
+  - 1.24.17
+  - 1.25.16
+  - 1.26.13
+  - 1.27.10
+  - 1.28.6