Skip to content

Commit

Permalink
Prevent multiple component versions (#2094)
Browse files Browse the repository at this point in the history
* Use --set forceIncompatibleKubernetes=true so show-docker-images always works

* Remove deprecated remote_docker_version, use cimg/base:stable

* Add verify_image_tags.py and verify-image-tags pre-commit hook
  • Loading branch information
danielhoherd authored and rishkarajgi committed Jan 31, 2024
1 parent a805460 commit d67a3e9
Show file tree
Hide file tree
Showing 6 changed files with 72 additions and 23 deletions.
15 changes: 6 additions & 9 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ parameters:
jobs:
trivy-scan-docker:
docker:
- image: docker:20.10.24-git
- image: cimg/base:stable
shell: /bin/sh -leo pipefail
parameters:
docker_image:
Expand All @@ -47,7 +47,7 @@ jobs:
ASTRO_SEC_ENDPOINT: << parameters.report_url >>
steps:
- setup_remote_docker:
version: 20.10.24
docker_layer_caching: true
- checkout
- run:
name: Pull Docker image
Expand Down Expand Up @@ -86,14 +86,14 @@ jobs:

twistcli-scan-docker:
docker:
- image: docker:20.10.24-git
- image: cimg/base:stable
shell: /bin/sh -leo pipefail
parameters:
docker_image:
type: string
steps:
- setup_remote_docker:
version: 20.10.24
docker_layer_caching: true
- run:
name: Pull Docker image
command: docker pull << parameters.docker_image >>
Expand Down Expand Up @@ -151,7 +151,6 @@ jobs:
steps:
- setup_remote_docker:
docker_layer_caching: true
version: 20.10.24
- checkout
- run:
name: Create chart-tests-cache-key.txt
Expand Down Expand Up @@ -333,11 +332,11 @@ jobs:

check-commander-airflow-version:
docker:
- image: cimg/base:2022.09
- image: cimg/base:stable
steps:
- checkout
- setup_remote_docker:
version: 20.10.24
docker_layer_caching: true
- run:
name: Check that commander image uses same Airflow chart version
command: make validate-commander-airflow-version
Expand Down Expand Up @@ -587,7 +586,6 @@ commands:
steps:
- setup_remote_docker:
docker_layer_caching: true
version: 20.10.24
- attach_workspace:
at: /tmp/workspace
- run:
Expand Down Expand Up @@ -641,7 +639,6 @@ commands:
- checkout
- setup_remote_docker:
docker_layer_caching: true
version: 20.10.24
- run:
name: Build the Docker image
command: |
Expand Down
15 changes: 6 additions & 9 deletions .circleci/config.yml.j2
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ parameters:
jobs:
trivy-scan-docker:
docker:
- image: docker:{{ remote_docker_version }}-git
- image: cimg/base:stable
shell: /bin/sh -leo pipefail
parameters:
docker_image:
Expand All @@ -45,7 +45,7 @@ jobs:
ASTRO_SEC_ENDPOINT: << parameters.report_url >>
steps:
- setup_remote_docker:
version: {{ remote_docker_version }}
docker_layer_caching: true
- checkout
- run:
name: Pull Docker image
Expand Down Expand Up @@ -84,14 +84,14 @@ jobs:

twistcli-scan-docker:
docker:
- image: docker:{{ remote_docker_version }}-git
- image: cimg/base:stable
shell: /bin/sh -leo pipefail
parameters:
docker_image:
type: string
steps:
- setup_remote_docker:
version: {{ remote_docker_version }}
docker_layer_caching: true
- run:
name: Pull Docker image
command: docker pull << parameters.docker_image >>
Expand Down Expand Up @@ -149,7 +149,6 @@ jobs:
steps:
- setup_remote_docker:
docker_layer_caching: true
version: {{ remote_docker_version }}
- checkout
- run:
name: Create chart-tests-cache-key.txt
Expand Down Expand Up @@ -268,11 +267,11 @@ jobs:

check-commander-airflow-version:
docker:
- image: cimg/base:2022.09
- image: cimg/base:stable
steps:
- checkout
- setup_remote_docker:
version: {{ remote_docker_version }}
docker_layer_caching: true
- run:
name: Check that commander image uses same Airflow chart version
command: make validate-commander-airflow-version
Expand Down Expand Up @@ -448,7 +447,6 @@ commands:
steps:
- setup_remote_docker:
docker_layer_caching: true
version: {{ remote_docker_version }}
- attach_workspace:
at: /tmp/workspace
- run:
Expand Down Expand Up @@ -502,7 +500,6 @@ commands:
- checkout
- setup_remote_docker:
docker_layer_caching: true
version: {{ remote_docker_version }}
- run:
name: Build the Docker image
command: |
Expand Down
6 changes: 1 addition & 5 deletions .circleci/generate_circleci_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,16 +11,13 @@
metadata = yaml.safe_load((Path(__file__).parents[1] / "metadata.yaml").read_text())
kube_versions = metadata["test_k8s_versions"]

# https://circleci.com/docs/2.0/building-docker-images/#docker-version
ci_remote_docker_version = "20.10.24"

# https://circleci.com/developer/machine/image/ubuntu-2204
machine_image_version = "ubuntu-2204:2023.07.2"
ci_runner_version = "2023-11"


def list_docker_images(path):
command = f"cd {path} && helm template . -f tests/enable_all_features.yaml 2>/dev/null | awk '/image: / {{print $2}}' | sed 's/\"//g' | sort -u"
command = f"cd {path} && helm template . --set forceIncompatibleKubernetes=true -f tests/enable_all_features.yaml 2>/dev/null | awk '/image: / {{print $2}}' | sed 's/\"//g' | sort -u"
docker_images_output = subprocess.check_output(command, shell=True)
docker_image_list = docker_images_output.decode("utf-8").strip().split("\n")

Expand All @@ -42,7 +39,6 @@ def main():
kube_versions=kube_versions,
docker_images=docker_images,
machine_image_version=machine_image_version,
remote_docker_version=ci_remote_docker_version,
ci_runner_version=ci_runner_version,
)
with open(config_path, "w") as circle_ci_config_file:
Expand Down
6 changes: 6 additions & 0 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,12 @@ exclude: '(venv|\.vscode|tests/k8s_schema)' # regex
repos:
- repo: local
hooks:
- id: verify-image-tags
name: Verify image tags
entry: python3
args: [bin/verify_image_tags.py]
language: system
files: ^.*\.(tpl|yaml)$
- id: circle-config-yaml
name: Checks for consistency between config.yml and config.yml.j2
language: python
Expand Down
2 changes: 2 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -77,13 +77,15 @@ update-requirements: ## Update all requirements.txt files
show-docker-images: ## Show all docker images and versions used in the helm chart
@helm template . \
-f tests/enable_all_features.yaml \
--set forceIncompatibleKubernetes=true \
2>/dev/null \
| gawk '/image: / {match($$2, /(([^"]*):[^"]*)/, a) ; printf "https://%s %s\n", a[2], a[1] ;}' | sort -u | column -t

.PHONY: show-docker-images-with-private-registry
show-docker-images-with-private-registry: ## Show all docker images and versions used in the helm chart with a privateRegistry set
@helm template . \
-f tests/enable_all_features.yaml \
--set forceIncompatibleKubernetes=true \
--set global.privateRegistry.enabled=True \
--set global.privateRegistry.repository=example.com/the-private-registry \
2>/dev/null \
Expand Down
51 changes: 51 additions & 0 deletions bin/verify_image_tags.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
#!/usr/bin/env python
"""Verify that every image has only one tag version."""

import subprocess
from pathlib import Path


GIT_ROOT = next(
iter([x for x in Path(__file__).resolve().parents if (x / ".git").is_dir()]), None
)
command = "helm template . --set forceIncompatibleKubernetes=true -f tests/enable_all_features.yaml | grep -o 'quay.io/astronomer[^\"]*' | sort -u"

result = subprocess.run(
command,
shell=True,
cwd=GIT_ROOT,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)

if result.returncode != 0:
print(f"Error running the command:\n{result.stderr}")
raise SystemExit(1)

output_lines = result.stdout.strip().split("\n")

image_dict = {}
images_with_multiple_tags = []

for line in output_lines:
if ":" not in line:
continue

image = line.strip()
parts = image.rsplit(":", 1)
image_name = parts[0]
tag = parts[1].strip(" \"'\t\n\r")

if image_name not in image_dict:
image_dict[image_name] = []

if tag and tag not in image_dict[image_name]:
image_dict[image_name].append(tag)

for image, tags in image_dict.items():
if len(tags) > 1:
print(f"ERROR: image {image} has multiple tags: {', '.join(tags)}")

if any(len(tags) > 1 for tags in image_dict.values()):
raise SystemExit(1)

0 comments on commit d67a3e9

Please sign in to comment.