From 80843e0f11c5a35f846635002ad0656535947557 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Mon, 24 Nov 2025 14:54:46 +0100 Subject: [PATCH 1/5] annotate --- Makefile | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Makefile b/Makefile index 582cf5c2..63f2e005 100644 --- a/Makefile +++ b/Makefile @@ -23,16 +23,21 @@ HELM_CHART_ARTIFACT := target/helm/${OPERATOR_NAME}-${VERSION}.tgz SHELL=/usr/bin/env bash -euo pipefail +# used in build.yml render-readme: scripts/render_readme.sh +# run by developer when changing doc templates like getting_started render-docs: scripts/docs_templating.sh ## Docker related targets +# called by the "docker" target docker-build: docker build --force-rm --build-arg VERSION=${VERSION} -t "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}:${VERSION}-${ARCH}" -f docker/Dockerfile . +# called by the "docker" target +# called by build.yml docker-publish: # Push to Harbor # We need to use "value" here to prevent the variable from being recursively expanded by make (username contains a dollar sign, since it's a Harbor bot) @@ -61,9 +66,11 @@ docker-publish: cosign attest -y --predicate sbom.merged.json --type cyclonedx "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}@$$REPO_DIGEST_OF_IMAGE" # This assumes "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}:${VERSION}-amd64 and "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}:${VERSION}-arm64 are built and pushed +# called by build.yml - TODO use the reusable action docker-manifest-list-build: docker manifest create "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}:${VERSION}" --amend "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}:${VERSION}-amd64" --amend "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}:${VERSION}-arm64" +# called by build.yml - TODO use the reusable action docker-manifest-list-publish: # Push to Harbor # We need to use "value" here to prevent the variable from being recursively expanded by make (username contains a dollar sign, since it's a Harbor bot) @@ -75,11 +82,17 @@ docker-manifest-list-publish: cosign sign -y "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}:${VERSION}@$$DIGEST_HARBOR" # TODO remove if not used/needed +# Remove this, covered by CI docker: docker-build docker-publish +# used in build.yml +# Not needed print-docker-tag: @echo "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}:${VERSION}" +# used by publish +# used in build.yml +# Not needed helm-publish: # Push to Harbor # We need to use "value" here to prevent the variable from being recursively expanded by make (username contains a dollar sign, since it's a Harbor bot) @@ -97,6 +110,7 @@ helm-publish: # Uses the keyless signing flow with Github Actions as identity provider\ cosign sign -y "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_CHARTS}/${HELM_CHART_NAME}@$$REPO_DIGEST_OF_ARTIFACT" +# TODO: make a reusable action for this helm-package: mkdir -p target/helm && helm package --destination target/helm deploy/helm/${OPERATOR_NAME} @@ -117,6 +131,7 @@ config: cp -r deploy/config-spec/* "deploy/helm/${OPERATOR_NAME}/configs";\ fi +# Maxi uses this occasionally crds: mkdir -p deploy/helm/"${OPERATOR_NAME}"/crds cargo run --bin stackable-"${OPERATOR_NAME}" -- crd | yq eval '.metadata.annotations["helm.sh/resource-policy"]="keep"' - > "deploy/helm/${OPERATOR_NAME}/crds/crds.yaml" From 81406f0cdf07088bf23ce938d435567dc1ff512c Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Mon, 24 Nov 2025 15:03:05 +0100 Subject: [PATCH 2/5] remove ci stuff NOTE: This will be removed in operator-templating, and requires changes to the workflows that are currently being worked on. --- Makefile | 95 -------------------------------------------------------- 1 file changed, 95 deletions(-) diff --git a/Makefile b/Makefile index 63f2e005..00429e48 100644 --- a/Makefile +++ b/Makefile @@ -9,17 +9,11 @@ .PHONY: build publish -TAG := $(shell git rev-parse --short HEAD) OPERATOR_NAME := airflow-operator VERSION := $(shell cargo metadata --format-version 1 | jq -r '.packages[] | select(.name=="stackable-${OPERATOR_NAME}") | .version') -ARCH := $(shell uname -m | sed -e 's#x86_64#amd64#' | sed -e 's#aarch64#arm64#') OCI_REGISTRY_HOSTNAME := oci.stackable.tech OCI_REGISTRY_PROJECT_IMAGES := sdp -OCI_REGISTRY_PROJECT_CHARTS := sdp-charts -# This will be overwritten by an environmental variable if called from the github action -HELM_CHART_NAME := ${OPERATOR_NAME} -HELM_CHART_ARTIFACT := target/helm/${OPERATOR_NAME}-${VERSION}.tgz SHELL=/usr/bin/env bash -euo pipefail @@ -31,89 +25,6 @@ render-readme: render-docs: scripts/docs_templating.sh -## Docker related targets -# called by the "docker" target -docker-build: - docker build --force-rm --build-arg VERSION=${VERSION} -t "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}:${VERSION}-${ARCH}" -f docker/Dockerfile . - -# called by the "docker" target -# called by build.yml -docker-publish: - # Push to Harbor - # We need to use "value" here to prevent the variable from being recursively expanded by make (username contains a dollar sign, since it's a Harbor bot) - docker login --username '${value OCI_REGISTRY_SDP_USERNAME}' --password '${OCI_REGISTRY_SDP_PASSWORD}' '${OCI_REGISTRY_HOSTNAME}' - DOCKER_OUTPUT=$$(docker push --all-tags '${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}');\ - # Obtain the digest of the pushed image from the output of `docker push`, because signing by tag is deprecated and will be removed from cosign in the future\ - REPO_DIGEST_OF_IMAGE=$$(echo "$$DOCKER_OUTPUT" | awk '/^${VERSION}-${ARCH}: digest: sha256:[0-9a-f]{64} size: [0-9]+$$/ { print $$3 }');\ - if [ -z "$$REPO_DIGEST_OF_IMAGE" ]; then\ - echo 'Could not find repo digest for container image: ${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}:${VERSION}-${ARCH}';\ - exit 1;\ - fi;\ - # This generates a signature and publishes it to the registry, next to the image\ - # Uses the keyless signing flow with Github Actions as identity provider\ - cosign sign -y "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}@$$REPO_DIGEST_OF_IMAGE";\ - # Generate the SBOM for the operator image, this leverages the already generated SBOM for the operator binary by cargo-cyclonedx\ - syft scan --output cyclonedx-json@1.5=sbom.json --select-catalogers "-cargo-auditable-binary-cataloger,+sbom-cataloger" --scope all-layers --source-name "${OPERATOR_NAME}" --source-version "${VERSION}-${ARCH}" "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}@$$REPO_DIGEST_OF_IMAGE";\ - # Determine the PURL for the container image\ - URLENCODED_REPO_DIGEST_OF_IMAGE=$$(echo "$$REPO_DIGEST_OF_IMAGE" | sed 's/:/%3A/g');\ - PURL="pkg:oci/${OPERATOR_NAME}@$$URLENCODED_REPO_DIGEST_OF_IMAGE?arch=${ARCH}&repository_url=${OCI_REGISTRY_HOSTNAME}%2F${OCI_REGISTRY_PROJECT_IMAGES}%2F${OPERATOR_NAME}";\ - # Get metadata from the image\ - IMAGE_DESCRIPTION=$$(docker inspect --format='{{.Config.Labels.description}}' "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}:${VERSION}-${ARCH}");\ - IMAGE_NAME=$$(docker inspect --format='{{.Config.Labels.name}}' "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}:${VERSION}-${ARCH}");\ - # Merge the SBOM with the metadata for the operator\ - jq -s '{"metadata":{"component":{"description":"'"$$IMAGE_NAME. $$IMAGE_DESCRIPTION"'","supplier":{"name":"Stackable GmbH","url":["https://stackable.tech/"]},"author":"Stackable GmbH","purl":"'"$$PURL"'","publisher":"Stackable GmbH"}}} * .[0]' sbom.json > sbom.merged.json;\ - # Attest the SBOM to the image\ - cosign attest -y --predicate sbom.merged.json --type cyclonedx "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}@$$REPO_DIGEST_OF_IMAGE" - -# This assumes "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}:${VERSION}-amd64 and "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}:${VERSION}-arm64 are built and pushed -# called by build.yml - TODO use the reusable action -docker-manifest-list-build: - docker manifest create "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}:${VERSION}" --amend "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}:${VERSION}-amd64" --amend "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}:${VERSION}-arm64" - -# called by build.yml - TODO use the reusable action -docker-manifest-list-publish: - # Push to Harbor - # We need to use "value" here to prevent the variable from being recursively expanded by make (username contains a dollar sign, since it's a Harbor bot) - docker login --username '${value OCI_REGISTRY_SDP_USERNAME}' --password '${OCI_REGISTRY_SDP_PASSWORD}' '${OCI_REGISTRY_HOSTNAME}' - DIGEST_HARBOR=$$(docker manifest push "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}:${VERSION}");\ - # Refer to image via its digest (oci.stackable.tech/sdp/airflow@sha256:0a1b2c...);\ - # This generates a signature and publishes it to the registry, next to the image\ - # Uses the keyless signing flow with Github Actions as identity provider\ - cosign sign -y "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}:${VERSION}@$$DIGEST_HARBOR" - -# TODO remove if not used/needed -# Remove this, covered by CI -docker: docker-build docker-publish - -# used in build.yml -# Not needed -print-docker-tag: - @echo "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}:${VERSION}" - -# used by publish -# used in build.yml -# Not needed -helm-publish: - # Push to Harbor - # We need to use "value" here to prevent the variable from being recursively expanded by make (username contains a dollar sign, since it's a Harbor bot) - helm registry login --username '${value OCI_REGISTRY_SDP_CHARTS_USERNAME}' --password '${OCI_REGISTRY_SDP_CHARTS_PASSWORD}' '${OCI_REGISTRY_HOSTNAME}' - # Obtain the digest of the pushed artifact from the output of `helm push`, because signing by tag is deprecated and will be removed from cosign in the future\ - HELM_OUTPUT=$$(helm push '${HELM_CHART_ARTIFACT}' 'oci://${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_CHARTS}' 2>&1);\ - REPO_DIGEST_OF_ARTIFACT=$$(echo "$$HELM_OUTPUT" | awk '/^Digest: sha256:[0-9a-f]{64}$$/ { print $$2 }');\ - if [ -z "$$REPO_DIGEST_OF_ARTIFACT" ]; then\ - echo 'Could not find repo digest for helm chart: ${HELM_CHART_NAME}';\ - exit 1;\ - fi;\ - # Login to Harbor, needed for cosign to be able to push the signature for the Helm chart\ - docker login --username '${value OCI_REGISTRY_SDP_CHARTS_USERNAME}' --password '${OCI_REGISTRY_SDP_CHARTS_PASSWORD}' '${OCI_REGISTRY_HOSTNAME}';\ - # This generates a signature and publishes it to the registry, next to the chart artifact\ - # Uses the keyless signing flow with Github Actions as identity provider\ - cosign sign -y "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_CHARTS}/${HELM_CHART_NAME}@$$REPO_DIGEST_OF_ARTIFACT" - -# TODO: make a reusable action for this -helm-package: - mkdir -p target/helm && helm package --destination target/helm deploy/helm/${OPERATOR_NAME} - ## Chart related targets compile-chart: version crds config @@ -131,7 +42,6 @@ config: cp -r deploy/config-spec/* "deploy/helm/${OPERATOR_NAME}/configs";\ fi -# Maxi uses this occasionally crds: mkdir -p deploy/helm/"${OPERATOR_NAME}"/crds cargo run --bin stackable-"${OPERATOR_NAME}" -- crd | yq eval '.metadata.annotations["helm.sh/resource-policy"]="keep"' - > "deploy/helm/${OPERATOR_NAME}/crds/crds.yaml" @@ -150,11 +60,6 @@ regenerate-nix: build: regenerate-charts regenerate-nix helm-package docker-build -# This target is used by the CI -# It doesn't make use of any nix dependencies and thus aviods building the -# operator unnecessarily often. -build-ci: regenerate-charts helm-package docker-build - publish: docker-publish helm-publish check-nix: From 073948408001fb74cb61792441714bb06ff5e02b Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Mon, 24 Nov 2025 15:07:11 +0100 Subject: [PATCH 3/5] remove unneeded comments --- Makefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Makefile b/Makefile index 00429e48..d0e2fc02 100644 --- a/Makefile +++ b/Makefile @@ -17,11 +17,9 @@ OCI_REGISTRY_PROJECT_IMAGES := sdp SHELL=/usr/bin/env bash -euo pipefail -# used in build.yml render-readme: scripts/render_readme.sh -# run by developer when changing doc templates like getting_started render-docs: scripts/docs_templating.sh From 8bd3b691e5bd7c0a263726e5882d41c206b7d2ad Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Mon, 24 Nov 2025 16:10:20 +0100 Subject: [PATCH 4/5] restore docker-build --- Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d0e2fc02..0575a836 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,10 @@ render-readme: render-docs: scripts/docs_templating.sh +## Docker related targets +docker-build: + docker build --force-rm --build-arg VERSION=${VERSION} -t "${OCI_REGISTRY_HOSTNAME}/${OCI_REGISTRY_PROJECT_IMAGES}/${OPERATOR_NAME}:${VERSION}-${ARCH}" -f docker/Dockerfile . + ## Chart related targets compile-chart: version crds config @@ -56,7 +60,7 @@ regenerate-charts: chart-clean compile-chart regenerate-nix: nix run --extra-experimental-features "nix-command flakes" -f . regenerateNixLockfiles -build: regenerate-charts regenerate-nix helm-package docker-build +build: regenerate-charts regenerate-nix docker-build publish: docker-publish helm-publish From 192c7e46c67c5224f1e6db518365d0674b7ea5f7 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Mon, 24 Nov 2025 16:12:27 +0100 Subject: [PATCH 5/5] remove unusable targets --- Makefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Makefile b/Makefile index 0575a836..9a9864d9 100644 --- a/Makefile +++ b/Makefile @@ -62,8 +62,6 @@ regenerate-nix: build: regenerate-charts regenerate-nix docker-build -publish: docker-publish helm-publish - check-nix: @which nix || (echo "Error: 'nix' is not installed. Please install it to proceed."; exit 1)