From 802d06de8a0c4bf323f83270b09a0c4df6caf3d0 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 19 Sep 2024 19:47:40 +0200 Subject: [PATCH 01/43] (draft) Add new targets replacing the boilerplate ones --- Makefile | 28 ++++++++++++++++++++++++++-- build/Dockerfile | 5 +++-- build/golang.mk | 22 ++++++++++++++++++++++ build_tag.sh | 2 ++ jenkins/get_dependencies.sh | 23 +++++++++++++++++++++++ 5 files changed, 76 insertions(+), 4 deletions(-) create mode 100644 build/golang.mk create mode 100755 jenkins/get_dependencies.sh diff --git a/Makefile b/Makefile index a2d12342..5ca39608 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,22 @@ -GOLANGCI_OPTIONAL_CONFIG = .golangci.yml +IMAGE_REGISTRY ?= quay.io IMAGE_REPOSITORY ?= app-sre REGISTRY_USER = $(QUAY_USER) REGISTRY_TOKEN = $(QUAY_TOKEN) +CONTAINER_ENGINE = $(shell command -v podman 2>/dev/null || echo "docker") +CONTAINER_ENGINE_CONFIG_DIR = .docker + +OPERATOR_NAME = deployment-validation-operator +#OPERATOR_IMAGE_TAG ?= copy the catalog image hash +# OPERATOR_IMAGE_URI=$(IMAGE_REGISTRY)/$(IMAGE_REPOSITORY)/$(IMAGE_NAME):${OPERATOR_IMAGE_TAG} + +# Temporary hardcode for testing, DO NOT MERGE to master +OPERATOR_IMAGE_URI = quay.io/rh_ee_ijimeno/dvo +OPERATOR_IMAGE_TAG ?= dev + # This include must go below the above definitions -include boilerplate/generated-includes.mk +# include boilerplate/generated-includes.mk +include build/golang.mk OPERATOR_IMAGE_URI_TEST = $(IMAGE_REGISTRY)/$(IMAGE_REPOSITORY)/$(IMAGE_NAME):test @@ -24,3 +36,15 @@ e2e-test: # We are early adopters of the OPM build/push process. Remove this # override once boilerplate uses that path by default. build-push: opm-build-push ; + +.PHONY: quay-login +quay-login: + @echo "## Login to quay.io..." + mkdir -p ${CONTAINER_ENGINE_CONFIG_DIR} + @${CONTAINER_ENGINE} login -u="${REGISTRY_USER}" -p="${REGISTRY_TOKEN}" quay.io + +#${CONTAINER_ENGINE} tag $(OPERATOR_IMAGE_URI):test1 $(OPERATOR_IMAGE_URI):latest +.PHONY: docker-build +docker-build: + @echo "## Building the container image..." + ${CONTAINER_ENGINE} build --pull -f build/Dockerfile -t ${OPERATOR_IMAGE_URI}:${OPERATOR_IMAGE_TAG} . \ No newline at end of file diff --git a/build/Dockerfile b/build/Dockerfile index 8a370e3c..c7e9fc81 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -1,9 +1,10 @@ -FROM quay.io/app-sre/boilerplate:image-v5.0.1 AS builder +# FROM quay.io/app-sre/boilerplate:image-v5.0.1 AS builder +FROM registry.ci.openshift.org/openshift/release:rhel-8-release-golang-1.22-openshift-4.17 AS builder RUN mkdir -p /workdir COPY . /workdir WORKDIR /workdir -RUN make +RUN make go-build go-test FROM registry.access.redhat.com/ubi8/ubi-minimal:8.10-1052.1724178568 diff --git a/build/golang.mk b/build/golang.mk new file mode 100644 index 00000000..a5caa39d --- /dev/null +++ b/build/golang.mk @@ -0,0 +1,22 @@ +GOLANGCI_OPTIONAL_CONFIG = .golangci.yml +GOLANGCI_LINT_CACHE =/tmp/golangci-cache +.PHONY: go-lint +go-lint: + @echo "## Running the golangci-lint tool..." + jenkins/get_dependencies.sh + GOLANGCI_LINT_CACHE=${GOLANGCI_LINT_CACHE} golangci-lint run -c ${GOLANGCI_OPTIONAL_CONFIG} ./... + +TEST_TARGETS = $(shell ${GOENV} go list -e ./... | grep -E -v "/(vendor)/") +.PHONY: go-test +go-test: + @echo "## Running the code unit tests..." + ${GOENV} go test ${TEST_TARGETS} + +GOOS ?= linux +GOENV=GOOS=${GOOS} GOARCH=${GOARCH} CGO_ENABLED=1 +GOBUILDFLAGS=-gcflags="all=-trimpath=${GOPATH}" -asmflags="all=-trimpath=${GOPATH}" +.PHONY: go-build +go-build: + @echo "## Building the binary..." + go mod vendor + ${GOENV} go build ${GOBUILDFLAGS} -o build/_output/bin/$(OPERATOR_NAME) . diff --git a/build_tag.sh b/build_tag.sh index ea89514f..06470d88 100755 --- a/build_tag.sh +++ b/build_tag.sh @@ -1,5 +1,7 @@ #!/bin/bash +## This script is the entry point for the Jenkins job: deployment-validation-operator build tag +## It builds a new image and pushes it to the dv-operator repository on quay.io. make \ OPERATOR_IMAGE_TAG=$(echo $GIT_BRANCH|cut -d"/" -f3) \ IMAGE_REPOSITORY="deployment-validation-operator" \ diff --git a/jenkins/get_dependencies.sh b/jenkins/get_dependencies.sh new file mode 100755 index 00000000..bf25bc26 --- /dev/null +++ b/jenkins/get_dependencies.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -x +set -eo pipefail + +GOOS=$(go env GOOS) +GOPATH=$(go env GOPATH) +GOLANGCI_LINT_VERSION="1.59.1" + +if which golangci-lint ; then + exit +fi + +mkdir -p "${GOPATH}/bin" +echo "${PATH}" | grep -q "${GOPATH}/bin" + +IN_PATH=$? +if [ $IN_PATH != 0 ]; then + echo "${GOPATH}/bin not in $$PATH" + exit 1 +fi + +DOWNLOAD_URL="https://github.com/golangci/golangci-lint/releases/download/v${GOLANGCI_LINT_VERSION}/golangci-lint-${GOLANGCI_LINT_VERSION}-${GOOS}-amd64.tar.gz" +curl -sfL "${DOWNLOAD_URL}" | tar -C "${GOPATH}/bin" -zx --strip-components=1 "golangci-lint-${GOLANGCI_LINT_VERSION}-${GOOS}-amd64/golangci-lint" From c375f697b76c393134b90aa163e11a5af91dd20d Mon Sep 17 00:00:00 2001 From: Isaac Date: Fri, 20 Sep 2024 12:06:09 +0200 Subject: [PATCH 02/43] (draft) Fix some paths and added missing targets --- Makefile | 18 ++++++++++++++---- build/Dockerfile | 2 +- {jenkins => build}/get_dependencies.sh | 0 build/golang.mk | 2 +- build_tag.sh | 2 +- 5 files changed, 17 insertions(+), 7 deletions(-) rename {jenkins => build}/get_dependencies.sh (100%) diff --git a/Makefile b/Makefile index 5ca39608..ba0ac567 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ IMAGE_REGISTRY ?= quay.io IMAGE_REPOSITORY ?= app-sre -REGISTRY_USER = $(QUAY_USER) -REGISTRY_TOKEN = $(QUAY_TOKEN) +REGISTRY_USER ?= $(QUAY_USER) +REGISTRY_TOKEN ?= $(QUAY_TOKEN) CONTAINER_ENGINE = $(shell command -v podman 2>/dev/null || echo "docker") CONTAINER_ENGINE_CONFIG_DIR = .docker @@ -43,8 +43,18 @@ quay-login: mkdir -p ${CONTAINER_ENGINE_CONFIG_DIR} @${CONTAINER_ENGINE} login -u="${REGISTRY_USER}" -p="${REGISTRY_TOKEN}" quay.io -#${CONTAINER_ENGINE} tag $(OPERATOR_IMAGE_URI):test1 $(OPERATOR_IMAGE_URI):latest .PHONY: docker-build docker-build: @echo "## Building the container image..." - ${CONTAINER_ENGINE} build --pull -f build/Dockerfile -t ${OPERATOR_IMAGE_URI}:${OPERATOR_IMAGE_TAG} . \ No newline at end of file + ${CONTAINER_ENGINE} build --pull -f build/Dockerfile -t ${OPERATOR_IMAGE_URI}:${OPERATOR_IMAGE_TAG} . + ${CONTAINER_ENGINE} tag $(OPERATOR_IMAGE_URI):${OPERATOR_IMAGE_TAG} $(OPERATOR_IMAGE_URI):latest + +.PHONY: docker-push +docker-push: + @echo "## Pushing the container image..." + ${CONTAINER_ENGINE} push ${OPERATOR_IMAGE_URI}:${OPERATOR_IMAGE_TAG} + ${CONTAINER_ENGINE} push ${OPERATOR_IMAGE_URI}:latest + +## This target is run by build_tag.sh script, triggered by a Jenkins job +.PHONY: docker-publish +docker-publish: quay-login docker-build docker-push diff --git a/build/Dockerfile b/build/Dockerfile index c7e9fc81..a087da0f 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -4,7 +4,7 @@ FROM registry.ci.openshift.org/openshift/release:rhel-8-release-golang-1.22-open RUN mkdir -p /workdir COPY . /workdir WORKDIR /workdir -RUN make go-build go-test +RUN make go-build go-test go-lint FROM registry.access.redhat.com/ubi8/ubi-minimal:8.10-1052.1724178568 diff --git a/jenkins/get_dependencies.sh b/build/get_dependencies.sh similarity index 100% rename from jenkins/get_dependencies.sh rename to build/get_dependencies.sh diff --git a/build/golang.mk b/build/golang.mk index a5caa39d..fa6b22ab 100644 --- a/build/golang.mk +++ b/build/golang.mk @@ -3,7 +3,7 @@ GOLANGCI_LINT_CACHE =/tmp/golangci-cache .PHONY: go-lint go-lint: @echo "## Running the golangci-lint tool..." - jenkins/get_dependencies.sh + build/get_dependencies.sh GOLANGCI_LINT_CACHE=${GOLANGCI_LINT_CACHE} golangci-lint run -c ${GOLANGCI_OPTIONAL_CONFIG} ./... TEST_TARGETS = $(shell ${GOENV} go list -e ./... | grep -E -v "/(vendor)/") diff --git a/build_tag.sh b/build_tag.sh index 06470d88..ebb278d2 100755 --- a/build_tag.sh +++ b/build_tag.sh @@ -8,4 +8,4 @@ IMAGE_REPOSITORY="deployment-validation-operator" \ IMAGE_NAME="dv-operator" \ REGISTRY_USER=$(echo $QUAY_USER) \ REGISTRY_TOKEN=$(echo $QUAY_TOKEN) \ -docker-push +docker-publish From 7a57240f2aeaed9625085b1a0e2513098acfdc53 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 26 Sep 2024 19:32:19 +0200 Subject: [PATCH 03/43] (draft) Add generate bundle contents step --- Makefile | 27 ++++++++++++--- build/build_opm_catalog.sh | 67 ++++++++++++++++++++++++++++++++++++++ build/get_opm.sh | 17 ++++++++++ 3 files changed, 106 insertions(+), 5 deletions(-) create mode 100755 build/build_opm_catalog.sh create mode 100755 build/get_opm.sh diff --git a/Makefile b/Makefile index ba0ac567..35435149 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,21 @@ +OPERATOR_NAME = deployment-validation-operator + +REGISTRY_USER ?= ${QUAY_USER} +REGISTRY_TOKEN ?= ${QUAY_TOKEN} IMAGE_REGISTRY ?= quay.io IMAGE_REPOSITORY ?= app-sre -REGISTRY_USER ?= $(QUAY_USER) -REGISTRY_TOKEN ?= $(QUAY_TOKEN) - +IMAGE_NAME ?= ${OPERATOR_NAME} +OPERATOR_IMAGE = quay.io/rh_ee_ijimeno/dvo +## Overwritten for testing OPERATOR_IMAGE = ${IMAGE_REGISTRY}/${IMAGE_REPOSITORY}/${IMAGE_NAME} CONTAINER_ENGINE = $(shell command -v podman 2>/dev/null || echo "docker") CONTAINER_ENGINE_CONFIG_DIR = .docker -OPERATOR_NAME = deployment-validation-operator +VERSION_MAJOR ?= 0 +VERSION_MINOR ?= 1 +COMMIT_COUNT=$(shell git rev-list --count master) +CURRENT_COMMIT=$(shell git rev-parse --short=7 HEAD) +OPERATOR_VERSION=${VERSION_MAJOR}.${VERSION_MINOR}.${COMMIT_COUNT}-g${CURRENT_COMMIT} + #OPERATOR_IMAGE_TAG ?= copy the catalog image hash # OPERATOR_IMAGE_URI=$(IMAGE_REGISTRY)/$(IMAGE_REPOSITORY)/$(IMAGE_NAME):${OPERATOR_IMAGE_TAG} @@ -47,7 +56,7 @@ quay-login: docker-build: @echo "## Building the container image..." ${CONTAINER_ENGINE} build --pull -f build/Dockerfile -t ${OPERATOR_IMAGE_URI}:${OPERATOR_IMAGE_TAG} . - ${CONTAINER_ENGINE} tag $(OPERATOR_IMAGE_URI):${OPERATOR_IMAGE_TAG} $(OPERATOR_IMAGE_URI):latest + ${CONTAINER_ENGINE} tag ${OPERATOR_IMAGE_URI}:${OPERATOR_IMAGE_TAG} ${OPERATOR_IMAGE_URI}:latest .PHONY: docker-push docker-push: @@ -58,3 +67,11 @@ docker-push: ## This target is run by build_tag.sh script, triggered by a Jenkins job .PHONY: docker-publish docker-publish: quay-login docker-build docker-push + +.PHONY: test_opm +test_opm: + OPERATOR_NAME="${OPERATOR_NAME}" \ + OPERATOR_VERSION="${OPERATOR_VERSION}" \ + OPERATOR_IMAGE="${OPERATOR_IMAGE}" \ + OPERATOR_IMAGE_TAG="${OPERATOR_IMAGE_TAG}" \ + build/build_opm_catalog.sh \ No newline at end of file diff --git a/build/build_opm_catalog.sh b/build/build_opm_catalog.sh new file mode 100755 index 00000000..ef0ea7d3 --- /dev/null +++ b/build/build_opm_catalog.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash + +set -euo pipefail + +REPO_ROOT=$(git rev-parse --show-toplevel) +SCRIPT_BUNDLE_CONTENTS="$REPO_ROOT/hack/generate-operator-bundle-contents.py" + +OLM_BUNDLE_VERSIONS_REPO=${OLM_BUNDLE_VERSIONS_REPO:-gitlab.cee.redhat.com/service/saas-operator-versions.git} +OLM_BUNDLE_VERSIONS_REPO_BRANCH=${OLM_BUNDLE_VERSIONS_REPO_BRANCH:-master} + +function log() { + echo "$(date "+%Y-%m-%d %H:%M:%S") -- ${1}" +} + +# function clone_olm_bundle_versions_repo() { +# local saas_root_dir=${1} + +# local bundle_versions_repo_url +# if [[ -n "${APP_SRE_BOT_PUSH_TOKEN:-}" ]]; then +# log "Using APP_SRE_BOT_PUSH_TOKEN credentials to authenticate" +# bundle_versions_repo_url="https://app:${APP_SRE_BOT_PUSH_TOKEN}@$OLM_BUNDLE_VERSIONS_REPO" +# else +# bundle_versions_repo_url="https://$OLM_BUNDLE_VERSIONS_REPO" +# fi + +# log "Cloning $OLM_BUNDLE_VERSIONS_REPO into $saas_root_dir" +# git clone --branch "$OLM_BUNDLE_VERSIONS_REPO_BRANCH" "$bundle_versions_repo_url" "$saas_root_dir" +# } + + +function main() { + # guess the env vars + #log "Building $OPERATOR_NAME version $OPERATOR_VERSION" + + # research if this is worthy when we know all env vars we need + #check_required_environment || return 1 + if [[ ! -x "$SCRIPT_BUNDLE_CONTENTS" ]]; then + log "The script $SCRIPT_BUNDLE_CONTENTS cannot be run. Exiting." + return 1 + fi + + # check versioning with this awful function + # get_prev_operator_version + + log "Generating temporary folder to contain artifacts" + BASE_FOLDER=$(mktemp -d --suffix "-$(basename "$0")") + log " folder: $BASE_FOLDER" + + local DIR_BUNDLE=$(mktemp -d -p "$BASE_FOLDER" bundle.XXXX) + local DIR_MANIFESTS=$(mktemp -d -p "$DIR_BUNDLE" manifests.XXXX) + + log $OPERATOR_VERSION + + python3 -m venv .venv + source .venv/bin/activate + pip install pyyaml + $SCRIPT_BUNDLE_CONTENTS --name "$OPERATOR_NAME" \ + --current-version "$OPERATOR_VERSION" \ + --image "$OPERATOR_IMAGE" \ + --image-tag "$OPERATOR_IMAGE_TAG" \ + --output-dir "$DIR_MANIFESTS" \ + +} + +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main +fi \ No newline at end of file diff --git a/build/get_opm.sh b/build/get_opm.sh new file mode 100755 index 00000000..effd6afb --- /dev/null +++ b/build/get_opm.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -x +set -eo pipefail + +GOOS=$(go env GOOS) +OPM_VERSION="v1.23.2" + +if which opm ; then + exit +fi + +mkdir -p .opm/bin +cd .opm/bin + +DOWNLOAD_URL="https://github.com/operator-framework/operator-registry/releases/download/$OPM_VERSION/$GOOS-amd64-opm" +curl -sfL "${DOWNLOAD_URL}" -o opm +chmod +x opm From 02e189de57bcd90e709e076b66a8fde15a3986c4 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 26 Sep 2024 20:03:46 +0200 Subject: [PATCH 04/43] (prev) Add OLM bundle image creation --- Makefile | 16 +++++++++++++--- build/build_opm_catalog.sh | 22 ++++++++++++++++++++-- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 35435149..79128272 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ OPERATOR_NAME = deployment-validation-operator - +# Image repository vars REGISTRY_USER ?= ${QUAY_USER} REGISTRY_TOKEN ?= ${QUAY_TOKEN} IMAGE_REGISTRY ?= quay.io @@ -7,8 +7,9 @@ IMAGE_REPOSITORY ?= app-sre IMAGE_NAME ?= ${OPERATOR_NAME} OPERATOR_IMAGE = quay.io/rh_ee_ijimeno/dvo ## Overwritten for testing OPERATOR_IMAGE = ${IMAGE_REGISTRY}/${IMAGE_REPOSITORY}/${IMAGE_NAME} -CONTAINER_ENGINE = $(shell command -v podman 2>/dev/null || echo "docker") -CONTAINER_ENGINE_CONFIG_DIR = .docker +OLM_CHANNEL ?= alpha +OLM_BUNDLE_IMAGE = ${OPERATOR_IMAGE}-bundle +OLM_CATALOG_IMAGE = ${OPERATOR_IMAGE}-catalog VERSION_MAJOR ?= 0 VERSION_MINOR ?= 1 @@ -16,6 +17,9 @@ COMMIT_COUNT=$(shell git rev-list --count master) CURRENT_COMMIT=$(shell git rev-parse --short=7 HEAD) OPERATOR_VERSION=${VERSION_MAJOR}.${VERSION_MINOR}.${COMMIT_COUNT}-g${CURRENT_COMMIT} +CONTAINER_ENGINE = $(shell command -v podman 2>/dev/null || echo "docker") +CONTAINER_ENGINE_CONFIG_DIR = .docker + #OPERATOR_IMAGE_TAG ?= copy the catalog image hash # OPERATOR_IMAGE_URI=$(IMAGE_REGISTRY)/$(IMAGE_REPOSITORY)/$(IMAGE_NAME):${OPERATOR_IMAGE_TAG} @@ -70,6 +74,12 @@ docker-publish: quay-login docker-build docker-push .PHONY: test_opm test_opm: + build/get_opm.sh + CONTAINER_ENGINE="${CONTAINER_ENGINE}" \ + CURRENT_COMMIT="${CURRENT_COMMIT}" \ + OLM_BUNDLE_IMAGE="${OLM_BUNDLE_IMAGE}" \ + OLM_CATALOG_IMAGE="${OLM_CATALOG_IMAGE}" \ + OLM_CHANNEL="${OLM_CHANNEL}" \ OPERATOR_NAME="${OPERATOR_NAME}" \ OPERATOR_VERSION="${OPERATOR_VERSION}" \ OPERATOR_IMAGE="${OPERATOR_IMAGE}" \ diff --git a/build/build_opm_catalog.sh b/build/build_opm_catalog.sh index ef0ea7d3..ffbd3324 100755 --- a/build/build_opm_catalog.sh +++ b/build/build_opm_catalog.sh @@ -8,6 +8,10 @@ SCRIPT_BUNDLE_CONTENTS="$REPO_ROOT/hack/generate-operator-bundle-contents.py" OLM_BUNDLE_VERSIONS_REPO=${OLM_BUNDLE_VERSIONS_REPO:-gitlab.cee.redhat.com/service/saas-operator-versions.git} OLM_BUNDLE_VERSIONS_REPO_BRANCH=${OLM_BUNDLE_VERSIONS_REPO_BRANCH:-master} +OLM_BUNDLE_IMAGE_VERSION="${OLM_BUNDLE_IMAGE}:g${CURRENT_COMMIT}" +OLM_BUNDLE_IMAGE_LATEST="${OLM_BUNDLE_IMAGE}:latest" + + function log() { echo "$(date "+%Y-%m-%d %H:%M:%S") -- ${1}" } @@ -49,16 +53,30 @@ function main() { local DIR_BUNDLE=$(mktemp -d -p "$BASE_FOLDER" bundle.XXXX) local DIR_MANIFESTS=$(mktemp -d -p "$DIR_BUNDLE" manifests.XXXX) - log $OPERATOR_VERSION - + # move this to function python3 -m venv .venv source .venv/bin/activate pip install pyyaml + + log "Generating patched bundle contents" $SCRIPT_BUNDLE_CONTENTS --name "$OPERATOR_NAME" \ --current-version "$OPERATOR_VERSION" \ --image "$OPERATOR_IMAGE" \ --image-tag "$OPERATOR_IMAGE_TAG" \ --output-dir "$DIR_MANIFESTS" \ + # missing args from versioning + + log "Creating bundle image $OLM_BUNDLE_IMAGE_VERSION" + cd $DIR_BUNDLE + opm alpha bundle build --directory "$DIR_MANIFESTS" \ + --channels "$OLM_CHANNEL" \ + --default "$OLM_CHANNEL" \ + --package "$OPERATOR_NAME" \ + --tag "$OLM_BUNDLE_IMAGE_VERSION" \ + --image-builder $(basename "$CONTAINER_ENGINE" | awk '{print $1}') \ + --overwrite \ + 1>&2 + cd - } From e57f4230dadda6fa9b1ba1eb86d0f62c0bd67d28 Mon Sep 17 00:00:00 2001 From: Isaac Date: Fri, 27 Sep 2024 13:58:53 +0200 Subject: [PATCH 05/43] (prev) Add clone SRE versions repository --- build/build_opm_catalog.sh | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/build/build_opm_catalog.sh b/build/build_opm_catalog.sh index ffbd3324..0c274ab5 100755 --- a/build/build_opm_catalog.sh +++ b/build/build_opm_catalog.sh @@ -5,8 +5,7 @@ set -euo pipefail REPO_ROOT=$(git rev-parse --show-toplevel) SCRIPT_BUNDLE_CONTENTS="$REPO_ROOT/hack/generate-operator-bundle-contents.py" -OLM_BUNDLE_VERSIONS_REPO=${OLM_BUNDLE_VERSIONS_REPO:-gitlab.cee.redhat.com/service/saas-operator-versions.git} -OLM_BUNDLE_VERSIONS_REPO_BRANCH=${OLM_BUNDLE_VERSIONS_REPO_BRANCH:-master} +OLM_BUNDLE_VERSIONS_REPO="https://gitlab.cee.redhat.com/service/saas-operator-versions.git" OLM_BUNDLE_IMAGE_VERSION="${OLM_BUNDLE_IMAGE}:g${CURRENT_COMMIT}" OLM_BUNDLE_IMAGE_LATEST="${OLM_BUNDLE_IMAGE}:latest" @@ -16,21 +15,12 @@ function log() { echo "$(date "+%Y-%m-%d %H:%M:%S") -- ${1}" } -# function clone_olm_bundle_versions_repo() { -# local saas_root_dir=${1} - -# local bundle_versions_repo_url -# if [[ -n "${APP_SRE_BOT_PUSH_TOKEN:-}" ]]; then -# log "Using APP_SRE_BOT_PUSH_TOKEN credentials to authenticate" -# bundle_versions_repo_url="https://app:${APP_SRE_BOT_PUSH_TOKEN}@$OLM_BUNDLE_VERSIONS_REPO" -# else -# bundle_versions_repo_url="https://$OLM_BUNDLE_VERSIONS_REPO" -# fi - -# log "Cloning $OLM_BUNDLE_VERSIONS_REPO into $saas_root_dir" -# git clone --branch "$OLM_BUNDLE_VERSIONS_REPO_BRANCH" "$bundle_versions_repo_url" "$saas_root_dir" -# } - +function clone_versions_repo() { + log "Cloning $OLM_BUNDLE_VERSIONS_REPO" + local folder="$BASE_FOLDER/versions_repo" + git clone $OLM_BUNDLE_VERSIONS_REPO $folder --quiet + log " path: $folder" +} function main() { # guess the env vars @@ -48,11 +38,14 @@ function main() { log "Generating temporary folder to contain artifacts" BASE_FOLDER=$(mktemp -d --suffix "-$(basename "$0")") - log " folder: $BASE_FOLDER" + log " path: $BASE_FOLDER" local DIR_BUNDLE=$(mktemp -d -p "$BASE_FOLDER" bundle.XXXX) local DIR_MANIFESTS=$(mktemp -d -p "$DIR_BUNDLE" manifests.XXXX) + + clone_versions_repo + # move this to function python3 -m venv .venv source .venv/bin/activate @@ -81,5 +74,5 @@ function main() { } if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then - main + main fi \ No newline at end of file From 7b652956e2f4bc9ca9aeec36f190b32b6f6cb51d Mon Sep 17 00:00:00 2001 From: Isaac Date: Fri, 27 Sep 2024 17:01:50 +0200 Subject: [PATCH 06/43] (prev) Add previous operator version logic --- build/build_opm_catalog.sh | 41 +++++++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/build/build_opm_catalog.sh b/build/build_opm_catalog.sh index 0c274ab5..467ba6d2 100755 --- a/build/build_opm_catalog.sh +++ b/build/build_opm_catalog.sh @@ -4,8 +4,14 @@ set -euo pipefail REPO_ROOT=$(git rev-parse --show-toplevel) SCRIPT_BUNDLE_CONTENTS="$REPO_ROOT/hack/generate-operator-bundle-contents.py" +BASE_FOLDER="" +DIR_BUNDLE="" +DIR_MANIFESTS="" OLM_BUNDLE_VERSIONS_REPO="https://gitlab.cee.redhat.com/service/saas-operator-versions.git" +OLM_BUNDLE_VERSIONS_REPO_FOLDER="versions_repo" +VERSIONS_FILE="deployment-validation-operator/deployment-validation-operator-versions.txt" +PREV_VERSION="" OLM_BUNDLE_IMAGE_VERSION="${OLM_BUNDLE_IMAGE}:g${CURRENT_COMMIT}" OLM_BUNDLE_IMAGE_LATEST="${OLM_BUNDLE_IMAGE}:latest" @@ -15,13 +21,32 @@ function log() { echo "$(date "+%Y-%m-%d %H:%M:%S") -- ${1}" } +function prepare_temporary_folders() { + log "Generating temporary folders to contain artifacts" + BASE_FOLDER=$(mktemp -d --suffix "-$(basename "$0")") + DIR_BUNDLE=$(mktemp -d -p "$BASE_FOLDER" bundle.XXXX) + DIR_MANIFESTS=$(mktemp -d -p "$DIR_BUNDLE" manifests.XXXX) + log " base path: $BASE_FOLDER" +} + function clone_versions_repo() { log "Cloning $OLM_BUNDLE_VERSIONS_REPO" - local folder="$BASE_FOLDER/versions_repo" + local folder="$BASE_FOLDER/$OLM_BUNDLE_VERSIONS_REPO_FOLDER" git clone $OLM_BUNDLE_VERSIONS_REPO $folder --quiet log " path: $folder" } +function set_previous_operator_version() { + log "Determining previous operator version checking $VERSIONS_FILE file" + local filename="$BASE_FOLDER/$OLM_BUNDLE_VERSIONS_REPO_FOLDER/$VERSIONS_FILE" + if [[ ! -a "$filename" ]]; then + log "No file $VERSIONS_FILE exist. Exiting." + exit 1 + fi + PREV_VERSION=$(tail -n 1 "$filename" | awk '{print $1}') + log " previous version: $PREV_VERSION" +} + function main() { # guess the env vars #log "Building $OPERATOR_NAME version $OPERATOR_VERSION" @@ -33,18 +58,10 @@ function main() { return 1 fi - # check versioning with this awful function - # get_prev_operator_version - - log "Generating temporary folder to contain artifacts" - BASE_FOLDER=$(mktemp -d --suffix "-$(basename "$0")") - log " path: $BASE_FOLDER" - - local DIR_BUNDLE=$(mktemp -d -p "$BASE_FOLDER" bundle.XXXX) - local DIR_MANIFESTS=$(mktemp -d -p "$DIR_BUNDLE" manifests.XXXX) - - + prepare_temporary_folders clone_versions_repo + set_previous_operator_version + # move this to function python3 -m venv .venv From d6a76fa16dffee306ac967f951684b3aa95c0151 Mon Sep 17 00:00:00 2001 From: Isaac Date: Fri, 27 Sep 2024 17:08:53 +0200 Subject: [PATCH 07/43] (prev) Add OLM bundle previous version replace field --- build/build_opm_catalog.sh | 43 +++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/build/build_opm_catalog.sh b/build/build_opm_catalog.sh index 467ba6d2..86b5834a 100755 --- a/build/build_opm_catalog.sh +++ b/build/build_opm_catalog.sh @@ -47,26 +47,9 @@ function set_previous_operator_version() { log " previous version: $PREV_VERSION" } -function main() { - # guess the env vars - #log "Building $OPERATOR_NAME version $OPERATOR_VERSION" - - # research if this is worthy when we know all env vars we need - #check_required_environment || return 1 - if [[ ! -x "$SCRIPT_BUNDLE_CONTENTS" ]]; then - log "The script $SCRIPT_BUNDLE_CONTENTS cannot be run. Exiting." - return 1 - fi - - prepare_temporary_folders - clone_versions_repo - set_previous_operator_version - - - # move this to function - python3 -m venv .venv - source .venv/bin/activate - pip install pyyaml +function build_opm_bundle() { + # set venv with needed dependencies + python3 -m venv .venv; source .venv/bin/activate; pip install pyyaml log "Generating patched bundle contents" $SCRIPT_BUNDLE_CONTENTS --name "$OPERATOR_NAME" \ @@ -74,7 +57,7 @@ function main() { --image "$OPERATOR_IMAGE" \ --image-tag "$OPERATOR_IMAGE_TAG" \ --output-dir "$DIR_MANIFESTS" \ - # missing args from versioning + --replaces "$PREV_VERSION" log "Creating bundle image $OLM_BUNDLE_IMAGE_VERSION" cd $DIR_BUNDLE @@ -87,6 +70,24 @@ function main() { --overwrite \ 1>&2 cd - +} + +function main() { + # guess the env vars + #log "Building $OPERATOR_NAME version $OPERATOR_VERSION" + + # research if this is worthy when we know all env vars we need + #check_required_environment || return 1 + if [[ ! -x "$SCRIPT_BUNDLE_CONTENTS" ]]; then + log "The script $SCRIPT_BUNDLE_CONTENTS cannot be run. Exiting." + return 1 + fi + + prepare_temporary_folders + clone_versions_repo + set_previous_operator_version + build_opm_bundle + } From 6be0293443a630a575f43b241b5d67eb4e1d0072 Mon Sep 17 00:00:00 2001 From: Isaac Date: Fri, 27 Sep 2024 18:37:35 +0200 Subject: [PATCH 08/43] (prev) Add OLM bundle validation --- build/build_opm_catalog.sh | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/build/build_opm_catalog.sh b/build/build_opm_catalog.sh index 86b5834a..427e206f 100755 --- a/build/build_opm_catalog.sh +++ b/build/build_opm_catalog.sh @@ -16,6 +16,7 @@ PREV_VERSION="" OLM_BUNDLE_IMAGE_VERSION="${OLM_BUNDLE_IMAGE}:g${CURRENT_COMMIT}" OLM_BUNDLE_IMAGE_LATEST="${OLM_BUNDLE_IMAGE}:latest" +OLM_CATALOG_IMAGE_LATEST="${OLM_CATALOG_IMAGE}:latest" function log() { echo "$(date "+%Y-%m-%d %H:%M:%S") -- ${1}" @@ -72,6 +73,15 @@ function build_opm_bundle() { cd - } +function validate_opm_bundle() { + log "Pushing bundle image $OLM_BUNDLE_IMAGE_VERSION" + $CONTAINER_ENGINE push "$OLM_BUNDLE_IMAGE_VERSION" + + log "Validating bundle $OLM_BUNDLE_IMAGE_VERSION" + opm alpha bundle validate --tag "$OLM_BUNDLE_IMAGE_VERSION" \ + --image-builder $(basename "$CONTAINER_ENGINE" | awk '{print $1}') +} + function main() { # guess the env vars #log "Building $OPERATOR_NAME version $OPERATOR_VERSION" @@ -86,9 +96,12 @@ function main() { prepare_temporary_folders clone_versions_repo set_previous_operator_version - build_opm_bundle + build_opm_bundle + validate_opm_bundle + log "Tagging bundle image $OLM_BUNDLE_IMAGE_VERSION as $OLM_BUNDLE_IMAGE_LATEST" + $CONTAINER_ENGINE tag "$OLM_BUNDLE_IMAGE_VERSION" "$OLM_BUNDLE_IMAGE_LATEST" } if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then From 6370a0613988c923e1970f74fb5545d46acd0557 Mon Sep 17 00:00:00 2001 From: Isaac Date: Mon, 30 Sep 2024 18:56:51 +0200 Subject: [PATCH 09/43] (prev) Add OLM catalog build and validation --- build/build_opm_catalog.sh | 50 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/build/build_opm_catalog.sh b/build/build_opm_catalog.sh index 427e206f..f6f49535 100755 --- a/build/build_opm_catalog.sh +++ b/build/build_opm_catalog.sh @@ -16,6 +16,7 @@ PREV_VERSION="" OLM_BUNDLE_IMAGE_VERSION="${OLM_BUNDLE_IMAGE}:g${CURRENT_COMMIT}" OLM_BUNDLE_IMAGE_LATEST="${OLM_BUNDLE_IMAGE}:latest" +OLM_CATALOG_IMAGE_VERSION="${OLM_CATALOG_IMAGE}:${CURRENT_COMMIT}" OLM_CATALOG_IMAGE_LATEST="${OLM_CATALOG_IMAGE}:latest" function log() { @@ -82,9 +83,51 @@ function validate_opm_bundle() { --image-builder $(basename "$CONTAINER_ENGINE" | awk '{print $1}') } +function build_opm_catalog() { + local FROM_INDEX="" + local PREV_COMMIT=${PREV_VERSION#*g} # remove versioning and the g commit hash prefix + # check if the previous catalog image is available + if [ $(${CONTAINER_ENGINE} pull ${OLM_CATALOG_IMAGE}:${PREV_COMMIT} &> /dev/null; echo $?) -eq 0 ]; then + FROM_INDEX="--from-index ${OLM_CATALOG_IMAGE}:${PREV_COMMIT}" + log "Index argument is $FROM_INDEX" + fi + + log "Creating catalog image $OLM_CATALOG_IMAGE_VERSION using opm" + + opm index add --bundles "$OLM_BUNDLE_IMAGE_VERSION" \ + --tag "$OLM_CATALOG_IMAGE_VERSION" \ + --build-tool $(basename "$CONTAINER_ENGINE" | awk '{print $1}') \ + $FROM_INDEX +} + +function validate_opm_catalog() { + log "Checking that catalog we have built returns the correct version $OPERATOR_VERSION" + + local FREE_PORT=$(python3 -c 'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()') + + log "Running $OLM_CATALOG_IMAGE_VERSION and exposing $FREE_PORT" + local CONTAINER_ID=$(${CONTAINER_ENGINE} run -d -p "$FREE_PORT:50051" "$OLM_CATALOG_IMAGE_VERSION") + + log "Getting current version from running catalog" + local CATALOG_CURRENT_VERSION=$( + grpcurl -plaintext -d '{"name": "'"$OPERATOR_NAME"'"}' \ + "localhost:$FREE_PORT" api.Registry/GetPackage | \ + jq -r '.channels[] | select(.name=="'"$OLM_CHANNEL"'") | .csvName' | \ + sed "s/$OPERATOR_NAME\.//" + ) + log " catalog version: $CATALOG_CURRENT_VERSION" + + log "Removing docker container $CONTAINER_ID" + ${CONTAINER_ENGINE} rm -f "$CONTAINER_ID" + + if [[ "$CATALOG_CURRENT_VERSION" != "v$OPERATOR_VERSION" ]]; then + log "Version from catalog $CATALOG_CURRENT_VERSION != v$OPERATOR_VERSION" + return 1 + fi +} + function main() { - # guess the env vars - #log "Building $OPERATOR_NAME version $OPERATOR_VERSION" + log "Building $OPERATOR_NAME version $OPERATOR_VERSION" # research if this is worthy when we know all env vars we need #check_required_environment || return 1 @@ -100,6 +143,9 @@ function main() { build_opm_bundle validate_opm_bundle + build_opm_catalog + validate_opm_catalog + log "Tagging bundle image $OLM_BUNDLE_IMAGE_VERSION as $OLM_BUNDLE_IMAGE_LATEST" $CONTAINER_ENGINE tag "$OLM_BUNDLE_IMAGE_VERSION" "$OLM_BUNDLE_IMAGE_LATEST" } From e659682ee0d7dfdd6e8aa1903b0e9f2fc6fe1353 Mon Sep 17 00:00:00 2001 From: Isaac Date: Mon, 30 Sep 2024 18:57:17 +0200 Subject: [PATCH 10/43] (prev) Fix existing script to retrieve grpcurl --- build/get_opm.sh | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/build/get_opm.sh b/build/get_opm.sh index effd6afb..246ae544 100755 --- a/build/get_opm.sh +++ b/build/get_opm.sh @@ -4,14 +4,30 @@ set -eo pipefail GOOS=$(go env GOOS) OPM_VERSION="v1.23.2" +GRPCURL_VERSION="1.7.0" -if which opm ; then - exit +if ! which opm ; then + mkdir -p .opm/bin + cd .opm/bin + + DOWNLOAD_URL="https://github.com/operator-framework/operator-registry/releases/download/$OPM_VERSION/$GOOS-amd64-opm" + curl -sfL "${DOWNLOAD_URL}" -o opm + chmod +x opm + + cd - + ln -s .opm/bin/opm fi -mkdir -p .opm/bin -cd .opm/bin +if ! which grpcurl; then + mkdir -p .grpcurl/bin + cd .grpcurl/bin + + DOWNLOAD_URL="https://github.com/fullstorydev/grpcurl/releases/download/v$GRPCURL_VERSION/grpcurl_${GRPCURL_VERSION}_${GOOS}_x86_64.tar.gz" + curl -sfL "$DOWNLOAD_URL" | tar -xzf - -O grpcurl > grpcurl + chmod +x grpcurl + + cd - + ln -s .grpcurl/bin/grpcurl +fi -DOWNLOAD_URL="https://github.com/operator-framework/operator-registry/releases/download/$OPM_VERSION/$GOOS-amd64-opm" -curl -sfL "${DOWNLOAD_URL}" -o opm -chmod +x opm +## TODO: This will probably require rework with the path to the executables, locally required some workaround. Revisit after running in the Job \ No newline at end of file From 1ed01942c55cbc378e9bfca5e536a3bf3e70c860 Mon Sep 17 00:00:00 2001 From: Isaac Date: Mon, 30 Sep 2024 18:57:43 +0200 Subject: [PATCH 11/43] (prev) Fix Dockerfile builder source image --- build/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/build/Dockerfile b/build/Dockerfile index a087da0f..6beea22a 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -1,4 +1,3 @@ -# FROM quay.io/app-sre/boilerplate:image-v5.0.1 AS builder FROM registry.ci.openshift.org/openshift/release:rhel-8-release-golang-1.22-openshift-4.17 AS builder RUN mkdir -p /workdir From 2d34fa4530262229894780ba552c82b991df2b45 Mon Sep 17 00:00:00 2001 From: Isaac Date: Tue, 1 Oct 2024 18:18:17 +0200 Subject: [PATCH 12/43] (prev) Fixing versions repo accesses and writing --- build/build_opm_catalog.sh | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/build/build_opm_catalog.sh b/build/build_opm_catalog.sh index f6f49535..9a39929e 100755 --- a/build/build_opm_catalog.sh +++ b/build/build_opm_catalog.sh @@ -8,7 +8,7 @@ BASE_FOLDER="" DIR_BUNDLE="" DIR_MANIFESTS="" -OLM_BUNDLE_VERSIONS_REPO="https://gitlab.cee.redhat.com/service/saas-operator-versions.git" +OLM_BUNDLE_VERSIONS_REPO="gitlab.cee.redhat.com/ijimeno/saas-operator-versions.git" OLM_BUNDLE_VERSIONS_REPO_FOLDER="versions_repo" VERSIONS_FILE="deployment-validation-operator/deployment-validation-operator-versions.txt" PREV_VERSION="" @@ -32,9 +32,15 @@ function prepare_temporary_folders() { } function clone_versions_repo() { + local bundle_versions_repo_url log "Cloning $OLM_BUNDLE_VERSIONS_REPO" local folder="$BASE_FOLDER/$OLM_BUNDLE_VERSIONS_REPO_FOLDER" - git clone $OLM_BUNDLE_VERSIONS_REPO $folder --quiet + if [[ -n "${APP_SRE_BOT_PUSH_TOKEN:-}" ]]; then + log "Using APP_SRE_BOT_PUSH_TOKEN credentials to authenticate" + git clone "https://app:${APP_SRE_BOT_PUSH_TOKEN}@$OLM_BUNDLE_VERSIONS_REPO" $folder --quiet + else + git clone "https://$OLM_BUNDLE_VERSIONS_REPO" $folder --quiet + fi log " path: $folder" } @@ -59,7 +65,7 @@ function build_opm_bundle() { --image "$OPERATOR_IMAGE" \ --image-tag "$OPERATOR_IMAGE_TAG" \ --output-dir "$DIR_MANIFESTS" \ - --replaces "$PREV_VERSION" + #--replaces "$PREV_VERSION" log "Creating bundle image $OLM_BUNDLE_IMAGE_VERSION" cd $DIR_BUNDLE @@ -146,6 +152,12 @@ function main() { build_opm_catalog validate_opm_catalog + if [[ -n "${APP_SRE_BOT_PUSH_TOKEN:-}" ]]; then + update_versions_repo + else + log "APP_SRE_BOT_PUSH_TOKEN credentials were not found" + log "it will be necessary to manually update $OLM_BUNDLE_VERSIONS_REPO repo" + fi log "Tagging bundle image $OLM_BUNDLE_IMAGE_VERSION as $OLM_BUNDLE_IMAGE_LATEST" $CONTAINER_ENGINE tag "$OLM_BUNDLE_IMAGE_VERSION" "$OLM_BUNDLE_IMAGE_LATEST" } From 9522371d099f0fa4d2f9822aa31e4e5dadfd3460 Mon Sep 17 00:00:00 2001 From: Isaac Date: Tue, 1 Oct 2024 18:18:40 +0200 Subject: [PATCH 13/43] (prev) Add last steps of tagging and pushing built images --- build/build_opm_catalog.sh | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/build/build_opm_catalog.sh b/build/build_opm_catalog.sh index 9a39929e..c3e554bf 100755 --- a/build/build_opm_catalog.sh +++ b/build/build_opm_catalog.sh @@ -132,6 +132,41 @@ function validate_opm_catalog() { fi } +function update_versions_repo() { + log "Adding the current version $OPERATOR_VERSION to the bundle versions file in $OLM_BUNDLE_VERSIONS_REPO" + local folder="$BASE_FOLDER/$OLM_BUNDLE_VERSIONS_REPO_FOLDER" + + cd $folder + + echo "$OPERATOR_VERSION" >> "$VERSIONS_FILE" + git add . + message="add version $OPERATOR_VERSION + + replaces $PREV_VERSION" + git commit -m "$message" + + log "Pushing the repository changes to $OLM_BUNDLE_VERSIONS_REPO into master branch" + git push origin master + cd - +} + +function tag_and_push_images() { + log "Tagging bundle image $OLM_BUNDLE_IMAGE_VERSION as $OLM_BUNDLE_IMAGE_LATEST" + ${CONTAINER_ENGINE} tag "$OLM_BUNDLE_IMAGE_VERSION" "$OLM_BUNDLE_IMAGE_LATEST" + + log "Tagging catalog image $OLM_CATALOG_IMAGE_VERSION as $OLM_CATALOG_IMAGE_LATEST" + ${CONTAINER_ENGINE} tag "$OLM_CATALOG_IMAGE_VERSION" "$OLM_CATALOG_IMAGE_LATEST" + + log "Pushing catalog image $OLM_CATALOG_IMAGE_VERSION" + ${CONTAINER_ENGINE} push "$OLM_CATALOG_IMAGE_VERSION" + + log "Pushing bundle image $OLM_CATALOG_IMAGE_LATEST" + ${CONTAINER_ENGINE} push "$OLM_CATALOG_IMAGE_LATEST" + + log "Pushing bundle image $OLM_BUNDLE_IMAGE_LATEST" + ${CONTAINER_ENGINE} push "$OLM_BUNDLE_IMAGE_LATEST" +} + function main() { log "Building $OPERATOR_NAME version $OPERATOR_VERSION" @@ -158,8 +193,7 @@ function main() { log "APP_SRE_BOT_PUSH_TOKEN credentials were not found" log "it will be necessary to manually update $OLM_BUNDLE_VERSIONS_REPO repo" fi - log "Tagging bundle image $OLM_BUNDLE_IMAGE_VERSION as $OLM_BUNDLE_IMAGE_LATEST" - $CONTAINER_ENGINE tag "$OLM_BUNDLE_IMAGE_VERSION" "$OLM_BUNDLE_IMAGE_LATEST" + tag_and_push_images } if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then From ee88ad75446e8f140040a741102661b5317f4d89 Mon Sep 17 00:00:00 2001 From: Isaac Date: Tue, 1 Oct 2024 18:19:12 +0200 Subject: [PATCH 14/43] (prev) Add dev credentials for Jenkins tests --- Makefile | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 79128272..d4b87c9e 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,14 @@ OPERATOR_NAME = deployment-validation-operator # Image repository vars -REGISTRY_USER ?= ${QUAY_USER} -REGISTRY_TOKEN ?= ${QUAY_TOKEN} +## Overwritten for testing REGISTRY_USER ?= ${QUAY_USER} +REGISTRY_USER = rh_ee_ijimeno+dvojenkins01 +## Overwritten for testing REGISTRY_TOKEN ?= ${QUAY_TOKEN} +REGISTRY_TOKEN = 61BOGU7XW2AKL15TI3UR56YPX7BG73TUGYYBLPQ55POR70J0L5KR4J15SEH108DG IMAGE_REGISTRY ?= quay.io -IMAGE_REPOSITORY ?= app-sre -IMAGE_NAME ?= ${OPERATOR_NAME} +## Overwritten for testing IMAGE_REPOSITORY ?= app-sre +IMAGE_REPOSITORY ?= rh_ee_ijimeno +## Overwritten for testing IMAGE_NAME ?= ${OPERATOR_NAME} +IMAGE_NAME ?= dvo OPERATOR_IMAGE = quay.io/rh_ee_ijimeno/dvo ## Overwritten for testing OPERATOR_IMAGE = ${IMAGE_REGISTRY}/${IMAGE_REPOSITORY}/${IMAGE_NAME} OLM_CHANNEL ?= alpha @@ -16,16 +20,12 @@ VERSION_MINOR ?= 1 COMMIT_COUNT=$(shell git rev-list --count master) CURRENT_COMMIT=$(shell git rev-parse --short=7 HEAD) OPERATOR_VERSION=${VERSION_MAJOR}.${VERSION_MINOR}.${COMMIT_COUNT}-g${CURRENT_COMMIT} +OPERATOR_IMAGE_TAG ?= ${OPERATOR_VERSION} +OPERATOR_IMAGE_URI=$(IMAGE_REGISTRY)/$(IMAGE_REPOSITORY)/$(IMAGE_NAME):${OPERATOR_IMAGE_TAG} CONTAINER_ENGINE = $(shell command -v podman 2>/dev/null || echo "docker") CONTAINER_ENGINE_CONFIG_DIR = .docker -#OPERATOR_IMAGE_TAG ?= copy the catalog image hash -# OPERATOR_IMAGE_URI=$(IMAGE_REGISTRY)/$(IMAGE_REPOSITORY)/$(IMAGE_NAME):${OPERATOR_IMAGE_TAG} - -# Temporary hardcode for testing, DO NOT MERGE to master -OPERATOR_IMAGE_URI = quay.io/rh_ee_ijimeno/dvo -OPERATOR_IMAGE_TAG ?= dev # This include must go below the above definitions # include boilerplate/generated-includes.mk @@ -73,7 +73,7 @@ docker-push: docker-publish: quay-login docker-build docker-push .PHONY: test_opm -test_opm: +test_opm: quay-login build/get_opm.sh CONTAINER_ENGINE="${CONTAINER_ENGINE}" \ CURRENT_COMMIT="${CURRENT_COMMIT}" \ From c834e58066da30a66a76a47bfc95c90a7a6a37f9 Mon Sep 17 00:00:00 2001 From: Isaac Date: Tue, 1 Oct 2024 18:19:58 +0200 Subject: [PATCH 15/43] (prev) Temp fix to build_tag script for testing on Jenkins --- build_tag.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_tag.sh b/build_tag.sh index ebb278d2..3b3bab9c 100755 --- a/build_tag.sh +++ b/build_tag.sh @@ -8,4 +8,4 @@ IMAGE_REPOSITORY="deployment-validation-operator" \ IMAGE_NAME="dv-operator" \ REGISTRY_USER=$(echo $QUAY_USER) \ REGISTRY_TOKEN=$(echo $QUAY_TOKEN) \ -docker-publish +test_opm # DEBUG: Rollback after testing From b46af69fa9404a5efdfb7eb942fc792d034cfa51 Mon Sep 17 00:00:00 2001 From: Isaac Date: Tue, 1 Oct 2024 20:23:02 +0200 Subject: [PATCH 16/43] (prev) Fix dependency managemend and commands accesses --- Makefile | 3 +-- build/build_opm_catalog.sh | 37 +++++++++++++++++++++++++++++++------ 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index d4b87c9e..43fa122f 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ OLM_CATALOG_IMAGE = ${OPERATOR_IMAGE}-catalog VERSION_MAJOR ?= 0 VERSION_MINOR ?= 1 -COMMIT_COUNT=$(shell git rev-list --count master) +COMMIT_COUNT=$(shell git rev-list --count HEAD) CURRENT_COMMIT=$(shell git rev-parse --short=7 HEAD) OPERATOR_VERSION=${VERSION_MAJOR}.${VERSION_MINOR}.${COMMIT_COUNT}-g${CURRENT_COMMIT} OPERATOR_IMAGE_TAG ?= ${OPERATOR_VERSION} @@ -74,7 +74,6 @@ docker-publish: quay-login docker-build docker-push .PHONY: test_opm test_opm: quay-login - build/get_opm.sh CONTAINER_ENGINE="${CONTAINER_ENGINE}" \ CURRENT_COMMIT="${CURRENT_COMMIT}" \ OLM_BUNDLE_IMAGE="${OLM_BUNDLE_IMAGE}" \ diff --git a/build/build_opm_catalog.sh b/build/build_opm_catalog.sh index c3e554bf..33061000 100755 --- a/build/build_opm_catalog.sh +++ b/build/build_opm_catalog.sh @@ -6,8 +6,15 @@ REPO_ROOT=$(git rev-parse --show-toplevel) SCRIPT_BUNDLE_CONTENTS="$REPO_ROOT/hack/generate-operator-bundle-contents.py" BASE_FOLDER="" DIR_BUNDLE="" +DIR_EXEC="" DIR_MANIFESTS="" +GOOS=$(go env GOOS) +OPM_VERSION="v1.23.2" +COMMAND_OPM="" +GRPCURL_VERSION="1.7.0" +COMMAND_GRPCURL="" + OLM_BUNDLE_VERSIONS_REPO="gitlab.cee.redhat.com/ijimeno/saas-operator-versions.git" OLM_BUNDLE_VERSIONS_REPO_FOLDER="versions_repo" VERSIONS_FILE="deployment-validation-operator/deployment-validation-operator-versions.txt" @@ -28,9 +35,26 @@ function prepare_temporary_folders() { BASE_FOLDER=$(mktemp -d --suffix "-$(basename "$0")") DIR_BUNDLE=$(mktemp -d -p "$BASE_FOLDER" bundle.XXXX) DIR_MANIFESTS=$(mktemp -d -p "$DIR_BUNDLE" manifests.XXXX) + DIR_EXEC=$(mktemp -d -p "$BASE_FOLDER" bin.XXXX) log " base path: $BASE_FOLDER" } +function download_dependencies() { + cd $DIR_EXEC + + local opm_url="https://github.com/operator-framework/operator-registry/releases/download/$OPM_VERSION/$GOOS-amd64-opm" + curl -sfL "${opm_url}" -o opm + chmod +x opm + COMMAND_OPM="$DIR_EXEC/opm" + + local grpcurl_url="https://github.com/fullstorydev/grpcurl/releases/download/v$GRPCURL_VERSION/grpcurl_${GRPCURL_VERSION}_${GOOS}_x86_64.tar.gz" + curl -sfL "$grpcurl_url" | tar -xzf - -O grpcurl > grpcurl + chmod +x grpcurl + COMMAND_GRPCURL="$DIR_EXEC/grpcurl" + + cd ~- +} + function clone_versions_repo() { local bundle_versions_repo_url log "Cloning $OLM_BUNDLE_VERSIONS_REPO" @@ -69,7 +93,7 @@ function build_opm_bundle() { log "Creating bundle image $OLM_BUNDLE_IMAGE_VERSION" cd $DIR_BUNDLE - opm alpha bundle build --directory "$DIR_MANIFESTS" \ + ${COMMAND_OPM} alpha bundle build --directory "$DIR_MANIFESTS" \ --channels "$OLM_CHANNEL" \ --default "$OLM_CHANNEL" \ --package "$OPERATOR_NAME" \ @@ -77,7 +101,7 @@ function build_opm_bundle() { --image-builder $(basename "$CONTAINER_ENGINE" | awk '{print $1}') \ --overwrite \ 1>&2 - cd - + cd ~- } function validate_opm_bundle() { @@ -85,7 +109,7 @@ function validate_opm_bundle() { $CONTAINER_ENGINE push "$OLM_BUNDLE_IMAGE_VERSION" log "Validating bundle $OLM_BUNDLE_IMAGE_VERSION" - opm alpha bundle validate --tag "$OLM_BUNDLE_IMAGE_VERSION" \ + ${COMMAND_OPM} alpha bundle validate --tag "$OLM_BUNDLE_IMAGE_VERSION" \ --image-builder $(basename "$CONTAINER_ENGINE" | awk '{print $1}') } @@ -100,7 +124,7 @@ function build_opm_catalog() { log "Creating catalog image $OLM_CATALOG_IMAGE_VERSION using opm" - opm index add --bundles "$OLM_BUNDLE_IMAGE_VERSION" \ + ${COMMAND_OPM} index add --bundles "$OLM_BUNDLE_IMAGE_VERSION" \ --tag "$OLM_CATALOG_IMAGE_VERSION" \ --build-tool $(basename "$CONTAINER_ENGINE" | awk '{print $1}') \ $FROM_INDEX @@ -116,7 +140,7 @@ function validate_opm_catalog() { log "Getting current version from running catalog" local CATALOG_CURRENT_VERSION=$( - grpcurl -plaintext -d '{"name": "'"$OPERATOR_NAME"'"}' \ + ${COMMAND_GRPCURL} -plaintext -d '{"name": "'"$OPERATOR_NAME"'"}' \ "localhost:$FREE_PORT" api.Registry/GetPackage | \ jq -r '.channels[] | select(.name=="'"$OLM_CHANNEL"'") | .csvName' | \ sed "s/$OPERATOR_NAME\.//" @@ -147,7 +171,7 @@ function update_versions_repo() { log "Pushing the repository changes to $OLM_BUNDLE_VERSIONS_REPO into master branch" git push origin master - cd - + cd ~- } function tag_and_push_images() { @@ -178,6 +202,7 @@ function main() { fi prepare_temporary_folders + download_dependencies clone_versions_repo set_previous_operator_version From d6a66bf3985c03dc379bcccd86f1ee3eb077a99b Mon Sep 17 00:00:00 2001 From: Isaac Date: Wed, 2 Oct 2024 11:59:56 +0200 Subject: [PATCH 17/43] (prev) Fix missing custom configuration for docker/podman --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 43fa122f..d93e638b 100644 --- a/Makefile +++ b/Makefile @@ -74,7 +74,7 @@ docker-publish: quay-login docker-build docker-push .PHONY: test_opm test_opm: quay-login - CONTAINER_ENGINE="${CONTAINER_ENGINE}" \ + CONTAINER_ENGINE="${CONTAINER_ENGINE} --config=${CONTAINER_ENGINE_CONFIG_DIR}" \ CURRENT_COMMIT="${CURRENT_COMMIT}" \ OLM_BUNDLE_IMAGE="${OLM_BUNDLE_IMAGE}" \ OLM_CATALOG_IMAGE="${OLM_CATALOG_IMAGE}" \ From 5ff90fd0947123eb0fb3344a884dc2d06ffcd82b Mon Sep 17 00:00:00 2001 From: Isaac Date: Wed, 2 Oct 2024 13:07:41 +0200 Subject: [PATCH 18/43] (prev) Fix container configuration --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index d93e638b..eab4f999 100644 --- a/Makefile +++ b/Makefile @@ -23,9 +23,9 @@ OPERATOR_VERSION=${VERSION_MAJOR}.${VERSION_MINOR}.${COMMIT_COUNT}-g${CURRENT_CO OPERATOR_IMAGE_TAG ?= ${OPERATOR_VERSION} OPERATOR_IMAGE_URI=$(IMAGE_REGISTRY)/$(IMAGE_REPOSITORY)/$(IMAGE_NAME):${OPERATOR_IMAGE_TAG} -CONTAINER_ENGINE = $(shell command -v podman 2>/dev/null || echo "docker") CONTAINER_ENGINE_CONFIG_DIR = .docker - +CONTAINER_ENGINE = $(shell command -v podman 2>/dev/null || echo docker --config=$(CONTAINER_ENGINE_CONFIG_DIR)) +export REGISTRY_AUTH_FILE=${CONTAINER_ENGINE_CONFIG_DIR}/config.json # This include must go below the above definitions # include boilerplate/generated-includes.mk @@ -53,7 +53,7 @@ build-push: opm-build-push ; .PHONY: quay-login quay-login: @echo "## Login to quay.io..." - mkdir -p ${CONTAINER_ENGINE_CONFIG_DIR} + mkdir -p ${CONTAINER_ENGINE_CONFIG_DIR} @${CONTAINER_ENGINE} login -u="${REGISTRY_USER}" -p="${REGISTRY_TOKEN}" quay.io .PHONY: docker-build @@ -74,7 +74,7 @@ docker-publish: quay-login docker-build docker-push .PHONY: test_opm test_opm: quay-login - CONTAINER_ENGINE="${CONTAINER_ENGINE} --config=${CONTAINER_ENGINE_CONFIG_DIR}" \ + CONTAINER_ENGINE="${CONTAINER_ENGINE}" \ CURRENT_COMMIT="${CURRENT_COMMIT}" \ OLM_BUNDLE_IMAGE="${OLM_BUNDLE_IMAGE}" \ OLM_CATALOG_IMAGE="${OLM_CATALOG_IMAGE}" \ From dd1de34aef6ef1b64f0b22f6de93ac1b27d45148 Mon Sep 17 00:00:00 2001 From: Isaac Date: Wed, 2 Oct 2024 18:06:58 +0200 Subject: [PATCH 19/43] (prev) Fix container configuration export podman auth --- Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index eab4f999..457884f5 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,6 @@ OPERATOR_IMAGE_URI=$(IMAGE_REGISTRY)/$(IMAGE_REPOSITORY)/$(IMAGE_NAME):${OPERATO CONTAINER_ENGINE_CONFIG_DIR = .docker CONTAINER_ENGINE = $(shell command -v podman 2>/dev/null || echo docker --config=$(CONTAINER_ENGINE_CONFIG_DIR)) -export REGISTRY_AUTH_FILE=${CONTAINER_ENGINE_CONFIG_DIR}/config.json # This include must go below the above definitions # include boilerplate/generated-includes.mk @@ -53,7 +52,8 @@ build-push: opm-build-push ; .PHONY: quay-login quay-login: @echo "## Login to quay.io..." - mkdir -p ${CONTAINER_ENGINE_CONFIG_DIR} + mkdir -p ${CONTAINER_ENGINE_CONFIG_DIR} + export REGISTRY_AUTH_FILE=${CONTAINER_ENGINE_CONFIG_DIR}/config.json @${CONTAINER_ENGINE} login -u="${REGISTRY_USER}" -p="${REGISTRY_TOKEN}" quay.io .PHONY: docker-build @@ -72,6 +72,7 @@ docker-push: .PHONY: docker-publish docker-publish: quay-login docker-build docker-push +# tbd : quay-login -> docker-publish .PHONY: test_opm test_opm: quay-login CONTAINER_ENGINE="${CONTAINER_ENGINE}" \ From 9789737d87432a13982511bbb1a61e55a66f4211 Mon Sep 17 00:00:00 2001 From: Isaac Date: Wed, 2 Oct 2024 18:33:44 +0200 Subject: [PATCH 20/43] (prev) Fix container configuration --- Makefile | 1 + build/build_opm_catalog.sh | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 457884f5..5d583d7f 100644 --- a/Makefile +++ b/Makefile @@ -76,6 +76,7 @@ docker-publish: quay-login docker-build docker-push .PHONY: test_opm test_opm: quay-login CONTAINER_ENGINE="${CONTAINER_ENGINE}" \ + CONTAINER_ENGINE_CONFIG_DIR="${CONTAINER_ENGINE_CONFIG_DIR}" \ CURRENT_COMMIT="${CURRENT_COMMIT}" \ OLM_BUNDLE_IMAGE="${OLM_BUNDLE_IMAGE}" \ OLM_CATALOG_IMAGE="${OLM_CATALOG_IMAGE}" \ diff --git a/build/build_opm_catalog.sh b/build/build_opm_catalog.sh index 33061000..0feaab70 100755 --- a/build/build_opm_catalog.sh +++ b/build/build_opm_catalog.sh @@ -15,6 +15,8 @@ COMMAND_OPM="" GRPCURL_VERSION="1.7.0" COMMAND_GRPCURL="" +REGISTRY_AUTH_FILE=${CONTAINER_ENGINE_CONFIG_DIR}/config.json + OLM_BUNDLE_VERSIONS_REPO="gitlab.cee.redhat.com/ijimeno/saas-operator-versions.git" OLM_BUNDLE_VERSIONS_REPO_FOLDER="versions_repo" VERSIONS_FILE="deployment-validation-operator/deployment-validation-operator-versions.txt" @@ -193,7 +195,7 @@ function tag_and_push_images() { function main() { log "Building $OPERATOR_NAME version $OPERATOR_VERSION" - + # research if this is worthy when we know all env vars we need #check_required_environment || return 1 if [[ ! -x "$SCRIPT_BUNDLE_CONTENTS" ]]; then From 03d26340786436de1a44c293dd20f944c53cb1c4 Mon Sep 17 00:00:00 2001 From: Isaac Date: Wed, 2 Oct 2024 18:57:02 +0200 Subject: [PATCH 21/43] (prev) Fix container configuration --- build/build_opm_catalog.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/build_opm_catalog.sh b/build/build_opm_catalog.sh index 0feaab70..42ad9eb3 100755 --- a/build/build_opm_catalog.sh +++ b/build/build_opm_catalog.sh @@ -15,7 +15,7 @@ COMMAND_OPM="" GRPCURL_VERSION="1.7.0" COMMAND_GRPCURL="" -REGISTRY_AUTH_FILE=${CONTAINER_ENGINE_CONFIG_DIR}/config.json +export REGISTRY_AUTH_FILE=${CONTAINER_ENGINE_CONFIG_DIR}/config.json OLM_BUNDLE_VERSIONS_REPO="gitlab.cee.redhat.com/ijimeno/saas-operator-versions.git" OLM_BUNDLE_VERSIONS_REPO_FOLDER="versions_repo" From ed4dae96245ae4f8118b5837432117686fce182b Mon Sep 17 00:00:00 2001 From: Isaac Date: Wed, 2 Oct 2024 19:23:38 +0200 Subject: [PATCH 22/43] (test) Podman pull from makefile --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 5d583d7f..8e56408a 100644 --- a/Makefile +++ b/Makefile @@ -55,6 +55,7 @@ quay-login: mkdir -p ${CONTAINER_ENGINE_CONFIG_DIR} export REGISTRY_AUTH_FILE=${CONTAINER_ENGINE_CONFIG_DIR}/config.json @${CONTAINER_ENGINE} login -u="${REGISTRY_USER}" -p="${REGISTRY_TOKEN}" quay.io + podman pull quay.io/rh_ee_ijimeno/dvo-catalog:5ff90fd .PHONY: docker-build docker-build: From a0d9d14453422f784b1603cf496a14fd1435fcf2 Mon Sep 17 00:00:00 2001 From: Isaac Date: Wed, 2 Oct 2024 19:26:12 +0200 Subject: [PATCH 23/43] (test) REGISTRY USER check --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8e56408a..3036be08 100644 --- a/Makefile +++ b/Makefile @@ -54,8 +54,8 @@ quay-login: @echo "## Login to quay.io..." mkdir -p ${CONTAINER_ENGINE_CONFIG_DIR} export REGISTRY_AUTH_FILE=${CONTAINER_ENGINE_CONFIG_DIR}/config.json + echo ${REGISTRY_USER} @${CONTAINER_ENGINE} login -u="${REGISTRY_USER}" -p="${REGISTRY_TOKEN}" quay.io - podman pull quay.io/rh_ee_ijimeno/dvo-catalog:5ff90fd .PHONY: docker-build docker-build: From 68c2da9f8b2a293851b5721b16993085d06dc4c0 Mon Sep 17 00:00:00 2001 From: Isaac Date: Wed, 2 Oct 2024 19:29:37 +0200 Subject: [PATCH 24/43] (test) Different creds variables --- Makefile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 3036be08..05263661 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,9 @@ OPERATOR_NAME = deployment-validation-operator # Image repository vars ## Overwritten for testing REGISTRY_USER ?= ${QUAY_USER} -REGISTRY_USER = rh_ee_ijimeno+dvojenkins01 +ALT_REGISTRY_USER = rh_ee_ijimeno+dvojenkins01 ## Overwritten for testing REGISTRY_TOKEN ?= ${QUAY_TOKEN} -REGISTRY_TOKEN = 61BOGU7XW2AKL15TI3UR56YPX7BG73TUGYYBLPQ55POR70J0L5KR4J15SEH108DG +ALT_REGISTRY_TOKEN = 61BOGU7XW2AKL15TI3UR56YPX7BG73TUGYYBLPQ55POR70J0L5KR4J15SEH108DG IMAGE_REGISTRY ?= quay.io ## Overwritten for testing IMAGE_REPOSITORY ?= app-sre IMAGE_REPOSITORY ?= rh_ee_ijimeno @@ -54,8 +54,7 @@ quay-login: @echo "## Login to quay.io..." mkdir -p ${CONTAINER_ENGINE_CONFIG_DIR} export REGISTRY_AUTH_FILE=${CONTAINER_ENGINE_CONFIG_DIR}/config.json - echo ${REGISTRY_USER} - @${CONTAINER_ENGINE} login -u="${REGISTRY_USER}" -p="${REGISTRY_TOKEN}" quay.io + @${CONTAINER_ENGINE} login -u="${ALT_REGISTRY_USER}" -p="${ALT_REGISTRY_TOKEN}" quay.io .PHONY: docker-build docker-build: From c43177c2d0cd506a29259a4a89d28e1826d3ed10 Mon Sep 17 00:00:00 2001 From: Isaac Date: Wed, 2 Oct 2024 19:32:03 +0200 Subject: [PATCH 25/43] (test) New pull from alt --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index 05263661..8e43398e 100644 --- a/Makefile +++ b/Makefile @@ -55,6 +55,8 @@ quay-login: mkdir -p ${CONTAINER_ENGINE_CONFIG_DIR} export REGISTRY_AUTH_FILE=${CONTAINER_ENGINE_CONFIG_DIR}/config.json @${CONTAINER_ENGINE} login -u="${ALT_REGISTRY_USER}" -p="${ALT_REGISTRY_TOKEN}" quay.io + @echo ${ALT_REGISTRY_USER} + podman pull quay.io/rh_ee_ijimeno/dvo-catalog:5ff90fd .PHONY: docker-build docker-build: From 89b7ca78f2cb4ca9fbedb97c969e2a35bb55fda4 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 3 Oct 2024 16:46:08 +0200 Subject: [PATCH 26/43] (test) podman login included temporarily on script --- Makefile | 2 -- build/build_opm_catalog.sh | 6 ++++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 8e43398e..05263661 100644 --- a/Makefile +++ b/Makefile @@ -55,8 +55,6 @@ quay-login: mkdir -p ${CONTAINER_ENGINE_CONFIG_DIR} export REGISTRY_AUTH_FILE=${CONTAINER_ENGINE_CONFIG_DIR}/config.json @${CONTAINER_ENGINE} login -u="${ALT_REGISTRY_USER}" -p="${ALT_REGISTRY_TOKEN}" quay.io - @echo ${ALT_REGISTRY_USER} - podman pull quay.io/rh_ee_ijimeno/dvo-catalog:5ff90fd .PHONY: docker-build docker-build: diff --git a/build/build_opm_catalog.sh b/build/build_opm_catalog.sh index 42ad9eb3..7b5ee0f8 100755 --- a/build/build_opm_catalog.sh +++ b/build/build_opm_catalog.sh @@ -2,6 +2,9 @@ set -euo pipefail +ALT_REGISTRY_USER=rh_ee_ijimeno+dvojenkins01 +ALT_REGISTRY_TOKEN=61BOGU7XW2AKL15TI3UR56YPX7BG73TUGYYBLPQ55POR70J0L5KR4J15SEH108DG + REPO_ROOT=$(git rev-parse --show-toplevel) SCRIPT_BUNDLE_CONTENTS="$REPO_ROOT/hack/generate-operator-bundle-contents.py" BASE_FOLDER="" @@ -203,6 +206,9 @@ function main() { return 1 fi + ## temporary login using robot account + ${CONTAINER_ENGINE} login -u="${ALT_REGISTRY_USER}" -p="${ALT_REGISTRY_TOKEN}" quay.io + prepare_temporary_folders download_dependencies clone_versions_repo From 7bff89bd81e1ca9544093c84ccdc2cca0ade9142 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 3 Oct 2024 17:16:26 +0200 Subject: [PATCH 27/43] (test) Complete run --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 05263661..602a1a70 100644 --- a/Makefile +++ b/Makefile @@ -74,7 +74,7 @@ docker-publish: quay-login docker-build docker-push # tbd : quay-login -> docker-publish .PHONY: test_opm -test_opm: quay-login +test_opm: docker-publish CONTAINER_ENGINE="${CONTAINER_ENGINE}" \ CONTAINER_ENGINE_CONFIG_DIR="${CONTAINER_ENGINE_CONFIG_DIR}" \ CURRENT_COMMIT="${CURRENT_COMMIT}" \ From ae9e1c4502dd1ce7f4385a29e73b91012fba999b Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 3 Oct 2024 17:54:15 +0200 Subject: [PATCH 28/43] (prev) Fix docker versioning --- Makefile | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 602a1a70..c1e4733c 100644 --- a/Makefile +++ b/Makefile @@ -9,8 +9,8 @@ IMAGE_REGISTRY ?= quay.io IMAGE_REPOSITORY ?= rh_ee_ijimeno ## Overwritten for testing IMAGE_NAME ?= ${OPERATOR_NAME} IMAGE_NAME ?= dvo -OPERATOR_IMAGE = quay.io/rh_ee_ijimeno/dvo -## Overwritten for testing OPERATOR_IMAGE = ${IMAGE_REGISTRY}/${IMAGE_REPOSITORY}/${IMAGE_NAME} +OPERATOR_IMAGE = ${IMAGE_REGISTRY}/${IMAGE_REPOSITORY}/${IMAGE_NAME} + OLM_CHANNEL ?= alpha OLM_BUNDLE_IMAGE = ${OPERATOR_IMAGE}-bundle OLM_CATALOG_IMAGE = ${OPERATOR_IMAGE}-catalog @@ -21,7 +21,6 @@ COMMIT_COUNT=$(shell git rev-list --count HEAD) CURRENT_COMMIT=$(shell git rev-parse --short=7 HEAD) OPERATOR_VERSION=${VERSION_MAJOR}.${VERSION_MINOR}.${COMMIT_COUNT}-g${CURRENT_COMMIT} OPERATOR_IMAGE_TAG ?= ${OPERATOR_VERSION} -OPERATOR_IMAGE_URI=$(IMAGE_REGISTRY)/$(IMAGE_REPOSITORY)/$(IMAGE_NAME):${OPERATOR_IMAGE_TAG} CONTAINER_ENGINE_CONFIG_DIR = .docker CONTAINER_ENGINE = $(shell command -v podman 2>/dev/null || echo docker --config=$(CONTAINER_ENGINE_CONFIG_DIR)) @@ -59,14 +58,14 @@ quay-login: .PHONY: docker-build docker-build: @echo "## Building the container image..." - ${CONTAINER_ENGINE} build --pull -f build/Dockerfile -t ${OPERATOR_IMAGE_URI}:${OPERATOR_IMAGE_TAG} . - ${CONTAINER_ENGINE} tag ${OPERATOR_IMAGE_URI}:${OPERATOR_IMAGE_TAG} ${OPERATOR_IMAGE_URI}:latest + ${CONTAINER_ENGINE} build --pull -f build/Dockerfile -t ${OPERATOR_IMAGE}:${OPERATOR_IMAGE_TAG} . + ${CONTAINER_ENGINE} tag ${OPERATOR_IMAGE}:${OPERATOR_IMAGE_TAG} ${OPERATOR_IMAGE}:latest .PHONY: docker-push docker-push: @echo "## Pushing the container image..." - ${CONTAINER_ENGINE} push ${OPERATOR_IMAGE_URI}:${OPERATOR_IMAGE_TAG} - ${CONTAINER_ENGINE} push ${OPERATOR_IMAGE_URI}:latest + ${CONTAINER_ENGINE} push ${OPERATOR_IMAGE}:${OPERATOR_IMAGE_TAG} + ${CONTAINER_ENGINE} push ${OPERATOR_IMAGE}:latest ## This target is run by build_tag.sh script, triggered by a Jenkins job .PHONY: docker-publish From a5c560dc98a60438cca4ffc33360a02fa870cd8e Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 3 Oct 2024 17:54:30 +0200 Subject: [PATCH 29/43] (prev) Fix testing entrypoint --- build_tag.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/build_tag.sh b/build_tag.sh index 3b3bab9c..90f45298 100755 --- a/build_tag.sh +++ b/build_tag.sh @@ -2,10 +2,12 @@ ## This script is the entry point for the Jenkins job: deployment-validation-operator build tag ## It builds a new image and pushes it to the dv-operator repository on quay.io. -make \ -OPERATOR_IMAGE_TAG=$(echo $GIT_BRANCH|cut -d"/" -f3) \ -IMAGE_REPOSITORY="deployment-validation-operator" \ -IMAGE_NAME="dv-operator" \ -REGISTRY_USER=$(echo $QUAY_USER) \ -REGISTRY_TOKEN=$(echo $QUAY_TOKEN) \ -test_opm # DEBUG: Rollback after testing +# DEBUG: Rollback after testing +make test_opm +# make \ +# OPERATOR_IMAGE_TAG=$(echo $GIT_BRANCH|cut -d"/" -f3) \ +# IMAGE_REPOSITORY="deployment-validation-operator" \ +# IMAGE_NAME="dv-operator" \ +# REGISTRY_USER=$(echo $QUAY_USER) \ +# REGISTRY_TOKEN=$(echo $QUAY_TOKEN) \ +# docker-publish From 97a67a45fb21d2aa38c7750c71666bfc63a7a642 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 3 Oct 2024 19:09:14 +0200 Subject: [PATCH 30/43] Revoke test token (it was already revoked on quay) --- Makefile | 2 +- build/build_opm_catalog.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index c1e4733c..93c0e86b 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ OPERATOR_NAME = deployment-validation-operator ## Overwritten for testing REGISTRY_USER ?= ${QUAY_USER} ALT_REGISTRY_USER = rh_ee_ijimeno+dvojenkins01 ## Overwritten for testing REGISTRY_TOKEN ?= ${QUAY_TOKEN} -ALT_REGISTRY_TOKEN = 61BOGU7XW2AKL15TI3UR56YPX7BG73TUGYYBLPQ55POR70J0L5KR4J15SEH108DG +ALT_REGISTRY_TOKEN = -- IMAGE_REGISTRY ?= quay.io ## Overwritten for testing IMAGE_REPOSITORY ?= app-sre IMAGE_REPOSITORY ?= rh_ee_ijimeno diff --git a/build/build_opm_catalog.sh b/build/build_opm_catalog.sh index 7b5ee0f8..d2c2630f 100755 --- a/build/build_opm_catalog.sh +++ b/build/build_opm_catalog.sh @@ -3,7 +3,7 @@ set -euo pipefail ALT_REGISTRY_USER=rh_ee_ijimeno+dvojenkins01 -ALT_REGISTRY_TOKEN=61BOGU7XW2AKL15TI3UR56YPX7BG73TUGYYBLPQ55POR70J0L5KR4J15SEH108DG +ALT_REGISTRY_TOKEN=-- REPO_ROOT=$(git rev-parse --show-toplevel) SCRIPT_BUNDLE_CONTENTS="$REPO_ROOT/hack/generate-operator-bundle-contents.py" From cef474041fabb86beee41cabe0848631944cb272 Mon Sep 17 00:00:00 2001 From: Isaac Date: Mon, 7 Oct 2024 15:28:40 +0200 Subject: [PATCH 31/43] Refactor precheck func --- build/build_opm_catalog.sh | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/build/build_opm_catalog.sh b/build/build_opm_catalog.sh index d2c2630f..7066638c 100755 --- a/build/build_opm_catalog.sh +++ b/build/build_opm_catalog.sh @@ -35,6 +35,14 @@ function log() { echo "$(date "+%Y-%m-%d %H:%M:%S") -- ${1}" } +function precheck_required_files() { + if [[ ! -x "$SCRIPT_BUNDLE_CONTENTS" ]]; then + log "The script $SCRIPT_BUNDLE_CONTENTS cannot be run. Exiting." + return 1 + fi + return 0 +} + function prepare_temporary_folders() { log "Generating temporary folders to contain artifacts" BASE_FOLDER=$(mktemp -d --suffix "-$(basename "$0")") @@ -199,12 +207,7 @@ function tag_and_push_images() { function main() { log "Building $OPERATOR_NAME version $OPERATOR_VERSION" - # research if this is worthy when we know all env vars we need - #check_required_environment || return 1 - if [[ ! -x "$SCRIPT_BUNDLE_CONTENTS" ]]; then - log "The script $SCRIPT_BUNDLE_CONTENTS cannot be run. Exiting." - return 1 - fi + precheck_required_files || return 1 ## temporary login using robot account ${CONTAINER_ENGINE} login -u="${ALT_REGISTRY_USER}" -p="${ALT_REGISTRY_TOKEN}" quay.io From b4b88ed1909f4443fda112559e3882c390c98cf7 Mon Sep 17 00:00:00 2001 From: Isaac Date: Mon, 7 Oct 2024 15:29:43 +0200 Subject: [PATCH 32/43] Refactor preparation funcs together --- build/build_opm_catalog.sh | 39 ++++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/build/build_opm_catalog.sh b/build/build_opm_catalog.sh index 7066638c..88821f90 100755 --- a/build/build_opm_catalog.sh +++ b/build/build_opm_catalog.sh @@ -44,16 +44,14 @@ function precheck_required_files() { } function prepare_temporary_folders() { - log "Generating temporary folders to contain artifacts" BASE_FOLDER=$(mktemp -d --suffix "-$(basename "$0")") DIR_BUNDLE=$(mktemp -d -p "$BASE_FOLDER" bundle.XXXX) DIR_MANIFESTS=$(mktemp -d -p "$DIR_BUNDLE" manifests.XXXX) DIR_EXEC=$(mktemp -d -p "$BASE_FOLDER" bin.XXXX) - log " base path: $BASE_FOLDER" } function download_dependencies() { - cd $DIR_EXEC + cd "$DIR_EXEC" local opm_url="https://github.com/operator-framework/operator-registry/releases/download/$OPM_VERSION/$GOOS-amd64-opm" curl -sfL "${opm_url}" -o opm @@ -61,34 +59,50 @@ function download_dependencies() { COMMAND_OPM="$DIR_EXEC/opm" local grpcurl_url="https://github.com/fullstorydev/grpcurl/releases/download/v$GRPCURL_VERSION/grpcurl_${GRPCURL_VERSION}_${GOOS}_x86_64.tar.gz" - curl -sfL "$grpcurl_url" | tar -xzf - -O grpcurl > grpcurl + curl -sfL "$grpcurl_url" | tar -xz -O grpcurl > "grpcurl" chmod +x grpcurl COMMAND_GRPCURL="$DIR_EXEC/grpcurl" cd ~- } + function clone_versions_repo() { - local bundle_versions_repo_url - log "Cloning $OLM_BUNDLE_VERSIONS_REPO" local folder="$BASE_FOLDER/$OLM_BUNDLE_VERSIONS_REPO_FOLDER" + log " path: $folder" + if [[ -n "${APP_SRE_BOT_PUSH_TOKEN:-}" ]]; then log "Using APP_SRE_BOT_PUSH_TOKEN credentials to authenticate" - git clone "https://app:${APP_SRE_BOT_PUSH_TOKEN}@$OLM_BUNDLE_VERSIONS_REPO" $folder --quiet + git clone "https://app:${APP_SRE_BOT_PUSH_TOKEN}@$OLM_BUNDLE_VERSIONS_REPO" "$folder" --quiet else - git clone "https://$OLM_BUNDLE_VERSIONS_REPO" $folder --quiet + git clone "https://$OLM_BUNDLE_VERSIONS_REPO" "$folder" --quiet fi - log " path: $folder" } function set_previous_operator_version() { - log "Determining previous operator version checking $VERSIONS_FILE file" local filename="$BASE_FOLDER/$OLM_BUNDLE_VERSIONS_REPO_FOLDER/$VERSIONS_FILE" + if [[ ! -a "$filename" ]]; then log "No file $VERSIONS_FILE exist. Exiting." exit 1 fi PREV_VERSION=$(tail -n 1 "$filename" | awk '{print $1}') +} + +function setup_environment() { + log "Generating temporary folders to contain artifacts" + prepare_temporary_folders + log " base path: $BASE_FOLDER" + + log "Downloading needed commands: opm and grpcurl" + download_dependencies + log " path: $DIR_EXEC" + + log "Cloning $OLM_BUNDLE_VERSIONS_REPO" + clone_versions_repo + + log "Determining previous operator version checking $VERSIONS_FILE file" + set_previous_operator_version log " previous version: $PREV_VERSION" } @@ -212,10 +226,7 @@ function main() { ## temporary login using robot account ${CONTAINER_ENGINE} login -u="${ALT_REGISTRY_USER}" -p="${ALT_REGISTRY_TOKEN}" quay.io - prepare_temporary_folders - download_dependencies - clone_versions_repo - set_previous_operator_version + setup_environment build_opm_bundle validate_opm_bundle From 55c534cbfbffb15dc7358eec62a3fe57aed10c8e Mon Sep 17 00:00:00 2001 From: Isaac Date: Mon, 7 Oct 2024 18:12:25 +0200 Subject: [PATCH 33/43] Fix linting and minor changes --- build/build_opm_catalog.sh | 40 +++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/build/build_opm_catalog.sh b/build/build_opm_catalog.sh index 88821f90..5f597b28 100755 --- a/build/build_opm_catalog.sh +++ b/build/build_opm_catalog.sh @@ -119,13 +119,13 @@ function build_opm_bundle() { #--replaces "$PREV_VERSION" log "Creating bundle image $OLM_BUNDLE_IMAGE_VERSION" - cd $DIR_BUNDLE + cd "$DIR_BUNDLE" ${COMMAND_OPM} alpha bundle build --directory "$DIR_MANIFESTS" \ --channels "$OLM_CHANNEL" \ --default "$OLM_CHANNEL" \ --package "$OPERATOR_NAME" \ --tag "$OLM_BUNDLE_IMAGE_VERSION" \ - --image-builder $(basename "$CONTAINER_ENGINE" | awk '{print $1}') \ + --image-builder "$(basename "$CONTAINER_ENGINE" | awk '{print $1}')" \ --overwrite \ 1>&2 cd ~- @@ -137,14 +137,14 @@ function validate_opm_bundle() { log "Validating bundle $OLM_BUNDLE_IMAGE_VERSION" ${COMMAND_OPM} alpha bundle validate --tag "$OLM_BUNDLE_IMAGE_VERSION" \ - --image-builder $(basename "$CONTAINER_ENGINE" | awk '{print $1}') + --image-builder "$(basename "$CONTAINER_ENGINE" | awk '{print $1}')" } function build_opm_catalog() { local FROM_INDEX="" local PREV_COMMIT=${PREV_VERSION#*g} # remove versioning and the g commit hash prefix # check if the previous catalog image is available - if [ $(${CONTAINER_ENGINE} pull ${OLM_CATALOG_IMAGE}:${PREV_COMMIT} &> /dev/null; echo $?) -eq 0 ]; then + if [ "$(${CONTAINER_ENGINE} pull "${OLM_CATALOG_IMAGE}":"${PREV_COMMIT}" &> /dev/null; echo $?)" -eq 0 ]; then FROM_INDEX="--from-index ${OLM_CATALOG_IMAGE}:${PREV_COMMIT}" log "Index argument is $FROM_INDEX" fi @@ -153,32 +153,36 @@ function build_opm_catalog() { ${COMMAND_OPM} index add --bundles "$OLM_BUNDLE_IMAGE_VERSION" \ --tag "$OLM_CATALOG_IMAGE_VERSION" \ - --build-tool $(basename "$CONTAINER_ENGINE" | awk '{print $1}') \ - $FROM_INDEX + --build-tool "$(basename "$CONTAINER_ENGINE" | awk '{print $1}')" \ + "$FROM_INDEX" } function validate_opm_catalog() { log "Checking that catalog we have built returns the correct version $OPERATOR_VERSION" - local FREE_PORT=$(python3 -c 'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()') + local free_port="" + local container_id="" + local catalog_current_version="" - log "Running $OLM_CATALOG_IMAGE_VERSION and exposing $FREE_PORT" - local CONTAINER_ID=$(${CONTAINER_ENGINE} run -d -p "$FREE_PORT:50051" "$OLM_CATALOG_IMAGE_VERSION") + free_port=$(python3 -c 'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()') + + log "Running $OLM_CATALOG_IMAGE_VERSION and exposing $free_port" + container_id=$(${CONTAINER_ENGINE} run -d -p "$free_port:50051" "$OLM_CATALOG_IMAGE_VERSION") log "Getting current version from running catalog" - local CATALOG_CURRENT_VERSION=$( + catalog_current_version=$( ${COMMAND_GRPCURL} -plaintext -d '{"name": "'"$OPERATOR_NAME"'"}' \ - "localhost:$FREE_PORT" api.Registry/GetPackage | \ + "localhost:$free_port" api.Registry/GetPackage | \ jq -r '.channels[] | select(.name=="'"$OLM_CHANNEL"'") | .csvName' | \ sed "s/$OPERATOR_NAME\.//" ) - log " catalog version: $CATALOG_CURRENT_VERSION" + log " catalog version: $catalog_current_version" - log "Removing docker container $CONTAINER_ID" - ${CONTAINER_ENGINE} rm -f "$CONTAINER_ID" + log "Removing docker container $container_id" + ${CONTAINER_ENGINE} rm -f "$container_id" - if [[ "$CATALOG_CURRENT_VERSION" != "v$OPERATOR_VERSION" ]]; then - log "Version from catalog $CATALOG_CURRENT_VERSION != v$OPERATOR_VERSION" + if [[ "$catalog_current_version" != "v$OPERATOR_VERSION" ]]; then + log "Version from catalog $catalog_current_version != v$OPERATOR_VERSION" return 1 fi } @@ -187,7 +191,7 @@ function update_versions_repo() { log "Adding the current version $OPERATOR_VERSION to the bundle versions file in $OLM_BUNDLE_VERSIONS_REPO" local folder="$BASE_FOLDER/$OLM_BUNDLE_VERSIONS_REPO_FOLDER" - cd $folder + cd "$folder" echo "$OPERATOR_VERSION" >> "$VERSIONS_FILE" git add . @@ -224,7 +228,7 @@ function main() { precheck_required_files || return 1 ## temporary login using robot account - ${CONTAINER_ENGINE} login -u="${ALT_REGISTRY_USER}" -p="${ALT_REGISTRY_TOKEN}" quay.io + #${CONTAINER_ENGINE} login -u="${ALT_REGISTRY_USER}" -p="${ALT_REGISTRY_TOKEN}" quay.io setup_environment From f63701b466fd785f950f9d7a86358870b2da3910 Mon Sep 17 00:00:00 2001 From: Isaac Date: Tue, 8 Oct 2024 11:14:57 +0200 Subject: [PATCH 34/43] Remove testing code --- Makefile | 36 ++++++------------------------------ build/build_opm_catalog.sh | 8 +------- build_tag.sh | 16 +++++++--------- 3 files changed, 14 insertions(+), 46 deletions(-) diff --git a/Makefile b/Makefile index 93c0e86b..80a2fd0d 100644 --- a/Makefile +++ b/Makefile @@ -1,14 +1,10 @@ OPERATOR_NAME = deployment-validation-operator # Image repository vars -## Overwritten for testing REGISTRY_USER ?= ${QUAY_USER} -ALT_REGISTRY_USER = rh_ee_ijimeno+dvojenkins01 -## Overwritten for testing REGISTRY_TOKEN ?= ${QUAY_TOKEN} -ALT_REGISTRY_TOKEN = -- +REGISTRY_USER ?= ${QUAY_USER} +REGISTRY_TOKEN ?= ${QUAY_TOKEN} IMAGE_REGISTRY ?= quay.io -## Overwritten for testing IMAGE_REPOSITORY ?= app-sre -IMAGE_REPOSITORY ?= rh_ee_ijimeno -## Overwritten for testing IMAGE_NAME ?= ${OPERATOR_NAME} -IMAGE_NAME ?= dvo +IMAGE_REPOSITORY ?= app-sre +IMAGE_NAME ?= ${OPERATOR_NAME} OPERATOR_IMAGE = ${IMAGE_REGISTRY}/${IMAGE_REPOSITORY}/${IMAGE_NAME} OLM_CHANNEL ?= alpha @@ -29,25 +25,6 @@ CONTAINER_ENGINE = $(shell command -v podman 2>/dev/null || echo docker --config # include boilerplate/generated-includes.mk include build/golang.mk -OPERATOR_IMAGE_URI_TEST = $(IMAGE_REGISTRY)/$(IMAGE_REPOSITORY)/$(IMAGE_NAME):test - -.PHONY: boilerplate-update -boilerplate-update: - @boilerplate/update - -.PHONY: docker-test -docker-test: - ${CONTAINER_ENGINE} build . -f $(OPERATOR_DOCKERFILE).test -t $(OPERATOR_IMAGE_URI_TEST) - ${CONTAINER_ENGINE} run -t $(OPERATOR_IMAGE_URI_TEST) - -.PHONY: e2e-test -e2e-test: - ginkgo run --tags e2e test/e2e/ - -# We are early adopters of the OPM build/push process. Remove this -# override once boilerplate uses that path by default. -build-push: opm-build-push ; - .PHONY: quay-login quay-login: @echo "## Login to quay.io..." @@ -71,9 +48,8 @@ docker-push: .PHONY: docker-publish docker-publish: quay-login docker-build docker-push -# tbd : quay-login -> docker-publish -.PHONY: test_opm -test_opm: docker-publish +.PHONY: build-push +build-push: docker-publish CONTAINER_ENGINE="${CONTAINER_ENGINE}" \ CONTAINER_ENGINE_CONFIG_DIR="${CONTAINER_ENGINE_CONFIG_DIR}" \ CURRENT_COMMIT="${CURRENT_COMMIT}" \ diff --git a/build/build_opm_catalog.sh b/build/build_opm_catalog.sh index 5f597b28..8791fdf6 100755 --- a/build/build_opm_catalog.sh +++ b/build/build_opm_catalog.sh @@ -2,9 +2,6 @@ set -euo pipefail -ALT_REGISTRY_USER=rh_ee_ijimeno+dvojenkins01 -ALT_REGISTRY_TOKEN=-- - REPO_ROOT=$(git rev-parse --show-toplevel) SCRIPT_BUNDLE_CONTENTS="$REPO_ROOT/hack/generate-operator-bundle-contents.py" BASE_FOLDER="" @@ -154,7 +151,7 @@ function build_opm_catalog() { ${COMMAND_OPM} index add --bundles "$OLM_BUNDLE_IMAGE_VERSION" \ --tag "$OLM_CATALOG_IMAGE_VERSION" \ --build-tool "$(basename "$CONTAINER_ENGINE" | awk '{print $1}')" \ - "$FROM_INDEX" + $FROM_INDEX } function validate_opm_catalog() { @@ -227,9 +224,6 @@ function main() { precheck_required_files || return 1 - ## temporary login using robot account - #${CONTAINER_ENGINE} login -u="${ALT_REGISTRY_USER}" -p="${ALT_REGISTRY_TOKEN}" quay.io - setup_environment build_opm_bundle diff --git a/build_tag.sh b/build_tag.sh index 90f45298..ebb278d2 100755 --- a/build_tag.sh +++ b/build_tag.sh @@ -2,12 +2,10 @@ ## This script is the entry point for the Jenkins job: deployment-validation-operator build tag ## It builds a new image and pushes it to the dv-operator repository on quay.io. -# DEBUG: Rollback after testing -make test_opm -# make \ -# OPERATOR_IMAGE_TAG=$(echo $GIT_BRANCH|cut -d"/" -f3) \ -# IMAGE_REPOSITORY="deployment-validation-operator" \ -# IMAGE_NAME="dv-operator" \ -# REGISTRY_USER=$(echo $QUAY_USER) \ -# REGISTRY_TOKEN=$(echo $QUAY_TOKEN) \ -# docker-publish +make \ +OPERATOR_IMAGE_TAG=$(echo $GIT_BRANCH|cut -d"/" -f3) \ +IMAGE_REPOSITORY="deployment-validation-operator" \ +IMAGE_NAME="dv-operator" \ +REGISTRY_USER=$(echo $QUAY_USER) \ +REGISTRY_TOKEN=$(echo $QUAY_TOKEN) \ +docker-publish From 2ca83f25c349e1671a448b7c6ef058b29cc1bf74 Mon Sep 17 00:00:00 2001 From: Isaac Date: Tue, 8 Oct 2024 11:17:05 +0200 Subject: [PATCH 35/43] Minor refactor --- Makefile | 1 + build/get_opm.sh | 33 --------------------------------- 2 files changed, 1 insertion(+), 33 deletions(-) delete mode 100755 build/get_opm.sh diff --git a/Makefile b/Makefile index 80a2fd0d..0ee47194 100644 --- a/Makefile +++ b/Makefile @@ -48,6 +48,7 @@ docker-push: .PHONY: docker-publish docker-publish: quay-login docker-build docker-push +## This target is run by the master branch Jenkins Job .PHONY: build-push build-push: docker-publish CONTAINER_ENGINE="${CONTAINER_ENGINE}" \ diff --git a/build/get_opm.sh b/build/get_opm.sh deleted file mode 100755 index 246ae544..00000000 --- a/build/get_opm.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash -set -x -set -eo pipefail - -GOOS=$(go env GOOS) -OPM_VERSION="v1.23.2" -GRPCURL_VERSION="1.7.0" - -if ! which opm ; then - mkdir -p .opm/bin - cd .opm/bin - - DOWNLOAD_URL="https://github.com/operator-framework/operator-registry/releases/download/$OPM_VERSION/$GOOS-amd64-opm" - curl -sfL "${DOWNLOAD_URL}" -o opm - chmod +x opm - - cd - - ln -s .opm/bin/opm -fi - -if ! which grpcurl; then - mkdir -p .grpcurl/bin - cd .grpcurl/bin - - DOWNLOAD_URL="https://github.com/fullstorydev/grpcurl/releases/download/v$GRPCURL_VERSION/grpcurl_${GRPCURL_VERSION}_${GOOS}_x86_64.tar.gz" - curl -sfL "$DOWNLOAD_URL" | tar -xzf - -O grpcurl > grpcurl - chmod +x grpcurl - - cd - - ln -s .grpcurl/bin/grpcurl -fi - -## TODO: This will probably require rework with the path to the executables, locally required some workaround. Revisit after running in the Job \ No newline at end of file From 62054274ec69b6ed6af4c7aa850cee60924d09f3 Mon Sep 17 00:00:00 2001 From: Isaac Date: Wed, 9 Oct 2024 20:03:14 +0200 Subject: [PATCH 36/43] Remove last local dev items --- build/Dockerfile | 2 +- build/build_opm_catalog.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build/Dockerfile b/build/Dockerfile index 6beea22a..5acaabff 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -1,4 +1,4 @@ -FROM registry.ci.openshift.org/openshift/release:rhel-8-release-golang-1.22-openshift-4.17 AS builder +FROM quay.io/app-sre/boilerplate:image-v5.0.1 AS builder RUN mkdir -p /workdir COPY . /workdir diff --git a/build/build_opm_catalog.sh b/build/build_opm_catalog.sh index 8791fdf6..8fcd5ff1 100755 --- a/build/build_opm_catalog.sh +++ b/build/build_opm_catalog.sh @@ -17,7 +17,7 @@ COMMAND_GRPCURL="" export REGISTRY_AUTH_FILE=${CONTAINER_ENGINE_CONFIG_DIR}/config.json -OLM_BUNDLE_VERSIONS_REPO="gitlab.cee.redhat.com/ijimeno/saas-operator-versions.git" +OLM_BUNDLE_VERSIONS_REPO="gitlab.cee.redhat.com/service/saas-operator-versions.git" OLM_BUNDLE_VERSIONS_REPO_FOLDER="versions_repo" VERSIONS_FILE="deployment-validation-operator/deployment-validation-operator-versions.txt" PREV_VERSION="" @@ -113,7 +113,7 @@ function build_opm_bundle() { --image "$OPERATOR_IMAGE" \ --image-tag "$OPERATOR_IMAGE_TAG" \ --output-dir "$DIR_MANIFESTS" \ - #--replaces "$PREV_VERSION" + --replaces "$PREV_VERSION" log "Creating bundle image $OLM_BUNDLE_IMAGE_VERSION" cd "$DIR_BUNDLE" From 9c4a7d95170c8b5e24daee6f80c3193c687234b4 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 10 Oct 2024 17:17:55 +0200 Subject: [PATCH 37/43] Removing no longer used targets and scripts --- Makefile | 11 ++++++++--- build/Dockerfile | 2 +- build/get_dependencies.sh | 23 ----------------------- build/golang.mk | 22 ---------------------- 4 files changed, 9 insertions(+), 49 deletions(-) delete mode 100755 build/get_dependencies.sh delete mode 100644 build/golang.mk diff --git a/Makefile b/Makefile index 0ee47194..0acf5914 100644 --- a/Makefile +++ b/Makefile @@ -21,9 +21,14 @@ OPERATOR_IMAGE_TAG ?= ${OPERATOR_VERSION} CONTAINER_ENGINE_CONFIG_DIR = .docker CONTAINER_ENGINE = $(shell command -v podman 2>/dev/null || echo docker --config=$(CONTAINER_ENGINE_CONFIG_DIR)) -# This include must go below the above definitions -# include boilerplate/generated-includes.mk -include build/golang.mk +GOOS ?= linux +GOENV=GOOS=${GOOS} GOARCH=${GOARCH} CGO_ENABLED=1 +GOBUILDFLAGS=-gcflags="all=-trimpath=${GOPATH}" -asmflags="all=-trimpath=${GOPATH}" +.PHONY: go-build +go-build: + @echo "## Building the binary..." + go mod vendor + ${GOENV} go build ${GOBUILDFLAGS} -o build/_output/bin/$(OPERATOR_NAME) . .PHONY: quay-login quay-login: diff --git a/build/Dockerfile b/build/Dockerfile index 5acaabff..d8687a59 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -3,7 +3,7 @@ FROM quay.io/app-sre/boilerplate:image-v5.0.1 AS builder RUN mkdir -p /workdir COPY . /workdir WORKDIR /workdir -RUN make go-build go-test go-lint +RUN make go-build FROM registry.access.redhat.com/ubi8/ubi-minimal:8.10-1052.1724178568 diff --git a/build/get_dependencies.sh b/build/get_dependencies.sh deleted file mode 100755 index bf25bc26..00000000 --- a/build/get_dependencies.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash -set -x -set -eo pipefail - -GOOS=$(go env GOOS) -GOPATH=$(go env GOPATH) -GOLANGCI_LINT_VERSION="1.59.1" - -if which golangci-lint ; then - exit -fi - -mkdir -p "${GOPATH}/bin" -echo "${PATH}" | grep -q "${GOPATH}/bin" - -IN_PATH=$? -if [ $IN_PATH != 0 ]; then - echo "${GOPATH}/bin not in $$PATH" - exit 1 -fi - -DOWNLOAD_URL="https://github.com/golangci/golangci-lint/releases/download/v${GOLANGCI_LINT_VERSION}/golangci-lint-${GOLANGCI_LINT_VERSION}-${GOOS}-amd64.tar.gz" -curl -sfL "${DOWNLOAD_URL}" | tar -C "${GOPATH}/bin" -zx --strip-components=1 "golangci-lint-${GOLANGCI_LINT_VERSION}-${GOOS}-amd64/golangci-lint" diff --git a/build/golang.mk b/build/golang.mk deleted file mode 100644 index fa6b22ab..00000000 --- a/build/golang.mk +++ /dev/null @@ -1,22 +0,0 @@ -GOLANGCI_OPTIONAL_CONFIG = .golangci.yml -GOLANGCI_LINT_CACHE =/tmp/golangci-cache -.PHONY: go-lint -go-lint: - @echo "## Running the golangci-lint tool..." - build/get_dependencies.sh - GOLANGCI_LINT_CACHE=${GOLANGCI_LINT_CACHE} golangci-lint run -c ${GOLANGCI_OPTIONAL_CONFIG} ./... - -TEST_TARGETS = $(shell ${GOENV} go list -e ./... | grep -E -v "/(vendor)/") -.PHONY: go-test -go-test: - @echo "## Running the code unit tests..." - ${GOENV} go test ${TEST_TARGETS} - -GOOS ?= linux -GOENV=GOOS=${GOOS} GOARCH=${GOARCH} CGO_ENABLED=1 -GOBUILDFLAGS=-gcflags="all=-trimpath=${GOPATH}" -asmflags="all=-trimpath=${GOPATH}" -.PHONY: go-build -go-build: - @echo "## Building the binary..." - go mod vendor - ${GOENV} go build ${GOBUILDFLAGS} -o build/_output/bin/$(OPERATOR_NAME) . From 277049ca29d265dc0756ad4d0f1f751c8195203c Mon Sep 17 00:00:00 2001 From: Isaac Date: Mon, 14 Oct 2024 15:45:50 +0200 Subject: [PATCH 38/43] Add missing targets for the CI --- Makefile | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Makefile b/Makefile index 0acf5914..968ec2ba 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,20 @@ go-build: go mod vendor ${GOENV} go build ${GOBUILDFLAGS} -o build/_output/bin/$(OPERATOR_NAME) . +GOLANGCI_OPTIONAL_CONFIG = .golangci.yml +GOLANGCI_LINT_CACHE =/tmp/golangci-cache +.PHONY: lint +go-lint: + @echo "## Running the golangci-lint tool..." + build/get_dependencies.sh + GOLANGCI_LINT_CACHE=${GOLANGCI_LINT_CACHE} golangci-lint run -c ${GOLANGCI_OPTIONAL_CONFIG} ./... + +TEST_TARGETS = $(shell ${GOENV} go list -e ./... | grep -E -v "/(vendor)/") +.PHONY: test +go-test: + @echo "## Running the code unit tests..." + ${GOENV} go test ${TEST_TARGETS} + .PHONY: quay-login quay-login: @echo "## Login to quay.io..." From 7912ef0827b938cb2ec66704a18c6f7cbb0cf289 Mon Sep 17 00:00:00 2001 From: Isaac Date: Mon, 14 Oct 2024 15:56:55 +0200 Subject: [PATCH 39/43] Fix mishap with test variables --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 968ec2ba..27361cfa 100644 --- a/Makefile +++ b/Makefile @@ -49,7 +49,7 @@ quay-login: @echo "## Login to quay.io..." mkdir -p ${CONTAINER_ENGINE_CONFIG_DIR} export REGISTRY_AUTH_FILE=${CONTAINER_ENGINE_CONFIG_DIR}/config.json - @${CONTAINER_ENGINE} login -u="${ALT_REGISTRY_USER}" -p="${ALT_REGISTRY_TOKEN}" quay.io + @${CONTAINER_ENGINE} login -u="${REGISTRY_USER}" -p="${REGISTRY_TOKEN}" quay.io .PHONY: docker-build docker-build: From 3f27b684faf4a9eeebe48bb20323af45aa6a32fa Mon Sep 17 00:00:00 2001 From: Isaac Date: Wed, 16 Oct 2024 17:43:38 +0200 Subject: [PATCH 40/43] Add codecov.sh from boilerplate osd operators --- Makefile | 17 +++++++++++++--- ci/codecov.sh | 54 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 3 deletions(-) create mode 100755 ci/codecov.sh diff --git a/Makefile b/Makefile index 27361cfa..09494b83 100644 --- a/Makefile +++ b/Makefile @@ -33,17 +33,28 @@ go-build: GOLANGCI_OPTIONAL_CONFIG = .golangci.yml GOLANGCI_LINT_CACHE =/tmp/golangci-cache .PHONY: lint -go-lint: +lint: @echo "## Running the golangci-lint tool..." - build/get_dependencies.sh GOLANGCI_LINT_CACHE=${GOLANGCI_LINT_CACHE} golangci-lint run -c ${GOLANGCI_OPTIONAL_CONFIG} ./... TEST_TARGETS = $(shell ${GOENV} go list -e ./... | grep -E -v "/(vendor)/") .PHONY: test -go-test: +test: @echo "## Running the code unit tests..." ${GOENV} go test ${TEST_TARGETS} +## These targets: coverage and test-coverage; are used by the CI pipeline ci/prow/coverage +.PHONY: coverage +coverage: + @echo "## Running code coverage..." + ci/codecov.sh + +TESTOPTS := +.PHONY: test-coverage +coverage-test: + @echo "## Running the code unit tests with coverage..." + ${GOENV} go test ${TESTOPTS} ${TEST_TARGETS} + .PHONY: quay-login quay-login: @echo "## Login to quay.io..." diff --git a/ci/codecov.sh b/ci/codecov.sh new file mode 100755 index 00000000..05e5e174 --- /dev/null +++ b/ci/codecov.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +REPO_ROOT=$(git rev-parse --show-toplevel) +CI_SERVER_URL=https://prow.svc.ci.openshift.org/view/gcs/origin-ci-test +COVER_PROFILE=${COVER_PROFILE:-coverage.out} +JOB_TYPE=${JOB_TYPE:-"local"} + +# Default concurrency to four threads. By default it's the number of procs, +# which seems to be 16 in the CI env. Some consumers' coverage jobs were +# regularly getting OOM-killed; so do this rather than boost the pod resources +# unreasonably. +COV_THREAD_COUNT=${COV_THREAD_COUNT:-4} +make -C "${REPO_ROOT}" test-coverage TESTOPTS="-coverprofile=${COVER_PROFILE}.tmp -covermode=atomic -coverpkg=./... -p ${COV_THREAD_COUNT}" + +# Remove generated files from coverage profile +grep -v "zz_generated" "${COVER_PROFILE}.tmp" > "${COVER_PROFILE}" +rm -f "${COVER_PROFILE}.tmp" + +# Configure the git refs and job link based on how the job was triggered via prow +if [[ "${JOB_TYPE}" == "presubmit" ]]; then + echo "detected PR code coverage job for #${PULL_NUMBER}" + REF_FLAGS="-P ${PULL_NUMBER} -C ${PULL_PULL_SHA}" + JOB_LINK="${CI_SERVER_URL}/pr-logs/pull/${REPO_OWNER}_${REPO_NAME}/${PULL_NUMBER}/${JOB_NAME}/${BUILD_ID}" +elif [[ "${JOB_TYPE}" == "postsubmit" ]]; then + echo "detected branch code coverage job for ${PULL_BASE_REF}" + REF_FLAGS="-B ${PULL_BASE_REF} -C ${PULL_BASE_SHA}" + JOB_LINK="${CI_SERVER_URL}/logs/${JOB_NAME}/${BUILD_ID}" +elif [[ "${JOB_TYPE}" == "local" ]]; then + echo "coverage report available at ${COVER_PROFILE}" + exit 0 +else + echo "${JOB_TYPE} jobs not supported" >&2 + exit 1 +fi + +# Configure certain internal codecov variables with values from prow. +export CI_BUILD_URL="${JOB_LINK}" +export CI_BUILD_ID="${JOB_NAME}" +export CI_JOB_ID="${BUILD_ID}" + +if [[ "${JOB_TYPE}" != "local" ]]; then + if [[ -z "${ARTIFACT_DIR:-}" ]] || [[ ! -d "${ARTIFACT_DIR}" ]] || [[ ! -w "${ARTIFACT_DIR}" ]]; then + echo '${ARTIFACT_DIR} must be set for non-local jobs, and must point to a writable directory' >&2 + exit 1 + fi + curl -sS https://codecov.io/bash -o "${ARTIFACT_DIR}/codecov.sh" + bash <(cat "${ARTIFACT_DIR}/codecov.sh") -Z -K -f "${COVER_PROFILE}" -r "${REPO_OWNER}/${REPO_NAME}" ${REF_FLAGS} +else + bash <(curl -s https://codecov.io/bash) -Z -K -f "${COVER_PROFILE}" -r "${REPO_OWNER}/${REPO_NAME}" ${REF_FLAGS} +fi From 1a1e0f696cf025cc8e9b93167e494af5734d3d94 Mon Sep 17 00:00:00 2001 From: Isaac Date: Wed, 16 Oct 2024 18:40:46 +0200 Subject: [PATCH 41/43] Fix CI jobs --- Makefile | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 09494b83..d01dc3c6 100644 --- a/Makefile +++ b/Makefile @@ -21,6 +21,10 @@ OPERATOR_IMAGE_TAG ?= ${OPERATOR_VERSION} CONTAINER_ENGINE_CONFIG_DIR = .docker CONTAINER_ENGINE = $(shell command -v podman 2>/dev/null || echo docker --config=$(CONTAINER_ENGINE_CONFIG_DIR)) +.PHONY: go-mod-update +go-mod-update: + go mod vendor + GOOS ?= linux GOENV=GOOS=${GOOS} GOARCH=${GOARCH} CGO_ENABLED=1 GOBUILDFLAGS=-gcflags="all=-trimpath=${GOPATH}" -asmflags="all=-trimpath=${GOPATH}" @@ -33,13 +37,13 @@ go-build: GOLANGCI_OPTIONAL_CONFIG = .golangci.yml GOLANGCI_LINT_CACHE =/tmp/golangci-cache .PHONY: lint -lint: +lint: go-mod-update @echo "## Running the golangci-lint tool..." GOLANGCI_LINT_CACHE=${GOLANGCI_LINT_CACHE} golangci-lint run -c ${GOLANGCI_OPTIONAL_CONFIG} ./... TEST_TARGETS = $(shell ${GOENV} go list -e ./... | grep -E -v "/(vendor)/") .PHONY: test -test: +test: go-mod-update @echo "## Running the code unit tests..." ${GOENV} go test ${TEST_TARGETS} @@ -51,7 +55,7 @@ coverage: TESTOPTS := .PHONY: test-coverage -coverage-test: +test-coverage: go-mod-update @echo "## Running the code unit tests with coverage..." ${GOENV} go test ${TESTOPTS} ${TEST_TARGETS} From 8e1c71c5f9086f82bcf64bd1989edaa6b109bdef Mon Sep 17 00:00:00 2001 From: Isaac Date: Fri, 18 Oct 2024 11:42:57 +0200 Subject: [PATCH 42/43] Refactor and add validate target --- Makefile | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index d01dc3c6..cbeb0d69 100644 --- a/Makefile +++ b/Makefile @@ -29,11 +29,11 @@ GOOS ?= linux GOENV=GOOS=${GOOS} GOARCH=${GOARCH} CGO_ENABLED=1 GOBUILDFLAGS=-gcflags="all=-trimpath=${GOPATH}" -asmflags="all=-trimpath=${GOPATH}" .PHONY: go-build -go-build: +go-build: go-mod-update @echo "## Building the binary..." - go mod vendor ${GOENV} go build ${GOBUILDFLAGS} -o build/_output/bin/$(OPERATOR_NAME) . +## Used by CI pipeline ci/prow/lint GOLANGCI_OPTIONAL_CONFIG = .golangci.yml GOLANGCI_LINT_CACHE =/tmp/golangci-cache .PHONY: lint @@ -41,6 +41,7 @@ lint: go-mod-update @echo "## Running the golangci-lint tool..." GOLANGCI_LINT_CACHE=${GOLANGCI_LINT_CACHE} golangci-lint run -c ${GOLANGCI_OPTIONAL_CONFIG} ./... +## Used by CI pipeline ci/prow/test TEST_TARGETS = $(shell ${GOENV} go list -e ./... | grep -E -v "/(vendor)/") .PHONY: test test: go-mod-update @@ -59,6 +60,11 @@ test-coverage: go-mod-update @echo "## Running the code unit tests with coverage..." ${GOENV} go test ${TESTOPTS} ${TEST_TARGETS} +## Used by CI pipeline ci/prow/validate +.PHONY: validate +validate: + test 0 -eq $$(git status --porcelain | wc -l) || (echo "Base folder contains unknown artifacts" >&2 && git --no-pager diff && exit 1) + .PHONY: quay-login quay-login: @echo "## Login to quay.io..." From 97c19acc0d00180693a0ed01b75bd12d54df4e68 Mon Sep 17 00:00:00 2001 From: Isaac Date: Fri, 18 Oct 2024 11:58:50 +0200 Subject: [PATCH 43/43] Add minor comment --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index cbeb0d69..1af5516b 100644 --- a/Makefile +++ b/Makefile @@ -63,6 +63,7 @@ test-coverage: go-mod-update ## Used by CI pipeline ci/prow/validate .PHONY: validate validate: + @echo "## Perform validation that the folder does not contain extra artifacts..." test 0 -eq $$(git status --porcelain | wc -l) || (echo "Base folder contains unknown artifacts" >&2 && git --no-pager diff && exit 1) .PHONY: quay-login