diff --git a/Makefile b/Makefile index a2d12342..1af5516b 100644 --- a/Makefile +++ b/Makefile @@ -1,26 +1,105 @@ -GOLANGCI_OPTIONAL_CONFIG = .golangci.yml +OPERATOR_NAME = deployment-validation-operator +# Image repository vars +REGISTRY_USER ?= ${QUAY_USER} +REGISTRY_TOKEN ?= ${QUAY_TOKEN} +IMAGE_REGISTRY ?= quay.io IMAGE_REPOSITORY ?= app-sre -REGISTRY_USER = $(QUAY_USER) -REGISTRY_TOKEN = $(QUAY_TOKEN) +IMAGE_NAME ?= ${OPERATOR_NAME} +OPERATOR_IMAGE = ${IMAGE_REGISTRY}/${IMAGE_REPOSITORY}/${IMAGE_NAME} + +OLM_CHANNEL ?= alpha +OLM_BUNDLE_IMAGE = ${OPERATOR_IMAGE}-bundle +OLM_CATALOG_IMAGE = ${OPERATOR_IMAGE}-catalog + +VERSION_MAJOR ?= 0 +VERSION_MINOR ?= 1 +COMMIT_COUNT=$(shell git rev-list --count HEAD) +CURRENT_COMMIT=$(shell git rev-parse --short=7 HEAD) +OPERATOR_VERSION=${VERSION_MAJOR}.${VERSION_MINOR}.${COMMIT_COUNT}-g${CURRENT_COMMIT} +OPERATOR_IMAGE_TAG ?= ${OPERATOR_VERSION} + +CONTAINER_ENGINE_CONFIG_DIR = .docker +CONTAINER_ENGINE = $(shell command -v podman 2>/dev/null || echo docker --config=$(CONTAINER_ENGINE_CONFIG_DIR)) + +.PHONY: go-mod-update +go-mod-update: + go mod vendor + +GOOS ?= linux +GOENV=GOOS=${GOOS} GOARCH=${GOARCH} CGO_ENABLED=1 +GOBUILDFLAGS=-gcflags="all=-trimpath=${GOPATH}" -asmflags="all=-trimpath=${GOPATH}" +.PHONY: go-build +go-build: go-mod-update + @echo "## Building the binary..." + ${GOENV} go build ${GOBUILDFLAGS} -o build/_output/bin/$(OPERATOR_NAME) . + +## Used by CI pipeline ci/prow/lint +GOLANGCI_OPTIONAL_CONFIG = .golangci.yml +GOLANGCI_LINT_CACHE =/tmp/golangci-cache +.PHONY: lint +lint: go-mod-update + @echo "## Running the golangci-lint tool..." + GOLANGCI_LINT_CACHE=${GOLANGCI_LINT_CACHE} golangci-lint run -c ${GOLANGCI_OPTIONAL_CONFIG} ./... + +## Used by CI pipeline ci/prow/test +TEST_TARGETS = $(shell ${GOENV} go list -e ./... | grep -E -v "/(vendor)/") +.PHONY: test +test: go-mod-update + @echo "## Running the code unit tests..." + ${GOENV} go test ${TEST_TARGETS} + +## These targets: coverage and test-coverage; are used by the CI pipeline ci/prow/coverage +.PHONY: coverage +coverage: + @echo "## Running code coverage..." + ci/codecov.sh + +TESTOPTS := +.PHONY: test-coverage +test-coverage: go-mod-update + @echo "## Running the code unit tests with coverage..." + ${GOENV} go test ${TESTOPTS} ${TEST_TARGETS} -# This include must go below the above definitions -include boilerplate/generated-includes.mk +## Used by CI pipeline ci/prow/validate +.PHONY: validate +validate: + @echo "## Perform validation that the folder does not contain extra artifacts..." + test 0 -eq $$(git status --porcelain | wc -l) || (echo "Base folder contains unknown artifacts" >&2 && git --no-pager diff && exit 1) -OPERATOR_IMAGE_URI_TEST = $(IMAGE_REGISTRY)/$(IMAGE_REPOSITORY)/$(IMAGE_NAME):test +.PHONY: quay-login +quay-login: + @echo "## Login to quay.io..." + mkdir -p ${CONTAINER_ENGINE_CONFIG_DIR} + export REGISTRY_AUTH_FILE=${CONTAINER_ENGINE_CONFIG_DIR}/config.json + @${CONTAINER_ENGINE} login -u="${REGISTRY_USER}" -p="${REGISTRY_TOKEN}" quay.io -.PHONY: boilerplate-update -boilerplate-update: - @boilerplate/update +.PHONY: docker-build +docker-build: + @echo "## Building the container image..." + ${CONTAINER_ENGINE} build --pull -f build/Dockerfile -t ${OPERATOR_IMAGE}:${OPERATOR_IMAGE_TAG} . + ${CONTAINER_ENGINE} tag ${OPERATOR_IMAGE}:${OPERATOR_IMAGE_TAG} ${OPERATOR_IMAGE}:latest -.PHONY: docker-test -docker-test: - ${CONTAINER_ENGINE} build . -f $(OPERATOR_DOCKERFILE).test -t $(OPERATOR_IMAGE_URI_TEST) - ${CONTAINER_ENGINE} run -t $(OPERATOR_IMAGE_URI_TEST) +.PHONY: docker-push +docker-push: + @echo "## Pushing the container image..." + ${CONTAINER_ENGINE} push ${OPERATOR_IMAGE}:${OPERATOR_IMAGE_TAG} + ${CONTAINER_ENGINE} push ${OPERATOR_IMAGE}:latest -.PHONY: e2e-test -e2e-test: - ginkgo run --tags e2e test/e2e/ +## This target is run by build_tag.sh script, triggered by a Jenkins job +.PHONY: docker-publish +docker-publish: quay-login docker-build docker-push -# We are early adopters of the OPM build/push process. Remove this -# override once boilerplate uses that path by default. -build-push: opm-build-push ; +## This target is run by the master branch Jenkins Job +.PHONY: build-push +build-push: docker-publish + CONTAINER_ENGINE="${CONTAINER_ENGINE}" \ + CONTAINER_ENGINE_CONFIG_DIR="${CONTAINER_ENGINE_CONFIG_DIR}" \ + CURRENT_COMMIT="${CURRENT_COMMIT}" \ + OLM_BUNDLE_IMAGE="${OLM_BUNDLE_IMAGE}" \ + OLM_CATALOG_IMAGE="${OLM_CATALOG_IMAGE}" \ + OLM_CHANNEL="${OLM_CHANNEL}" \ + OPERATOR_NAME="${OPERATOR_NAME}" \ + OPERATOR_VERSION="${OPERATOR_VERSION}" \ + OPERATOR_IMAGE="${OPERATOR_IMAGE}" \ + OPERATOR_IMAGE_TAG="${OPERATOR_IMAGE_TAG}" \ + build/build_opm_catalog.sh \ No newline at end of file diff --git a/build/Dockerfile b/build/Dockerfile index 8a370e3c..d8687a59 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -3,7 +3,7 @@ FROM quay.io/app-sre/boilerplate:image-v5.0.1 AS builder RUN mkdir -p /workdir COPY . /workdir WORKDIR /workdir -RUN make +RUN make go-build FROM registry.access.redhat.com/ubi8/ubi-minimal:8.10-1052.1724178568 diff --git a/build/build_opm_catalog.sh b/build/build_opm_catalog.sh new file mode 100755 index 00000000..8fcd5ff1 --- /dev/null +++ b/build/build_opm_catalog.sh @@ -0,0 +1,246 @@ +#!/usr/bin/env bash + +set -euo pipefail + +REPO_ROOT=$(git rev-parse --show-toplevel) +SCRIPT_BUNDLE_CONTENTS="$REPO_ROOT/hack/generate-operator-bundle-contents.py" +BASE_FOLDER="" +DIR_BUNDLE="" +DIR_EXEC="" +DIR_MANIFESTS="" + +GOOS=$(go env GOOS) +OPM_VERSION="v1.23.2" +COMMAND_OPM="" +GRPCURL_VERSION="1.7.0" +COMMAND_GRPCURL="" + +export REGISTRY_AUTH_FILE=${CONTAINER_ENGINE_CONFIG_DIR}/config.json + +OLM_BUNDLE_VERSIONS_REPO="gitlab.cee.redhat.com/service/saas-operator-versions.git" +OLM_BUNDLE_VERSIONS_REPO_FOLDER="versions_repo" +VERSIONS_FILE="deployment-validation-operator/deployment-validation-operator-versions.txt" +PREV_VERSION="" + +OLM_BUNDLE_IMAGE_VERSION="${OLM_BUNDLE_IMAGE}:g${CURRENT_COMMIT}" +OLM_BUNDLE_IMAGE_LATEST="${OLM_BUNDLE_IMAGE}:latest" + +OLM_CATALOG_IMAGE_VERSION="${OLM_CATALOG_IMAGE}:${CURRENT_COMMIT}" +OLM_CATALOG_IMAGE_LATEST="${OLM_CATALOG_IMAGE}:latest" + +function log() { + echo "$(date "+%Y-%m-%d %H:%M:%S") -- ${1}" +} + +function precheck_required_files() { + if [[ ! -x "$SCRIPT_BUNDLE_CONTENTS" ]]; then + log "The script $SCRIPT_BUNDLE_CONTENTS cannot be run. Exiting." + return 1 + fi + return 0 +} + +function prepare_temporary_folders() { + BASE_FOLDER=$(mktemp -d --suffix "-$(basename "$0")") + DIR_BUNDLE=$(mktemp -d -p "$BASE_FOLDER" bundle.XXXX) + DIR_MANIFESTS=$(mktemp -d -p "$DIR_BUNDLE" manifests.XXXX) + DIR_EXEC=$(mktemp -d -p "$BASE_FOLDER" bin.XXXX) +} + +function download_dependencies() { + cd "$DIR_EXEC" + + local opm_url="https://github.com/operator-framework/operator-registry/releases/download/$OPM_VERSION/$GOOS-amd64-opm" + curl -sfL "${opm_url}" -o opm + chmod +x opm + COMMAND_OPM="$DIR_EXEC/opm" + + local grpcurl_url="https://github.com/fullstorydev/grpcurl/releases/download/v$GRPCURL_VERSION/grpcurl_${GRPCURL_VERSION}_${GOOS}_x86_64.tar.gz" + curl -sfL "$grpcurl_url" | tar -xz -O grpcurl > "grpcurl" + chmod +x grpcurl + COMMAND_GRPCURL="$DIR_EXEC/grpcurl" + + cd ~- +} + + +function clone_versions_repo() { + local folder="$BASE_FOLDER/$OLM_BUNDLE_VERSIONS_REPO_FOLDER" + log " path: $folder" + + if [[ -n "${APP_SRE_BOT_PUSH_TOKEN:-}" ]]; then + log "Using APP_SRE_BOT_PUSH_TOKEN credentials to authenticate" + git clone "https://app:${APP_SRE_BOT_PUSH_TOKEN}@$OLM_BUNDLE_VERSIONS_REPO" "$folder" --quiet + else + git clone "https://$OLM_BUNDLE_VERSIONS_REPO" "$folder" --quiet + fi +} + +function set_previous_operator_version() { + local filename="$BASE_FOLDER/$OLM_BUNDLE_VERSIONS_REPO_FOLDER/$VERSIONS_FILE" + + if [[ ! -a "$filename" ]]; then + log "No file $VERSIONS_FILE exist. Exiting." + exit 1 + fi + PREV_VERSION=$(tail -n 1 "$filename" | awk '{print $1}') +} + +function setup_environment() { + log "Generating temporary folders to contain artifacts" + prepare_temporary_folders + log " base path: $BASE_FOLDER" + + log "Downloading needed commands: opm and grpcurl" + download_dependencies + log " path: $DIR_EXEC" + + log "Cloning $OLM_BUNDLE_VERSIONS_REPO" + clone_versions_repo + + log "Determining previous operator version checking $VERSIONS_FILE file" + set_previous_operator_version + log " previous version: $PREV_VERSION" +} + +function build_opm_bundle() { + # set venv with needed dependencies + python3 -m venv .venv; source .venv/bin/activate; pip install pyyaml + + log "Generating patched bundle contents" + $SCRIPT_BUNDLE_CONTENTS --name "$OPERATOR_NAME" \ + --current-version "$OPERATOR_VERSION" \ + --image "$OPERATOR_IMAGE" \ + --image-tag "$OPERATOR_IMAGE_TAG" \ + --output-dir "$DIR_MANIFESTS" \ + --replaces "$PREV_VERSION" + + log "Creating bundle image $OLM_BUNDLE_IMAGE_VERSION" + cd "$DIR_BUNDLE" + ${COMMAND_OPM} alpha bundle build --directory "$DIR_MANIFESTS" \ + --channels "$OLM_CHANNEL" \ + --default "$OLM_CHANNEL" \ + --package "$OPERATOR_NAME" \ + --tag "$OLM_BUNDLE_IMAGE_VERSION" \ + --image-builder "$(basename "$CONTAINER_ENGINE" | awk '{print $1}')" \ + --overwrite \ + 1>&2 + cd ~- +} + +function validate_opm_bundle() { + log "Pushing bundle image $OLM_BUNDLE_IMAGE_VERSION" + $CONTAINER_ENGINE push "$OLM_BUNDLE_IMAGE_VERSION" + + log "Validating bundle $OLM_BUNDLE_IMAGE_VERSION" + ${COMMAND_OPM} alpha bundle validate --tag "$OLM_BUNDLE_IMAGE_VERSION" \ + --image-builder "$(basename "$CONTAINER_ENGINE" | awk '{print $1}')" +} + +function build_opm_catalog() { + local FROM_INDEX="" + local PREV_COMMIT=${PREV_VERSION#*g} # remove versioning and the g commit hash prefix + # check if the previous catalog image is available + if [ "$(${CONTAINER_ENGINE} pull "${OLM_CATALOG_IMAGE}":"${PREV_COMMIT}" &> /dev/null; echo $?)" -eq 0 ]; then + FROM_INDEX="--from-index ${OLM_CATALOG_IMAGE}:${PREV_COMMIT}" + log "Index argument is $FROM_INDEX" + fi + + log "Creating catalog image $OLM_CATALOG_IMAGE_VERSION using opm" + + ${COMMAND_OPM} index add --bundles "$OLM_BUNDLE_IMAGE_VERSION" \ + --tag "$OLM_CATALOG_IMAGE_VERSION" \ + --build-tool "$(basename "$CONTAINER_ENGINE" | awk '{print $1}')" \ + $FROM_INDEX +} + +function validate_opm_catalog() { + log "Checking that catalog we have built returns the correct version $OPERATOR_VERSION" + + local free_port="" + local container_id="" + local catalog_current_version="" + + free_port=$(python3 -c 'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()') + + log "Running $OLM_CATALOG_IMAGE_VERSION and exposing $free_port" + container_id=$(${CONTAINER_ENGINE} run -d -p "$free_port:50051" "$OLM_CATALOG_IMAGE_VERSION") + + log "Getting current version from running catalog" + catalog_current_version=$( + ${COMMAND_GRPCURL} -plaintext -d '{"name": "'"$OPERATOR_NAME"'"}' \ + "localhost:$free_port" api.Registry/GetPackage | \ + jq -r '.channels[] | select(.name=="'"$OLM_CHANNEL"'") | .csvName' | \ + sed "s/$OPERATOR_NAME\.//" + ) + log " catalog version: $catalog_current_version" + + log "Removing docker container $container_id" + ${CONTAINER_ENGINE} rm -f "$container_id" + + if [[ "$catalog_current_version" != "v$OPERATOR_VERSION" ]]; then + log "Version from catalog $catalog_current_version != v$OPERATOR_VERSION" + return 1 + fi +} + +function update_versions_repo() { + log "Adding the current version $OPERATOR_VERSION to the bundle versions file in $OLM_BUNDLE_VERSIONS_REPO" + local folder="$BASE_FOLDER/$OLM_BUNDLE_VERSIONS_REPO_FOLDER" + + cd "$folder" + + echo "$OPERATOR_VERSION" >> "$VERSIONS_FILE" + git add . + message="add version $OPERATOR_VERSION + + replaces $PREV_VERSION" + git commit -m "$message" + + log "Pushing the repository changes to $OLM_BUNDLE_VERSIONS_REPO into master branch" + git push origin master + cd ~- +} + +function tag_and_push_images() { + log "Tagging bundle image $OLM_BUNDLE_IMAGE_VERSION as $OLM_BUNDLE_IMAGE_LATEST" + ${CONTAINER_ENGINE} tag "$OLM_BUNDLE_IMAGE_VERSION" "$OLM_BUNDLE_IMAGE_LATEST" + + log "Tagging catalog image $OLM_CATALOG_IMAGE_VERSION as $OLM_CATALOG_IMAGE_LATEST" + ${CONTAINER_ENGINE} tag "$OLM_CATALOG_IMAGE_VERSION" "$OLM_CATALOG_IMAGE_LATEST" + + log "Pushing catalog image $OLM_CATALOG_IMAGE_VERSION" + ${CONTAINER_ENGINE} push "$OLM_CATALOG_IMAGE_VERSION" + + log "Pushing bundle image $OLM_CATALOG_IMAGE_LATEST" + ${CONTAINER_ENGINE} push "$OLM_CATALOG_IMAGE_LATEST" + + log "Pushing bundle image $OLM_BUNDLE_IMAGE_LATEST" + ${CONTAINER_ENGINE} push "$OLM_BUNDLE_IMAGE_LATEST" +} + +function main() { + log "Building $OPERATOR_NAME version $OPERATOR_VERSION" + + precheck_required_files || return 1 + + setup_environment + + build_opm_bundle + validate_opm_bundle + + build_opm_catalog + validate_opm_catalog + + if [[ -n "${APP_SRE_BOT_PUSH_TOKEN:-}" ]]; then + update_versions_repo + else + log "APP_SRE_BOT_PUSH_TOKEN credentials were not found" + log "it will be necessary to manually update $OLM_BUNDLE_VERSIONS_REPO repo" + fi + tag_and_push_images +} + +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main +fi \ No newline at end of file diff --git a/build_tag.sh b/build_tag.sh index ea89514f..ebb278d2 100755 --- a/build_tag.sh +++ b/build_tag.sh @@ -1,9 +1,11 @@ #!/bin/bash +## This script is the entry point for the Jenkins job: deployment-validation-operator build tag +## It builds a new image and pushes it to the dv-operator repository on quay.io. make \ OPERATOR_IMAGE_TAG=$(echo $GIT_BRANCH|cut -d"/" -f3) \ IMAGE_REPOSITORY="deployment-validation-operator" \ IMAGE_NAME="dv-operator" \ REGISTRY_USER=$(echo $QUAY_USER) \ REGISTRY_TOKEN=$(echo $QUAY_TOKEN) \ -docker-push +docker-publish diff --git a/ci/codecov.sh b/ci/codecov.sh new file mode 100755 index 00000000..05e5e174 --- /dev/null +++ b/ci/codecov.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +REPO_ROOT=$(git rev-parse --show-toplevel) +CI_SERVER_URL=https://prow.svc.ci.openshift.org/view/gcs/origin-ci-test +COVER_PROFILE=${COVER_PROFILE:-coverage.out} +JOB_TYPE=${JOB_TYPE:-"local"} + +# Default concurrency to four threads. By default it's the number of procs, +# which seems to be 16 in the CI env. Some consumers' coverage jobs were +# regularly getting OOM-killed; so do this rather than boost the pod resources +# unreasonably. +COV_THREAD_COUNT=${COV_THREAD_COUNT:-4} +make -C "${REPO_ROOT}" test-coverage TESTOPTS="-coverprofile=${COVER_PROFILE}.tmp -covermode=atomic -coverpkg=./... -p ${COV_THREAD_COUNT}" + +# Remove generated files from coverage profile +grep -v "zz_generated" "${COVER_PROFILE}.tmp" > "${COVER_PROFILE}" +rm -f "${COVER_PROFILE}.tmp" + +# Configure the git refs and job link based on how the job was triggered via prow +if [[ "${JOB_TYPE}" == "presubmit" ]]; then + echo "detected PR code coverage job for #${PULL_NUMBER}" + REF_FLAGS="-P ${PULL_NUMBER} -C ${PULL_PULL_SHA}" + JOB_LINK="${CI_SERVER_URL}/pr-logs/pull/${REPO_OWNER}_${REPO_NAME}/${PULL_NUMBER}/${JOB_NAME}/${BUILD_ID}" +elif [[ "${JOB_TYPE}" == "postsubmit" ]]; then + echo "detected branch code coverage job for ${PULL_BASE_REF}" + REF_FLAGS="-B ${PULL_BASE_REF} -C ${PULL_BASE_SHA}" + JOB_LINK="${CI_SERVER_URL}/logs/${JOB_NAME}/${BUILD_ID}" +elif [[ "${JOB_TYPE}" == "local" ]]; then + echo "coverage report available at ${COVER_PROFILE}" + exit 0 +else + echo "${JOB_TYPE} jobs not supported" >&2 + exit 1 +fi + +# Configure certain internal codecov variables with values from prow. +export CI_BUILD_URL="${JOB_LINK}" +export CI_BUILD_ID="${JOB_NAME}" +export CI_JOB_ID="${BUILD_ID}" + +if [[ "${JOB_TYPE}" != "local" ]]; then + if [[ -z "${ARTIFACT_DIR:-}" ]] || [[ ! -d "${ARTIFACT_DIR}" ]] || [[ ! -w "${ARTIFACT_DIR}" ]]; then + echo '${ARTIFACT_DIR} must be set for non-local jobs, and must point to a writable directory' >&2 + exit 1 + fi + curl -sS https://codecov.io/bash -o "${ARTIFACT_DIR}/codecov.sh" + bash <(cat "${ARTIFACT_DIR}/codecov.sh") -Z -K -f "${COVER_PROFILE}" -r "${REPO_OWNER}/${REPO_NAME}" ${REF_FLAGS} +else + bash <(curl -s https://codecov.io/bash) -Z -K -f "${COVER_PROFILE}" -r "${REPO_OWNER}/${REPO_NAME}" ${REF_FLAGS} +fi