From fc683b129de7d3613bf6da7dc1ecebfd35e48c66 Mon Sep 17 00:00:00 2001 From: Todd Short Date: Mon, 24 Nov 2025 11:24:39 -0500 Subject: [PATCH] Add upgrade e2es for standard to experimental Signed-off-by: Todd Short --- .github/workflows/e2e.yaml | 130 +++++++++++++------------------------ Makefile | 67 ++++++++++++++----- 2 files changed, 95 insertions(+), 102 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 9977fe590..6055cd859 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -9,105 +9,67 @@ on: - main jobs: - extension-developer-e2e: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - - uses: actions/setup-go@v6 - with: - go-version-file: go.mod - - - name: Run the extension developer e2e test - run: make test-extension-developer-e2e - - e2e-kind: + e2e: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + test: + - name: extension-developer-e2e + make-target: test-extension-developer-e2e + use-artifacts: false + use-codecov: false + - name: e2e + make-target: test-e2e + use-artifacts: true + use-codecov: true + - name: experimental-e2e + make-target: test-experimental-e2e + use-artifacts: true + use-codecov: true + - name: upgrade-st2st-e2e + make-target: test-upgrade-st2st-e2e + use-artifacts: true + use-codecov: false + - name: upgrade-ex2ex-e2e + make-target: test-upgrade-ex2ex-e2e + use-artifacts: true + use-codecov: false + - name: upgrade-st2ex-e2e + make-target: test-upgrade-st2ex-e2e + use-artifacts: true + use-codecov: false + - name: st2ex-e2e + make-target: test-st2ex-e2e + use-artifacts: true + use-codecov: false + name: ${{ matrix.test.name }} steps: - uses: actions/checkout@v6 - with: - fetch-depth: 0 - uses: actions/setup-go@v6 with: go-version-file: go.mod - - name: Run e2e tests - run: ARTIFACT_PATH=/tmp/artifacts E2E_SUMMARY_OUTPUT=$GITHUB_STEP_SUMMARY make test-e2e + - name: Run e2e test + run: | + if [[ "${{ matrix.test.use-artifacts }}" == "true" ]]; then + ARTIFACT_PATH=/tmp/artifacts E2E_SUMMARY_OUTPUT=$GITHUB_STEP_SUMMARY make ${{ matrix.test.make-target }} + else + make ${{ matrix.test.make-target }} + fi - uses: actions/upload-artifact@v5 - if: failure() + if: failure() && matrix.test.use-artifacts == true with: - name: e2e-artifacts + name: ${{ matrix.test.name }}-artifacts path: /tmp/artifacts/ - uses: codecov/codecov-action@v5.5.1 + if: matrix.test.use-codecov == true with: disable_search: true - files: coverage/e2e.out - flags: e2e + files: coverage/${{ matrix.test.name }}.out + flags: ${{ matrix.test.name }} token: ${{ secrets.CODECOV_TOKEN }} - experimental-e2e: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - - uses: actions/setup-go@v6 - with: - go-version-file: go.mod - - - name: Run e2e tests - run: ARTIFACT_PATH=/tmp/artifacts E2E_SUMMARY_OUTPUT=$GITHUB_STEP_SUMMARY make test-experimental-e2e - - - uses: actions/upload-artifact@v5 - if: failure() - with: - name: experimental-e2e-artifacts - path: /tmp/artifacts/ - - - uses: codecov/codecov-action@v5.5.1 - with: - disable_search: true - files: coverage/experimental-e2e.out - flags: experimental-e2e - token: ${{ secrets.CODECOV_TOKEN }} - - upgrade-e2e: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - - uses: actions/setup-go@v6 - with: - go-version-file: go.mod - - - name: Run the upgrade e2e test - run: ARTIFACT_PATH=/tmp/artifacts make test-upgrade-e2e - - - uses: actions/upload-artifact@v5 - if: failure() - with: - name: upgrade-e2e-artifacts - path: /tmp/artifacts/ - - upgrade-experimental-e2e: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - - uses: actions/setup-go@v6 - with: - go-version-file: go.mod - - - name: Run the upgrade e2e test - run: ARTIFACT_PATH=/tmp/artifacts make test-upgrade-experimental-e2e - - - uses: actions/upload-artifact@v5 - if: failure() - with: - name: upgrade-experimental-e2e-artifacts - path: /tmp/artifacts/ - diff --git a/Makefile b/Makefile index c91f952b7..c7d9ea5cf 100644 --- a/Makefile +++ b/Makefile @@ -303,6 +303,7 @@ test-extension-developer-e2e: run-internal image-registry extension-developer-e2 .PHONY: run-latest-release run-latest-release: + @echo -e "\n\U23EC Using $(RELEASE_INSTALL) as release installer\n" curl -L -s https://github.com/operator-framework/operator-controller/releases/latest/download/$(notdir $(RELEASE_INSTALL)) | bash -s .PHONY: pre-upgrade-setup @@ -316,24 +317,41 @@ post-upgrade-checks: TEST_UPGRADE_E2E_TASKS := kind-cluster run-latest-release image-registry pre-upgrade-setup docker-build kind-load kind-deploy post-upgrade-checks kind-clean -.PHONY: test-upgrade-e2e -test-upgrade-e2e: SOURCE_MANIFEST := $(STANDARD_MANIFEST) -test-upgrade-e2e: RELEASE_INSTALL := $(STANDARD_RELEASE_INSTALL) -test-upgrade-e2e: KIND_CLUSTER_NAME := operator-controller-upgrade-e2e -test-upgrade-e2e: export MANIFEST := $(STANDARD_RELEASE_MANIFEST) -test-upgrade-e2e: export TEST_CLUSTER_CATALOG_NAME := test-catalog -test-upgrade-e2e: export TEST_CLUSTER_EXTENSION_NAME := test-package -test-upgrade-e2e: $(TEST_UPGRADE_E2E_TASKS) #HELP Run upgrade e2e tests on a local kind cluster - -.PHONY: test-upgrade-experimental-e2e -test-upgrade-experimental-e2e: SOURCE_MANIFEST := $(EXPERIMENTAL_MANIFEST) -test-upgrade-experimental-e2e: RELEASE_INSTALL := $(EXPERIMENTAL_RELEASE_INSTALL) -test-upgrade-experimental-e2e: KIND_CLUSTER_NAME := operator-controller-upgrade-experimental-e2e -test-upgrade-experimental-e2e: export MANIFEST := $(EXPERIMENTAL_RELEASE_MANIFEST) -test-upgrade-experimental-e2e: export TEST_CLUSTER_CATALOG_NAME := test-catalog -test-upgrade-experimental-e2e: export TEST_CLUSTER_EXTENSION_NAME := test-package -test-upgrade-experimental-e2e: $(TEST_UPGRADE_E2E_TASKS) #HELP Run upgrade e2e tests on a local kind cluster - +.PHONY: test-upgrade-st2st-e2e +test-upgrade-st2st-e2e: SOURCE_MANIFEST := $(STANDARD_MANIFEST) +test-upgrade-st2st-e2e: RELEASE_INSTALL := $(STANDARD_RELEASE_INSTALL) +test-upgrade-st2st-e2e: KIND_CLUSTER_NAME := operator-controller-upgrade-st2st-e2e +test-upgrade-st2st-e2e: export MANIFEST := $(STANDARD_RELEASE_MANIFEST) +test-upgrade-st2st-e2e: export TEST_CLUSTER_CATALOG_NAME := test-catalog +test-upgrade-st2st-e2e: export TEST_CLUSTER_EXTENSION_NAME := test-package +test-upgrade-st2st-e2e: $(TEST_UPGRADE_E2E_TASKS) #HELP Run upgrade (standard -> standard) e2e tests on a local kind cluster + +.PHONY: test-upgrade-ex2ex-e2e +test-upgrade-ex2ex-e2e: SOURCE_MANIFEST := $(EXPERIMENTAL_MANIFEST) +test-upgrade-ex2ex-e2e: RELEASE_INSTALL := $(EXPERIMENTAL_RELEASE_INSTALL) +test-upgrade-ex2ex-e2e: KIND_CLUSTER_NAME := operator-controller-upgrade-ex2ex-e2e +test-upgrade-ex2ex-e2e: export MANIFEST := $(EXPERIMENTAL_RELEASE_MANIFEST) +test-upgrade-ex2ex-e2e: export TEST_CLUSTER_CATALOG_NAME := test-catalog +test-upgrade-ex2ex-e2e: export TEST_CLUSTER_EXTENSION_NAME := test-package +test-upgrade-ex2ex-e2e: $(TEST_UPGRADE_E2E_TASKS) #HELP Run upgrade (experimental -> experimental) e2e tests on a local kind cluster + +.PHONY: test-upgrade-st2ex-e2e +test-upgrade-st2ex-e2e: SOURCE_MANIFEST := $(EXPERIMENTAL_MANIFEST) +test-upgrade-st2ex-e2e: RELEASE_INSTALL := $(STANDARD_RELEASE_INSTALL) +test-upgrade-st2ex-e2e: KIND_CLUSTER_NAME := operator-controller-upgrade-st2ex-e2e +test-upgrade-st2ex-e2e: export MANIFEST := $(EXPERIMENTAL_RELEASE_MANIFEST) +test-upgrade-st2ex-e2e: export TEST_CLUSTER_CATALOG_NAME := test-catalog +test-upgrade-st2ex-e2e: export TEST_CLUSTER_EXTENSION_NAME := test-package +test-upgrade-st2ex-e2e: $(TEST_UPGRADE_E2E_TASKS) #HELP Run upgrade (standard -> experimental) e2e tests on a local kind cluster + +.PHONY: test-st2ex-e2e +test-st2ex-e2e: SOURCE_MANIFEST := $(STANDARD_MANIFEST) +test-st2ex-e2e: RELEASE_INSTALL := $(STANDARD_RELEASE_INSTALL) +test-st2ex-e2e: KIND_CLUSTER_NAME := operator-controller-st2ex-e2e +test-st2ex-e2e: export MANIFEST := $(STANDARD_RELEASE_MANIFEST) +test-st2ex-e2e: export TEST_CLUSTER_CATALOG_NAME := test-catalog +test-st2ex-e2e: export TEST_CLUSTER_EXTENSION_NAME := test-package +test-st2ex-e2e: run-internal image-registry pre-upgrade-setup kind-deploy-experimental post-upgrade-checks kind-clean #HELP Run swichover (standard -> experimental) e2e tests on a local kind cluster .PHONY: e2e-coverage e2e-coverage: @@ -375,6 +393,18 @@ kind-deploy: manifests cp $(CATALOGS_MANIFEST) $(DEFAULT_CATALOG) envsubst '$$DEFAULT_CATALOG,$$CERT_MGR_VERSION,$$INSTALL_DEFAULT_CATALOGS,$$MANIFEST' < scripts/install.tpl.sh | bash -s +.PHONY: kind-deploy-experimental +kind-deploy-experimental: export DEFAULT_CATALOG := $(RELEASE_CATALOGS) +kind-deploy-experimental: SOURCE_MANIFEST := $(EXPERIMENTAL_MANIFEST) +kind-deploy-experimental: export MANIFEST := $(EXPERIMENTAL_RELEASE_MANIFEST) +kind-deploy-experimental: NAMESPACE := olmv1-system +# Have to be a _completely_ separate recipe, rather than having `kind-deploy` as a dependency, because `make` will think it was already built +kind-deploy-experimental: manifests + @echo -e "\n\U1F4D8 Using $(SOURCE_MANIFEST) as source manifest\n" + sed "s/cert-git-version/cert-$(VERSION)/g" $(SOURCE_MANIFEST) > $(MANIFEST) + cp $(CATALOGS_MANIFEST) $(DEFAULT_CATALOG) + envsubst '$$DEFAULT_CATALOG,$$CERT_MGR_VERSION,$$INSTALL_DEFAULT_CATALOGS,$$MANIFEST' < scripts/install.tpl.sh | bash -s + .PHONY: kind-cluster kind-cluster: $(KIND) kind-verify-versions #EXHELP Standup a kind cluster. -$(KIND) delete cluster --name $(KIND_CLUSTER_NAME) @@ -452,6 +482,7 @@ run-experimental: export MANIFEST := $(EXPERIMENTAL_RELEASE_MANIFEST) run-experimental: run-internal #HELP Build the operator-controller then deploy it with the experimental manifest into a new kind cluster. CATD_NAMESPACE := olmv1-system +.PHONY: wait wait: kubectl wait --for=condition=Available --namespace=$(CATD_NAMESPACE) deployment/catalogd-controller-manager --timeout=60s kubectl wait --for=condition=Ready --namespace=$(CATD_NAMESPACE) certificate/catalogd-service-cert # Avoid upgrade test flakes when reissuing cert