fix helm upgrade (#1201)

Signed-off-by: Sertac Ozercan <sozercan@gmail.com>

Co-authored-by: Rita Zhang <rita.z.zhang@gmail.com>

.github/workflows/upgrade.yaml

@@ -9,7 +9,7 @@ on:
       - "manifest_staging/charts/**"
 
 env:
-  BASE_RELEASE: 3.1.1
+  BASE_RELEASE: 3.2.3
 
 jobs:
   helm_upgrade:
@@ -35,14 +35,24 @@ jobs:
 
       - name: Run e2e after install
         run: |
-          make test-e2e
+          # running tests matching the base release
+          git clone https://github.com/open-policy-agent/gatekeeper.git -b v${{ env.BASE_RELEASE }} --sparse --depth 1 /tmp/gatekeeper-tests-v${{ env.BASE_RELEASE }}
+          cd /tmp/gatekeeper-tests-v${{ env.BASE_RELEASE }}
+          git sparse-checkout set test/bats
+          bats -t test/bats/test.bats
 
       - name: Save logs
         if: ${{ always() }}
         run: |
           kubectl logs -n gatekeeper-system -l control-plane=controller-manager --tail=-1 > logs-${{ matrix.HELM_VERSION }}-controller-pre-upgrade.json
           kubectl logs -n gatekeeper-system -l control-plane=audit-controller --tail=-1 > logs-${{ matrix.HELM_VERSION }}-audit-pre-upgrade.json
 
+      - name: Cleanup
+        run: |
+          kubectl delete ns gatekeeper-test-playground gatekeeper-excluded-namespace || true
+          kubectl delete constrainttemplates --all || true
+          kubectl delete configs --all || true
+
       - name: Upgrade Helm version
         run: |
           make e2e-build-load-image IMG=gatekeeper-e2e-helm:latest

Makefile

@@ -13,6 +13,7 @@ KIND_VERSION ?= 0.8.1
 KUBERNETES_VERSION ?= v1.19.0
 KUSTOMIZE_VERSION ?= 3.8.8
 BATS_VERSION ?= 1.2.1
+BATS_TESTS_FILE ?= test/bats/test.bats
 KUBECTL_KUSTOMIZE_VERSION ?= 1.20.1-${KUSTOMIZE_VERSION}
 HELM_VERSION ?= 2.17.0
 HELM_ARGS ?=
@@ -92,7 +93,7 @@ test:
 	docker build --pull .staging/test -t gatekeeper-test && docker run -t gatekeeper-test
 
 test-e2e:
-	bats -t test/bats/test.bats
+	bats -t ${BATS_TESTS_FILE}
 
 e2e-bootstrap:
 	# Download and install kind

test/bats/test.bats

@@ -2,7 +2,7 @@
 
 load helpers
 
-BATS_TESTS_DIR=test/bats/tests
+BATS_TESTS_DIR=${BATS_TESTS_DIR:-test/bats/tests}
 WAIT_TIME=120
 SLEEP_TIME=1
 CLEAN_CMD="echo cleaning..."