ci: merge upgrade test into integration script

.github/workflows/.reusable-integration-test.yml

@@ -49,6 +49,7 @@ jobs:
             "pre-config",
             "other-ns",
             "configured-cert",
+            "upgrade",
           ]
     services:
       alerting-endpoint:
@@ -173,88 +174,3 @@ jobs:
         if: failure()
         run: |
           kubectl logs -n connaisseur -lapp.kubernetes.io/name=connaisseur --prefix=true
-
-  upgrade-test:
-    runs-on: ubuntu-latest
-    if: inputs.skip_integration_tests != 'non-required'
-    permissions:
-      packages: read
-    env:
-      IMAGE: ${{ inputs.build_image }}
-      COSIGN_PUBLIC_KEY: ${{ inputs.cosign_public_key }}
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
-      - name: Login with registry
-        uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v2.1.0
-        with:
-          registry: ${{ inputs.build_registry }}
-          username: ${{ inputs.repo_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Install yq and bash
-        run: |
-          sudo snap install yq
-      - uses: ./.github/actions/k8s-version-config
-        name: Setup k8s cluster
-        with:
-          k8s-version: v1.25
-      - name: Configure Cluster
-        run: |
-          kubectl create ns connaisseur
-          kubectl create secret generic ${IMAGEPULLSECRET} \
-            --from-file=.dockerconfigjson=$HOME/.docker/config.json \
-            --type=kubernetes.io/dockerconfigjson \
-            --namespace=connaisseur
-      - name: Checkout code (master)
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
-        with:
-          ref: "master"
-      - name: Checkout required current files
-        uses: Bhacaz/checkout-files@e3e34e7daef91a5f237485bb88a260aee4be29dd # v2
-        with:
-          files: tests/integration/ghcr-values.yaml
-          branch: ${{ github.head_ref || github.ref_name }}
-      - name: Configure Connaisseur (master)
-        run: |
-          COSIGN_PUBLIC_KEY="$(printf -- "${COSIGN_PUBLIC_KEY//<br>/\\n          }")"
-          envsubst < tests/integration/ghcr-values.yaml > update
-          yq '. *+ load("update")' -i helm/values.yaml
-          rm update
-          yq e '.' helm/values.yaml
-      - name: Install Connaisseur (master)
-        run: |
-          make install
-      - name: Get image name & version (master)
-        run: |
-          kubectl get pods -n connaisseur -o jsonpath="{.items[*].spec.containers[*].image}"
-      - name: Run integration tests (master)
-        run: |
-          bash tests/integration/upgrade-integration-test.sh
-      - name: Display k8s logs if integration test failed
-        if: failure()
-        run: |
-          kubectl logs -n connaisseur -lapp.kubernetes.io/name=connaisseur --prefix=true
-      - name: Checkout code
-        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
-      - name: Configure current version
-        run: |
-          envsubst < tests/integration/var-img.yaml > update
-          yq eval-all --inplace 'select(fileIndex == 0) * select(fileIndex == 1)' helm/values.yaml update
-          yq e '.' helm/values.yaml
-          rm update
-      - name: Upgrade Connaisseur to current branch
-        run: |
-          make upgrade
-      - name: Get image name & version (branch)
-        run: |
-          kubectl get pods -n connaisseur -o jsonpath="{.items[*].spec.containers[*].image}"
-      - name: Run integration tests (branch)
-        run: |
-          bash tests/integration/upgrade-integration-test.sh
-      - name: Display k8s logs if integration test failed
-        if: failure()
-        run: |
-          kubectl logs -n connaisseur -lapp.kubernetes.io/name=connaisseur --prefix=true
-      - name: Uninstall Connaisseur
-        run: |
-          make uninstall

tests/integration/cases.yaml

@@ -19,14 +19,14 @@ test_cases:
     type: deploy
     ref: securesystemsengineering/testimage:signed
     namespace: default
-    expected_msg: pod/pod-rs created
+    expected_msg: pod/pod-rs-${RAND} created
     expected_result: VALID
   - id: rsds
     txt: Testing signed image with designated signer...
     type: deploy
     ref: securesystemsengineering/testimage:special_sig
     namespace: default
-    expected_msg: pod/pod-rsds created
+    expected_msg: pod/pod-rsds-${RAND} created
     expected_result: VALID
   - id: rsmds
     txt: Testing image with missing designated signer...
@@ -61,7 +61,7 @@ test_cases:
     type: deploy
     ref: securesystemsengineering/testimage:signed@sha256:c5327b291d702719a26c6cf8cc93f72e7902df46547106a9930feda2c002a4a7
     namespace: default
-    expected_msg: pod/pod-rstd created
+    expected_msg: pod/pod-rstd-${RAND} created
     expected_result: VALID
   cosign:
   - id: cu
@@ -83,7 +83,7 @@ test_cases:
     type: deploy
     ref: securesystemsengineering/testimage:co-signed
     namespace: default
-    expected_msg: pod/pod-cs created
+    expected_msg: pod/pod-cs-${RAND} created
     expected_result: null
   - id: cstd
     txt: Testing signed cosign image with tag and digest...
@@ -102,7 +102,7 @@ test_cases:
     type: deploy
     ref: securesystemsengineering/testimage:multi-cosigned-alice-bob-charlie
     namespace: default
-    expected_msg: pod/pod-mc-s created
+    expected_msg: pod/pod-mc-s-${RAND} created
     expected_result: null
   - id: mct2-u
     txt: Testing multi-cosigned image `threshold` => 2, not reached...
@@ -116,7 +116,7 @@ test_cases:
     type: deploy
     ref: securesystemsengineering/testimage:multi-cosigned-bob-charlie
     namespace: default
-    expected_msg: pod/pod-mct2-s created
+    expected_msg: pod/pod-mct2-s-${RAND} created
     expected_result: null
   - id: mcr-u
     txt: Testing multi-cosigned image `required` signers => ['alice', 'charlie'], not reached...
@@ -130,7 +130,7 @@ test_cases:
     type: deploy
     ref: securesystemsengineering/testimage:multi-cosigned-charlie-alice
     namespace: default
-    expected_msg: pod/pod-mcr-s created
+    expected_msg: pod/pod-mcr-s-${RAND} created
     expected_result: null
   rekor-cosigned:
   - id: rcu
@@ -152,7 +152,7 @@ test_cases:
     type: deploy
     ref: securesystemsengineering/testimage:rekor-cosigned-tl
     namespace: default
-    expected_msg: pod/pod-rcstl created
+    expected_msg: pod/pod-rcstl-${RAND} created
     expected_result: null
   ignore-namespace-val:
   - id: iuu
@@ -167,14 +167,14 @@ test_cases:
     type: deploy
     ref: securesystemsengineering/testimage:signed
     namespace: default
-    expected_msg: pod/pod-isu created
+    expected_msg: pod/pod-isu-${RAND} created
     expected_result: null
   - id: iui
     txt: Testing unsigned image in ignored namespace...
     type: deploy
     ref: securesystemsengineering/testimage:unsigned
     namespace: ignoredns
-    expected_msg: pod/pod-iui created
+    expected_msg: pod/pod-iui-${RAND} created
     expected_result: null
   validate-namespace-val:
   - id: vue
@@ -189,14 +189,14 @@ test_cases:
     type: deploy
     ref: securesystemsengineering/testimage:signed
     namespace: validatedns
-    expected_msg: pod/pod-vse created
+    expected_msg: pod/pod-vse-${RAND} created
     expected_result: null
   - id: vuu
     txt: Testing unsigned image in unlabelled namespace...
     type: deploy
     ref: securesystemsengineering/testimage:unsigned
     namespace: default
-    expected_msg: pod/pod-vuu created
+    expected_msg: pod/pod-vuu-${RAND} created
     expected_result: null
   deployment:
   - id: d1s
@@ -275,15 +275,15 @@ test_cases:
     type: deploy
     ref: securesystemsengineering/testimage:signed
     namespace: default
-    expected_msg: pod/pod-pnv1s created
+    expected_msg: pod/pod-pnv1s-${RAND} created
     expected_result: null
   - id: poff
     txt: Testing signed official docker image...
     type: deploy
     # choose official image that doesn't exit, so we can check ready status
     ref: docker.io/library/nginx
     namespace: default
-    expected_msg: pod/pod-poff created
+    expected_msg: pod/pod-poff-${RAND} created
     expected_result: null
   certificate:
   - id: x509u
@@ -298,5 +298,5 @@ test_cases:
     type: deploy
     ref: securesystemsengineering/testimage:signed
     namespace: default
-    expected_msg: pod/pod-x509s created
+    expected_msg: pod/pod-x509s-${RAND} created
     expected_result: VALID

tests/integration/integration-test.sh

@@ -21,21 +21,24 @@ COSIGN_PUBLIC_KEY="$(printf -- "${COSIGN_PUBLIC_KEY//<br>/\\n          }")"
 
 ## Join ghcr integration yaml
 if [[ -n "${IMAGE+x}" && -n "${IMAGEPULLSECRET+x}" ]]; then
-  yq '. *+ load("tests/integration/var-img.yaml")' tests/integration/ghcr-values.yaml > ghcr-tmp
-  envsubst < ghcr-tmp > ghcr-values
+  yq '. *+ load("tests/integration/var-img.yaml")' tests/integration/ghcr-values.yaml >ghcr-tmp
+  envsubst <ghcr-tmp >ghcr-values
+  envsubst <tests/integration/ghcr-values.yaml >ghcr-validator
   rm ghcr-tmp
 else
-  echo "" > ghcr-values
+  echo "" >ghcr-values
 fi
 
 ### SINGLE TEST CASE ####################################
 single_test() { # ID TXT TYP REF NS MSG RES
   echo -n "[$1] $2"
-  i=0 # intialize iterator
-  while : ; do
-    i=$((i+1))
+  i=0                                                              # intialize iterator
+  export RAND=$(head -c 5 /dev/urandom | hexdump -ve '1/1 "%.2x"') # creating a random index to label the pods and avoid name collision for repeated runs
+  MSG=$(envsubst <<<"$6")                                          # in case RAND is to be used, it needs to be added as ${RAND} to cases.yaml (and maybe deployment file)
+  while :; do
+    i=$((i + 1))
     if [[ "$3" == "deploy" ]]; then
-      kubectl run pod-$1 --image="$4" --namespace="$5" -luse="connaisseur-integration-test" >output.log 2>&1 || true
+      kubectl run pod-$1-${RAND} --image="$4" --namespace="$5" -luse="connaisseur-integration-test" >output.log 2>&1 || true
     elif [[ "$3" == "workload" ]]; then
       envsubst <tests/integration/workload-objects/$4.yaml | kubectl apply -f - >output.log 2>&1 || true
     else
@@ -44,7 +47,7 @@ single_test() { # ID TXT TYP REF NS MSG RES
     # if the webhook couldn't be called, try again.
     [[ ("$(cat output.log)" =~ "failed calling webhook") && $i -lt $RETRY ]] || break
   done
-  if [[ ! "$(cat output.log)" =~ "$6" ]]; then
+  if [[ ! "$(cat output.log)" =~ "${MSG}" ]]; then
     echo -e ${FAILED}
     echo "::group::Output"
     cat output.log
@@ -61,7 +64,7 @@ single_test() { # ID TXT TYP REF NS MSG RES
   fi
 
   # 3 tries on first test, 2 tries on second, 1 try for all subsequential
-  RETRY=$((RETRY-1))
+  RETRY=$((RETRY - 1))
 }
 
 ### MULTI TEST CASE FROM FILE ####################################
@@ -115,12 +118,12 @@ workload_test() { # WORKLOAD_KIND
       # NAME                     READY   UP-TO-DATE   AVAILABLE   AGE
       # coredns                  2/2     2            2           177d
       STATUSES=$(kubectl get ${KIND})
-      NUMBER_UNREADY=$(( $(echo "${STATUSES}" | awk '{print $2}' | awk -F "/" '$1!=$2 { print $0 }' | wc -l) - 1)) # Preserving header row for better readability
+      NUMBER_UNREADY=$(($(echo "${STATUSES}" | awk '{print $2}' | awk -F "/" '$1!=$2 { print $0 }' | wc -l) - 1)) # Preserving header row for better readability
     elif [[ "${KIND}" == "ReplicaSet" || "${KIND}" == "DaemonSet" || "${KIND}" == "ReplicationController" ]]; then
       # NAME                 DESIRED   CURRENT   READY   AGE
       # coredns-558bd4d5db   2         2         2       177d
       STATUSES=$(kubectl get ${KIND} | awk '$2!=$4 {print $0}')
-      NUMBER_UNREADY=$(( $(echo "${STATUSES}" | wc -l) - 1 )) # Preserving header row for better readability
+      NUMBER_UNREADY=$(($(echo "${STATUSES}" | wc -l) - 1)) # Preserving header row for better readability
     elif [[ "${KIND}" == "Job" || "${KIND}" == "CronJob" ]]; then
       # NAME                        COMPLETIONS   DURATION   AGE
       # cronjob-signed-1674474300   0/1           30s        30s
@@ -205,6 +208,21 @@ create_imagepullsecret_in_ns() { # NAMESPACE # CREATE
 }
 
 ### INSTALLING CONNAISSEUR ####################################
+helm_install_release() { # NAMESPACE
+  create_imagepullsecret_in_ns ${1}
+  echo -n "Installing released Connaisseur..."
+  helm repo add connaisseur https://sse-secure-systems.github.io/connaisseur/charts
+  helm show values connaisseur/connaisseur >release.yaml
+  yq '. *+ load("ghcr-validator")' release.yaml >release_patched.yaml
+  helm install connaisseur connaisseur/connaisseur --atomic \
+    --namespace ${1} --values release_patched.yaml >/dev/null || {
+    echo -e "${FAILED}"
+    exit 1
+  }
+  echo -e "${SUCCESS}"
+  sleep ${TIMEOUT}
+}
+
 make_install() {
   create_imagepullsecret_in_ns "connaisseur"
   echo -n "Installing Connaisseur..."
@@ -288,8 +306,7 @@ make_uninstall() {
 
 helm_uninstall() {
   echo -n 'Uninstalling Connaisseur...'
-  helm uninstall connaisseur -n connaisseur >/dev/null &&
-    kubectl delete ns connaisseur >/dev/null || {
+  helm uninstall connaisseur -n connaisseur >/dev/null || {
     echo -e "${FAILED}"
     exit 1
   }
@@ -307,22 +324,22 @@ update_values_minimal() {
 }
 
 update_via_env_vars() {
-  envsubst < tests/integration/update.yaml > update
+  envsubst <tests/integration/update.yaml >update
   yq '. *+ load("ghcr-values")' -i update
   yq eval-all --inplace 'select(fileIndex == 0) * select(fileIndex == 1)' helm/values.yaml update
   rm update
 }
 
 update_helm_for_workloads() {
-  envsubst < tests/integration/update-for-workloads.yaml > update
+  envsubst <tests/integration/update-for-workloads.yaml >update
   yq '. *+ load("ghcr-values")' -i update
   yq eval-all --inplace 'select(fileIndex == 0) * select(fileIndex == 1)' helm/values.yaml update
   rm update
 }
 
-debug_vaules() {
+debug_vaules() { #PATH
   echo "::group::values.yaml"
-  cat helm/values.yaml
+  cat "$1"
   echo "::endgroup::"
 }
 
@@ -332,7 +349,8 @@ regular_int_test() {
 
   ### EDGE CASE TAG IN RELEASES AND TARGETS ####################################
   echo -n "[edge1] Testing edge case of tag defined in both targets and release json file..."
-  DEPLOYED_SHA=$(kubectl get pod pod-rs -o yaml | yq e '.spec.containers[0].image' - | sed 's/.*sha256://')
+  POD=$(kubectl get pods -o name | grep "pod-rs-")
+  DEPLOYED_SHA=$(kubectl get "${POD}" -o yaml | yq e '.spec.containers[0].image' - | sed 's/.*sha256://')
   if [[ "${DEPLOYED_SHA}" != 'c5327b291d702719a26c6cf8cc93f72e7902df46547106a9930feda2c002a4a7' ]]; then
     echo -e "${FAILED}"
     EXIT="1"
@@ -520,13 +538,21 @@ case $1 in
   update_via_env_vars
   make_install
   certificate_int_test
-  # Clean up such that next test doesn't run into existing test pods
-  kubectl delete pods -luse="connaisseur-integration-test" -A >/dev/null
   yq eval-all --inplace 'select(fileIndex == 0) * select(fileIndex == 1)' helm/values.yaml tests/integration/update-cert.yaml
   make_upgrade
   certificate_check
   certificate_int_test
   ;;
+"upgrade")
+  helm_install_release "connaisseur"
+  debug_vaules "release_patched.yaml"
+  pre_config_int_test
+  update_values_minimal
+  debug_vaules "helm/values.yaml"
+  helm_upgrade_namespace "connaisseur"
+  pre_config_int_test
+  helm_uninstall
+  ;;
 *)
   echo "Invalid test case. Exiting..."
   exit 1