From e227a5cb0fe706306c29ba8cae89bf244e1dea3d Mon Sep 17 00:00:00 2001 From: Christoph Hamsen Date: Sat, 4 Mar 2023 14:26:42 +0100 Subject: [PATCH] ci: merge upgrade test into integration script --- .../workflows/.reusable-integration-test.yml | 86 +------------------ tests/integration/cases.yaml | 30 +++---- tests/integration/integration-test.sh | 66 +++++++++----- 3 files changed, 62 insertions(+), 120 deletions(-) diff --git a/.github/workflows/.reusable-integration-test.yml b/.github/workflows/.reusable-integration-test.yml index ba15217da..21eab7150 100644 --- a/.github/workflows/.reusable-integration-test.yml +++ b/.github/workflows/.reusable-integration-test.yml @@ -49,6 +49,7 @@ jobs: "pre-config", "other-ns", "configured-cert", + "upgrade", ] services: alerting-endpoint: @@ -173,88 +174,3 @@ jobs: if: failure() run: | kubectl logs -n connaisseur -lapp.kubernetes.io/name=connaisseur --prefix=true - - upgrade-test: - runs-on: ubuntu-latest - if: inputs.skip_integration_tests != 'non-required' - permissions: - packages: read - env: - IMAGE: ${{ inputs.build_image }} - COSIGN_PUBLIC_KEY: ${{ inputs.cosign_public_key }} - steps: - - name: Checkout code - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 - - name: Login with registry - uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v2.1.0 - with: - registry: ${{ inputs.build_registry }} - username: ${{ inputs.repo_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Install yq and bash - run: | - sudo snap install yq - - uses: ./.github/actions/k8s-version-config - name: Setup k8s cluster - with: - k8s-version: v1.25 - - name: Configure Cluster - run: | - kubectl create ns connaisseur - kubectl create secret generic ${IMAGEPULLSECRET} \ - --from-file=.dockerconfigjson=$HOME/.docker/config.json \ - --type=kubernetes.io/dockerconfigjson \ - --namespace=connaisseur - - name: Checkout code (master) - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 - with: - ref: "master" - - name: Checkout required current files - uses: Bhacaz/checkout-files@e3e34e7daef91a5f237485bb88a260aee4be29dd # v2 - with: - files: tests/integration/ghcr-values.yaml - branch: ${{ github.head_ref || github.ref_name }} - - name: Configure Connaisseur (master) - run: | - COSIGN_PUBLIC_KEY="$(printf -- "${COSIGN_PUBLIC_KEY//
/\\n }")" - envsubst < tests/integration/ghcr-values.yaml > update - yq '. *+ load("update")' -i helm/values.yaml - rm update - yq e '.' helm/values.yaml - - name: Install Connaisseur (master) - run: | - make install - - name: Get image name & version (master) - run: | - kubectl get pods -n connaisseur -o jsonpath="{.items[*].spec.containers[*].image}" - - name: Run integration tests (master) - run: | - bash tests/integration/upgrade-integration-test.sh - - name: Display k8s logs if integration test failed - if: failure() - run: | - kubectl logs -n connaisseur -lapp.kubernetes.io/name=connaisseur --prefix=true - - name: Checkout code - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 - - name: Configure current version - run: | - envsubst < tests/integration/var-img.yaml > update - yq eval-all --inplace 'select(fileIndex == 0) * select(fileIndex == 1)' helm/values.yaml update - yq e '.' helm/values.yaml - rm update - - name: Upgrade Connaisseur to current branch - run: | - make upgrade - - name: Get image name & version (branch) - run: | - kubectl get pods -n connaisseur -o jsonpath="{.items[*].spec.containers[*].image}" - - name: Run integration tests (branch) - run: | - bash tests/integration/upgrade-integration-test.sh - - name: Display k8s logs if integration test failed - if: failure() - run: | - kubectl logs -n connaisseur -lapp.kubernetes.io/name=connaisseur --prefix=true - - name: Uninstall Connaisseur - run: | - make uninstall diff --git a/tests/integration/cases.yaml b/tests/integration/cases.yaml index 939c31373..253a61d15 100644 --- a/tests/integration/cases.yaml +++ b/tests/integration/cases.yaml @@ -19,14 +19,14 @@ test_cases: type: deploy ref: securesystemsengineering/testimage:signed namespace: default - expected_msg: pod/pod-rs created + expected_msg: pod/pod-rs-${RAND} created expected_result: VALID - id: rsds txt: Testing signed image with designated signer... type: deploy ref: securesystemsengineering/testimage:special_sig namespace: default - expected_msg: pod/pod-rsds created + expected_msg: pod/pod-rsds-${RAND} created expected_result: VALID - id: rsmds txt: Testing image with missing designated signer... @@ -61,7 +61,7 @@ test_cases: type: deploy ref: securesystemsengineering/testimage:signed@sha256:c5327b291d702719a26c6cf8cc93f72e7902df46547106a9930feda2c002a4a7 namespace: default - expected_msg: pod/pod-rstd created + expected_msg: pod/pod-rstd-${RAND} created expected_result: VALID cosign: - id: cu @@ -83,7 +83,7 @@ test_cases: type: deploy ref: securesystemsengineering/testimage:co-signed namespace: default - expected_msg: pod/pod-cs created + expected_msg: pod/pod-cs-${RAND} created expected_result: null - id: cstd txt: Testing signed cosign image with tag and digest... @@ -102,7 +102,7 @@ test_cases: type: deploy ref: securesystemsengineering/testimage:multi-cosigned-alice-bob-charlie namespace: default - expected_msg: pod/pod-mc-s created + expected_msg: pod/pod-mc-s-${RAND} created expected_result: null - id: mct2-u txt: Testing multi-cosigned image `threshold` => 2, not reached... @@ -116,7 +116,7 @@ test_cases: type: deploy ref: securesystemsengineering/testimage:multi-cosigned-bob-charlie namespace: default - expected_msg: pod/pod-mct2-s created + expected_msg: pod/pod-mct2-s-${RAND} created expected_result: null - id: mcr-u txt: Testing multi-cosigned image `required` signers => ['alice', 'charlie'], not reached... @@ -130,7 +130,7 @@ test_cases: type: deploy ref: securesystemsengineering/testimage:multi-cosigned-charlie-alice namespace: default - expected_msg: pod/pod-mcr-s created + expected_msg: pod/pod-mcr-s-${RAND} created expected_result: null rekor-cosigned: - id: rcu @@ -152,7 +152,7 @@ test_cases: type: deploy ref: securesystemsengineering/testimage:rekor-cosigned-tl namespace: default - expected_msg: pod/pod-rcstl created + expected_msg: pod/pod-rcstl-${RAND} created expected_result: null ignore-namespace-val: - id: iuu @@ -167,14 +167,14 @@ test_cases: type: deploy ref: securesystemsengineering/testimage:signed namespace: default - expected_msg: pod/pod-isu created + expected_msg: pod/pod-isu-${RAND} created expected_result: null - id: iui txt: Testing unsigned image in ignored namespace... type: deploy ref: securesystemsengineering/testimage:unsigned namespace: ignoredns - expected_msg: pod/pod-iui created + expected_msg: pod/pod-iui-${RAND} created expected_result: null validate-namespace-val: - id: vue @@ -189,14 +189,14 @@ test_cases: type: deploy ref: securesystemsengineering/testimage:signed namespace: validatedns - expected_msg: pod/pod-vse created + expected_msg: pod/pod-vse-${RAND} created expected_result: null - id: vuu txt: Testing unsigned image in unlabelled namespace... type: deploy ref: securesystemsengineering/testimage:unsigned namespace: default - expected_msg: pod/pod-vuu created + expected_msg: pod/pod-vuu-${RAND} created expected_result: null deployment: - id: d1s @@ -275,7 +275,7 @@ test_cases: type: deploy ref: securesystemsengineering/testimage:signed namespace: default - expected_msg: pod/pod-pnv1s created + expected_msg: pod/pod-pnv1s-${RAND} created expected_result: null - id: poff txt: Testing signed official docker image... @@ -283,7 +283,7 @@ test_cases: # choose official image that doesn't exit, so we can check ready status ref: docker.io/library/nginx namespace: default - expected_msg: pod/pod-poff created + expected_msg: pod/pod-poff-${RAND} created expected_result: null certificate: - id: x509u @@ -298,5 +298,5 @@ test_cases: type: deploy ref: securesystemsengineering/testimage:signed namespace: default - expected_msg: pod/pod-x509s created + expected_msg: pod/pod-x509s-${RAND} created expected_result: VALID diff --git a/tests/integration/integration-test.sh b/tests/integration/integration-test.sh index af9a9fefe..dbc13839b 100755 --- a/tests/integration/integration-test.sh +++ b/tests/integration/integration-test.sh @@ -21,21 +21,24 @@ COSIGN_PUBLIC_KEY="$(printf -- "${COSIGN_PUBLIC_KEY//
/\\n }")" ## Join ghcr integration yaml if [[ -n "${IMAGE+x}" && -n "${IMAGEPULLSECRET+x}" ]]; then - yq '. *+ load("tests/integration/var-img.yaml")' tests/integration/ghcr-values.yaml > ghcr-tmp - envsubst < ghcr-tmp > ghcr-values + yq '. *+ load("tests/integration/var-img.yaml")' tests/integration/ghcr-values.yaml >ghcr-tmp + envsubst ghcr-values + envsubst ghcr-validator rm ghcr-tmp else - echo "" > ghcr-values + echo "" >ghcr-values fi ### SINGLE TEST CASE #################################### single_test() { # ID TXT TYP REF NS MSG RES echo -n "[$1] $2" - i=0 # intialize iterator - while : ; do - i=$((i+1)) + i=0 # intialize iterator + export RAND=$(head -c 5 /dev/urandom | hexdump -ve '1/1 "%.2x"') # creating a random index to label the pods and avoid name collision for repeated runs + MSG=$(envsubst <<<"$6") # in case RAND is to be used, it needs to be added as ${RAND} to cases.yaml (and maybe deployment file) + while :; do + i=$((i + 1)) if [[ "$3" == "deploy" ]]; then - kubectl run pod-$1 --image="$4" --namespace="$5" -luse="connaisseur-integration-test" >output.log 2>&1 || true + kubectl run pod-$1-${RAND} --image="$4" --namespace="$5" -luse="connaisseur-integration-test" >output.log 2>&1 || true elif [[ "$3" == "workload" ]]; then envsubst output.log 2>&1 || true else @@ -44,7 +47,7 @@ single_test() { # ID TXT TYP REF NS MSG RES # if the webhook couldn't be called, try again. [[ ("$(cat output.log)" =~ "failed calling webhook") && $i -lt $RETRY ]] || break done - if [[ ! "$(cat output.log)" =~ "$6" ]]; then + if [[ ! "$(cat output.log)" =~ "${MSG}" ]]; then echo -e ${FAILED} echo "::group::Output" cat output.log @@ -61,7 +64,7 @@ single_test() { # ID TXT TYP REF NS MSG RES fi # 3 tries on first test, 2 tries on second, 1 try for all subsequential - RETRY=$((RETRY-1)) + RETRY=$((RETRY - 1)) } ### MULTI TEST CASE FROM FILE #################################### @@ -115,12 +118,12 @@ workload_test() { # WORKLOAD_KIND # NAME READY UP-TO-DATE AVAILABLE AGE # coredns 2/2 2 2 177d STATUSES=$(kubectl get ${KIND}) - NUMBER_UNREADY=$(( $(echo "${STATUSES}" | awk '{print $2}' | awk -F "/" '$1!=$2 { print $0 }' | wc -l) - 1)) # Preserving header row for better readability + NUMBER_UNREADY=$(($(echo "${STATUSES}" | awk '{print $2}' | awk -F "/" '$1!=$2 { print $0 }' | wc -l) - 1)) # Preserving header row for better readability elif [[ "${KIND}" == "ReplicaSet" || "${KIND}" == "DaemonSet" || "${KIND}" == "ReplicationController" ]]; then # NAME DESIRED CURRENT READY AGE # coredns-558bd4d5db 2 2 2 177d STATUSES=$(kubectl get ${KIND} | awk '$2!=$4 {print $0}') - NUMBER_UNREADY=$(( $(echo "${STATUSES}" | wc -l) - 1 )) # Preserving header row for better readability + NUMBER_UNREADY=$(($(echo "${STATUSES}" | wc -l) - 1)) # Preserving header row for better readability elif [[ "${KIND}" == "Job" || "${KIND}" == "CronJob" ]]; then # NAME COMPLETIONS DURATION AGE # cronjob-signed-1674474300 0/1 30s 30s @@ -205,6 +208,21 @@ create_imagepullsecret_in_ns() { # NAMESPACE # CREATE } ### INSTALLING CONNAISSEUR #################################### +helm_install_release() { # NAMESPACE + create_imagepullsecret_in_ns ${1} + echo -n "Installing released Connaisseur..." + helm repo add connaisseur https://sse-secure-systems.github.io/connaisseur/charts + helm show values connaisseur/connaisseur >release.yaml + yq '. *+ load("ghcr-validator")' release.yaml >release_patched.yaml + helm install connaisseur connaisseur/connaisseur --atomic \ + --namespace ${1} --values release_patched.yaml >/dev/null || { + echo -e "${FAILED}" + exit 1 + } + echo -e "${SUCCESS}" + sleep ${TIMEOUT} +} + make_install() { create_imagepullsecret_in_ns "connaisseur" echo -n "Installing Connaisseur..." @@ -288,8 +306,7 @@ make_uninstall() { helm_uninstall() { echo -n 'Uninstalling Connaisseur...' - helm uninstall connaisseur -n connaisseur >/dev/null && - kubectl delete ns connaisseur >/dev/null || { + helm uninstall connaisseur -n connaisseur >/dev/null || { echo -e "${FAILED}" exit 1 } @@ -307,22 +324,22 @@ update_values_minimal() { } update_via_env_vars() { - envsubst < tests/integration/update.yaml > update + envsubst update yq '. *+ load("ghcr-values")' -i update yq eval-all --inplace 'select(fileIndex == 0) * select(fileIndex == 1)' helm/values.yaml update rm update } update_helm_for_workloads() { - envsubst < tests/integration/update-for-workloads.yaml > update + envsubst update yq '. *+ load("ghcr-values")' -i update yq eval-all --inplace 'select(fileIndex == 0) * select(fileIndex == 1)' helm/values.yaml update rm update } -debug_vaules() { +debug_vaules() { #PATH echo "::group::values.yaml" - cat helm/values.yaml + cat "$1" echo "::endgroup::" } @@ -332,7 +349,8 @@ regular_int_test() { ### EDGE CASE TAG IN RELEASES AND TARGETS #################################### echo -n "[edge1] Testing edge case of tag defined in both targets and release json file..." - DEPLOYED_SHA=$(kubectl get pod pod-rs -o yaml | yq e '.spec.containers[0].image' - | sed 's/.*sha256://') + POD=$(kubectl get pods -o name | grep "pod-rs-") + DEPLOYED_SHA=$(kubectl get "${POD}" -o yaml | yq e '.spec.containers[0].image' - | sed 's/.*sha256://') if [[ "${DEPLOYED_SHA}" != 'c5327b291d702719a26c6cf8cc93f72e7902df46547106a9930feda2c002a4a7' ]]; then echo -e "${FAILED}" EXIT="1" @@ -520,13 +538,21 @@ case $1 in update_via_env_vars make_install certificate_int_test - # Clean up such that next test doesn't run into existing test pods - kubectl delete pods -luse="connaisseur-integration-test" -A >/dev/null yq eval-all --inplace 'select(fileIndex == 0) * select(fileIndex == 1)' helm/values.yaml tests/integration/update-cert.yaml make_upgrade certificate_check certificate_int_test ;; +"upgrade") + helm_install_release "connaisseur" + debug_vaules "release_patched.yaml" + pre_config_int_test + update_values_minimal + debug_vaules "helm/values.yaml" + helm_upgrade_namespace "connaisseur" + pre_config_int_test + helm_uninstall + ;; *) echo "Invalid test case. Exiting..." exit 1