test: re-work test strategy for integration tests (#3443)

Signed-off-by: odubajDT <ondrej.dubaj@dynatrace.com>

.github/actions/deploy-keptn-on-cluster/action.yml

@@ -18,14 +18,10 @@ inputs:
     required: false
     description: "Name of the kind cluster"
     default: "test-cluster"
-  scheduling-gates:
+  values-file:
     required: false
-    description: "Use scheduling gates instead of scheduler"
-    default: "gates_off"
-  allowed-namespaces:
-    required: false
-    description: "Decides whether to allow only certain namespaces"
-    default: "allowed_ns_off"
+    description: "Values file used for Keptn installation configuration"
+    default: "values.yaml"
   cert-manager-io-enabled:
     required: false
     description: "Decides whether to use cert-manager.io"
@@ -64,49 +60,6 @@ runs:
           kind load image-archive $image/$image -n ${{ inputs.cluster-name }}
         done
 
-    - name: Install Keptn with helm
-      if: inputs.cert-manager-io-enabled == 'cert_manager_io_off'
-      env:
-        RELEASE_REGISTRY: "localhost:5000/keptn"
-      shell: bash
-      run: |
-        echo "Installing Keptn using helm"
-        helm version
-
-        helm repo add keptn "https://charts.lifecycle.keptn.sh"
-        helm repo update
-
-        for chart_dir in ./lifecycle-operator/chart \
-                ./metrics-operator/chart \
-                ./keptn-cert-manager/chart \
-                ./chart; do
-            cd "$chart_dir"
-            echo "updating charts for" $chart_dir
-            helm dependency update
-            helm dependency build
-            cd -  # Return to the previous directory
-        done
-
-        cd .github/actions/deploy-keptn-on-cluster
-
-        if [ "${{ inputs.allowed-namespaces }}" == "allowed_ns_on" ]; then
-          echo "  allowedNamespaces: [allowed-ns-test]" >> values.yaml
-        fi
-
-        if [ "${{ inputs.scheduling-gates }}" == "gates_on" ]; then
-          echo "  schedulingGatesEnabled: true" >> values.yaml
-        fi
-
-        export TAG=${{ inputs.runtime_tag }}
-        envsubst < values.yaml > tmp.yaml
-
-        echo "installing with values.yaml file:"
-        cat tmp.yaml
-
-        helm install -n keptn-system --create-namespace keptn ../../../chart \
-          --values ./tmp.yaml \
-          --debug --wait --timeout 1m
-
     - name: Install cert-manager.io
       if: inputs.cert-manager-io-enabled == 'cert_manager_io_on'
       env:
@@ -118,8 +71,10 @@ runs:
         kubectl wait --for=condition=Available deployment/cert-manager-webhook -n cert-manager --timeout=120s
         kubectl wait --for=condition=ready pod -l app=webhook -n cert-manager --timeout=60s
 
-    - name: Install Keptn using cert-manager.io with helm
-      if: inputs.cert-manager-io-enabled == 'cert_manager_io_on'
+        kubectl create ns keptn-system
+        kubectl apply -f .github/actions/deploy-keptn-on-cluster/values/certificate.yaml
+
+    - name: Install Keptn with helm
       env:
         RELEASE_REGISTRY: "localhost:5000/keptn"
       shell: bash
@@ -132,6 +87,7 @@ runs:
 
         for chart_dir in ./lifecycle-operator/chart \
                 ./metrics-operator/chart \
+                ./keptn-cert-manager/chart \
                 ./chart; do
             cd "$chart_dir"
             echo "updating charts for" $chart_dir
@@ -142,23 +98,12 @@ runs:
 
         cd .github/actions/deploy-keptn-on-cluster
 
-        if [ "${{ inputs.allowed-namespaces }}" == "allowed_ns_on" ]; then
-          echo "  allowedNamespaces: [allowed-ns-test]" >> values-cert-manager-io.yaml
-        fi
-
-        if [ "${{ inputs.scheduling-gates }}" == "gates_on" ]; then
-          echo "  schedulingGatesEnabled: true" >> values-cert-manager-io.yaml
-        fi
-
         export TAG=${{ inputs.runtime_tag }}
-        envsubst < values-cert-manager-io.yaml > tmp.yaml
+        envsubst < values/${{ inputs.values-file }} > tmp.yaml
 
-        echo "installing with values-cert-manager-io.yaml file:"
+        echo "installing with values.yaml file:"
         cat tmp.yaml
 
-        kubectl create ns keptn-system
-        kubectl apply -f certificate.yaml
-
-        helm install -n keptn-system keptn ../../../chart \
+        helm install -n keptn-system --create-namespace keptn ../../../chart \
           --values ./tmp.yaml \
           --debug --wait --timeout 1m

.github/actions/deploy-keptn-on-cluster/values/values-integration-allowed-namespaces.yaml

@@ -0,0 +1,24 @@
+global:
+  imageRegistry: "localhost:5000"
+
+certManager:
+  imagePullPolicy: Never
+  image:
+    tag: $TAG
+
+metricsOperator:
+  enabled: false
+
+lifecycleOperator:
+  lifecycleOperator:
+    imagePullPolicy: Never
+    image:
+      tag: $TAG
+    env:
+      functionRunnerImage: localhost:5000/keptn/deno-runtime:$TAG
+      pythonRunnerImage: localhost:5000/keptn/python-runtime:$TAG
+  scheduler:
+    imagePullPolicy: Never
+    image:
+      tag: $TAG
+  allowedNamespaces: [allowed-ns-test]

.github/actions/deploy-keptn-on-cluster/values/values-integration-cert-manager.yaml

@@ -0,0 +1,23 @@
+global:
+  imageRegistry: "localhost:5000"
+
+certManager:
+  imagePullPolicy: Never
+  image:
+    tag: $TAG
+
+metricsOperator:
+  enabled: false
+
+lifecycleOperator:
+  lifecycleOperator:
+    imagePullPolicy: Never
+    image:
+      tag: $TAG
+    env:
+      functionRunnerImage: localhost:5000/keptn/deno-runtime:$TAG
+      pythonRunnerImage: localhost:5000/keptn/python-runtime:$TAG
+  scheduler:
+    imagePullPolicy: Never
+    image:
+      tag: $TAG

.github/actions/deploy-keptn-on-cluster/values/values-integration-gates_off-cert_manager_io_off.yaml

@@ -0,0 +1,27 @@
+global:
+  imageRegistry: "localhost:5000"
+
+certManager:
+  imagePullPolicy: Never
+  image:
+    tag: $TAG
+
+metricsOperator:
+  imagePullPolicy: Never
+  image:
+    tag: $TAG
+
+lifecycleOperator:
+  promotionTasksEnabled: true
+  lifecycleOperator:
+    imagePullPolicy: Never
+    image:
+      tag: $TAG
+    env:
+      functionRunnerImage: localhost:5000/keptn/deno-runtime:$TAG
+      pythonRunnerImage: localhost:5000/keptn/python-runtime:$TAG
+  scheduler:
+    imagePullPolicy: Never
+    image:
+      tag: $TAG
+  schedulingGatesEnabled: false

.github/actions/deploy-keptn-on-cluster/values-cert-manager-io.yaml → .github/actions/deploy-keptn-on-cluster/values/values-integration-gates_off-cert_manager_io_on.yaml

@@ -22,3 +22,4 @@ lifecycleOperator:
     imagePullPolicy: Never
     image:
       tag: $TAG
+  schedulingGatesEnabled: false

.github/actions/deploy-keptn-on-cluster/values/values-integration-gates_on-cert_manager_io_off.yaml

@@ -0,0 +1,27 @@
+global:
+  imageRegistry: "localhost:5000"
+
+certManager:
+  imagePullPolicy: Never
+  image:
+    tag: $TAG
+
+metricsOperator:
+  imagePullPolicy: Never
+  image:
+    tag: $TAG
+
+lifecycleOperator:
+  promotionTasksEnabled: true
+  lifecycleOperator:
+    imagePullPolicy: Never
+    image:
+      tag: $TAG
+    env:
+      functionRunnerImage: localhost:5000/keptn/deno-runtime:$TAG
+      pythonRunnerImage: localhost:5000/keptn/python-runtime:$TAG
+  scheduler:
+    imagePullPolicy: Never
+    image:
+      tag: $TAG
+  schedulingGatesEnabled: true

.github/actions/deploy-keptn-on-cluster/values/values-integration-lifecycle.yaml

@@ -0,0 +1,25 @@
+global:
+  imageRegistry: "localhost:5000"
+
+certManager:
+  imagePullPolicy: Never
+  image:
+    tag: $TAG
+
+metricsOperator:
+  enabled: false
+
+lifecycleOperator:
+  promotionTasksEnabled: true
+  lifecycleOperator:
+    imagePullPolicy: Never
+    image:
+      tag: $TAG
+    env:
+      functionRunnerImage: localhost:5000/keptn/deno-runtime:$TAG
+      pythonRunnerImage: localhost:5000/keptn/python-runtime:$TAG
+  scheduler:
+    imagePullPolicy: Never
+    image:
+      tag: $TAG
+  schedulingGatesEnabled: true

.github/actions/deploy-keptn-on-cluster/values/values-integration-metrics.yaml

@@ -0,0 +1,15 @@
+global:
+  imageRegistry: "localhost:5000"
+
+certManager:
+  imagePullPolicy: Never
+  image:
+    tag: $TAG
+
+metricsOperator:
+  imagePullPolicy: Never
+  image:
+    tag: $TAG
+
+lifecycleOperator:
+  enabled: false

.github/workflows/CI.yaml

@@ -208,15 +208,56 @@ jobs:
     strategy:
       matrix:
         scheduling-gates: [gates_on, gates_off]
-        allowed-namespaces: [allowed_ns_on, allowed_ns_off]
         cert-manager-io-enabled: [cert_manager_io_on, cert_manager_io_off]
+        exclude:
+          - scheduling-gates: gates_on
+            cert-manager-io-enabled: cert_manager_io_on
     with:
       runtime_tag: dev-${{ needs.prepare_ci_run.outputs.DATETIME }}
       scheduling-gates: ${{ matrix.scheduling-gates }}
-      allowed-namespaces: ${{ matrix.allowed-namespaces }}
       cert-manager-io-enabled: ${{ matrix.cert-manager-io-enabled }}
     uses: ./.github/workflows/integration-test.yml
 
+  integration_tests_allowed_namespaces:
+    name: Integration Tests Allowed Namespaces
+    needs:
+      - prepare_ci_run
+      - build_image
+    with:
+      runtime_tag: dev-${{ needs.prepare_ci_run.outputs.DATETIME }}
+      type: allowed-namespaces
+    uses: ./.github/workflows/integration-test-component.yml
+
+  integration_tests_lifecycle:
+    name: Integration Tests Lifecycle
+    needs:
+      - prepare_ci_run
+      - build_image
+    with:
+      runtime_tag: dev-${{ needs.prepare_ci_run.outputs.DATETIME }}
+      type: lifecycle
+    uses: ./.github/workflows/integration-test-component.yml
+
+  integration_tests_metrics:
+    name: Integration Tests Metrics
+    needs:
+      - prepare_ci_run
+      - build_image
+    with:
+      runtime_tag: dev-${{ needs.prepare_ci_run.outputs.DATETIME }}
+      type: metrics
+    uses: ./.github/workflows/integration-test-component.yml
+
+  integration_tests_cert_manager:
+    name: Integration Tests Cert Manager
+    needs:
+      - prepare_ci_run
+      - build_image
+    with:
+      runtime_tag: dev-${{ needs.prepare_ci_run.outputs.DATETIME }}
+      type: cert-manager
+    uses: ./.github/workflows/integration-test-component.yml
+
   load-tests:
     name: Load Tests
     needs: [prepare_ci_run, build_image]

.github/workflows/integration-test-component.yml

@@ -0,0 +1,57 @@
+name: Integration-Test-Component
+on:
+  workflow_call:
+    inputs:
+      runtime_tag:
+        description: "Tag for the deno and python runner images"
+        type: "string"
+        required: true
+      type:
+        description: "Type of the tests that are going to be executed"
+        type: "string"
+        required: true
+
+# Declare default permissions as read only.
+permissions: read-all
+
+env:
+  GO_VERSION: "~1.21"
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  run-integration-test-component:
+    name: Run Chainsaw Tests
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Check out code
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+      - name: Setup cluster
+        uses: ./.github/actions/deploy-keptn-on-cluster
+        with:
+          runtime_tag: ${{ inputs.runtime_tag }}
+          values-file: values-integration-${{ inputs.type }}.yaml
+
+      - name: Install and expose Prometheus
+        uses: ./.github/actions/deploy-prometheus-on-cluster
+
+      - name: Install Chainsaw
+        uses: kyverno/action-install-chainsaw@995cddaee7702e849270b84fa44cdcebe7462da8 # v0.1.9
+
+      - name: Run Integration Tests ${{ inputs.type }}
+        working-directory: .
+        run: make integration-test-${{ inputs.type }}
+
+      - name: Create reports
+        if: failure()
+        working-directory: ./.github/scripts
+        run: ./create-reports-full.sh
+
+      - name: Upload cluster logs
+        if: failure()
+        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4
+        with:
+          name: logs-integration-tests-${{ inputs.type }}
+          path: .github/scripts/logs