feat: Improve the Slack notification action

.gitattributes

@@ -0,0 +1 @@
+send-slack-notification/templates/* linguist-language=yaml

.github/workflows/pr_actions-smoke-test.yml

@@ -6,12 +6,13 @@ on:
   pull_request:
     paths:
       - .github/workflows/pr_actions-smoke-test.yml
-      - build-container-image/action.yml
-      - build-product-image/action.yml
-      - free-disk-space/action.yml
-      - publish-image/action.yml
-      - publish-index-manifest/action.yml
-      - shard/action.yml
+      - build-container-image/action.yaml
+      - build-product-image/action.yaml
+      - free-disk-space/action.yaml
+      - publish-image/action.yaml
+      - publish-index-manifest/action.yaml
+      - send-slack-notification/action.yaml
+      - shard/action.yaml
       - smoke/*
 
 jobs:
@@ -94,3 +95,23 @@ jobs:
           image-registry-password: ${{ secrets.HARBOR_ROBOT_STACKABLE_GITHUB_ACTION_BUILD_SECRET }}
           image-repository: stackable/smoke
           image-index-manifest-tag: ${{ matrix.versions }}-stackable0.0.0-dev
+
+  notify:
+    name: Failure Notification
+    needs: [generate_matrix, build, publish_manifests]
+    runs-on: ubuntu-latest
+    if: failure() || github.run_attempt > 1
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          persist-credentials: false
+
+      - name: Send Notification
+        uses: ./send-slack-notification
+        with:
+          publish-manifests-result: ${{ needs.publish_manifests.result }}
+          build-result: ${{ needs.build.result }}
+          slack-token: ${{ secrets.SLACK_CONTAINER_IMAGE_TOKEN }}
+          channel-id: C07UG6JH44F # notifications-container-images
+          type: container-image-build

run-integration-test/README.md

@@ -105,6 +105,9 @@ profiles:
 
 - `start-time`
 - `end-time`
+- `health`: The health of the integration tests. Contains three comma-separated values: Slack emoji,
+  GitHub emoji, and success rate.
+- `failed-tests`: A (potentially empty) plain text list of failed tests.
 
 [supported-clusters]: https://docs.replicated.com/vendor/testing-supported-clusters
 [run-integration-test]: ./action.yaml

run-integration-test/action.yaml

@@ -38,6 +38,12 @@ outputs:
   end-time:
     description: The date and time this integration test finished.
     value: ${{ steps.end-time.outputs.END_TIME }}
+  health:
+    description: The health of this integration test over the last few tries.
+    value: ${{ steps.health.outputs.HEALTH }}
+  failed-tests:
+    description: The (potentially empty) list of failed tests
+    value: ${{ steps.failed-tests.outputs.FAILED_TESTS }}
 runs:
   using: composite
   steps:
@@ -65,8 +71,8 @@ runs:
 
         # Run interu to expand parameters into GITHUB_ENV
         if [ "$TEST_MODE" == "profile" ]; then
-          [ -n "${TEST_SUITE:-}" ] && echo ::warning::The test-suite input is ignored, because a profile is selected.
-          [ -n "${TEST:-}" ] && echo ::warning::The test input is ignored, because a profile is selected.
+          [ -n "${TEST_SUITE:-}" ] && echo "::warning::The test-suite input is ignored, because a profile is selected."
+          [ -n "${TEST:-}" ] && echo "::warning::The test input is ignored, because a profile is selected."
 
           interu --instances "$GITHUB_ACTION_PATH/instances.yaml" profile "$TEST_MODE_INPUT" --check-test-definitions --output "$GITHUB_ENV"
         else
@@ -209,6 +215,7 @@ runs:
         echo "START_TIME=$(date +'%Y-%m-%dT%H:%M:%S')" | tee -a "$GITHUB_OUTPUT"
 
     - name: Run Integration Test
+      id: integration-test
       env:
         REF_NAME: ${{ github.ref_name }}
         GH_TOKEN: ${{ github.token }}
@@ -217,12 +224,12 @@ runs:
         set -euo pipefail
 
         OPERATOR_VERSION=$("$GITHUB_ACTION_PATH/../.scripts/actions/get_operator_version.sh" "$REF_NAME")
-        python ./scripts/run-tests --skip-tests --operator "$OPERATOR_NAME=$OPERATOR_VERSION"
+        python ./scripts/run-tests --skip-tests --operator "$OPERATOR_NAME=$OPERATOR_VERSION" | tee -a test-output.log
 
         [ -n "${BEKU_TEST_SUITE:-}" ] && ARGS+=" --test-suite $BEKU_TEST_SUITE"
         [ -n "${BEKU_TEST:-}" ] && ARGS+=" --test $BEKU_TEST"
 
-        python ./scripts/run-tests --skip-release --log-level debug --parallel "$BEKU_TEST_PARALLELISM" ${ARGS:-}
+        python ./scripts/run-tests --skip-release --log-level debug --parallel "$BEKU_TEST_PARALLELISM" ${ARGS:-} | tee -a test-output.log
 
     - name: Record Test End Time
       id: end-time
@@ -240,3 +247,37 @@ runs:
         # See: https://github.com/replicatedhq/replicated-actions/tree/main/remove-cluster#inputs
         api-token: ${{ inputs.replicated-api-token }}
         cluster-id: ${{ steps.prepare-replicated-cluster.outputs.cluster-id }}
+
+    - name: Extract Failed Tests
+      id: failed-tests
+      if: steps.integration-test.conclusion == 'failure'
+      shell: bash
+      run: |
+        # Only look at the last 200 lines of test output which should be more than enough to capture all test results
+        FAILED_TESTS=$(tail --lines 200 test-output.log | grep -E '\s{8}--- FAIL:' | sed -e 's|^.*kuttl/harness/||')
+        echo "FAILED_TESTS=$FAILED_TESTS" | tee -a "$GITHUB_OUTPUT"
+
+    - name: Calculate Health
+      if: always()
+      id: health
+      env:
+        INTEGRATION_TEST_CONCLUSION: ${{ steps.integration-test.conclusion }}
+        WORKFLOW_NAME: "Integration Test"
+        GH_TOKEN: ${{ github.token }}
+        LAST_TRIES_LIMIT: "4"
+      shell: bash
+      run: |
+        set -euo pipefail
+
+        # First, we retrieve the number of successes of the last (currently) 4 runs. Afterwards we
+        # add 1 to the number of successes if this run succeeded. This ultimately represents the
+        # number of successes in the last 5 runs, similar to what Jenkins does. This score is then
+        # turned into an appropriate "weather" emoji and provided as an output to follow-up steps.
+        LAST_RUNS=$(gh run list --limit "$LAST_TRIES_LIMIT" --workflow "$WORKFLOW_NAME" --json conclusion)
+        SUCCESSES=$(echo $LAST_RUNS | jq '[.[] | select(.conclusion == "success")] | length')
+        export LAST_TRIES_TOTAL=$(echo $LAST_RUNS | jq 'length')
+
+        [ "$INTEGRATION_TEST_CONCLUSION" == "success" ] && SUCCESSES=$(echo "$SUCCESSES+1" | bc)
+        export SUCCESSES="$SUCCESSES"
+
+        echo "HEALTH=$(cat "${GITHUB_ACTION_PATH}/health" | envsubst | bc)" | tee -a "$GITHUB_OUTPUT"

run-integration-test/health

@@ -0,0 +1,35 @@
+/*
+  Calculate the percentage of successes of the last few tries (usually 5) and
+  return emojis suited for Slack and Github and the rate of successes formatted
+  like <successes>/<total_tries>.
+*/
+
+/* Set the scale to 1 to get 1 digit after the decimal point */
+scale = 1
+
+/* Calculate the percentage of successes */
+/* The variables below are replaced by envsubst */
+total_tries = ${LAST_TRIES_TOTAL}+1
+successes = ${SUCCESSES}
+health = successes/total_tries
+
+/* Convert the percentage into "weather" icons */
+if (health > 0.8) {
+  print ":sunny:,:sunny:,", successes, "/", total_tries
+} else {
+  if (health > 0.6) {
+    print ":partly_sunny:,:partly_sunny:,", successes, "/", total_tries
+  } else {
+    if (health > 0.4) {
+      print ":cloud:,:cloud:,", successes, "/", total_tries
+    } else {
+      if (health > 0.2) {
+        print ":rain_cloud:,:cloud_with_rain:,", successes, "/", total_tries
+      } else {
+        print ":thunder_cloud_and_rain:,:cloud_with_lightning_and_rain:,", successes, "/", total_tries
+      }
+    }
+  }
+}
+
+quit

send-slack-notification/README.md

@@ -6,7 +6,11 @@ This action sends a Slack message to notify about container image build failures
 Subsequent attempts of the same workflow run are automatically threaded in Slack.
 The color of the message is automatically selected based on the provided results.
 
-Example usage (workflow):
+## Supported notifications
+
+Currently, two types of notifications are supported.
+
+### Container image builds
 
 ```yaml
 jobs:
@@ -19,22 +23,46 @@ jobs:
       - name: Send Notification
         uses: stackabletech/actions/send-slack-notification
         with:
-          build-result: ${{ needs.job_2.result }}
+          type: container-image-build
+          channel-id: DEADBEEF
+          build-result: ${{ needs.job_1.result }}
           publish-manifests-result: ${{ needs.job_2.result }}
           slack-token: ${{ secrets.MY_SECRET }}
 ```
 
+### Integration tests for operators
+
+```yaml
+jobs:
+  notify:
+    name: Failure Notification
+    needs: [job_1]
+    runs-on: ubuntu-latest
+    if: failure() || github.run_attempt > 1
+    steps:
+      - name: Send Notification
+        uses: stackabletech/actions/send-slack-notification
+        with:
+          type: integration-test
+          channel-id: DEADBEEF
+          failed-tests: ${{ needs.job_1.failed-tests }}
+          test-result: ${{ needs.job_1.result }}
+          test-health: ${{ needs.job_1.health }}
+          slack-token: ${{ secrets.MY_SECRET }}
+```
+
 ## Inputs and Outputs
 
 > [!TIP]
 > For descriptions of the inputs and outputs, see the complete [send-slack-notification] action.
 
 ### Inputs
 
-- `channel-id` (defaults to `C07UG6JH44F`)
-- `build-result` (required, e.g. `success`)
-- `publish-manifests-result` (required, e.g. `failure`)
+- `type` (required, supported values: `container-image-build` and `integration-test`)
+- `channel-id` (required)
 - `slack-token` (required)
+- `build-result` (optional, e.g. `success`)
+- `publish-manifests-result` (optional, e.g. `failure`)
 
 ### Outputs
 

send-slack-notification/action.yaml

@@ -1,19 +1,65 @@
 ---
 name: Send Notification via Slack
-description: ""
+description: "This action sends notifications for different types of workflows"
 inputs:
   channel-id:
     description: The channel to sent the message to
-    default: C07UG6JH44F # notifications-container-images
+  type:
+    description: |
+      The type of notification to send. Supported types are `container-image-build`
+      and `integration-test`.
   build-result:
     description: The result of the build job
   publish-manifests-result:
     description: The result of the publish manifests job
+  test-result:
+    description: The result of an integration test
+  test-health:
+    description: The health of the past few integration tests
+  failed-tests:
+    description: The list (plain text) of failed tests
   slack-token:
     description: The Slack token
 runs:
   using: composite
   steps:
+    - name: Validate Inputs
+      env:
+        NOTIFICATION_TYPE: ${{ inputs.type }}
+
+        PUBLISH_MANIFESTS_RESULT: ${{ inputs.publish-manifests-result }}
+        BUILD_RESULT: ${{ inputs.build-result }}
+
+        FAILED_TESTS: ${{ inputs.failed-tests }}
+        TEST_RESULT: ${{ inputs.test-result }}
+        TEST_HEALTH: ${{ inputs.test-health }}
+      shell: bash
+      run: |
+        if [ -z "${NOTIFICATION_TYPE:-}" ]; then
+          echo "The type input must be provided"
+          exit 1
+        fi
+
+        if [ "$NOTIFICATION_TYPE" == "container-image-build" ]; then
+          [ -z "${PUBLISH_MANIFESTS_RESULT:-}" ] && echo "The publish-manifests-result input must be provided" && exit 1
+          [ -z "${BUILD_RESULT:-}" ] && echo "The build-result input must be provided" && exit 1
+
+          echo "PUBLISH_MANIFESTS_RESULT=$PUBLISH_MANIFESTS_RESULT" | tee -a "$GITHUB_ENV"
+          echo "BUILD_RESULT=$BUILD_RESULT" | tee -a "$GITHUB_ENV"
+        elif [ "$NOTIFICATION_TYPE" == "integration-test" ]; then
+          [ -z "${TEST_RESULT:-}" ] && echo "The test-result input must be provided" && exit 1
+          [ -z "${TEST_HEALTH:-}" ] && echo "The test-health input must be provided" && exit 1
+
+          echo "FAILED_TESTS=$FAILED_TESTS" | tee -a "$GITHUB_ENV"
+          echo "TEST_RESULT=$TEST_RESULT" | tee -a "$GITHUB_ENV"
+          echo "TEST_HEALTH=$TEST_HEALTH" | tee -a "$GITHUB_ENV"
+        else
+          echo "Supported notification types are: 'container-image-build' and 'integration-test'"
+          exit 1
+        fi
+
+        echo "NOTIFICATION_TYPE=$NOTIFICATION_TYPE" | tee -a "$GITHUB_ENV"
+
     - name: Retrieve Slack Thread ID
       id: retrieve-slack-thread-id
       continue-on-error: true
@@ -29,48 +75,52 @@ runs:
 
     - name: Format message
       env:
-        PUBLISH_MANIFESTS_RESULT: ${{ inputs.publish-manifests-result }}
         GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }}
-        GITHUB_WORKFLOW: ${{ github.workflow  }}
-        BUILD_RESULT: ${{ inputs.build-result }}
+        GITHUB_REPOSITORY: ${{ github.repository }}
+        GITHUB_WORKFLOW: ${{ github.workflow }}
       shell: bash
       run: |
-        if [ "$PUBLISH_MANIFESTS_RESULT" = "failure" ] || [ "$BUILD_RESULT" = "failure" ]; then
-          MESSAGE_VERB=failed
-          MESSAGE_COLOR=aa0000
-        else
-          MESSAGE_VERB=succeeded
-          MESSAGE_COLOR=10c400
-        fi
+        if [ "$NOTIFICATION_TYPE" == "container-image-build" ]; then
+          # TODO (@Techassi): Also add success template
+          if [ "$PUBLISH_MANIFESTS_RESULT" = "failure" ] || [ "$BUILD_RESULT" = "failure" ]; then
+            MESSAGE_VERB=failed
+            echo "MESSAGE_VERB=$MESSAGE_VERB" | tee -a "$GITHUB_ENV"
+
+            echo "MESSAGE_COLOR=aa0000" | tee -a "$GITHUB_ENV"
+          else
+            MESSAGE_VERB=succeeded
+            echo "MESSAGE_VERB=$MESSAGE_VERB" | tee -a "$GITHUB_ENV"
 
-        echo "MESSAGE_TEXT=*$GITHUB_WORKFLOW* $MESSAGE_VERB (attempt $GITHUB_RUN_ATTEMPT)" | tee -a "$GITHUB_ENV"
-        echo "MESSAGE_COLOR=$MESSAGE_COLOR" | tee -a "$GITHUB_ENV"
-        echo "MESSAGE_VERB=$MESSAGE_VERB" | tee -a "$GITHUB_ENV"
+            echo "MESSAGE_COLOR=10c400" | tee -a "$GITHUB_ENV"
+          fi
+
+          echo "MESSAGE_TEXT=*$GITHUB_WORKFLOW* $MESSAGE_VERB (attempt $GITHUB_RUN_ATTEMPT)" | tee -a "$GITHUB_ENV"
+          echo -e "MESSAGE_TEMPLATE<<EOF\n$(cat ${GITHUB_ACTION_PATH}/templates/container-image-build/failure.tpl)\nEOF" | tee -a "$GITHUB_ENV"
+        elif [ "$NOTIFICATION_TYPE" == "integration-test" ]; then
+          echo "HEALTH_SLACK_EMOJI=$(cut -d ',' -f 1 $TEST_HEALTH)" | tee -a "$GITHUB_ENV"
+          echo "HEALTH_RATE=$(cut -d ',' -f 3 $TEST_HEALTH)" | tee -a "$GITHUB_ENV"
+
+          if [ "$TEST_RESULT" == "failure" ]; then
+            echo "MESSAGE_TEXT=The integration test for *GITHUB_REPOSITORY* failed" | tee -a "$GITHUB_ENV"
+            echo -e "MESSAGE_TEMPLATE<<EOF\n$(cat ${GITHUB_ACTION_PATH}/templates/integration-test/failure.tpl)\nEOF" | tee -a "$GITHUB_ENV"
+          else
+            echo "MESSAGE_TEXT=The integration test for *GITHUB_REPOSITORY* succeeded" | tee -a "$GITHUB_ENV"
+            echo -e "MESSAGE_TEMPLATE<<EOF\n$(cat ${GITHUB_ACTION_PATH}/templates/integration-test/success.tpl)\nEOF" | tee -a "$GITHUB_ENV"
+          fi
+        fi
 
     - name: Send Notification
       id: send-notification
       uses: slackapi/slack-github-action@b0fa283ad8fea605de13dc3f449259339835fc52 # 2.1.0
+      env:
+        SLACK_THREAD_YAML: |
+          ${{ steps.retrieve-slack-thread-id.outcome == 'success' && format('thread_ts: "{0}"', env.SLACK_THREAD_ID) || '' }}
+        CHANNEL_ID: ${{ inputs.channel-id }}
       with:
         method: chat.postMessage
         token: ${{ inputs.slack-token }}
-        payload: |
-          channel: "${{ inputs.channel-id }}"
-          text: "${{ env.MESSAGE_TEXT }}"
-          ${{ steps.retrieve-slack-thread-id.outcome == 'success' && format('thread_ts: "{0}"', env.SLACK_THREAD_ID) || '' }}
-          attachments:
-            - pretext: "See the details below for a summary of which job(s) ${{ env.MESSAGE_VERB }}."
-              color: "${{ env.MESSAGE_COLOR }}"
-              fields:
-                - title: Build/Publish Image
-                  short: true
-                  value: "${{ inputs.build-result }}"
-                - title: Build/Publish Manifests
-                  short: true
-                  value: "${{ inputs.publish-manifests-result }}"
-              actions:
-                - type: button
-                  text: Go to workflow run
-                  url: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}"
+        payload: ${{ env.MESSAGE_TEMPLATE }}
+        payload-templated: true
 
     - name: Save Slack Thread ID to File
       if: steps.retrieve-slack-thread-id.outcome == 'failure'