conformance.yml

# ------------------------------------------------------------
# Copyright 2021 The Dapr Authors
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#     http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ------------------------------------------------------------

name: Components Conformance Tests

on:
  repository_dispatch:
    types: [conformance-test]
  workflow_dispatch:
  schedule:
    - cron: '0 */8 * * *'
  pull_request:
    branches:
      - master
      - 'release-*'

jobs:
  # Based on whether this is a PR or a scheduled run, we will run a different
  # subset of the conformance tests. This allows all the tests not requiring
  # secrets to be executed on pull requests.
  generate-matrix:
    runs-on: ubuntu-latest
    steps:
    - name: Parse repository_dispatch payload
      if: github.event_name == 'repository_dispatch'
      working-directory: ${{ github.workspace }}
      run: |
        if [ ${{ github.event.client_payload.command }} = "ok-to-test" ]; then
          echo "CHECKOUT_REF=${{ github.event.client_payload.pull_head_ref }}" >> $GITHUB_ENV
          echo "PR_NUMBER=${{ github.event.client_payload.issue.number }}" >> $GITHUB_ENV
        fi

    - name: Install yq
      run: |
        sudo snap install yq

    - name: Specify components not requiring secrets nor certs
      id: pr-components
      run: |
        PR_COMPONENTS=$(yq -I0 --tojson eval - << EOF
        - bindings.cron
        - bindings.http
        - bindings.influx
        - bindings.kafka-wurstmeister
        - bindings.kafka-confluent
        - bindings.mqtt-emqx
        - bindings.mqtt-mosquitto
        - bindings.mqtt-vernemq
        - bindings.postgres
        - bindings.redis.v6
        - bindings.redis.v7
        - bindings.kubemq
        - bindings.rabbitmq
        - pubsub.aws.snssqs.docker
        - pubsub.hazelcast
        - pubsub.in-memory
        - pubsub.mqtt-emqx
        - pubsub.mqtt-mosquitto
        - pubsub.mqtt-vernemq
        - pubsub.natsstreaming
        - pubsub.pulsar
        - pubsub.rabbitmq
        - pubsub.redis.v6
        - pubsub.kafka-wurstmeister
        - pubsub.kafka-confluent
        - pubsub.kubemq
        - pubsub.solace
        - secretstores.kubernetes
        - secretstores.localenv
        - secretstores.localfile
        - secretstores.hashicorp.vault
        - state.cassandra
        - state.memcached
        - state.mongodb
        - state.mysql.mysql
        - state.mysql.mariadb
        - state.postgresql
        - state.redis.v6
        - state.redis.v7
        - state.sqlserver
        - state.in-memory
        - state.cockroachdb
        - workflows.temporal
        - state.rethinkdb
        EOF
        )
        echo "pr-components=$PR_COMPONENTS" >> $GITHUB_OUTPUT

    - name: Specify components requiring secrets or certs
      id: cron-components
      run: |
        if [ "${{ github.event_name }}" = "pull_request" ]; then
          echo "cron-components=[]" >> $GITHUB_OUTPUT
          exit
        fi
        # Unfortunately, Azure secrets can't have underscores in
        # names, while environment variables with hyphens ('-') are
        # troublesome.
        #
        # We work around here by leveraging the fact that
        # environment variable names are case sensitive, so
        # CamelCase would still work.
        #
        # That is slightly better than something like
        # AZURECOSMOSDBMASTERKEY, which is extremely hard to read
        # and errorprone.
        #
        # Only list the secrets you need for the component.
        CRON_COMPONENTS=$(yq -I0 --tojson eval - << EOF
        - component: state.azure.blobstorage
          required-secrets: AzureBlobStorageAccessKey,AzureBlobStorageAccount
        - component: state.azure.cosmosdb
          required-secrets: AzureCosmosDBMasterKey,AzureCosmosDBUrl,AzureCosmosDB,AzureCosmosDBCollection
        - component: state.azure.sql
          required-secrets: AzureResourceGroupName, AzureSqlServerName, AzureSqlServerConnectionString
        - component: state.azure.tablestorage.storage
          required-secrets: AzureBlobStorageAccessKey,AzureBlobStorageAccount
        - component: state.azure.tablestorage.cosmosdb
          required-secrets: AzureCosmosDBTableAPI,AzureCosmosDBTableAPIMasterKey
        - component: pubsub.azure.eventhubs
          required-secrets: AzureEventHubsPubsubNamespaceConnectionString,AzureEventHubsPubsubConsumerGroup,AzureBlobStorageAccessKey,AzureBlobStorageAccount,AzureEventHubsPubsubContainer
        - component: pubsub.azure.servicebus.topics
          required-secrets: AzureServiceBusConnectionString
        - component: pubsub.azure.servicebus.queues
          required-secrets: AzureServiceBusConnectionString
        - component: bindings.azure.blobstorage
          required-secrets: AzureBlobStorageAccessKey,AzureBlobStorageAccount
        - component: bindings.azure.eventgrid
          required-secrets: AzureEventGridNgrokToken,AzureEventGridAccessKey,AzureEventGridTopicEndpoint,AzureEventGridScope,AzureEventGridClientSecret,AzureEventGridClientId,AzureEventGridTenantId,AzureEventGridSubscriptionId
        - component: bindings.azure.eventhubs
          required-secrets: AzureEventHubsBindingsConnectionString,AzureEventHubsBindingsConsumerGroup,AzureBlobStorageAccessKey,AzureBlobStorageAccount,AzureEventHubsBindingsContainer
        - component: bindings.azure.servicebusqueues
          required-secrets: AzureServiceBusConnectionString
        - component: bindings.azure.storagequeues
          required-secrets: AzureBlobStorageAccessKey,AzureBlobStorageAccount,AzureBlobStorageQueue
        - component: secretstores.azure.keyvault.certificate
          required-secrets: AzureKeyVaultName,AzureKeyVaultSecretStoreTenantId,AzureKeyVaultSecretStoreClientId
          required-certs: AzureKeyVaultSecretStoreCert
        - component: secretstores.azure.keyvault.serviceprincipal
          required-secrets: AzureKeyVaultName,AzureKeyVaultSecretStoreTenantId,AzureKeyVaultSecretStoreServicePrincipalClientId,AzureKeyVaultSecretStoreServicePrincipalClientSecret
        - component: bindings.azure.cosmosdb
          required-secrets: AzureCosmosDBMasterKey,AzureCosmosDBUrl,AzureCosmosDB,AzureCosmosDBCollection
        - component: pubsub.aws.snssqs.terraform
          terraform-dir: pubsub/aws/snssqs        
        - component: state.cloudflare.workerskv
        EOF
        )
        echo "cron-components=$CRON_COMPONENTS" >> $GITHUB_OUTPUT

    - name: Create PR comment
      if: env.PR_NUMBER != ''
      uses: artursouza/sticky-pull-request-comment@v2.2.0
      with:
        header: ${{ github.run_id }}
        number: ${{ env.PR_NUMBER }}
        GITHUB_TOKEN: ${{ secrets.DAPR_BOT_TOKEN }}
        message: |
          # Components conformance test

          🔗 **[Link to Action run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})**

          Commit ref: ${{ env.CHECKOUT_REF }}

    outputs:
      pr-components: ${{ steps.pr-components.outputs.pr-components }}
      cron-components: ${{ steps.cron-components.outputs.cron-components }}

  conformance:
    name: ${{ matrix.component }} conformance
    runs-on: ubuntu-latest
    env:
      # Version of Node.js to use
      # Currently used by the Cloudflare components
      NODE_VERSION: 18.x
      UNIQUE_ID: ${{github.run_id}}-${{github.run_attempt}}
    defaults:
      run:
        shell: bash
    needs: generate-matrix

    strategy:
      fail-fast: false # Keep running even if one component fails
      matrix:
        component: ${{ fromJson(needs.generate-matrix.outputs.pr-components) }}
        include: ${{ fromJson(needs.generate-matrix.outputs.cron-components) }}

    steps:
    - name: Set default payload repo and ref
      working-directory: ${{ github.workspace }}
      run: |
        echo "CHECKOUT_REPO=${{ github.repository }}" >> $GITHUB_ENV
        echo "CHECKOUT_REF=${{ github.ref }}" >> $GITHUB_ENV

    - name: Parse repository_dispatch payload
      if: github.event_name == 'repository_dispatch'
      working-directory: ${{ github.workspace }}
      run: |
        if [ ${{ github.event.client_payload.command }} = "ok-to-test" ]; then
          echo "CHECKOUT_REPO=${{ github.event.client_payload.pull_head_repo }}" >> $GITHUB_ENV
          echo "CHECKOUT_REF=${{ github.event.client_payload.pull_head_ref }}" >> $GITHUB_ENV
          echo "PR_NUMBER=${{ github.event.client_payload.issue.number }}" >> $GITHUB_ENV
        fi

    - name: Check out code onto GOPATH
      uses: actions/checkout@v3
      with:
        repository: ${{ env.CHECKOUT_REPO }}
        ref: ${{ env.CHECKOUT_REF }}

    - name: Setup test output
      run: |
        export TEST_OUTPUT_FILE_PREFIX=$GITHUB_WORKSPACE/test_report
        echo "TEST_OUTPUT_FILE_PREFIX=$TEST_OUTPUT_FILE_PREFIX" >> $GITHUB_ENV

    - uses: Azure/login@v1
      with:
        creds: ${{ secrets.AZURE_CREDENTIALS }}
      if: matrix.required-secrets != ''

    # Set this GitHub secret to your KeyVault, and grant the KeyVault policy to your Service Principal:
    #    az keyvault set-policy -n $AZURE_KEYVAULT --secret-permissions get list --spn $SPN_CLIENT_ID
    # Using az cli to query keyvault as Azure/get-keyvault-secrets@v1 is deprecated
    - name: Setup secrets
      id: get-azure-secrets
      if: matrix.required-secrets != ''
      env:
        VAULT_NAME: ${{ secrets.AZURE_KEYVAULT }}
      run: |
        secrets="${{ matrix.required-secrets }}"
        for secretName in $(echo -n $secrets | tr ',' ' '); do
          value=$(az keyvault secret show \
            --name $secretName \
            --vault-name $VAULT_NAME \
            --query value \
            --output tsv)
          echo "::add-mask::$value"
          echo "$secretName=$value" >> $GITHUB_OUTPUT
          echo "$secretName=$value" >> $GITHUB_ENV
        done

    - name: Start ngrok
      if: contains(matrix.component, 'azure.eventgrid')
      run: |
        wget https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip
        unzip -qq ngrok-stable-linux-amd64.zip
        ./ngrok authtoken ${{ env.AzureEventGridNgrokToken }}
        ./ngrok http -log=stdout --log-level debug -host-header=localhost 9000 > /tmp/ngrok.log &
        sleep 10
        export NGROK_ENDPOINT=`cat /tmp/ngrok.log |  grep -Eom1 'https://.*' | sed 's/\s.*//'`
        echo "Ngrok's endpoint: ${NGROK_ENDPOINT}"
        echo "AzureEventGridSubscriberEndpoint=${NGROK_ENDPOINT}/api/events" >> $GITHUB_ENV
        cat /tmp/ngrok.log
        # Schedule trigger to kill ngrok
        bash -c "sleep 500 && pkill ngrok" &

    # Download the required certificates into files, and set env var pointing to their names
    - name: Setup certs
      if: matrix.required-certs != ''
      run: |
        for CERT_NAME in $(echo "${{ matrix.required-certs }}" | sed 's/,/ /g'); do
          CERT_FILE=$(mktemp --suffix .pfx)
          echo "Downloading cert $CERT_NAME into file $CERT_FILE"
          rm $CERT_FILE && \
            az keyvault secret download --vault-name ${{ secrets.AZURE_KEYVAULT }} --name $CERT_NAME --encoding base64 --file $CERT_FILE
          echo 'Setting $CERT_NAME to' "$CERT_FILE"
          echo "$CERT_NAME=$CERT_FILE" >> $GITHUB_ENV
        done

    - name: Get current time
      run: |
        echo "CURRENT_TIME=$(date --rfc-3339=date)" >> ${GITHUB_ENV}

    - name: Setup Terraform
      uses: hashicorp/setup-terraform@v2
      if: matrix.terraform-dir != ''

    - name: Configure AWS Credentials
      uses: aws-actions/configure-aws-credentials@v1
      with:
        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY }}
        aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY }}
        aws-region: us-west-1
      if: matrix.terraform-dir != ''

    - name: Terraform Init
      id: init
      run: terraform init
      working-directory: "./.github/infrastructure/terraform/conformance/${{ matrix.terraform-dir }}"
      if: matrix.terraform-dir != ''
      
    - name: Terraform Validate
      id: validate
      run: terraform validate -no-color
      working-directory: "./.github/infrastructure/terraform/conformance/${{ matrix.terraform-dir }}"
      if: matrix.terraform-dir != ''

    - name: Terraform Plan
      id: plan
      run: terraform plan -no-color -var="UNIQUE_ID=${{env.UNIQUE_ID}}" -var="TIMESTAMP=${{env.CURRENT_TIME}}"
      working-directory: "./.github/infrastructure/terraform/conformance/${{ matrix.terraform-dir }}"
      if: matrix.terraform-dir != ''

    - name: Terraform Apply
      run: terraform apply -auto-approve -var="UNIQUE_ID=${{env.UNIQUE_ID}}" -var="TIMESTAMP=${{env.CURRENT_TIME}}"
      working-directory: "./.github/infrastructure/terraform/conformance/${{ matrix.terraform-dir }}"
      if: matrix.terraform-dir != ''
      continue-on-error: true

    - name: Create aws.snssqs variables
      run: |
        PUBSUB_AWS_SNSSQS_QUEUE="testQueue-${{ env.UNIQUE_ID }}"
        echo "PUBSUB_AWS_SNSSQS_QUEUE=$PUBSUB_AWS_SNSSQS_QUEUE" >> $GITHUB_ENV
        PUBSUB_AWS_SNSSQS_TOPIC="testTopic-${{ env.UNIQUE_ID }}"
        echo "PUBSUB_AWS_SNSSQS_TOPIC=$PUBSUB_AWS_SNSSQS_TOPIC" >> $GITHUB_ENV
        PUBSUB_AWS_SNSSQS_TOPIC_MULTI_1="multiTopic1-${{ env.UNIQUE_ID }}"
        echo "PUBSUB_AWS_SNSSQS_TOPIC_MULTI_1=$PUBSUB_AWS_SNSSQS_TOPIC_MULTI_1" >> $GITHUB_ENV
        PUBSUB_AWS_SNSSQS_TOPIC_MULTI_2="multiTopic2-${{ env.UNIQUE_ID }}"
        echo "PUBSUB_AWS_SNSSQS_TOPIC_MULTI_2=$PUBSUB_AWS_SNSSQS_TOPIC_MULTI_2" >> $GITHUB_ENV
      if: contains(matrix.component, 'snssqs')

    - name: Start Redis 6 with Redis JSON
      run: docker-compose -f ./.github/infrastructure/docker-compose-redisjson.yml -p redis up -d
      if: contains(matrix.component, 'redis.v6')
  
    - name: Start Redis 7
      run: docker-compose -f ./.github/infrastructure/docker-compose-redis7.yml -p redis up -d
      if: contains(matrix.component, 'redis.v7')

    - name: Start Temporal
      run: docker-compose -f ./.github/infrastructure/docker-compose-temporal.yml -p temporal up -d
      if: contains(matrix.component, 'temporal')

    - name: Start MongoDB
      uses: supercharge/mongodb-github-action@1.3.0
      with:
        mongodb-version: 4.2
        mongodb-replica-set: test-rs
      if: contains(matrix.component, 'mongodb')

    - name: Start sqlserver
      run: docker-compose -f ./.github/infrastructure/docker-compose-sqlserver.yml -p sqlserver up -d
      if: contains(matrix.component, 'sqlserver')

    - name: Start kafka
      run: docker-compose -f ./.github/infrastructure/docker-compose-kafka.yml -p kafka up -d
      if: contains(matrix.component, 'wurstmeister')

    - name: Start kafka Confluent
      run: docker-compose -f ./.github/infrastructure/docker-compose-confluent.yml -p confluent up -d
      if: contains(matrix.component, 'confluent')

    - name: Start memcached
      run: docker-compose -f ./.github/infrastructure/docker-compose-memcached.yml -p memcached up -d
      if: contains(matrix.component, 'memcached')

    - name: Start natsstreaming
      run: docker-compose -f ./.github/infrastructure/docker-compose-natsstreaming.yml -p natsstreaming up -d
      if: contains(matrix.component, 'natsstreaming')

    - name: Start pulsar
      run: docker-compose -f ./.github/infrastructure/docker-compose-pulsar.yml -p pulsar up -d
      if: contains(matrix.component, 'pulsar')

    - name: Start Eclipse Mosquitto (MQTT)
      run: docker-compose -f ./.github/infrastructure/docker-compose-mosquitto.yml -p mosquitto up -d
      if: contains(matrix.component, 'mqtt-mosquitto')

    - name: Start EMQ X (MQTT)
      run: docker-compose -f ./.github/infrastructure/docker-compose-emqx.yml -p emqx up -d
      if: contains(matrix.component, 'mqtt-emqx')

    - name: Start VerneMQ (MQTT)
      run: docker-compose -f ./.github/infrastructure/docker-compose-vernemq.yml -p vernemq up -d
      if: contains(matrix.component, 'mqtt-vernemq')

    - name: Start hazelcast
      run: docker-compose -f ./.github/infrastructure/docker-compose-hazelcast.yml -p hazelcast up -d
      if: contains(matrix.component, 'hazelcast')

    - name: Start rabbitmq
      run: docker-compose -f ./.github/infrastructure/docker-compose-rabbitmq.yml -p rabbitmq up -d
      if: contains(matrix.component, 'rabbitmq')

    - name: Install Node.js ${{ env.NODE_VERSION }}
      if: contains(matrix.component, 'cloudflare')
      uses: actions/setup-node@v3
      with:
        node-version: ${{ env.NODE_VERSION }}

    - name: Check Cloudflare Workers code
      if: contains(matrix.component, 'cloudflare')
      run: |
        # Build the Worker
        (
          cd internal/component/cloudflare/worker-src;
          npm ci;
          npm run build;
        )
        # Check no changes
        git diff --exit-code ./internal/component/cloudflare/workers/code \
          || (echo "The source code of the Cloudflare Worker has changed, but the Worker has not been recompiled. Please re-compile the Worker by running 'npm ci && npm run build' in 'internal/component/cloudflare/worker-src'" && exit 1)

    - name: Setup Cloudflare KV
      if: matrix.component == 'state.cloudflare.workerskv'
      env:
        CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
        CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
      run: |
        CloudflareWorkerName="daprconfkv${{ github.run_id }}${{ github.run_attempt }}"
        CloudflareKVNamespaceID=$( curl -s -X POST "https://api.cloudflare.com/client/v4/accounts/${CLOUDFLARE_ACCOUNT_ID}/storage/kv/namespaces" \
          -H "Authorization: Bearer ${CLOUDFLARE_API_TOKEN}" \
          -H "Content-Type: application/json" \
          --data "{\"title\":\"${CloudflareWorkerName}\"}" \
            | jq -r ".result.id" )

        echo "CloudflareWorkerName=${CloudflareWorkerName}" >> $GITHUB_ENV
        echo "CloudflareAPIToken=${CLOUDFLARE_API_TOKEN}" >> $GITHUB_ENV
        echo "CloudflareAccountID=${CLOUDFLARE_ACCOUNT_ID}" >> $GITHUB_ENV
        echo "CloudflareKVNamespaceID=${CloudflareKVNamespaceID}" >> $GITHUB_ENV

    - name: Start aws snssqs
      run: docker-compose -f ./.github/infrastructure/docker-compose-snssqs.yml -p snssqs up -d
      if: contains(matrix.component, 'aws.snssqs.docker')

    - name: Start influxdb
      run: |
        export INFLUX_TOKEN=$(openssl rand -base64 32)
        echo "INFLUX_TOKEN=$INFLUX_TOKEN" >> $GITHUB_ENV
        docker-compose -f ./.github/infrastructure/docker-compose-influxdb.yml -p influxdb up -d
      if: contains(matrix.component, 'influx')

    - name: Start mysql
      run: |
        docker-compose -f ./.github/infrastructure/docker-compose-mysql.yml -p mysql up -d
      if: contains(matrix.component, 'mysql.mysql')

    - name: Start mariadb
      run: |
        docker-compose -f ./.github/infrastructure/docker-compose-mariadb.yml -p mariadb up -d
      if: contains(matrix.component, 'mysql.mariadb')

    - name: Start KinD
      uses: helm/kind-action@v1.4.0
      if: contains(matrix.component, 'kubernetes')

    - name: Start postgresql
      run: |
        docker-compose -f ./.github/infrastructure/docker-compose-postgresql.yml -p postgresql up -d
      if: contains(matrix.component, 'postgres')

    - name: Start cassandra
      run: |
        docker-compose -f ./.github/infrastructure/docker-compose-cassandra.yml -p cassandra up -d
      if: contains(matrix.component, 'cassandra')

    - name: Start cockroachdb
      run: |
        docker-compose -f ./.github/infrastructure/docker-compose-cockroachdb.yml -p cockroachdb up -d
      if: contains(matrix.component, 'cockroachdb')

    - name: Start vault
      run: |
        docker-compose -f ./.github/infrastructure/docker-compose-hashicorp-vault.yml -p vault up -d
      if: contains(matrix.component, 'hashicorp.vault')

    - name: Start rethinkdb
      run: |
        docker-compose -f ./.github/infrastructure/docker-compose-rethinkdb.yml -p rethinkdb up -d
      if: contains(matrix.component, 'rethinkdb')

    - name: Start kubemq
      run: docker-compose -f ./.github/infrastructure/docker-compose-kubemq.yml -p kubemq up -d
      if: contains(matrix.component, 'kubemq')
    
    - name: Start solace
      run: docker-compose -f ./.github/infrastructure/docker-compose-solace.yml -p solace up -d
      if: contains(matrix.component, 'solace')

    - name: Start nats with JetStream
      run: |
        docker-compose -f ./.github/infrastructure/docker-compose-jetstream.yml up -p jetstream -d
      if: contains(matrix.component, 'jetstream')

    - name: Setup KinD test data
      if: contains(matrix.component, 'kubernetes')
      run: |
        kubectl apply -f tests/config/kind-data.yaml
        echo "NAMESPACE=default" >> $GITHUB_ENV

    - name: Set up Go
      uses: actions/setup-go@v3
      with:
        go-version: '^1.19'

    - name: Download Go dependencies
      run: |
        go mod download
        go install gotest.tools/gotestsum@latest

    - name: Generate Azure SQL DB name
      run: |
        # Use UUID with `-` stripped out for DB names to prevent collisions between workflows
        export AzureSqlServerDbName=$(cat /proc/sys/kernel/random/uuid | sed -E 's/-//g')
        echo "AzureSqlServerDbName=$AzureSqlServerDbName" >> $GITHUB_ENV
      if: contains(matrix.component, 'azure.sql')

    - name: Run tests
      continue-on-error: true
      env:
        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY }}
        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY }}  
      run: |
        set -e 
        KIND=$(echo ${{ matrix.component }} | cut -d. -f1)
        NAME=$(echo ${{ matrix.component }} | cut -d. -f2-)
        KIND_UPPER="$(tr '[:lower:]' '[:upper:]' <<< ${KIND:0:1})${KIND:1}"

        if [ "${KIND}" = "secretstores" ]; then
          KIND_UPPER=SecretStore
        fi

        echo "Running tests for Test${KIND_UPPER}Conformance/${KIND}/${NAME} ... "

        set +e
        gotestsum --jsonfile ${{ env.TEST_OUTPUT_FILE_PREFIX }}_conformance.json \
          --junitfile ${{ env.TEST_OUTPUT_FILE_PREFIX }}_conformance.xml --format standard-verbose -- \
          -p 2 -count=1 -timeout=15m -tags=conftests ./tests/conformance --run="Test${KIND_UPPER}Conformance/${NAME}"

        status=$?
        echo "Completed tests for Test${KIND_UPPER}Conformance/${KIND}/${NAME} ... "
        if test $status -ne 0; then
          echo "Setting CONFORMANCE_FAILURE"
          echo "CONFORMANCE_FAILURE=true" >> $GITHUB_ENV
        fi
        set -e

        # Fail the step if we found no test to run
        if grep -q "warning: no tests to run" ${{ env.TEST_OUTPUT_FILE_PREFIX }}_conformance.json ; then
          echo "::error:: No test was found for component ${{ matrix.component }}"
          exit -1
        fi

    - name: Stop ngrok
      if: contains(matrix.component, 'azure.eventgrid')
      continue-on-error: true
      run: |
        set +e
        echo "GET ngrok tunnels:"
        curl http://localhost:4040/api/tunnels
        echo "GET ngrok http requests:"
        curl http://localhost:4040/api/requests/http
        pkill ngrok
        cat /tmp/ngrok.log

    - name: Cleanup Azure SQL test DB instance
      if: contains(matrix.component, 'azure.sql')
      continue-on-error: true
      run: |
        # Wait for the creation of the DB by the test to propagate to ARM, otherwise deletion succeeds as no-op.
        # The wait should be under 30s, but is capped at 1m as flakiness here results in an accumulation of expensive DB instances over time.
        # Also note that the deletion call only blocks until the request is process, do not rely on it for mutex on the same DB,
        # deletion may be ongoing in sequential runs.
        sleep 1m
        az sql db delete --resource-group ${{ env.AzureResourceGroupName }} --server ${{ env.AzureSqlServerName }} -n ${{ env.AzureSqlServerDbName }} --yes

    - name: Cleanup Cloudflare KV
      if: always() && matrix.component == 'state.cloudflare.workerskv'
      continue-on-error: true
      env:
        CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
        CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
      run: |
        # Delete the Worker
        curl -X DELETE "https://api.cloudflare.com/client/v4/accounts/${CLOUDFLARE_ACCOUNT_ID}/workers/scripts/${{ env.CloudflareWorkerName }}" \
          -H "Authorization: Bearer ${CLOUDFLARE_API_TOKEN}"
        # Delete the KV namespace
        curl -X DELETE "https://api.cloudflare.com/client/v4/accounts/${CLOUDFLARE_ACCOUNT_ID}/storage/kv/namespaces/${{ env.CloudflareKVNamespaceID }}" \
          -H "Authorization: Bearer ${CLOUDFLARE_API_TOKEN}"

    # Download the required certificates into files, and set env var pointing to their names
    - name: Clean up certs
      if: matrix.required-certs != ''
      run: |
        for CERT_NAME in $(echo "${{ matrix.required-certs }}" | sed 's/,/ /g'); do
          CERT_FILE=$(printenv $CERT_NAME)

          echo "Cleaning up the certificate file $CERT_FILE..."
          rm $CERT_FILE
        done

    - name: Terraform Destroy
      continue-on-error: true    
      run: terraform destroy -auto-approve -var="UNIQUE_ID=${{env.UNIQUE_ID}}" -var="TIMESTAMP=${{env.CURRENT_TIME}}"
      working-directory: "./.github/infrastructure/terraform/conformance/${{ matrix.terraform-dir }}"
      if: matrix.terraform-dir != ''

    - name: Check conformance test passed
      continue-on-error: false
      run: |
        echo "CONFORMANCE_FAILURE=$CONFORMANCE_FAILURE"
        if [[ -v CONFORMANCE_FAILURE ]]; then
          exit 1
        fi

    - name: Prepare test result info
      if: always()
      run: |
        mkdir -p tmp/result_files
        echo "Writing to tmp/result_files/${{ matrix.component }}.txt"
        if [[ "${{ env.CONFORMANCE_FAILURE }}" == "true" ]]; then
          echo "0" >> "tmp/result_files/${{ matrix.component }}.txt"
        else
          echo "1" >> "tmp/result_files/${{ matrix.component }}.txt"
        fi

    - name: Upload result files
      uses: actions/upload-artifact@v3
      if: always()
      with:
        name: result_files
        path: tmp/result_files
        retention-days: 1

    # Upload logs for test analytics to consume
    - name: Upload test results
      if: always()
      uses: actions/upload-artifact@master
      with:
        name: ${{ matrix.component }}_conformance_test
        path: ${{ env.TEST_OUTPUT_FILE_PREFIX }}_conformance.*

  post_job:
    name: Post-completion
    runs-on: ubuntu-latest
    if: always()
    needs:
      - conformance
      - generate-matrix
    steps:
      - name: Parse repository_dispatch payload
        if: github.event_name == 'repository_dispatch'
        working-directory: ${{ github.workspace }}
        run: |
          if [ ${{ github.event.client_payload.command }} = "ok-to-test" ]; then
            echo "CHECKOUT_REF=${{ github.event.client_payload.pull_head_ref }}" >> $GITHUB_ENV
            echo "PR_NUMBER=${{ github.event.client_payload.issue.number }}" >> $GITHUB_ENV
          fi

      - name: Download test result artifact
        if: always() && env.PR_NUMBER != ''
        uses: actions/download-artifact@v3
        continue-on-error: true
        id: testresults
        with:
          name: result_files
          path: tmp/result_files

      - name: Build message
        if: always() && env.PR_NUMBER != ''
        # Abusing of the github-script action to be able to write this in JS
        uses: actions/github-script@v6
        with:
          script: |
            const prComponents = ('${{ needs.generate-matrix.outputs.pr-components }}' && JSON.parse('${{ needs.generate-matrix.outputs.pr-components }}')) || []
            const cronComponents = ('${{ needs.generate-matrix.outputs.cron-components }}' && JSON.parse('${{ needs.generate-matrix.outputs.cron-components }}')) || []
            const allComponents = [...prComponents, ...cronComponents]
            const basePath = '${{ steps.testresults.outputs.download-path }}'
            const testType = 'conformance'

            const fs = require('fs')
            const path = require('path')

            let message = `# Components ${testType} test

            🔗 **[Link to Action run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})**

            Commit ref: ${{ env.CHECKOUT_REF }}`

            let allSuccess = true
            let allFound = true
            let notSuccess = []
            let notFound = []
            for (let i = 0; i < allComponents.length; i++) {
                let component = allComponents[i]
                if (!component) {
                    continue
                }
                if (typeof component == 'object') {
                    component = component.component
                }
                let found = false
                let success = false
                try {
                    let read =fs.readFileSync(path.join(basePath, component + '.txt'), 'utf8')
                    read = read.split('\n')[0]
                    switch (read) {
                        case '1':
                            found = true
                            success = true
                            break
                        case '0':
                            found = true
                            success = false
                    }
                } catch (e) {
                    // ignore errors, leave found = false
                }

                if (!found) {
                    allFound = false
                    notFound.push(component)
                }
                if (!success) {
                    allSuccess = false
                    notSuccess.push(component)
                }
            }

            if (allSuccess) {
                if (allFound) {
                    message += '\n\n' + `# ✅ All ${testType} tests passed

            All tests have reported a successful status` + '\n\n'
                } else {
                    message += '\n\n' + `# ⚠️ Some ${testType} tests did not report status

            Although there were no failures reported, some tests did not report a status:` + '\n\n'
                    for (let i = 0; i < notFound.length; i++) {
                        message += '- ' + notFound[i] + '\n'
                    }
                    message += '\n'
                }
            } else {
                message += '\n\n' + `# ❌ Some ${testType} tests failed

            These tests failed:` + '\n\n'
                for (let i = 0; i < notSuccess.length; i++) {
                    message += '- ' + notSuccess[i] + '\n'
                }
                message += '\n'

                if (!allFound) {
                    message += 'Additionally, some tests did not report a status:\n\n'
                    for (let i = 0; i < notFound.length; i++) {
                        message += '- ' + notFound[i] + '\n'
                    }
                    message += '\n'
                }
            }

            fs.writeFileSync('message.txt', message)

      - name: Replace PR comment
        if: env.PR_NUMBER != ''
        uses: artursouza/sticky-pull-request-comment@v2.2.0
        with:
          header: ${{ github.run_id }}
          number: ${{ env.PR_NUMBER }}
          GITHUB_TOKEN: ${{ secrets.DAPR_BOT_TOKEN }}
          path: message.txt