Stable Components Certification Tests #1112
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # ------------------------------------------------------------ | |
| # Copyright 2021 The Dapr Authors | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| # ------------------------------------------------------------ | |
| name: Stable Components Certification Tests | |
| on: | |
| repository_dispatch: | |
| types: [certification-test] | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '25 */8 * * *' | |
| push: | |
| branches: | |
| - 'release-*' | |
| pull_request: | |
| branches: | |
| - 'master' | |
| - 'release-*' | |
| jobs: | |
| # Based on whether this is a PR or a scheduled run, we will run a different | |
| # subset of the certification tests. This allows all the tests not requiring | |
| # secrets to be executed on pull requests. | |
| generate-matrix: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Parse repository_dispatch payload | |
| if: github.event_name == 'repository_dispatch' | |
| working-directory: ${{ github.workspace }} | |
| run: | | |
| if [ ${{ github.event.client_payload.command }} = "ok-to-test" ]; then | |
| echo "CHECKOUT_REF=${{ github.event.client_payload.pull_head_ref }}" >> $GITHUB_ENV | |
| echo "PR_NUMBER=${{ github.event.client_payload.issue.number }}" >> $GITHUB_ENV | |
| fi | |
| - name: Check out code | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: ${{ env.CHECKOUT_REPO }} | |
| ref: ${{ env.CHECKOUT_REF }} | |
| - name: Generate test matrix | |
| id: generate-matrix | |
| env: | |
| VAULT_NAME: ${{ secrets.AZURE_KEYVAULT }} | |
| run: | | |
| if [ -z "$VAULT_NAME" ]; then | |
| # Do not include cloud tests when credentials are not available | |
| node .github/scripts/test-info.mjs certification false | |
| else | |
| # Include cloud tests | |
| node .github/scripts/test-info.mjs certification true | |
| fi | |
| - name: Create PR comment | |
| if: env.PR_NUMBER != '' | |
| uses: artursouza/sticky-pull-request-comment@da9e86aa2a80e4ae3b854d251add33bd6baabcba | |
| with: | |
| header: ${{ github.run_id }} | |
| number: ${{ env.PR_NUMBER }} | |
| GITHUB_TOKEN: ${{ secrets.DAPR_BOT_TOKEN }} | |
| message: | | |
| # Components certification test | |
| 🔗 **[Link to Action run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})** | |
| Commit ref: ${{ env.CHECKOUT_REF }} | |
| outputs: | |
| test-matrix: ${{ steps.generate-matrix.outputs.test-matrix }} | |
| certification: | |
| name: ${{ matrix.component }} certification | |
| # Add "id-token" with the intended permissions. | |
| # Needed by the 'Authenticate to Google Cloud' step. | |
| permissions: | |
| contents: 'read' | |
| id-token: 'write' | |
| runs-on: ubuntu-22.04 | |
| env: | |
| UNIQUE_ID: ${{github.run_id}}-${{github.run_attempt}} | |
| GOCOV_VER: "v1.1.0" | |
| GOTESTSUM_VER: "v1.9.0" | |
| defaults: | |
| run: | |
| shell: bash | |
| needs: | |
| - generate-matrix | |
| strategy: | |
| fail-fast: false # Keep running even if one component fails | |
| matrix: | |
| include: ${{ fromJson(needs.generate-matrix.outputs.test-matrix) }} | |
| steps: | |
| - name: Set default payload repo and ref | |
| run: | | |
| echo "CHECKOUT_REPO=${{ github.repository }}" >> $GITHUB_ENV | |
| echo "CHECKOUT_REF=${{ github.ref }}" >> $GITHUB_ENV | |
| - name: Parse repository_dispatch payload | |
| if: github.event_name == 'repository_dispatch' | |
| run: | | |
| if [ ${{ github.event.client_payload.command }} = "ok-to-test" ]; then | |
| echo "CHECKOUT_REPO=${{ github.event.client_payload.pull_head_repo }}" >> $GITHUB_ENV | |
| echo "CHECKOUT_REF=${{ github.event.client_payload.pull_head_ref }}" >> $GITHUB_ENV | |
| fi | |
| - name: Check out code | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: ${{ env.CHECKOUT_REPO }} | |
| ref: ${{ env.CHECKOUT_REF }} | |
| - name: Configure environment | |
| run: | | |
| # Output file | |
| echo "TEST_OUTPUT_FILE_PREFIX=$GITHUB_WORKSPACE/test_report" >> $GITHUB_ENV | |
| # Certification test and source path | |
| TEST_COMPONENT=$(echo "${{ matrix.component }}" | sed -E 's/\./\//g') | |
| echo "TEST_PATH=tests/certification/${TEST_COMPONENT}" >> $GITHUB_ENV | |
| SOURCE_PATH="github.com/dapr/components-contrib/${TEST_COMPONENT}" | |
| echo "SOURCE_PATH=$SOURCE_PATH" >> $GITHUB_ENV | |
| # converts slashes to dots in this string, so that it doesn't consider them sub-folders | |
| SOURCE_PATH_LINEAR=$(echo "$SOURCE_PATH" |sed 's#/#\.#g') | |
| echo "SOURCE_PATH_LINEAR=$SOURCE_PATH_LINEAR" >> $GITHUB_ENV | |
| # Current time (used by Terraform) | |
| echo "CURRENT_TIME=$(date --rfc-3339=date)" >> ${GITHUB_ENV} | |
| - uses: Azure/login@v1 | |
| with: | |
| creds: ${{ secrets.AZURE_CREDENTIALS }} | |
| if: matrix.required-secrets != '' | |
| # Set this GitHub secret to your KeyVault, and grant the KeyVault policy to your Service Principal: | |
| # az keyvault set-policy -n $AZURE_KEYVAULT --secret-permissions get list --spn $SPN_CLIENT_ID | |
| # Using az cli to query keyvault as Azure/get-keyvault-secrets@v1 is deprecated | |
| - name: Setup secrets | |
| if: matrix.required-secrets != '' | |
| env: | |
| VAULT_NAME: ${{ secrets.AZURE_KEYVAULT }} | |
| run: | | |
| secrets="${{ matrix.required-secrets }}" | |
| for secretName in $(echo -n $secrets | tr ',' ' '); do | |
| value=$(az keyvault secret show \ | |
| --name $secretName \ | |
| --vault-name $VAULT_NAME \ | |
| --query value \ | |
| --output tsv) | |
| echo "::add-mask::$value" | |
| echo "$secretName=$value" >> $GITHUB_OUTPUT | |
| echo "$secretName=$value" >> $GITHUB_ENV | |
| done | |
| # Authenticate with GCP Workload Identity Pool | |
| # Exports GCP ENV Vars: | |
| # - GCP_PROJECT | |
| # - GOOGLE_APPLICATION_CREDENTIALS | |
| - id: 'auth' | |
| if: matrix.require-gcp-credentials == 'true' | |
| name: 'Authenticate to Google Cloud' | |
| uses: 'google-github-actions/auth@v1' | |
| with: | |
| token_format: 'access_token' | |
| workload_identity_provider: ${{ secrets.GCP_WIF_PROVIDER_NAME }} | |
| service_account: ${{ secrets.GCP_WIF_SA_EMAIL }} | |
| create_credentials_file: true | |
| export_environment_variables: true | |
| cleanup_credentials: true | |
| # Download the required certificates into files, and set env var pointing to their names | |
| - name: Setup certs | |
| if: matrix.required-certs != '' | |
| working-directory: ${{ env.TEST_PATH }} | |
| run: | | |
| for CERT_NAME in $(echo "${{ matrix.required-certs }}" | sed 's/,/ /g'); do | |
| CERT_FILE=$(mktemp --suffix .pfx) | |
| echo "Downloading cert $CERT_NAME into file $CERT_FILE" | |
| rm $CERT_FILE && \ | |
| az keyvault secret download --vault-name ${{ secrets.AZURE_KEYVAULT }} --name $CERT_NAME --encoding base64 --file $CERT_FILE | |
| echo 'Setting $CERT_NAME to' "$CERT_FILE" | |
| echo "$CERT_NAME=$CERT_FILE" >> $GITHUB_ENV | |
| done | |
| - name: Setup Terraform | |
| uses: hashicorp/setup-terraform@v2.0.3 | |
| if: matrix.require-terraform == 'true' | |
| - name: Set Cloudflare env vars | |
| if: matrix.require-cloudflare-credentials == 'true' | |
| run: | | |
| echo "CLOUDFLARE_ACCOUNT_ID=${{ secrets.CLOUDFLARE_ACCOUNT_ID }}" >> $GITHUB_ENV | |
| echo "CLOUDFLARE_API_TOKEN=${{ secrets.CLOUDFLARE_API_TOKEN }}" >> $GITHUB_ENV | |
| - name: Set AWS env vars | |
| if: matrix.require-aws-credentials == 'true' | |
| run: | | |
| echo "AWS_REGION=us-west-1" >> $GITHUB_ENV | |
| echo "AWS_ACCESS_KEY=${{ secrets.AWS_ACCESS_KEY }}" >> $GITHUB_ENV | |
| echo "AWS_SECRET_KEY=${{ secrets.AWS_SECRET_KEY }}" >> $GITHUB_ENV | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| if: matrix.require-aws-credentials == 'true' | |
| with: | |
| aws-access-key-id: "${{ secrets.AWS_ACCESS_KEY }}" | |
| aws-secret-access-key: "${{ secrets.AWS_SECRET_KEY }}" | |
| aws-region: "${{ env.AWS_REGION }}" | |
| - name: Set up Go | |
| id: setup-go | |
| uses: actions/setup-go@v3 | |
| with: | |
| go-version-file: 'go.mod' | |
| - name: Download Go dependencies | |
| working-directory: ${{ env.TEST_PATH }} | |
| run: | | |
| go mod download | |
| go install github.com/axw/gocov/gocov@${{ env.GOCOV_VER }} | |
| go install gotest.tools/gotestsum@${{ env.GOTESTSUM_VER }} | |
| - name: Run setup script | |
| if: matrix.setup-script != '' | |
| run: .github/scripts/components-scripts/${{ matrix.setup-script }} | |
| - name: Catch setup failures | |
| if: failure() | |
| run: | | |
| echo "CERTIFICATION_FAILURE=true" >> $GITHUB_ENV | |
| - name: Run tests | |
| continue-on-error: false | |
| working-directory: ${{ env.TEST_PATH }} | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY }} | |
| AWS_REGION: "${{ env.AWS_REGION }}" | |
| run: | | |
| echo "Running certification tests for ${{ matrix.component }} ... " | |
| echo "Source Pacakge: " ${{ matrix.source-pkg }} | |
| export GOLANG_PROTOBUF_REGISTRATION_CONFLICT=ignore | |
| set +e | |
| gotestsum --jsonfile ${{ env.TEST_OUTPUT_FILE_PREFIX }}_certification.json \ | |
| --junitfile ${{ env.TEST_OUTPUT_FILE_PREFIX }}_certification.xml --format standard-quiet -- \ | |
| -coverprofile=cover.out -covermode=set -tags=certtests -timeout=30m -coverpkg=${{ matrix.source-pkg }} | |
| status=$? | |
| echo "Completed certification tests for ${{ matrix.component }} ... " | |
| if test $status -ne 0; then | |
| echo "Setting CERTIFICATION_FAILURE" | |
| export CERTIFICATION_FAILURE=true | |
| fi | |
| set -e | |
| mkdir -p tmp/cert_code_cov_files | |
| cat cover.out >> tmp/cert_code_cov_files/${{ env.SOURCE_PATH_LINEAR }}.out | |
| cat tmp/cert_code_cov_files/${{ env.SOURCE_PATH_LINEAR }}.out | |
| # Fail the step if we found no test to run | |
| if grep -q "\[no test files\]" ${{ env.TEST_OUTPUT_FILE_PREFIX }}_certification.json ; then | |
| echo "::error:: No certification test file was found for component ${{ matrix.component }}" | |
| exit -1 | |
| fi | |
| for CERT_NAME in $(echo "${{ matrix.required-certs }}" | sed 's/,/ /g'); do | |
| CERT_FILE=$(printenv $CERT_NAME) | |
| echo "Cleaning up the certificate file $CERT_FILE..." | |
| rm $CERT_FILE || true | |
| done | |
| if [[ -v CERTIFICATION_FAILURE ]]; then | |
| echo "CERTIFICATION_FAILURE=true" >> $GITHUB_ENV | |
| exit 1 | |
| else | |
| echo "CERTIFICATION_FAILURE=false" >> $GITHUB_ENV | |
| fi | |
| - name: Upload Cert Coverage Report File | |
| uses: actions/upload-artifact@v3 | |
| if: github.event_name == 'schedule' | |
| with: | |
| name: cert_code_cov | |
| path: ${{ env.TEST_PATH }}/tmp/cert_code_cov_files | |
| retention-days: 7 | |
| - name: Prepare test result info | |
| if: always() | |
| run: | | |
| mkdir -p tmp/result_files | |
| echo "Writing to tmp/result_files/${{ matrix.component }}.txt" | |
| if [[ "${{ env.CERTIFICATION_FAILURE }}" == "true" ]]; then | |
| echo "0" >> "tmp/result_files/${{ matrix.component }}.txt" | |
| else | |
| echo "1" >> "tmp/result_files/${{ matrix.component }}.txt" | |
| fi | |
| - name: Upload result files | |
| uses: actions/upload-artifact@v3 | |
| if: always() | |
| with: | |
| name: result_files | |
| path: tmp/result_files | |
| retention-days: 1 | |
| # Upload logs for test analytics to consume | |
| - name: Upload test results | |
| if: always() | |
| uses: actions/upload-artifact@master | |
| with: | |
| name: ${{ matrix.component }}_certification_test | |
| path: ${{ env.TEST_OUTPUT_FILE_PREFIX }}_certification.* | |
| - name: Run destroy script | |
| if: always() && matrix.destroy-script != '' | |
| run: .github/scripts/components-scripts/${{ matrix.destroy-script }} | |
| post_job: | |
| name: Post-completion | |
| runs-on: ubuntu-22.04 | |
| if: always() | |
| needs: | |
| - certification | |
| - generate-matrix | |
| steps: | |
| - name: Parse repository_dispatch payload | |
| if: github.event_name == 'repository_dispatch' | |
| working-directory: ${{ github.workspace }} | |
| run: | | |
| if [ ${{ github.event.client_payload.command }} = "ok-to-test" ]; then | |
| echo "CHECKOUT_REF=${{ github.event.client_payload.pull_head_ref }}" >> $GITHUB_ENV | |
| echo "PR_NUMBER=${{ github.event.client_payload.issue.number }}" >> $GITHUB_ENV | |
| fi | |
| - name: Download test result artifact | |
| if: always() && env.PR_NUMBER != '' | |
| uses: actions/download-artifact@v3 | |
| continue-on-error: true | |
| id: testresults | |
| with: | |
| name: result_files | |
| path: tmp/result_files | |
| - name: Build message | |
| if: always() && env.PR_NUMBER != '' | |
| # Abusing of the github-script action to be able to write this in JS | |
| uses: actions/github-script@v6 | |
| with: | |
| script: | | |
| const allComponents = JSON.parse('${{ needs.generate-matrix.outputs.test-matrix }}') | |
| const basePath = '${{ steps.testresults.outputs.download-path }}' | |
| const testType = 'certification' | |
| const fs = require('fs') | |
| const path = require('path') | |
| let message = `# Components ${testType} test | |
| 🔗 **[Link to Action run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})** | |
| Commit ref: ${{ env.CHECKOUT_REF }}` | |
| let allSuccess = true | |
| let allFound = true | |
| let notSuccess = [] | |
| let notFound = [] | |
| for (let i = 0; i < allComponents.length; i++) { | |
| let component = allComponents[i] | |
| if (!component) { | |
| continue | |
| } | |
| if (typeof component == 'object') { | |
| component = component.component | |
| } | |
| let found = false | |
| let success = false | |
| try { | |
| let read = fs.readFileSync(path.join(basePath, component + '.txt'), 'utf8') | |
| read = read.split('\n')[0] | |
| switch (read) { | |
| case '1': | |
| found = true | |
| success = true | |
| break | |
| case '0': | |
| found = true | |
| success = false | |
| } | |
| } catch (e) { | |
| // ignore errors, leave found = false | |
| } | |
| if (!found) { | |
| allFound = false | |
| notFound.push(component) | |
| } | |
| if (!success) { | |
| allSuccess = false | |
| notSuccess.push(component) | |
| } | |
| } | |
| if (allSuccess) { | |
| if (allFound) { | |
| message += '\n\n' + `# ✅ All ${testType} tests passed | |
| All tests have reported a successful status` + '\n\n' | |
| } else { | |
| message += '\n\n' + `# ⚠️ Some ${testType} tests did not report status | |
| Although there were no failures reported, some tests did not report a status:` + '\n\n' | |
| for (let i = 0; i < notFound.length; i++) { | |
| message += '- ' + notFound[i] + '\n' | |
| } | |
| message += '\n' | |
| } | |
| } else { | |
| message += '\n\n' + `# ❌ Some ${testType} tests failed | |
| These tests failed:` + '\n\n' | |
| for (let i = 0; i < notSuccess.length; i++) { | |
| message += '- ' + notSuccess[i] + '\n' | |
| } | |
| message += '\n' | |
| if (!allFound) { | |
| message += 'Additionally, some tests did not report a status:\n\n' | |
| for (let i = 0; i < notFound.length; i++) { | |
| message += '- ' + notFound[i] + '\n' | |
| } | |
| message += '\n' | |
| } | |
| } | |
| fs.writeFileSync('message.txt', message) | |
| - name: Replace PR comment | |
| if: always() && env.PR_NUMBER != '' | |
| uses: artursouza/sticky-pull-request-comment@da9e86aa2a80e4ae3b854d251add33bd6baabcba | |
| with: | |
| header: ${{ github.run_id }} | |
| number: ${{ env.PR_NUMBER }} | |
| GITHUB_TOKEN: ${{ secrets.DAPR_BOT_TOKEN }} | |
| path: message.txt |