Stable Components Certification Tests #546
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# ------------------------------------------------------------ | |
# Copyright 2021 The Dapr Authors | |
# Licensed under the Apache License, Version 2.0 (the "License"); | |
# you may not use this file except in compliance with the License. | |
# You may obtain a copy of the License at | |
# http://www.apache.org/licenses/LICENSE-2.0 | |
# Unless required by applicable law or agreed to in writing, software | |
# distributed under the License is distributed on an "AS IS" BASIS, | |
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
# See the License for the specific language governing permissions and | |
# limitations under the License. | |
# ------------------------------------------------------------ | |
name: Stable Components Certification Tests | |
on: | |
repository_dispatch: | |
types: [certification-test] | |
workflow_dispatch: | |
schedule: | |
- cron: '25 */8 * * *' | |
push: | |
branches: | |
- 'release-*' | |
pull_request: | |
branches: | |
# TODO: REMOVE "master" BEFORE MERGING | |
- 'master' | |
- 'release-*' | |
jobs: | |
# Based on whether this is a PR or a scheduled run, we will run a different | |
# subset of the certification tests. This allows all the tests not requiring | |
# secrets to be executed on pull requests. | |
generate-matrix: | |
runs-on: ubuntu-22.04 | |
steps: | |
- name: Parse repository_dispatch payload | |
if: github.event_name == 'repository_dispatch' | |
working-directory: ${{ github.workspace }} | |
run: | | |
if [ ${{ github.event.client_payload.command }} = "ok-to-test" ]; then | |
echo "CHECKOUT_REF=${{ github.event.client_payload.pull_head_ref }}" >> $GITHUB_ENV | |
echo "PR_NUMBER=${{ github.event.client_payload.issue.number }}" >> $GITHUB_ENV | |
fi | |
- name: Check out code | |
uses: actions/checkout@v3 | |
with: | |
repository: ${{ env.CHECKOUT_REPO }} | |
ref: ${{ env.CHECKOUT_REF }} | |
- name: Generate test matrix | |
id: generate-matrix | |
env: | |
VAULT_NAME: ${{ secrets.AZURE_KEYVAULT }} | |
run: | | |
if [ -z "$VAULT_NAME" ]; then | |
# Do not include cloud tests when credentials are not available | |
node .github/scripts/test-info.mjs certification false | |
else | |
# Include cloud tests | |
node .github/scripts/test-info.mjs certification true | |
fi | |
- name: Create PR comment | |
if: env.PR_NUMBER != '' | |
uses: artursouza/sticky-pull-request-comment@da9e86aa2a80e4ae3b854d251add33bd6baabcba | |
with: | |
header: ${{ github.run_id }} | |
number: ${{ env.PR_NUMBER }} | |
GITHUB_TOKEN: ${{ secrets.DAPR_BOT_TOKEN }} | |
message: | | |
# Components certification test | |
🔗 **[Link to Action run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})** | |
Commit ref: ${{ env.CHECKOUT_REF }} | |
outputs: | |
test-matrix: ${{ steps.generate-matrix.outputs.test-matrix }} | |
certification: | |
name: ${{ matrix.component }} certification | |
runs-on: ubuntu-22.04 | |
env: | |
UNIQUE_ID: ${{github.run_id}}-${{github.run_attempt}} | |
defaults: | |
run: | |
shell: bash | |
needs: | |
- generate-matrix | |
strategy: | |
fail-fast: false # Keep running even if one component fails | |
matrix: | |
include: ${{ fromJson(needs.generate-matrix.outputs.test-matrix) }} | |
steps: | |
- name: Set default payload repo and ref | |
run: | | |
echo "CHECKOUT_REPO=${{ github.repository }}" >> $GITHUB_ENV | |
echo "CHECKOUT_REF=${{ github.ref }}" >> $GITHUB_ENV | |
- name: Parse repository_dispatch payload | |
if: github.event_name == 'repository_dispatch' | |
run: | | |
if [ ${{ github.event.client_payload.command }} = "ok-to-test" ]; then | |
echo "CHECKOUT_REPO=${{ github.event.client_payload.pull_head_repo }}" >> $GITHUB_ENV | |
echo "CHECKOUT_REF=${{ github.event.client_payload.pull_head_ref }}" >> $GITHUB_ENV | |
fi | |
- name: Check out code | |
uses: actions/checkout@v3 | |
with: | |
repository: ${{ env.CHECKOUT_REPO }} | |
ref: ${{ env.CHECKOUT_REF }} | |
- name: Configure environment | |
run: | | |
# Output file | |
echo "TEST_OUTPUT_FILE_PREFIX=$GITHUB_WORKSPACE/test_report" >> $GITHUB_ENV | |
# Certification test and source path | |
TEST_COMPONENT=$(echo "${{ matrix.component }}" | sed -E 's/\./\//g') | |
echo "TEST_PATH=tests/certification/${TEST_COMPONENT}" >> $GITHUB_ENV | |
SOURCE_PATH="github.com/dapr/components-contrib/${TEST_COMPONENT}" | |
echo "SOURCE_PATH=$SOURCE_PATH" >> $GITHUB_ENV | |
# converts slashes to dots in this string, so that it doesn't consider them sub-folders | |
SOURCE_PATH_LINEAR=$(echo "$SOURCE_PATH" |sed 's#/#\.#g') | |
echo "SOURCE_PATH_LINEAR=$SOURCE_PATH_LINEAR" >> $GITHUB_ENV | |
# Current time (used by Terraform) | |
echo "CURRENT_TIME=$(date --rfc-3339=date)" >> ${GITHUB_ENV} | |
- uses: Azure/login@v1 | |
with: | |
creds: ${{ secrets.AZURE_CREDENTIALS }} | |
if: matrix.required-secrets != '' | |
# Set this GitHub secret to your KeyVault, and grant the KeyVault policy to your Service Principal: | |
# az keyvault set-policy -n $AZURE_KEYVAULT --secret-permissions get list --spn $SPN_CLIENT_ID | |
# Using az cli to query keyvault as Azure/get-keyvault-secrets@v1 is deprecated | |
- name: Setup secrets | |
if: matrix.required-secrets != '' | |
env: | |
VAULT_NAME: ${{ secrets.AZURE_KEYVAULT }} | |
run: | | |
secrets="${{ matrix.required-secrets }}" | |
for secretName in $(echo -n $secrets | tr ',' ' '); do | |
value=$(az keyvault secret show \ | |
--name $secretName \ | |
--vault-name $VAULT_NAME \ | |
--query value \ | |
--output tsv) | |
echo "::add-mask::$value" | |
echo "$secretName=$value" >> $GITHUB_OUTPUT | |
echo "$secretName=$value" >> $GITHUB_ENV | |
done | |
# Download the required certificates into files, and set env var pointing to their names | |
- name: Setup certs | |
if: matrix.required-certs != '' | |
working-directory: ${{ env.TEST_PATH }} | |
run: | | |
for CERT_NAME in $(echo "${{ matrix.required-certs }}" | sed 's/,/ /g'); do | |
CERT_FILE=$(mktemp --suffix .pfx) | |
echo "Downloading cert $CERT_NAME into file $CERT_FILE" | |
rm $CERT_FILE && \ | |
az keyvault secret download --vault-name ${{ secrets.AZURE_KEYVAULT }} --name $CERT_NAME --encoding base64 --file $CERT_FILE | |
echo 'Setting $CERT_NAME to' "$CERT_FILE" | |
echo "$CERT_NAME=$CERT_FILE" >> $GITHUB_ENV | |
done | |
- name: Setup Terraform | |
uses: hashicorp/setup-terraform@v2.0.3 | |
if: matrix.require-terraform == 'true' | |
- name: Set Cloudflare env vars | |
if: matrix.require-cloudflare-credentials == 'true' | |
run: | | |
echo "CLOUDFLARE_ACCOUNT_ID=${{ secrets.CLOUDFLARE_ACCOUNT_ID }}" >> $GITHUB_ENV | |
echo "CLOUDFLARE_API_TOKEN=${{ secrets.CLOUDFLARE_API_TOKEN }}" >> $GITHUB_ENV | |
- name: Set AWS env vars | |
if: matrix.require-aws-credentials == 'true' | |
run: | | |
echo "AWS_REGION=us-west-1" >> $GITHUB_ENV | |
echo "AWS_ACCESS_KEY=${{ secrets.AWS_ACCESS_KEY }}" >> $GITHUB_ENV | |
echo "AWS_SECRET_KEY=${{ secrets.AWS_SECRET_KEY }}" >> $GITHUB_ENV | |
- name: Configure AWS Credentials | |
uses: aws-actions/configure-aws-credentials@v1 | |
if: matrix.require-aws-credentials == 'true' | |
with: | |
aws-access-key-id: "${{ secrets.AWS_ACCESS_KEY }}" | |
aws-secret-access-key: "${{ secrets.AWS_SECRET_KEY }}" | |
aws-region: "${{ env.AWS_REGION }}" | |
- name: Set up Go | |
id: setup-go | |
uses: actions/setup-go@v3 | |
with: | |
go-version-file: 'go.mod' | |
- name: Download Go dependencies | |
working-directory: ${{ env.TEST_PATH }} | |
run: | | |
go mod download | |
go install gotest.tools/gotestsum@latest | |
go install github.com/axw/gocov/gocov@v1.1.0 | |
- name: Check that go mod tidy is up-to-date | |
working-directory: ${{ env.TEST_PATH }} | |
run: | | |
# Get just the major version and major update (e.g. "1.20") from the installed Go version | |
GOVER=$(echo "${{ steps.setup-go.outputs.go-version }}" | cut -d '.' -f 1,2) | |
go mod tidy -compat=$GOVER | |
git diff --exit-code ./go.mod | |
git diff --exit-code ./go.sum | |
- name: Run setup script | |
if: matrix.setup-script != '' | |
run: .github/scripts/components-scripts/${{ matrix.setup-script }} | |
- name: Catch setup failures | |
if: failure() | |
run: | | |
echo "CERTIFICATION_FAILURE=true" >> $GITHUB_ENV | |
- name: Run tests | |
continue-on-error: false | |
working-directory: ${{ env.TEST_PATH }} | |
env: | |
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY }} | |
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY }} | |
AWS_REGION: "${{ env.AWS_REGION }}" | |
run: | | |
echo "Running certification tests for ${{ matrix.component }} ... " | |
export GOLANG_PROTOBUF_REGISTRATION_CONFLICT=ignore | |
set +e | |
gotestsum --jsonfile ${{ env.TEST_OUTPUT_FILE_PREFIX }}_certification.json \ | |
--junitfile ${{ env.TEST_OUTPUT_FILE_PREFIX }}_certification.xml --format standard-quiet -- \ | |
-coverprofile=cover.out -covermode=set -tags=certtests -coverpkg=${{ env.SOURCE_PATH }} | |
status=$? | |
echo "Completed certification tests for ${{ matrix.component }} ... " | |
if test $status -ne 0; then | |
echo "Setting CERTIFICATION_FAILURE" | |
export CERTIFICATION_FAILURE=true | |
fi | |
set -e | |
COVERAGE_REPORT=$(gocov convert cover.out | gocov report) | |
COVERAGE_LINE=$(echo $COVERAGE_REPORT | grep -oP '(?<=Total Coverage:).*') # example: "80.00% (40/50)" | |
COVERAGE_PERCENTAGE=$(echo $COVERAGE_LINE | grep -oP '([0-9\.]*)' | head -n 1) # example "80.00" | |
echo "COVERAGE_LINE=$COVERAGE_LINE" >> $GITHUB_ENV | |
echo "COMPONENT_PERCENTAGE=$COVERAGE_PERCENTAGE" >> $GITHUB_ENV | |
# Fail the step if we found no test to run | |
if grep -q "\[no test files\]" ${{ env.TEST_OUTPUT_FILE_PREFIX }}_certification.json ; then | |
echo "::error:: No certification test file was found for component ${{ matrix.component }}" | |
exit -1 | |
fi | |
for CERT_NAME in $(echo "${{ matrix.required-certs }}" | sed 's/,/ /g'); do | |
CERT_FILE=$(printenv $CERT_NAME) | |
echo "Cleaning up the certificate file $CERT_FILE..." | |
rm $CERT_FILE || true | |
done | |
if [[ -v CERTIFICATION_FAILURE ]]; then | |
echo "CERTIFICATION_FAILURE=true" >> $GITHUB_ENV | |
exit 1 | |
else | |
echo "CERTIFICATION_FAILURE=false" >> $GITHUB_ENV | |
fi | |
- name: Prepare test result info | |
if: always() | |
run: | | |
mkdir -p tmp/result_files | |
echo "Writing to tmp/result_files/${{ matrix.component }}.txt" | |
if [[ "${{ env.CERTIFICATION_FAILURE }}" == "true" ]]; then | |
echo "0" >> "tmp/result_files/${{ matrix.component }}.txt" | |
else | |
echo "1" >> "tmp/result_files/${{ matrix.component }}.txt" | |
fi | |
- name: Upload result files | |
uses: actions/upload-artifact@v3 | |
if: always() | |
with: | |
name: result_files | |
path: tmp/result_files | |
retention-days: 1 | |
- name: Prepare Cert Coverage Info | |
if: github.event_name == 'schedule' | |
run: | | |
mkdir -p tmp/cov_files | |
echo "${{ env.COVERAGE_LINE }}" >> tmp/cov_files/${{ env.SOURCE_PATH_LINEAR }}.txt | |
- name: Upload Cert Coverage Artifact | |
uses: actions/upload-artifact@v3 | |
if: github.event_name == 'schedule' | |
with: | |
name: certtest_cov | |
path: tmp/cov_files | |
retention-days: 1 | |
- name: Component Coverage Discord Notification | |
if: github.event_name == 'schedule' | |
env: | |
DISCORD_WEBHOOK: ${{ secrets.DISCORD_MONITORING_WEBHOOK_URL }} | |
uses: Ilshidur/action-discord@0c4b27844ba47cb1c7bee539c8eead5284ce9fa9 | |
continue-on-error: true | |
with: | |
args: 'Cert Test Coverage for {{ SOURCE_PATH }} is {{ COVERAGE_LINE }}' | |
# Upload logs for test analytics to consume | |
- name: Upload test results | |
if: always() | |
uses: actions/upload-artifact@master | |
with: | |
name: ${{ matrix.component }}_certification_test | |
path: ${{ env.TEST_OUTPUT_FILE_PREFIX }}_certification.* | |
- name: Run destroy script | |
if: always() && matrix.destroy-script != '' | |
run: .github/scripts/components-scripts/${{ matrix.destroy-script }} | |
post_job: | |
name: Post-completion | |
runs-on: ubuntu-22.04 | |
if: always() | |
needs: | |
- certification | |
- generate-matrix | |
steps: | |
- name: Parse repository_dispatch payload | |
if: github.event_name == 'repository_dispatch' | |
working-directory: ${{ github.workspace }} | |
run: | | |
if [ ${{ github.event.client_payload.command }} = "ok-to-test" ]; then | |
echo "CHECKOUT_REF=${{ github.event.client_payload.pull_head_ref }}" >> $GITHUB_ENV | |
echo "PR_NUMBER=${{ github.event.client_payload.issue.number }}" >> $GITHUB_ENV | |
fi | |
- name: Download test result artifact | |
if: always() && env.PR_NUMBER != '' | |
uses: actions/download-artifact@v3 | |
continue-on-error: true | |
id: testresults | |
with: | |
name: result_files | |
path: tmp/result_files | |
- name: Build message | |
if: always() && env.PR_NUMBER != '' | |
# Abusing of the github-script action to be able to write this in JS | |
uses: actions/github-script@v6 | |
with: | |
script: | | |
const allComponents = JSON.parse('${{ needs.generate-matrix.outputs.test-matrix }}') | |
const basePath = '${{ steps.testresults.outputs.download-path }}' | |
const testType = 'certification' | |
const fs = require('fs') | |
const path = require('path') | |
let message = `# Components ${testType} test | |
🔗 **[Link to Action run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})** | |
Commit ref: ${{ env.CHECKOUT_REF }}` | |
let allSuccess = true | |
let allFound = true | |
let notSuccess = [] | |
let notFound = [] | |
for (let i = 0; i < allComponents.length; i++) { | |
let component = allComponents[i] | |
if (!component) { | |
continue | |
} | |
if (typeof component == 'object') { | |
component = component.component | |
} | |
let found = false | |
let success = false | |
try { | |
let read = fs.readFileSync(path.join(basePath, component + '.txt'), 'utf8') | |
read = read.split('\n')[0] | |
switch (read) { | |
case '1': | |
found = true | |
success = true | |
break | |
case '0': | |
found = true | |
success = false | |
} | |
} catch (e) { | |
// ignore errors, leave found = false | |
} | |
if (!found) { | |
allFound = false | |
notFound.push(component) | |
} | |
if (!success) { | |
allSuccess = false | |
notSuccess.push(component) | |
} | |
} | |
if (allSuccess) { | |
if (allFound) { | |
message += '\n\n' + `# ✅ All ${testType} tests passed | |
All tests have reported a successful status` + '\n\n' | |
} else { | |
message += '\n\n' + `# ⚠️ Some ${testType} tests did not report status | |
Although there were no failures reported, some tests did not report a status:` + '\n\n' | |
for (let i = 0; i < notFound.length; i++) { | |
message += '- ' + notFound[i] + '\n' | |
} | |
message += '\n' | |
} | |
} else { | |
message += '\n\n' + `# ❌ Some ${testType} tests failed | |
These tests failed:` + '\n\n' | |
for (let i = 0; i < notSuccess.length; i++) { | |
message += '- ' + notSuccess[i] + '\n' | |
} | |
message += '\n' | |
if (!allFound) { | |
message += 'Additionally, some tests did not report a status:\n\n' | |
for (let i = 0; i < notFound.length; i++) { | |
message += '- ' + notFound[i] + '\n' | |
} | |
message += '\n' | |
} | |
} | |
fs.writeFileSync('message.txt', message) | |
- name: Replace PR comment | |
if: always() && env.PR_NUMBER != '' | |
uses: artursouza/sticky-pull-request-comment@da9e86aa2a80e4ae3b854d251add33bd6baabcba | |
with: | |
header: ${{ github.run_id }} | |
number: ${{ env.PR_NUMBER }} | |
GITHUB_TOKEN: ${{ secrets.DAPR_BOT_TOKEN }} | |
path: message.txt | |
- name: Download Cert Coverage Artifact | |
uses: actions/download-artifact@v3 | |
continue-on-error: true | |
if: success() && github.event_name == 'schedule' | |
id: download | |
with: | |
name: certtest_cov | |
path: tmp/cov_files | |
- name: Calculate total coverage | |
if: success() && github.event_name == 'schedule' | |
run: | | |
threshold=60.0 | |
echo "threshold=$threshold" >> $GITHUB_ENV | |
aboveThreshold=0 | |
totalFiles=0 | |
ls "${{steps.download.outputs.download-path}}" | while read f; do | |
while read LINE; | |
do | |
ratio=$(echo $LINE | cut -d "(" -f2 | cut -d ")" -f1) | |
prcnt=$(echo $LINE | cut -d "(" -f1 | cut -d ")" -f1) | |
tempPrcnt=$(echo $prcnt | cut -d'%' -f1) | |
if [ $tempPrcnt \> $threshold ]; then aboveThreshold=$(($aboveThreshold+1)); fi | |
totalFiles=$(($totalFiles+1)) | |
tempNumerator=$(echo $ratio | cut -d'/' -f1) | |
tempDenominator=$(echo $ratio | cut -d'/' -f2) | |
export numerator=$(($numerator+$tempNumerator)) | |
export denominator=$(($denominator+$tempDenominator)) | |
totalPer=$(awk "BEGIN { print (($numerator / $denominator) * 100) }") | |
echo "totalPer=$totalPer" >> $GITHUB_ENV | |
echo "aboveThreshold=$aboveThreshold" >> $GITHUB_ENV | |
echo "totalFiles=$totalFiles" >> $GITHUB_ENV | |
done < "${{steps.download.outputs.download-path}}/$f" | |
done | |
continue-on-error: true | |
- name: Final Coverage Discord Notification | |
if: success() && github.event_name == 'schedule' | |
env: | |
DISCORD_WEBHOOK: ${{ secrets.DISCORD_MONITORING_WEBHOOK_URL }} | |
uses: Ilshidur/action-discord@0c4b27844ba47cb1c7bee539c8eead5284ce9fa9 | |
continue-on-error: true | |
with: | |
args: 'Total Coverage for Certification Tests is {{ totalPer }}%. {{ aboveThreshold }} out of {{ totalFiles }} components have certification tests with code coverage > {{ threshold }}%' |