Unit, Smoke, Connected and Integration tests #6682
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Unit, Smoke, Connected and Integration tests | |
| env: | |
| K8_CLUSTER: 'integration-master' | |
| # This must be defined for the bash redirection | |
| GOOGLE_APPLICATION_CREDENTIALS: 'jade-dev-account.json' | |
| # This must be defined for the bash redirection | |
| GOOGLE_SA_CERT: 'jade-dev-account.pem' | |
| # This will make test logs not use JSON | |
| TDR_LOG_APPENDER: 'Console-Standard' | |
| # The default Azure credentials to use to authenticate | |
| AZURE_CREDENTIALS_APPLICATIONID: 0e29ec36-04e8-44d5-ae7c-50dc15135571 | |
| AZURE_CREDENTIALS_HOMETENANTID: fad90753-2022-4456-9b0a-c7e5b934e408 | |
| JADE_USER_EMAIL: connected-tdr-user@notarealemail.org | |
| RBS_CLIENT_CREDENTIAL_FILE_PATH: rbs-tools-sa.json | |
| AZURE_CREDENTIALS_SECRET: ${{ secrets.AZURE_CREDENTIALS_SECRET }} | |
| AZURE_SYNAPSE_SQLADMINUSER: ${{ secrets.AZURE_SYNAPSE_SQLADMINUSER }} | |
| AZURE_SYNAPSE_SQLADMINPASSWORD: ${{ secrets.AZURE_SYNAPSE_SQLADMINPASSWORD }} | |
| AZURE_SYNAPSE_WORKSPACENAME: tdr-snps-int-east-us-ondemand.sql.azuresynapse.net | |
| CACHE_PATHS: | | |
| build/jacoco | |
| build/reports | |
| build/spotless | |
| build/test-results | |
| build/jacocoHtml | |
| on: | |
| pull_request: | |
| branches: | |
| - develop | |
| paths: | |
| - '!*' | |
| - 'src/**' | |
| - 'gradle**' | |
| - 'gradle/**' | |
| - '**.gradle' | |
| - 'Dockerfile' | |
| - 'datarepo-clienttests/**' | |
| - '.github/workflows/int-and-connected-test-run.yml' | |
| - '.swagger-codegen-ignore' | |
| workflow_dispatch: {} | |
| schedule: | |
| - cron: '0 4 * * *' # run at 4 AM UTC, 12PM EST. | |
| concurrency: | |
| # Don't run this workflow concurrently on the same branch | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| # For PRs, don't wait for completion of existing runs, cancel them instead | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| jobs: | |
| test_check: | |
| name: "Checkout, verify and run unit tests" | |
| outputs: | |
| job-status: ${{ job.status }} | |
| timeout-minutes: 60 | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest] | |
| if: "!contains( github.event.pull_request.labels.*.name, 'skip-ci')" | |
| runs-on: ${{ matrix.os }} | |
| ## skips if pr label is 'skip-ci' | |
| # run a local Postgres container in Docker for the basic check tests | |
| services: | |
| postgres: | |
| image: postgres:11 | |
| env: | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: postgres | |
| ports: | |
| - 5432:5432 | |
| options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 | |
| steps: | |
| - name: "Checkout code" | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: "Cache build" | |
| uses: actions/cache@v3 | |
| with: | |
| path: ${{ env.CACHE_PATHS }} | |
| key: ${{ runner.os }}-build-unit }} | |
| - name: "Run unit tests and sonar scan via Gradle" | |
| uses: broadinstitute/datarepo-actions/actions/main@0.74.0 | |
| with: | |
| actions_subcommand: 'gradleinttest' | |
| pgport: ${{ job.services.postgres.ports[5432] }} | |
| test_to_run: 'check' | |
| sa_b64_credentials: ${{ secrets.SA_B64_CREDENTIALS }} | |
| sonar_token: ${{ secrets.SONAR_TOKEN }} | |
| test_connected: | |
| name: "Run connected tests" | |
| outputs: | |
| job-status: ${{ job.status }} | |
| timeout-minutes: 180 | |
| needs: test_check | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest] | |
| if: "!contains( github.event.pull_request.labels.*.name, 'skip-ci')" | |
| runs-on: ${{ matrix.os }} | |
| ## skips if pr label is 'skip-ci' | |
| # run a local Postgres container in Docker for the basic check tests | |
| services: | |
| postgres: | |
| image: postgres:11 | |
| env: | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: postgres | |
| ports: | |
| - 5432:5432 | |
| options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 | |
| steps: | |
| - name: "Checkout code" | |
| uses: actions/checkout@v3 | |
| - name: "Cache build" | |
| uses: actions/cache@v3 | |
| with: | |
| path: ${{ env.CACHE_PATHS }} | |
| key: ${{ runner.os }}-build-connected | |
| - name: "Write RBS SA to a file" | |
| run: | | |
| # write vault token | |
| base64 --decode <<< ${{ secrets.B64_RBS_APPLICATION_CREDENTIALS }} > ${RBS_CLIENT_CREDENTIAL_FILE_PATH} | |
| - name: "Run connected tests via Gradle" | |
| uses: broadinstitute/datarepo-actions/actions/main@0.74.0 | |
| with: | |
| actions_subcommand: 'gradleinttest' | |
| pgport: ${{ job.services.postgres.ports[5432] }} | |
| test_to_run: 'testConnected' | |
| sa_b64_credentials: ${{ secrets.SA_B64_CREDENTIALS }} | |
| - name: "Temp: Archive all junit test reports" | |
| uses: actions/upload-artifact@v2 | |
| if: always() | |
| with: | |
| name: junit-test-reports-for-connected | |
| path: build/reports | |
| retention-days: 5 | |
| deploy_test_integration: | |
| name: "Run integration and smoke tests" | |
| outputs: | |
| job-status: ${{ job.status }} | |
| api_image_tag: ${{ steps.configuration.outputs.git_hash }} | |
| timeout-minutes: 300 | |
| needs: test_check | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest] | |
| if: "!contains( github.event.pull_request.labels.*.name, 'skip-ci')" | |
| runs-on: ${{ matrix.os }} | |
| ## skips if pr label is 'skip-ci' | |
| # run a local Postgres container in Docker for the basic check tests | |
| services: | |
| postgres: | |
| image: postgres:11 | |
| env: | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: postgres | |
| ports: | |
| - 5432:5432 | |
| options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 | |
| steps: | |
| - name: "Checkout code" | |
| uses: actions/checkout@v3 | |
| - name: "Cache build" | |
| uses: actions/cache@v3 | |
| with: | |
| path: ${{ env.CACHE_PATHS }} | |
| key: ${{ runner.os }}-build-integration | |
| - name: "Whitelist Runner IP" | |
| uses: broadinstitute/datarepo-actions/actions/main@0.74.0 | |
| with: | |
| actions_subcommand: 'gcp_whitelist' | |
| sa_b64_credentials: ${{ secrets.SA_B64_CREDENTIALS }} | |
| - name: "Check for an available namespace to deploy API to and set state lock" | |
| uses: broadinstitute/datarepo-actions/actions/main@0.74.0 | |
| with: | |
| actions_subcommand: 'k8_checknamespace' | |
| k8_namespaces: 'integration-1,integration-2,integration-3,integration-6' | |
| sa_b64_credentials: ${{ secrets.SA_B64_CREDENTIALS }} | |
| - name: "Build docker container via Gradle" | |
| uses: broadinstitute/datarepo-actions/actions/main@0.74.0 | |
| with: | |
| actions_subcommand: 'gradlebuild' # creates gcr build with git_hash tag | |
| sa_b64_credentials: ${{ secrets.SA_B64_CREDENTIALS }} | |
| - name: "Deploy to cluster with Helm" | |
| uses: broadinstitute/datarepo-actions/actions/main@0.74.0 | |
| with: | |
| actions_subcommand: 'helmdeploy' | |
| helm_create_secret_manager_secret_version: 0.0.8 | |
| helm_datarepo_api_chart_version: 0.0.730 | |
| helm_datarepo_ui_chart_version: 0.0.357 | |
| helm_gcloud_sqlproxy_chart_version: 0.19.13 | |
| helm_oidc_proxy_chart_version: 0.0.44 | |
| sa_b64_credentials: ${{ secrets.SA_B64_CREDENTIALS }} | |
| - name: "Fetch gitHash for deployed integration version" | |
| id: configuration | |
| run: | | |
| git_hash=$(git rev-parse --short HEAD) | |
| echo "git_hash=${git_hash}" >> $GITHUB_OUTPUT | |
| echo "Latest git hash for this branch: $git_hash" | |
| - name: "Wait for deployment to come back online" | |
| uses: broadinstitute/datarepo-actions/actions/wait-for-deployment@0.74.0 | |
| timeout-minutes: 20 | |
| env: | |
| DESIRED_GITHASH: ${{ steps.configuration.outputs.git_hash }} | |
| DEPLOYMENT_TYPE: 'api' | |
| - name: "Run Test Runner smoke tests via Gradle" | |
| uses: broadinstitute/datarepo-actions/actions/main@0.74.0 | |
| with: | |
| actions_subcommand: 'gradletestrunnersmoketest' | |
| sa_b64_credentials: ${{ secrets.SA_B64_CREDENTIALS }} | |
| - name: "Run integration tests via Gradle" | |
| uses: broadinstitute/datarepo-actions/actions/main@0.74.0 | |
| with: | |
| actions_subcommand: 'gradleinttest' | |
| pgport: ${{ job.services.postgres.ports[5432] }} | |
| test_to_run: 'testIntegration' | |
| sa_b64_credentials: ${{ secrets.SA_B64_CREDENTIALS }} | |
| env: | |
| AZURE_CREDENTIALS_APPLICATIONID: ${{ env.AZURE_CREDENTIALS_APPLICATIONID }} | |
| AZURE_CREDENTIALS_HOMETENANTID: ${{ env.AZURE_CREDENTIALS_HOMETENANTID }} | |
| - name: "Clean state lock from used Namespace on API deploy" | |
| if: always() | |
| uses: broadinstitute/datarepo-actions/actions/main@0.74.0 | |
| with: | |
| actions_subcommand: 'k8_checknamespace_clean' | |
| sa_b64_credentials: ${{ secrets.SA_B64_CREDENTIALS }} | |
| - name: "Clean whitelisted Runner IP" | |
| if: always() | |
| uses: broadinstitute/datarepo-actions/actions/main@0.74.0 | |
| with: | |
| actions_subcommand: 'gcp_whitelist_clean' | |
| sa_b64_credentials: ${{ secrets.SA_B64_CREDENTIALS }} | |
| report-to-sherlock: | |
| uses: broadinstitute/sherlock/.github/workflows/client-report-app-version.yaml@main | |
| needs: deploy_test_integration | |
| # Always attempt to run if pull_request, as we want to report the appVersion even if the tests fail. | |
| # never run on cron or other runs as we don't want extranaeous build reporting. | |
| if: github.event_name == 'pull_request' | |
| with: | |
| new-version: ${{ needs.deploy_test_integration.outputs.api_image_tag }} | |
| chart-name: 'datarepo' | |
| permissions: | |
| contents: 'read' | |
| id-token: 'write' | |
| publish_test_reports: | |
| name: "Save execution reports" | |
| timeout-minutes: 60 | |
| needs: | |
| - test_check | |
| - test_connected | |
| - deploy_test_integration | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest] | |
| if: always() | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: "Load unit test cache" | |
| uses: actions/cache@v3 | |
| with: | |
| path: ${{ env.CACHE_PATHS }} | |
| key: ${{ runner.os }}-build-unit | |
| - name: "Load connected test cache" | |
| uses: actions/cache@v3 | |
| with: | |
| path: ${{ env.CACHE_PATHS }} | |
| key: ${{ runner.os }}-build-connected | |
| - name: "Load integration test cache" | |
| uses: actions/cache@v3 | |
| with: | |
| path: ${{ env.CACHE_PATHS }} | |
| key: ${{ runner.os }}-build-integration | |
| - name: "Archive code coverage results" | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: code-coverage-report | |
| path: build/jacocoHtml | |
| retention-days: 10 | |
| - name: "Archive all junit test reports" | |
| uses: actions/upload-artifact@v2 | |
| if: always() | |
| with: | |
| name: junit-test-reports | |
| path: build/reports | |
| retention-days: 10 | |
| report-workflow: | |
| uses: broadinstitute/sherlock/.github/workflows/client-report-workflow.yaml@main | |
| if: ${{ github.ref == 'refs/heads/develop' }} | |
| with: | |
| relates-to-chart-releases: 'datarepo-dev' | |
| notify-slack-channels-upon-workflow-failure: ${{ vars.SLACK_NOTIFICATION_CHANNELS }} | |
| notify-slack-channels-upon-workflow-retry: ${{ vars.SLACK_NOTIFICATION_CHANNELS }} | |
| permissions: | |
| id-token: write |