[DC-1225] Add scripts directory and move files #6894
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Unit, Smoke, Connected and Integration tests | |
| on: | |
| workflow_dispatch: {} | |
| pull_request: | |
| schedule: | |
| - cron: '0 4 * * *' # run at 4 AM UTC, 12PM EST. | |
| # do not allow concurrent runs of this workflow on the same branch | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| jobs: | |
| test_unit: | |
| name: Unit tests | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up JDK 17 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: '17' | |
| cache: 'gradle' | |
| - name: Run unit tests | |
| env: | |
| # required for unit tests | |
| GOOGLE_APPLICATION_CREDENTIALS: jade-dev-account.json | |
| # required for sonarqube reports | |
| SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} | |
| # output plain logs instead of json | |
| TDR_LOG_APPENDER: 'Console-Standard' | |
| run: | | |
| # extract service account credentials | |
| base64 --decode <<< ${{ secrets.SA_B64_CREDENTIALS }} > ${GOOGLE_APPLICATION_CREDENTIALS} | |
| # assemble code, run unit tests, and generate scan | |
| ./gradlew --scan assemble check jacocoTestReport sonar | |
| test_connected: | |
| name: Connected tests | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 180 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up JDK 17 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: '17' | |
| cache: 'gradle' | |
| - name: Run connected tests | |
| env: | |
| # required for azure tests | |
| AZURE_CREDENTIALS_APPLICATIONID: 0e29ec36-04e8-44d5-ae7c-50dc15135571 | |
| AZURE_CREDENTIALS_HOMETENANTID: fad90753-2022-4456-9b0a-c7e5b934e408 | |
| AZURE_CREDENTIALS_SECRET: ${{ secrets.AZURE_CREDENTIALS_SECRET }} | |
| # required for synapse tests | |
| AZURE_SYNAPSE_SQLADMINPASSWORD: ${{ secrets.AZURE_SYNAPSE_SQLADMINPASSWORD }} | |
| AZURE_SYNAPSE_SQLADMINUSER: ${{ secrets.AZURE_SYNAPSE_SQLADMINUSER }} | |
| AZURE_SYNAPSE_WORKSPACENAME: tdr-snps-int-east-us-ondemand.sql.azuresynapse.net | |
| # required for connected tests | |
| GOOGLE_APPLICATION_CREDENTIALS: jade-dev-account.json | |
| # required data project for snapshotTest | |
| GOOGLE_CLOUD_DATA_PROJECT: broad-jade-integration-data | |
| # required for testAzureBillingProfile | |
| # uses an azure marketplace app with this hardcoded deployment email | |
| JADE_USER_EMAIL: connected-tdr-user@notarealemail.org | |
| # required for rbs tests | |
| RBS_CLIENT_CREDENTIAL_FILE_PATH: rbs-tools-sa.json | |
| # output plain logs instead of json | |
| TDR_LOG_APPENDER: 'Console-Standard' | |
| run: | | |
| # extract service account credentials | |
| base64 --decode <<< ${{ secrets.SA_B64_CREDENTIALS }} > ${GOOGLE_APPLICATION_CREDENTIALS} | |
| base64 --decode <<< ${{ secrets.B64_RBS_APPLICATION_CREDENTIALS }} > ${RBS_CLIENT_CREDENTIAL_FILE_PATH} | |
| # assemble code and run connected tests | |
| ./gradlew --scan assemble testConnected | |
| test_integration: | |
| name: Integration tests | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 300 | |
| services: | |
| postgres: | |
| image: postgres:11 | |
| env: | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: postgres | |
| ports: | |
| - 5432:5432 | |
| options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up JDK 17 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: '17' | |
| cache: 'gradle' | |
| - name: Run integration tests | |
| env: | |
| # required for azure tests | |
| AZURE_CREDENTIALS_APPLICATIONID: 0e29ec36-04e8-44d5-ae7c-50dc15135571 | |
| AZURE_CREDENTIALS_HOMETENANTID: fad90753-2022-4456-9b0a-c7e5b934e408 | |
| AZURE_CREDENTIALS_SECRET: ${{ secrets.AZURE_CREDENTIALS_SECRET }} | |
| AZURE_SYNAPSE_SQLADMINUSER: ${{ secrets.AZURE_SYNAPSE_SQLADMINUSER }} | |
| AZURE_SYNAPSE_SQLADMINPASSWORD: ${{ secrets.AZURE_SYNAPSE_SQLADMINPASSWORD }} | |
| AZURE_SYNAPSE_WORKSPACENAME: tdr-snps-int-east-us-ondemand.sql.azuresynapse.net | |
| # required for integration tests | |
| GOOGLE_APPLICATION_CREDENTIALS: jade-dev-account.json | |
| GOOGLE_SA_CERT: jade-dev-account.pem | |
| IT_JADE_API_URL: http://localhost:8080 | |
| # postgres connection details | |
| PGHOST: 127.0.0.1 | |
| PGPASSWORD: postgres | |
| # required for integration tests | |
| RBS_CLIENT_CREDENTIAL_FILE_PATH: rbs-tools-sa.json | |
| RBS_INSTANCEURL: https://buffer.tools.integ.envs.broadinstitute.org | |
| RBS_POOLID: datarepo_v1 | |
| # output plain logs instead of json | |
| TDR_LOG_APPENDER: 'Console-Standard' | |
| run: | | |
| # wait for postgres to be ready | |
| pg_isready -h ${PGHOST} -t 30 | |
| # create the datarepo and stairway databases | |
| psql -U postgres -f ./scripts/init-db/postgres-init.sql | |
| # extract service account credentials | |
| base64 --decode <<< ${{ secrets.SA_B64_CREDENTIALS }} > ${GOOGLE_APPLICATION_CREDENTIALS} | |
| jq -r .private_key ${GOOGLE_APPLICATION_CREDENTIALS} > ${GOOGLE_SA_CERT} | |
| chmod 644 ${GOOGLE_SA_CERT} | |
| base64 --decode <<< ${{ secrets.B64_RBS_APPLICATION_CREDENTIALS }} > ${RBS_CLIENT_CREDENTIAL_FILE_PATH} | |
| # build code and run local api | |
| ./gradlew bootJar | |
| export DATA_REPO_JAR=$(find . -type f -name jade-data-repo-*-SNAPSHOT.jar) | |
| java -jar ${DATA_REPO_JAR} > local-api-output.log & | |
| # wait until api is ready | |
| timeout 30 bash -c 'until curl -s ${IT_JADE_API_URL}/status; do sleep 1; done' | |
| # run integration tests | |
| ./gradlew --scan testIntegration | |
| - name: Upload API logs | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: integration-test-api-logs | |
| path: local-api-output.log | |
| retention-days: 7 | |
| git_hash: | |
| name: Extract git hash | |
| runs-on: ubuntu-latest | |
| outputs: | |
| version: ${{ steps.config.outputs.GIT_HASH }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Get the latest git hash | |
| id: config | |
| run: | | |
| GIT_HASH=$(git rev-parse --short HEAD) | |
| echo "GIT_HASH=${GIT_HASH}" >> $GITHUB_OUTPUT | |
| echo "Latest git hash in branch is ${GIT_HASH}" | |
| report-to-sherlock: | |
| # only runs on pull requests and reports the appVersion even if tests fail | |
| if: github.event_name == 'pull_request' | |
| uses: broadinstitute/sherlock/.github/workflows/client-report-app-version.yaml@main | |
| needs: git_hash | |
| with: | |
| new-version: ${{ needs.git_hash.outputs.version }} | |
| chart-name: 'datarepo' | |
| permissions: | |
| contents: read | |
| id-token: write | |
| report-workflow: | |
| if: github.ref == 'refs/heads/develop' | |
| uses: broadinstitute/sherlock/.github/workflows/client-report-workflow.yaml@main | |
| # dependency is not required but makes the action run page more readable | |
| needs: git_hash | |
| with: | |
| relates-to-chart-releases: 'datarepo-dev' | |
| notify-slack-channels-upon-workflow-failure: ${{ vars.SLACK_NOTIFICATION_CHANNELS }} | |
| notify-slack-channels-upon-workflow-retry: ${{ vars.SLACK_NOTIFICATION_CHANNELS }} | |
| permissions: | |
| id-token: write |