Update Perf Env and Run Nightly Test Runner Tests #1476
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Update Perf Env and Run Nightly Test Runner Tests | |
| env: | |
| GOOGLE_APPLICATION_CREDENTIALS: /tmp/jade-dev-account.json | |
| GOOGLE_CLOUD_PROJECT: broad-jade-perf | |
| GOOGLE_CLOUD_DATA_PROJECT: broad-jade-perf-data2 | |
| JADE_USER_EMAIL: dev-tdr-user@notarealemail.org | |
| TEST_RUNNER_SERVER_SPECIFICATION_FILE: perf.json | |
| GOOGLE_ZONE: us-central1 | |
| K8_CLUSTER: jade-master-us-central1 | |
| K8_NAMESPACES: perf | |
| TDR_LOG_APPENDER: Console-Standard | |
| AZURE_CREDENTIALS_APPLICATIONID: 22cb243c-f1a5-43d8-8f12-6566bcce6542 | |
| on: | |
| workflow_dispatch: {} | |
| schedule: | |
| - cron: '0 4 * * *' # run at 4 AM UTC, 12PM EST. | |
| jobs: | |
| test-runner-perf: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: "Fetch latest semantic version from data-repo dev" | |
| id: "read_property" | |
| run: | | |
| CURRENT_VERSION=$(curl -s -X GET "https://jade-perf.datarepo-perf.broadinstitute.org/configuration" -H "accept: application/json" | jq -r '.semVer|rtrimstr("-SNAPSHOT")') | |
| echo "Current Version: $CURRENT_VERSION" | |
| echo "::set-output name=CURRENT_SEMVER::$CURRENT_VERSION" | |
| LATEST_VERSION=$(curl -s -X GET "https://jade.datarepo-dev.broadinstitute.org/configuration" -H "accept: application/json" | jq -r '.semVer|rtrimstr("-SNAPSHOT")') | |
| echo "Latest Version: $LATEST_VERSION" | |
| echo "::set-output name=LATEST_VERSION::$LATEST_VERSION" | |
| LATEST_GITHASH=$(curl -s -X GET "https://jade.datarepo-dev.broadinstitute.org/configuration" -H "accept: application/json" | jq -r '.gitHash') | |
| echo "Latest git hash: $LATEST_GITHASH" | |
| echo "::set-output name=LATEST_GITHASH::$LATEST_GITHASH" | |
| - name: "Checkout jade-data-repo ${{ steps.read_property.outputs.LATEST_VERSION }} branch" | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ steps.read_property.outputs.LATEST_VERSION }} | |
| - name: "Setup Java 17" | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: 'temurin' | |
| java-version: '17' | |
| cache: 'gradle' | |
| - name: "Import Vault perf secrets" | |
| uses: hashicorp/vault-action@v2.5.0 | |
| with: | |
| url: ${{ secrets.VAULT_ADDR }} | |
| method: approle | |
| roleId: ${{ secrets.PERF_ROLE_ID }} | |
| secretId: ${{ secrets.PERF_SECRET_ID }} | |
| secrets: | | |
| secret/dsde/datarepo/perf/helm-azure applicationsecret | AZURE_CREDENTIALS_SECRET ; | |
| secret/dsde/datarepo/perf/datarepo-api-sa key | B64_APPLICATION_CREDENTIALS ; | |
| - name: "Perform IAM policy cleanup for perf" | |
| run: | | |
| # write vault tokens | |
| base64 --decode <<< ${B64_APPLICATION_CREDENTIALS} > ${GOOGLE_APPLICATION_CREDENTIALS} | |
| gcloud auth activate-service-account --key-file ${GOOGLE_APPLICATION_CREDENTIALS} | |
| ./tools/cleanupPolicies.sh ${GOOGLE_CLOUD_DATA_PROJECT} | |
| - name: "Import Vault dev secrets" | |
| uses: hashicorp/vault-action@v2.5.0 | |
| with: | |
| url: ${{ secrets.VAULT_ADDR }} | |
| method: approle | |
| roleId: ${{ secrets.ROLE_ID }} | |
| secretId: ${{ secrets.SECRET_ID }} | |
| secrets: | | |
| secret/dsde/datarepo/dev/sa-key-b64 sa | B64_APPLICATION_CREDENTIALS ; | |
| - name: "Configure credentials" | |
| run: | | |
| # write vault tokens | |
| base64 --decode <<< ${B64_APPLICATION_CREDENTIALS} > ${GOOGLE_APPLICATION_CREDENTIALS} | |
| gcloud config set project ${GOOGLE_CLOUD_PROJECT} --quiet | |
| gcloud config set compute/zone ${GOOGLE_ZONE} --quiet | |
| gcloud auth activate-service-account --key-file ${GOOGLE_APPLICATION_CREDENTIALS} | |
| gcloud auth configure-docker --quiet | |
| if [[ -n "${K8_CLUSTER}" ]]; then | |
| gcloud container clusters get-credentials ${K8_CLUSTER} --zone ${GOOGLE_ZONE} | |
| fi | |
| - name: "Set up Cloud SDK" | |
| uses: google-github-actions/setup-gcloud@v1 | |
| with: | |
| install_components: 'gke-gcloud-auth-plugin' | |
| - name: 'Get GKE Credentials' | |
| uses: 'google-github-actions/get-gke-credentials@v1' | |
| with: | |
| cluster_name: ${{ env.K8_CLUSTER }} | |
| location: ${{ env.GOOGLE_ZONE }} | |
| project_id: ${{ env.GOOGLE_CLOUD_PROJECT }} | |
| - name: "Whitelist Runner IP" | |
| uses: broadinstitute/datarepo-actions/actions/main@0.68.0 | |
| env: | |
| GOOGLE_SA_CERT: 'jade-dev-account.pem' | |
| with: | |
| actions_subcommand: 'gcp_whitelist' | |
| role_id: ${{ secrets.ROLE_ID }} | |
| secret_id: ${{ secrets.SECRET_ID }} | |
| google_project: broad-jade-perf | |
| - name: "Check that perf namespace is available and set state lock" | |
| uses: broadinstitute/datarepo-actions/actions/main@0.68.0 | |
| with: | |
| actions_subcommand: 'k8_checknamespace' | |
| k8_namespaces: 'perf' | |
| - name: '[Update API version on Perf] Checkout datarepo-helm-definitions repo' | |
| if: ${{ steps.read_property.outputs.LATEST_VERSION != steps.read_property.outputs.CURRENT_SEMVER }} | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: 'broadinstitute/datarepo-helm-definitions' | |
| token: ${{ secrets.BROADBOT_TOKEN }} | |
| path: datarepo-helm-definitions | |
| - name: "[Update API version on Perf] Update perf image tag with semVer" | |
| if: ${{ steps.read_property.outputs.LATEST_VERSION != steps.read_property.outputs.CURRENT_SEMVER }} | |
| uses: docker://mikefarah/yq:3.3.4 | |
| with: | |
| args: yq w -i datarepo-helm-definitions/perf/datarepo/datarepo-api.yaml image.tag ${{ steps.read_property.outputs.LATEST_VERSION }} | |
| - name: "[Update API version on Perf] [datarepo-helm-definitions] Merge version update" | |
| if: ${{ steps.read_property.outputs.LATEST_VERSION != steps.read_property.outputs.CURRENT_SEMVER }} | |
| uses: broadinstitute/datarepo-actions/actions/merger@0.67.0 | |
| env: | |
| COMMIT_MESSAGE: "Perf Datarepo version update: ${{ steps.read_property.outputs.LATEST_VERSION }}" | |
| GITHUB_REPO: datarepo-helm-definitions | |
| SWITCH_DIRECTORIES: "true" | |
| MERGE_BRANCH: master | |
| - name: "[Update API version on Perf] Install Helmfile" | |
| if: ${{ steps.read_property.outputs.LATEST_VERSION != steps.read_property.outputs.CURRENT_SEMVER }} | |
| uses: broadinstitute/setup-helmfile@v0.6.0 #Forked from mamezou-tech/setup-helmfile | |
| - name: "[Update API version on Perf] Use helmfile reapply helm for api pod to update version" | |
| if: ${{ steps.read_property.outputs.LATEST_VERSION != steps.read_property.outputs.CURRENT_SEMVER }} | |
| run: | | |
| helmfile --version | |
| cd ${GITHUB_WORKSPACE}/${workingDir}/datarepo-helm-definitions/perf | |
| echo "Apply helm updates, including updated data-repo version" | |
| helmfile apply | |
| cd ${GITHUB_WORKSPACE}/${workingDir} | |
| - name: "[Clear Perf Database] Import Perf Database Secret" | |
| uses: hashicorp/vault-action@v2.5.0 | |
| with: | |
| url: ${{ secrets.VAULT_ADDR }} | |
| method: approle | |
| roleId: ${{ secrets.PERF_ROLE_ID }} | |
| secretId: ${{ secrets.PERF_SECRET_ID }} | |
| #Yes, they both point to the stairway database password | |
| secrets: | | |
| secret/dsde/datarepo/perf/datarepo-sql-db stairwaypassword | ORG_GRADLE_PROJECT_dbDatarepoPassword ; | |
| secret/dsde/datarepo/perf/datarepo-sql-db stairwaypassword | ORG_GRADLE_PROJECT_dbStairwayPassword ; | |
| - name: "[Clear Perf Database] Perform DropAll" | |
| env: | |
| ORG_GRADLE_PROJECT_dbDatarepoUri: jdbc:postgresql://localhost:5434/datarepo | |
| ORG_GRADLE_PROJECT_dbStairwayUri: jdbc:postgresql://localhost:5434/stairway | |
| run: | | |
| # port foward to perf database | |
| kubectl port-forward --namespace perf svc/perf-jade-gcloud-sqlproxy 5434:5432 & | |
| PID=$! | |
| sleep 10 | |
| echo "After giving the proxy 10 seconds to spin up, run drop all on perf database" | |
| cd ${GITHUB_WORKSPACE}/${workingDir} | |
| ./gradlew dropAll | |
| kill ${PID} | |
| - name: "[Clear Perf Database] Cycle pods to initialize databases" | |
| run: | | |
| echo "Cycle API pods to trigger migrations to initialize databases" | |
| kubectl delete pods -n perf -l app.kubernetes.io/name=datarepo-api | |
| echo "Sleep 45 seconds to give the pods a chance to start cycling before checking if up and on correct version" | |
| sleep 45 | |
| - name: "Wait for Perf Cluster to come back up with correct version" | |
| uses: broadinstitute/datarepo-actions/actions/wait-for-deployment@0.67.0 | |
| env: | |
| NAMESPACEINUSE: perf | |
| IT_JADE_API_URL: "https://jade-perf.datarepo-perf.broadinstitute.org" | |
| DESIRED_GITHASH: ${{ steps.read_property.outputs.LATEST_GITHASH }} | |
| DEPLOYMENT_TYPE: 'api' | |
| - name: "Build and run Test Runner" | |
| run: | | |
| cd ${GITHUB_WORKSPACE}/${workingDir} | |
| echo "Building Data Repo client library" | |
| export TEST_RUNNER_SERVER_SPECIFICATION_FILE="perf.json" | |
| ENABLE_SUBPROJECT_TASKS=1 ./gradlew :datarepo-client:clean :datarepo-client:assemble | |
| cd ${GITHUB_WORKSPACE}/${workingDir}/datarepo-clienttests | |
| export ORG_GRADLE_PROJECT_datarepoclientjar=$(find .. -type f -name "datarepo-client*.jar") | |
| echo "ORG_GRADLE_PROJECT_datarepoclientjar = ${ORG_GRADLE_PROJECT_datarepoclientjar}" | |
| echo "Running test suite" | |
| ./gradlew runTest --args="suites/NightlyPerfWorkflow.json tmp/TestRunnerResults" | |
| echo "Collecting measurements" | |
| ./gradlew collectMeasurements --args="NightlyPerfWorkflow.json tmp/TestRunnerResults" | |
| echo "Uploading results" | |
| ./gradlew uploadResults --args="BroadJadeDev.json tmp/TestRunnerResults" | |
| cd ${GITHUB_WORKSPACE}/${workingDir} | |
| - name: 'Re-obtain GKE Credentials' | |
| if: always() | |
| uses: 'google-github-actions/get-gke-credentials@v1' | |
| with: | |
| cluster_name: ${{ env.K8_CLUSTER }} | |
| location: ${{ env.GOOGLE_ZONE }} | |
| project_id: ${{ env.GOOGLE_CLOUD_PROJECT }} | |
| - name: "Unlock perf namespace if we locked it in this run" | |
| if: always() # the task always runs, but the lock is only cleared if this run set it. | |
| uses: broadinstitute/datarepo-actions/actions/main@0.68.0 | |
| with: | |
| actions_subcommand: 'k8_checknamespace_clean' | |
| - name: "Clean whitelisted Runner IP" | |
| if: always() | |
| uses: broadinstitute/datarepo-actions/actions/main@0.68.0 | |
| env: | |
| GOOGLE_SA_CERT: 'jade-dev-account.pem' | |
| with: | |
| actions_subcommand: 'gcp_whitelist_clean' | |
| google_project: broad-jade-perf | |
| - name: "Notify Jade Slack" | |
| if: always() | |
| uses: broadinstitute/action-slack@v3.15.0 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| with: | |
| status: ${{ job.status }} | |
| channel: "#jade-alerts" | |
| username: "Data Repo tests" | |
| text: "Perf tests" | |
| fields: repo,message,commit,author,action,eventName,ref,workflow,job,took |