GEOS-Chem submod update: Merge PR #2187 (Update intgration/parallel t… #444
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Release pipeline: | |
| # | |
| # This pipeline triggers the cloud-based benchmarking workflow | |
| # upon pushes to the dev environment. The benchmarking workflow | |
| # infrastructure code can be found in the following repository: | |
| # https://github.com/geoschem/gc-cloud-infrastructure | |
| # | |
| # This pipeline is triggered by pushes to dev | |
| # | |
| # Notes: | |
| # - This workflow requires aws credentials necessary to | |
| # trigger the benchmarking step function via the aws cli. | |
| # The credentials need step function permissions and can | |
| # be added to the repo as an action secret called | |
| # AWS_SECRET_ACCESS_KEY and AWS_ACCESS_KEY_ID. | |
| name: cloud_benchmarking | |
| on: | |
| push: | |
| branches: | |
| - dev** | |
| tags: | |
| - '*' | |
| pull_request: | |
| branches: | |
| - dev** | |
| jobs: | |
| trigger_step_function: | |
| runs-on: ubuntu-latest # aws cli comes pre-installed | |
| steps: | |
| # for now both use Spot instances -- may need to update to use on demand | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set Initial Variables | |
| # By default we use 1Hr benchmarks | |
| run: | | |
| echo "TIME_PERIOD=1Hr" >> $GITHUB_ENV | |
| echo "RESOLUTION=24" >> $GITHUB_ENV | |
| echo "GITHUB_SHA_SHORT=`echo ${GITHUB_SHA} | cut -c1-7`" >> $GITHUB_ENV | |
| echo "COMMIT_NAME=`echo ${GITHUB_SHA} | cut -c1-7`" >> $GITHUB_ENV | |
| # conditionally overwrite variables if a tag was the triggering event | |
| - name: Reset Initial Variables for pull request | |
| run: | | |
| echo "GITHUB_SHA_SHORT=`echo ${{ github.event.pull_request.head.sha }} | cut -c1-7`" >> $GITHUB_ENV | |
| echo "COMMIT_NAME=`echo ${{ github.event.pull_request.head.sha }} | cut -c1-7`" >> $GITHUB_ENV | |
| if: github.event_name == 'pull_request' | |
| - name: Reset Variables For Tags | |
| # We do a 1Month benchmark for tags | |
| run: | | |
| echo "TIME_PERIOD=1Mon" >> $GITHUB_ENV | |
| echo "COMMIT_NAME=${GITHUB_REF##*/}" >> $GITHUB_ENV | |
| if: startsWith(github.ref, 'refs/tags/') | |
| - name: Generate Primary Key | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_DEFAULT_REGION: us-east-1 | |
| AWS_DEFAULT_OUTPUT: json | |
| run: | | |
| cd ${{ github.workspace }} | |
| echo "DEV_PRIMARY_KEY=gchp-c${RESOLUTION}-${TIME_PERIOD}-`git describe --tags`" >> $GITHUB_ENV | |
| echo "REF_PRIMARY_KEY=`./.github/workflows/findRefKey.sh ${RESOLUTION} ${TIME_PERIOD} ${GITHUB_SHA}`" >> $GITHUB_ENV | |
| - name: Trigger Step Function | |
| env: | |
| # Set config options for aws cli | |
| NUM_CORES: 62 | |
| EC2_TYPE: SPOT # SPOT or DEMAND | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_DEFAULT_REGION: us-east-1 | |
| AWS_DEFAULT_OUTPUT: json | |
| # Note the step function input is sent in as a single json string | |
| run: | | |
| export WORKFLOW_ARN=`aws stepfunctions list-state-machines --output json \ | |
| | jq -r '.stateMachines[] | select(.name == "benchmarks-cloud-workflow").stateMachineArn'` | |
| aws stepfunctions start-execution \ | |
| --name ${DEV_PRIMARY_KEY} \ | |
| --state-machine-arn ${WORKFLOW_ARN} \ | |
| --input "{\"event\": {"` | |
| `"\"nameSuffix\": \"${GITHUB_SHA_SHORT}\","` | |
| `"\"primaryKey\": \"${DEV_PRIMARY_KEY}\","` | |
| `"\"simulationType\": \"gchp\","` | |
| `"\"runType\": \"${EC2_TYPE}\","` | |
| `"\"timePeriod\": \"${TIME_PERIOD}\","` | |
| `"\"tag\": \"${COMMIT_NAME}\","` | |
| `"\"numCores\": \"${NUM_CORES}\","` | |
| `"\"memory\": \"80000\","` | |
| `"\"resolution\": \"${RESOLUTION}\","` | |
| `"\"sendEmailNotification\": \"true\""` | |
| `"},"` | |
| `"\"plotting\": {"` | |
| `"\"devPrimaryKey\": \"${DEV_PRIMARY_KEY}\","` | |
| `"\"refPrimaryKey\": \"${REF_PRIMARY_KEY}\""` | |
| `"}}" |