ci-post-land #2471
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ci-post-land | |
| on: | |
| create: | |
| branches: [latest, release-*] | |
| push: | |
| branches: [main, latest, release-*] # gha-test-* | |
| defaults: | |
| run: | |
| shell: bash | |
| env: | |
| max_threads: 16 | |
| pre_command: cd /opt/git/diem/ | |
| jobs: | |
| prepare: | |
| runs-on: ubuntu-20.04 | |
| outputs: | |
| changes-target-branch: ${{ steps.changes.outputs.changes-target-branch }} | |
| changes-base-git-rev: ${{ steps.changes.outputs.changes-base-git-rev }} | |
| changes-base-githash: ${{ steps.changes.outputs.changes-base-githash }} | |
| changes-pull-request-number: ${{ steps.changes.outputs.changes-pull-request-number }} | |
| rust-changes: ${{ steps.rust-changes.outputs.changes-found }} | |
| base-image-changes: ${{ steps.base-image-changes.outputs.changes-found }} | |
| steps: | |
| - uses: actions/checkout@v2.4.0 | |
| with: | |
| # This ensures that the tip of the PR is checked out instead of the merge between the base ref and the tip | |
| # On `push` this value will be empty and will "do-the-right-thing" | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| fetch-depth: 0 #get all the history!!! | |
| - name: Git Hooks and Checks | |
| run: ./scripts/git-checks.sh | |
| - id: changes | |
| name: determine changes | |
| uses: diem/actions/changes@faadd16607b77dfa2231a8f366883e01717b3225 | |
| with: | |
| workflow-file: ci-post-land.yml | |
| github-token: ${{secrets.GITHUB_TOKEN}} | |
| - id: rust-changes | |
| name: find rust/cargo changes. | |
| uses: diem/actions/matches@faadd16607b77dfa2231a8f366883e01717b3225 | |
| with: | |
| pattern: '^documentation\|^docker\|^scripts\|^developers.diem.com' | |
| invert: "true" | |
| - id: base-image-changes | |
| name: find dev-setup.sh/base docker image changes | |
| uses: diem/actions/matches@faadd16607b77dfa2231a8f366883e01717b3225 | |
| with: | |
| pattern: '.github/workflows/ci-post-land.yml\|.github/actions/dockerhub_login/action.yml\|docker/ci/github/Dockerfile\|scripts/dev_setup.sh\|rust-toolchain' | |
| # NOTE: update-sscache-osx removed as cost savings effort. It strongly resembled update-sccache-ubuntu | |
| # except for a 'runs-on: macos-11', and no container. Restore when there are more osx contributors. | |
| #update-sscache-osx: | |
| # needs: prepare | |
| # runs-on: macos-11 | |
| # environment: | |
| # name: Sccache | |
| # if: ${{ needs.prepare.outputs.rust-changes == 'true' }} | |
| # steps: | |
| # - uses: actions/checkout@v2.3.4 | |
| # with: | |
| # This ensures that the tip of the PR is checked out instead of the merge between the base ref and the tip | |
| # On `push` this value will be empty and will "do-the-right-thing" | |
| # ref: ${{ github.event.pull_request.head.sha }} | |
| # fetch-depth: 0 #get all the history!!! | |
| # - uses: ./.github/actions/build-setup | |
| # - uses: actions/cache@v2.1.6 | |
| # with: | |
| # path: "/opt/cargo/git\n/opt/cargo/registry\n/opt/cargo/.package-cache" | |
| # key: crates-${{ runner.os }}-${{ hashFiles('Cargo.lock') }} | |
| # restore-keys: "crates-${{ runner.os }}" | |
| # - name: build all unit test code. | |
| # run: | | |
| # $pre_command && cargo x test --no-run --jobs ${max_threads} --unit | |
| # env: | |
| # TARGET_BRANCH: ${{ needs.prepare.outputs.changes-target-branch }} | |
| # SCCACHE_AWS_ACCESS_KEY_ID: ${{ secrets.ENV_DIEM_S3_AWS_ACCESS_KEY_ID }} | |
| # SCCACHE_AWS_SECRET_ACCESS_KEY: ${{ secrets.ENV_DIEM_S3_AWS_SECRET_ACCESS_KEY }} | |
| # - uses: ./.github/actions/build-teardown | |
| update-sccache-ubuntu: | |
| needs: prepare | |
| environment: | |
| name: Sccache | |
| runs-on: testnet-runner-ubuntu-2x-2 | |
| container: | |
| image: diem/build_environment:${{ needs.prepare.outputs.changes-target-branch }} | |
| volumes: | |
| - "${{github.workspace}}:/opt/git/diem" | |
| if: ${{ needs.prepare.outputs.rust-changes == 'true' }} | |
| steps: | |
| - uses: actions/checkout@v2.4.0 | |
| with: | |
| # This ensures that the tip of the PR is checked out instead of the merge between the base ref and the tip | |
| # On `push` this value will be empty and will "do-the-right-thing" | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| fetch-depth: 0 #get all the history!!! | |
| - uses: ./.github/actions/build-setup | |
| - uses: actions/cache@v2.1.6 | |
| with: | |
| path: "/opt/cargo/git\n/opt/cargo/registry\n/opt/cargo/.package-cache" | |
| key: crates-${{ runner.os }}-${{ hashFiles('Cargo.lock') }} | |
| restore-keys: "crates-${{ runner.os }}" | |
| - name: build all unit test code. | |
| run: | | |
| $pre_command && cargo x test --no-run --jobs ${max_threads} --unit | |
| env: | |
| TARGET_BRANCH: ${{ needs.prepare.outputs.changes-target-branch }} | |
| SCCACHE_AWS_ACCESS_KEY_ID: ${{ secrets.ENV_DIEM_S3_AWS_ACCESS_KEY_ID }} | |
| SCCACHE_AWS_SECRET_ACCESS_KEY: ${{ secrets.ENV_DIEM_S3_AWS_SECRET_ACCESS_KEY }} | |
| - uses: ./.github/actions/build-teardown | |
| # In the future when the script can differentiate between target branches for each auto run then we'll | |
| # update this report to differentiate branches. | |
| unit-test-allure-report: | |
| name: Unit Test Reports | |
| runs-on: ubuntu-latest | |
| if: ${{ github.ref == 'refs/heads/main' }} | |
| environment: | |
| name: Sccache | |
| container: | |
| image: ghcr.io/diem/diem_build_environment:latest | |
| volumes: | |
| - "${{github.workspace}}:/opt/git/diem" | |
| steps: | |
| - uses: actions/checkout@v2.4.0 | |
| with: | |
| # This ensures that the tip of the PR is checked out instead of the merge between the base ref and the tip | |
| # On `push` this value will be empty and will "do-the-right-thing" | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| fetch-depth: 0 #get all the history!!! | |
| - uses: ./.github/actions/build-setup | |
| # build setup is called to ensure the latest docker image tooling is in place and the ~/.profile is called. | |
| - name: Publish unit test results | |
| uses: diem/actions/process-test-results@faadd16607b77dfa2231a8f366883e01717b3225 | |
| with: | |
| #used to read artifacts, annoying to need one. | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| artifacts: "unit-test-results codegen-unit-test-results" | |
| workflow_file: ci-test.yml | |
| branch: auto | |
| history: 20 | |
| recreate: false | |
| allure_configuration: ./allure-configuration | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.ENV_DIEM_S3_AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.ENV_DIEM_S3_AWS_SECRET_ACCESS_KEY }} | |
| AWS_DEFAULT_REGION: ${{ secrets.ENV_DIEM_S3_AWS_REGION }} | |
| build_ci_base_docker_image: | |
| needs: prepare | |
| runs-on: testnet-runner-ubuntu | |
| if: ${{ needs.prepare.outputs.base-image-changes == 'true' || github.event_name == 'create' }} | |
| continue-on-error: false | |
| env: | |
| REGISTRY: ghcr.io | |
| IMAGE_NAME: ${{ github.repository }}_build_environment | |
| IMAGE_TAG: ${{ needs.prepare.outputs.changes-target-branch }} | |
| steps: | |
| - name: Chown user | |
| run: | | |
| sudo chown -R $USER:$USER $GITHUB_WORKSPACE | |
| - name: docker prune | |
| run: | | |
| docker system prune -f | |
| - uses: actions/checkout@v2.4.0 | |
| with: | |
| # This ensures that the tip of the PR is checked out instead of the merge between the base ref and the tip | |
| # On `push` this value will be empty and will "do-the-right-thing" | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| fetch-depth: 0 #get all the history!!! | |
| - name: Log in to the Container registry | |
| uses: docker/login-action@v1 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: build image | |
| run: docker build -f docker/ci/github/Dockerfile -t ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} . | |
| - name: Push to ghcr | |
| run: | | |
| docker push ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} | |
| #run-forge-test-pre-release-suite: | |
| # needs: prepare | |
| # name: Run the pre-release suite of Forge Test | |
| # runs-on: [self-hosted, ACP] | |
| # if: ${{ startsWith(needs.prepare.outputs.changes-target-branch, 'release') }} | |
| # # The pre-release suite run time varies 1~1.5 hr. | |
| # timeout-minutes: 120 | |
| # steps: | |
| # - uses: actions/checkout@v2.4.0 | |
| # with: | |
| # fetch-depth: 0 #get all the history!!! | |
| # - name: set_env | |
| # id: set_env | |
| # run: | | |
| # HEAD_GIT_REV=$(git rev-parse --short=8 HEAD) | |
| # echo "HEAD_GIT_REV=$HEAD_GIT_REV" >> $GITHUB_ENV | |
| # IMAGE_TAG=$(echo ${GITHUB_REF#refs/heads/})_$HEAD_GIT_REV | |
| # echo $IMAGE_TAG | |
| # echo "IMAGE_TAG=$IMAGE_TAG" >> $GITHUB_ENV | |
| # - name: poll_images | |
| # # Poll until images are ready | |
| # env: | |
| # AWS_REGION: us-west-2 | |
| # RETRIES: 75 | |
| # run: | | |
| # set +e | |
| # retry=0 | |
| # status=1 | |
| # while [[ $status != 0 && $retry -lt $RETRIES ]]; do | |
| # status=0 | |
| # for image in diem/validator diem/validator_tcb diem/init diem/forge; do | |
| # aws ecr describe-images --region $AWS_REGION --repository-name $image --image-ids=imageTag=$IMAGE_TAG | |
| # status=$((status + $?)) | |
| # done | |
| # retry=$((retry + 1)) | |
| # if [[ $status != 0 ]] ; then | |
| # echo "CI has not pushed all images to ECR." | |
| # echo "Wait $((1*$retry)) of $((1*$RETRIES)) minutes before retry." | |
| # sleep 1m | |
| # fi | |
| # done | |
| # exit $status | |
| # - name: Run Forge Test | |
| # run: | | |
| # date | |
| # BASE_GIT_REV=$(git rev-parse $HEAD_GIT_REV^) | |
| # ./scripts/fgi/run --tag $IMAGE_TAG --timeout-secs 7200 \ | |
| # --env SLACK_CHANGELOG_URL=${{ secrets.WEBHOOK_FORGE }} \ | |
| # --changelog $BASE_GIT_REV $HEAD_GIT_REV \ | |
| # --suite pre_release | |
| # - name: Push alert | |
| # if: ${{ failure() }} | |
| # run: | | |
| # jq -n \ | |
| # --arg msg "*${{ github.job }}* job in ${{ github.workflow }} workflow failed with $IMAGE_TAG." \ | |
| # --arg url "https://github.com/${{ github.repository }}/actions/runs/${{github.run_id}}" \ | |
| # '{ | |
| # "attachments": [ | |
| # { | |
| # "text": $msg, | |
| # "actions": [ | |
| # { | |
| # "type": "button", | |
| # "text": "Visit Job", | |
| # "url": $url | |
| # } | |
| # ] | |
| # } | |
| # ] | |
| # }' > /tmp/payload | |
| # curl -X POST -H 'Content-type: application/json' -d @/tmp/payload ${{ secrets.WEBHOOK_FORGE }} | |
| build-release-images: | |
| needs: prepare | |
| runs-on: testnet-runner-ubuntu | |
| continue-on-error: false | |
| env: | |
| TAG: github-1 | |
| CHANGES_TARGET_BRANCH: ${{ needs.prepare.outputs.changes-target-branch }} | |
| environment: | |
| name: Docker | |
| url: https://hub.docker.com/u/diem | |
| steps: | |
| - name: Chown user | |
| run: | | |
| sudo chown -R $USER:$USER $GITHUB_WORKSPACE | |
| - name: docker prune | |
| run: | | |
| docker system prune -f | |
| - uses: actions/checkout@v2.4.0 | |
| with: | |
| fetch-depth: 0 #get all the history!!! | |
| #- name: setup_aws_ecr_login | |
| # run: | | |
| # echo 'AWS_ECR_ACCOUNT_URL=${{ secrets.ENV_ACP_ECR_AWS_ACCOUNT_NUM }}.dkr.ecr.${{ secrets.ENV_ACP_ECR_AWS_REGION }}.amazonaws.com' >> $GITHUB_ENV | |
| #- name: Configure AWS credentials | |
| # uses: aws-actions/configure-aws-credentials@v1 | |
| # with: | |
| # aws-access-key-id: ${{ secrets.ENV_ACP_ECR_AWS_ACCESS_KEY_ID }} | |
| # aws-secret-access-key: ${{ secrets.ENV_ACP_ECR_AWS_SECRET_ACCESS_KEY }} | |
| # aws-region: ${{ secrets.ENV_ACP_ECR_AWS_REGION }} | |
| #- name: Login to Amazon ECR | |
| # id: login-ecr | |
| # uses: aws-actions/amazon-ecr-login@v1.3.3 | |
| - name: Sign in to dockerhub, install image signing cert. | |
| uses: ./.github/actions/dockerhub_login | |
| with: | |
| username: ${{ secrets.ENV_DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.ENV_DOCKERHUB_PASSWORD }} | |
| key_material: ${{ secrets.ENV_DOCKERHUB_KEY_MATERIAL }} | |
| key_name: ${{ secrets.ENV_DOCKERHUB_KEY_NAME }} | |
| key_password: ${{ secrets.ENV_DOCKERHUB_KEY_PASSWORD }} | |
| - name: should pre build docker images (targeting a release branch)? | |
| if: ${{ github.ref == 'refs/heads/auto' }} | |
| run: | | |
| if [[ ! "$CHANGES_TARGET_BRANCH" =~ "^release-[0-9|.]+$" ]] && [[ ! "$CHANGES_TARGET_BRANCH" =~ "^gha-test-[0-9|.]+$" ]] ; then | |
| echo Targeting branch $CHANGES_TARGET_BRANCH will not pre-publish docker images. | |
| fi | |
| - name: pre-release docker images | |
| if: ${{ github.ref == 'refs/heads/auto' }} | |
| run: | | |
| BRANCH="$CHANGES_TARGET_BRANCH" | |
| success=0 | |
| tmpfile=$(mktemp) | |
| echo "Failed to push:" > "${tmpfile}" | |
| docker/build_push.sh -u -p -b ${BRANCH} -n client || success=$(echo "init" >> "${tmpfile}"; echo 1) | |
| docker/build_push.sh -u -p -b ${BRANCH} -n init || success=$(echo "init" >> "${tmpfile}"; echo 1) | |
| docker/build_push.sh -u -p -b ${BRANCH} -n faucet || success=$(echo "faucet" >> "${tmpfile}"; echo 1) | |
| docker/build_push.sh -u -p -b ${BRANCH} -n tools || success=$(echo "tools" >> "${tmpfile}"; echo 1) | |
| docker/build_push.sh -u -p -b ${BRANCH} -n validator || success=$(echo "validator" >> "${tmpfile}"; echo 1) | |
| docker/build_push.sh -u -p -b ${BRANCH} -n validator-tcb || success=$(echo "validator-tcb" >> "${tmpfile}"; echo 1) | |
| docker/build_push.sh -u -p -b ${BRANCH} -n forge || success=$(echo "forge" >> "${tmpfile}"; echo 1) | |
| if [[ "$success" == "1" ]]; then | |
| cat "${tmpfile}" | |
| fi | |
| exit $success | |
| env: | |
| DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ secrets.ENV_DOCKERHUB_KEY_PASSWORD }} | |
| - name: pull pre images (or build if not pullable) and push release docker images if not on auto branch. | |
| if: ${{ github.ref != 'refs/heads/auto' }} | |
| run: | | |
| set -x | |
| BRANCH=$(echo "$GITHUB_REF" | sed 's|.*/||' ) | |
| success=0 | |
| tmpfile=$(mktemp) | |
| echo "Failed to push:" >> "${tmpfile}" | |
| docker/build_push.sh -u -b ${BRANCH} -n client || success=$(echo "init" >> "${tmpfile}"; echo 1) | |
| docker/build_push.sh -u -b ${BRANCH} -n init || success=$(echo "init" >> "${tmpfile}"; echo 1) | |
| docker/build_push.sh -u -b ${BRANCH} -n faucet || success=$(echo "faucet" >> "${tmpfile}"; echo 1) | |
| docker/build_push.sh -u -b ${BRANCH} -n tools || success=$(echo "tools" >> "${tmpfile}"; echo 1) | |
| docker/build_push.sh -u -b ${BRANCH} -n validator || success=$(echo "validator" >> "${tmpfile}"; echo 1) | |
| docker/build_push.sh -u -b ${BRANCH} -n validator-tcb || success=$(echo "validator-tcb" >> "${tmpfile}"; echo 1) | |
| docker/build_push.sh -u -b ${BRANCH} -n forge || success=$(echo "forge" >> "${tmpfile}"; echo 1) | |
| if [[ "$success" == "1" ]]; then | |
| cat "${tmpfile}" | |
| fi | |
| exit $success | |
| env: | |
| DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE: ${{ secrets.ENV_DOCKERHUB_KEY_PASSWORD }} | |
| #- name: push to acp ecr | |
| # if: ${{ always() && github.ref != 'refs/heads/auto' }} | |
| # run: | | |
| # #push to acp ecr with standard names | |
| # BRANCH=$(echo "$GITHUB_REF" | sed 's|.*/||' ) | |
| # GIT_REV=$(git rev-parse --short=8 HEAD) | |
| # aws ecr get-login-password --region ${{ secrets.ENV_ACP_ECR_AWS_REGION }} | \ | |
| # docker login --username AWS --password-stdin "${AWS_ECR_ACCOUNT_URL}" | |
| # docker/docker_republish.sh -t ${BRANCH}_${GIT_REV} -r ${AWS_ECR_ACCOUNT_URL} -d | |
| rustdoc: | |
| needs: prepare | |
| if: ${{ needs.prepare.outputs.changes-target-branch == 'latest' && needs.prepare.outputs.rust-changes == 'true'}} | |
| runs-on: ubuntu-latest | |
| container: | |
| image: diem/build_environment:${{ needs.prepare.outputs.changes-target-branch }} | |
| volumes: | |
| - "${{github.workspace}}:/opt/git/diem" | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v2.3.4 | |
| - uses: ./.github/actions/build-setup | |
| - uses: actions/cache@v2.1.6 | |
| with: | |
| path: "/opt/cargo/git\n/opt/cargo/registry\n/opt/cargo/.package-cache" | |
| key: crates-${{ runner.os }}-${{ hashFiles('Cargo.lock') }} | |
| restore-keys: "crates-${{ runner.os }}" | |
| - name: Build Documentation | |
| # Build the rust crate docs | |
| # Use `RUSTC_BOOTSTRAP` in order to use the `--enable-index-page` flag of rustdoc | |
| # This is needed in order to generate a landing page `index.html` for workspaces | |
| run: cargo doc --no-deps --workspace --lib | |
| env: | |
| RUSTC_BOOTSTRAP: 1 | |
| RUSTDOCFLAGS: "-Z unstable-options --enable-index-page" | |
| CARGO_INCREMENTAL: 0 | |
| RUSTFLAGS: -D warnings | |
| - name: Deploy Docs | |
| uses: peaceiris/actions-gh-pages@v3.8.0 | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| publish_branch: gh-pages | |
| publish_dir: ./target/doc | |
| force_orphan: true |