Benchmark runtime weights #118
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark runtime weights | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| rebuild-docker: | |
| type: boolean | |
| # if the runtime-benchmarks image should be rebuilt or pulled from hub | |
| description: rebuild-docker | |
| required: true | |
| default: true | |
| litentry: | |
| type: boolean | |
| description: litentry | |
| required: true | |
| default: true | |
| litmus: | |
| type: boolean | |
| description: litmus | |
| required: true | |
| default: true | |
| rococo: | |
| type: boolean | |
| description: rococo | |
| required: true | |
| default: true | |
| pallets: | |
| description: pallets to benchmark, * for all, or comma listed (e.g. frame-system,pallet-proxy) | |
| default: "*" | |
| required: true | |
| env: | |
| INSTANCE_ID: ${{ secrets.BENCHMARK_INSTANCE_ID }} # remote AWS host to run benchmarking | |
| BENCHMARK_SSH_USER: ${{ secrets.BENCHMARK_SSH_USER }} | |
| BENCHMARK_SSH_KEYPATH: ${{ secrets.BENCHMARK_SSH_KEYPATH }} | |
| DOCKER_BUILDKIT: 1 | |
| jobs: | |
| ## build docker image with runtime-benchmarks feature and push it to the hub | |
| build-docker: | |
| if: ${{ github.event.inputs.rebuild-docker == 'true' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout codes on ${{ github.ref }} | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| # try to increase usable memory | |
| # https://github.com/actions/runner/issues/1051 | |
| - name: Set Swap Space | |
| uses: pierotofy/set-swap-space@master | |
| with: | |
| swap-size-gb: 10 | |
| - name: Build docker image | |
| run: | | |
| ./scripts/build-docker.sh production runtime-benchmarks --features=runtime-benchmarks | |
| - name: Dockerhub login | |
| uses: docker/login-action@v2 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_PASSWORD }} | |
| - name: Push docker image | |
| run: | | |
| docker push litentry/litentry-parachain:runtime-benchmarks | |
| ## run the benchmarking remotely | |
| benchmark: | |
| runs-on: jumphost | |
| needs: build-docker | |
| # see https://github.com/actions/runner/issues/491 | |
| if: | | |
| always() && | |
| (needs.build-docker.result == 'success' || needs.build-docker.result == 'skipped') | |
| steps: | |
| - name: Set env | |
| run: | | |
| chain="" | |
| if [ "${{ github.event.inputs.litmus }}" = "true" ]; then | |
| chain="$chain litmus" | |
| fi | |
| if [ "${{ github.event.inputs.litentry }}" = "true" ]; then | |
| chain="$chain litentry" | |
| fi | |
| if [ "${{ github.event.inputs.rococo }}" = "true" ]; then | |
| chain="$chain rococo" | |
| fi | |
| if [ "$chain" = "" ]; then | |
| echo "::error::Please select at least one chain." | |
| exit 1 | |
| fi | |
| echo "CHAIN=$chain" >> $GITHUB_ENV | |
| - name: Checkout codes on ${{ github.ref }} | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Pull docker image | |
| run: | | |
| docker pull litentry/litentry-parachain:runtime-benchmarks | |
| # TODO: maybe use GHA to start/stop remote instance | |
| - name: Start remote instance | |
| timeout-minutes: 10 | |
| id: start_instance | |
| run: | | |
| aws ec2 start-instances --region ap-southeast-1 --instance-ids ${{ env.INSTANCE_ID }} | |
| sleep 5 | |
| instance_status="aws ec2 describe-instance-status --region ap-southeast-1 --instance-ids ${{ env.INSTANCE_ID }} --query 'InstanceStatuses[0].InstanceStatus.Status' --output text" | |
| system_status="aws ec2 describe-instance-status --region ap-southeast-1 --instance-ids ${{ env.INSTANCE_ID }} --query 'InstanceStatuses[0].SystemStatus.Status' --output text" | |
| SECONDS=0 | |
| while : ; do | |
| if [ "$(eval $instance_status)" = "ok" ] && [ "$(eval $system_status)" = "ok" ]; then | |
| break | |
| else | |
| sleep 20 | |
| fi | |
| done | |
| echo "Remote instance reachable now after $SECONDS seconds" | |
| remote_ip=`aws ec2 describe-instances --region ap-southeast-1 --filters 'Name=instance-state-name,Values=running' 'Name=instance-id,Values=${{ env.INSTANCE_ID }}' --query 'Reservations[*].Instances[*].[PublicIpAddress]' --output text` | |
| echo "Running instances ip address: $remote_ip" | |
| echo "remote_ip=$remote_ip" >> $GITHUB_OUTPUT | |
| # exit status should propagate through ssh | |
| - name: Remotely benchmark pallets ${{ github.event.inputs.pallets }} for ${{ env.CHAIN }} | |
| timeout-minutes: 240 | |
| run: | | |
| # prepend the asterisk with \ to go through ssh | |
| echo "Running instances ip address: ${{ steps.start_instance.outputs.remote_ip }}" | |
| arg="${{ github.event.inputs.pallets }}" | |
| chain="${{ env.CHAIN }}" | |
| if [ "$arg" = "*" ]; then | |
| arg="\\$arg"; | |
| fi | |
| for c in $chain; do | |
| ssh -x -o StrictHostKeychecking=no ${{ secrets.BENCHMARK_INSTANCE_IP }} -l ${{ env.BENCHMARK_SSH_USER }} 'bash -s' < scripts/benchmark-weight-remote.sh "$c" "${GITHUB_REF#refs/heads/}" "$arg" | |
| echo "copy generated weights files back ..." | |
| scp -o StrictHostKeychecking=no "${{ env.BENCHMARK_SSH_USER }}"@"${{ secrets.BENCHMARK_INSTANCE_IP }}":/tmp/litentry-parachain/runtime/$c/src/weights/*.rs runtime/$c/src/weights/ | |
| done | |
| echo "======================" | |
| git status | |
| - name: Stop remote instance | |
| if: always() | |
| run: | | |
| aws ec2 stop-instances --region ap-southeast-1 --instance-ids ${{ env.INSTANCE_ID }} | |
| sleep 5 | |
| ret=`aws ec2 describe-instance-status --region ap-southeast-1 --instance-ids ${{ env.INSTANCE_ID }} | jq '.InstanceStatuses[0].InstanceState.Name'` | |
| echo "Remote instance running state: $ret" | |
| - name: Create auto PR | |
| uses: peter-evans/create-pull-request@v3 | |
| with: | |
| commit-message: "[benchmarking bot] Auto commit generated weights files" | |
| committer: benchmarking bot <noreply@github.com> | |
| author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> | |
| signoff: false | |
| branch: benchmarking-bot-${{ github.run_id }} | |
| delete-branch: true | |
| title: "[benchmarking bot] Update generated weights files" | |
| body: | | |
| This is an automatically created PR. | |
| It updates the weights files under `runtime/*/src/weights/*.rs` after running benchmarks on the remote machine: ${{ env.INSTANCE_ID }} | |
| Pallets: "${{ github.event.inputs.pallets }}" | |
| Chain: "${{ env.CHAIN }}" | |
| Github action run: https://github.com/litentry/litentry-parachain/actions/runs/${{ github.run_id }} | |
| labels: | | |
| automated-pr | |
| assignees: ${{ github.actor }} | |
| reviewers: ${{ github.actor }} | |
| draft: false |