chore: compare micro benchmarks #1831
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| types: [opened, synchronize, labeled] | |
| branches: | |
| - main | |
| workflow_dispatch: | |
| inputs: | |
| branch: | |
| description: "Branch to run benchmark on" | |
| required: true | |
| default: "main" | |
| jobs: | |
| benchmark: | |
| name: Macro Benchmarks | |
| if: contains(github.event.pull_request.labels.*.name, 'ci:benchmark') || github.event_name == 'push' | |
| runs-on: benchmarking-runner | |
| permissions: | |
| pull-requests: write | |
| contents: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install Stable Toolchain | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| toolchain: stable | |
| override: true | |
| components: rustfmt, clippy | |
| - name: Cache Rust Dependencies | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| target | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} | |
| - name: Run Cargo Build | |
| uses: actions-rs/cargo@v1 | |
| with: | |
| command: build | |
| args: --release | |
| - name: Run Tailcall | |
| run: | | |
| TAILCALL_LOG_LEVEL=error ./target/release/tailcall start ci-benchmark/benchmark.graphql & | |
| - name: Install Wrk | |
| run: | | |
| sudo apt-get install -y wrk | |
| - name: Run Test Query | |
| run: | | |
| curl -i -X POST -d '{"query": "{posts{title}}"}' http://localhost:8000/graphql -H "Content-Type: application/json" | |
| - name: Warmup Wrk | |
| working-directory: ci-benchmark | |
| run: | | |
| wrk -d 10 -t 4 -c 100 -s wrk.lua http://localhost:8000/graphql | |
| - id: run_wrk | |
| name: Run Wrk | |
| working-directory: ci-benchmark | |
| run: | | |
| wrk -d 30 -t 4 -c 100 -s wrk.lua http://localhost:8000/graphql > wrk-output.txt | |
| - id: convert_wrk_output_markdown | |
| name: Convert Output to Markdown | |
| working-directory: ci-benchmark | |
| run: | | |
| node wrk-output-to-md.js wrk-output.txt > wrk-output.md | |
| - id: cat_md | |
| name: Cat Markdown | |
| working-directory: ci-benchmark | |
| run: | | |
| cat wrk-output.md | |
| - id: get_comment_body | |
| name: Get Comment Body | |
| working-directory: ci-benchmark | |
| run: | | |
| body="$(cat wrk-output.md)" | |
| delimiter="$(openssl rand -hex 8)" | |
| echo "body<<$delimiter" >> $GITHUB_OUTPUT | |
| echo "$body" >> $GITHUB_OUTPUT | |
| echo "$delimiter" >> $GITHUB_OUTPUT | |
| - name: Create Commit Comment | |
| uses: peter-evans/commit-comment@v3 | |
| with: | |
| sha: ${{github.event.pull_request.head.sha}} | |
| body: ${{steps.get_comment_body.outputs.body}} | |
| lint: | |
| name: CI:Lint | |
| if: contains(github.event.pull_request.labels.*.name, 'ci:lint') | |
| runs-on: ubuntu-latest | |
| permissions: | |
| pull-requests: write | |
| contents: write | |
| steps: | |
| - name: Install Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: "18" | |
| - name: Install Prettier | |
| run: npm install --global prettier | |
| - name: Install Stable Toolchain | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| toolchain: nightly | |
| components: rustfmt, clippy | |
| - name: Cache Rust Dependencies | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| target | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} | |
| - uses: actions/checkout@v4 | |
| - name: Run lint script | |
| run: ./lint.sh --mode=fix | |
| - name: Commit and push if changed | |
| uses: stefanzweifel/git-auto-commit-action@v5 | |
| with: | |
| commit_message: "style: apply lint fixes" | |
| Cache_Benchmarks: | |
| name: Micro Benchmarks | |
| if: github.ref == 'refs/heads/main' | |
| permissions: | |
| pull-requests: write | |
| contents: write | |
| runs-on: benchmarking-runner | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| - name: Install Rust | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| toolchain: stable | |
| override: true | |
| - name: Run Benchmarks | |
| run: | | |
| cargo install cargo-criterion rust-script | |
| cargo criterion --message-format=json > benches/main_benchmarks.json | |
| ./scripts/json_to_md.rs benches/main_benchmarks.json > benches/main_benchmarks.md | |
| cat benches/main_benchmarks.md | |
| - name: Cache Criterion Benchmarks Json | |
| uses: actions/cache@v3 | |
| with: | |
| path: benches/main_benchmarks.json | |
| key: criterion_benchmarks_${{ github.sha }} | |
| Criterion_Compare: | |
| name: Comparing Micro Benchmarks | |
| if: github.ref != 'refs/heads/main' | |
| runs-on: benchmarking-runner | |
| permissions: | |
| pull-requests: write | |
| contents: write | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v2 | |
| - name: Install Rust | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| toolchain: stable | |
| override: true | |
| - name: Restore file | |
| uses: actions/cache@v3 | |
| with: | |
| path: benches/main_benchmarks.json | |
| key: criterion_benchmarks_${{ github.event.pull_request.base.sha }} | |
| fail-on-cache-miss: true | |
| - name: Run Criterion Benchmarks | |
| run: | | |
| cargo install cargo-criterion rust-script | |
| cargo criterion --message-format=json > benches/benchmarks.json | |
| ./scripts/json_to_md.rs benches/benchmarks.json > benches/change_benchmarks.md | |
| - name: Print Criterion Benchmarks | |
| run: cat benches/change_benchmarks.md | |
| - name: Print Benchmark Comparision | |
| run: ./scripts/criterion_compare.rs benches/main_benchmarks.json benches/benchmarks.json table | |
| - name: Check Degradation | |
| run: ./scripts/criterion_compare.rs benches/main_benchmarks.json benches/benchmarks.json check |