diff --git a/.github/workflows/benchmarkdotnet.yml b/.github/workflows/benchmarkdotnet.yml deleted file mode 100644 index a9f59f1..0000000 --- a/.github/workflows/benchmarkdotnet.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Benchmark.Net Example -on: - push: - branches: - - master - -permissions: - contents: write - deployments: write - -jobs: - benchmark: - name: Run Benchmark.Net benchmark example - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-dotnet@v3 - with: - dotnet-version: '6.0.101' # SDK Version to use. keep in line with examples/benchmarkdotnet/global.json - - name: Run benchmark - run: cd examples/benchmarkdotnet && dotnet run --exporters json --filter '*' - - - name: Store benchmark result - uses: rhysd/github-action-benchmark@v1 - with: - name: Benchmark.Net Benchmark - tool: 'benchmarkdotnet' - output-file-path: examples/benchmarkdotnet/BenchmarkDotNet.Artifacts/results/Sample.Benchmarks-report-full-compressed.json - github-token: ${{ secrets.GITHUB_TOKEN }} - auto-push: true - # Show alert with commit comment on detecting possible performance regression - alert-threshold: '200%' - comment-on-alert: true - fail-on-alert: true - alert-comment-cc-users: '@ktrz' - - - name: Store benchmark result - separate results repo - uses: rhysd/github-action-benchmark@v1 - with: - name: Benchmark.Net Benchmark - tool: 'benchmarkdotnet' - output-file-path: examples/benchmarkdotnet/BenchmarkDotNet.Artifacts/results/Sample.Benchmarks-report-full-compressed.json - github-token: ${{ secrets.BENCHMARK_ACTION_BOT_TOKEN }} - auto-push: true - # Show alert with commit comment on detecting possible performance regression - alert-threshold: '200%' - comment-on-alert: true - fail-on-alert: true - alert-comment-cc-users: '@ktrz' - gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' diff --git a/.github/workflows/benchmarkjs.yml b/.github/workflows/benchmarkjs.yml deleted file mode 100644 index 7c404ef..0000000 --- a/.github/workflows/benchmarkjs.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: Benchmark.js Example -on: - push: - branches: - - master - -permissions: - contents: write - deployments: write - -jobs: - benchmark: - name: Run JavaScript benchmark example - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - name: Run benchmark - run: cd examples/benchmarkjs && npm install && node bench.js | tee output.txt - - - name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 - with: - name: Benchmark.js Benchmark - tool: 'benchmarkjs' - output-file-path: examples/benchmarkjs/output.txt - github-token: ${{ secrets.GITHUB_TOKEN }} - auto-push: true - # Show alert with commit comment on detecting possible performance regression - alert-threshold: '200%' - comment-on-alert: true - fail-on-alert: true - alert-comment-cc-users: '@ktrz' - - - name: Store benchmark result - separate results repo - uses: benchmark-action/github-action-benchmark@v1 - with: - name: Benchmark.js Benchmark - tool: 'benchmarkjs' - output-file-path: examples/benchmarkjs/output.txt - github-token: ${{ secrets.BENCHMARK_ACTION_BOT_TOKEN }} - auto-push: true - # Show alert with commit comment on detecting possible performance regression - alert-threshold: '200%' - comment-on-alert: true - fail-on-alert: true - alert-comment-cc-users: '@ktrz' - gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml new file mode 100644 index 0000000..7887156 --- /dev/null +++ b/.github/workflows/build-and-test.yml @@ -0,0 +1,42 @@ +name: Build and test + +on: + workflow_dispatch: + pull_request: + +# Test all inputs +# TODO: ensure that the output matches +# TODO: add failing cases +jobs: + build-and-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: "build" + run: | + npm install + npm run build + - uses: ./ # local action in repo + with: + tool: 'cargo' + platform: 'default' + output-file-path: ./test/data/extract/cargo_output.txt + data-out-path: output.txt + - uses: ./ # local action in repo + with: + tool: 'cargo' + platform: 'default' + output-file-path: ./test/data/extract/cargo_output2.txt + data-out-path: output.txt + - uses: ./ # local action in repo + with: + tool: 'cargo' + platform: 'default' + output-file-path: ./test/data/extract/cargo_output3.txt + data-out-path: output.txt + - uses: ./ # local action in repo + with: + tool: 'cargo' + platform: 'default' + output-file-path: ./test/data/extract/criterion_output.txt + data-out-path: output.txt diff --git a/.github/workflows/catch2.yml b/.github/workflows/catch2.yml deleted file mode 100644 index 41bbe7b..0000000 --- a/.github/workflows/catch2.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: Catch2 C++ Example -on: - push: - branches: - - master - -permissions: - contents: write - deployments: write - -jobs: - benchmark: - name: Run Catch2 C++ Benchmark Framework example (v3.x) - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4 - - name: Build and run benchmarks with Catch2 - run: | - cd examples/catch2 - mkdir build && cd build - cmake -DCMAKE_BUILD_TYPE=Release .. - cmake --build . --config Release - ./Catch2_bench | tee ../benchmark_result.txt - - - name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 - with: - name: Catch2 Benchmark (v3) - tool: 'catch2' - output-file-path: examples/catch2/benchmark_result.txt - github-token: ${{ secrets.GITHUB_TOKEN }} - auto-push: true - # Show alert with commit comment on detecting possible performance regression - alert-threshold: '200%' - comment-on-alert: true - fail-on-alert: true - alert-comment-cc-users: '@ktrz' - - - name: Store benchmark result - separate results repo - uses: benchmark-action/github-action-benchmark@v1 - with: - name: Catch2 Benchmark (v3) - tool: 'catch2' - output-file-path: examples/catch2/benchmark_result.txt - github-token: ${{ secrets.BENCHMARK_ACTION_BOT_TOKEN }} - auto-push: true - # Show alert with commit comment on detecting possible performance regression - alert-threshold: '200%' - comment-on-alert: true - fail-on-alert: true - alert-comment-cc-users: '@ktrz' - gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' diff --git a/.github/workflows/catch2_v2.yml b/.github/workflows/catch2_v2.yml deleted file mode 100644 index d28b867..0000000 --- a/.github/workflows/catch2_v2.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: Catch2 C++ Example -on: - push: - branches: - - master - -permissions: - contents: write - deployments: write - -jobs: - benchmark: - name: Run Catch2 C++ Benchmark Framework example (v2.x) - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4 - - name: Build and run benchmarks with Catch2 - run: | - cd examples/catch2_v2 - mkdir build && cd build - cmake -DCMAKE_BUILD_TYPE=Release .. - cmake --build . --config Release - ./Catch2_bench | tee ../benchmark_result.txt - - - name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 - with: - name: Catch2 Benchmark - tool: 'catch2' - output-file-path: examples/catch2_v2/benchmark_result.txt - github-token: ${{ secrets.GITHUB_TOKEN }} - auto-push: true - # Show alert with commit comment on detecting possible performance regression - alert-threshold: '200%' - comment-on-alert: true - fail-on-alert: true - alert-comment-cc-users: '@ktrz' - - - name: Store benchmark result - separate results repo - uses: benchmark-action/github-action-benchmark@v1 - with: - name: Catch2 Benchmark - tool: 'catch2' - output-file-path: examples/catch2_v2/benchmark_result.txt - github-token: ${{ secrets.BENCHMARK_ACTION_BOT_TOKEN }} - auto-push: true - # Show alert with commit comment on detecting possible performance regression - alert-threshold: '200%' - comment-on-alert: true - fail-on-alert: true - alert-comment-cc-users: '@ktrz' - gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' diff --git a/.github/workflows/ci-results-repo.yml b/.github/workflows/ci-results-repo.yml deleted file mode 100644 index 59356f2..0000000 --- a/.github/workflows/ci-results-repo.yml +++ /dev/null @@ -1,462 +0,0 @@ -name: CI - separate results repo -on: - push: - branches: - - master - pull_request: - branches: - - master - -jobs: - benchmarkdotnet-framework: - name: Run Benchmark.Net .Net Benchmark Framework example - github.com/benchmark-action/github-action-benchmark-results - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-dotnet@v3 - with: - dotnet-version: '6.0.101' # SDK Version to use. keep in line with examples/benchmarkdotnet/global.json - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run build - - name: Run benchmark - run: cd examples/benchmarkdotnet && dotnet run --exporters json --filter '*' - - - uses: actions/checkout@v4 - with: - repository: benchmark-action/github-action-benchmark-results - ref: 'gh-pages' - path: 'dist/other-repo' - - name: Save previous data.js - run: | - cp ./dist/other-repo/dev/bench/data.js before_data.js - - - name: Store benchmark result - uses: ./ - with: - name: Benchmark.Net Benchmark - tool: 'benchmarkdotnet' - output-file-path: examples/benchmarkdotnet/BenchmarkDotNet.Artifacts/results/Sample.Benchmarks-report-full-compressed.json - fail-on-alert: true - gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' - github-token: ${{ secrets.GITHUB_TOKEN }} - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.Net Benchmark' './benchmark-data-repository' - - benchmarkjs: - name: Run JavaScript benchmark example - github.com/benchmark-action/github-action-benchmark-results - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run build - - name: Run benchmark - run: cd examples/benchmarkjs && npm install && node bench.js | tee output.txt - - - uses: actions/checkout@v4 - with: - repository: benchmark-action/github-action-benchmark-results - ref: 'gh-pages' - path: 'dist/other-repo' - - name: Save previous data.js - run: | - cp ./dist/other-repo/dev/bench/data.js before_data.js - - - name: Store benchmark result - uses: ./ - with: - name: Benchmark.js Benchmark - tool: 'benchmarkjs' - output-file-path: examples/benchmarkjs/output.txt - fail-on-alert: true - gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' - github-token: ${{ secrets.GITHUB_TOKEN }} - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.js Benchmark' './benchmark-data-repository' - - catch2-v2-framework: - name: Run Catch2 C++ Benchmark Framework example (v2.x) - github.com/benchmark-action/github-action-benchmark-results - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run build - - name: Run benchmark - run: | - cd examples/catch2_v2 - mkdir build && cd build - cmake -DCMAKE_BUILD_TYPE=Release .. - cmake --build . --config Release - ./Catch2_bench > ../benchmark_result.txt - - - uses: actions/checkout@v4 - with: - repository: benchmark-action/github-action-benchmark-results - ref: 'gh-pages' - path: 'dist/other-repo' - - name: Save previous data.js - run: | - cp ./dist/other-repo/dev/bench/data.js before_data.js - - - name: Store benchmark result - uses: ./ - with: - name: Catch2 Benchmark - tool: 'catch2' - output-file-path: examples/catch2_v2/benchmark_result.txt - fail-on-alert: true - gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' - github-token: ${{ secrets.GITHUB_TOKEN }} - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark' './benchmark-data-repository' - - catch2-v3-framework: - name: Run Catch2 C++ Benchmark Framework example (v3.x) - github.com/benchmark-action/github-action-benchmark-results - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run build - - name: Run benchmark - run: | - cd examples/catch2 - mkdir build && cd build - cmake -DCMAKE_BUILD_TYPE=Release .. - cmake --build . --config Release - ./Catch2_bench > ../benchmark_result.txt - - - uses: actions/checkout@v4 - with: - repository: benchmark-action/github-action-benchmark-results - ref: 'gh-pages' - path: 'dist/other-repo' - - name: Save previous data.js - run: | - cp ./dist/other-repo/dev/bench/data.js before_data.js - - - name: Store benchmark result - uses: ./ - with: - name: Catch2 Benchmark (v3) - tool: 'catch2' - output-file-path: examples/catch2/benchmark_result.txt - fail-on-alert: true - gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' - github-token: ${{ secrets.GITHUB_TOKEN }} - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark (v3)' './benchmark-data-repository' - - cpp-framework: - name: Run Google C++ Benchmark Framework example - github.com/benchmark-action/github-action-benchmark-results - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run build - - name: Cache Benchmark library - uses: actions/cache@v4 - with: - path: examples/cpp/benchmark - key: ${{ runner.os }}-googlebenchmark-v1.5.0 - - name: Run benchmark - run: | - cd examples/cpp - make json - - - uses: actions/checkout@v4 - with: - repository: benchmark-action/github-action-benchmark-results - ref: 'gh-pages' - path: 'dist/other-repo' - - name: Save previous data.js - run: | - cp ./dist/other-repo/dev/bench/data.js before_data.js - - - name: Store benchmark result - uses: ./ - with: - name: C++ Benchmark - tool: 'googlecpp' - output-file-path: examples/cpp/benchmark_result.json - fail-on-alert: true - gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' - github-token: ${{ secrets.GITHUB_TOKEN }} - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'C++ Benchmark' './benchmark-data-repository' - - go: - name: Run Go benchmark example - github.com/benchmark-action/github-action-benchmark-results - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - uses: actions/setup-go@v4 - with: - go-version: "stable" - - run: npm ci - - run: npm run build - - name: Run benchmark - run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt - - - uses: actions/checkout@v4 - with: - repository: benchmark-action/github-action-benchmark-results - ref: 'gh-pages' - path: 'dist/other-repo' - - name: Save previous data.js - run: | - cp ./dist/other-repo/dev/bench/data.js before_data.js - - - name: Store benchmark result - uses: ./ - with: - name: Go Benchmark - tool: 'go' - output-file-path: examples/go/output.txt - fail-on-alert: true - gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' - github-token: ${{ secrets.GITHUB_TOKEN }} - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Go Benchmark' './benchmark-data-repository' - - java-jmh: - name: Run JMH Java Benchmark Framework example - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - uses: actions/setup-java@v4 - with: - distribution: 'adopt' - java-version: '11' - - run: npm ci - - run: npm run build - - name: Run benchmark - run: | - cd examples/java - mvn clean verify - java -jar target/benchmarks.jar -wi 1 -i 3 -f 1 -rf json - - uses: actions/checkout@v4 - with: - repository: benchmark-action/github-action-benchmark-results - ref: 'gh-pages' - path: 'dist/other-repo' - - name: Save previous data.js - run: | - cp ./dist/other-repo/dev/bench/data.js before_data.js - - name: Store benchmark result - uses: ./ - with: - name: JMH Benchmark - tool: "jmh" - output-file-path: examples/java/jmh-result.json - fail-on-alert: true - gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' - github-token: ${{ secrets.GITHUB_TOKEN }} - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'JMH Benchmark' './benchmark-data-repository' - - julia-benchmark: - name: Run Julia benchmark example - github.com/benchmark-action/github-action-benchmark-results - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run build - - uses: julia-actions/setup-julia@v1 - with: - version: '1' - arch: x64 - - name: Run benchmark - run: | - cd examples/julia - julia --project --color=yes -e ' - using Pkg; - Pkg.instantiate(); - include("fib.jl")' - - - uses: actions/checkout@v4 - with: - repository: benchmark-action/github-action-benchmark-results - ref: 'gh-pages' - path: 'dist/other-repo' - - name: Save previous data.js - run: | - cp ./dist/other-repo/dev/bench/data.js before_data.js - - - name: Store benchmark result - uses: ./ - with: - name: Julia benchmark - tool: 'julia' - output-file-path: examples/julia/output.json - fail-on-alert: true - gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' - github-token: ${{ secrets.GITHUB_TOKEN }} - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Julia benchmark' './benchmark-data-repository' - - pytest-benchmark: - name: Run Pytest benchmark example - github.com/benchmark-action/github-action-benchmark-results - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - uses: actions/setup-python@v5 - with: - python-version: 3.9 - - run: npm ci - - run: npm run build - - name: Run benchmark - run: | - cd examples/pytest - pip install -r requirements.txt - pytest bench.py --benchmark-json output.json - - - uses: actions/checkout@v4 - with: - repository: benchmark-action/github-action-benchmark-results - ref: 'gh-pages' - path: 'dist/other-repo' - - name: Save previous data.js - run: | - cp ./dist/other-repo/dev/bench/data.js before_data.js - - - name: Store benchmark result - uses: ./ - with: - name: Python Benchmark with pytest-benchmark - tool: 'pytest' - output-file-path: examples/pytest/output.json - fail-on-alert: true - gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' - github-token: ${{ secrets.GITHUB_TOKEN }} - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Python Benchmark with pytest-benchmark' './benchmark-data-repository' - - rust: - name: Run Rust benchmark example - github.com/benchmark-action/github-action-benchmark-results - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run build - - run: rustup toolchain update nightly && rustup default nightly - - name: Run benchmark - run: cd examples/rust && cargo +nightly bench | tee output.txt - - - uses: actions/checkout@v4 - with: - repository: benchmark-action/github-action-benchmark-results - ref: 'gh-pages' - path: 'dist/other-repo' - - name: Save previous data.js - run: | - cp ./dist/other-repo/dev/bench/data.js before_data.js - - - name: Store benchmark result - uses: ./ - with: - name: Rust Benchmark - tool: 'cargo' - output-file-path: examples/rust/output.txt - fail-on-alert: true - gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' - github-token: ${{ secrets.GITHUB_TOKEN }} - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Rust Benchmark' './benchmark-data-repository' - - rust-criterion-rs-framework: - name: Run Criterion.rs benchmark example - github.com/benchmark-action/github-action-benchmark-results - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run build - - run: rustup toolchain update nightly && rustup default nightly - - name: Run benchmark - run: cd examples/criterion-rs && cargo +nightly bench -- --output-format bencher | tee output.txt - - - uses: actions/checkout@v4 - with: - repository: benchmark-action/github-action-benchmark-results - ref: 'gh-pages' - path: 'dist/other-repo' - - name: Save previous data.js - run: | - cp ./dist/other-repo/dev/bench/data.js before_data.js - - - name: Store benchmark result - uses: ./ - with: - name: Criterion.rs Benchmark - tool: 'cargo' - output-file-path: examples/criterion-rs/output.txt - fail-on-alert: true - gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' - github-token: ${{ secrets.GITHUB_TOKEN }} - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Criterion.rs Benchmark' './benchmark-data-repository' - - only-alert-with-cache: - name: Run alert check with actions/cache - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run build - - uses: actions/setup-go@v4 - with: - go-version: "stable" - - name: Run benchmark - run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt - - name: Download previous benchmark data - uses: actions/cache@v4 - with: - path: ./cache - key: ${{ runner.os }}-ci-cache-test - - name: Store benchmark result - uses: ./ - with: - name: Alert setup example with cache - tool: 'go' - output-file-path: examples/go/output.txt - external-data-json-path: ./cache/benchmark-data.json - github-token: ${{ secrets.GITHUB_TOKEN }} - alert-threshold: '200%' - comment-on-alert: true - fail-on-alert: true - alert-comment-cc-users: '@ktrz' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 274cf63..0000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,469 +0,0 @@ -name: CI -on: - push: - branches: - - master - pull_request: - branches: - - master - -jobs: - benchmarkdotnet-framework: - name: Run Benchmark.Net .Net Benchmark Framework example - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - uses: actions/setup-dotnet@v3 - with: - dotnet-version: '6.0.101' # SDK Version to use. keep in line with examples/benchmarkdotnet/global.json - - run: npm ci - - run: npm run build - - name: Run benchmark - run: cd examples/benchmarkdotnet && dotnet run --exporters json --filter '*' - - name: Save previous data.js - run: | - git fetch origin gh-pages - git checkout gh-pages - cp ./dev/bench/data.js before_data.js - git checkout - - - name: Store benchmark result - uses: ./ - with: - name: Benchmark.Net Benchmark - tool: 'benchmarkdotnet' - output-file-path: examples/benchmarkdotnet/BenchmarkDotNet.Artifacts/results/Sample.Benchmarks-report-full-compressed.json - skip-fetch-gh-pages: true - fail-on-alert: true - summary-always: true - github-token: ${{ secrets.GITHUB_TOKEN }} - comment-on-alert: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.Net Benchmark' - - benchmarkjs: - name: Run JavaScript benchmark example - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run build - - name: Run benchmark - run: cd examples/benchmarkjs && npm install && node bench.js | tee output.txt - - name: Save previous data.js - run: | - git fetch origin gh-pages - git checkout gh-pages - cp ./dev/bench/data.js before_data.js - git checkout - - - name: Store benchmark result - uses: ./ - with: - name: Benchmark.js Benchmark - tool: 'benchmarkjs' - output-file-path: examples/benchmarkjs/output.txt - skip-fetch-gh-pages: true - fail-on-alert: true - summary-always: true - github-token: ${{ secrets.GITHUB_TOKEN }} - comment-on-alert: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.js Benchmark' - - catch2-v2-framework: - name: Run Catch2 C++ Benchmark Framework example (v2.x) - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run build - - name: Run benchmark - run: | - cd examples/catch2_v2 - mkdir build && cd build - cmake -DCMAKE_BUILD_TYPE=Release .. - cmake --build . --config Release - ./Catch2_bench > ../benchmark_result.txt - - name: Save previous data.js - run: | - git fetch origin gh-pages - git checkout gh-pages - cp ./dev/bench/data.js before_data.js - git checkout - - - name: Store benchmark result - uses: ./ - with: - name: Catch2 Benchmark - tool: 'catch2' - output-file-path: examples/catch2_v2/benchmark_result.txt - skip-fetch-gh-pages: true - fail-on-alert: true - summary-always: true - github-token: ${{ secrets.GITHUB_TOKEN }} - comment-on-alert: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark' - - catch2-v3-framework: - name: Run Catch2 C++ Benchmark Framework example (v3.x) - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run build - - name: Run benchmark - run: | - cd examples/catch2 - mkdir build && cd build - cmake -DCMAKE_BUILD_TYPE=Release .. - cmake --build . --config Release - ./Catch2_bench > ../benchmark_result.txt - - name: Save previous data.js - run: | - git fetch origin gh-pages - git checkout gh-pages - cp ./dev/bench/data.js before_data.js - git checkout - - - name: Store benchmark result - uses: ./ - with: - name: Catch2 Benchmark (v3) - tool: 'catch2' - output-file-path: examples/catch2/benchmark_result.txt - skip-fetch-gh-pages: true - fail-on-alert: true - summary-always: true - github-token: ${{ secrets.GITHUB_TOKEN }} - comment-on-alert: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark (v3)' - - cpp-framework: - name: Run Google C++ Benchmark Framework example - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run build - - name: Cache Benchmark library - uses: actions/cache@v4 - with: - path: examples/cpp/benchmark - key: ${{ runner.os }}-googlebenchmark-v1.5.0 - - name: Run benchmark - run: | - cd examples/cpp - make json - - name: Save previous data.js - run: | - git fetch origin gh-pages - git checkout gh-pages - cp ./dev/bench/data.js before_data.js - git checkout - - - name: Store benchmark result - uses: ./ - with: - name: C++ Benchmark - tool: 'googlecpp' - output-file-path: examples/cpp/benchmark_result.json - skip-fetch-gh-pages: true - fail-on-alert: true - summary-always: true - github-token: ${{ secrets.GITHUB_TOKEN }} - comment-on-alert: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'C++ Benchmark' - - go: - name: Run Go benchmark example - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - uses: actions/setup-go@v4 - with: - go-version: "stable" - - run: npm ci - - run: npm run build - - name: Run benchmark - run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt - - name: Save previous data.js - run: | - git fetch origin gh-pages - git checkout gh-pages - cp ./dev/bench/data.js before_data.js - git checkout - - - name: Store benchmark result - uses: ./ - with: - name: Go Benchmark - tool: 'go' - output-file-path: examples/go/output.txt - skip-fetch-gh-pages: true - fail-on-alert: true - summary-always: true - github-token: ${{ secrets.GITHUB_TOKEN }} - comment-on-alert: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Go Benchmark' - - java-jmh: - name: Run JMH Java Benchmark Framework example - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - uses: actions/setup-java@v4 - with: - distribution: 'adopt' - java-version: '11' - - run: npm ci - - run: npm run build - - name: Run benchmark - run: | - cd examples/java - mvn clean verify - java -jar target/benchmarks.jar -wi 1 -i 3 -f 1 -rf json - - name: Save previous data.js - run: | - git fetch origin gh-pages - git checkout gh-pages - cp ./dev/bench/data.js before_data.js - git checkout - - - name: Store benchmark result - uses: ./ - with: - name: JMH Benchmark - tool: "jmh" - output-file-path: examples/java/jmh-result.json - skip-fetch-gh-pages: true - fail-on-alert: true - summary-always: true - github-token: ${{ secrets.GITHUB_TOKEN }} - comment-on-alert: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'JMH Benchmark' - - julia-benchmark: - name: Run Julia benchmark example - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run build - - uses: julia-actions/setup-julia@v1 - with: - version: '1' - arch: x64 - - name: Run benchmark - run: | - cd examples/julia - julia --project --color=yes -e ' - using Pkg; - Pkg.instantiate(); - include("fib.jl")' - - name: Save previous data.js - run: | - git fetch origin gh-pages - git checkout gh-pages - cp ./dev/bench/data.js before_data.js - git checkout - - - name: Store benchmark result - uses: ./ - with: - name: Julia benchmark - tool: 'julia' - output-file-path: examples/julia/output.json - skip-fetch-gh-pages: true - fail-on-alert: true - summary-always: true - github-token: ${{ secrets.GITHUB_TOKEN }} - comment-on-alert: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Julia benchmark' - - pytest-benchmark: - name: Run Pytest benchmark example - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - uses: actions/setup-python@v5 - with: - python-version: 3.9 - - run: npm ci - - run: npm run build - - name: Run benchmark - run: | - cd examples/pytest - pip install -r requirements.txt - pytest bench.py --benchmark-json output.json - - name: Save previous data.js - run: | - git fetch origin gh-pages - git checkout gh-pages - cp ./dev/bench/data.js before_data.js - git checkout - - - name: Store benchmark result - uses: ./ - with: - name: Python Benchmark with pytest-benchmark - tool: 'pytest' - output-file-path: examples/pytest/output.json - skip-fetch-gh-pages: true - fail-on-alert: true - summary-always: true - github-token: ${{ secrets.GITHUB_TOKEN }} - comment-on-alert: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Python Benchmark with pytest-benchmark' - - rust: - name: Run Rust benchmark example - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run build - - run: rustup toolchain update nightly && rustup default nightly - - name: Run benchmark - run: cd examples/rust && cargo +nightly bench | tee output.txt - - name: Save previous data.js - run: | - git fetch origin gh-pages - git checkout gh-pages - cp ./dev/bench/data.js before_data.js - git checkout - - - name: Store benchmark result - uses: ./ - with: - name: Rust Benchmark - tool: 'cargo' - output-file-path: examples/rust/output.txt - fail-on-alert: true - summary-always: true - github-token: ${{ secrets.GITHUB_TOKEN }} - comment-on-alert: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Rust Benchmark' - - rust-criterion-rs-framework: - name: Run Criterion.rs benchmark example - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run build - - run: rustup toolchain update nightly && rustup default nightly - - name: Run benchmark - run: cd examples/criterion-rs && cargo +nightly bench -- --output-format bencher | tee output.txt - - name: Save previous data.js - run: | - git fetch origin gh-pages - git checkout gh-pages - cp ./dev/bench/data.js before_data.js - git checkout - - - name: Store benchmark result - uses: ./ - with: - name: Criterion.rs Benchmark - tool: 'cargo' - output-file-path: examples/criterion-rs/output.txt - fail-on-alert: true - summary-always: true - github-token: ${{ secrets.GITHUB_TOKEN }} - comment-on-alert: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Criterion.rs Benchmark' - - only-alert-with-cache: - name: Run alert check with actions/cache - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run build - - uses: actions/setup-go@v4 - with: - go-version: "stable" - - name: Run benchmark - run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt - - name: Download previous benchmark data - uses: actions/cache@v4 - with: - path: ./cache - key: ${{ runner.os }}-ci-cache-test - - name: Store benchmark result - uses: ./ - with: - name: Alert setup example with cache - tool: 'go' - output-file-path: examples/go/output.txt - external-data-json-path: ./cache/benchmark-data.json - github-token: ${{ secrets.GITHUB_TOKEN }} - alert-threshold: '200%' - comment-on-alert: true - fail-on-alert: true - alert-comment-cc-users: '@ktrz' - unit-tests: - name: Run unit tests - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run coverage - - name: Apply yamllint - run: | - sudo pip install yamllint - yamllint --strict .github/workflows - - uses: codecov/codecov-action@v2 - - lint: - name: Run linting and formatting check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run lint - - run: npm run format:check diff --git a/.github/workflows/commit-comment.yml b/.github/workflows/commit-comment.yml deleted file mode 100644 index 7f30433..0000000 --- a/.github/workflows/commit-comment.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: Example for alert with commit comment -on: - push: - branches: - - master - -permissions: - contents: write - deployments: write - -jobs: - benchmark: - name: Run alert check without using Git branch - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v4 - with: - go-version: "stable" - - name: Run benchmark - run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt - - name: Download previous benchmark data - uses: actions/cache@v4 - with: - path: ./cache - key: ${{ runner.os }}-benchmark - - name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 - with: - name: Alert setup example with cache - tool: 'go' - output-file-path: examples/go/output.txt - external-data-json-path: ./cache/benchmark-data.json - github-token: ${{ secrets.GITHUB_TOKEN }} - alert-threshold: '200%' - comment-on-alert: true - fail-on-alert: true - alert-comment-cc-users: '@ktrz' diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml deleted file mode 100644 index 1034c0e..0000000 --- a/.github/workflows/cpp.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: C++ Example -on: - push: - branches: - - master - -permissions: - contents: write - deployments: write - -jobs: - benchmark: - name: Run Google C++ Benchmark Framework example - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4 - - name: Cache Benchmark library - uses: actions/cache@v4 - with: - path: examples/cpp/benchmark - key: ${{ runner.os }}-googlebenchmark-v1.5.0 - - name: Run benchmark - run: cd examples/cpp && make json - - - name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 - with: - name: C++ Benchmark - tool: 'googlecpp' - output-file-path: examples/cpp/benchmark_result.json - github-token: ${{ secrets.GITHUB_TOKEN }} - auto-push: true - # Show alert with commit comment on detecting possible performance regression - alert-threshold: '200%' - comment-on-alert: true - fail-on-alert: true - alert-comment-cc-users: '@ktrz' - - - name: Store benchmark result - separate results repo - uses: benchmark-action/github-action-benchmark@v1 - with: - name: C++ Benchmark - tool: 'googlecpp' - output-file-path: examples/cpp/benchmark_result.json - github-token: ${{ secrets.BENCHMARK_ACTION_BOT_TOKEN }} - auto-push: true - # Show alert with commit comment on detecting possible performance regression - alert-threshold: '200%' - comment-on-alert: true - fail-on-alert: true - alert-comment-cc-users: '@ktrz' - gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml deleted file mode 100644 index 4657de9..0000000 --- a/.github/workflows/go.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Go Example -on: - push: - branches: - - master - -permissions: - contents: write - deployments: write - -jobs: - benchmark: - name: Run Go benchmark example - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v4 - with: - go-version: "stable" - - name: Run benchmark - run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt - - - name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 - with: - name: Go Benchmark - tool: 'go' - output-file-path: examples/go/output.txt - github-token: ${{ secrets.GITHUB_TOKEN }} - auto-push: true - # Show alert with commit comment on detecting possible performance regression - alert-threshold: '200%' - comment-on-alert: true - fail-on-alert: true - alert-comment-cc-users: '@ktrz' - - - name: Store benchmark result - separate results repo - uses: benchmark-action/github-action-benchmark@v1 - with: - name: Go Benchmark - tool: 'go' - output-file-path: examples/go/output.txt - github-token: ${{ secrets.BENCHMARK_ACTION_BOT_TOKEN }} - auto-push: true - # Show alert with commit comment on detecting possible performance regression - alert-threshold: '200%' - comment-on-alert: true - fail-on-alert: true - alert-comment-cc-users: '@ktrz' - gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml deleted file mode 100644 index 03c706e..0000000 --- a/.github/workflows/java.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: JMH Example -on: - push: - branches: - - master - -permissions: - contents: write - deployments: write - -jobs: - benchmark: - name: Run JMH benchmark example - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - distribution: 'adopt' - java-version: '11' - - name: Run benchmark - run: | - cd examples/java - mvn clean verify - java -jar target/benchmarks.jar -wi 1 -i 3 -f 1 -rf json - - - name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 - with: - name: JMH Benchmark - tool: 'jmh' - output-file-path: examples/java/jmh-result.json - # Use personal access token instead of GITHUB_TOKEN due to https://github.community/t/github-action-not-triggering-gh-pages-upon-push/16096 - github-token: ${{ secrets.GITHUB_TOKEN }} - auto-push: true - # Show alert with commit comment on detecting possible performance regression - alert-threshold: '200%' - comment-on-alert: true - fail-on-alert: true - alert-comment-cc-users: '@michaelmior' - - - name: Store benchmark result - separate results repo - uses: benchmark-action/github-action-benchmark@v1 - with: - name: JMH Benchmark - tool: 'jmh' - output-file-path: examples/java/jmh-result.json - # Use personal access token instead of GITHUB_TOKEN due to https://github.community/t/github-action-not-triggering-gh-pages-upon-push/16096 - github-token: ${{ secrets.BENCHMARK_ACTION_BOT_TOKEN }} - auto-push: true - # Show alert with commit comment on detecting possible performance regression - alert-threshold: '200%' - comment-on-alert: true - fail-on-alert: true - alert-comment-cc-users: '@michaelmior' - gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' diff --git a/.github/workflows/julia.yml b/.github/workflows/julia.yml deleted file mode 100644 index 7676a6a..0000000 --- a/.github/workflows/julia.yml +++ /dev/null @@ -1,78 +0,0 @@ -name: Julia Example with BenchmarkTools.jl -on: - push: - branches: - - master - -permissions: - contents: write - deployments: write - -jobs: - benchmark: - name: Run julia benchmark example - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - version: - - '1' - os: - - ubuntu-latest - arch: - - x64 - steps: - - uses: actions/checkout@v4 - - uses: julia-actions/setup-julia@v1 - with: - version: ${{ matrix.version }} - arch: ${{ matrix.arch }} - - uses: actions/cache@v4 - env: - cache-name: cache-artifacts - with: - path: ~/.julia/artifacts - key: runner.os−test−env.cache−name−{{ hashFiles('**/Project.toml') }} - restore-keys: | - runner.os−test− - ${{ env.cache-name }}- - ${{ runner.os }}-test- - ${{ runner.os }}- - - name: Run benchmark - run: | - cd examples/julia - julia --project --color=yes -e ' - using Pkg; - Pkg.instantiate(); - include("fib.jl")' - - - name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 - with: - name: Julia benchmark result - tool: 'julia' - output-file-path: examples/julia/output.json - # Use personal access token instead of GITHUB_TOKEN due to https://github.community/t/github-action-not-triggering-gh-pages-upon-push/16096 - github-token: ${{ secrets.GITHUB_TOKEN }} - auto-push: true - # Show alert with commit comment on detecting possible performance regression - alert-threshold: '200%' - comment-on-alert: true - fail-on-alert: true - alert-comment-cc-users: '@ktrz,@findmyway' - - - name: Store benchmark result - separate results repo - uses: benchmark-action/github-action-benchmark@v1 - with: - name: Julia benchmark result - tool: 'julia' - output-file-path: examples/julia/output.json - # Use personal access token instead of GITHUB_TOKEN due to https://github.community/t/github-action-not-triggering-gh-pages-upon-push/16096 - github-token: ${{ secrets.BENCHMARK_ACTION_BOT_TOKEN }} - auto-push: true - # Show alert with commit comment on detecting possible performance regression - alert-threshold: '200%' - comment-on-alert: true - fail-on-alert: true - alert-comment-cc-users: '@ktrz,@findmyway' - gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' diff --git a/.github/workflows/minimal.yml b/.github/workflows/minimal.yml index a8edaba..5fe3060 100644 --- a/.github/workflows/minimal.yml +++ b/.github/workflows/minimal.yml @@ -1,33 +1,21 @@ name: Example for minimal setup -on: - push: - branches: - - master -permissions: - contents: write - deployments: write +on: + workflow_dispatch: + pull_request: jobs: - benchmark: - name: Run minimal steps to run github-action-benchmark + minimal-test: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-go@v4 - with: - go-version: "stable" - - name: Run benchmark - run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt - - name: Download previous benchmark data - uses: actions/cache@v4 - with: - path: ./cache - key: ${{ runner.os }}-benchmark - - name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 + - name: "build" + run: | + npm install + npm run build + - uses: ./ # local action in repo with: - tool: 'go' - output-file-path: examples/go/output.txt - external-data-json-path: ./cache/benchmark-data.json - fail-on-alert: true + tool: 'cargo' + platform: 'default' + output-file-path: ./test/data/extract/cargo_output.txt + data-out-path: output.txt diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml deleted file mode 100644 index d888854..0000000 --- a/.github/workflows/pytest.yml +++ /dev/null @@ -1,55 +0,0 @@ -name: Python Example with pytest -on: - push: - branches: - - master - -permissions: - contents: write - deployments: write - -jobs: - benchmark: - name: Run pytest-benchmark benchmark example - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: 3.9 - - name: Run benchmark - run: | - cd examples/pytest - pip install -r requirements.txt - pytest bench.py --benchmark-json output.json - - - name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 - with: - name: Python Benchmark with pytest-benchmark - tool: 'pytest' - output-file-path: examples/pytest/output.json - # Use personal access token instead of GITHUB_TOKEN due to https://github.community/t/github-action-not-triggering-gh-pages-upon-push/16096 - github-token: ${{ secrets.GITHUB_TOKEN }} - auto-push: true - # Show alert with commit comment on detecting possible performance regression - alert-threshold: '200%' - comment-on-alert: true - fail-on-alert: true - alert-comment-cc-users: '@ktrz' - - - name: Store benchmark result - separate results repo - uses: benchmark-action/github-action-benchmark@v1 - with: - name: Python Benchmark with pytest-benchmark - tool: 'pytest' - output-file-path: examples/pytest/output.json - # Use personal access token instead of GITHUB_TOKEN due to https://github.community/t/github-action-not-triggering-gh-pages-upon-push/16096 - github-token: ${{ secrets.BENCHMARK_ACTION_BOT_TOKEN }} - auto-push: true - # Show alert with commit comment on detecting possible performance regression - alert-threshold: '200%' - comment-on-alert: true - fail-on-alert: true - alert-comment-cc-users: '@ktrz' - gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' diff --git a/.github/workflows/rust-criterion-rs.yml b/.github/workflows/rust-criterion-rs.yml deleted file mode 100644 index 4034b9d..0000000 --- a/.github/workflows/rust-criterion-rs.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: Criterion.rs Example -on: - push: - branches: - - master - -permissions: - contents: write - deployments: write - -jobs: - benchmark: - name: Run Criterion.rs benchmark example - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - run: rustup toolchain update nightly && rustup default nightly - - name: Run benchmark - run: cd examples/criterion-rs && cargo +nightly bench -- --output-format bencher | tee output.txt - - - name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 - with: - name: Criterion.rs Benchmark - tool: 'cargo' - output-file-path: examples/criterion-rs/output.txt - github-token: ${{ secrets.GITHUB_TOKEN }} - auto-push: true - # Show alert with commit comment on detecting possible performance regression - alert-threshold: '200%' - comment-on-alert: true - fail-on-alert: true - alert-comment-cc-users: '@ktrz' - - - name: Store benchmark result - separate results repo - uses: benchmark-action/github-action-benchmark@v1 - with: - name: Criterion.rs Benchmark - tool: 'cargo' - output-file-path: examples/criterion-rs/output.txt - github-token: ${{ secrets.BENCHMARK_ACTION_BOT_TOKEN }} - auto-push: true - # Show alert with commit comment on detecting possible performance regression - alert-threshold: '200%' - comment-on-alert: true - fail-on-alert: true - alert-comment-cc-users: '@ktrz' - gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml deleted file mode 100644 index 61aa225..0000000 --- a/.github/workflows/rust.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: Rust Example -on: - push: - branches: - - master - -permissions: - contents: write - deployments: write - -jobs: - benchmark: - name: Run Rust benchmark example - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - run: rustup toolchain update nightly && rustup default nightly - - name: Run benchmark - run: cd examples/rust && cargo +nightly bench | tee output.txt - - - name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 - with: - name: Rust Benchmark - tool: 'cargo' - output-file-path: examples/rust/output.txt - github-token: ${{ secrets.GITHUB_TOKEN }} - auto-push: true - # Show alert with commit comment on detecting possible performance regression - alert-threshold: '200%' - comment-on-alert: true - fail-on-alert: true - alert-comment-cc-users: '@ktrz' - - - name: Store benchmark result - separate results repo - uses: benchmark-action/github-action-benchmark@v1 - with: - name: Rust Benchmark - tool: 'cargo' - output-file-path: examples/rust/output.txt - github-token: ${{ secrets.BENCHMARK_ACTION_BOT_TOKEN }} - auto-push: true - # Show alert with commit comment on detecting possible performance regression - alert-threshold: '200%' - summary-always: true - comment-on-alert: true - fail-on-alert: true - alert-comment-cc-users: '@ktrz' - gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' diff --git a/.github/workflows/typescript-tests.yml b/.github/workflows/typescript-tests.yml new file mode 100644 index 0000000..387500d --- /dev/null +++ b/.github/workflows/typescript-tests.yml @@ -0,0 +1,16 @@ +name: Run TypeScript tests + +on: + workflow_dispatch: + pull_request: + +jobs: + typescript-tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: "build and test" + run: | + npm install + npm run build + npm run test diff --git a/.github/workflows/validate-action-typings.yml b/.github/workflows/validate-action-typings.yml deleted file mode 100644 index 1eb0f1c..0000000 --- a/.github/workflows/validate-action-typings.yml +++ /dev/null @@ -1,15 +0,0 @@ -name: Validate action typings - -on: - push: - branches: [master] - pull_request: - branches: [master] - workflow_dispatch: - -jobs: - validate-typings: - runs-on: "ubuntu-latest" - steps: - - uses: actions/checkout@v4 - - uses: typesafegithub/github-actions-typing@v1 diff --git a/.gitignore b/.gitignore index 9578ebe..ea240a3 100644 --- a/.gitignore +++ b/.gitignore @@ -3,9 +3,6 @@ /examples/rust/target /examples/criterion-rs/Cargo.lock /examples/criterion-rs/target -/examples/benchmarkdotnet/bin -/examples/benchmarkdotnet/obj -/examples/benchmarkdotnet/BenchmarkDotNet.Artifacts /.nyc_output /coverage /dist diff --git a/README.md b/README.md index 7dacb6c..dfcc1df 100644 --- a/README.md +++ b/README.md @@ -1,676 +1,29 @@ -GitHub Action for Continuous Benchmarking -========================================= -[![Action Marketplace][release-badge]][marketplace] -[![Build Status][build-badge]][ci] -[![codecov][codecov-badge]][codecov] +# GitHub Action for extraction and transformation of benchmark data -[This repository][proj] provides a [GitHub Action][github-action] for continuous benchmarking. -If your project has some benchmark suites, this action collects data from the benchmark outputs -and monitor the results on GitHub Actions workflow. +GitHub Action to transform benchmark output into a standardized format, and write to a file. -- This action can store collected benchmark results in [GitHub pages][gh-pages] branch and provide - a chart view. Benchmark results are visualized on the GitHub pages of your project. -- This action can detect possible performance regressions by comparing benchmark results. When - benchmark results get worse than previous exceeding the specified threshold, it can raise an alert - via commit comment or workflow failure. +Currently supported sources: + - `cargo bench` + - `criterion` -This action currently supports the following tools: +## Fields +- `name` (required): name of the benchmark +- `tool` (required): tool used to get benchmark output. One of `["cargo"]` +- `platform` (required): a string describing the platform +- `output-file-path` (required): a path to a file containing the output of the benchmark tool +- `data-out-path` (required): the path where the output of the action should be written -- [`cargo bench`][cargo-bench] for Rust projects -- `go test -bench` for Go projects -- [benchmark.js][benchmarkjs] for JavaScript/TypeScript projects -- [pytest-benchmark][] for Python projects with [pytest][] -- [Google Benchmark Framework][google-benchmark] for C++ projects -- [Catch2][catch2] for C++ projects -- [BenchmarkTools.jl][] for Julia packages -- [Benchmark.Net][benchmarkdotnet] for .Net projects -- [benchmarkluau](https://github.com/Roblox/luau/tree/master/bench) for Luau projects -- [JMH][jmh] for Java projects -- Custom benchmarks where either 'biggerIsBetter' or 'smallerIsBetter' - -Multiple languages in the same repository are supported for polyglot projects. - -[Japanese Blog post](https://rhysd.hatenablog.com/entry/2019/11/11/131505) - - - -## Examples - -Example projects for each language are in [examples/](./examples) directory. Live example workflow -definitions are in [.github/workflows/](./.github/workflows) directory. Live workflows are: - -| Language | Workflow | Example Project | -|--------------|-----------------------------------------------------------------------------------------|------------------------------------------------| -| Rust | [![Rust Example Workflow][rust-badge]][rust-workflow-example] | [examples/rust](./examples/rust) | -| Go | [![Go Example Workflow][go-badge]][go-workflow-example] | [examples/go](./examples/go) | -| JavaScript | [![JavaScript Example Workflow][benchmarkjs-badge]][benchmarkjs-workflow-example] | [examples/benchmarkjs](./examples/benchmarkjs) | -| Python | [![pytest-benchmark Example Workflow][pytest-benchmark-badge]][pytest-workflow-example] | [examples/pytest](./examples/pytest) | -| C++ | [![C++ Example Workflow][cpp-badge]][cpp-workflow-example] | [examples/cpp](./examples/cpp) | -| C++ (Catch2) | [![C++ Catch2 Example Workflow][catch2-badge]][catch2-workflow-example] | [examples/catch2](./examples/catch2) | -| Julia | [![Julia Example][julia-badge]][julia-workflow-example] | [examples/julia](./examples/julia) | -| .Net | [![C# Benchmark.Net Example Workflow][benchmarkdotnet-badge]][benchmarkdotnet-workflow-example] | [examples/benchmarkdotnet](./examples/benchmarkdotnet) | -| Java | [![Java Example Workflow][java-badge]][java-workflow-example] | [examples/java](./examples/java) | -| Luau | Coming soon | Coming soon | - -All benchmark charts from above workflows are gathered in GitHub pages: - -https://benchmark-action.github.io/github-action-benchmark/dev/bench/ - -Additionally, even though there is no explicit example for them, you can use -`customBiggerIsBetter` and `customSmallerIsBetter` to use this -action and create your own graphs from your own benchmark data. The name in -these tools define which direction "is better" for your benchmarks. - -Every entry in the JSON file you provide only needs to provide `name`, `unit`, -and `value`. You can also provide optional `range` (results' variance) and -`extra` (any additional information that might be useful to your benchmark's -context) properties. Like this: +## Output data format +The output will be written to `data-out-path` in a standardized JSON format: ```json [ - { - "name": "My Custom Smaller Is Better Benchmark - CPU Load", - "unit": "Percent", - "value": 50 - }, { "name": "My Custom Smaller Is Better Benchmark - Memory Used", "unit": "Megabytes", + "platform": "ubuntu-latest", "value": 100, "range": "3", - "extra": "Value for Tooltip: 25\nOptional Num #2: 100\nAnything Else!" } ] ``` - -## Screenshots - -### Charts on GitHub Pages - -![page screenshot](https://raw.githubusercontent.com/rhysd/ss/master/github-action-benchmark/main.png) - -Mouseover on data point shows a tooltip. It includes - -- Commit hash -- Commit message -- Date and committer -- Benchmark value - -Clicking data point in chart opens the commit page on a GitHub repository. - -![tooltip](https://raw.githubusercontent.com/rhysd/ss/master/github-action-benchmark/tooltip.png) - -At bottom of the page, the download button is available for downloading benchmark results as a JSON file. - -![download button](https://raw.githubusercontent.com/rhysd/ss/master/github-action-benchmark/download.png) - - -### Alert comment on commit page - -This action can raise [an alert comment][alert-comment-example]. to the commit when its benchmark -results are worse than previous exceeding a specified threshold. - -![alert comment](https://raw.githubusercontent.com/rhysd/ss/master/github-action-benchmark/alert-comment.png) - - - -## Why? - -Since performance is important. Writing benchmarks is a popular and correct way to visualize a software -performance. Benchmarks help us to keep performance and to confirm the effects of optimizations. -For keeping the performance, it's important to monitor the benchmark results along with changes to -the software. To notice performance regression quickly, it's useful to monitor benchmarking results -continuously. - -However, there is no good free tool to watch the performance easily and continuously across languages -(as far as I looked into). So I built a new tool on top of GitHub Actions. - - - -## How to use - -This action takes a file that contains benchmark output. And it outputs the results to GitHub Pages -branch and/or alert commit comment. - - -### Minimal setup - -Let's start with a minimal workflow setup. For explanation, here let's say we have a Go project. But basic -setup is the same when you use other languages. For language-specific setup, please read the later section. - -```yaml -name: Minimal setup -on: - push: - branches: - - master - -jobs: - benchmark: - name: Performance regression check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v4 - with: - go-version: "stable" - # Run benchmark with `go test -bench` and stores the output to a file - - name: Run benchmark - run: go test -bench 'BenchmarkFib' | tee output.txt - # Download previous benchmark result from cache (if exists) - - name: Download previous benchmark data - uses: actions/cache@v4 - with: - path: ./cache - key: ${{ runner.os }}-benchmark - # Run `github-action-benchmark` action - - name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 - with: - # What benchmark tool the output.txt came from - tool: 'go' - # Where the output from the benchmark tool is stored - output-file-path: output.txt - # Where the previous data file is stored - external-data-json-path: ./cache/benchmark-data.json - # Workflow will fail when an alert happens - fail-on-alert: true - # Upload the updated cache file for the next job by actions/cache -``` - -The step which runs `github-action-benchmark` does followings: - -1. Extract benchmark result from the output in `output.txt` -2. Update the downloaded cache file with the extracted result -3. Compare the result with the previous result. If it gets worse than previous exceeding 200% threshold, - the workflow fails and the failure is notified to you - -By default, this action marks the result as performance regression when it is worse than the previous -exceeding 200% threshold. For example, if the previous benchmark result was 100 iter/ns and this time -it is 230 iter/ns, it means 230% worse than the previous and an alert will happen. The threshold can -be changed by `alert-threshold` input. - -A live workflow example is [here](.github/workflows/minimal.yml). And the results of the workflow can -be seen [here][minimal-workflow-example]. - - -### Commit comment - -In addition to the above setup, GitHub API token needs to be given to enable `comment-on-alert` feature. - -```yaml -- name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 - with: - tool: 'go' - output-file-path: output.txt - external-data-json-path: ./cache/benchmark-data.json - fail-on-alert: true - # GitHub API token to make a commit comment - github-token: ${{ secrets.GITHUB_TOKEN }} - # Enable alert commit comment - comment-on-alert: true - # Mention @rhysd in the commit comment - alert-comment-cc-users: '@rhysd' -``` - -`secrets.GITHUB_TOKEN` is [a GitHub API token automatically generated for each workflow run][help-github-token]. -It is necessary to send a commit comment when the benchmark result of the commit is detected as possible -performance regression. - -Now, in addition to making workflow fail, the step leaves a commit comment when it detects performance -regression [like this][alert-comment-example]. Though `alert-comment-cc-users` input is not mandatory for -this, I recommend to set it to make sure you can notice the comment via GitHub notification. Please note -that this value must be quoted like `'@rhysd'` because [`@` is an indicator in YAML syntax](https://yaml.org/spec/1.2/spec.html#id2772075). - -A live workflow example is [here](.github/workflows/commit-comment.yml). And the results of the workflow -can be seen [here][commit-comment-workflow-example]. - -### Job Summary - -Similar to the [Commit comment](#commit-comment) feature, Github Actions [Job Summaries](https://github.blog/2022-05-09-supercharging-github-actions-with-job-summaries/) are -also supported. In order to use Job Summaries, turn on the `summary-always` -option. - -```yaml -- name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 - with: - tool: 'cargo' - output-file-path: output.txt - external-data-json-path: ./cache/benchmark-data.json - fail-on-alert: true - # GitHub API token to make a commit comment - github-token: ${{ secrets.GITHUB_TOKEN }} - # Enable alert commit comment - comment-on-alert: true - # Enable Job Summary for PRs - summary-always: true - # Mention @rhysd in the commit comment - alert-comment-cc-users: '@rhysd' -``` - -### Charts on GitHub Pages - -It is useful to see how the benchmark results changed on each change in time-series charts. This action -provides a chart dashboard on GitHub pages. - -It requires some preparations before the workflow setup. - -You need to create a branch for GitHub Pages if you haven't created it yet. - -```sh -# Create a local branch -$ git checkout --orphan gh-pages -# Push it to create a remote branch -$ git push origin gh-pages:gh-pages -``` - -Now you're ready for workflow setup. - -```yaml -# Do not run this workflow on pull request since this workflow has permission to modify contents. -on: - push: - branches: - - master - -permissions: - # deployments permission to deploy GitHub pages website - deployments: write - # contents permission to update benchmark contents in gh-pages branch - contents: write - -jobs: - benchmark: - name: Performance regression check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v4 - with: - go-version: "stable" - # Run benchmark with `go test -bench` and stores the output to a file - - name: Run benchmark - run: go test -bench 'BenchmarkFib' | tee output.txt - # gh-pages branch is updated and pushed automatically with extracted benchmark data - - name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 - with: - name: My Project Go Benchmark - tool: 'go' - output-file-path: output.txt - # Access token to deploy GitHub Pages branch - github-token: ${{ secrets.GITHUB_TOKEN }} - # Push and deploy GitHub pages branch automatically - auto-push: true -``` - -The step which runs `github-action-benchmark` does followings: - -1. Extract benchmark result from the output in `output.txt` -2. Switch branch to `gh-pages` -3. Read existing benchmark results from `dev/bench/data.js` -4. Update `dev/bench/data.js` with the extracted benchmark result -5. Generate a commit to store the update in `gh-pages` branch -6. Push `gh-pages` branch to remote -7. Compare the results with previous results and make an alert if possible performance regression is detected - -After the first workflow run, you will get the first result on `https://you.github.io/repo/dev/bench` -[like this][examples-page]. - -By default, this action assumes that `gh-pages` is your GitHub Pages branch and that `/dev/bench` is -a path to put the benchmark dashboard page. If they don't fit your use case, please tweak them by -`gh-pages-branch`, `gh-repository` and `benchmark-data-dir-path` inputs. - -This action merges all benchmark results into one GitHub pages branch. If your workflows have multiple -steps to check benchmarks from multiple tools, please give `name` input to each step to make each -benchmark results identical. - -Please see the above ['Examples' section](#examples) to see live workflow examples for each language. - -If you don't want to pass GitHub API token to this action, it's still OK. - -```yaml -- name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 - with: - name: My Project Go Benchmark - tool: 'go' - output-file-path: output.txt - # Set auto-push to false since GitHub API token is not given - auto-push: false -# Push gh-pages branch by yourself -- name: Push benchmark result - run: git push 'https://you:${{ secrets.GITHUB_TOKEN }}@github.com/you/repo-name.git' gh-pages:gh-pages -``` - -Please add a step to push the branch to the remote. - - -### Tool specific setup - -Please read `README.md` files at each example directory. Usually, take stdout from a benchmark tool -and store it to file. Then specify the file path to `output-file-path` input. - -- [`cargo bench` for Rust projects](./examples/rust/README.md) -- [`go test` for Go projects](./examples/go/README.md) -- [Benchmark.js for JavaScript/TypeScript projects](./examples/benchmarkjs/README.md) -- [pytest-benchmark for Python projects with pytest](./examples/pytest/README.md) -- [Google Benchmark Framework for C++ projects](./examples/cpp/README.md) -- [catch2 for C++ projects](./examples/cpp/README.md) -- [BenchmarkTools.jl for Julia projects](./examples/julia/README.md) -- [Benchmark.Net for .Net projects](./examples/benchmarkdotnet/README.md) -- [benchmarkluau for Luau projects](#) - Examples for this are still a work in progress. - -These examples are run in workflows of this repository as described in the 'Examples' section above. - - -### Action inputs - -Input definitions are written in [action.yml](./action.yml). - -#### `name` (Required) - -- Type: String -- Default: `"Benchmark"` - -Name of the benchmark. This value must be identical across all benchmarks in your repository. - -#### `tool` (Required) - -- Type: String -- Default: N/A - -Tool for running benchmark. The value must be one of `"cargo"`, `"go"`, `"benchmarkjs"`, `"pytest"`, -`"googlecpp"`, `"catch2"`, `"julia"`, `"jmh"`, `"benchmarkdotnet"`,`"benchmarkluau"`, `"customBiggerIsBetter"`, `"customSmallerIsBetter"`. - -#### `output-file-path` (Required) - -- Type: String -- Default: N/A - -Path to a file which contains the output from benchmark tool. The path can be relative to repository root. - -#### `gh-pages-branch` (Required) - -- Type: String -- Default: `"gh-pages"` - -Name of your GitHub pages branch. - -Note: If you're using `docs/` directory of `master` branch for GitHub pages, please set `gh-pages-branch` -to `master` and `benchmark-data-dir-path` to the directory under `docs` like `docs/dev/bench`. - -#### `gh-repository` - -- Type: String - -Url to an optional different repository to store benchmark results (eg. `github.com/benchmark-action/github-action-benchmark-results`) - -NOTE: if you want to auto push to a different repository you need to use a separate Personal Access Token that has a write access to the specified repository. -If you are not using the `auto-push` option then you can avoid passing the `gh-token` if your data repository is public - -#### `benchmark-data-dir-path` (Required) - -- Type: String -- Default: `"dev/bench"` - -Path to a directory that contains benchmark files on the GitHub pages branch. For example, when this value -is set to `"path/to/bench"`, `https://you.github.io/repo-name/path/to/bench` will be available as benchmarks -dashboard page. If it does not contain `index.html`, this action automatically generates it at first run. -The path can be relative to repository root. - -#### `github-token` (Optional) - -- Type: String -- Default: N/A - -GitHub API access token. - -#### `ref` (Optional) - -- Type: String -- Default: N/A - -Ref to use for reporting the commit - -#### `auto-push` (Optional) - -- Type: Boolean -- Default: `false` - -If it is set to `true`, this action automatically pushes the generated commit to GitHub Pages branch. -Otherwise, you need to push it by your own. Please read 'Commit comment' section above for more details. - -#### `comment-always` (Optional) - -- Type: Boolean -- Default: `false` - -If it is set to `true`, this action will leave a commit comment comparing the current benchmark with previous. -`github-token` is necessary as well. - -#### `save-data-file` (Optional) - -- Type: Boolean -- Default: `true` - -If it is set to `false`, this action will not save the current benchmark to the external data file. -You can use this option to set up your action to compare the benchmarks between PR and base branch. - -#### `alert-threshold` (Optional) - -- Type: String -- Default: `"200%"` - -Percentage value like `"150%"`. It is a ratio indicating how worse the current benchmark result is. -For example, if we now get `150 ns/iter` and previously got `100 ns/iter`, it gets `150%` worse. - -If the current benchmark result is worse than previous exceeding the threshold, an alert will happen. -See `comment-on-alert` and `fail-on-alert` also. - -#### `comment-on-alert` (Optional) - -- Type: Boolean -- Default: `false` - -If it is set to `true`, this action will leave a commit comment when an alert happens [like this][alert-comment-example]. -`github-token` is necessary as well. For the threshold, please see `alert-threshold` also. - -#### `fail-on-alert` (Optional) - -- Type: Boolean -- Default: `false` - -If it is set to `true`, the workflow will fail when an alert happens. For the threshold for this, please -see `alert-threshold` and `fail-threshold` also. - -#### `fail-threshold` (Optional) - -- Type: String -- Default: The same value as `alert-threshold` - -Percentage value in the same format as `alert-threshold`. If this value is set, the threshold value -will be used to determine if the workflow should fail. Default value is set to the same value as -`alert-threshold` input. **This value must be equal or larger than `alert-threshold` value.** - -#### `alert-comment-cc-users` (Optional) - -- Type: String -- Default: N/A - -Comma-separated GitHub user names mentioned in alert commit comment like `"@foo,@bar"`. These users -will be mentioned in a commit comment when an alert happens. For configuring alerts, please see -`alert-threshold` and `comment-on-alert` also. - -#### `external-data-json-path` (Optional) - -- Type: String -- Default: N/A - -External JSON file which contains benchmark results until previous job run. When this value is set, -this action updates the file content instead of generating a Git commit in GitHub Pages branch. -This option is useful if you don't want to put benchmark results in GitHub Pages branch. Instead, -you need to keep the JSON file persistently among job runs. One option is using a workflow cache -with `actions/cache` action. Please read 'Minimal setup' section above. - -#### `max-items-in-chart` (Optional) - -- Type: Number -- Default: N/A - -Max number of data points in a chart for avoiding too busy chart. This value must be unsigned integer -larger than zero. If the number of benchmark results for some benchmark suite exceeds this value, -the oldest one will be removed before storing the results to file. By default this value is empty -which means there is no limit. - -#### `skip-fetch-gh-pages` (Optional) - -- Type: Boolean -- Default: `false` - -If set to `true`, the workflow will skip fetching branch defined with the `gh-pages-branch` variable. - - -### Action outputs - -No action output is set by this action for the parent GitHub workflow. - - -### Caveats - -#### Run only on your branches - -Please ensure that your benchmark workflow runs only on your branches. Please avoid running it on -pull requests. If a branch were pushed to GitHub pages branch on a pull request, anyone who creates -a pull request on your repository could modify your GitHub pages branch. - -For this, you can specify a branch that runs your benchmark workflow on `on:` section. Or set the -proper condition to `if:` section of step which pushes GitHub pages. - -e.g. Runs on only `master` branch - -```yaml -on: - push: - branches: - - master -``` - -e.g. Push when not running for a pull request - -```yaml -- name: Push benchmark result - run: git push ... - if: github.event_name != 'pull_request' -``` - -#### Stability of Virtual Environment - -As far as watching the benchmark results of examples in this repository, the amplitude of the benchmarks -is about +- 10~20%. If your benchmarks use some resources such as networks or file I/O, the amplitude -might be bigger. - -If the amplitude is not acceptable, please prepare a stable environment to run benchmarks. -GitHub action supports [self-hosted runners](https://docs.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners). - - -### Customizing the benchmarks result page - -This action creates the default `index.html` in the directory specified with `benchmark-data-dir-path` -input. By default, every benchmark test case has own chart on the page. Charts are drawn with -[Chart.js](https://www.chartjs.org/). - -If it does not fit your use case, please modify the HTML file or replace it with your favorite one. -Every benchmark data is stored in `window.BENCHMARK_DATA` so you can create your favorite view. - - -### Versioning - -This action conforms semantic versioning 2.0. - -For example, `benchmark-action/github-action-benchmark@v1` means the latest version of `1.x.y`. And -`benchmark-action/github-action-benchmark@v1.0.2` always uses `v1.0.2` even if a newer version is published. - -`master` branch of this repository is for development and does not work as action. - - -### Track updates of this action - -To notice new version releases, please [watch 'release only'][help-watch-release] at [this repository][proj]. -Every release will appear on your GitHub notifications page. - - - -## Future work - -- Support pull requests. Instead of updating GitHub pages, add a comment to the pull request to explain - benchmark results. -- Add more benchmark tools: - - [airspeed-velocity Python benchmarking tool](https://github.com/airspeed-velocity/asv) -- Allow uploading results to metrics services such as [mackerel](https://en.mackerel.io/) -- Show extracted benchmark data in the output from this action -- Add a table view in dashboard page to see all data points in table - - - -## Related actions - -- [lighthouse-ci-action][] is an action for [Lighthouse CI][lighthouse-ci]. If you're measuring performance - of your web application, using Lighthouse CI and lighthouse-ci-action would be better than using this - action. - - - -## License - -[the MIT License](./LICENSE.txt) - - - -[build-badge]: https://github.com/benchmark-action/github-action-benchmark/actions/workflows/ci.yml/badge.svg -[ci]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3ACI -[codecov-badge]: https://codecov.io/gh/benchmark-action/github-action-benchmark/branch/master/graph/badge.svg -[codecov]: https://app.codecov.io/gh/benchmark-action/github-action-benchmark -[release-badge]: https://img.shields.io/github/v/release/benchmark-action/github-action-benchmark.svg -[marketplace]: https://github.com/marketplace/actions/continuous-benchmark -[proj]: https://github.com/benchmark-action/github-action-benchmark -[rust-badge]: https://github.com/benchmark-action/github-action-benchmark/actions/workflows/rust.yml/badge.svg -[go-badge]: https://github.com/benchmark-action/github-action-benchmark/actions/workflows/go.yml/badge.svg -[benchmarkjs-badge]: https://github.com/benchmark-action/github-action-benchmark/actions/workflows/benchmarkjs.yml/badge.svg -[pytest-benchmark-badge]: https://github.com/benchmark-action/github-action-benchmark/actions/workflows/pytest.yml/badge.svg -[cpp-badge]: https://github.com/benchmark-action/github-action-benchmark/actions/workflows/cpp.yml/badge.svg -[catch2-badge]: https://github.com/benchmark-action/github-action-benchmark/actions/workflows/catch2.yml/badge.svg -[julia-badge]: https://github.com/benchmark-action/github-action-benchmark/actions/workflows/julia.yml/badge.svg -[java-badge]: https://github.com/benchmark-action/github-action-benchmark/actions/workflows/java.yml/badge.svg -[github-action]: https://github.com/features/actions -[cargo-bench]: https://doc.rust-lang.org/cargo/commands/cargo-bench.html -[benchmarkjs]: https://benchmarkjs.com/ -[gh-pages]: https://pages.github.com/ -[examples-page]: https://benchmark-action.github.io/github-action-benchmark/dev/bench/ -[pytest-benchmark]: https://pypi.org/project/pytest-benchmark/ -[pytest]: https://pypi.org/project/pytest/ -[alert-comment-example]: https://github.com/benchmark-action/github-action-benchmark/commit/077dde1c236baba9244caad4d9e82ea8399dae20#commitcomment-36047186 -[rust-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Rust+Example%22 -[go-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Go+Example%22 -[benchmarkjs-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Benchmark.js+Example%22 -[pytest-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Python+Example+with+pytest%22 -[cpp-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22C%2B%2B+Example%22 -[catch2-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Catch2+C%2B%2B+Example%22 -[julia-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Julia+Example+with+BenchmarkTools.jl%22 -[java-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22JMH+Example%22 -[help-watch-release]: https://docs.github.com/en/github/receiving-notifications-about-activity-on-github/watching-and-unwatching-releases-for-a-repository -[help-github-token]: https://docs.github.com/en/actions/security-guides/automatic-token-authentication -[minimal-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Example+for+minimal+setup%22 -[commit-comment-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Example+for+alert+with+commit+comment%22 -[google-benchmark]: https://github.com/google/benchmark -[catch2]: https://github.com/catchorg/Catch2 -[jmh]: https://openjdk.java.net/projects/code-tools/jmh/ -[lighthouse-ci-action]: https://github.com/treosh/lighthouse-ci-action -[lighthouse-ci]: https://github.com/GoogleChrome/lighthouse-ci -[BenchmarkTools.jl]: https://github.com/JuliaCI/BaseBenchmarks.jl -[benchmarkdotnet]: https://benchmarkdotnet.org -[benchmarkdotnet-badge]: https://github.com/benchmark-action/github-action-benchmark/actions/workflows/benchmarkdotnet.yml/badge.svg -[benchmarkdotnet-workflow-example]: https://github.com/rhysd/github-action-benchmark/actions?query=workflow%3A%22Benchmark.Net+Example%22 -[job-summaries]: https://github.blog/2022-05-09-supercharging-github-actions-with-job-summaries/ diff --git a/action-types.yml b/action-types.yml index e199a66..de67f57 100644 --- a/action-types.yml +++ b/action-types.yml @@ -5,53 +5,9 @@ inputs: type: enum allowed-values: - cargo - - go - - benchmarkjs - - pytest - - googlecpp - - catch2 - - julia - - jmh - - benchmarkdotnet - - benchmarkluau - - customBiggerIsBetter - - customSmallerIsBetter - output-file-path: - type: string - gh-pages-branch: - type: string - gh-repository: - type: string - benchmark-data-dir-path: + platform: type: string - github-token: - type: string - ref: - type: string - auto-push: - type: boolean - skip-fetch-gh-pages: - type: boolean - comment-always: - type: boolean - summary-always: - type: boolean - save-data-file: - type: boolean - comment-on-alert: - type: boolean - alert-threshold: - type: string - fail-on-alert: - type: boolean - fail-threshold: + output-file-path: type: string - alert-comment-cc-users: - type: list - separator: ',' - list-item: - type: string - external-data-json-path: + data-out-path: type: string - max-items-in-chart: - type: integer diff --git a/action.yml b/action.yml index daa6cd3..3b348ef 100644 --- a/action.yml +++ b/action.yml @@ -1,6 +1,6 @@ -name: 'Continuous Benchmark' -author: 'github-action-benchmark developers ' -description: 'Continuous Benchmark using GitHub pages as dash board for keeping performance' +name: 'Benchmark extract and transform' +author: 'Cryspen ' +description: 'Extract and transform `cargo` and other benchmark data for visualization' branding: icon: 'fast-forward' color: 'blue' @@ -11,75 +11,18 @@ inputs: required: true default: 'Benchmark' tool: - description: 'Tool to use get benchmark output. One of "cargo", "go", "benchmarkjs", "pytest", "googlecpp", "catch2", "julia", "benchmarkdotnet", "customBiggerIsBetter", "customSmallerIsBetter"' + description: 'Tool to use to get benchmark output. One of "cargo"...' + required: true + platform: + description: 'A string describing the platform' required: true output-file-path: description: 'A path to file which contains the benchmark output' required: true - gh-pages-branch: - description: 'Branch for gh-pages' - required: true - default: 'gh-pages' - gh-repository: - description: 'Url to an optional different repository to store benchmark results' - required: false - benchmark-data-dir-path: - description: 'Path to directory which contains benchmark files on GitHub pages branch' + data-out-path: + description: 'A path to the file where the output of the action should be written' required: true - default: 'dev/bench' - github-token: - description: 'GitHub API token to pull/push GitHub pages branch and deploy GitHub pages. For public repository, this must be personal access token for now. Please read README.md for more details' - required: false - ref: - description: 'optional Ref to use when finding commit' - required: false - auto-push: - description: 'Push GitHub Pages branch to remote automatically. This option requires github-token input' - required: false - default: false - skip-fetch-gh-pages: - description: 'Skip pulling GitHub Pages branch before generating an auto commit' - required: false - default: false - comment-always: - description: 'Leave a comment with benchmark result comparison. To enable this feature, github-token input must be given as well' - required: false - default: false - summary-always: - description: 'Leave a job summary with benchmark result comparison' - required: false - default: false - save-data-file: - description: 'Save the benchmark data to external file' - required: false - default: true - comment-on-alert: - description: 'Leave an alert comment when current benchmark result is worse than previous. Threshold is specified with alert-threshold input. To enable this feature, github-token input must be given as well' - required: false - default: false - alert-threshold: - description: 'Threshold which determines if an alert should happen or not. Percentage value such as "150%". For example, 150% means that an alert happens when current benchmark result is 1.5x worse than previous' - required: false - default: '200%' - fail-on-alert: - description: 'Workflow fails when alert comment happens' - required: false - # Note: Set to false by default since this action does not push to remote by default. When workflow - # fails and auto-push is not set, there is no chance to push the result to remote. - default: false - fail-threshold: - description: 'Threshold which determines if the current workflow fails. Format is the same as alert-threshold input. If this value is not specified, the same value as alert-threshold is used' - required: false - alert-comment-cc-users: - description: 'Comma separated GitHub user names which start with @ (e.g. "@foo,@bar"). They will be mentioned in commit comment for alert.' - required: false - external-data-json-path: - description: 'JSON data file for storing benchmark results. When this input is set, github-action-benchmark no longer uses Git branch to store data. Instead, it reads and appends benchmark data from/to the file. User must store the file anywhere' - required: false - max-items-in-chart: - description: 'Max data points in a benchmark chart to avoid making the chart too busy. Value must be unsigned integer. No limit by default' - required: false runs: using: 'node20' - main: 'dist/src/index.js' + main: 'dist/index.js' diff --git a/examples/benchmarkdotnet/Benchmarks.cs b/examples/benchmarkdotnet/Benchmarks.cs deleted file mode 100644 index d9f79c8..0000000 --- a/examples/benchmarkdotnet/Benchmarks.cs +++ /dev/null @@ -1,23 +0,0 @@ -using BenchmarkDotNet.Attributes; - -namespace Sample -{ - public class Benchmarks - { - public static int Fib(int n) { - switch (n) - { - case 0: return 0; - case 1: return 1; - default: - return Fib(n-2) + Fib(2-1); - } - } - - [Benchmark] - public void Fib10() => Fib(10); - - [Benchmark] - public void Fib20() => Fib(20); - } -} diff --git a/examples/benchmarkdotnet/Program.cs b/examples/benchmarkdotnet/Program.cs deleted file mode 100644 index f12c00f..0000000 --- a/examples/benchmarkdotnet/Program.cs +++ /dev/null @@ -1,9 +0,0 @@ -using BenchmarkDotNet.Running; - -namespace Sample -{ - public class Program - { - public static void Main(string[] args) => BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args); - } -} \ No newline at end of file diff --git a/examples/benchmarkdotnet/README.md b/examples/benchmarkdotnet/README.md deleted file mode 100644 index 8810c88..0000000 --- a/examples/benchmarkdotnet/README.md +++ /dev/null @@ -1,90 +0,0 @@ -C# example for benchmarking with `Benchmark.Net` -================================================ - -- [Workflow for this example](../../.github/workflows/benchmarkdotnet.yml) -- [Action log of this example](https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Benchmark.Net+example%22) -- [Benchmark results on GitHub pages](https://benchmark-action.github.io/github-action-benchmark/dev/bench/) - -This directory shows how to use [`github-action-benchmark`](https://github.com/benchmark-action/github-action-benchmark) -with [`BenchmarkDotNet`](https://benchmarkdotnet.org/). - -## Run benchmarks - -Official documentation for usage of `BenchmarkDotNet`: - -https://benchmarkdotnet.org/articles/overview.html - -You should add the `BenchmarkDotNet` package to your test project and configure your tests according to the [Getting Started](https://benchmarkdotnet.org/articles/guides/getting-started.html) docs. A simple test file might look like - - -```csharp -using System; -using System.Security.Cryptography; -using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Running; - -namespace MyBenchmarks -{ - [JsonExporterAttribute.Full] - [JsonExporterAttribute.FullCompressed] - public class Md5VsSha256 - { - private const int N = 10000; - private readonly byte[] data; - - private readonly SHA256 sha256 = SHA256.Create(); - private readonly MD5 md5 = MD5.Create(); - - public Md5VsSha256() - { - data = new byte[N]; - new Random(42).NextBytes(data); - } - - [Benchmark] - public byte[] Sha256() => sha256.ComputeHash(data); - - [Benchmark] - public byte[] Md5() => md5.ComputeHash(data); - } - - public class Program - { - public static void Main(string[] args) - { - var summary = BenchmarkRunner.Run(); - } - } -} -``` - -You can then run the tests using `dotnet run`. It's _very_ important that you ensure the JSON exporter is configured. You can do this by adding at least one of the exporter attributes in the example above, or by using the `BenchmarkSwitcher` type to run your tests, passing in your `args`, and using `--exporters json` from the command line. - -## Process benchmark results - -Store the benchmark results with step using the action. Please set `benchmarkdotnet` to `tool` input. - -By default, BenchmarkDotNet will output results files to the current directory in a structure like: - -``` -BenchmarkDotNet.Artifacts -├── Sample.Benchmarks-20200529-153703.log -├── Sample.Benchmarks-20200529-153729.log -└── results - ├── Sample.Benchmarks-report-full-compressed.json - ├── Sample.Benchmarks-report-github.md - ├── Sample.Benchmarks-report.csv - └── Sample.Benchmarks-report.html -``` - -You want to get the path of the `-report-full-compressed.json` report for use with this action. Once you have both pieces of data, use the action like so, replacing the output file path with your own path. - -```yaml -- name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 - with: - tool: 'benchmarkdotnet' - output-file-path: BenchmarkDotNet.Artifacts/results/Sample.Benchmarks-report-full-compressed.json -``` - -Please read [How to use section](https://github.com/benchmark-action/github-action-benchmark#how-to-use) for common usage. diff --git a/examples/benchmarkdotnet/Sample.csproj b/examples/benchmarkdotnet/Sample.csproj deleted file mode 100644 index 3f27694..0000000 --- a/examples/benchmarkdotnet/Sample.csproj +++ /dev/null @@ -1,17 +0,0 @@ - - - net6.0 - Exe - - - AnyCPU - portable - true - true - true - Release - - - - - \ No newline at end of file diff --git a/examples/benchmarkdotnet/global.json b/examples/benchmarkdotnet/global.json deleted file mode 100644 index e52d340..0000000 --- a/examples/benchmarkdotnet/global.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "sdk": { - "version": "6.0.101" - } -} diff --git a/examples/benchmarkjs/.gitignore b/examples/benchmarkjs/.gitignore deleted file mode 100644 index d502512..0000000 --- a/examples/benchmarkjs/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/node_modules -/package-lock.json diff --git a/examples/benchmarkjs/README.md b/examples/benchmarkjs/README.md deleted file mode 100644 index b4621f2..0000000 --- a/examples/benchmarkjs/README.md +++ /dev/null @@ -1,62 +0,0 @@ -JavaScript example for benchmarking with [benchmark.js][tool] -============================================================= - -- [Workflow for this example](../../.github/workflows/benchmarkjs.yml) -- [Action log of this example](https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Benchmark.js+Example%22) -- [Benchmark results on GitHub pages](https://benchmark-action.github.io/github-action-benchmark/dev/bench/) - -This directory shows how to use [`github-action-benchmark`](https://github.com/benchmark-action/github-action-benchmark) -with [benchmark.js][tool]. - -## Run benchmarks - -Official documentation for usage of benchmark.js: - -https://benchmarkjs.com/ - -Prepare script `bench.js` as follows: - -e.g. - -```javascript -const Benchmark = require('benchmark'); -const suite = new Benchmark.Suite(); - -suite - .add('some test case', () => { - // ... - }) - .on('cycle', event => { - // Output benchmark result by converting benchmark result to string - console.log(String(event.target)); - }) - .run(); -``` - -Ensure the output includes string values converted from benchmark results. -This action extracts measured values fron the output. - -Run the script in workflow: - -e.g. - -```yaml -- name: Run benchmark - run: node bench.js | tee output.txt -``` - -## Process benchmark results - -Store the benchmark results with step using the action. Please set `benchmarkjs` to `tool` input. - -```yaml -- name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 - with: - tool: 'benchmarkjs' - output-file-path: output.txt -``` - -Please read ['How to use' section](https://github.com/benchmark-action/github-action-benchmark#how-to-use) for common usage. - -[tool]: https://benchmarkjs.com/ diff --git a/examples/benchmarkjs/bench.js b/examples/benchmarkjs/bench.js deleted file mode 100644 index 2cc5183..0000000 --- a/examples/benchmarkjs/bench.js +++ /dev/null @@ -1,15 +0,0 @@ -const Benchmark = require('benchmark'); -const suite = new Benchmark.Suite(); -const { fib } = require('./index'); - -suite - .add('fib(10)', () => { - fib(10); - }) - .add('fib(20)', () => { - fib(20); - }) - .on('cycle', (event) => { - console.log(String(event.target)); - }) - .run(); diff --git a/examples/benchmarkjs/index.js b/examples/benchmarkjs/index.js deleted file mode 100644 index 868afdd..0000000 --- a/examples/benchmarkjs/index.js +++ /dev/null @@ -1,8 +0,0 @@ -function fib(n) { - if (n <= 1) { - return 1; - } - return fib(n - 2) + fib(n - 1); -} - -exports.fib = fib; diff --git a/examples/benchmarkjs/package.json b/examples/benchmarkjs/package.json deleted file mode 100644 index 2a27c0f..0000000 --- a/examples/benchmarkjs/package.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "name": "benchmark-example", - "private": true, - "version": "0.0.0", - "description": "", - "main": "index.js", - "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" - }, - "author": "rhysd (https://rhysd.github.io/)", - "license": "MIT", - "devDependencies": { - "benchmark": "^2.1.4" - } -} diff --git a/examples/catch2/.gitignore b/examples/catch2/.gitignore deleted file mode 100644 index 567609b..0000000 --- a/examples/catch2/.gitignore +++ /dev/null @@ -1 +0,0 @@ -build/ diff --git a/examples/catch2/CMakeLists.txt b/examples/catch2/CMakeLists.txt deleted file mode 100644 index 844f532..0000000 --- a/examples/catch2/CMakeLists.txt +++ /dev/null @@ -1,26 +0,0 @@ -cmake_minimum_required(VERSION 3.12) -project("Catch2_bench") - -include(FetchContent) - -FetchContent_Declare( - catch2 - GIT_REPOSITORY https://github.com/catchorg/Catch2.git - GIT_TAG v3.6.0) - -FetchContent_GetProperties(catch2) -if(NOT catch2_POPULATED) - FetchContent_Populate(catch2) - add_subdirectory(${catch2_SOURCE_DIR} ${catch2_BINARY_DIR}) -endif() - -add_executable(${PROJECT_NAME}) -target_sources(${PROJECT_NAME} PRIVATE catch2_bench.cpp) -target_link_libraries(${PROJECT_NAME} Catch2::Catch2WithMain) - -target_compile_options( - ${PROJECT_NAME} - PRIVATE - $<$:/DCATCH_CONFIG_ENABLE_BENCHMARKING> - $<$,$,$>:-DCATCH_CONFIG_ENABLE_BENCHMARKING> -) diff --git a/examples/catch2/README.md b/examples/catch2/README.md deleted file mode 100644 index b22944b..0000000 --- a/examples/catch2/README.md +++ /dev/null @@ -1,68 +0,0 @@ -C++ example for benchmarking with [Catch2 Framework][tool] -==================================================================== - -- [Workflow for this example](../../.github/workflows/catch2.yml) -- [Action log of this example](https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Catch2+C%2B%2B+Example%22) -- [Benchmark results on GitHub pages](https://benchmark-action.github.io/github-action-benchmark/dev/bench/) - -This directory shows how to use [`github-action-benchmark`][action] with [Catch2 Framework][tool]. - - - -## Run benchmarks - -Official documentation for usage of Catch2 Framework can be found in its repository: - -https://github.com/catchorg/Catch2 - -Since Catch2 is a header-only test framework, you don't need to build it in advance. -Download and put the headers in your `include` directory and write your benchmarks. - -```cpp -#define CATCH_CONFIG_MAIN -#include - -TEST_CASE("Fibonacci") { - // now let's benchmark: - BENCHMARK("Some benchmark") { - // Your benchmark goes here - }; -} -``` - -Build the source with C++ compiler and run the built executable to get the benchmark output. -Ensure to use `console` reporter for this. `xml` reporter may be supported in the future. - - - -## Process benchmark results - -Store the benchmark results with step using the action. Please set `catch2` to `tool` input. - -```yaml -- name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 - with: - tool: 'catch2' - output-file-path: benchmark_result.json -``` - -Please read ['How to use' section](https://github.com/benchmark-action/github-action-benchmark#how-to-use) for common usage. - - - -## Run this example - -To try this example, please use [cmake](./CMakeLists.txt) and `clang++`. - -```sh -$ mkdir build -$ cd build -$ cmake -DCMAKE_BUILD_TYPE=Release .. -$ cmake --build . --config Release -``` - -This will create `Catch2_bench` executable. The results are output to stdout. - -[tool]: https://github.com/catchorg/Catch2 -[action]: https://github.com/benchmark-action/github-action-benchmark diff --git a/examples/catch2/catch2_bench.cpp b/examples/catch2/catch2_bench.cpp deleted file mode 100644 index b0f58cb..0000000 --- a/examples/catch2/catch2_bench.cpp +++ /dev/null @@ -1,11 +0,0 @@ -#include "fib.hpp" -#include -#include - -TEST_CASE("Fibonacci") { - - // now let's benchmark: - BENCHMARK("Fibonacci 10") { return fib(10); }; - - BENCHMARK("Fibonacci 20") { return fib(20); }; -} diff --git a/examples/catch2/fib.hpp b/examples/catch2/fib.hpp deleted file mode 100644 index 559e886..0000000 --- a/examples/catch2/fib.hpp +++ /dev/null @@ -1,11 +0,0 @@ -#if !defined FIB_HPP_INCLUDED -#define FIB_HPP_INCLUDED - -int fib(int const i) { - if (i <= 1) { - return 1; - } - return fib(i - 2) + fib(i - 1); -} - -#endif // FIB_HPP_INCLUDED diff --git a/examples/catch2_v2/.gitignore b/examples/catch2_v2/.gitignore deleted file mode 100644 index 567609b..0000000 --- a/examples/catch2_v2/.gitignore +++ /dev/null @@ -1 +0,0 @@ -build/ diff --git a/examples/catch2_v2/CMakeLists.txt b/examples/catch2_v2/CMakeLists.txt deleted file mode 100644 index 5d0ba31..0000000 --- a/examples/catch2_v2/CMakeLists.txt +++ /dev/null @@ -1,26 +0,0 @@ -cmake_minimum_required(VERSION 3.12) -project("Catch2_bench") - -include(FetchContent) - -FetchContent_Declare( - catch2 - GIT_REPOSITORY https://github.com/catchorg/Catch2.git - GIT_TAG v2.13.10) - -FetchContent_GetProperties(catch2) -if(NOT catch2_POPULATED) - FetchContent_Populate(catch2) - add_subdirectory(${catch2_SOURCE_DIR} ${catch2_BINARY_DIR}) -endif() - -add_executable(${PROJECT_NAME}) -target_sources(${PROJECT_NAME} PRIVATE catch2_bench.cpp) -target_link_libraries(${PROJECT_NAME} Catch2::Catch2) - -target_compile_options( - ${PROJECT_NAME} - PRIVATE - $<$:/DCATCH_CONFIG_ENABLE_BENCHMARKING> - $<$,$,$>:-DCATCH_CONFIG_ENABLE_BENCHMARKING> -) diff --git a/examples/catch2_v2/README.md b/examples/catch2_v2/README.md deleted file mode 100644 index b22944b..0000000 --- a/examples/catch2_v2/README.md +++ /dev/null @@ -1,68 +0,0 @@ -C++ example for benchmarking with [Catch2 Framework][tool] -==================================================================== - -- [Workflow for this example](../../.github/workflows/catch2.yml) -- [Action log of this example](https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Catch2+C%2B%2B+Example%22) -- [Benchmark results on GitHub pages](https://benchmark-action.github.io/github-action-benchmark/dev/bench/) - -This directory shows how to use [`github-action-benchmark`][action] with [Catch2 Framework][tool]. - - - -## Run benchmarks - -Official documentation for usage of Catch2 Framework can be found in its repository: - -https://github.com/catchorg/Catch2 - -Since Catch2 is a header-only test framework, you don't need to build it in advance. -Download and put the headers in your `include` directory and write your benchmarks. - -```cpp -#define CATCH_CONFIG_MAIN -#include - -TEST_CASE("Fibonacci") { - // now let's benchmark: - BENCHMARK("Some benchmark") { - // Your benchmark goes here - }; -} -``` - -Build the source with C++ compiler and run the built executable to get the benchmark output. -Ensure to use `console` reporter for this. `xml` reporter may be supported in the future. - - - -## Process benchmark results - -Store the benchmark results with step using the action. Please set `catch2` to `tool` input. - -```yaml -- name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 - with: - tool: 'catch2' - output-file-path: benchmark_result.json -``` - -Please read ['How to use' section](https://github.com/benchmark-action/github-action-benchmark#how-to-use) for common usage. - - - -## Run this example - -To try this example, please use [cmake](./CMakeLists.txt) and `clang++`. - -```sh -$ mkdir build -$ cd build -$ cmake -DCMAKE_BUILD_TYPE=Release .. -$ cmake --build . --config Release -``` - -This will create `Catch2_bench` executable. The results are output to stdout. - -[tool]: https://github.com/catchorg/Catch2 -[action]: https://github.com/benchmark-action/github-action-benchmark diff --git a/examples/catch2_v2/catch2_bench.cpp b/examples/catch2_v2/catch2_bench.cpp deleted file mode 100644 index 4c28804..0000000 --- a/examples/catch2_v2/catch2_bench.cpp +++ /dev/null @@ -1,11 +0,0 @@ -#include "fib.hpp" -#define CATCH_CONFIG_MAIN -#include - -TEST_CASE("Fibonacci") { - - // now let's benchmark: - BENCHMARK("Fibonacci 10") { return fib(10); }; - - BENCHMARK("Fibonacci 20") { return fib(20); }; -} diff --git a/examples/catch2_v2/fib.hpp b/examples/catch2_v2/fib.hpp deleted file mode 100644 index 559e886..0000000 --- a/examples/catch2_v2/fib.hpp +++ /dev/null @@ -1,11 +0,0 @@ -#if !defined FIB_HPP_INCLUDED -#define FIB_HPP_INCLUDED - -int fib(int const i) { - if (i <= 1) { - return 1; - } - return fib(i - 2) + fib(i - 1); -} - -#endif // FIB_HPP_INCLUDED diff --git a/examples/cpp/.gitignore b/examples/cpp/.gitignore deleted file mode 100644 index a52ac63..0000000 --- a/examples/cpp/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -/benchmark -/a.out -/benchmark_result.json diff --git a/examples/cpp/Makefile b/examples/cpp/Makefile deleted file mode 100644 index 2d5f6d6..0000000 --- a/examples/cpp/Makefile +++ /dev/null @@ -1,27 +0,0 @@ -bench: a.out - ./a.out - -json: a.out - ./a.out --benchmark_format=json | tee benchmark_result.json - -a.out: benchmark/build/src/libbenchmark.a bench.cpp fib.hpp - clang++ -std=c++14 -O3 -I ./benchmark/include -L ./benchmark/build/src/ -pthread bench.cpp -l benchmark - -benchmark/build/src/libbenchmark.a: benchmark/build benchmark/googletest - cd ./benchmark/build && \ - cmake -DCMAKE_BUILD_TYPE=Release -DBENCHMARK_ENABLE_TESTING=true ../ && \ - make -j - -benchmark/build: benchmark - mkdir -p benchmark/build - -benchmark: - [ -d benchmark ] || git clone --depth=1 --single-branch --branch v1.5.0 https://github.com/google/benchmark.git benchmark - -benchmark/googletest: benchmark - [ -d benchmark/googletest ] || git clone --depth=1 --single-branch --branch release-1.10.0 https://github.com/google/googletest.git benchmark/googletest - -clean: - rm -rf a.out benchmark - -.PHONY: bench json clean diff --git a/examples/cpp/README.md b/examples/cpp/README.md deleted file mode 100644 index 5151b18..0000000 --- a/examples/cpp/README.md +++ /dev/null @@ -1,99 +0,0 @@ -C++ example for benchmarking with [Google Benchmark Framework][tool] -==================================================================== - -- [Workflow for this example](../../.github/workflows/cpp.yml) -- [Action log of this example](https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22C%2B%2B+Example%22) -- [Benchmark results on GitHub pages](https://benchmark-action.github.io/github-action-benchmark/dev/bench/) - -This directory shows how to use [`github-action-benchmark`][action] with [Google Benchmark Framework][tool]. - -## Run benchmarks - -Official documentation for usage of Google Benchmark Framework: - -https://github.com/google/benchmark - -Build and install `benchmark` library and write up your benchmark suites following instructions in -the above repository: - -```cpp -#include "benchmark/benchmark.h" - -static void bench1(benchmark::State &state) { - for (auto _ : state) { - // Your benchmark goes here - } -} - -// Register the function as a benchmark -BENCHMARK(bench1); - -// Run the benchmark -BENCHMARK_MAIN(); -``` - -Build the source with C++ compiler: - -```sh -$ clang++ -std=c++14 -O3 -l benchmark bench.cpp -``` - -And run built executable to output the result in JSON format: - -```sh -$ ./a.out --benchmark_format=json | tee benchmark_result.json -``` - -## Process benchmark results - -Store the benchmark results with step using the action. Please set `googlecpp` to `tool` input. - -```yaml -- name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 - with: - tool: 'googlecpp' - output-file-path: benchmark_result.json -``` - -Please read ['How to use' section](https://github.com/benchmark-action/github-action-benchmark#how-to-use) for common usage. - -## Run this example - -To try this example, please use [make](./Makefile) and `clang++`. - -```sh -$ make bench -``` - -`bench` subcommand prepares all dependencies, compiles `bench.cpp` and runs benchmarks. The results -are output to console. - -``` -2019-11-29 21:13:55 -Running ./a.out -Run on (4 X 2700 MHz CPU s) -CPU Caches: - L1 Data 32K (x2) - L1 Instruction 32K (x2) - L2 Unified 262K (x2) - L3 Unified 3145K (x1) -Load Average: 1.66, 1.98, 2.49 ------------------------------------------------------ -Benchmark Time CPU Iterations ------------------------------------------------------ -fib_10 210 ns 210 ns 3239181 -fib_20 27857 ns 27786 ns 25206 -``` - -To get JSON output for running [github-action-benchmark][action], please use another subcommand. - -```sh -$ make json -``` - -`json` subcommand outputs the benchmark results in JSON format and generates `benchmark_result.json` -file in current directory. - -[tool]: https://github.com/google/benchmark -[action]: https://github.com/benchmark-action/github-action-benchmark diff --git a/examples/cpp/bench.cpp b/examples/cpp/bench.cpp deleted file mode 100644 index e1b0b6f..0000000 --- a/examples/cpp/bench.cpp +++ /dev/null @@ -1,26 +0,0 @@ -#include "./fib.hpp" -#include "benchmark/benchmark.h" - -static void fib_10(benchmark::State &state) { - for (auto _ : state) { - // Suppress optimization otherwise this line is removed by DCE - int i = 10; - benchmark::DoNotOptimize(i); - benchmark::DoNotOptimize(fib(i)); - } -} - -static void fib_20(benchmark::State &state) { - for (auto _ : state) { - int i = 20; - benchmark::DoNotOptimize(i); - benchmark::DoNotOptimize(fib(i)); - } -} - -// Register the function as a benchmark -BENCHMARK(fib_10); -BENCHMARK(fib_20); - -// Run the benchmark -BENCHMARK_MAIN(); diff --git a/examples/cpp/fib.hpp b/examples/cpp/fib.hpp deleted file mode 100644 index 559e886..0000000 --- a/examples/cpp/fib.hpp +++ /dev/null @@ -1,11 +0,0 @@ -#if !defined FIB_HPP_INCLUDED -#define FIB_HPP_INCLUDED - -int fib(int const i) { - if (i <= 1) { - return 1; - } - return fib(i - 2) + fib(i - 1); -} - -#endif // FIB_HPP_INCLUDED diff --git a/examples/go/README.md b/examples/go/README.md deleted file mode 100644 index 1f8fca0..0000000 --- a/examples/go/README.md +++ /dev/null @@ -1,36 +0,0 @@ -Go example for benchmarking with `go test -bench` -================================================= - -- [Workflow for this example](../../.github/workflows/go.yml) -- [Action log of this example](https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Go+Example%22) -- [Benchmark results on GitHub pages](https://benchmark-action.github.io/github-action-benchmark/dev/bench/) - -This directory shows how to use [`github-action-benchmark`](https://github.com/benchmark-action/github-action-benchmark) -with `go test -bench` command. - -## Run benchmarks - -Official documentation for usage of `go test -bench`: - -https://pkg.go.dev/testing#hdr-Benchmarks - -e.g. - -```yaml -- name: Run benchmark - run: go test -bench 'Benchmark' | tee output.txt -``` - -## Process benchmark results - -Store the benchmark results with step using the action. Please set `go` to `tool` input. - -```yaml -- name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 - with: - tool: 'go' - output-file-path: output.txt -``` - -Please read ['How to use' section](https://github.com/benchmark-action/github-action-benchmark#how-to-use) for common usage. diff --git a/examples/go/fib.go b/examples/go/fib.go deleted file mode 100644 index 21a3676..0000000 --- a/examples/go/fib.go +++ /dev/null @@ -1,8 +0,0 @@ -package fib - -func Fib(u uint) uint { - if u <= 1 { - return 1 - } - return Fib(u-2) + Fib(u-1) -} diff --git a/examples/go/fib_test.go b/examples/go/fib_test.go deleted file mode 100644 index 5b92c20..0000000 --- a/examples/go/fib_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package fib - -import ( - "testing" -) - -func BenchmarkFib10(b *testing.B) { - for i := 0; i < b.N; i++ { - var _ = Fib(10) - } -} - -func BenchmarkFib20(b *testing.B) { - for i := 0; i < b.N; i++ { - var _ = Fib(20) - } -} - -func BenchmarkFib20WithAuxMetric(b *testing.B) { - for i := 0; i < b.N; i++ { - var _ = Fib(20) - } - b.ReportMetric(4.0, "auxMetricUnits") -} diff --git a/examples/go/go.mod b/examples/go/go.mod deleted file mode 100644 index b9be5c9..0000000 --- a/examples/go/go.mod +++ /dev/null @@ -1,3 +0,0 @@ -module github.com/github-action-benchmark/github-action-benchmark/examples/go - -go 1.20 diff --git a/examples/java/pom.xml b/examples/java/pom.xml deleted file mode 100644 index fd6f342..0000000 --- a/examples/java/pom.xml +++ /dev/null @@ -1,172 +0,0 @@ - - - - 4.0.0 - - org.openjdk.jmh.samples - jmh-sample - 1.0 - jar - - JMH benchmark sample: Java - - - - - - org.openjdk.jmh - jmh-core - ${jmh.version} - - - org.openjdk.jmh - jmh-generator-annprocess - ${jmh.version} - provided - - - - - UTF-8 - - - 1.29 - - - 1.8 - - - benchmarks - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.8.0 - - ${javac.target} - ${javac.target} - ${javac.target} - - - - org.apache.maven.plugins - maven-shade-plugin - 3.2.1 - - - package - - shade - - - ${uberjar.name} - - - org.openjdk.jmh.Main - - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - - - - - - maven-clean-plugin - 2.5 - - - maven-deploy-plugin - 2.8.1 - - - maven-install-plugin - 2.5.1 - - - maven-jar-plugin - 2.4 - - - maven-javadoc-plugin - 2.9.1 - - - maven-resources-plugin - 2.6 - - - maven-site-plugin - 3.3 - - - maven-source-plugin - 2.2.1 - - - maven-surefire-plugin - 2.17 - - - - - - diff --git a/examples/java/src/main/java/org/openjdk/jmh/samples/JMHSample_01_HelloWorld.java b/examples/java/src/main/java/org/openjdk/jmh/samples/JMHSample_01_HelloWorld.java deleted file mode 100644 index cd7af88..0000000 --- a/examples/java/src/main/java/org/openjdk/jmh/samples/JMHSample_01_HelloWorld.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Oracle designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -package org.openjdk.jmh.samples; - -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.runner.Runner; -import org.openjdk.jmh.runner.RunnerException; -import org.openjdk.jmh.runner.options.Options; -import org.openjdk.jmh.runner.options.OptionsBuilder; - -public class JMHSample_01_HelloWorld { - - /* - * This is our first benchmark method. - * - * The contract for the benchmark methods is very simple: - * annotate it with @Benchmark, and you are set to go. - * JMH will run the test by continuously calling this method, and measuring - * the performance metrics for its execution. - * - * The method names are non-essential, it matters they are marked with - * @Benchmark. You can have multiple benchmark methods - * within the same class. - * - * Note: if the benchmark method never finishes, then JMH run never - * finishes as well. If you throw the exception from the method body, - * the JMH run ends abruptly for this benchmark, and JMH will run - * the next benchmark down the list. - * - * Although this benchmark measures "nothing", it is the good showcase - * for the overheads the infrastructure bear on the code you measure - * in the method. There are no magical infrastructures which incur no - * overhead, and it's important to know what are the infra overheads - * you are dealing with. You might find this thought unfolded in future - * examples by having the "baseline" measurements to compare against. - */ - - @Benchmark - public void wellHelloThere() { - // this method was intentionally left blank. - } - - /* - * ============================== HOW TO RUN THIS TEST: ==================================== - * - * You are expected to see the run with large number of iterations, and - * very large throughput numbers. You can see that as the estimate of the - * harness overheads per method call. In most of our measurements, it is - * down to several cycles per call. - * - * a) Via command-line: - * $ mvn clean install - * $ java -jar target/benchmarks.jar ".*JMHSample_01.*" - * - * JMH generates self-contained JARs, bundling JMH together with it. - * The runtime options for the JMH are available with "-h": - * $ java -jar target/benchmarks.jar -h - * - * b) Via Java API: - */ - - public static void main(String[] args) throws RunnerException { - Options opt = new OptionsBuilder() - .include(".*" + JMHSample_01_HelloWorld.class.getSimpleName() + ".*") - .forks(1) - .build(); - - new Runner(opt).run(); - } -} diff --git a/examples/julia/.gitignore b/examples/julia/.gitignore deleted file mode 100644 index e76e6ee..0000000 --- a/examples/julia/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/output.json diff --git a/examples/julia/Manifest.toml b/examples/julia/Manifest.toml deleted file mode 100644 index 4103525..0000000 --- a/examples/julia/Manifest.toml +++ /dev/null @@ -1,116 +0,0 @@ -# This file is machine-generated - editing it directly is not advised - -julia_version = "1.10.0" -manifest_format = "2.0" -project_hash = "cdb03a69499471ffe0f32a9f377a6fd82c192b6f" - -[[deps.Artifacts]] -uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" - -[[deps.BenchmarkTools]] -deps = ["JSON", "Logging", "Printf", "Profile", "Statistics", "UUIDs"] -git-tree-sha1 = "f1f03a9fa24271160ed7e73051fba3c1a759b53f" -uuid = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" -version = "1.4.0" - -[[deps.CompilerSupportLibraries_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" -version = "1.0.5+1" - -[[deps.Dates]] -deps = ["Printf"] -uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" - -[[deps.JSON]] -deps = ["Dates", "Mmap", "Parsers", "Unicode"] -git-tree-sha1 = "31e996f0a15c7b280ba9f76636b3ff9e2ae58c9a" -uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" -version = "0.21.4" - -[[deps.Libdl]] -uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" - -[[deps.LinearAlgebra]] -deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] -uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" - -[[deps.Logging]] -uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" - -[[deps.Mmap]] -uuid = "a63ad114-7e13-5084-954f-fe012c677804" - -[[deps.OpenBLAS_jll]] -deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] -uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" -version = "0.3.23+2" - -[[deps.Parsers]] -deps = ["Dates", "PrecompileTools", "UUIDs"] -git-tree-sha1 = "8489905bcdbcfac64d1daa51ca07c0d8f0283821" -uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" -version = "2.8.1" - -[[deps.PrecompileTools]] -deps = ["Preferences"] -git-tree-sha1 = "03b4c25b43cb84cee5c90aa9b5ea0a78fd848d2f" -uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" -version = "1.2.0" - -[[deps.Preferences]] -deps = ["TOML"] -git-tree-sha1 = "00805cd429dcb4870060ff49ef443486c262e38e" -uuid = "21216c6a-2e73-6563-6e65-726566657250" -version = "1.4.1" - -[[deps.Printf]] -deps = ["Unicode"] -uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" - -[[deps.Profile]] -deps = ["Printf"] -uuid = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79" - -[[deps.Random]] -deps = ["SHA"] -uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" - -[[deps.SHA]] -uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" -version = "0.7.0" - -[[deps.Serialization]] -uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" - -[[deps.SparseArrays]] -deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] -uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" -version = "1.10.0" - -[[deps.Statistics]] -deps = ["LinearAlgebra", "SparseArrays"] -uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" -version = "1.10.0" - -[[deps.SuiteSparse_jll]] -deps = ["Artifacts", "Libdl", "libblastrampoline_jll"] -uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" -version = "7.2.1+1" - -[[deps.TOML]] -deps = ["Dates"] -uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" -version = "1.0.3" - -[[deps.UUIDs]] -deps = ["Random", "SHA"] -uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" - -[[deps.Unicode]] -uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" - -[[deps.libblastrampoline_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" -version = "5.8.0+1" diff --git a/examples/julia/Project.toml b/examples/julia/Project.toml deleted file mode 100644 index 05a4894..0000000 --- a/examples/julia/Project.toml +++ /dev/null @@ -1,2 +0,0 @@ -[deps] -BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" diff --git a/examples/julia/README.md b/examples/julia/README.md deleted file mode 100644 index e7bef72..0000000 --- a/examples/julia/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Julia example with `BenchmarkTools.jl` - -Please read the [docs](https://juliaci.github.io/BenchmarkTools.jl/stable/manual/) of `BenchmarkTools.jl` first. Expecially the [BenchmarkGroup](https://juliaci.github.io/BenchmarkTools.jl/stable/manual/#The-BenchmarkGroup-type) section. Generally speaking, we only need the `json` file exported by `BenchmarkTools.save`. You can checkout the [`fib.jl`](./fib.jl) file for how to do it. A [workflow](../../.github/workflows/julia.yml) for this example is also provided to help you integrate it in your project. - -**Note:** Currently we only support test suite after applying an estimation (`minimumm`,`median`, `mean`, `maximum`, `std`). \ No newline at end of file diff --git a/examples/julia/fib.jl b/examples/julia/fib.jl deleted file mode 100644 index 0d6c6bd..0000000 --- a/examples/julia/fib.jl +++ /dev/null @@ -1,15 +0,0 @@ -using BenchmarkTools - -fib(n) = n <= 1 ? 1 : fib(n - 2) + fib(n - 1) - -suite = BenchmarkGroup() - -suite["fib"] = BenchmarkGroup(["tag1", "tag2"]) - -suite["fib"][10] = @benchmarkable fib(10) -suite["fib"][20] = @benchmarkable fib(20) - -tune!(suite) -results = run(suite, verbose = true) - -BenchmarkTools.save("output.json", median(results)) diff --git a/examples/pytest/.gitignore b/examples/pytest/.gitignore deleted file mode 100644 index cde6fdd..0000000 --- a/examples/pytest/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -/venv -__pycache__ -/.benchmarks -/.pytest_cache diff --git a/examples/pytest/README.md b/examples/pytest/README.md deleted file mode 100644 index 9af2a61..0000000 --- a/examples/pytest/README.md +++ /dev/null @@ -1,60 +0,0 @@ -Python example for benchmarking with [pytest-benchmark][tool] -============================================================= - -- [Workflow for this example](../../.github/workflows/pytest.yml) -- [Action log of this example](https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Python+Example+with+pytest%22) -- [Benchmark results on GitHub pages](https://benchmark-action.github.io/github-action-benchmark/dev/bench/) - -This directory shows how to use [`github-action-benchmark`](https://github.com/benchmark-action/github-action-benchmark) -with [pytest-benchmark][tool]. - -## Run benchmarks - -Official documentation for usage of pytest-benchmark: - -https://pytest-benchmark.readthedocs.io/en/stable/ - -Install dependencies with `venv` package using Python3. - -```sh -$ python -m venv venv -$ source venv/bin/activate -$ pip install pytest pytest-benchmark -``` - -Prepare `bench.py` as follows: - -e.g. - -```python -import pytest - -def some_test_case(benchmark): - benchmark(some_func, args) -``` - -And run benchmarks with `--benchmark-json` in workflow. The JSON file will be an input to -github-action-benchmark. - -e.g. - -```yaml -- name: Run benchmark - run: pytest bench.py --benchmark-json output.json -``` - -## Process benchmark results - -Store the benchmark results with step using the action. Please set `pytest` to `tool` input. - -```yaml -- name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 - with: - tool: 'pytest' - output-file-path: output.json -``` - -Please read ['How to use' section](https://github.com/benchmark-action/github-action-benchmark#how-to-use) for common usage. - -[tool]: https://pypi.org/project/pytest-benchmark/ diff --git a/examples/pytest/bench.py b/examples/pytest/bench.py deleted file mode 100644 index da92239..0000000 --- a/examples/pytest/bench.py +++ /dev/null @@ -1,8 +0,0 @@ -from fib import fib -import pytest - -def test_fib_10(benchmark): - benchmark(fib, 10) - -def test_fib_20(benchmark): - benchmark(fib, 20) diff --git a/examples/pytest/fib/__init__.py b/examples/pytest/fib/__init__.py deleted file mode 100644 index 3cb9161..0000000 --- a/examples/pytest/fib/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -def fib(n): - if n <= 1: - return 1 - return fib(n - 2) + fib(n - 1) diff --git a/examples/pytest/requirements.txt b/examples/pytest/requirements.txt deleted file mode 100644 index b50c03a..0000000 --- a/examples/pytest/requirements.txt +++ /dev/null @@ -1,14 +0,0 @@ -atomicwrites==1.3.0 -attrs==19.3.0 -importlib-metadata==0.23 -more-itertools==7.2.0 -packaging==19.2 -pluggy==0.13.0 -py==1.10.0 -py-cpuinfo==5.0.0 -pyparsing==2.4.5 -pytest==5.2.4 -pytest-benchmark==3.2.2 -six==1.13.0 -wcwidth==0.1.7 -zipp==0.6.0 diff --git a/examples/pytest/setup.py b/examples/pytest/setup.py deleted file mode 100644 index d31d913..0000000 --- a/examples/pytest/setup.py +++ /dev/null @@ -1,11 +0,0 @@ -from setuptools import setup - -setup( - name='benchmark-example', - version='0.0.0', - url='https://github.com/benchmark-action/github-action-benchmark', - author='rhysd ', - author_email='github@users.noreply.github.com', - description='Benchmark example with timeit package', - packages=['fib'], -) diff --git a/package-lock.json b/package-lock.json index b027b81..dfd1dda 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,17 +1,16 @@ { - "name": "github-action-benchmark", + "name": "benchmark-data-extract-transform", "version": "0.0.0", "lockfileVersion": 2, "requires": true, "packages": { "": { - "name": "github-action-benchmark", + "name": "benchmark-data-extract-transform", "version": "0.0.0", "license": "MIT", "dependencies": { "@actions/core": "^1.10.0", "@actions/exec": "^1.1.1", - "@actions/github": "^5.1.1", "@actions/io": "^1.1.2" }, "devDependencies": { @@ -59,17 +58,6 @@ "@actions/io": "^1.0.1" } }, - "node_modules/@actions/github": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/@actions/github/-/github-5.1.1.tgz", - "integrity": "sha512-Nk59rMDoJaV+mHCOJPXuvB1zIbomlKS0dmSIqPGxd0enAXBnOfn4VWF+CGtRCwXZG9Epa54tZA7VIRlJDS8A6g==", - "dependencies": { - "@actions/http-client": "^2.0.1", - "@octokit/core": "^3.6.0", - "@octokit/plugin-paginate-rest": "^2.17.0", - "@octokit/plugin-rest-endpoint-methods": "^5.13.0" - } - }, "node_modules/@actions/http-client": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.1.0.tgz", @@ -97,80 +85,19 @@ } }, "node_modules/@babel/code-frame": { - "version": "7.23.5", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.23.5.tgz", - "integrity": "sha512-CgH3s1a96LipHCmSUmYFPwY7MNx8C3avkq7i4Wl3cfa662ldtUe4VM1TPXX70pfmrlWTb6jLqTYrZyT2ZTJBgA==", + "version": "7.26.2", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.26.2.tgz", + "integrity": "sha512-RJlIHRueQgwWitWgF8OdFYGZX328Ax5BCemNGlqHfplnRT9ESi8JkFlvaVYbS+UubVY6dpv87Fs2u5M29iNFVQ==", "dev": true, "dependencies": { - "@babel/highlight": "^7.23.4", - "chalk": "^2.4.2" + "@babel/helper-validator-identifier": "^7.25.9", + "js-tokens": "^4.0.0", + "picocolors": "^1.0.0" }, "engines": { "node": ">=6.9.0" } }, - "node_modules/@babel/code-frame/node_modules/ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "dev": true, - "dependencies": { - "color-convert": "^1.9.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/@babel/code-frame/node_modules/chalk": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", - "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", - "dev": true, - "dependencies": { - "ansi-styles": "^3.2.1", - "escape-string-regexp": "^1.0.5", - "supports-color": "^5.3.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/@babel/code-frame/node_modules/color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "dev": true, - "dependencies": { - "color-name": "1.1.3" - } - }, - "node_modules/@babel/code-frame/node_modules/color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", - "dev": true - }, - "node_modules/@babel/code-frame/node_modules/escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", - "dev": true, - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/@babel/code-frame/node_modules/supports-color": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", - "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", - "dev": true, - "dependencies": { - "has-flag": "^3.0.0" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/@babel/compat-data": { "version": "7.23.5", "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.23.5.tgz", @@ -373,18 +300,18 @@ } }, "node_modules/@babel/helper-string-parser": { - "version": "7.23.4", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.23.4.tgz", - "integrity": "sha512-803gmbQdqwdf4olxrX4AJyFBV/RTr3rSmOj0rKwesmzlfhYNDEs+/iOcznzpNWlJlIlTJC2QfPFcHB6DlzdVLQ==", + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.25.9.tgz", + "integrity": "sha512-4A/SCr/2KLd5jrtOMFzaKjVtAei3+2r/NChoBNoZ3EyP/+GlhoaEGoWOZUmFmoITP7zOJyHIMm+DYRd8o3PvHA==", "dev": true, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-validator-identifier": { - "version": "7.22.20", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz", - "integrity": "sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A==", + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.25.9.tgz", + "integrity": "sha512-Ed61U6XJc3CVRfkERJWDz4dJwKe7iLmmJsbOGu9wSloNSFttHV0I8g6UAgb7qnK5ly5bGLPd4oXZlxCdANBOWQ==", "dev": true, "engines": { "node": ">=6.9.0" @@ -400,100 +327,26 @@ } }, "node_modules/@babel/helpers": { - "version": "7.23.9", - "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.23.9.tgz", - "integrity": "sha512-87ICKgU5t5SzOT7sBMfCOZQ2rHjRU+Pcb9BoILMYz600W6DkVRLFBPwQ18gwUVvggqXivaUakpnxWQGbpywbBQ==", - "dev": true, - "dependencies": { - "@babel/template": "^7.23.9", - "@babel/traverse": "^7.23.9", - "@babel/types": "^7.23.9" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/highlight": { - "version": "7.23.4", - "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.23.4.tgz", - "integrity": "sha512-acGdbYSfp2WheJoJm/EBBBLh/ID8KDc64ISZ9DYtBmC8/Q204PZJLHyzeB5qMzJ5trcOkybd78M4x2KWsUq++A==", + "version": "7.26.10", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.26.10.tgz", + "integrity": "sha512-UPYc3SauzZ3JGgj87GgZ89JVdC5dj0AoetR5Bw6wj4niittNyFh6+eOGonYvJ1ao6B8lEa3Q3klS7ADZ53bc5g==", "dev": true, "dependencies": { - "@babel/helper-validator-identifier": "^7.22.20", - "chalk": "^2.4.2", - "js-tokens": "^4.0.0" + "@babel/template": "^7.26.9", + "@babel/types": "^7.26.10" }, "engines": { "node": ">=6.9.0" } }, - "node_modules/@babel/highlight/node_modules/ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "dev": true, - "dependencies": { - "color-convert": "^1.9.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/@babel/highlight/node_modules/chalk": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", - "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", - "dev": true, - "dependencies": { - "ansi-styles": "^3.2.1", - "escape-string-regexp": "^1.0.5", - "supports-color": "^5.3.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/@babel/highlight/node_modules/color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "dev": true, - "dependencies": { - "color-name": "1.1.3" - } - }, - "node_modules/@babel/highlight/node_modules/color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", - "dev": true - }, - "node_modules/@babel/highlight/node_modules/escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", - "dev": true, - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/@babel/highlight/node_modules/supports-color": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", - "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "node_modules/@babel/parser": { + "version": "7.26.10", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.26.10.tgz", + "integrity": "sha512-6aQR2zGE/QFi8JpDLjUZEPYOs7+mhKXm86VaKFiLP35JQwQb6bwUE+XbvkH0EptsYhbNBSUGaUBLKqxH1xSgsA==", "dev": true, "dependencies": { - "has-flag": "^3.0.0" + "@babel/types": "^7.26.10" }, - "engines": { - "node": ">=4" - } - }, - "node_modules/@babel/parser": { - "version": "7.23.9", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.23.9.tgz", - "integrity": "sha512-9tcKgqKbs3xGJ+NtKF2ndOBBLVwPjl1SHxPQkd36r3Dlirw3xWUeGaTbqr7uGZcTaxkVNwc+03SVP7aCdWrTlA==", - "dev": true, "bin": { "parser": "bin/babel-parser.js" }, @@ -679,14 +532,14 @@ } }, "node_modules/@babel/template": { - "version": "7.23.9", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.23.9.tgz", - "integrity": "sha512-+xrD2BWLpvHKNmX2QbpdpsBaWnRxahMwJjO+KZk2JOElj5nSmKezyS1B4u+QbHMTX69t4ukm6hh9lsYQ7GHCKA==", + "version": "7.26.9", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.26.9.tgz", + "integrity": "sha512-qyRplbeIpNZhmzOysF/wFMuP9sctmh2cFzRAZOn1YapxBsE1i9bJIY586R/WBLfLcmcBlM8ROBiQURnnNy+zfA==", "dev": true, "dependencies": { - "@babel/code-frame": "^7.23.5", - "@babel/parser": "^7.23.9", - "@babel/types": "^7.23.9" + "@babel/code-frame": "^7.26.2", + "@babel/parser": "^7.26.9", + "@babel/types": "^7.26.9" }, "engines": { "node": ">=6.9.0" @@ -723,14 +576,13 @@ } }, "node_modules/@babel/types": { - "version": "7.23.9", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.23.9.tgz", - "integrity": "sha512-dQjSq/7HaSjRM43FFGnv5keM2HsxpmyV1PfaSVm0nzzjwwTmjOe6J4bC8e3+pTEIgHaHj+1ZlLThRJ2auc/w1Q==", + "version": "7.26.10", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.26.10.tgz", + "integrity": "sha512-emqcG3vHrpxUKTrxcblR36dcrcoRDvKmnL/dCL6ZsHaShW80qxCAcNhzQZrpeM765VzEos+xOi4s+r4IXzTwdQ==", "dev": true, "dependencies": { - "@babel/helper-string-parser": "^7.23.4", - "@babel/helper-validator-identifier": "^7.22.20", - "to-fast-properties": "^2.0.0" + "@babel/helper-string-parser": "^7.25.9", + "@babel/helper-validator-identifier": "^7.25.9" }, "engines": { "node": ">=6.9.0" @@ -1210,107 +1062,6 @@ "node": ">= 8" } }, - "node_modules/@octokit/auth-token": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-2.5.0.tgz", - "integrity": "sha512-r5FVUJCOLl19AxiuZD2VRZ/ORjp/4IN98Of6YJoJOkY75CIBuYfmiNHGrDwXr+aLGG55igl9QrxX3hbiXlLb+g==", - "dependencies": { - "@octokit/types": "^6.0.3" - } - }, - "node_modules/@octokit/core": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/@octokit/core/-/core-3.6.0.tgz", - "integrity": "sha512-7RKRKuA4xTjMhY+eG3jthb3hlZCsOwg3rztWh75Xc+ShDWOfDDATWbeZpAHBNRpm4Tv9WgBMOy1zEJYXG6NJ7Q==", - "dependencies": { - "@octokit/auth-token": "^2.4.4", - "@octokit/graphql": "^4.5.8", - "@octokit/request": "^5.6.3", - "@octokit/request-error": "^2.0.5", - "@octokit/types": "^6.0.3", - "before-after-hook": "^2.2.0", - "universal-user-agent": "^6.0.0" - } - }, - "node_modules/@octokit/endpoint": { - "version": "6.0.12", - "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-6.0.12.tgz", - "integrity": "sha512-lF3puPwkQWGfkMClXb4k/eUT/nZKQfxinRWJrdZaJO85Dqwo/G0yOC434Jr2ojwafWJMYqFGFa5ms4jJUgujdA==", - "dependencies": { - "@octokit/types": "^6.0.3", - "is-plain-object": "^5.0.0", - "universal-user-agent": "^6.0.0" - } - }, - "node_modules/@octokit/graphql": { - "version": "4.8.0", - "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-4.8.0.tgz", - "integrity": "sha512-0gv+qLSBLKF0z8TKaSKTsS39scVKF9dbMxJpj3U0vC7wjNWFuIpL/z76Qe2fiuCbDRcJSavkXsVtMS6/dtQQsg==", - "dependencies": { - "@octokit/request": "^5.6.0", - "@octokit/types": "^6.0.3", - "universal-user-agent": "^6.0.0" - } - }, - "node_modules/@octokit/openapi-types": { - "version": "12.11.0", - "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-12.11.0.tgz", - "integrity": "sha512-VsXyi8peyRq9PqIz/tpqiL2w3w80OgVMwBHltTml3LmVvXiphgeqmY9mvBw9Wu7e0QWk/fqD37ux8yP5uVekyQ==" - }, - "node_modules/@octokit/plugin-paginate-rest": { - "version": "2.21.3", - "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-2.21.3.tgz", - "integrity": "sha512-aCZTEf0y2h3OLbrgKkrfFdjRL6eSOo8komneVQJnYecAxIej7Bafor2xhuDJOIFau4pk0i/P28/XgtbyPF0ZHw==", - "dependencies": { - "@octokit/types": "^6.40.0" - }, - "peerDependencies": { - "@octokit/core": ">=2" - } - }, - "node_modules/@octokit/plugin-rest-endpoint-methods": { - "version": "5.16.2", - "resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-5.16.2.tgz", - "integrity": "sha512-8QFz29Fg5jDuTPXVtey05BLm7OB+M8fnvE64RNegzX7U+5NUXcOcnpTIK0YfSHBg8gYd0oxIq3IZTe9SfPZiRw==", - "dependencies": { - "@octokit/types": "^6.39.0", - "deprecation": "^2.3.1" - }, - "peerDependencies": { - "@octokit/core": ">=3" - } - }, - "node_modules/@octokit/request": { - "version": "5.6.3", - "resolved": "https://registry.npmjs.org/@octokit/request/-/request-5.6.3.tgz", - "integrity": "sha512-bFJl0I1KVc9jYTe9tdGGpAMPy32dLBXXo1dS/YwSCTL/2nd9XeHsY616RE3HPXDVk+a+dBuzyz5YdlXwcDTr2A==", - "dependencies": { - "@octokit/endpoint": "^6.0.1", - "@octokit/request-error": "^2.1.0", - "@octokit/types": "^6.16.1", - "is-plain-object": "^5.0.0", - "node-fetch": "^2.6.7", - "universal-user-agent": "^6.0.0" - } - }, - "node_modules/@octokit/request-error": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-2.1.0.tgz", - "integrity": "sha512-1VIvgXxs9WHSjicsRwq8PlR2LR2x6DwsJAaFgzdi0JfJoGSO8mYI/cHJQ+9FbN21aa+DrgNLnwObmyeSC8Rmpg==", - "dependencies": { - "@octokit/types": "^6.0.3", - "deprecation": "^2.0.0", - "once": "^1.4.0" - } - }, - "node_modules/@octokit/types": { - "version": "6.41.0", - "resolved": "https://registry.npmjs.org/@octokit/types/-/types-6.41.0.tgz", - "integrity": "sha512-eJ2jbzjdijiL3B4PrSQaSjuF2sPEQPVCPzBvTHJD9Nz+9dw2SGH4K4xeQJ77YfTq5bRQ+bD8wT11JbeDPmxmGg==", - "dependencies": { - "@octokit/openapi-types": "^12.11.0" - } - }, "node_modules/@sinclair/typebox": { "version": "0.27.8", "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz", @@ -2035,11 +1786,6 @@ "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", "dev": true }, - "node_modules/before-after-hook": { - "version": "2.2.3", - "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.2.3.tgz", - "integrity": "sha512-NzUnlZexiaH/46WDhANlyR2bXRopNg4F/zuSA3OpZnllCUgRaOF2znDioDWrmbNVsuZk6l9pMquQB38cfBZwkQ==" - }, "node_modules/boolbase": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", @@ -2057,12 +1803,12 @@ } }, "node_modules/braces": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", - "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", "dev": true, "dependencies": { - "fill-range": "^7.0.1" + "fill-range": "^7.1.1" }, "engines": { "node": ">=8" @@ -2336,9 +2082,9 @@ } }, "node_modules/cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", "dev": true, "dependencies": { "path-key": "^3.1.0", @@ -2509,11 +2255,6 @@ "node": ">= 0.4" } }, - "node_modules/deprecation": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/deprecation/-/deprecation-2.3.1.tgz", - "integrity": "sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ==" - }, "node_modules/detect-newline": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", @@ -3289,9 +3030,9 @@ } }, "node_modules/fill-range": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", - "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", "dev": true, "dependencies": { "to-regex-range": "^5.0.1" @@ -3512,15 +3253,6 @@ "node": ">= 0.4.0" } }, - "node_modules/has-flag": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", - "dev": true, - "engines": { - "node": ">=4" - } - }, "node_modules/has-symbols": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.0.tgz", @@ -3771,14 +3503,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-plain-object": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz", - "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/is-regex": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.4.tgz", @@ -4709,13 +4433,13 @@ } }, "node_modules/micromatch": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.4.tgz", - "integrity": "sha512-pRmzw/XUcwXGpD9aI9q/0XOwLNygjETJ8y0ao0wdqprrzDa4YnxLcz7fQRZr8voh8V10kGhABbNcHVk5wHgWwg==", + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", "dev": true, "dependencies": { - "braces": "^3.0.1", - "picomatch": "^2.2.3" + "braces": "^3.0.3", + "picomatch": "^2.3.1" }, "engines": { "node": ">=8.6" @@ -4754,44 +4478,6 @@ "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=", "dev": true }, - "node_modules/node-fetch": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.9.tgz", - "integrity": "sha512-DJm/CJkZkRjKKj4Zi4BsKVZh3ValV5IR5s7LVZnW+6YMh0W1BfNA8XSs6DLMGYlId5F3KnA70uu2qepcR08Qqg==", - "dependencies": { - "whatwg-url": "^5.0.0" - }, - "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } - } - }, - "node_modules/node-fetch/node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" - }, - "node_modules/node-fetch/node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" - }, - "node_modules/node-fetch/node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, "node_modules/node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", @@ -4877,6 +4563,7 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "dev": true, "dependencies": { "wrappy": "1" } @@ -5032,9 +4719,9 @@ "dev": true }, "node_modules/picomatch": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.0.tgz", - "integrity": "sha512-lY1Q/PiJGC2zOv/z391WOTD+Z02bCgsFfvxoXXf6h7kv9o+WmsmzYqrAwY63sNgOxE4xEdq0WyUnXfKeBrSvYw==", + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", "dev": true, "engines": { "node": ">=8.6" @@ -5680,15 +5367,6 @@ "integrity": "sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw==", "dev": true }, - "node_modules/to-fast-properties": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", - "integrity": "sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==", - "dev": true, - "engines": { - "node": ">=4" - } - }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", @@ -5813,11 +5491,6 @@ "integrity": "sha512-8Y75pvTYkLJW2hWQHXxoqRgV7qb9B+9vFEtidML+7koHUFapnVJAZ6cKs+Qjz5Aw3aZWHMC6u0wJE3At+nSGwA==", "dev": true }, - "node_modules/universal-user-agent": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.0.tgz", - "integrity": "sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w==" - }, "node_modules/update-browserslist-db": { "version": "1.0.13", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.13.tgz", @@ -5966,7 +5639,8 @@ "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", + "dev": true }, "node_modules/write-file-atomic": { "version": "4.0.2", @@ -6054,17 +5728,6 @@ "@actions/io": "^1.0.1" } }, - "@actions/github": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/@actions/github/-/github-5.1.1.tgz", - "integrity": "sha512-Nk59rMDoJaV+mHCOJPXuvB1zIbomlKS0dmSIqPGxd0enAXBnOfn4VWF+CGtRCwXZG9Epa54tZA7VIRlJDS8A6g==", - "requires": { - "@actions/http-client": "^2.0.1", - "@octokit/core": "^3.6.0", - "@octokit/plugin-paginate-rest": "^2.17.0", - "@octokit/plugin-rest-endpoint-methods": "^5.13.0" - } - }, "@actions/http-client": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.1.0.tgz", @@ -6089,65 +5752,14 @@ } }, "@babel/code-frame": { - "version": "7.23.5", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.23.5.tgz", - "integrity": "sha512-CgH3s1a96LipHCmSUmYFPwY7MNx8C3avkq7i4Wl3cfa662ldtUe4VM1TPXX70pfmrlWTb6jLqTYrZyT2ZTJBgA==", + "version": "7.26.2", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.26.2.tgz", + "integrity": "sha512-RJlIHRueQgwWitWgF8OdFYGZX328Ax5BCemNGlqHfplnRT9ESi8JkFlvaVYbS+UubVY6dpv87Fs2u5M29iNFVQ==", "dev": true, "requires": { - "@babel/highlight": "^7.23.4", - "chalk": "^2.4.2" - }, - "dependencies": { - "ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "dev": true, - "requires": { - "color-convert": "^1.9.0" - } - }, - "chalk": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", - "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", - "dev": true, - "requires": { - "ansi-styles": "^3.2.1", - "escape-string-regexp": "^1.0.5", - "supports-color": "^5.3.0" - } - }, - "color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "dev": true, - "requires": { - "color-name": "1.1.3" - } - }, - "color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", - "dev": true - }, - "escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", - "dev": true - }, - "supports-color": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", - "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", - "dev": true, - "requires": { - "has-flag": "^3.0.0" - } - } + "@babel/helper-validator-identifier": "^7.25.9", + "js-tokens": "^4.0.0", + "picocolors": "^1.0.0" } }, "@babel/compat-data": { @@ -6307,15 +5919,15 @@ } }, "@babel/helper-string-parser": { - "version": "7.23.4", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.23.4.tgz", - "integrity": "sha512-803gmbQdqwdf4olxrX4AJyFBV/RTr3rSmOj0rKwesmzlfhYNDEs+/iOcznzpNWlJlIlTJC2QfPFcHB6DlzdVLQ==", + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.25.9.tgz", + "integrity": "sha512-4A/SCr/2KLd5jrtOMFzaKjVtAei3+2r/NChoBNoZ3EyP/+GlhoaEGoWOZUmFmoITP7zOJyHIMm+DYRd8o3PvHA==", "dev": true }, "@babel/helper-validator-identifier": { - "version": "7.22.20", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz", - "integrity": "sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A==", + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.25.9.tgz", + "integrity": "sha512-Ed61U6XJc3CVRfkERJWDz4dJwKe7iLmmJsbOGu9wSloNSFttHV0I8g6UAgb7qnK5ly5bGLPd4oXZlxCdANBOWQ==", "dev": true }, "@babel/helper-validator-option": { @@ -6325,85 +5937,24 @@ "dev": true }, "@babel/helpers": { - "version": "7.23.9", - "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.23.9.tgz", - "integrity": "sha512-87ICKgU5t5SzOT7sBMfCOZQ2rHjRU+Pcb9BoILMYz600W6DkVRLFBPwQ18gwUVvggqXivaUakpnxWQGbpywbBQ==", + "version": "7.26.10", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.26.10.tgz", + "integrity": "sha512-UPYc3SauzZ3JGgj87GgZ89JVdC5dj0AoetR5Bw6wj4niittNyFh6+eOGonYvJ1ao6B8lEa3Q3klS7ADZ53bc5g==", "dev": true, "requires": { - "@babel/template": "^7.23.9", - "@babel/traverse": "^7.23.9", - "@babel/types": "^7.23.9" + "@babel/template": "^7.26.9", + "@babel/types": "^7.26.10" } }, - "@babel/highlight": { - "version": "7.23.4", - "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.23.4.tgz", - "integrity": "sha512-acGdbYSfp2WheJoJm/EBBBLh/ID8KDc64ISZ9DYtBmC8/Q204PZJLHyzeB5qMzJ5trcOkybd78M4x2KWsUq++A==", + "@babel/parser": { + "version": "7.26.10", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.26.10.tgz", + "integrity": "sha512-6aQR2zGE/QFi8JpDLjUZEPYOs7+mhKXm86VaKFiLP35JQwQb6bwUE+XbvkH0EptsYhbNBSUGaUBLKqxH1xSgsA==", "dev": true, "requires": { - "@babel/helper-validator-identifier": "^7.22.20", - "chalk": "^2.4.2", - "js-tokens": "^4.0.0" - }, - "dependencies": { - "ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "dev": true, - "requires": { - "color-convert": "^1.9.0" - } - }, - "chalk": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", - "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", - "dev": true, - "requires": { - "ansi-styles": "^3.2.1", - "escape-string-regexp": "^1.0.5", - "supports-color": "^5.3.0" - } - }, - "color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "dev": true, - "requires": { - "color-name": "1.1.3" - } - }, - "color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", - "dev": true - }, - "escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", - "dev": true - }, - "supports-color": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", - "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", - "dev": true, - "requires": { - "has-flag": "^3.0.0" - } - } + "@babel/types": "^7.26.10" } }, - "@babel/parser": { - "version": "7.23.9", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.23.9.tgz", - "integrity": "sha512-9tcKgqKbs3xGJ+NtKF2ndOBBLVwPjl1SHxPQkd36r3Dlirw3xWUeGaTbqr7uGZcTaxkVNwc+03SVP7aCdWrTlA==", - "dev": true - }, "@babel/plugin-syntax-async-generators": { "version": "7.8.4", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz", @@ -6531,14 +6082,14 @@ } }, "@babel/template": { - "version": "7.23.9", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.23.9.tgz", - "integrity": "sha512-+xrD2BWLpvHKNmX2QbpdpsBaWnRxahMwJjO+KZk2JOElj5nSmKezyS1B4u+QbHMTX69t4ukm6hh9lsYQ7GHCKA==", + "version": "7.26.9", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.26.9.tgz", + "integrity": "sha512-qyRplbeIpNZhmzOysF/wFMuP9sctmh2cFzRAZOn1YapxBsE1i9bJIY586R/WBLfLcmcBlM8ROBiQURnnNy+zfA==", "dev": true, "requires": { - "@babel/code-frame": "^7.23.5", - "@babel/parser": "^7.23.9", - "@babel/types": "^7.23.9" + "@babel/code-frame": "^7.26.2", + "@babel/parser": "^7.26.9", + "@babel/types": "^7.26.9" } }, "@babel/traverse": { @@ -6568,14 +6119,13 @@ } }, "@babel/types": { - "version": "7.23.9", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.23.9.tgz", - "integrity": "sha512-dQjSq/7HaSjRM43FFGnv5keM2HsxpmyV1PfaSVm0nzzjwwTmjOe6J4bC8e3+pTEIgHaHj+1ZlLThRJ2auc/w1Q==", + "version": "7.26.10", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.26.10.tgz", + "integrity": "sha512-emqcG3vHrpxUKTrxcblR36dcrcoRDvKmnL/dCL6ZsHaShW80qxCAcNhzQZrpeM765VzEos+xOi4s+r4IXzTwdQ==", "dev": true, "requires": { - "@babel/helper-string-parser": "^7.23.4", - "@babel/helper-validator-identifier": "^7.22.20", - "to-fast-properties": "^2.0.0" + "@babel/helper-string-parser": "^7.25.9", + "@babel/helper-validator-identifier": "^7.25.9" } }, "@bcoe/v8-coverage": { @@ -6956,101 +6506,6 @@ "fastq": "^1.6.0" } }, - "@octokit/auth-token": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-2.5.0.tgz", - "integrity": "sha512-r5FVUJCOLl19AxiuZD2VRZ/ORjp/4IN98Of6YJoJOkY75CIBuYfmiNHGrDwXr+aLGG55igl9QrxX3hbiXlLb+g==", - "requires": { - "@octokit/types": "^6.0.3" - } - }, - "@octokit/core": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/@octokit/core/-/core-3.6.0.tgz", - "integrity": "sha512-7RKRKuA4xTjMhY+eG3jthb3hlZCsOwg3rztWh75Xc+ShDWOfDDATWbeZpAHBNRpm4Tv9WgBMOy1zEJYXG6NJ7Q==", - "requires": { - "@octokit/auth-token": "^2.4.4", - "@octokit/graphql": "^4.5.8", - "@octokit/request": "^5.6.3", - "@octokit/request-error": "^2.0.5", - "@octokit/types": "^6.0.3", - "before-after-hook": "^2.2.0", - "universal-user-agent": "^6.0.0" - } - }, - "@octokit/endpoint": { - "version": "6.0.12", - "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-6.0.12.tgz", - "integrity": "sha512-lF3puPwkQWGfkMClXb4k/eUT/nZKQfxinRWJrdZaJO85Dqwo/G0yOC434Jr2ojwafWJMYqFGFa5ms4jJUgujdA==", - "requires": { - "@octokit/types": "^6.0.3", - "is-plain-object": "^5.0.0", - "universal-user-agent": "^6.0.0" - } - }, - "@octokit/graphql": { - "version": "4.8.0", - "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-4.8.0.tgz", - "integrity": "sha512-0gv+qLSBLKF0z8TKaSKTsS39scVKF9dbMxJpj3U0vC7wjNWFuIpL/z76Qe2fiuCbDRcJSavkXsVtMS6/dtQQsg==", - "requires": { - "@octokit/request": "^5.6.0", - "@octokit/types": "^6.0.3", - "universal-user-agent": "^6.0.0" - } - }, - "@octokit/openapi-types": { - "version": "12.11.0", - "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-12.11.0.tgz", - "integrity": "sha512-VsXyi8peyRq9PqIz/tpqiL2w3w80OgVMwBHltTml3LmVvXiphgeqmY9mvBw9Wu7e0QWk/fqD37ux8yP5uVekyQ==" - }, - "@octokit/plugin-paginate-rest": { - "version": "2.21.3", - "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-2.21.3.tgz", - "integrity": "sha512-aCZTEf0y2h3OLbrgKkrfFdjRL6eSOo8komneVQJnYecAxIej7Bafor2xhuDJOIFau4pk0i/P28/XgtbyPF0ZHw==", - "requires": { - "@octokit/types": "^6.40.0" - } - }, - "@octokit/plugin-rest-endpoint-methods": { - "version": "5.16.2", - "resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-5.16.2.tgz", - "integrity": "sha512-8QFz29Fg5jDuTPXVtey05BLm7OB+M8fnvE64RNegzX7U+5NUXcOcnpTIK0YfSHBg8gYd0oxIq3IZTe9SfPZiRw==", - "requires": { - "@octokit/types": "^6.39.0", - "deprecation": "^2.3.1" - } - }, - "@octokit/request": { - "version": "5.6.3", - "resolved": "https://registry.npmjs.org/@octokit/request/-/request-5.6.3.tgz", - "integrity": "sha512-bFJl0I1KVc9jYTe9tdGGpAMPy32dLBXXo1dS/YwSCTL/2nd9XeHsY616RE3HPXDVk+a+dBuzyz5YdlXwcDTr2A==", - "requires": { - "@octokit/endpoint": "^6.0.1", - "@octokit/request-error": "^2.1.0", - "@octokit/types": "^6.16.1", - "is-plain-object": "^5.0.0", - "node-fetch": "^2.6.7", - "universal-user-agent": "^6.0.0" - } - }, - "@octokit/request-error": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-2.1.0.tgz", - "integrity": "sha512-1VIvgXxs9WHSjicsRwq8PlR2LR2x6DwsJAaFgzdi0JfJoGSO8mYI/cHJQ+9FbN21aa+DrgNLnwObmyeSC8Rmpg==", - "requires": { - "@octokit/types": "^6.0.3", - "deprecation": "^2.0.0", - "once": "^1.4.0" - } - }, - "@octokit/types": { - "version": "6.41.0", - "resolved": "https://registry.npmjs.org/@octokit/types/-/types-6.41.0.tgz", - "integrity": "sha512-eJ2jbzjdijiL3B4PrSQaSjuF2sPEQPVCPzBvTHJD9Nz+9dw2SGH4K4xeQJ77YfTq5bRQ+bD8wT11JbeDPmxmGg==", - "requires": { - "@octokit/openapi-types": "^12.11.0" - } - }, "@sinclair/typebox": { "version": "0.27.8", "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz", @@ -7603,11 +7058,6 @@ "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", "dev": true }, - "before-after-hook": { - "version": "2.2.3", - "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.2.3.tgz", - "integrity": "sha512-NzUnlZexiaH/46WDhANlyR2bXRopNg4F/zuSA3OpZnllCUgRaOF2znDioDWrmbNVsuZk6l9pMquQB38cfBZwkQ==" - }, "boolbase": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", @@ -7625,12 +7075,12 @@ } }, "braces": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", - "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", "dev": true, "requires": { - "fill-range": "^7.0.1" + "fill-range": "^7.1.1" } }, "browserslist": { @@ -7817,9 +7267,9 @@ } }, "cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", "dev": true, "requires": { "path-key": "^3.1.0", @@ -7941,11 +7391,6 @@ "object-keys": "^1.0.12" } }, - "deprecation": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/deprecation/-/deprecation-2.3.1.tgz", - "integrity": "sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ==" - }, "detect-newline": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", @@ -8512,9 +7957,9 @@ } }, "fill-range": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", - "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", "dev": true, "requires": { "to-regex-range": "^5.0.1" @@ -8672,12 +8117,6 @@ "function-bind": "^1.1.1" } }, - "has-flag": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", - "dev": true - }, "has-symbols": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.0.tgz", @@ -8855,11 +8294,6 @@ "integrity": "sha512-zohwelOAur+5uXtk8O3GPQ1eAcu4ZX3UwxQhUlfFFMNpUd83gXgjbhJh6HmB6LUNV/ieOLQuDwJO3dWJosUeMw==", "dev": true }, - "is-plain-object": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz", - "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==" - }, "is-regex": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.4.tgz", @@ -9576,13 +9010,13 @@ "dev": true }, "micromatch": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.4.tgz", - "integrity": "sha512-pRmzw/XUcwXGpD9aI9q/0XOwLNygjETJ8y0ao0wdqprrzDa4YnxLcz7fQRZr8voh8V10kGhABbNcHVk5wHgWwg==", + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", "dev": true, "requires": { - "braces": "^3.0.1", - "picomatch": "^2.2.3" + "braces": "^3.0.3", + "picomatch": "^2.3.1" } }, "mimic-fn": { @@ -9612,35 +9046,6 @@ "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=", "dev": true }, - "node-fetch": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.9.tgz", - "integrity": "sha512-DJm/CJkZkRjKKj4Zi4BsKVZh3ValV5IR5s7LVZnW+6YMh0W1BfNA8XSs6DLMGYlId5F3KnA70uu2qepcR08Qqg==", - "requires": { - "whatwg-url": "^5.0.0" - }, - "dependencies": { - "tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" - }, - "webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" - }, - "whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "requires": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - } - } - }, "node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", @@ -9705,6 +9110,7 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "dev": true, "requires": { "wrappy": "1" } @@ -9820,9 +9226,9 @@ "dev": true }, "picomatch": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.0.tgz", - "integrity": "sha512-lY1Q/PiJGC2zOv/z391WOTD+Z02bCgsFfvxoXXf6h7kv9o+WmsmzYqrAwY63sNgOxE4xEdq0WyUnXfKeBrSvYw==", + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", "dev": true }, "pirates": { @@ -10262,12 +9668,6 @@ "integrity": "sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw==", "dev": true }, - "to-fast-properties": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", - "integrity": "sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==", - "dev": true - }, "to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", @@ -10337,11 +9737,6 @@ "integrity": "sha512-8Y75pvTYkLJW2hWQHXxoqRgV7qb9B+9vFEtidML+7koHUFapnVJAZ6cKs+Qjz5Aw3aZWHMC6u0wJE3At+nSGwA==", "dev": true }, - "universal-user-agent": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.0.tgz", - "integrity": "sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w==" - }, "update-browserslist-db": { "version": "1.0.13", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.13.tgz", @@ -10446,7 +9841,8 @@ "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", + "dev": true }, "write-file-atomic": { "version": "4.0.2", diff --git a/package.json b/package.json index faa2b4a..e7c5c9d 100644 --- a/package.json +++ b/package.json @@ -1,9 +1,9 @@ { - "name": "github-action-benchmark", + "name": "benchmark-data-extract-transform", "version": "0.0.0", "private": true, "description": "", - "main": "dist/src/index.js", + "main": "dist/index.js", "scripts": { "build": "tsc -p tsconfig.build.json", "lint": "eslint '**/*.{ts,js}'", @@ -17,23 +17,22 @@ }, "repository": { "type": "git", - "url": "git+https://github.com/benchmark-action/github-action-benchmark.git" + "url": "git+https://github.com/cryspen/benchmark-data-extract-transform.git" }, "keywords": [ "github", "action", "benchmark" ], - "author": "github-action-benchmark developers ", + "author": "Cryspen ", "license": "MIT", "bugs": { - "url": "https://github.com/benchmark-action/github-action-benchmark/issues" + "url": "https://github.com/cryspen/benchmark-data-extract-transform/issues" }, - "homepage": "https://github.com/benchmark-action/github-action-benchmark#readme", + "homepage": "https://github.com/cryspen/benchmark-data-extract-transform#readme", "dependencies": { "@actions/core": "^1.10.0", "@actions/exec": "^1.1.1", - "@actions/github": "^5.1.1", "@actions/io": "^1.1.2" }, "devDependencies": { diff --git a/scripts/ci_validate_modification.ts b/scripts/ci_validate_modification.ts deleted file mode 100644 index f9b8acc..0000000 --- a/scripts/ci_validate_modification.ts +++ /dev/null @@ -1,291 +0,0 @@ -import * as path from 'path'; -import { promises as fs } from 'fs'; -import * as cp from 'child_process'; -import { BenchmarkSuites, DataJson, SCRIPT_PREFIX } from '../src/write'; -import { VALID_TOOLS } from '../src/config'; -import { Benchmark } from '../src/extract'; -import { diff, Diff, DiffArray, DiffEdit, DiffNew } from 'deep-diff'; -import { getServerUrl } from '../src/git'; -import assert from 'assert'; -import deepEq = require('deep-equal'); - -function help(): never { - throw new Error( - 'Usage: node ci_validate_modification.js before_data.js "benchmark name" [benchmark-data-repository-directory]', - ); -} - -async function exec(cmd: string): Promise { - console.log(`+ ${cmd}`); - return new Promise((resolve, reject) => { - cp.exec(cmd, (err, stdout, stderr) => { - if (err) { - reject(new Error(`Exec '${cmd}' failed with error ${err.message}. Stderr: '${stderr}'`)); - return; - } - resolve(stdout); - }); - }); -} - -async function readDataJson(file: string): Promise { - const content = await fs.readFile(file, 'utf8'); - return JSON.parse(content.slice(SCRIPT_PREFIX.length)); -} - -function validateDataJson(data: DataJson) { - const { lastUpdate, repoUrl, entries: suites } = data; - const now = Date.now(); - if (lastUpdate > now) { - throw new Error(`Last update is not correct: ${lastUpdate} v.s. ${now}`); - } - - const serverUrl = getServerUrl(repoUrl); - const repoUrlMatcher = new RegExp(`^${serverUrl}/[^/]+/github-action-benchmark$`); - const commitUrlMatcher = new RegExp(`^${serverUrl}/[^/]+/github-action-benchmark/commit/`); - if (!repoUrlMatcher.test(repoUrl)) { - throw new Error(`repoUrl is not correct: ${repoUrl}`); - } - - for (const benchName of Object.keys(suites)) { - for (const suite of suites[benchName]) { - const { commit, tool, date, benches } = suite; - if (!(VALID_TOOLS as ReadonlyArray).includes(tool)) { - throw new Error(`Invalid tool ${tool}`); - } - if (!commitUrlMatcher.test(commit.url) && !/\/pull\/\d+\/commits\/[a-f0-9]+$/.test(commit.url)) { - throw new Error(`Invalid commit url: ${commit.url}`); - } - if (!commit.url.endsWith(commit.id)) { - throw new Error(`Commit ID ${commit.id} does not match to URL ${commit.url}`); - } - if (date > now) { - throw new Error(`Benchmark date is not correct: ${date} v.s. ${now}`); - } - for (const bench of benches) { - const { name, value, unit, range, extra } = bench; - const json = JSON.stringify(bench); - if (!name) { - throw new Error(`Benchmark result name is invalid: ${name} (${json})`); - } - if (typeof value !== 'number' || isNaN(value)) { - throw new Error(`Benchmark result value is invalid: ${value} (${json})`); - } - if (typeof unit !== 'string') { - throw new Error(`Benchmark result unit is invalid: ${unit} (${json})`); - } - if (range && typeof range !== 'string') { - throw new Error(`Benchmark result range is invalid: ${range} (${json})`); - } - if (extra && typeof extra !== 'string') { - throw new Error(`Benchmark result extra is invalid: ${extra} (${json})`); - } - } - } - } -} - -function assertNumberDiffEdit(diff: Diff): asserts diff is DiffEdit { - if (diff.kind !== 'E') { - throw new Error(`Given diff is not DiffEdit: ${JSON.stringify(diff)}`); - } - if (typeof diff.lhs !== 'number') { - throw new Error(`Given DiffEdit's lhs is not for number: ${diff.lhs}`); - } - if (typeof diff.rhs !== 'number') { - throw new Error(`Given DiffEdit's rhs is not for number: ${diff.rhs}`); - } -} - -function validateLastUpdateMod(diff: Diff) { - assertNumberDiffEdit(diff); - if (!deepEq(diff.path, ['lastUpdate'])) { - throw new Error(`Not diff for lastUpdate: ${JSON.stringify(diff.path)}`); - } - const { lhs, rhs } = diff; - if (lhs >= rhs) { - throw new Error(`Update of datetime is not correct. New is older: ${lhs} v.s. ${rhs}`); - } -} - -function assertDiffArray(diff: Diff): asserts diff is DiffArray { - if (diff.kind !== 'A') { - throw new Error(`Given diff is not DiffArray: ${JSON.stringify(diff)}`); - } -} - -function assertDiffNewBench(diff: Diff): asserts diff is DiffNew { - if (diff.kind !== 'N') { - throw new Error(`Given diff is not DiffNew: ${JSON.stringify(diff)}`); - } - const { rhs } = diff; - if (typeof rhs !== 'object' || rhs === null) { - throw new Error(`DiffNew for Benchmark object is actually not a object: ${rhs}`); - } - for (const prop of ['commit', 'date', 'tool', 'benches']) { - if (!(prop in rhs)) { - throw new Error(`Not a valid benchmark object in DiffNew: ${JSON.stringify(rhs)}`); - } - } -} - -function validateBenchmarkResultMod(diff: Diff, expectedBenchName: string, afterSuites: BenchmarkSuites) { - if (!(expectedBenchName in afterSuites)) { - throw new Error(`data.js after action does not contain '${expectedBenchName}' benchmark`); - } - - const benchSuites = afterSuites[expectedBenchName]; - if (benchSuites.length === 0) { - throw new Error('Benchmark suite is empty after action'); - } - - if (diff.kind === 'N') { - // Previous data does not exist. This case occurs only once when new tool support is added. - // Ignore checks. - return; - } - - assertDiffArray(diff); - - if (!deepEq(diff.path, ['entries', expectedBenchName])) { - throw new Error(`Diff path is not expected for adding new benchmark: ${JSON.stringify(diff.path)}`); - } - - diff = diff.item; - assertDiffNewBench(diff); - - const added: Benchmark = diff.rhs; - const last = benchSuites[benchSuites.length - 1]; - if (last.commit.id !== added.commit.id) { - throw new Error( - `Newly added benchmark ${JSON.stringify(added)} is not the last one in data.js ${JSON.stringify(last)}`, - ); - } - - for (const suite of benchSuites) { - if (suite.date > added.date) { - throw new Error(`Older suite's date ${JSON.stringify(suite)} is newer than added ${JSON.stringify(added)}`); - } - - if (suite.tool !== added.tool) { - throw new Error(`Tool is different between ${JSON.stringify(suite)} and ${JSON.stringify(added)}`); - } - - for (const addedBench of added.benches) { - for (const prevBench of suite.benches) { - if (prevBench.name === addedBench.name) { - if (prevBench.unit !== addedBench.unit) { - throw new Error( - `Unit is different between previous benchmark and newly added benchmark: ${JSON.stringify( - prevBench, - )} v.v. ${JSON.stringify(addedBench)}`, - ); - } - } - } - } - } -} - -function validateDiff(beforeJson: DataJson, afterJson: DataJson, expectedBenchName: string) { - const diffs = diff(beforeJson, afterJson); - console.log('Validating diffs:', diffs); - - if (!diffs || diffs.length !== 2) { - throw new Error('Number of diffs are incorrect. Exact 2 diffs are expected'); - } - - console.log('Validating lastUpdate modification'); - validateLastUpdateMod(diffs[0]); - - console.log('Validating benchmark result modification'); - validateBenchmarkResultMod(diffs[1], expectedBenchName, afterJson.entries); - - console.log('👌'); -} - -async function main() { - console.log('Start validating modifications by action with args', process.argv); - - if (process.argv.length !== 4 && process.argv.length !== 5) { - help(); - } - - if (['-h', '--help'].includes(process.argv[2])) { - help(); - } - - console.log('Checking pre-condition'); - const stats = await fs.stat(path.resolve('.git')); - if (!stats.isDirectory()) { - throw new Error('This script must be run at root directory of repository'); - } - - const beforeDataJs = path.resolve(process.argv[2]); - const expectedBenchName = process.argv[3]; - const benchmarkDataDirectory = process.argv[4]; - - const additionalGitParams = benchmarkDataDirectory - ? `--work-tree=${benchmarkDataDirectory} --git-dir=${benchmarkDataDirectory}/.git` - : ''; - - console.log('Validating modifications by action'); - console.log(` data.js before action: ${beforeDataJs}`); - - console.log('Reading data.js before action as JSON'); - const beforeJson = await readDataJson(beforeDataJs); - - console.log('Validating current branch'); - const branch = await exec(`git ${additionalGitParams} rev-parse --abbrev-ref HEAD`); - if (branch === 'gh-pages') { - throw new Error(`Current branch is still on '${branch}'`); - } - - console.log('Retrieving data.js after action'); - await exec(`git ${additionalGitParams} checkout gh-pages`); - const latestCommitLog = await exec(`git ${additionalGitParams} log -n 1`); - - console.log('Validating auto commit'); - const commitLogLines = latestCommitLog.split('\n'); - - const commitAuthorLine = commitLogLines[1]; - if (!commitAuthorLine.startsWith('Author: github-action-benchmark')) { - throw new Error(`Unexpected auto commit author in log '${latestCommitLog}'`); - } - - const commitMessageLine = commitLogLines[4]; - const reCommitMessage = new RegExp( - `add ${expectedBenchName.replace( - /[.*+?^=!:${}()|[\]/\\]/g, - '\\$&', - )} \\([^)]+\\) benchmark result for [0-9a-f]+$`, - ); - if (!reCommitMessage.test(commitMessageLine)) { - throw new Error(`Unexpected auto commit message in log '${latestCommitLog}'`); - } - - const dataResults = await Promise.allSettled([ - readDataJson('benchmark-data-repository/dev/bench/data.js'), - readDataJson('dev/bench/data.js'), - ]); - - const jsonResults = dataResults - .filter((res): res is PromiseFulfilledResult => res.status === 'fulfilled') - .map((res) => res.value); - - assert(jsonResults.length > 0 && jsonResults.length <= 2, 'Maximum 2 data.js files should be present in the repo'); - - const afterJson = jsonResults[0]; - await exec(`git ${additionalGitParams} checkout -`); - - console.log('Validating data.js both before/after action'); - validateDataJson(beforeJson); - validateDataJson(afterJson); - - validateDiff(beforeJson, afterJson, expectedBenchName); -} - -main().catch((err) => { - console.error(err); - process.exit(110); -}); diff --git a/scripts/prepare-release.sh b/scripts/prepare-release.sh index 2cab042..817b184 100755 --- a/scripts/prepare-release.sh +++ b/scripts/prepare-release.sh @@ -25,8 +25,8 @@ if ! git diff --cached --quiet; then fi branch="$(git symbolic-ref --short HEAD)" -if [[ "$branch" != "master" ]]; then - echo 'Current branch is not master. Please move to master before running this script' >&2 +if [[ "$branch" != "main" ]]; then + echo 'Current branch is not main. Please move to main before running this script' >&2 exit 1 fi diff --git a/src/comment/benchmarkCommentTags.ts b/src/comment/benchmarkCommentTags.ts deleted file mode 100644 index 6e88bff..0000000 --- a/src/comment/benchmarkCommentTags.ts +++ /dev/null @@ -1,13 +0,0 @@ -const BENCHMARK_COMMENT_TAG = 'github-benchmark-action-comment'; - -export function benchmarkStartTag(commentId: string) { - return ``; -} - -export function benchmarkEndTag(commentId: string) { - return ``; -} - -export function wrapBodyWithBenchmarkTags(commentId: string, body: string) { - return `${benchmarkStartTag(commentId)}\n${body}\n${benchmarkEndTag(commentId)}`; -} diff --git a/src/comment/findExistingPRReviewId.ts b/src/comment/findExistingPRReviewId.ts deleted file mode 100644 index b41c594..0000000 --- a/src/comment/findExistingPRReviewId.ts +++ /dev/null @@ -1,28 +0,0 @@ -import * as github from '@actions/github'; -import { benchmarkStartTag } from './benchmarkCommentTags'; -import * as core from '@actions/core'; - -export async function findExistingPRReviewId( - repoOwner: string, - repoName: string, - pullRequestNumber: number, - benchName: string, - token: string, -) { - core.debug('findExistingPRReviewId start'); - const client = github.getOctokit(token); - - const existingReviewsResponse = await client.rest.pulls.listReviews({ - owner: repoOwner, - repo: repoName, - // eslint-disable-next-line @typescript-eslint/naming-convention - pull_number: pullRequestNumber, - }); - - const existingReview = existingReviewsResponse.data.find((review) => - review.body.startsWith(benchmarkStartTag(benchName)), - ); - - core.debug('findExistingPRReviewId start'); - return existingReview?.id; -} diff --git a/src/comment/leaveCommitComment.ts b/src/comment/leaveCommitComment.ts deleted file mode 100644 index c25b671..0000000 --- a/src/comment/leaveCommitComment.ts +++ /dev/null @@ -1,25 +0,0 @@ -import * as github from '@actions/github'; -import * as core from '@actions/core'; -import { wrapBodyWithBenchmarkTags } from './benchmarkCommentTags'; - -export async function leaveCommitComment( - repoOwner: string, - repoName: string, - commitId: string, - body: string, - commentId: string, - token: string, -) { - core.debug('leaveCommitComment start'); - const client = github.getOctokit(token); - const response = await client.rest.repos.createCommitComment({ - owner: repoOwner, - repo: repoName, - // eslint-disable-next-line @typescript-eslint/naming-convention - commit_sha: commitId, - body: wrapBodyWithBenchmarkTags(commentId, body), - }); - console.log(`Comment was sent to ${response.url}. Response:`, response.status, response.data); - core.debug('leaveCommitComment end'); - return response; -} diff --git a/src/comment/leavePRComment.ts b/src/comment/leavePRComment.ts deleted file mode 100644 index d9bb94f..0000000 --- a/src/comment/leavePRComment.ts +++ /dev/null @@ -1,72 +0,0 @@ -import * as github from '@actions/github'; -import { wrapBodyWithBenchmarkTags } from './benchmarkCommentTags'; -import { findExistingPRReviewId } from './findExistingPRReviewId'; -import * as core from '@actions/core'; - -export async function leavePRComment( - repoOwner: string, - repoName: string, - pullRequestNumber: number, - body: string, - commentId: string, - token: string, -) { - try { - core.debug('leavePRComment start'); - const client = github.getOctokit(token); - - const bodyWithTags = wrapBodyWithBenchmarkTags(commentId, body); - - const existingCommentId = await findExistingPRReviewId( - repoOwner, - repoName, - pullRequestNumber, - commentId, - token, - ); - - if (!existingCommentId) { - core.debug('creating new pr comment'); - const createReviewResponse = await client.rest.pulls.createReview({ - owner: repoOwner, - repo: repoName, - // eslint-disable-next-line @typescript-eslint/naming-convention - pull_number: pullRequestNumber, - event: 'COMMENT', - body: bodyWithTags, - }); - - console.log( - `Comment was created via ${createReviewResponse.url}. Response:`, - createReviewResponse.status, - createReviewResponse.data, - ); - - core.debug('leavePRComment end'); - return createReviewResponse; - } - - core.debug('updating existing pr comment'); - const updateReviewResponse = await client.rest.pulls.updateReview({ - owner: repoOwner, - repo: repoName, - // eslint-disable-next-line @typescript-eslint/naming-convention - pull_number: pullRequestNumber, - // eslint-disable-next-line @typescript-eslint/naming-convention - review_id: existingCommentId, - body: bodyWithTags, - }); - - console.log( - `Comment was updated via ${updateReviewResponse.url}. Response:`, - updateReviewResponse.status, - updateReviewResponse.data, - ); - core.debug('leavePRComment end'); - - return updateReviewResponse; - } catch (err) { - console.log('error', err); - throw err; - } -} diff --git a/src/config.ts b/src/config.ts index 0ce3bd5..54f0ee7 100644 --- a/src/config.ts +++ b/src/config.ts @@ -6,42 +6,13 @@ import * as path from 'path'; export type ToolType = typeof VALID_TOOLS[number]; export interface Config { name: string; + platform: string; tool: ToolType; outputFilePath: string; - ghPagesBranch: string; - ghRepository: string | undefined; - benchmarkDataDirPath: string; - githubToken: string | undefined; - autoPush: boolean; - skipFetchGhPages: boolean; - commentAlways: boolean; - summaryAlways: boolean; - saveDataFile: boolean; - commentOnAlert: boolean; - alertThreshold: number; - failOnAlert: boolean; - failThreshold: number; - alertCommentCcUsers: string[]; - externalDataJsonPath: string | undefined; - maxItemsInChart: number | null; - ref: string | undefined; + dataOutPath: string; } -export const VALID_TOOLS = [ - 'cargo', - 'go', - 'benchmarkjs', - 'benchmarkluau', - 'pytest', - 'googlecpp', - 'catch2', - 'julia', - 'jmh', - 'benchmarkdotnet', - 'customBiggerIsBetter', - 'customSmallerIsBetter', -] as const; -const RE_UINT = /^\d+$/; +export const VALID_TOOLS = ['cargo'] as const; function validateToolType(tool: string): asserts tool is ToolType { if ((VALID_TOOLS as ReadonlyArray).includes(tool)) { @@ -85,19 +56,12 @@ async function validateOutputFilePath(filePath: string): Promise { throw new Error(`Invalid value for 'output-file-path' input: ${err}`); } } - -function validateGhPagesBranch(branch: string) { - if (branch) { - return; - } - throw new Error(`Branch value must not be empty for 'gh-pages-branch' input`); -} - -function validateBenchmarkDataDirPath(dirPath: string): string { +async function validateDataOutPath(filePath: string): Promise { try { - return resolvePath(dirPath); - } catch (e) { - throw new Error(`Invalid value for 'benchmark-data-dir-path': ${e}`); + // TODO: validate + return filePath; + } catch (err) { + throw new Error(`Invalid value for 'data-out-path' input: ${err}`); } } @@ -107,185 +71,31 @@ function validateName(name: string) { } throw new Error('Name must not be empty'); } - -function validateGitHubToken(inputName: string, githubToken: string | undefined, todo: string) { - if (!githubToken) { - throw new Error(`'${inputName}' is enabled but 'github-token' is not set. Please give API token ${todo}`); - } -} - -function getBoolInput(name: string): boolean { - const input = core.getInput(name); - if (!input) { - return false; - } - if (input !== 'true' && input !== 'false') { - throw new Error(`'${name}' input must be boolean value 'true' or 'false' but got '${input}'`); - } - return input === 'true'; -} - -function getPercentageInput(name: string): number | null { - const input = core.getInput(name); - if (!input) { - return null; - } - if (!input.endsWith('%')) { - throw new Error(`'${name}' input must ends with '%' for percentage value (e.g. '200%')`); - } - - const percentage = parseFloat(input.slice(0, -1)); // Omit '%' at last - if (isNaN(percentage)) { - throw new Error(`Specified value '${input.slice(0, -1)}' in '${name}' input cannot be parsed as float number`); - } - - return percentage / 100; -} - -function getCommaSeparatedInput(name: string): string[] { - const input = core.getInput(name); - if (!input) { - return []; - } - return input.split(',').map((s) => s.trim()); -} - -function validateAlertCommentCcUsers(users: string[]) { - for (const u of users) { - if (!u.startsWith('@')) { - throw new Error(`User name in 'alert-comment-cc-users' input must start with '@' but got '${u}'`); - } - } -} - -async function isDir(path: string) { - try { - const s = await fs.stat(path); - return s.isDirectory(); - } catch (_) { - return false; - } -} - -async function validateExternalDataJsonPath(path: string | undefined, autoPush: boolean): Promise { - if (!path) { - return Promise.resolve(undefined); - } - if (autoPush) { - throw new Error( - 'auto-push must be false when external-data-json-path is set since this action reads/writes the given JSON file and never pushes to remote', - ); - } - try { - const p = resolvePath(path); - if (await isDir(p)) { - throw new Error(`Specified path '${p}' must be file but it is actually directory`); - } - return p; - } catch (err) { - throw new Error(`Invalid value for 'external-data-json-path' input: ${err}`); - } -} - -function getUintInput(name: string): number | null { - const input = core.getInput(name); - if (!input) { - return null; - } - if (!RE_UINT.test(input)) { - throw new Error(`'${name}' input must be unsigned integer but got '${input}'`); - } - const i = parseInt(input, 10); - if (isNaN(i)) { - throw new Error(`Unsigned integer value '${input}' in '${name}' input was parsed as NaN`); - } - return i; -} - -function validateMaxItemsInChart(max: number | null) { - if (max !== null && max <= 0) { - throw new Error(`'max-items-in-chart' input value must be one or more but got ${max}`); - } -} - -function validateAlertThreshold(alertThreshold: number | null, failThreshold: number | null): asserts alertThreshold { - if (alertThreshold === null) { - throw new Error("'alert-threshold' input must not be empty"); - } - if (failThreshold && alertThreshold > failThreshold) { - throw new Error( - `'alert-threshold' value must be smaller than 'fail-threshold' value but got ${alertThreshold} > ${failThreshold}`, - ); +function validatePlatform(platform: string) { + if (platform) { + return; } + throw new Error('Platform must not be empty'); } export async function configFromJobInput(): Promise { const tool: string = core.getInput('tool'); let outputFilePath: string = core.getInput('output-file-path'); - const ghPagesBranch: string = core.getInput('gh-pages-branch'); - const ghRepository: string = core.getInput('gh-repository'); - let benchmarkDataDirPath: string = core.getInput('benchmark-data-dir-path'); + let dataOutPath: string = core.getInput('data-out-path'); const name: string = core.getInput('name'); - const githubToken: string | undefined = core.getInput('github-token') || undefined; - const ref: string | undefined = core.getInput('ref') || undefined; - const autoPush = getBoolInput('auto-push'); - const skipFetchGhPages = getBoolInput('skip-fetch-gh-pages'); - const commentAlways = getBoolInput('comment-always'); - const summaryAlways = getBoolInput('summary-always'); - const saveDataFile = getBoolInput('save-data-file'); - const commentOnAlert = getBoolInput('comment-on-alert'); - const alertThreshold = getPercentageInput('alert-threshold'); - const failOnAlert = getBoolInput('fail-on-alert'); - const alertCommentCcUsers = getCommaSeparatedInput('alert-comment-cc-users'); - let externalDataJsonPath: undefined | string = core.getInput('external-data-json-path'); - const maxItemsInChart = getUintInput('max-items-in-chart'); - let failThreshold = getPercentageInput('fail-threshold'); + const platform: string = core.getInput('platform'); + validateName(name); + validatePlatform(platform); validateToolType(tool); outputFilePath = await validateOutputFilePath(outputFilePath); - validateGhPagesBranch(ghPagesBranch); - benchmarkDataDirPath = validateBenchmarkDataDirPath(benchmarkDataDirPath); - validateName(name); - if (autoPush) { - validateGitHubToken('auto-push', githubToken, 'to push GitHub pages branch to remote'); - } - if (commentAlways) { - validateGitHubToken('comment-always', githubToken, 'to send commit comment'); - } - if (commentOnAlert) { - validateGitHubToken('comment-on-alert', githubToken, 'to send commit comment on alert'); - } - if (ghRepository) { - validateGitHubToken('gh-repository', githubToken, 'to clone the repository'); - } - validateAlertThreshold(alertThreshold, failThreshold); - validateAlertCommentCcUsers(alertCommentCcUsers); - externalDataJsonPath = await validateExternalDataJsonPath(externalDataJsonPath, autoPush); - validateMaxItemsInChart(maxItemsInChart); - if (failThreshold === null) { - failThreshold = alertThreshold; - } + dataOutPath = await validateDataOutPath(dataOutPath); return { name, tool, outputFilePath, - ghPagesBranch, - ghRepository, - benchmarkDataDirPath, - githubToken, - autoPush, - skipFetchGhPages, - commentAlways, - summaryAlways, - saveDataFile, - commentOnAlert, - alertThreshold, - failOnAlert, - alertCommentCcUsers, - externalDataJsonPath, - maxItemsInChart, - failThreshold, - ref, + dataOutPath, + platform, }; } diff --git a/src/default_index_html.ts b/src/default_index_html.ts deleted file mode 100644 index a6246d5..0000000 --- a/src/default_index_html.ts +++ /dev/null @@ -1,282 +0,0 @@ -export const DEFAULT_INDEX_HTML = String.raw` - - - - - - Benchmarks - - - - -
- - - - - - - -`; diff --git a/src/extract.ts b/src/extract.ts index ec7bc35..fcda046 100644 --- a/src/extract.ts +++ b/src/extract.ts @@ -1,7 +1,6 @@ /* eslint-disable @typescript-eslint/naming-convention */ import { promises as fs } from 'fs'; -import * as github from '@actions/github'; -import { Config, ToolType } from './config'; +import { Config } from './config'; export interface BenchmarkResult { name: string; @@ -9,308 +8,10 @@ export interface BenchmarkResult { range?: string; unit: string; extra?: string; + platform: string; } -interface GitHubUser { - email?: string; - name?: string; - username?: string; -} - -interface Commit { - author: GitHubUser; - committer: GitHubUser; - distinct?: unknown; // Unused - id: string; - message: string; - timestamp?: string; - tree_id?: unknown; // Unused - url: string; -} - -interface PullRequest { - [key: string]: any; - number: number; - html_url?: string; - body?: string; -} - -export interface Benchmark { - commit: Commit; - date: number; - tool: ToolType; - benches: BenchmarkResult[]; -} - -export interface GoogleCppBenchmarkJson { - context: { - date: string; - host_name: string; - executable: string; - num_cpus: number; - mhz_per_cpu: number; - cpu_scaling_enabled: boolean; - caches: unknown; - load_avg: number[]; - library_build_type: 'release' | 'debug'; - }; - benchmarks: Array<{ - name: string; - run_name: string; - run_type: string; - repetitions: number; - repetition_index: number; - threads: number; - iterations: number; - real_time: number; - cpu_time: number; - time_unit: string; - }>; -} - -export interface PytestBenchmarkJson { - machine_info: { - node: string; - processor: string; - machine: string; - python_compiler: string; - python_implementation: string; - python_implementation_version: string; - python_version: string; - python_build: string[]; - release: string; - system: string; - cpu: { - vendor_id: string; - hardware: string; - brand: string; - }; - }; - commit_info: { - id: string; - time: string; - author_time: string; - dirty: boolean; - project: string; - branch: string; - }; - benchmarks: Array<{ - group: null | string; - name: string; - fullname: string; - params: null | string[]; - param: null | string; - extra_info: object; - options: { - disable_gc: boolean; - time: string; - min_rounds: number; - max_time: number; - min_time: number; - warmup: boolean; - }; - stats: { - min: number; - max: number; - mean: number; - stddev: number; - rounds: number; - median: number; - irq: number; - q1: number; - q3: number; - irq_outliers: number; - stddev_outliers: number; - outliers: string; - ld15iqr: number; - hd15iqr: number; - ops: number; - total: number; - data: number[]; - iterations: number; - }; - }>; - datetime: string; - version: string; -} - -type JuliaBenchmark = JuliaBenchmarkGroup | JuliaBenchmarkTrialEstimate | JuliaBenchmarkOther; - -type JuliaBenchmarkOther = [string, unknown]; - -type JuliaBenchmarkTrialEstimate = [ - 'TrialEstimate', - { - params: [ - string, - { - seconds: number; - samples: number; - evals: number; - overhead: number; - gctrial: boolean; - gcsample: boolean; - time_tolerance: number; - memory_tolerance: number; - }, - ]; - time: number; - gctime: number; - memory: number; - allocs: number; - }, -]; - -type JuliaBenchmarkGroup = [ - 'BenchmarkGroup', - { - data: Record; - tags: string[]; - }, -]; - -type JuliaBenchmarkJson = [object, JuliaBenchmarkGroup[]]; - -export interface JmhBenchmarkJson { - jmhVersion: string; - benchmark: string; - mode: string; - threads: number; - forks: number; - jvm: string; - jvmArgs: string[]; - jdkVersion: string; - vmName: string; - vmVersion: string; - warmupIterations: number; - warmupTime: string; - warmupBatchSize: number; - measurementIterations: number; - measurementTime: string; - measurementBatchSize: number; - params: object; - primaryMetric: { - score: number; - scoreError: number; - scoreConfidence: number[]; - scorePercentiles: { - 0.0: number; - 50.0: number; - 90.0: number; - 95.0: number; - 99.0: number; - 99.9: number; - 99.99: number; - 99.999: number; - 99.9999: number; - 100.0: number; - }; - scoreUnit: string; - rawData: number[][]; - }; -} - -export interface BenchmarkDotnetBenchmark { - FullName: string; - Statistics: { - StandardDeviation: number; - Mean: number; - }; -} - -export interface BenchmarkDotNetBenchmarkJson { - Benchmarks: BenchmarkDotnetBenchmark[]; -} - -function getHumanReadableUnitValue(seconds: number): [number, string] { - if (seconds < 1.0e-6) { - return [seconds * 1e9, 'nsec']; - } else if (seconds < 1.0e-3) { - return [seconds * 1e6, 'usec']; - } else if (seconds < 1.0) { - return [seconds * 1e3, 'msec']; - } else { - return [seconds, 'sec']; - } -} - -function getCommitFromPullRequestPayload(pr: PullRequest): Commit { - // On pull_request hook, head_commit is not available - const id: string = pr.head.sha; - const username: string = pr.head.user.login; - const user = { - name: username, // XXX: Fallback, not correct - username, - }; - - return { - author: user, - committer: user, - id, - message: pr.title, - timestamp: pr.head.repo.updated_at, - url: `${pr.html_url}/commits/${id}`, - }; -} - -async function getCommitFromGitHubAPIRequest(githubToken: string, ref?: string): Promise { - const octocat = github.getOctokit(githubToken); - - const { status, data } = await octocat.rest.repos.getCommit({ - owner: github.context.repo.owner, - repo: github.context.repo.repo, - ref: ref ?? github.context.ref, - }); - - if (!(status === 200 || status === 304)) { - throw new Error(`Could not fetch the head commit. Received code: ${status}`); - } - - const { commit } = data; - - return { - author: { - name: commit.author?.name, - username: data.author?.login, - email: commit.author?.email, - }, - committer: { - name: commit.committer?.name, - username: data.committer?.login, - email: commit.committer?.email, - }, - id: data.sha, - message: commit.message, - timestamp: commit.author?.date, - url: data.html_url, - }; -} - -async function getCommit(githubToken?: string, ref?: string): Promise { - if (github.context.payload.head_commit) { - return github.context.payload.head_commit; - } - - const pr = github.context.payload.pull_request; - - if (pr) { - return getCommitFromPullRequestPayload(pr); - } - - if (!githubToken) { - throw new Error( - `No commit information is found in payload: ${JSON.stringify( - github.context.payload, - null, - 2, - )}. Also, no 'github-token' provided, could not fallback to GitHub API Request.`, - ); - } - - return getCommitFromGitHubAPIRequest(githubToken, ref); -} - -function extractCargoResult(output: string): BenchmarkResult[] { +function extractCargoResult(config: Config, output: string): BenchmarkResult[] { const lines = output.split(/\r?\n/g); const ret = []; const reExtract = /^test (.+)\s+\.\.\. bench:\s+([0-9,.]+) (\w+\/\w+) \(\+\/- ([0-9,.]+)\)$/; @@ -332,405 +33,27 @@ function extractCargoResult(output: string): BenchmarkResult[] { value, range: `± ${range}`, unit: unit, + platform: config.platform, }); } return ret; } -function extractGoResult(output: string): BenchmarkResult[] { - const lines = output.split(/\r?\n/g); - const ret = []; - // Example: - // BenchmarkFib20-8 30000 41653 ns/op - // BenchmarkDoWithConfigurer1-8 30000000 42.3 ns/op - - // Example if someone has used the ReportMetric function to add additional metrics to each benchmark: - // BenchmarkThing-16 1 95258906556 ns/op 64.02 UnitsForMeasure2 31.13 UnitsForMeasure3 - - // reference, "Proposal: Go Benchmark Data Format": https://go.googlesource.com/proposal/+/master/design/14313-benchmark-format.md - // "A benchmark result line has the general form: [ ...]" - // "The fields are separated by runs of space characters (as defined by unicode.IsSpace), so the line can be parsed with strings.Fields. The line must have an even number of fields, and at least four." - const reExtract = - /^(?Benchmark\w+[\w()$%^&*-=|,[\]{}"#]*?)(?-\d+)?\s+(?\d+)\s+(?.+)$/; - - for (const line of lines) { - const m = line.match(reExtract); - if (m?.groups) { - const procs = m.groups.procs !== undefined ? m.groups.procs.slice(1) : null; - const times = m.groups.times; - const remainder = m.groups.remainder; - - const pieces = remainder.split(/[ \t]+/); - - // This is done for backwards compatibility with Go benchmarks that had multiple metrics in output, - // but they were not extracted properly before v1.18.0 - if (pieces.length > 2) { - pieces.unshift(pieces[0], remainder.slice(remainder.indexOf(pieces[1]))); - } - - for (let i = 0; i < pieces.length; i = i + 2) { - let extra = `${times} times`.replace(/\s\s+/g, ' '); - if (procs !== null) { - extra += `\n${procs} procs`; - } - const value = parseFloat(pieces[i]); - const unit = pieces[i + 1]; - let name; - if (i > 0) { - name = m.groups.name + ' - ' + unit; - } else { - name = m.groups.name; - } - ret.push({ name, value, unit, extra }); - } - } - } - - return ret; -} - -function extractBenchmarkJsResult(output: string): BenchmarkResult[] { - const lines = output.split(/\r?\n/g); - const ret = []; - // Example: - // fib(20) x 11,465 ops/sec ±1.12% (91 runs sampled) - // createObjectBuffer with 200 comments x 81.61 ops/sec ±1.70% (69 runs sampled) - const reExtract = /^ x ([0-9,.]+)\s+(\S+)\s+((?:±|\+-)[^%]+%) \((\d+) runs sampled\)$/; // Note: Extract parts after benchmark name - const reComma = /,/g; - - for (const line of lines) { - const idx = line.lastIndexOf(' x '); - if (idx === -1) { - continue; - } - const name = line.slice(0, idx); - const rest = line.slice(idx); - - const m = rest.match(reExtract); - if (m === null) { - continue; - } - - const value = parseFloat(m[1].replace(reComma, '')); - const unit = m[2]; - const range = m[3]; - const extra = `${m[4]} samples`; - - ret.push({ name, value, range, unit, extra }); - } - - return ret; -} - -function extractPytestResult(output: string): BenchmarkResult[] { - try { - const json: PytestBenchmarkJson = JSON.parse(output); - return json.benchmarks.map((bench) => { - const stats = bench.stats; - const name = bench.fullname; - const value = stats.ops; - const unit = 'iter/sec'; - const range = `stddev: ${stats.stddev}`; - const [mean, meanUnit] = getHumanReadableUnitValue(stats.mean); - const extra = `mean: ${mean} ${meanUnit}\nrounds: ${stats.rounds}`; - return { name, value, unit, range, extra }; - }); - } catch (err: any) { - throw new Error( - `Output file for 'pytest' must be JSON file generated by --benchmark-json option: ${err.message}`, - ); - } -} - -function extractGoogleCppResult(output: string): BenchmarkResult[] { - let json: GoogleCppBenchmarkJson; - try { - json = JSON.parse(output); - } catch (err: any) { - throw new Error( - `Output file for 'googlecpp' must be JSON file generated by --benchmark_format=json option: ${err.message}`, - ); - } - return json.benchmarks.map((b) => { - const name = b.name; - const value = b.real_time; - const unit = b.time_unit + '/iter'; - const extra = `iterations: ${b.iterations}\ncpu: ${b.cpu_time} ${b.time_unit}\nthreads: ${b.threads}`; - return { name, value, unit, extra }; - }); -} - -function extractCatch2Result(output: string): BenchmarkResult[] { - // Example: - - // benchmark name samples iterations estimated <-- Start benchmark section - // mean low mean high mean <-- Ignored - // std dev low std dev high std dev <-- Ignored - // ----------------------------------------------------- <-- Ignored - // Fibonacci 20 100 2 8.4318 ms <-- Start actual benchmark - // 43.186 us 41.402 us 46.246 us <-- Actual benchmark data - // 11.719 us 7.847 us 17.747 us <-- Ignored - - const reTestCaseStart = /^benchmark name +samples +iterations +(estimated|est run time)/; - const reBenchmarkStart = /(\d+) +(\d+) +(?:\d+(\.\d+)?) (?:ns|ms|us|s)\s*$/; - const reBenchmarkValues = - /^ +(\d+(?:\.\d+)?) (ns|us|ms|s) +(?:\d+(?:\.\d+)?) (?:ns|us|ms|s) +(?:\d+(?:\.\d+)?) (?:ns|us|ms|s)/; - const reEmptyLine = /^\s*$/; - const reSeparator = /^-+$/; - - const lines = output.split(/\r?\n/g); - lines.reverse(); - let lnum = 0; - function nextLine(): [string | null, number] { - return [lines.pop() ?? null, ++lnum]; - } - - function extractBench(): BenchmarkResult | null { - const startLine = nextLine()[0]; - if (startLine === null) { - return null; - } - - const start = startLine.match(reBenchmarkStart); - if (start === null) { - return null; // No more benchmark found. Go to next benchmark suite - } - - const extra = `${start[1]} samples\n${start[2]} iterations`; - const name = startLine.slice(0, start.index).trim(); - - const [meanLine, meanLineNum] = nextLine(); - const mean = meanLine?.match(reBenchmarkValues); - if (!mean) { - throw new Error( - `Mean values cannot be retrieved for benchmark '${name}' on parsing input '${ - meanLine ?? 'EOF' - }' at line ${meanLineNum}`, - ); - } - - const value = parseFloat(mean[1]); - const unit = mean[2]; - - const [stdDevLine, stdDevLineNum] = nextLine(); - const stdDev = stdDevLine?.match(reBenchmarkValues); - if (!stdDev) { - throw new Error( - `Std-dev values cannot be retrieved for benchmark '${name}' on parsing '${ - stdDevLine ?? 'EOF' - }' at line ${stdDevLineNum}`, - ); - } - - const range = '± ' + stdDev[1].trim(); - - // Skip empty line - const [emptyLine, emptyLineNum] = nextLine(); - if (emptyLine === null || !reEmptyLine.test(emptyLine)) { - throw new Error( - `Empty line is not following after 'std dev' line of benchmark '${name}' at line ${emptyLineNum}`, - ); - } - - return { name, value, range, unit, extra }; - } - - const ret = []; - while (lines.length > 0) { - // Search header of benchmark section - const line = nextLine()[0]; - if (line === null) { - break; // All lines were eaten - } - if (!reTestCaseStart.test(line)) { - continue; - } - - // Eat until a separator line appears - for (;;) { - const [line, num] = nextLine(); - if (line === null) { - throw new Error(`Separator '------' does not appear after benchmark suite at line ${num}`); - } - if (reSeparator.test(line)) { - break; - } - } - - let benchFound = false; - for (;;) { - const res = extractBench(); - if (res === null) { - break; - } - ret.push(res); - benchFound = true; - } - if (!benchFound) { - throw new Error(`No benchmark found for bench suite. Possibly mangled output from Catch2:\n\n${output}`); - } - } - - return ret; -} - -function extractJuliaBenchmarkHelper([_, bench]: JuliaBenchmarkGroup, labels: string[] = []): BenchmarkResult[] { - const res: BenchmarkResult[] = []; - for (const key in bench.data) { - const value = bench.data[key]; - if (value[0] === 'BenchmarkGroup') { - res.push(...extractJuliaBenchmarkHelper(value as JuliaBenchmarkGroup, [...labels, key])); - } else if (value[0] === 'TrialEstimate') { - const v = value as JuliaBenchmarkTrialEstimate; - res.push({ - name: [...labels, key].join('/'), - value: v[1].time, - unit: 'ns', - extra: `gctime=${v[1].gctime}\nmemory=${v[1].memory}\nallocs=${v[1].allocs}\nparams=${JSON.stringify( - v[1].params[1], - )}`, - }); - } else if (value[0] === 'Trial') { - throw new Error( - `Only TrialEstimate is supported currently. You need to apply apply an estimation (minimum/median/mean/maximum/std) before saving the JSON file.`, - ); - } else { - throw new Error(`Unsupported type ${value[0]}`); - } - } - return res; -} - -function extractJuliaBenchmarkResult(output: string): BenchmarkResult[] { - let json: JuliaBenchmarkJson; - try { - json = JSON.parse(output); - } catch (err: any) { - throw new Error( - `Output file for 'julia' must be JSON file generated by BenchmarkTools.save("output.json", suit::BenchmarkGroup) : ${err.message}`, - ); - } - - const res: BenchmarkResult[] = []; - for (const group of json[1]) { - res.push(...extractJuliaBenchmarkHelper(group)); - } - - return res; -} - -function extractJmhResult(output: string): BenchmarkResult[] { - let json: JmhBenchmarkJson[]; - try { - json = JSON.parse(output); - } catch (err: any) { - throw new Error(`Output file for 'jmh' must be JSON file generated by -rf json option: ${err.message}`); - } - return json.map((b) => { - const name = b.benchmark; - const value = b.primaryMetric.score; - const unit = b.primaryMetric.scoreUnit; - const params = b.params ? ' ( ' + JSON.stringify(b.params) + ' )' : ''; - const extra = `iterations: ${b.measurementIterations}\nforks: ${b.forks}\nthreads: ${b.threads}`; - return { name: name + params, value, unit, extra }; - }); -} - -function extractBenchmarkDotnetResult(output: string): BenchmarkResult[] { - let json: BenchmarkDotNetBenchmarkJson; - try { - json = JSON.parse(output); - } catch (err: any) { - throw new Error( - `Output file for 'benchmarkdotnet' must be JSON file generated by '--exporters json' option or by adding the JsonExporter to your run config: ${err.message}`, - ); - } - - return json.Benchmarks.map((benchmark) => { - const name = benchmark.FullName; - const value = benchmark.Statistics.Mean; - const stdDev = benchmark.Statistics.StandardDeviation; - const range = `± ${stdDev}`; - return { name, value, unit: 'ns', range }; - }); -} - -function extractCustomBenchmarkResult(output: string): BenchmarkResult[] { - try { - const json: BenchmarkResult[] = JSON.parse(output); - return json.map(({ name, value, unit, range, extra }) => { - return { name, value, unit, range, extra }; - }); - } catch (err: any) { - throw new Error( - `Output file for 'custom-(bigger|smaller)-is-better' must be JSON file containing an array of entries in BenchmarkResult format: ${err.message}`, - ); - } -} - -function extractLuauBenchmarkResult(output: string): BenchmarkResult[] { - const lines = output.split(/\n/); - const results: BenchmarkResult[] = []; - - output; - for (const line of lines) { - if (!line.startsWith('SUCCESS')) continue; - const [_0, name, _2, valueStr, _4, range, _6, extra] = line.split(/\s+/); - - results.push({ - name: name, - value: parseFloat(valueStr), - unit: valueStr.replace(/.[0-9]+/g, ''), - range: `±${range}`, - extra: extra, - }); - } - - return results; +export async function localWriteBenchmark(benches: BenchmarkResult[], config: Config) { + const data = JSON.stringify(benches); + const outPath = config.dataOutPath; + await fs.writeFile(outPath, data); } -export async function extractResult(config: Config): Promise { +export async function extractData(config: Config): Promise { const output = await fs.readFile(config.outputFilePath, 'utf8'); - const { tool, githubToken, ref } = config; + const { tool } = config; let benches: BenchmarkResult[]; switch (tool) { case 'cargo': - benches = extractCargoResult(output); - break; - case 'go': - benches = extractGoResult(output); - break; - case 'benchmarkjs': - benches = extractBenchmarkJsResult(output); - break; - case 'pytest': - benches = extractPytestResult(output); - break; - case 'googlecpp': - benches = extractGoogleCppResult(output); - break; - case 'catch2': - benches = extractCatch2Result(output); - break; - case 'julia': - benches = extractJuliaBenchmarkResult(output); - break; - case 'jmh': - benches = extractJmhResult(output); - break; - case 'benchmarkdotnet': - benches = extractBenchmarkDotnetResult(output); - break; - case 'customBiggerIsBetter': - benches = extractCustomBenchmarkResult(output); - break; - case 'customSmallerIsBetter': - benches = extractCustomBenchmarkResult(output); - break; - case 'benchmarkluau': - benches = extractLuauBenchmarkResult(output); + benches = extractCargoResult(config, output); break; default: throw new Error(`FATAL: Unexpected tool: '${tool}'`); @@ -740,12 +63,5 @@ export async function extractResult(config: Config): Promise { throw new Error(`No benchmark result was found in ${config.outputFilePath}. Benchmark output was '${output}'`); } - const commit = await getCommit(githubToken, ref); - - return { - commit, - date: Date.now(), - tool, - benches, - }; + return benches; } diff --git a/src/git.ts b/src/git.ts deleted file mode 100644 index 1a228ac..0000000 --- a/src/git.ts +++ /dev/null @@ -1,168 +0,0 @@ -import { exec } from '@actions/exec'; -import * as core from '@actions/core'; -import * as github from '@actions/github'; -import { URL } from 'url'; - -const DEFAULT_GITHUB_URL = 'https://github.com'; - -interface ExecResult { - stdout: string; - stderr: string; - code: number | null; -} - -async function capture(cmd: string, args: string[]): Promise { - const res: ExecResult = { - stdout: '', - stderr: '', - code: null, - }; - - try { - const code = await exec(cmd, args, { - listeners: { - stdout(data) { - res.stdout += data.toString(); - }, - stderr(data) { - res.stderr += data.toString(); - }, - }, - }); - res.code = code; - return res; - } catch (err) { - const msg = `Command '${cmd}' failed with args '${args.join(' ')}': ${res.stderr}: ${err}`; - core.debug(`@actions/exec.exec() threw an error: ${msg}`); - throw new Error(msg); - } -} - -export function getServerUrlObj(repositoryUrl: string | undefined): URL { - const urlValue = - repositoryUrl && repositoryUrl.trim().length > 0 - ? repositoryUrl - : process.env['GITHUB_SERVER_URL'] ?? DEFAULT_GITHUB_URL; - return new URL(urlValue); -} - -export function getServerUrl(repositoryUrl: string | undefined): string { - return getServerUrlObj(repositoryUrl).origin; -} - -export function getServerName(repositoryUrl: string | undefined): string { - return getServerUrlObj(repositoryUrl).hostname; -} - -export async function cmd(additionalGitOptions: string[], ...args: string[]): Promise { - core.debug(`Executing Git: ${args.join(' ')}`); - const serverUrl = getServerUrl(github.context.payload.repository?.html_url); - const userArgs = [ - ...additionalGitOptions, - '-c', - 'user.name=github-action-benchmark', - '-c', - 'user.email=github@users.noreply.github.com', - '-c', - `http.${serverUrl}/.extraheader=`, // This config is necessary to support actions/checkout@v2 (#9) - ]; - const res = await capture('git', userArgs.concat(args)); - if (res.code !== 0) { - throw new Error(`Command 'git ${args.join(' ')}' failed: ${JSON.stringify(res)}`); - } - return res.stdout; -} - -function getCurrentRepoRemoteUrl(token: string): string { - const { repo, owner } = github.context.repo; - const serverName = getServerName(github.context.payload.repository?.html_url); - return getRepoRemoteUrl(token, `${serverName}/${owner}/${repo}`); -} - -function getRepoRemoteUrl(token: string, repoUrl: string): string { - return `https://x-access-token:${token}@${repoUrl}.git`; -} - -export async function push( - token: string, - repoUrl: string | undefined, - branch: string, - additionalGitOptions: string[] = [], - ...options: string[] -): Promise { - core.debug(`Executing 'git push' to branch '${branch}' with token and options '${options.join(' ')}'`); - - const remote = repoUrl ? getRepoRemoteUrl(token, repoUrl) : getCurrentRepoRemoteUrl(token); - let args = ['push', remote, `${branch}:${branch}`, '--no-verify']; - if (options.length > 0) { - args = args.concat(options); - } - - return cmd(additionalGitOptions, ...args); -} - -export async function pull( - token: string | undefined, - branch: string, - additionalGitOptions: string[] = [], - ...options: string[] -): Promise { - core.debug(`Executing 'git pull' to branch '${branch}' with token and options '${options.join(' ')}'`); - - const remote = token !== undefined ? getCurrentRepoRemoteUrl(token) : 'origin'; - let args = ['pull', remote, branch]; - if (options.length > 0) { - args = args.concat(options); - } - - return cmd(additionalGitOptions, ...args); -} - -export async function fetch( - token: string | undefined, - branch: string, - additionalGitOptions: string[] = [], - ...options: string[] -): Promise { - core.debug(`Executing 'git fetch' for branch '${branch}' with token and options '${options.join(' ')}'`); - - const remote = token !== undefined ? getCurrentRepoRemoteUrl(token) : 'origin'; - let args = ['fetch', remote, `${branch}:${branch}`]; - if (options.length > 0) { - args = args.concat(options); - } - - return cmd(additionalGitOptions, ...args); -} - -export async function clone( - token: string, - ghRepository: string, - baseDirectory: string, - additionalGitOptions: string[] = [], - ...options: string[] -): Promise { - core.debug(`Executing 'git clone' to directory '${baseDirectory}' with token and options '${options.join(' ')}'`); - - const remote = getRepoRemoteUrl(token, ghRepository); - let args = ['clone', remote, baseDirectory]; - if (options.length > 0) { - args = args.concat(options); - } - - return cmd(additionalGitOptions, ...args); -} -export async function checkout( - ghRef: string, - additionalGitOptions: string[] = [], - ...options: string[] -): Promise { - core.debug(`Executing 'git checkout' to ref '${ghRef}' with token and options '${options.join(' ')}'`); - - let args = ['checkout', ghRef]; - if (options.length > 0) { - args = args.concat(options); - } - - return cmd(additionalGitOptions, ...args); -} diff --git a/src/index.ts b/src/index.ts index 2d22153..13b7983 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,18 +1,17 @@ import * as core from '@actions/core'; import { configFromJobInput } from './config'; -import { extractResult } from './extract'; -import { writeBenchmark } from './write'; +import { extractData, localWriteBenchmark } from './extract'; async function main() { const config = await configFromJobInput(); core.debug(`Config extracted from job: ${config}`); - const bench = await extractResult(config); - core.debug(`Benchmark result was extracted: ${bench}`); + const benches = await extractData(config); + core.debug(`Benchmark result was extracted: ${benches}`); - await writeBenchmark(bench, config); + await localWriteBenchmark(benches, config); - console.log('github-action-benchmark was run successfully!', '\nData:', bench); + console.log('action was run successfully!', '\nData:', benches); } main().catch((e) => core.setFailed(e.message)); diff --git a/src/write.ts b/src/write.ts deleted file mode 100644 index e1ad4ea..0000000 --- a/src/write.ts +++ /dev/null @@ -1,573 +0,0 @@ -import { promises as fs } from 'fs'; -import * as path from 'path'; -import * as io from '@actions/io'; -import * as core from '@actions/core'; -import * as github from '@actions/github'; -import * as git from './git'; -import { Benchmark, BenchmarkResult } from './extract'; -import { Config, ToolType } from './config'; -import { DEFAULT_INDEX_HTML } from './default_index_html'; -import { leavePRComment } from './comment/leavePRComment'; -import { leaveCommitComment } from './comment/leaveCommitComment'; - -export type BenchmarkSuites = { [name: string]: Benchmark[] }; -export interface DataJson { - lastUpdate: number; - repoUrl: string; - entries: BenchmarkSuites; -} - -export const SCRIPT_PREFIX = 'window.BENCHMARK_DATA = '; -const DEFAULT_DATA_JSON = { - lastUpdate: 0, - repoUrl: '', - entries: {}, -}; - -async function loadDataJs(dataPath: string): Promise { - try { - const script = await fs.readFile(dataPath, 'utf8'); - const json = script.slice(SCRIPT_PREFIX.length); - const parsed = JSON.parse(json); - core.debug(`Loaded data.js at ${dataPath}`); - return parsed; - } catch (err) { - console.log(`Could not find data.js at ${dataPath}. Using empty default: ${err}`); - return { ...DEFAULT_DATA_JSON }; - } -} - -async function storeDataJs(dataPath: string, data: DataJson) { - const script = SCRIPT_PREFIX + JSON.stringify(data, null, 2); - await fs.writeFile(dataPath, script, 'utf8'); - core.debug(`Overwrote ${dataPath} for adding new data`); -} - -async function addIndexHtmlIfNeeded(additionalGitArguments: string[], dir: string, baseDir: string) { - const indexHtmlRelativePath = path.join(dir, 'index.html'); - const indexHtmlFullPath = path.join(baseDir, indexHtmlRelativePath); - try { - await fs.stat(indexHtmlFullPath); - core.debug(`Skipped to create default index.html since it is already existing: ${indexHtmlFullPath}`); - return; - } catch (_) { - // Continue - } - - await fs.writeFile(indexHtmlFullPath, DEFAULT_INDEX_HTML, 'utf8'); - await git.cmd(additionalGitArguments, 'add', indexHtmlRelativePath); - console.log('Created default index.html at', indexHtmlFullPath); -} - -function biggerIsBetter(tool: ToolType): boolean { - switch (tool) { - case 'cargo': - return false; - case 'go': - return false; - case 'benchmarkjs': - return true; - case 'benchmarkluau': - return false; - case 'pytest': - return true; - case 'googlecpp': - return false; - case 'catch2': - return false; - case 'julia': - return false; - case 'jmh': - return false; - case 'benchmarkdotnet': - return false; - case 'customBiggerIsBetter': - return true; - case 'customSmallerIsBetter': - return false; - } -} - -interface Alert { - current: BenchmarkResult; - prev: BenchmarkResult; - ratio: number; -} - -function findAlerts(curSuite: Benchmark, prevSuite: Benchmark, threshold: number): Alert[] { - core.debug(`Comparing current:${curSuite.commit.id} and prev:${prevSuite.commit.id} for alert`); - - const alerts = []; - for (const current of curSuite.benches) { - const prev = prevSuite.benches.find((b) => b.name === current.name); - if (prev === undefined) { - core.debug(`Skipped because benchmark '${current.name}' is not found in previous benchmarks`); - continue; - } - - const ratio = getRatio(curSuite.tool, prev, current); - - if (ratio > threshold) { - core.warning( - `Performance alert! Previous value was ${prev.value} and current value is ${current.value}.` + - ` It is ${ratio}x worse than previous exceeding a ratio threshold ${threshold}`, - ); - alerts.push({ current, prev, ratio }); - } - } - - return alerts; -} - -function getCurrentRepoMetadata() { - const { repo, owner } = github.context.repo; - const serverUrl = git.getServerUrl(github.context.payload.repository?.html_url); - return { - name: repo, - owner: { - login: owner, - }, - // eslint-disable-next-line @typescript-eslint/naming-convention - html_url: `${serverUrl}/${owner}/${repo}`, - }; -} - -function floatStr(n: number) { - if (!Number.isFinite(n)) { - return `${n > 0 ? '+' : '-'}∞`; - } - - if (Number.isInteger(n)) { - return n.toFixed(0); - } - - if (n > 0.1) { - return n.toFixed(2); - } - - return n.toString(); -} - -function strVal(b: BenchmarkResult): string { - let s = `\`${b.value}\` ${b.unit}`; - if (b.range) { - s += ` (\`${b.range}\`)`; - } - return s; -} - -function commentFooter(): string { - const repoMetadata = getCurrentRepoMetadata(); - const repoUrl = repoMetadata.html_url ?? ''; - const actionUrl = repoUrl + '/actions?query=workflow%3A' + encodeURIComponent(github.context.workflow); - - return `This comment was automatically generated by [workflow](${actionUrl}) using [github-action-benchmark](https://github.com/marketplace/actions/continuous-benchmark).`; -} - -export function buildComment( - benchName: string, - curSuite: Benchmark, - prevSuite: Benchmark, - expandableDetails = true, -): string { - const lines = [ - `# ${benchName}`, - '', - expandableDetails ? '
' : '', - '', - `| Benchmark suite | Current: ${curSuite.commit.id} | Previous: ${prevSuite.commit.id} | Ratio |`, - '|-|-|-|-|', - ]; - - for (const current of curSuite.benches) { - let line; - const prev = prevSuite.benches.find((i) => i.name === current.name); - - if (prev) { - const ratio = getRatio(curSuite.tool, prev, current); - - line = `| \`${current.name}\` | ${strVal(current)} | ${strVal(prev)} | \`${floatStr(ratio)}\` |`; - } else { - line = `| \`${current.name}\` | ${strVal(current)} | | |`; - } - - lines.push(line); - } - - // Footer - lines.push('', expandableDetails ? '
' : '', '', commentFooter()); - - return lines.join('\n'); -} - -function buildAlertComment( - alerts: Alert[], - benchName: string, - curSuite: Benchmark, - prevSuite: Benchmark, - threshold: number, - cc: string[], -): string { - // Do not show benchmark name if it is the default value 'Benchmark'. - const benchmarkText = benchName === 'Benchmark' ? '' : ` **'${benchName}'**`; - const title = threshold === 0 ? '# Performance Report' : '# :warning: **Performance Alert** :warning:'; - const thresholdString = floatStr(threshold); - const lines = [ - title, - '', - `Possible performance regression was detected for benchmark${benchmarkText}.`, - `Benchmark result of this commit is worse than the previous benchmark result exceeding threshold \`${thresholdString}\`.`, - '', - `| Benchmark suite | Current: ${curSuite.commit.id} | Previous: ${prevSuite.commit.id} | Ratio |`, - '|-|-|-|-|', - ]; - - for (const alert of alerts) { - const { current, prev, ratio } = alert; - const line = `| \`${current.name}\` | ${strVal(current)} | ${strVal(prev)} | \`${floatStr(ratio)}\` |`; - lines.push(line); - } - - // Footer - lines.push('', commentFooter()); - - if (cc.length > 0) { - lines.push('', `CC: ${cc.join(' ')}`); - } - - return lines.join('\n'); -} - -async function leaveComment(commitId: string, body: string, commentId: string, token: string) { - core.debug('Sending comment:\n' + body); - - const repoMetadata = getCurrentRepoMetadata(); - const pr = github.context.payload.pull_request; - - return await (pr?.number - ? leavePRComment(repoMetadata.owner.login, repoMetadata.name, pr.number, body, commentId, token) - : leaveCommitComment(repoMetadata.owner.login, repoMetadata.name, commitId, body, commentId, token)); -} - -async function handleComment(benchName: string, curSuite: Benchmark, prevSuite: Benchmark, config: Config) { - const { commentAlways, githubToken } = config; - - if (!commentAlways) { - core.debug('Comment check was skipped because comment-always is disabled'); - return; - } - - if (!githubToken) { - throw new Error("'comment-always' input is set but 'github-token' input is not set"); - } - - core.debug('Commenting about benchmark comparison'); - - const body = buildComment(benchName, curSuite, prevSuite); - - await leaveComment(curSuite.commit.id, body, `${benchName} Summary`, githubToken); -} - -async function handleAlert(benchName: string, curSuite: Benchmark, prevSuite: Benchmark, config: Config) { - const { alertThreshold, githubToken, commentOnAlert, failOnAlert, alertCommentCcUsers, failThreshold } = config; - - if (!commentOnAlert && !failOnAlert) { - core.debug('Alert check was skipped because both comment-on-alert and fail-on-alert were disabled'); - return; - } - - const alerts = findAlerts(curSuite, prevSuite, alertThreshold); - if (alerts.length === 0) { - core.debug('No performance alert found happily'); - return; - } - - core.debug(`Found ${alerts.length} alerts`); - const body = buildAlertComment(alerts, benchName, curSuite, prevSuite, alertThreshold, alertCommentCcUsers); - let message = body; - - if (commentOnAlert) { - if (!githubToken) { - throw new Error("'comment-on-alert' input is set but 'github-token' input is not set"); - } - const res = await leaveComment(curSuite.commit.id, body, `${benchName} Alert`, githubToken); - const url = res.data.html_url; - message = body + `\nComment was generated at ${url}`; - } - - if (failOnAlert) { - // Note: alertThreshold is smaller than failThreshold. It was checked in config.ts - const len = alerts.length; - const threshold = floatStr(failThreshold); - const failures = alerts.filter((a) => a.ratio > failThreshold); - if (failures.length > 0) { - core.debug('Mark this workflow as fail since one or more fatal alerts found'); - if (failThreshold !== alertThreshold) { - // Prepend message that explains how these alerts were detected with different thresholds - message = `${failures.length} of ${len} alerts exceeded the failure threshold \`${threshold}\` specified by fail-threshold input:\n\n${message}`; - } - throw new Error(message); - } else { - core.debug( - `${len} alerts exceeding the alert threshold ${alertThreshold} were found but` + - ` none of them exceeded the failure threshold ${threshold}`, - ); - } - } -} - -function addBenchmarkToDataJson( - benchName: string, - bench: Benchmark, - data: DataJson, - maxItems: number | null, -): Benchmark | null { - const repoMetadata = getCurrentRepoMetadata(); - const htmlUrl = repoMetadata.html_url ?? ''; - - let prevBench: Benchmark | null = null; - data.lastUpdate = Date.now(); - data.repoUrl = htmlUrl; - - // Add benchmark result - if (data.entries[benchName] === undefined) { - data.entries[benchName] = [bench]; - core.debug(`No suite was found for benchmark '${benchName}' in existing data. Created`); - } else { - const suites = data.entries[benchName]; - // Get last suite which has different commit ID for alert comment - for (const e of suites.slice().reverse()) { - if (e.commit.id !== bench.commit.id) { - prevBench = e; - break; - } - } - - suites.push(bench); - - if (maxItems !== null && suites.length > maxItems) { - suites.splice(0, suites.length - maxItems); - core.debug( - `Number of data items for '${benchName}' was truncated to ${maxItems} due to max-items-in-charts`, - ); - } - } - - return prevBench; -} - -function isRemoteRejectedError(err: unknown): err is Error { - if (err instanceof Error) { - return ['[remote rejected]', '[rejected]'].some((l) => err.message.includes(l)); - } - return false; -} - -async function writeBenchmarkToGitHubPagesWithRetry( - bench: Benchmark, - config: Config, - retry: number, -): Promise { - const { - name, - tool, - ghPagesBranch, - ghRepository, - benchmarkDataDirPath, - githubToken, - autoPush, - skipFetchGhPages, - maxItemsInChart, - } = config; - const rollbackActions = new Array<() => Promise>(); - - // FIXME: This payload is not available on `schedule:` or `workflow_dispatch:` events. - const isPrivateRepo = github.context.payload.repository?.private ?? false; - - let benchmarkBaseDir = './'; - let extraGitArguments: string[] = []; - - if (githubToken && !skipFetchGhPages && ghRepository) { - benchmarkBaseDir = './benchmark-data-repository'; - await git.clone(githubToken, ghRepository, benchmarkBaseDir); - rollbackActions.push(async () => { - await io.rmRF(benchmarkBaseDir); - }); - extraGitArguments = [`--work-tree=${benchmarkBaseDir}`, `--git-dir=${benchmarkBaseDir}/.git`]; - await git.checkout(ghPagesBranch, extraGitArguments); - } else if (!skipFetchGhPages && (!isPrivateRepo || githubToken)) { - await git.pull(githubToken, ghPagesBranch); - } else if (isPrivateRepo && !skipFetchGhPages) { - core.warning( - "'git pull' was skipped. If you want to ensure GitHub Pages branch is up-to-date " + - "before generating a commit, please set 'github-token' input to pull GitHub pages branch", - ); - } else { - console.warn('NOTHING EXECUTED:', { - skipFetchGhPages, - ghRepository, - isPrivateRepo, - githubToken: !!githubToken, - }); - } - - // `benchmarkDataDirPath` is an absolute path at this stage, - // so we need to convert it to relative to be able to prepend the `benchmarkBaseDir` - const benchmarkDataRelativeDirPath = path.relative(process.cwd(), benchmarkDataDirPath); - const benchmarkDataDirFullPath = path.join(benchmarkBaseDir, benchmarkDataRelativeDirPath); - - const dataPath = path.join(benchmarkDataDirFullPath, 'data.js'); - - await io.mkdirP(benchmarkDataDirFullPath); - - const data = await loadDataJs(dataPath); - const prevBench = addBenchmarkToDataJson(name, bench, data, maxItemsInChart); - - await storeDataJs(dataPath, data); - - await git.cmd(extraGitArguments, 'add', path.join(benchmarkDataRelativeDirPath, 'data.js')); - await addIndexHtmlIfNeeded(extraGitArguments, benchmarkDataRelativeDirPath, benchmarkBaseDir); - await git.cmd(extraGitArguments, 'commit', '-m', `add ${name} (${tool}) benchmark result for ${bench.commit.id}`); - - if (githubToken && autoPush) { - try { - await git.push(githubToken, ghRepository, ghPagesBranch, extraGitArguments); - console.log( - `Automatically pushed the generated commit to ${ghPagesBranch} branch since 'auto-push' is set to true`, - ); - } catch (err: unknown) { - if (!isRemoteRejectedError(err)) { - throw err; - } - // Fall through - - core.warning(`Auto-push failed because the remote ${ghPagesBranch} was updated after git pull`); - - if (retry > 0) { - core.debug('Rollback the auto-generated commit before retry'); - await git.cmd(extraGitArguments, 'reset', '--hard', 'HEAD~1'); - - // we need to rollback actions in order so not running them concurrently - for (const action of rollbackActions) { - await action(); - } - - core.warning( - `Retrying to generate a commit and push to remote ${ghPagesBranch} with retry count ${retry}...`, - ); - return await writeBenchmarkToGitHubPagesWithRetry(bench, config, retry - 1); // Recursively retry - } else { - core.warning(`Failed to add benchmark data to '${name}' data: ${JSON.stringify(bench)}`); - throw new Error( - `Auto-push failed 3 times since the remote branch ${ghPagesBranch} rejected pushing all the time. Last exception was: ${err.message}`, - ); - } - } - } else { - core.debug( - `Auto-push to ${ghPagesBranch} is skipped because it requires both 'github-token' and 'auto-push' inputs`, - ); - } - - return prevBench; -} - -async function writeBenchmarkToGitHubPages(bench: Benchmark, config: Config): Promise { - const { ghPagesBranch, skipFetchGhPages, ghRepository, githubToken } = config; - if (!ghRepository) { - if (!skipFetchGhPages) { - await git.fetch(githubToken, ghPagesBranch); - } - await git.cmd([], 'switch', ghPagesBranch); - } - try { - return await writeBenchmarkToGitHubPagesWithRetry(bench, config, 10); - } finally { - if (!ghRepository) { - // `git switch` does not work for backing to detached head - await git.cmd([], 'checkout', '-'); - } - } -} - -async function loadDataJson(jsonPath: string): Promise { - try { - const content = await fs.readFile(jsonPath, 'utf8'); - const json: DataJson = JSON.parse(content); - core.debug(`Loaded external JSON file at ${jsonPath}`); - return json; - } catch (err) { - core.warning( - `Could not find external JSON file for benchmark data at ${jsonPath}. Using empty default: ${err}`, - ); - return { ...DEFAULT_DATA_JSON }; - } -} - -async function writeBenchmarkToExternalJson( - bench: Benchmark, - jsonFilePath: string, - config: Config, -): Promise { - const { name, maxItemsInChart, saveDataFile } = config; - const data = await loadDataJson(jsonFilePath); - const prevBench = addBenchmarkToDataJson(name, bench, data, maxItemsInChart); - - if (!saveDataFile) { - core.debug('Skipping storing benchmarks in external data file'); - return prevBench; - } - - try { - const jsonDirPath = path.dirname(jsonFilePath); - await io.mkdirP(jsonDirPath); - await fs.writeFile(jsonFilePath, JSON.stringify(data, null, 2), 'utf8'); - } catch (err) { - throw new Error(`Could not store benchmark data as JSON at ${jsonFilePath}: ${err}`); - } - - return prevBench; -} - -export async function writeBenchmark(bench: Benchmark, config: Config) { - const { name, externalDataJsonPath } = config; - const prevBench = externalDataJsonPath - ? await writeBenchmarkToExternalJson(bench, externalDataJsonPath, config) - : await writeBenchmarkToGitHubPages(bench, config); - - // Put this after `git push` for reducing possibility to get conflict on push. Since sending - // comment take time due to API call, do it after updating remote branch. - if (prevBench === null) { - core.debug('Alert check was skipped because previous benchmark result was not found'); - } else { - await handleComment(name, bench, prevBench, config); - await handleSummary(name, bench, prevBench, config); - await handleAlert(name, bench, prevBench, config); - } -} - -async function handleSummary(benchName: string, currBench: Benchmark, prevBench: Benchmark, config: Config) { - const { summaryAlways } = config; - - if (!summaryAlways) { - core.debug('Summary was skipped because summary-always is disabled'); - return; - } - - const body = buildComment(benchName, currBench, prevBench, false); - - const summary = core.summary.addRaw(body); - - core.debug('Writing a summary about benchmark comparison'); - core.debug(summary.stringify()); - - await summary.write(); -} - -function getRatio(tool: ToolType, prev: BenchmarkResult, current: BenchmarkResult) { - if (prev.value === 0 && current.value === 0) return 1; - - return biggerIsBetter(tool) - ? prev.value / current.value // e.g. current=100, prev=200 - : current.value / prev.value; // e.g. current=200, prev=100 -} diff --git a/test/__snapshots__/extract.spec.ts.snap b/test/__snapshots__/extract.spec.ts.snap index 76d1e91..d403a75 100644 --- a/test/__snapshots__/extract.spec.ts.snap +++ b/test/__snapshots__/extract.spec.ts.snap @@ -1,970 +1,289 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`extractResult() extracts benchmark output from benchmarkdotnet - benchmarkdotnet.json 1`] = ` -{ - "benches": [ - { - "name": "Sample.Benchmarks.Fib10", - "range": "± 0.1981212530783709", - "unit": "ns", - "value": 24.4202085009643, - }, - { - "name": "Sample.Benchmarks.Fib20", - "range": "± 0.7903737021529575", - "unit": "ns", - "value": 51.52008151549559, - }, - ], - "commit": { - "author": null, - "committer": null, - "id": "123456789abcdef", - "message": "this is dummy", - "timestamp": "dummy timestamp", - "url": "https://github.com/dummy/repo", - }, - "date": 1712131503296, - "tool": "benchmarkdotnet", -} -`; - -exports[`extractResult() extracts benchmark output from benchmarkjs - benchmarkjs_output.txt 1`] = ` -{ - "benches": [ - { - "extra": "93 samples", - "name": "fib(10)", - "range": "±0.74%", - "unit": "ops/sec", - "value": 1431759, - }, - { - "extra": "96 samples", - "name": "fib(20)", - "range": "±0.32%", - "unit": "ops/sec", - "value": 12146, - }, - { - "extra": "69 samples", - "name": "createObjectBuffer with 200 comments", - "range": "±1.70%", - "unit": "ops/sec", - "value": 81.61, - }, - ], - "commit": { - "author": null, - "committer": null, - "id": "123456789abcdef", - "message": "this is dummy", - "timestamp": "dummy timestamp", - "url": "https://github.com/dummy/repo", - }, - "date": 1712131503296, - "tool": "benchmarkjs", -} -`; - -exports[`extractResult() extracts benchmark output from benchmarkluau - benchmarkluau_output.txt 1`] = ` -{ - "benches": [ - { - "extra": "luau", - "name": "base64", - "range": "±0.636%", - "unit": "ms", - "value": 15.041, - }, - { - "extra": "luau", - "name": "chess", - "range": "±0.212%", - "unit": "ms", - "value": 69.56, - }, - { - "extra": "luau", - "name": "life", - "range": "±0.187%", - "unit": "ms", - "value": 85.089, - }, - ], - "commit": { - "author": null, - "committer": null, - "id": "123456789abcdef", - "message": "this is dummy", - "timestamp": "dummy timestamp", - "url": "https://github.com/dummy/repo", - }, - "date": 1712131503296, - "tool": "benchmarkluau", -} -`; - -exports[`extractResult() extracts benchmark output from cargo - cargo_output.txt 1`] = ` -{ - "benches": [ - { - "name": "bench_fib_10", - "range": "± 24", - "unit": "ns/iter", - "value": 135, - }, - { - "name": "bench_fib_20", - "range": "± 755", - "unit": "ns/iter", - "value": 18149, - }, - ], - "commit": { - "author": null, - "committer": null, - "id": "123456789abcdef", - "message": "this is dummy", - "timestamp": "dummy timestamp", - "url": "https://github.com/dummy/repo", - }, - "date": 1712131503296, - "tool": "cargo", -} -`; - -exports[`extractResult() extracts benchmark output from cargo - cargo_output_units.txt 1`] = ` -{ - "benches": [ - { - "name": "cmov", - "range": "± 14", - "unit": "cycles/iter", - "value": 2835, - }, - { - "name": "cmov2", - "range": "± 19", - "unit": "cycles/iter", - "value": 2845, - }, - { - "name": "mov", - "range": "± 17", - "unit": "cycles/iter", - "value": 1508, - }, - { - "name": "upload", - "range": "± 420", - "unit": "MS/s", - "value": 1911, - }, - { - "name": "download", - "range": "± 69", - "unit": "MS/s", - "value": 9001, - }, - ], - "commit": { - "author": null, - "committer": null, - "id": "123456789abcdef", - "message": "this is dummy", - "timestamp": "dummy timestamp", - "url": "https://github.com/dummy/repo", - }, - "date": 1712131503296, - "tool": "cargo", -} -`; - -exports[`extractResult() extracts benchmark output from cargo - cargo_output2.txt 1`] = ` -{ - "benches": [ - { - "name": "bench_engine_new", - "range": "± 70126", - "unit": "ns/iter", - "value": 211834, - }, - { - "name": "bench_engine_new_raw", - "range": "± 18", - "unit": "ns/iter", - "value": 197, - }, - { - "name": "bench_engine_new_raw_core", - "range": "± 31", - "unit": "ns/iter", - "value": 196, - }, - { - "name": "bench_engine_register_fn", - "range": "± 82", - "unit": "ns/iter", - "value": 493, - }, - ], - "commit": { - "author": null, - "committer": null, - "id": "123456789abcdef", - "message": "this is dummy", - "timestamp": "dummy timestamp", - "url": "https://github.com/dummy/repo", - }, - "date": 1712131503296, - "tool": "cargo", -} -`; - -exports[`extractResult() extracts benchmark output from cargo - cargo_output3.txt 1`] = ` -{ - "benches": [ - { - "name": "bench_fib_10", - "range": "± 2.21", - "unit": "ns/iter", - "value": 148.7, - }, - { - "name": "bench_fib_20", - "range": "± 440.25", - "unit": "ns/iter", - "value": 18794.12, - }, - ], - "commit": { - "author": null, - "committer": null, - "id": "123456789abcdef", - "message": "this is dummy", - "timestamp": "dummy timestamp", - "url": "https://github.com/dummy/repo", - }, - "date": 1712131503296, - "tool": "cargo", -} -`; - -exports[`extractResult() extracts benchmark output from cargo - criterion_output.txt 1`] = ` -{ - "benches": [ - { - "name": "Create Realm", - "range": "± 4", - "unit": "ns/iter", - "value": 329, - }, - { - "name": "Symbols (Execution)", - "range": "± 47", - "unit": "ns/iter", - "value": 3268, - }, - { - "name": "For loop (Execution)", - "range": "± 123", - "unit": "ns/iter", - "value": 12314, - }, - { - "name": "Fibonacci (Execution)", - "range": "± 10166", - "unit": "ns/iter", - "value": 1672496, - }, - ], - "commit": { - "author": null, - "committer": null, - "id": "123456789abcdef", - "message": "this is dummy", - "timestamp": "dummy timestamp", - "url": "https://github.com/dummy/repo", - }, - "date": 1712131503296, - "tool": "cargo", -} -`; - -exports[`extractResult() extracts benchmark output from catch2 - catch2_output_v2.txt 1`] = ` -{ - "benches": [ - { - "extra": "100 samples -208 iterations", - "name": "Fibonacci 10", - "range": "± 19", - "unit": "ns", - "value": 344, - }, - { - "extra": "100 samples -2 iterations", - "name": "Fibonacci 20", - "range": "± 3.256", - "unit": "us", - "value": 41.731, - }, - { - "extra": "100 samples -1961 iterations", - "name": "Fibonacci~ 5!", - "range": "± 4", - "unit": "ns", - "value": 36, - }, - { - "extra": "100 samples -20 iterations", - "name": "Fibonacci-15_bench", - "range": "± 362", - "unit": "us", - "value": 3.789, - }, - ], - "commit": { - "author": null, - "committer": null, - "id": "123456789abcdef", - "message": "this is dummy", - "timestamp": "dummy timestamp", - "url": "https://github.com/dummy/repo", - }, - "date": 1712131503296, - "tool": "catch2", -} -`; - -exports[`extractResult() extracts benchmark output from catch2 - catch2_output_v3.txt 1`] = ` -{ - "benches": [ - { - "extra": "100 samples -208 iterations", - "name": "Fibonacci 10", - "range": "± 19", - "unit": "ns", - "value": 344, - }, - { - "extra": "100 samples -2 iterations", - "name": "Fibonacci 20", - "range": "± 3.256", - "unit": "us", - "value": 41.731, - }, - { - "extra": "100 samples -1961 iterations", - "name": "Fibonacci~ 5!", - "range": "± 4", - "unit": "ns", - "value": 36, - }, - { - "extra": "100 samples -20 iterations", - "name": "Fibonacci-15_bench", - "range": "± 362", - "unit": "us", - "value": 3.789, - }, - ], - "commit": { - "author": null, - "committer": null, - "id": "123456789abcdef", - "message": "this is dummy", - "timestamp": "dummy timestamp", - "url": "https://github.com/dummy/repo", - }, - "date": 1712131503296, - "tool": "catch2", -} -`; - -exports[`extractResult() extracts benchmark output from catch2 - issue16_output.txt 1`] = ` -{ - "benches": [ - { - "extra": "100 samples -76353 iterations", - "name": "Fibonacci 10", - "range": "± 0", - "unit": "ns", - "value": 0, - }, - { - "extra": "100 samples -75814 iterations", - "name": "Fibonacci 20", - "range": "± 0", - "unit": "ns", - "value": 1, - }, - ], - "commit": { - "author": null, - "committer": null, - "id": "123456789abcdef", - "message": "this is dummy", - "timestamp": "dummy timestamp", - "url": "https://github.com/dummy/repo", - }, - "date": 1712131503296, - "tool": "catch2", -} -`; - -exports[`extractResult() extracts benchmark output from customBiggerIsBetter - customBiggerIsBetter_output.json 1`] = ` -{ - "benches": [ - { - "extra": undefined, - "name": "My Custom Bigger Is Better Benchmark - Throughput", - "range": undefined, - "unit": "req/s", - "value": 70, - }, - { - "extra": "Optional Value #1: 25 -Helpful Num #2: 100 -Anything Else!", - "name": "My Custom Bigger Is Better Benchmark - Free Memory", - "range": "3", - "unit": "Megabytes", - "value": 150, - }, - ], - "commit": { - "author": null, - "committer": null, - "id": "123456789abcdef", - "message": "this is dummy", - "timestamp": "dummy timestamp", - "url": "https://github.com/dummy/repo", - }, - "date": 1712131503296, - "tool": "customBiggerIsBetter", -} +exports[`extractData() extracts benchmark output from cargo - cargo_output.txt 1`] = ` +[ + { + "name": "bench_fib_10", + "platform": "ubuntu-latest", + "range": "± 24", + "unit": "ns/iter", + "value": 135, + }, + { + "name": "bench_fib_20", + "platform": "ubuntu-latest", + "range": "± 755", + "unit": "ns/iter", + "value": 18149, + }, +] `; -exports[`extractResult() extracts benchmark output from customSmallerIsBetter - customSmallerIsBetter_output.json 1`] = ` -{ - "benches": [ - { - "extra": "My Optional Information for the tooltip", - "name": "My Custom Smaller Is Better Benchmark - CPU Load", - "range": "5%", - "unit": "Percent", - "value": 50, - }, - { - "extra": undefined, - "name": "My Custom Smaller Is Better Benchmark - Memory Used", - "range": undefined, - "unit": "Megabytes", - "value": 100, - }, - ], - "commit": { - "author": null, - "committer": null, - "id": "123456789abcdef", - "message": "this is dummy", - "timestamp": "dummy timestamp", - "url": "https://github.com/dummy/repo", - }, - "date": 1712131503296, - "tool": "customSmallerIsBetter", -} +exports[`extractData() extracts benchmark output from cargo - cargo_output_units.txt 1`] = ` +[ + { + "name": "cmov", + "platform": "ubuntu-latest", + "range": "± 14", + "unit": "cycles/iter", + "value": 2835, + }, + { + "name": "cmov2", + "platform": "ubuntu-latest", + "range": "± 19", + "unit": "cycles/iter", + "value": 2845, + }, + { + "name": "mov", + "platform": "ubuntu-latest", + "range": "± 17", + "unit": "cycles/iter", + "value": 1508, + }, + { + "name": "upload", + "platform": "ubuntu-latest", + "range": "± 420", + "unit": "MS/s", + "value": 1911, + }, + { + "name": "download", + "platform": "ubuntu-latest", + "range": "± 69", + "unit": "MS/s", + "value": 9001, + }, +] `; -exports[`extractResult() extracts benchmark output from go - go_fiber_output.txt 1`] = ` -{ - "benches": [ - { - "extra": "4846809 times -4 procs", - "name": "Benchmark_Ctx_Accepts/run-[]string{".xml"}", - "unit": "ns/op 0 B/op 0 allocs/op", - "value": 247.6, - }, - { - "extra": "4846809 times -4 procs", - "name": "Benchmark_Ctx_Accepts/run-[]string{".xml"} - ns/op", - "unit": "ns/op", - "value": 247.6, - }, - { - "extra": "4846809 times -4 procs", - "name": "Benchmark_Ctx_Accepts/run-[]string{".xml"} - B/op", - "unit": "B/op", - "value": 0, - }, - { - "extra": "4846809 times -4 procs", - "name": "Benchmark_Ctx_Accepts/run-[]string{".xml"} - allocs/op", - "unit": "allocs/op", - "value": 0, - }, - { - "extra": "3900769 times -4 procs", - "name": "Benchmark_Ctx_Accepts/run-[]string{"json",_"xml"}", - "unit": "ns/op 0 B/op 0 allocs/op", - "value": 307.1, - }, - { - "extra": "3900769 times -4 procs", - "name": "Benchmark_Ctx_Accepts/run-[]string{"json",_"xml"} - ns/op", - "unit": "ns/op", - "value": 307.1, - }, - { - "extra": "3900769 times -4 procs", - "name": "Benchmark_Ctx_Accepts/run-[]string{"json",_"xml"} - B/op", - "unit": "B/op", - "value": 0, - }, - { - "extra": "3900769 times -4 procs", - "name": "Benchmark_Ctx_Accepts/run-[]string{"json",_"xml"} - allocs/op", - "unit": "allocs/op", - "value": 0, - }, - { - "extra": "5118646 times -4 procs", - "name": "Benchmark_Ctx_Accepts/run-[]string{"application/json",_"application/xml"}", - "unit": "ns/op 0 B/op 0 allocs/op", - "value": 316.1, - }, - { - "extra": "5118646 times -4 procs", - "name": "Benchmark_Ctx_Accepts/run-[]string{"application/json",_"application/xml"} - ns/op", - "unit": "ns/op", - "value": 316.1, - }, - { - "extra": "5118646 times -4 procs", - "name": "Benchmark_Ctx_Accepts/run-[]string{"application/json",_"application/xml"} - B/op", - "unit": "B/op", - "value": 0, - }, - { - "extra": "5118646 times -4 procs", - "name": "Benchmark_Ctx_Accepts/run-[]string{"application/json",_"application/xml"} - allocs/op", - "unit": "allocs/op", - "value": 0, - }, - { - "extra": "3067818 times -4 procs", - "name": "Benchmark_Utils_GetOffer/mime_extension#01", - "unit": "ns/op 48 B/op 2 allocs/op", - "value": 390.7, - }, - { - "extra": "3067818 times -4 procs", - "name": "Benchmark_Utils_GetOffer/mime_extension#01 - ns/op", - "unit": "ns/op", - "value": 390.7, - }, - { - "extra": "3067818 times -4 procs", - "name": "Benchmark_Utils_GetOffer/mime_extension#01 - B/op", - "unit": "B/op", - "value": 48, - }, - { - "extra": "3067818 times -4 procs", - "name": "Benchmark_Utils_GetOffer/mime_extension#01 - allocs/op", - "unit": "allocs/op", - "value": 2, - }, - { - "extra": "1992943 times -4 procs", - "name": "Benchmark_Utils_GetOffer/mime_extension#02", - "unit": "ns/op 48 B/op 2 allocs/op", - "value": 602.1, - }, - { - "extra": "1992943 times -4 procs", - "name": "Benchmark_Utils_GetOffer/mime_extension#02 - ns/op", - "unit": "ns/op", - "value": 602.1, - }, - { - "extra": "1992943 times -4 procs", - "name": "Benchmark_Utils_GetOffer/mime_extension#02 - B/op", - "unit": "B/op", - "value": 48, - }, - { - "extra": "1992943 times -4 procs", - "name": "Benchmark_Utils_GetOffer/mime_extension#02 - allocs/op", - "unit": "allocs/op", - "value": 2, - }, - { - "extra": "4180965 times -4 procs", - "name": "Benchmark_Utils_GetOffer/mime_extension#03", - "unit": "ns/op 0 B/op 0 allocs/op", - "value": 286.3, - }, - { - "extra": "4180965 times -4 procs", - "name": "Benchmark_Utils_GetOffer/mime_extension#03 - ns/op", - "unit": "ns/op", - "value": 286.3, - }, - { - "extra": "4180965 times -4 procs", - "name": "Benchmark_Utils_GetOffer/mime_extension#03 - B/op", - "unit": "B/op", - "value": 0, - }, - { - "extra": "4180965 times -4 procs", - "name": "Benchmark_Utils_GetOffer/mime_extension#03 - allocs/op", - "unit": "allocs/op", - "value": 0, - }, - ], - "commit": { - "author": null, - "committer": null, - "id": "123456789abcdef", - "message": "this is dummy", - "timestamp": "dummy timestamp", - "url": "https://github.com/dummy/repo", - }, - "date": 1712131503296, - "tool": "go", -} +exports[`extractData() extracts benchmark output from cargo - cargo_output2.txt 1`] = ` +[ + { + "name": "bench_engine_new", + "platform": "ubuntu-latest", + "range": "± 70126", + "unit": "ns/iter", + "value": 211834, + }, + { + "name": "bench_engine_new_raw", + "platform": "ubuntu-latest", + "range": "± 18", + "unit": "ns/iter", + "value": 197, + }, + { + "name": "bench_engine_new_raw_core", + "platform": "ubuntu-latest", + "range": "± 31", + "unit": "ns/iter", + "value": 196, + }, + { + "name": "bench_engine_register_fn", + "platform": "ubuntu-latest", + "range": "± 82", + "unit": "ns/iter", + "value": 493, + }, +] `; -exports[`extractResult() extracts benchmark output from go - go_output.txt 1`] = ` -{ - "benches": [ - { - "extra": "5000000 times -8 procs", - "name": "BenchmarkFib10", - "unit": "ns/op", - "value": 325, - }, - { - "extra": "30000 times", - "name": "BenchmarkFib20", - "unit": "ns/op", - "value": 40537.123, - }, - { - "extra": "5000000 times -8 procs", - "name": "BenchmarkFib/my_tabled_benchmark_-_10", - "unit": "ns/op", - "value": 325, - }, - { - "extra": "30000 times", - "name": "BenchmarkFib/my_tabled_benchmark_-_20", - "unit": "ns/op", - "value": 40537.123, - }, - { - "extra": "30001 times", - "name": "BenchmarkFib/my/tabled/benchmark_-_20", - "unit": "ns/op", - "value": 40537.456, - }, - { - "extra": "30001 times", - "name": "BenchmarkFib/my/tabled/benchmark_-_20,var1=13,var2=14", - "unit": "ns/op", - "value": 40537.456, - }, - { - "extra": "4587789 times -16 procs", - "name": "BenchmarkFib11", - "unit": "ns/op 3.000 auxMetricUnits", - "value": 262.7, - }, - { - "extra": "4587789 times -16 procs", - "name": "BenchmarkFib11 - ns/op", - "unit": "ns/op", - "value": 262.7, - }, - { - "extra": "4587789 times -16 procs", - "name": "BenchmarkFib11 - auxMetricUnits", - "unit": "auxMetricUnits", - "value": 3, - }, - { - "extra": "37653 times -16 procs", - "name": "BenchmarkFib22", - "unit": "ns/op 4.000 auxMetricUnits", - "value": 31915, - }, - { - "extra": "37653 times -16 procs", - "name": "BenchmarkFib22 - ns/op", - "unit": "ns/op", - "value": 31915, - }, - { - "extra": "37653 times -16 procs", - "name": "BenchmarkFib22 - auxMetricUnits", - "unit": "auxMetricUnits", - "value": 4, - }, - ], - "commit": { - "author": null, - "committer": null, - "id": "123456789abcdef", - "message": "this is dummy", - "timestamp": "dummy timestamp", - "url": "https://github.com/dummy/repo", - }, - "date": 1712131503296, - "tool": "go", -} +exports[`extractData() extracts benchmark output from cargo - cargo_output3.txt 1`] = ` +[ + { + "name": "bench_fib_10", + "platform": "ubuntu-latest", + "range": "± 2.21", + "unit": "ns/iter", + "value": 148.7, + }, + { + "name": "bench_fib_20", + "platform": "ubuntu-latest", + "range": "± 440.25", + "unit": "ns/iter", + "value": 18794.12, + }, +] `; -exports[`extractResult() extracts benchmark output from googlecpp - googlecpp_output.json 1`] = ` -{ - "benches": [ - { - "extra": "iterations: 3070566 -cpu: 213.65507206163295 ns -threads: 1", - "name": "fib_10", - "unit": "ns/iter", - "value": 214.98980114547953, - }, - { - "extra": "iterations: 23968 -cpu: 27364.90320427236 ns -threads: 1", - "name": "fib_20", - "unit": "ns/iter", - "value": 27455.600415007055, - }, - ], - "commit": { - "author": null, - "committer": null, - "id": "123456789abcdef", - "message": "this is dummy", - "timestamp": "dummy timestamp", - "url": "https://github.com/dummy/repo", - }, - "date": 1712131503296, - "tool": "googlecpp", -} +exports[`extractData() extracts benchmark output from cargo - criterion_output.txt 1`] = ` +[ + { + "name": "Create Realm", + "platform": "ubuntu-latest", + "range": "± 4", + "unit": "ns/iter", + "value": 329, + }, + { + "name": "Symbols (Execution)", + "platform": "ubuntu-latest", + "range": "± 47", + "unit": "ns/iter", + "value": 3268, + }, + { + "name": "For loop (Execution)", + "platform": "ubuntu-latest", + "range": "± 123", + "unit": "ns/iter", + "value": 12314, + }, + { + "name": "Fibonacci (Execution)", + "platform": "ubuntu-latest", + "range": "± 10166", + "unit": "ns/iter", + "value": 1672496, + }, +] `; -exports[`extractResult() extracts benchmark output from jmh - jmh_output.json 1`] = ` -{ - "benches": [ - { - "extra": "iterations: 3 -forks: 1 -threads: 1", - "name": "org.openjdk.jmh.samples.JMHSample_01_HelloWorld.wellHelloThere", - "unit": "ops/s", - "value": 3376238873.1228185, - }, - ], - "commit": { - "author": null, - "committer": null, - "id": "123456789abcdef", - "message": "this is dummy", - "timestamp": "dummy timestamp", - "url": "https://github.com/dummy/repo", - }, - "date": 1712131503296, - "tool": "jmh", -} +exports[`localWriteBenchmark() extracts benchmark output from cargo - cargo_output.txt 1`] = ` +[ + { + "name": "bench_fib_10", + "platform": "ubuntu-latest", + "range": "± 24", + "unit": "ns/iter", + "value": 135, + }, + { + "name": "bench_fib_20", + "platform": "ubuntu-latest", + "range": "± 755", + "unit": "ns/iter", + "value": 18149, + }, +] `; -exports[`extractResult() extracts benchmark output from jmh - jmh_output_2.json 1`] = ` -{ - "benches": [ - { - "extra": "iterations: 3 -forks: 1 -threads: 1", - "name": "org.openjdk.jmh.samples.JMHSample_01_HelloWorld.wellHelloThere ( {"paramA":"17","paramB":"33"} )", - "unit": "ops/s", - "value": 3376238873.1228185, - }, - ], - "commit": { - "author": null, - "committer": null, - "id": "123456789abcdef", - "message": "this is dummy", - "timestamp": "dummy timestamp", - "url": "https://github.com/dummy/repo", - }, - "date": 1712131503296, - "tool": "jmh", -} +exports[`localWriteBenchmark() extracts benchmark output from cargo - cargo_output_units.txt 1`] = ` +[ + { + "name": "cmov", + "platform": "ubuntu-latest", + "range": "± 14", + "unit": "cycles/iter", + "value": 2835, + }, + { + "name": "cmov2", + "platform": "ubuntu-latest", + "range": "± 19", + "unit": "cycles/iter", + "value": 2845, + }, + { + "name": "mov", + "platform": "ubuntu-latest", + "range": "± 17", + "unit": "cycles/iter", + "value": 1508, + }, + { + "name": "upload", + "platform": "ubuntu-latest", + "range": "± 420", + "unit": "MS/s", + "value": 1911, + }, + { + "name": "download", + "platform": "ubuntu-latest", + "range": "± 69", + "unit": "MS/s", + "value": 9001, + }, +] `; -exports[`extractResult() extracts benchmark output from julia - julia_output.json 1`] = ` -{ - "benches": [ - { - "extra": "gctime=0 -memory=0 -allocs=0 -params={"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":390,"gcsample":false,"seconds":5,"overhead":0,"memory_tolerance":0.01}", - "name": "fib/10", - "unit": "ns", - "value": 246.03846153846155, - }, - { - "extra": "gctime=0 -memory=0 -allocs=0 -params={"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5,"overhead":0,"memory_tolerance":0.01}", - "name": "fib/20", - "unit": "ns", - "value": 31028, - }, - ], - "commit": { - "author": null, - "committer": null, - "id": "123456789abcdef", - "message": "this is dummy", - "timestamp": "dummy timestamp", - "url": "https://github.com/dummy/repo", - }, - "date": 1712131503296, - "tool": "julia", -} +exports[`localWriteBenchmark() extracts benchmark output from cargo - cargo_output2.txt 1`] = ` +[ + { + "name": "bench_engine_new", + "platform": "ubuntu-latest", + "range": "± 70126", + "unit": "ns/iter", + "value": 211834, + }, + { + "name": "bench_engine_new_raw", + "platform": "ubuntu-latest", + "range": "± 18", + "unit": "ns/iter", + "value": 197, + }, + { + "name": "bench_engine_new_raw_core", + "platform": "ubuntu-latest", + "range": "± 31", + "unit": "ns/iter", + "value": 196, + }, + { + "name": "bench_engine_register_fn", + "platform": "ubuntu-latest", + "range": "± 82", + "unit": "ns/iter", + "value": 493, + }, +] `; -exports[`extractResult() extracts benchmark output from pytest - pytest_output.json 1`] = ` -{ - "benches": [ - { - "extra": "mean: 24.08868133322941 usec -rounds: 38523", - "name": "bench.py::test_fib_10", - "range": "stddev: 0.000006175090189861328", - "unit": "iter/sec", - "value": 41513.272817492856, - }, - { - "extra": "mean: 2.9850306726618627 msec -rounds: 278", - "name": "bench.py::test_fib_20", - "range": "stddev: 0.0001745301654140968", - "unit": "iter/sec", - "value": 335.0049328331567, - }, - ], - "commit": { - "author": null, - "committer": null, - "id": "123456789abcdef", - "message": "this is dummy", - "timestamp": "dummy timestamp", - "url": "https://github.com/dummy/repo", - }, - "date": 1712131503296, - "tool": "pytest", -} +exports[`localWriteBenchmark() extracts benchmark output from cargo - cargo_output3.txt 1`] = ` +[ + { + "name": "bench_fib_10", + "platform": "ubuntu-latest", + "range": "± 2.21", + "unit": "ns/iter", + "value": 148.7, + }, + { + "name": "bench_fib_20", + "platform": "ubuntu-latest", + "range": "± 440.25", + "unit": "ns/iter", + "value": 18794.12, + }, +] `; -exports[`extractResult() extracts benchmark output from pytest - pytest_several_units.json 1`] = ` -{ - "benches": [ - { - "extra": "mean: 149.95610248628836 nsec -rounds: 68536", - "name": "bench.py::test_fib_1", - "range": "stddev: 2.9351731952139377e-8", - "unit": "iter/sec", - "value": 6668618.238403659, - }, - { - "extra": "mean: 28.85754012484424 usec -rounds: 20025", - "name": "bench.py::test_fib_10", - "range": "stddev: 0.000005235937482008476", - "unit": "iter/sec", - "value": 34652.98828915334, - }, - { - "extra": "mean: 3.611916368852473 msec -rounds: 122", - "name": "bench.py::test_fib_20", - "range": "stddev: 0.0003737982822178215", - "unit": "iter/sec", - "value": 276.8613383807958, - }, - { - "extra": "mean: 2.0038430469999997 sec -rounds: 5", - "name": "bench.py::test_sleep_2", - "range": "stddev: 0.0018776587251587858", - "unit": "iter/sec", - "value": 0.49904108083570886, - }, - ], - "commit": { - "author": null, - "committer": null, - "id": "123456789abcdef", - "message": "this is dummy", - "timestamp": "dummy timestamp", - "url": "https://github.com/dummy/repo", - }, - "date": 1712131503296, - "tool": "pytest", -} +exports[`localWriteBenchmark() extracts benchmark output from cargo - criterion_output.txt 1`] = ` +[ + { + "name": "Create Realm", + "platform": "ubuntu-latest", + "range": "± 4", + "unit": "ns/iter", + "value": 329, + }, + { + "name": "Symbols (Execution)", + "platform": "ubuntu-latest", + "range": "± 47", + "unit": "ns/iter", + "value": 3268, + }, + { + "name": "For loop (Execution)", + "platform": "ubuntu-latest", + "range": "± 123", + "unit": "ns/iter", + "value": 12314, + }, + { + "name": "Fibonacci (Execution)", + "platform": "ubuntu-latest", + "range": "± 10166", + "unit": "ns/iter", + "value": 1672496, + }, +] `; diff --git a/test/buildComment.test.ts b/test/buildComment.test.ts deleted file mode 100644 index ea9431c..0000000 --- a/test/buildComment.test.ts +++ /dev/null @@ -1,209 +0,0 @@ -import { buildComment } from '../src/write'; -import { FakedOctokit, fakedRepos } from './fakedOctokit'; -import dedent from 'dedent'; - -interface RepositoryPayloadSubset { - private: boolean; - html_url: string; -} - -const gitHubContext = { - repo: { - repo: 'repo', - owner: 'user', - }, - payload: { - repository: { - private: false, - html_url: 'https://github.com/user/repo', - } as RepositoryPayloadSubset | null, - }, - workflow: 'Workflow name', -}; - -jest.mock('@actions/github', () => ({ - get context() { - return gitHubContext; - }, - getOctokit(token: string) { - return new FakedOctokit(token); - }, -})); - -afterEach(function () { - fakedRepos.clear(); -}); - -describe('buildComment', () => { - it('should build comment when previous benchmarks are 0 and biggerIsBetter returns false', () => { - const comment = buildComment( - 'TestSuite', - { - date: 12345, - commit: { - id: 'testCommitIdCurrent', - author: { - name: 'TestUser', - }, - committer: { - name: 'TestUser', - }, - message: 'Test current commit message', - url: 'https://test.previous.commit.url', - }, - tool: 'cargo', - benches: [ - { - value: 0, - name: 'TestBench<1>', - unit: 'testUnit', - }, - { - value: 1, - name: 'TestBench<2>', - unit: 'testUnit', - }, - { - value: -1, - name: 'TestBench<3>', - unit: 'testUnit', - }, - ], - }, - { - date: 12345, - commit: { - id: 'testCommitIdPrevious', - author: { - name: 'TestUser', - }, - committer: { - name: 'TestUser', - }, - message: 'Test previous commit message', - url: 'https://test.previous.commit.url', - }, - tool: 'cargo', - benches: [ - { - value: 0, - name: 'TestBench<1>', - unit: 'testUnit', - }, - { - value: 0, - name: 'TestBench<2>', - unit: 'testUnit', - }, - { - value: 0, - name: 'TestBench<3>', - unit: 'testUnit', - }, - ], - }, - ); - - expect(comment).toEqual(dedent` - # TestSuite - -
- - | Benchmark suite | Current: testCommitIdCurrent | Previous: testCommitIdPrevious | Ratio | - |-|-|-|-| - | \`TestBench<1>\` | \`0\` testUnit | \`0\` testUnit | \`1\` | - | \`TestBench<2>\` | \`1\` testUnit | \`0\` testUnit | \`+∞\` | - | \`TestBench<3>\` | \`-1\` testUnit | \`0\` testUnit | \`-∞\` | - -
- - This comment was automatically generated by [workflow](https://github.com/user/repo/actions?query=workflow%3AWorkflow%20name) using [github-action-benchmark](https://github.com/marketplace/actions/continuous-benchmark). - `); - }); - - it('should build comment when current benchmarks are 0 and biggerIsBetter returns true', () => { - const comment = buildComment( - 'TestSuite', - { - date: 12345, - commit: { - id: 'testCommitIdCurrent', - author: { - name: 'TestUser', - }, - committer: { - name: 'TestUser', - }, - message: 'Test current commit message', - url: 'https://test.previous.commit.url', - }, - tool: 'benchmarkjs', - benches: [ - { - value: 0, - name: 'TestBench<1>', - unit: 'testUnit', - }, - { - value: 0, - name: 'TestBench<2>', - unit: 'testUnit', - }, - { - value: 0, - name: 'TestBench<3>', - unit: 'testUnit', - }, - ], - }, - { - date: 12345, - commit: { - id: 'testCommitIdPrevious', - author: { - name: 'TestUser', - }, - committer: { - name: 'TestUser', - }, - message: 'Test previous commit message', - url: 'https://test.previous.commit.url', - }, - tool: 'benchmarkjs', - benches: [ - { - value: 0, - name: 'TestBench<1>', - unit: 'testUnit', - }, - { - value: 1, - name: 'TestBench<2>', - unit: 'testUnit', - }, - { - value: -1, - name: 'TestBench<3>', - unit: 'testUnit', - }, - ], - }, - ); - - expect(comment).toEqual(dedent` - # TestSuite - -
- - | Benchmark suite | Current: testCommitIdCurrent | Previous: testCommitIdPrevious | Ratio | - |-|-|-|-| - | \`TestBench<1>\` | \`0\` testUnit | \`0\` testUnit | \`1\` | - | \`TestBench<2>\` | \`0\` testUnit | \`1\` testUnit | \`+∞\` | - | \`TestBench<3>\` | \`0\` testUnit | \`-1\` testUnit | \`-∞\` | - -
- - This comment was automatically generated by [workflow](https://github.com/user/repo/actions?query=workflow%3AWorkflow%20name) using [github-action-benchmark](https://github.com/marketplace/actions/continuous-benchmark). - `); - }); -}); diff --git a/test/config.spec.ts b/test/config.spec.ts index 24094fd..8621f08 100644 --- a/test/config.spec.ts +++ b/test/config.spec.ts @@ -33,17 +33,8 @@ describe('configFromJobInput()', function () { name: 'Benchmark', tool: 'cargo', 'output-file-path': 'out.txt', - 'gh-pages-branch': 'gh-pages', - 'benchmark-data-dir-path': '.', - 'github-token': '', - 'auto-push': 'false', - 'skip-fetch-gh-pages': 'false', - 'comment-on-alert': 'false', - 'alert-threshold': '200%', - 'fail-on-alert': 'false', - 'alert-comment-cc-users': '', - 'external-data-json-path': '', - 'max-items-in-chart': '', + 'data-out-path': 'test.txt', // TODO + platform: 'any', }; const validationTests: Array<{ @@ -71,97 +62,6 @@ describe('configFromJobInput()', function () { inputs: { ...defaultInputs, 'output-file-path': '.' }, expected: /Specified path '.*' is not a file/, }, - { - what: 'wrong GitHub pages branch name', - inputs: { ...defaultInputs, 'gh-pages-branch': '' }, - expected: /^Error: Branch value must not be empty/, - }, - // Cannot check 'benchmark-data-dir-path' invalidation because it throws an error only when - // current working directory is not obtainable. - { - what: 'auto-push is set but github-token is not set', - inputs: { ...defaultInputs, 'auto-push': 'true', 'github-token': '' }, - expected: /'auto-push' is enabled but 'github-token' is not set/, - }, - { - what: 'auto-push is set to other than boolean', - inputs: { ...defaultInputs, 'auto-push': 'hello', 'github-token': 'dummy' }, - expected: /'auto-push' input must be boolean value 'true' or 'false' but got 'hello'/, - }, - { - what: 'alert-threshold does not have percentage value', - inputs: { ...defaultInputs, 'alert-threshold': '1.2' }, - expected: /'alert-threshold' input must ends with '%' for percentage value/, - }, - { - what: 'alert-threshold does not have correct percentage number', - inputs: { ...defaultInputs, 'alert-threshold': 'foo%' }, - expected: /Specified value 'foo' in 'alert-threshold' input cannot be parsed as float number/, - }, - { - what: 'comment-on-alert is set but github-token is not set', - inputs: { ...defaultInputs, 'comment-on-alert': 'true', 'github-token': '' }, - expected: /'comment-on-alert' is enabled but 'github-token' is not set/, - }, - { - what: 'user names in alert-comment-cc-users is not starting with @', - inputs: { ...defaultInputs, 'alert-comment-cc-users': '@foo,bar' }, - expected: /User name in 'alert-comment-cc-users' input must start with '@' but got 'bar'/, - }, - { - what: 'external data file is actually directory', - inputs: { ...defaultInputs, 'external-data-json-path': '.' }, - expected: /must be file but it is actually directory/, - }, - { - what: 'both external-data-json-path and auto-push are set at the same time', - inputs: { - ...defaultInputs, - 'external-data-json-path': 'external.json', - 'auto-push': 'true', - 'github-token': 'dummy', - }, - expected: /auto-push must be false when external-data-json-path is set/, - }, - { - what: 'invalid integer value for max-items-in-chart', - inputs: { - ...defaultInputs, - 'max-items-in-chart': '3.14', - }, - expected: /'max-items-in-chart' input must be unsigned integer but got '3.14'/, - }, - { - what: 'max-items-in-chart must not be zero', - inputs: { - ...defaultInputs, - 'max-items-in-chart': '0', - }, - expected: /'max-items-in-chart' input value must be one or more/, - }, - { - what: 'alert-threshold must not be empty', - inputs: { - ...defaultInputs, - 'alert-threshold': '', - }, - expected: /'alert-threshold' input must not be empty/, - }, - { - what: 'fail-threshold does not have percentage value', - inputs: { ...defaultInputs, 'fail-threshold': '1.2' }, - expected: /'fail-threshold' input must ends with '%' for percentage value/, - }, - { - what: 'fail-threshold does not have correct percentage number', - inputs: { ...defaultInputs, 'fail-threshold': 'foo%' }, - expected: /Specified value 'foo' in 'fail-threshold' input cannot be parsed as float number/, - }, - { - what: 'fail-threshold is smaller than alert-threshold', - inputs: { ...defaultInputs, 'alert-threshold': '150%', 'fail-threshold': '120%' }, - expected: /'alert-threshold' value must be smaller than 'fail-threshold' value but got 1.5 > 1.2/, - }, ]; it.each(validationTests)('validates $what', async function (test) { @@ -172,35 +72,13 @@ describe('configFromJobInput()', function () { interface ExpectedResult { name: string; tool: string; - ghPagesBranch: string; - ghRepository: string | undefined; - githubToken: string | undefined; - autoPush: boolean; - skipFetchGhPages: boolean; - commentOnAlert: boolean; - alertThreshold: number; - failOnAlert: boolean; - alertCommentCcUsers: string[]; - hasExternalDataJsonPath: boolean; - maxItemsInChart: null | number; - failThreshold: number | null; + platform: string; } const defaultExpected: ExpectedResult = { name: 'Benchmark', tool: 'cargo', - ghPagesBranch: 'gh-pages', - ghRepository: undefined, - autoPush: false, - skipFetchGhPages: false, - githubToken: undefined, - commentOnAlert: false, - alertThreshold: 2, - failOnAlert: false, - alertCommentCcUsers: [], - hasExternalDataJsonPath: false, - maxItemsInChart: null, - failThreshold: null, + platform: 'platform', }; const returnedConfigTests: Array<{ @@ -213,79 +91,11 @@ describe('configFromJobInput()', function () { inputs: { ...defaultInputs, tool }, expected: { ...defaultExpected, tool }, })), - ...( - [ - ['auto-push', 'autoPush'], - ['skip-fetch-gh-pages', 'skipFetchGhPages'], - ['comment-on-alert', 'commentOnAlert'], - ['fail-on-alert', 'failOnAlert'], - ] as const - ) - .map(([name, prop]) => - ['true', 'false'].map((v) => ({ - what: `boolean input ${name} set to '${v}'`, - inputs: { ...defaultInputs, 'github-token': 'dummy', [name]: v }, - expected: { ...defaultExpected, githubToken: 'dummy', [prop]: v === 'true' }, - })), - ) - .flat(), { what: 'with specified name', inputs: { ...defaultInputs, name: 'My Name is...' }, expected: { ...defaultExpected, name: 'My Name is...' }, }, - { - what: 'with specified GitHub Pages branch', - inputs: { ...defaultInputs, 'gh-pages-branch': 'master' }, - expected: { ...defaultExpected, ghPagesBranch: 'master' }, - }, - ...( - [ - ['150%', 1.5], - ['0%', 0], - ['123.4%', 1.234], - ] as Array<[string, number]> - ).map(([v, e]) => ({ - what: `with alert threshold ${v}`, - inputs: { ...defaultInputs, 'alert-threshold': v }, - expected: { ...defaultExpected, alertThreshold: e }, - })), - ...( - [ - ['@foo', ['@foo']], - ['@foo,@bar', ['@foo', '@bar']], - ['@foo, @bar ', ['@foo', '@bar']], - ] as Array<[string, string[]]> - ).map(([v, e]) => ({ - what: `with comment CC users ${v}`, - inputs: { ...defaultInputs, 'alert-comment-cc-users': v }, - expected: { ...defaultExpected, alertCommentCcUsers: e }, - })), - { - what: 'external JSON file', - inputs: { ...defaultInputs, 'external-data-json-path': 'external.json' }, - expected: { ...defaultExpected, hasExternalDataJsonPath: true }, - }, - { - what: 'max items in chart', - inputs: { ...defaultInputs, 'max-items-in-chart': '50' }, - expected: { ...defaultExpected, maxItemsInChart: 50 }, - }, - { - what: 'different failure threshold from alert threshold', - inputs: { ...defaultInputs, 'fail-threshold': '300%' }, - expected: { ...defaultExpected, failThreshold: 3.0 }, - }, - { - what: 'boolean value parsing an empty input as false', - inputs: { - ...defaultInputs, - 'skip-fetch-gh-pages': '', - 'comment-on-alert': '', - 'fail-on-alert': '', - }, - expected: defaultExpected, - }, ]; it.each(returnedConfigTests)('returns validated config with $what', async function (test) { @@ -293,28 +103,6 @@ describe('configFromJobInput()', function () { const actual = await configFromJobInput(); A.equal(actual.name, test.expected.name); A.equal(actual.tool, test.expected.tool); - A.equal(actual.ghPagesBranch, test.expected.ghPagesBranch); - A.equal(actual.githubToken, test.expected.githubToken); - A.equal(actual.skipFetchGhPages, test.expected.skipFetchGhPages); - A.equal(actual.commentOnAlert, test.expected.commentOnAlert); - A.equal(actual.failOnAlert, test.expected.failOnAlert); - A.equal(actual.alertThreshold, test.expected.alertThreshold); - A.deepEqual(actual.alertCommentCcUsers, test.expected.alertCommentCcUsers); - A.ok(path.isAbsolute(actual.outputFilePath), actual.outputFilePath); - A.ok(path.isAbsolute(actual.benchmarkDataDirPath), actual.benchmarkDataDirPath); - A.equal(actual.maxItemsInChart, test.expected.maxItemsInChart); - if (test.expected.failThreshold === null) { - A.equal(actual.failThreshold, test.expected.alertThreshold); - } else { - A.equal(actual.failThreshold, test.expected.failThreshold); - } - - if (test.expected.hasExternalDataJsonPath) { - A.equal(typeof actual.externalDataJsonPath, 'string'); - A.ok(path.isAbsolute(actual.externalDataJsonPath as string), actual.externalDataJsonPath); - } else { - A.equal(actual.externalDataJsonPath, undefined); - } }); it('resolves relative paths in config', async function () { @@ -329,22 +117,17 @@ describe('configFromJobInput()', function () { A.equal(config.tool, 'cargo'); A.ok(path.isAbsolute(config.outputFilePath), config.outputFilePath); A.ok(config.outputFilePath.endsWith('out.txt'), config.outputFilePath); - A.ok(path.isAbsolute(config.benchmarkDataDirPath), config.benchmarkDataDirPath); - A.ok(config.benchmarkDataDirPath.endsWith('output'), config.benchmarkDataDirPath); }); it('does not change absolute paths in config', async function () { const outFile = path.resolve('out.txt'); - const dataDir = path.resolve('path/to/output'); mockInputs({ ...defaultInputs, 'output-file-path': outFile, - 'benchmark-data-dir-path': dataDir, }); const config = await configFromJobInput(); A.equal(config.outputFilePath, outFile); - A.equal(config.benchmarkDataDirPath, dataDir); }); it('resolves home directory in output directory path', async function () { @@ -357,18 +140,14 @@ describe('configFromJobInput()', function () { const cwd = path.join('~', absCwd.slice(home.length)); const file = path.join(cwd, 'out.txt'); - const dir = path.join(cwd, 'outdir'); mockInputs({ ...defaultInputs, 'output-file-path': file, - 'benchmark-data-dir-path': dir, }); const config = await configFromJobInput(); A.ok(path.isAbsolute(config.outputFilePath), config.outputFilePath); A.equal(config.outputFilePath, path.join(absCwd, 'out.txt')); - A.ok(path.isAbsolute(config.benchmarkDataDirPath), config.benchmarkDataDirPath); - A.equal(config.benchmarkDataDirPath, path.join(absCwd, 'outdir')); }); }); diff --git a/test/data/extract/benchmarkdotnet.json b/test/data/extract/benchmarkdotnet.json deleted file mode 100644 index 7af6bf1..0000000 --- a/test/data/extract/benchmarkdotnet.json +++ /dev/null @@ -1,488 +0,0 @@ -{ - "Title": "Sample.Benchmarks-20200529-164431", - "HostEnvironmentInfo": { - "BenchmarkDotNetCaption": "BenchmarkDotNet", - "BenchmarkDotNetVersion": "0.12.1", - "OsVersion": "macOS Catalina 10.15.4 (19E287) [Darwin 19.4.0]", - "ProcessorName": "Intel Core i7-7820HQ CPU 2.90GHz (Kaby Lake)", - "PhysicalProcessorCount": 1, - "PhysicalCoreCount": 4, - "LogicalCoreCount": 8, - "RuntimeVersion": ".NET Core 3.0.0 (CoreCLR 4.700.19.46205, CoreFX 4.700.19.46214)", - "Architecture": "X64", - "HasAttachedDebugger": false, - "HasRyuJit": true, - "Configuration": "RELEASE", - "DotNetCliVersion": "3.1.201", - "ChronometerFrequency": { - "Hertz": 1000000000 - }, - "HardwareTimerKind": "Unknown" - }, - "Benchmarks": [ - { - "DisplayInfo": "Benchmarks.Fib10: DefaultJob", - "Namespace": "Sample", - "Type": "Benchmarks", - "Method": "Fib10", - "MethodTitle": "Fib10", - "Parameters": "", - "FullName": "Sample.Benchmarks.Fib10", - "Statistics": { - "OriginalValues": [ - 24.805462032556534, - 24.505721479654312, - 24.38324511051178, - 24.31508642435074, - 24.604057878255844, - 24.411026656627655, - 24.184963196516037, - 24.06129601597786, - 24.39676508307457, - 24.65345013141632, - 24.45925670862198, - 24.501324713230133, - 24.42797040939331, - 24.17329317331314 - ], - "N": 14, - "Min": 24.06129601597786, - "LowerFence": 24.073381807655096, - "Q1": 24.332126095891, - "Median": 24.419498533010483, - "Mean": 24.4202085009643, - "Q3": 24.504622288048267, - "UpperFence": 24.76336657628417, - "Max": 24.805462032556534, - "InterquartileRange": 0.17249619215726852, - "LowerOutliers": [ - 24.06129601597786 - ], - "UpperOutliers": [ - 24.805462032556534 - ], - "AllOutliers": [ - 24.06129601597786, - 24.805462032556534 - ], - "StandardError": 0.05295013214697114, - "Variance": 0.03925203092134389, - "StandardDeviation": 0.1981212530783709, - "Skewness": 0.016247051802435818, - "Kurtosis": 2.3053970890769357, - "ConfidenceInterval": { - "N": 14, - "Mean": 24.4202085009643, - "StandardError": 0.05295013214697114, - "Level": 12, - "Margin": 0.22349359775222813, - "Lower": 24.196714903212072, - "Upper": 24.643702098716528 - }, - "Percentiles": { - "P0": 24.06129601597786, - "P25": 24.332126095891, - "P50": 24.419498533010483, - "P67": 24.489124991893767, - "P80": 24.545056039094924, - "P85": 24.60652749091387, - "P90": 24.638632455468176, - "P95": 24.706654296815394, - "P100": 24.805462032556534 - } - }, - "Measurements": [ - { - "IterationMode": "Overhead", - "IterationStage": "Jitting", - "LaunchIndex": 1, - "IterationIndex": 1, - "Operations": 1, - "Nanoseconds": 337843 - }, - { - "IterationMode": "Workload", - "IterationStage": "Jitting", - "LaunchIndex": 1, - "IterationIndex": 1, - "Operations": 1, - "Nanoseconds": 221293 - }, - { - "IterationMode": "Overhead", - "IterationStage": "Jitting", - "LaunchIndex": 1, - "IterationIndex": 2, - "Operations": 16, - "Nanoseconds": 275111 - }, - { - "IterationMode": "Workload", - "IterationStage": "Jitting", - "LaunchIndex": 1, - "IterationIndex": 2, - "Operations": 16, - "Nanoseconds": 237080 - }, - { - "IterationMode": "Workload", - "IterationStage": "Pilot", - "LaunchIndex": 1, - "IterationIndex": 1, - "Operations": 16, - "Nanoseconds": 1354 - }, - { - "IterationMode": "Workload", - "IterationStage": "Pilot", - "LaunchIndex": 1, - "IterationIndex": 2, - "Operations": 32, - "Nanoseconds": 1662 - }, - { - "IterationMode": "Workload", - "IterationStage": "Pilot", - "LaunchIndex": 1, - "IterationIndex": 3, - "Operations": 64, - "Nanoseconds": 2584 - }, - { - "IterationMode": "Workload", - "IterationStage": "Pilot", - "LaunchIndex": 1, - "IterationIndex": 4, - "Operations": 128, - "Nanoseconds": 4247 - }, - { - "IterationMode": "Workload", - "IterationStage": "Pilot", - "LaunchIndex": 1, - "IterationIndex": 5, - "Operations": 256, - "Nanoseconds": 10800 - }, - { - "IterationMode": "Workload", - "IterationStage": "Pilot", - "LaunchIndex": 1, - "IterationIndex": 6, - "Operations": 512, - "Nanoseconds": 18079 - }, - { - "IterationMode": "Workload", - "IterationStage": "Pilot", - "LaunchIndex": 1, - "IterationIndex": 7, - "Operations": 1024, - "Nanoseconds": 28336 - }, - { - "IterationMode": "Workload", - "IterationStage": "Pilot", - "LaunchIndex": 1, - "IterationIndex": 8, - "Operations": 2048, - "Nanoseconds": 64383 - }, - { - "IterationMode": "Workload", - "IterationStage": "Pilot", - "LaunchIndex": 1, - "IterationIndex": 9, - "Operations": 4096, - "Nanoseconds": 110378 - }, - { - "IterationMode": "Workload", - "IterationStage": "Pilot", - "LaunchIndex": 1, - "IterationIndex": 10, - "Operations": 8192, - "Nanoseconds": 220190 - }, - { - "IterationMode": "Workload", - "IterationStage": "Pilot", - "LaunchIndex": 1, - "IterationIndex": 11, - "Operations": 16384, - "Nanoseconds": 439325 - }, - { - "IterationMode": "Workload", - "IterationStage": "Pilot", - "LaunchIndex": 1, - "IterationIndex": 12, - "Operations": 32768, - "Nanoseconds": 902024 - }, - { - "IterationMode": "Workload", - "IterationStage": "Pilot", - "LaunchIndex": 1, - "IterationIndex": 13, - "Operations": 65536, - "Nanoseconds": 1815215 - }, - { - "IterationMode": "Workload", - "IterationStage": "Pilot", - "LaunchIndex": 1, - "IterationIndex": 14, - "Operations": 131072, - "Nanoseconds": 3632048 - }, - { - "IterationMode": "Workload", - "IterationStage": "Pilot", - "LaunchIndex": 1, - "IterationIndex": 15, - "Operations": 262144, - "Nanoseconds": 7122212 - }, - { - "IterationMode": "Workload", - "IterationStage": "Pilot", - "LaunchIndex": 1, - "IterationIndex": 16, - "Operations": 524288, - "Nanoseconds": 16029873 - } - ] - }, - { - "DisplayInfo": "Benchmarks.Fib20: DefaultJob", - "Namespace": "Sample", - "Type": "Benchmarks", - "Method": "Fib20", - "MethodTitle": "Fib20", - "Parameters": "", - "FullName": "Sample.Benchmarks.Fib20", - "Statistics": { - "OriginalValues": [ - 51.17876464128494, - 51.03414672613144, - 51.7533193230629, - 52.16337263584137, - 51.40195590257645, - 50.61139816045761, - 50.41881966590881, - 50.895901918411255, - 51.100995659828186, - 52.09306198358536, - 53.3868932723999, - 51.78464049100876, - 51.93778932094574 - ], - "N": 13, - "Min": 50.41881966590881, - "LowerFence": 49.67868283390999, - "Q1": 51.03414672613144, - "Median": 51.40195590257645, - "Mean": 51.52008151549559, - "Q3": 51.93778932094574, - "UpperFence": 53.29325321316719, - "Max": 53.3868932723999, - "InterquartileRange": 0.9036425948143005, - "LowerOutliers": [], - "UpperOutliers": [ - 53.3868932723999 - ], - "AllOutliers": [ - 53.3868932723999 - ], - "StandardError": 0.21921022383775318, - "Variance": 0.624690589054972, - "StandardDeviation": 0.7903737021529575, - "Skewness": 0.7019263889809103, - "Kurtosis": 2.9306043686334626, - "ConfidenceInterval": { - "N": 13, - "Mean": 51.52008151549559, - "StandardError": 0.21921022383775318, - "Level": 12, - "Margin": 0.9465039937371241, - "Lower": 50.573577521758466, - "Upper": 52.466585509232715 - }, - "Percentiles": { - "P0": 50.41881966590881, - "P25": 51.03414672613144, - "P50": 51.40195590257645, - "P67": 51.79076644420624, - "P80": 52.03095291852951, - "P85": 52.10712411403656, - "P90": 52.14931050539016, - "P95": 52.65278089046478, - "P100": 53.3868932723999 - } - }, - "Measurements": [ - { - "IterationMode": "Overhead", - "IterationStage": "Jitting", - "LaunchIndex": 1, - "IterationIndex": 1, - "Operations": 1, - "Nanoseconds": 312333 - }, - { - "IterationMode": "Workload", - "IterationStage": "Jitting", - "LaunchIndex": 1, - "IterationIndex": 1, - "Operations": 1, - "Nanoseconds": 208793 - }, - { - "IterationMode": "Overhead", - "IterationStage": "Jitting", - "LaunchIndex": 1, - "IterationIndex": 2, - "Operations": 16, - "Nanoseconds": 239695 - }, - { - "IterationMode": "Workload", - "IterationStage": "Jitting", - "LaunchIndex": 1, - "IterationIndex": 2, - "Operations": 16, - "Nanoseconds": 221420 - }, - { - "IterationMode": "Workload", - "IterationStage": "Pilot", - "LaunchIndex": 1, - "IterationIndex": 1, - "Operations": 16, - "Nanoseconds": 1425 - }, - { - "IterationMode": "Workload", - "IterationStage": "Pilot", - "LaunchIndex": 1, - "IterationIndex": 2, - "Operations": 32, - "Nanoseconds": 2093 - }, - { - "IterationMode": "Workload", - "IterationStage": "Pilot", - "LaunchIndex": 1, - "IterationIndex": 3, - "Operations": 64, - "Nanoseconds": 3815 - }, - { - "IterationMode": "Workload", - "IterationStage": "Pilot", - "LaunchIndex": 1, - "IterationIndex": 4, - "Operations": 128, - "Nanoseconds": 7027 - }, - { - "IterationMode": "Workload", - "IterationStage": "Pilot", - "LaunchIndex": 1, - "IterationIndex": 5, - "Operations": 256, - "Nanoseconds": 13778 - }, - { - "IterationMode": "Workload", - "IterationStage": "Pilot", - "LaunchIndex": 1, - "IterationIndex": 6, - "Operations": 512, - "Nanoseconds": 27459 - }, - { - "IterationMode": "Workload", - "IterationStage": "Pilot", - "LaunchIndex": 1, - "IterationIndex": 7, - "Operations": 1024, - "Nanoseconds": 55682 - }, - { - "IterationMode": "Workload", - "IterationStage": "Pilot", - "LaunchIndex": 1, - "IterationIndex": 8, - "Operations": 2048, - "Nanoseconds": 106437 - }, - { - "IterationMode": "Workload", - "IterationStage": "Pilot", - "LaunchIndex": 1, - "IterationIndex": 9, - "Operations": 4096, - "Nanoseconds": 212168 - }, - { - "IterationMode": "Workload", - "IterationStage": "Pilot", - "LaunchIndex": 1, - "IterationIndex": 10, - "Operations": 8192, - "Nanoseconds": 437760 - }, - { - "IterationMode": "Workload", - "IterationStage": "Pilot", - "LaunchIndex": 1, - "IterationIndex": 11, - "Operations": 16384, - "Nanoseconds": 853166 - }, - { - "IterationMode": "Workload", - "IterationStage": "Pilot", - "LaunchIndex": 1, - "IterationIndex": 12, - "Operations": 32768, - "Nanoseconds": 1704807 - }, - { - "IterationMode": "Workload", - "IterationStage": "Pilot", - "LaunchIndex": 1, - "IterationIndex": 13, - "Operations": 65536, - "Nanoseconds": 3993641 - }, - { - "IterationMode": "Workload", - "IterationStage": "Pilot", - "LaunchIndex": 1, - "IterationIndex": 14, - "Operations": 131072, - "Nanoseconds": 7413920 - }, - { - "IterationMode": "Workload", - "IterationStage": "Pilot", - "LaunchIndex": 1, - "IterationIndex": 15, - "Operations": 262144, - "Nanoseconds": 15015792 - }, - { - "IterationMode": "Workload", - "IterationStage": "Pilot", - "LaunchIndex": 1, - "IterationIndex": 16, - "Operations": 524288, - "Nanoseconds": 27841809 - } - ] - } - ] -} \ No newline at end of file diff --git a/test/data/extract/benchmarkjs_output.txt b/test/data/extract/benchmarkjs_output.txt deleted file mode 100644 index cba58f3..0000000 --- a/test/data/extract/benchmarkjs_output.txt +++ /dev/null @@ -1,6 +0,0 @@ -fib(10) x 1,431,759 ops/sec ±0.74% (93 runs sampled) -fib(20) x 12,146 ops/sec ±0.32% (96 runs sampled) - -this x line should be ignored - -createObjectBuffer with 200 comments x 81.61 ops/sec ±1.70% (69 runs sampled) diff --git a/test/data/extract/benchmarkluau_output.txt b/test/data/extract/benchmarkluau_output.txt deleted file mode 100644 index 87c138e..0000000 --- a/test/data/extract/benchmarkluau_output.txt +++ /dev/null @@ -1,12 +0,0 @@ -SUCCESS: base64 : 15.041ms +/- 0.636% on luau -SUCCESS: chess : 69.560ms +/- 0.212% on luau -SUCCESS: life : 85.089ms +/- 0.187% on luau - -==================================================RESULTS================================================== -Test | Min | Average | StdDev% | Driver ------------------------+-------------+-------------+------------+-------- -base64 | 14.831ms | 15.041ms | 0.636% | luau -chess | 69.106ms | 69.560ms | 0.212% | luau -life | 84.696ms | 85.089ms | 0.187% | luau -Total | 168.630ms | 169.690ms | --- | luau ---- diff --git a/test/data/extract/catch2_output_v2.txt b/test/data/extract/catch2_output_v2.txt deleted file mode 100644 index 8365a86..0000000 --- a/test/data/extract/catch2_output_v2.txt +++ /dev/null @@ -1,47 +0,0 @@ - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Catch2_bench is a Catch v2.11.0 host application. -Run with -? for options - -------------------------------------------------------------------------------- -Fibonacci -------------------------------------------------------------------------------- -/Users/rhayasd/Develop/github.com/benchmark-action/github-action-benchmark/examples/catch2/catch2_bench.cpp:5 -............................................................................... - -benchmark name samples iterations estimated - mean low mean high mean - std dev low std dev high std dev -------------------------------------------------------------------------------- -Fibonacci 10 100 208 7.1968 ms - 344 ns 341 ns 349 ns - 19 ns 11 ns 29 ns - -Fibonacci 20 100 2 8.3712 ms - 41.731 us 41.25 us 42.622 us - 3.256 us 2.163 us 5.353 us - - -------------------------------------------------------------------------------- -More Fibonacci -------------------------------------------------------------------------------- -/Users/rhayasd/Develop/github.com/benchmark-action/github-action-benchmark/examples/catch2/catch2_bench.cpp:13 -............................................................................... - -benchmark name samples iterations estimated - mean low mean high mean - std dev low std dev high std dev -------------------------------------------------------------------------------- -Fibonacci~ 5! 100 1961 7.0596 ms - 36 ns 35 ns 37 ns - 4 ns 3 ns 6 ns - -Fibonacci-15_bench 100 20 7.48 ms - 3.789 us 3.734 us 3.888 us - 362 ns 234 ns 539 ns - - -=============================================================================== -test cases: 2 | 2 passed -assertions: - none - - diff --git a/test/data/extract/catch2_output_v3.txt b/test/data/extract/catch2_output_v3.txt deleted file mode 100644 index 5f05be7..0000000 --- a/test/data/extract/catch2_output_v3.txt +++ /dev/null @@ -1,47 +0,0 @@ - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Catch2_bench is a Catch v2.11.0 host application. -Run with -? for options - -------------------------------------------------------------------------------- -Fibonacci -------------------------------------------------------------------------------- -/Users/rhayasd/Develop/github.com/benchmark-action/github-action-benchmark/examples/catch2/catch2_bench.cpp:5 -............................................................................... - -benchmark name samples iterations est run time - mean low mean high mean - std dev low std dev high std dev -------------------------------------------------------------------------------- -Fibonacci 10 100 208 7.1968 ms - 344 ns 341 ns 349 ns - 19 ns 11 ns 29 ns - -Fibonacci 20 100 2 8.3712 ms - 41.731 us 41.25 us 42.622 us - 3.256 us 2.163 us 5.353 us - - -------------------------------------------------------------------------------- -More Fibonacci -------------------------------------------------------------------------------- -/Users/rhayasd/Develop/github.com/benchmark-action/github-action-benchmark/examples/catch2/catch2_bench.cpp:13 -............................................................................... - -benchmark name samples iterations estimated - mean low mean high mean - std dev low std dev high std dev -------------------------------------------------------------------------------- -Fibonacci~ 5! 100 1961 7.0596 ms - 36 ns 35 ns 37 ns - 4 ns 3 ns 6 ns - -Fibonacci-15_bench 100 20 7.48 ms - 3.789 us 3.734 us 3.888 us - 362 ns 234 ns 539 ns - - -=============================================================================== -test cases: 2 | 2 passed -assertions: - none - - diff --git a/test/data/extract/customBiggerIsBetter_output.json b/test/data/extract/customBiggerIsBetter_output.json deleted file mode 100644 index 9994f44..0000000 --- a/test/data/extract/customBiggerIsBetter_output.json +++ /dev/null @@ -1,14 +0,0 @@ -[ - { - "name": "My Custom Bigger Is Better Benchmark - Throughput", - "unit": "req/s", - "value": 70 - }, - { - "name": "My Custom Bigger Is Better Benchmark - Free Memory", - "unit": "Megabytes", - "value": 150, - "range": "3", - "extra": "Optional Value #1: 25\nHelpful Num #2: 100\nAnything Else!" - } -] diff --git a/test/data/extract/customSmallerIsBetter_output.json b/test/data/extract/customSmallerIsBetter_output.json deleted file mode 100644 index 86103a7..0000000 --- a/test/data/extract/customSmallerIsBetter_output.json +++ /dev/null @@ -1,14 +0,0 @@ -[ - { - "name": "My Custom Smaller Is Better Benchmark - CPU Load", - "unit": "Percent", - "value": 50, - "range": "5%", - "extra": "My Optional Information for the tooltip" - }, - { - "name": "My Custom Smaller Is Better Benchmark - Memory Used", - "unit": "Megabytes", - "value": 100 - } -] diff --git a/test/data/extract/go_fiber_output.txt b/test/data/extract/go_fiber_output.txt deleted file mode 100644 index 92c4e76..0000000 --- a/test/data/extract/go_fiber_output.txt +++ /dev/null @@ -1,6 +0,0 @@ -Benchmark_Ctx_Accepts/run-[]string{".xml"}-4 4846809 247.6 ns/op 0 B/op 0 allocs/op -Benchmark_Ctx_Accepts/run-[]string{"json",_"xml"}-4 3900769 307.1 ns/op 0 B/op 0 allocs/op -Benchmark_Ctx_Accepts/run-[]string{"application/json",_"application/xml"}-4 5118646 316.1 ns/op 0 B/op 0 allocs/op -Benchmark_Utils_GetOffer/mime_extension#01-4 3067818 390.7 ns/op 48 B/op 2 allocs/op -Benchmark_Utils_GetOffer/mime_extension#02-4 1992943 602.1 ns/op 48 B/op 2 allocs/op -Benchmark_Utils_GetOffer/mime_extension#03-4 4180965 286.3 ns/op 0 B/op 0 allocs/op diff --git a/test/data/extract/go_output.txt b/test/data/extract/go_output.txt deleted file mode 100644 index 5b5d107..0000000 --- a/test/data/extract/go_output.txt +++ /dev/null @@ -1,13 +0,0 @@ -goos: darwin -goarch: amd64 -BenchmarkFib10-8 5000000 325 ns/op -BenchmarkFib20 30000 40537.123 ns/op -BenchmarkFib/my_tabled_benchmark_-_10-8 5000000 325 ns/op -BenchmarkFib/my_tabled_benchmark_-_20 30000 40537.123 ns/op -BenchmarkFib/my/tabled/benchmark_-_20 30001 40537.456 ns/op -BenchmarkFib/my/tabled/benchmark_-_20,var1=13,var2=14 30001 40537.456 ns/op -BenchmarkFib11-16 4587789 262.7 ns/op 3.000 auxMetricUnits -BenchmarkFib22-16 37653 31915 ns/op 4.000 auxMetricUnits -PASS -PASS -ok _/Users/rhayasd/Develop/github.com/benchmark-action/github-action-benchmark/examples/go 3.614s diff --git a/test/data/extract/googlecpp_output.json b/test/data/extract/googlecpp_output.json deleted file mode 100644 index bccdea0..0000000 --- a/test/data/extract/googlecpp_output.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "context": { - "date": "2019-11-29 21:26:59", - "host_name": "Corgi.local", - "executable": "./a.out", - "num_cpus": 4, - "mhz_per_cpu": 2700, - "cpu_scaling_enabled": false, - "caches": [ - { - "type": "Data", - "level": 1, - "size": 32768, - "num_sharing": 2 - }, - { - "type": "Instruction", - "level": 1, - "size": 32768, - "num_sharing": 2 - }, - { - "type": "Unified", - "level": 2, - "size": 262144, - "num_sharing": 2 - }, - { - "type": "Unified", - "level": 3, - "size": 3145728, - "num_sharing": 4 - } - ], - "load_avg": [1.68408,1.73779,2.02783], - "library_build_type": "release" - }, - "benchmarks": [ - { - "name": "fib_10", - "run_name": "fib_10", - "run_type": "iteration", - "repetitions": 0, - "repetition_index": 0, - "threads": 1, - "iterations": 3070566, - "real_time": 2.1498980114547953e+02, - "cpu_time": 2.1365507206163295e+02, - "time_unit": "ns" - }, - { - "name": "fib_20", - "run_name": "fib_20", - "run_type": "iteration", - "repetitions": 0, - "repetition_index": 0, - "threads": 1, - "iterations": 23968, - "real_time": 2.7455600415007055e+04, - "cpu_time": 2.7364903204272359e+04, - "time_unit": "ns" - } - ] -} diff --git a/test/data/extract/invalid.txt b/test/data/extract/invalid.txt new file mode 100644 index 0000000..e69de29 diff --git a/test/data/extract/issue16_output.txt b/test/data/extract/issue16_output.txt deleted file mode 100644 index e2faa50..0000000 --- a/test/data/extract/issue16_output.txt +++ /dev/null @@ -1,28 +0,0 @@ - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -bench.exe is a Catch v2.11.1 host application. -Run with -? for options - -------------------------------------------------------------------------------- -Fibonacci -------------------------------------------------------------------------------- -D:\a\bench_action_test\bench_action_test\catch2_bench.cpp(5) -............................................................................... - -benchmark name samples iterations estimated - mean low mean high mean - std dev low std dev high std dev -------------------------------------------------------------------------------- -Fibonacci 10 100 76353 0 ns - 0 ns 0 ns 0 ns - 0 ns 0 ns 0 ns - -Fibonacci 20 100 75814 0 ns - 1 ns 1 ns 1 ns - 0 ns 0 ns 0 ns - - -=============================================================================== -test cases: 1 | 1 passed -assertions: - none - - diff --git a/test/data/extract/jmh_output.json b/test/data/extract/jmh_output.json deleted file mode 100644 index d9714b2..0000000 --- a/test/data/extract/jmh_output.json +++ /dev/null @@ -1,53 +0,0 @@ -[ - { - "jmhVersion" : "1.29", - "benchmark" : "org.openjdk.jmh.samples.JMHSample_01_HelloWorld.wellHelloThere", - "mode" : "thrpt", - "threads" : 1, - "forks" : 1, - "jvm" : "/home/mmior/.jabba/jdk/1.8.202/jre/bin/java", - "jvmArgs" : [ - ], - "jdkVersion" : "1.8.0_202", - "vmName" : "Java HotSpot(TM) 64-Bit Server VM", - "vmVersion" : "25.202-b08", - "warmupIterations" : 1, - "warmupTime" : "10 s", - "warmupBatchSize" : 1, - "measurementIterations" : 3, - "measurementTime" : "10 s", - "measurementBatchSize" : 1, - "primaryMetric" : { - "score" : 3.3762388731228185E9, - "scoreError" : 1.4287985743935993E7, - "scoreConfidence" : [ - 3.3619508873788824E9, - 3.3905268588667545E9 - ], - "scorePercentiles" : { - "0.0" : 3.375353964729294E9, - "50.0" : 3.37651988895058E9, - "90.0" : 3.376842765688582E9, - "95.0" : 3.376842765688582E9, - "99.0" : 3.376842765688582E9, - "99.9" : 3.376842765688582E9, - "99.99" : 3.376842765688582E9, - "99.999" : 3.376842765688582E9, - "99.9999" : 3.376842765688582E9, - "100.0" : 3.376842765688582E9 - }, - "scoreUnit" : "ops/s", - "rawData" : [ - [ - 3.376842765688582E9, - 3.37651988895058E9, - 3.375353964729294E9 - ] - ] - }, - "secondaryMetrics" : { - } - } -] - - diff --git a/test/data/extract/jmh_output_2.json b/test/data/extract/jmh_output_2.json deleted file mode 100644 index 882e30a..0000000 --- a/test/data/extract/jmh_output_2.json +++ /dev/null @@ -1,57 +0,0 @@ -[ - { - "jmhVersion" : "1.29", - "benchmark" : "org.openjdk.jmh.samples.JMHSample_01_HelloWorld.wellHelloThere", - "mode" : "thrpt", - "threads" : 1, - "forks" : 1, - "jvm" : "/home/mmior/.jabba/jdk/1.8.202/jre/bin/java", - "jvmArgs" : [ - ], - "jdkVersion" : "1.8.0_202", - "vmName" : "Java HotSpot(TM) 64-Bit Server VM", - "vmVersion" : "25.202-b08", - "warmupIterations" : 1, - "warmupTime" : "10 s", - "warmupBatchSize" : 1, - "measurementIterations" : 3, - "measurementTime" : "10 s", - "measurementBatchSize" : 1, - "params" : { - "paramA" : "17", - "paramB" : "33" - }, - "primaryMetric" : { - "score" : 3.3762388731228185E9, - "scoreError" : 1.4287985743935993E7, - "scoreConfidence" : [ - 3.3619508873788824E9, - 3.3905268588667545E9 - ], - "scorePercentiles" : { - "0.0" : 3.375353964729294E9, - "50.0" : 3.37651988895058E9, - "90.0" : 3.376842765688582E9, - "95.0" : 3.376842765688582E9, - "99.0" : 3.376842765688582E9, - "99.9" : 3.376842765688582E9, - "99.99" : 3.376842765688582E9, - "99.999" : 3.376842765688582E9, - "99.9999" : 3.376842765688582E9, - "100.0" : 3.376842765688582E9 - }, - "scoreUnit" : "ops/s", - "rawData" : [ - [ - 3.376842765688582E9, - 3.37651988895058E9, - 3.375353964729294E9 - ] - ] - }, - "secondaryMetrics" : { - } - } -] - - diff --git a/test/data/extract/julia_output.json b/test/data/extract/julia_output.json deleted file mode 100644 index 8ec08ac..0000000 --- a/test/data/extract/julia_output.json +++ /dev/null @@ -1,71 +0,0 @@ -[ - { - "Julia": "1.7.0-rc3", - "BenchmarkTools": "1.0.0" - }, - [ - [ - "BenchmarkGroup", - { - "data": { - "fib": [ - "BenchmarkGroup", - { - "data": { - "20": [ - "TrialEstimate", - { - "allocs": 0, - "time": 31028.0, - "memory": 0, - "params": [ - "Parameters", - { - "gctrial": true, - "time_tolerance": 0.05, - "samples": 10000, - "evals": 1, - "gcsample": false, - "seconds": 5.0, - "overhead": 0.0, - "memory_tolerance": 0.01 - } - ], - "gctime": 0.0 - } - ], - "10": [ - "TrialEstimate", - { - "allocs": 0, - "time": 246.03846153846155, - "memory": 0, - "params": [ - "Parameters", - { - "gctrial": true, - "time_tolerance": 0.05, - "samples": 10000, - "evals": 390, - "gcsample": false, - "seconds": 5.0, - "overhead": 0.0, - "memory_tolerance": 0.01 - } - ], - "gctime": 0.0 - } - ] - }, - "tags": [ - "tag1", - "tag2" - ] - } - ] - }, - "tags": [] - } - ] - ] -] diff --git a/test/data/extract/pytest_output.json b/test/data/extract/pytest_output.json deleted file mode 100644 index 611a2d1..0000000 --- a/test/data/extract/pytest_output.json +++ /dev/null @@ -1,118 +0,0 @@ -{ - "machine_info": { - "node": "Corgi.local", - "processor": "i386", - "machine": "x86_64", - "python_compiler": "Clang 11.0.0 (clang-1100.0.33.8)", - "python_implementation": "CPython", - "python_implementation_version": "3.7.5", - "python_version": "3.7.5", - "python_build": [ - "default", - "Nov 1 2019 02:16:32" - ], - "release": "18.7.0", - "system": "Darwin", - "cpu": { - "vendor_id": "GenuineIntel", - "hardware": "unknown", - "brand": "Intel(R) Core(TM) i7-4771 CPU @ 3.50GHz" - } - }, - "commit_info": { - "id": "9d55ab0434c9088746ae2d134702d782bf088644", - "time": "2019-11-16T12:10:30+09:00", - "author_time": "2019-11-16T12:10:30+09:00", - "dirty": true, - "project": "github-action-benchmark", - "branch": "pytest" - }, - "benchmarks": [ - { - "group": null, - "name": "test_fib_10", - "fullname": "bench.py::test_fib_10", - "params": null, - "param": null, - "extra_info": {}, - "options": { - "disable_gc": false, - "timer": "perf_counter", - "min_rounds": 5, - "max_time": 1.0, - "min_time": 5e-06, - "warmup": false - }, - "stats": { - "min": 2.2067000000181025e-05, - "max": 0.0003125999999999962, - "mean": 2.408868133322941e-05, - "stddev": 6.175090189861328e-06, - "rounds": 38523, - "median": 2.2300000000363696e-05, - "iqr": 1.2670000003600634e-06, - "q1": 2.2255999999831744e-05, - "q3": 2.3523000000191807e-05, - "iqr_outliers": 2896, - "stddev_outliers": 2523, - "outliers": "2523;2896", - "ld15iqr": 2.2067000000181025e-05, - "hd15iqr": 2.5426000000106086e-05, - "ops": 41513.272817492856, - "total": 0.9279682709999966, - "data": [ - 3.215300000025678e-05, - 2.8706999999794647e-05, - 2.489499999969169e-05, - 2.487300000009185e-05, - "... omitted to keep this file small" - ], - "iterations": 1 - } - }, - { - "group": null, - "name": "test_fib_20", - "fullname": "bench.py::test_fib_20", - "params": null, - "param": null, - "extra_info": {}, - "options": { - "disable_gc": false, - "timer": "perf_counter", - "min_rounds": 5, - "max_time": 1.0, - "min_time": 5e-06, - "warmup": false - }, - "stats": { - "min": 0.0027349359999999656, - "max": 0.003903139999999805, - "mean": 0.002985030672661863, - "stddev": 0.0001745301654140968, - "rounds": 278, - "median": 0.0029605825000003083, - "iqr": 0.00021968899999968983, - "q1": 0.0028489629999999266, - "q3": 0.0030686519999996165, - "iqr_outliers": 5, - "stddev_outliers": 89, - "outliers": "89;5", - "ld15iqr": 0.0027349359999999656, - "hd15iqr": 0.0034360730000000395, - "ops": 335.0049328331567, - "total": 0.8298385269999979, - "data": [ - 0.0032874509999998303, - 0.00317211499999992, - 0.0029934199999996913, - 0.00316091800000029, - "... omitted to keep this file small" - ], - "iterations": 1 - } - } - ], - "datetime": "2019-11-17T02:22:16.817862", - "version": "3.2.2" -} diff --git a/test/data/extract/pytest_several_units.json b/test/data/extract/pytest_several_units.json deleted file mode 100644 index 0e39a7b..0000000 --- a/test/data/extract/pytest_several_units.json +++ /dev/null @@ -1,193 +0,0 @@ -{ - "machine_info": { - "node": "Corgi.local", - "processor": "i386", - "machine": "x86_64", - "python_compiler": "Clang 8.1.0 (clang-802.0.42)", - "python_implementation": "CPython", - "python_implementation_version": "3.7.5", - "python_version": "3.7.5", - "python_build": ["default", "Nov 28 2019 13:59:33"], - "release": "16.7.0", - "system": "Darwin", - "cpu": { - "vendor_id": "GenuineIntel", - "hardware": "unknown", - "brand": "Intel(R) Core(TM) i5-5257U CPU @ 2.70GHz" - } - }, - "commit_info": { - "id": "b8e9e2af2493a1ee2f3fa758a9572cd9972e6db7", - "time": "2019-12-06T16:36:53+09:00", - "author_time": "2019-12-06T16:30:53+09:00", - "dirty": true, - "project": "github-action-benchmark", - "branch": "master" - }, - "benchmarks": [ - { - "group": null, - "name": "test_fib_1", - "fullname": "bench.py::test_fib_1", - "params": null, - "param": null, - "extra_info": {}, - "options": { - "disable_gc": false, - "timer": "perf_counter", - "min_rounds": 5, - "max_time": 1, - "min_time": 0.000005, - "warmup": false - }, - "stats": { - "min": 1.422099999981441e-7, - "max": 0.000001708769999999582, - "mean": 1.4995610248628836e-7, - "stddev": 2.9351731952139377e-8, - "rounds": 68536, - "median": 1.437600000020467e-7, - "iqr": 6.699999977044534e-10, - "q1": 1.4348000000108384e-7, - "q3": 1.441499999987883e-7, - "iqr_outliers": 10729, - "stddev_outliers": 1999, - "outliers": "1999;10729", - "ld15iqr": 1.4247999999827953e-7, - "hd15iqr": 1.4515999999797912e-7, - "ops": 6668618.238403659, - "total": 0.010277391440000352, - "data": [ - 1.4538999999924586e-7, - 1.4500999999889076e-7, - 1.4345999999942904e-7, - 1.4399000000109297e-7, - "... omitted to keep this file small" - ], - "iterations": 100 - } - }, - { - "group": null, - "name": "test_fib_10", - "fullname": "bench.py::test_fib_10", - "params": null, - "param": null, - "extra_info": {}, - "options": { - "disable_gc": false, - "timer": "perf_counter", - "min_rounds": 5, - "max_time": 1, - "min_time": 0.000005, - "warmup": false - }, - "stats": { - "min": 0.000027831000000144712, - "max": 0.00019098799999994753, - "mean": 0.00002885754012484424, - "stddev": 0.000005235937482008476, - "rounds": 20025, - "median": 0.000028052999999861328, - "iqr": 1.2500000012849455e-7, - "q1": 0.00002800099999999972, - "q3": 0.000028126000000128215, - "iqr_outliers": 1664, - "stddev_outliers": 510, - "outliers": "510;1664", - "ld15iqr": 0.000027831000000144712, - "hd15iqr": 0.000028314000000140283, - "ops": 34652.98828915334, - "total": 0.5778722410000059, - "data": [ - 0.00004389500000012703, - 0.00003724299999996461, - 0.000042176000000004876, - 0.000039165000000007666, - "... omitted to keep this file small" - ], - "iterations": 1 - } - }, - { - "group": null, - "name": "test_fib_20", - "fullname": "bench.py::test_fib_20", - "params": null, - "param": null, - "extra_info": {}, - "options": { - "disable_gc": false, - "timer": "perf_counter", - "min_rounds": 5, - "max_time": 1, - "min_time": 0.000005, - "warmup": false - }, - "stats": { - "min": 0.0034516560000001917, - "max": 0.005351305000000028, - "mean": 0.003611916368852473, - "stddev": 0.0003737982822178215, - "rounds": 122, - "median": 0.003471104000000169, - "iqr": 0.00010164100000009668, - "q1": 0.003456519999999852, - "q3": 0.0035581609999999486, - "iqr_outliers": 18, - "stddev_outliers": 8, - "outliers": "8;18", - "ld15iqr": 0.0034516560000001917, - "hd15iqr": 0.003772042000000031, - "ops": 276.8613383807958, - "total": 0.44065379700000173, - "data": [ - 0.0034733610000001747, - 0.005047906999999796, - 0.0035023750000000575, - 0.0035686130000001093, - "... omitted to keep this file small" - ], - "iterations": 1 - } - }, - { - "group": null, - "name": "test_sleep_2", - "fullname": "bench.py::test_sleep_2", - "params": null, - "param": null, - "extra_info": {}, - "options": { - "disable_gc": false, - "timer": "perf_counter", - "min_rounds": 5, - "max_time": 1, - "min_time": 0.000005, - "warmup": false - }, - "stats": { - "min": 2.001044877, - "max": 2.005173128, - "mean": 2.0038430469999997, - "stddev": 0.0018776587251587858, - "rounds": 5, - "median": 2.005124842999999, - "iqr": 0.002820521000000298, - "q1": 2.0023191352499996, - "q3": 2.00513965625, - "iqr_outliers": 0, - "stddev_outliers": 1, - "outliers": "1;0", - "ld15iqr": 2.001044877, - "hd15iqr": 2.005173128, - "ops": 0.49904108083570886, - "total": 10.019215234999999, - "data": [2.005173128, 2.0051284989999996, 2.005124842999999, 2.0027438879999995, 2.001044877], - "iterations": 1 - } - } - ], - "datetime": "2019-12-06T08:58:58.115974", - "version": "3.2.2" -} diff --git a/test/data/write/.gitignore b/test/data/write/.gitignore deleted file mode 100644 index 49f9590..0000000 --- a/test/data/write/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -/data-dir/data.js -/with-index-html/data.js -/new-data-dir diff --git a/test/data/write/benchmark-data-repository/data-dir/original_data.js b/test/data/write/benchmark-data-repository/data-dir/original_data.js deleted file mode 100644 index 9b5d2df..0000000 --- a/test/data/write/benchmark-data-repository/data-dir/original_data.js +++ /dev/null @@ -1,23 +0,0 @@ -window.BENCHMARK_DATA = { - "lastUpdate": 1574927128603, - "repoUrl": "https://github.com/user/repo", - "entries": { - "Test benchmark": [ - { - "commit": { - "author": { "email": "dummy@example.com", "name": "User", "username": "user" }, - "committer": { "email": "dummy@example.com", "name": "User", "username": "user" }, - "distinct": false, - "id": "prev commit id", - "message": "dummy message", - "timestamp": "dummy stamp", - "tree_id": "dummy tree id", - "url": "https://github.com/user/repo/commit/prev commit id" - }, - "date": 1574927127603, - "tool": "cargo", - "benches": [{ "name": "bench_fib_10", "range": "± 20", "unit": "ns/iter", "value": 100 }] - } - ] - } -} diff --git a/test/data/write/data-dir/original_data.js b/test/data/write/data-dir/original_data.js deleted file mode 100644 index 9b5d2df..0000000 --- a/test/data/write/data-dir/original_data.js +++ /dev/null @@ -1,23 +0,0 @@ -window.BENCHMARK_DATA = { - "lastUpdate": 1574927128603, - "repoUrl": "https://github.com/user/repo", - "entries": { - "Test benchmark": [ - { - "commit": { - "author": { "email": "dummy@example.com", "name": "User", "username": "user" }, - "committer": { "email": "dummy@example.com", "name": "User", "username": "user" }, - "distinct": false, - "id": "prev commit id", - "message": "dummy message", - "timestamp": "dummy stamp", - "tree_id": "dummy tree id", - "url": "https://github.com/user/repo/commit/prev commit id" - }, - "date": 1574927127603, - "tool": "cargo", - "benches": [{ "name": "bench_fib_10", "range": "± 20", "unit": "ns/iter", "value": 100 }] - } - ] - } -} diff --git a/test/data/write/with-index-html/index.html b/test/data/write/with-index-html/index.html deleted file mode 100644 index 51f3600..0000000 --- a/test/data/write/with-index-html/index.html +++ /dev/null @@ -1,4 +0,0 @@ - - - Hello! - diff --git a/test/data/write/with-index-html/original_data.js b/test/data/write/with-index-html/original_data.js deleted file mode 100644 index 718a4cd..0000000 --- a/test/data/write/with-index-html/original_data.js +++ /dev/null @@ -1,23 +0,0 @@ -window.BENCHMARK_DATA = { - "lastUpdate": 1574927128603, - "repoUrl": "https://github.com/user/repo", - "entries": { - "Test benchmark": [ - { - "commit": { - "author": { "email": "dummy@example.com", "name": "User", "username": "user" }, - "committer": { "email": "dummy@example.com", "name": "User", "username": "user" }, - "distinct": false, - "id": "prev commit id", - "message": "dummy message", - "timestamp": "dummy stamp", - "tree_id": "dummy tree id", - "url": "https://github.com/user/repo/commit/prev commit id" - }, - "date": 1574927127603, - "tool": "cargo", - "benches": [{ "name": "bench_fib_10", "range": "+/- 20", "unit": "ns/iter", "value": 100 }] - } - ] - } -} diff --git a/test/default_index_html.spec.ts b/test/default_index_html.spec.ts deleted file mode 100644 index 1ec1a50..0000000 --- a/test/default_index_html.spec.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { strict as A } from 'assert'; -import * as cheerio from 'cheerio'; -import { Parser as JsParser } from 'acorn'; -import { DEFAULT_INDEX_HTML } from '../src/default_index_html'; - -describe('DEFAULT_INDEX_HTML', function () { - it('is valid HTML and its script is valid as JavaScript', function () { - // Verify HTML syntax - const q = cheerio.load(DEFAULT_INDEX_HTML); - const s = q('#main-script'); - A.ok(s); - const src = s.html(); - A.ok(src); - - // Verify JavaScript syntax. It raises an error if invalid - JsParser.parse(src as string); - }); -}); diff --git a/test/extract.spec.ts b/test/extract.spec.ts index 48a5c27..c5fe36e 100644 --- a/test/extract.spec.ts +++ b/test/extract.spec.ts @@ -1,60 +1,9 @@ import * as path from 'path'; import { strict as A } from 'assert'; import { Config, ToolType } from '../src/config'; -import { extractResult } from '../src/extract'; - -const dummyWebhookPayload = { - head_commit: { - author: null, - committer: null, - id: '123456789abcdef', - message: 'this is dummy', - timestamp: 'dummy timestamp', - url: 'https://github.com/dummy/repo', - }, -} as { [key: string]: any }; -let dummyCommitData = {}; -let lastCommitRequestData = {}; -class DummyGitHub { - rest = { - repos: { - getCommit: (data: any) => { - lastCommitRequestData = data; - return { - status: 200, - data: dummyCommitData, - }; - }, - }, - }; -} -const dummyGitHubContext = { - payload: dummyWebhookPayload, - repo: { - owner: 'dummy', - repo: 'repo', - }, - ref: 'abcd1234', -}; - -jest.mock('@actions/github', () => ({ - get context() { - return dummyGitHubContext; - }, - getOctokit() { - return new DummyGitHub(); - }, -})); - -describe('extractResult()', function () { - afterAll(function () { - jest.unmock('@actions/github'); - }); - - afterEach(function () { - dummyGitHubContext.payload = dummyWebhookPayload; - }); +import { extractData, localWriteBenchmark } from '../src/extract'; +describe('extractData()', function () { const normalCases: Array<{ tool: ToolType; file: string; @@ -79,70 +28,6 @@ describe('extractResult()', function () { tool: 'cargo', file: 'criterion_output.txt', }, - { - tool: 'catch2', - file: 'catch2_output_v2.txt', - }, - { - tool: 'catch2', - file: 'catch2_output_v3.txt', - }, - { - tool: 'go', - file: 'go_output.txt', - }, - { - tool: 'go', - file: 'go_fiber_output.txt', - }, - { - tool: 'benchmarkjs', - file: 'benchmarkjs_output.txt', - }, - { - tool: 'benchmarkluau', - file: 'benchmarkluau_output.txt', - }, - { - tool: 'pytest', - file: 'pytest_output.json', - }, - { - tool: 'googlecpp', - file: 'googlecpp_output.json', - }, - { - tool: 'pytest', - file: 'pytest_several_units.json', - }, - { - tool: 'catch2', - file: 'issue16_output.txt', - }, - { - tool: 'julia', - file: 'julia_output.json', - }, - { - tool: 'jmh', - file: 'jmh_output.json', - }, - { - tool: 'jmh', - file: 'jmh_output_2.json', - }, - { - tool: 'benchmarkdotnet', - file: 'benchmarkdotnet.json', - }, - { - tool: 'customBiggerIsBetter', - file: 'customBiggerIsBetter_output.json', - }, - { - tool: 'customSmallerIsBetter', - file: 'customSmallerIsBetter_output.json', - }, ]; it.each(normalCases)(`extracts benchmark output from $tool - $file`, async function (test) { @@ -151,10 +36,11 @@ describe('extractResult()', function () { }); const outputFilePath = path.join(__dirname, 'data', 'extract', test.file); const config = { + platform: 'ubuntu-latest', tool: test.tool, outputFilePath, } as Config; - const bench = await extractResult(config); + const bench = await extractData(config); expect(bench).toMatchSnapshot(); @@ -163,199 +49,78 @@ describe('extractResult()', function () { it('raises an error on unexpected tool', async function () { const config = { + platform: 'ubuntu-latest', tool: 'foo' as any, - outputFilePath: path.join(__dirname, 'data', 'extract', 'go_output.txt'), + outputFilePath: path.join(__dirname, 'data', 'extract', 'cargo_output_units.txt'), } as Config; - await A.rejects(extractResult(config), /^Error: FATAL: Unexpected tool: 'foo'$/); + await A.rejects(extractData(config), /^Error: FATAL: Unexpected tool: 'foo'$/); }); it('raises an error when output file is not readable', async function () { const config = { - tool: 'go', + platform: 'ubuntu-latest', + tool: 'cargo', outputFilePath: 'path/does/not/exist.txt', } as Config; - await A.rejects(extractResult(config)); + await A.rejects(extractData(config)); }); it('raises an error when no output found', async function () { const config = { + platform: 'ubuntu-latest', tool: 'cargo', - outputFilePath: path.join(__dirname, 'data', 'extract', 'go_output.txt'), + outputFilePath: path.join(__dirname, 'data', 'extract', 'invalid.txt'), } as Config; - await A.rejects(extractResult(config), /^Error: No benchmark result was found in /); + await A.rejects(extractData(config), /^Error: No benchmark result was found in /); }); +}); - it.each(['pytest', 'googlecpp', 'jmh', 'customBiggerIsBetter', 'customSmallerIsBetter'] as const)( - `raises an error when output file is not in JSON with tool '%s'`, - async function (tool) { - const file = 'go_output.txt'; - // Note: go_output.txt is not in JSON format! - const outputFilePath = path.join(__dirname, 'data', 'extract', file); - const config = { tool, outputFilePath } as Config; - await A.rejects(extractResult(config), /must be JSON file/); +describe('localWriteBenchmark()', function () { + const normalCases: Array<{ + tool: ToolType; + file: string; + }> = [ + { + tool: 'cargo', + file: 'cargo_output.txt', }, - ); - - it('collects the commit information from pull_request payload as fallback', async function () { - dummyGitHubContext.payload = { - pull_request: { - title: 'this is title', - html_url: 'https://github.com/dummy/repo/pull/1', - head: { - sha: 'abcdef0123456789', - user: { - login: 'user', - }, - repo: { - updated_at: 'repo updated at timestamp', - }, - }, - }, - }; - const outputFilePath = path.join(__dirname, 'data', 'extract', 'go_output.txt'); - const config = { - tool: 'go', - outputFilePath, - } as Config; - const { commit } = await extractResult(config); - const expectedUser = { - name: 'user', - username: 'user', - }; - A.deepEqual(commit.author, expectedUser); - A.deepEqual(commit.committer, expectedUser); - A.equal(commit.id, 'abcdef0123456789'); - A.equal(commit.message, 'this is title'); - A.equal(commit.timestamp, 'repo updated at timestamp'); - A.equal(commit.url, 'https://github.com/dummy/repo/pull/1/commits/abcdef0123456789'); - }); - - it('collects the commit information from specified ref via REST API as fallback when githubToken and ref provided', async function () { - dummyGitHubContext.payload = {}; - dummyCommitData = { - author: { - login: 'testAuthorLogin', - }, - committer: { - login: 'testCommitterLogin', - }, - commit: { - author: { - name: 'test author', - date: 'author updated at timestamp', - email: 'author@testdummy.com', - }, - committer: { - name: 'test committer', - // We use the `author.date` instead. - // date: 'committer updated at timestamp', - email: 'committer@testdummy.com', - }, - message: 'test message', - }, - sha: 'abcd1234', - html_url: 'https://github.com/dymmy/repo/commit/abcd1234', - }; - const outputFilePath = path.join(__dirname, 'data', 'extract', 'go_output.txt'); - const config = { - tool: 'go', - outputFilePath, - githubToken: 'abcd1234', - ref: 'refs/pull/123/head', - } as Config; - - const { commit } = await extractResult(config); + { + tool: 'cargo', + file: 'cargo_output2.txt', + }, + { + tool: 'cargo', + file: 'cargo_output3.txt', + }, + { + tool: 'cargo', + file: 'cargo_output_units.txt', + }, + { + tool: 'cargo', + file: 'criterion_output.txt', + }, + ]; - const expectedCommit = { - id: 'abcd1234', - message: 'test message', - timestamp: 'author updated at timestamp', - url: 'https://github.com/dymmy/repo/commit/abcd1234', - author: { - name: 'test author', - username: 'testAuthorLogin', - email: 'author@testdummy.com', - }, - committer: { - name: 'test committer', - username: 'testCommitterLogin', - email: 'committer@testdummy.com', - }, - }; - A.deepEqual(lastCommitRequestData, { - owner: 'dummy', - repo: 'repo', - ref: 'refs/pull/123/head', + it.each(normalCases)(`extracts benchmark output from $tool - $file`, async function (test) { + jest.useFakeTimers({ + now: 1712131503296, }); - A.deepEqual(commit, expectedCommit); - }); - - it('collects the commit information from current head via REST API as fallback when githubToken is provided', async function () { - dummyGitHubContext.payload = {}; - dummyCommitData = { - author: { - login: 'testAuthorLogin', - }, - committer: { - login: 'testCommitterLogin', - }, - commit: { - author: { - name: 'test author', - date: 'author updated at timestamp', - email: 'author@testdummy.com', - }, - committer: { - name: 'test committer', - // We use the `author.date` instead. - // date: 'committer updated at timestamp', - email: 'committer@testdummy.com', - }, - message: 'test message', - }, - sha: 'abcd1235', - html_url: 'https://github.com/dymmy/repo/commit/abcd1234', - }; - const outputFilePath = path.join(__dirname, 'data', 'extract', 'go_output.txt'); + const outputFilePath = path.join(__dirname, 'data', 'extract', test.file); + const dataOutPath = 'test.txt'; // TODO: tempfile const config = { - tool: 'go', + platform: 'ubuntu-latest', + tool: test.tool, outputFilePath, - githubToken: 'abcd1234', + dataOutPath, } as Config; + const benches = await extractData(config); - const { commit } = await extractResult(config); + expect(benches).toMatchSnapshot(); - const expectedCommit = { - id: 'abcd1235', - message: 'test message', - timestamp: 'author updated at timestamp', - url: 'https://github.com/dymmy/repo/commit/abcd1234', - author: { - name: 'test author', - username: 'testAuthorLogin', - email: 'author@testdummy.com', - }, - committer: { - name: 'test committer', - username: 'testCommitterLogin', - email: 'committer@testdummy.com', - }, - }; - A.deepEqual(lastCommitRequestData, { - owner: 'dummy', - repo: 'repo', - ref: 'abcd1234', - }); - A.deepEqual(commit, expectedCommit); - }); + // write out + localWriteBenchmark(benches, config); - it('raises an error when commit information is not found in webhook payload and no githubToken is provided', async function () { - dummyGitHubContext.payload = {}; - const outputFilePath = path.join(__dirname, 'data', 'extract', 'go_output.txt'); - const config = { - tool: 'go', - outputFilePath, - } as Config; - await A.rejects(extractResult(config), /^Error: No commit information is found in payload/); + jest.useRealTimers(); }); }); diff --git a/test/fakedOctokit.ts b/test/fakedOctokit.ts deleted file mode 100644 index 70526f5..0000000 --- a/test/fakedOctokit.ts +++ /dev/null @@ -1,34 +0,0 @@ -type OctokitOpts = { owner: string; repo: string; commit_sha: string; body: string }; -class FakedOctokitRepos { - spyOpts: OctokitOpts[]; - constructor() { - this.spyOpts = []; - } - createCommitComment(opt: OctokitOpts) { - this.spyOpts.push(opt); - return Promise.resolve({ - status: 201, - data: { - html_url: 'https://dummy-comment-url', - }, - }); - } - lastCall(): OctokitOpts { - return this.spyOpts[this.spyOpts.length - 1]; - } - clear() { - this.spyOpts = []; - } -} - -export const fakedRepos = new FakedOctokitRepos(); - -export class FakedOctokit { - rest = { - repos: fakedRepos, - }; - opt: { token: string }; - constructor(token: string) { - this.opt = { token }; - } -} diff --git a/test/git.spec.ts b/test/git.spec.ts deleted file mode 100644 index 91e09b8..0000000 --- a/test/git.spec.ts +++ /dev/null @@ -1,220 +0,0 @@ -import { deepStrictEqual as eq, notDeepStrictEqual as neq, strict as A } from 'assert'; -import { cmd, getServerUrl, pull, push, fetch } from '../src/git'; - -interface ExecOptions { - listeners: { - stdout(b: Buffer): void; - stderr(b: Buffer): void; - }; -} - -class FakedExec { - lastArgs: [string, string[], ExecOptions] | null; - stdout: string; - stderr: string | null; - exitCode: number; - error: string | null; - - constructor() { - this.lastArgs = null; - this.stdout = 'this is test'; - this.stderr = null; - this.exitCode = 0; - this.error = null; - } - - reset() { - this.lastArgs = null; - this.stdout = 'this is test'; - this.stderr = null; - this.exitCode = 0; - this.error = null; - } -} - -const fakedExec = new FakedExec(); -const gitHubContext = { - repo: { - repo: 'repo', - owner: 'user', - }, - payload: { - repository: { - html_url: 'https://github.com/user/benchmark-action/github-action-benchmark', - }, - }, -} as { - repo: { - repo: string; - owner: string; - }; - payload: { - repository: { - html_url: string; - }; - }; -}; - -jest.mock('@actions/exec', () => ({ - exec: (c: string, a: string[], o: ExecOptions) => { - fakedExec.lastArgs = [c, a, o]; - o.listeners.stdout(Buffer.from(fakedExec.stdout)); - if (fakedExec.stderr !== null) { - o.listeners.stderr(Buffer.from(fakedExec.stderr)); - } - if (fakedExec.error === null) { - return Promise.resolve(fakedExec.exitCode); - } else { - return Promise.reject(new Error(fakedExec.error)); - } - }, -})); -jest.mock('@actions/core', () => ({ - debug: () => { - /* do nothing */ - }, -})); -jest.mock('@actions/github', () => ({ - get context() { - return gitHubContext; - }, -})); - -const ok: (x: any) => asserts x = A.ok; -const serverUrl = getServerUrl(gitHubContext.payload.repository?.html_url); -const userArgs = [ - '-c', - 'user.name=github-action-benchmark', - '-c', - 'user.email=github@users.noreply.github.com', - '-c', - `http.${serverUrl}/.extraheader=`, -]; - -describe('git', function () { - afterAll(function () { - jest.unmock('@actions/exec'); - jest.unmock('@actions/core'); - jest.unmock('@actions/github'); - }); - - afterEach(function () { - fakedExec.reset(); - }); - - describe('cmd()', function () { - it('runs Git command successfully', async function () { - const stdout = await cmd([], 'log', '--oneline'); - const args = fakedExec.lastArgs; - - eq(stdout, 'this is test'); - ok(args); - eq(args[0], 'git'); - eq(args[1], userArgs.concat(['log', '--oneline'])); - ok('listeners' in (args[2] as object)); - }); - - it('raises an error when command returns non-zero exit code', async function () { - fakedExec.exitCode = 101; - await A.rejects(() => cmd([], 'show'), /^Error: Command 'git show' failed: /); - neq(fakedExec.lastArgs, null); - }); - - it('raises an error with stderr output', async function () { - fakedExec.exitCode = 101; - fakedExec.stderr = 'this is error output!'; - await A.rejects(() => cmd([], 'show'), /this is error output!/); - }); - - it('raises an error when exec.exec() threw an error', async function () { - fakedExec.error = 'this is error from exec.exec'; - fakedExec.stderr = 'this is stderr output!'; - await A.rejects(() => cmd([], 'show'), /this is error from exec\.exec/); - await A.rejects(() => cmd([], 'show'), /this is stderr output!/); - }); - }); - - describe('push()', function () { - it('runs `git push` with given branch and options', async function () { - const stdout = await push('this-is-token', undefined, 'my-branch', [], 'opt1', 'opt2'); - const args = fakedExec.lastArgs; - - eq(stdout, 'this is test'); - ok(args); - eq(args[0], 'git'); - eq( - args[1], - userArgs.concat([ - 'push', - 'https://x-access-token:this-is-token@github.com/user/repo.git', - 'my-branch:my-branch', - '--no-verify', - 'opt1', - 'opt2', - ]), - ); - }); - }); - - describe('pull()', function () { - it('runs `git pull` with given branch and options with token', async function () { - const stdout = await pull('this-is-token', 'my-branch', [], 'opt1', 'opt2'); - const args = fakedExec.lastArgs; - - eq(stdout, 'this is test'); - ok(args); - eq(args[0], 'git'); - eq( - args[1], - userArgs.concat([ - 'pull', - 'https://x-access-token:this-is-token@github.com/user/repo.git', - 'my-branch', - 'opt1', - 'opt2', - ]), - ); - }); - - it('runs `git pull` with given branch and options without token', async function () { - const stdout = await pull(undefined, 'my-branch', [], 'opt1', 'opt2'); - const args = fakedExec.lastArgs; - - eq(stdout, 'this is test'); - ok(args); - eq(args[0], 'git'); - eq(args[1], userArgs.concat(['pull', 'origin', 'my-branch', 'opt1', 'opt2'])); - }); - }); - - describe('fetch()', function () { - it('runs `git fetch` with given branch and options with token', async function () { - const stdout = await fetch('this-is-token', 'my-branch', [], 'opt1', 'opt2'); - const args = fakedExec.lastArgs; - - eq(stdout, 'this is test'); - ok(args); - eq(args[0], 'git'); - eq( - args[1], - userArgs.concat([ - 'fetch', - 'https://x-access-token:this-is-token@github.com/user/repo.git', - 'my-branch:my-branch', - 'opt1', - 'opt2', - ]), - ); - }); - - it('runs `git fetch` with given branch and options without token', async function () { - const stdout = await fetch(undefined, 'my-branch', [], 'opt1', 'opt2'); - const args = fakedExec.lastArgs; - - eq(stdout, 'this is test'); - ok(args); - eq(args[0], 'git'); - eq(args[1], userArgs.concat(['fetch', 'origin', 'my-branch:my-branch', 'opt1', 'opt2'])); - }); - }); -}); diff --git a/test/write.spec.ts b/test/write.spec.ts deleted file mode 100644 index e88543c..0000000 --- a/test/write.spec.ts +++ /dev/null @@ -1,1345 +0,0 @@ -import * as path from 'path'; -import { promises as fs } from 'fs'; -import * as cheerio from 'cheerio'; -import markdownit from 'markdown-it'; -import rimraf from 'rimraf'; -import { Config } from '../src/config'; -import { Benchmark } from '../src/extract'; -import { DataJson, writeBenchmark } from '../src/write'; -import { expect } from '@jest/globals'; -import { FakedOctokit, fakedRepos } from './fakedOctokit'; -import { wrapBodyWithBenchmarkTags } from '../src/comment/benchmarkCommentTags'; - -const ok: (x: any, msg?: string) => asserts x = (x, msg) => { - try { - expect(x).toBeTruthy(); - } catch (err) { - if (msg) { - throw Error(msg); - } - throw err; - } -}; - -type GitFunc = 'cmd' | 'push' | 'pull' | 'fetch' | 'clone' | 'checkout'; -class GitSpy { - history: [GitFunc, unknown[]][]; - pushFailure: null | string; - pushFailureCount: number; - - constructor() { - this.history = []; - this.pushFailure = null; - this.pushFailureCount = 0; - } - - call(func: GitFunc, args: unknown[]) { - this.history.push([func, args]); - } - - clear() { - this.history = []; - this.pushFailure = null; - this.pushFailureCount = 0; - } - - mayFailPush() { - if (this.pushFailure !== null && this.pushFailureCount > 0) { - --this.pushFailureCount; - throw new Error(this.pushFailure); - } - } -} -const gitSpy = new GitSpy(); - -interface RepositoryPayloadSubset { - private: boolean; - html_url: string; -} - -const gitHubContext = { - repo: { - repo: 'repo', - owner: 'user', - }, - payload: { - repository: { - private: false, - html_url: 'https://github.com/user/repo', - } as RepositoryPayloadSubset | null, - }, - workflow: 'Workflow name', -}; - -jest.mock('@actions/core', () => ({ - debug: () => { - /* do nothing */ - }, - warning: () => { - /* do nothing */ - }, -})); -jest.mock('@actions/github', () => ({ - get context() { - return gitHubContext; - }, - getOctokit(token: string) { - return new FakedOctokit(token); - }, -})); -jest.mock('../src/git', () => ({ - ...jest.requireActual('../src/git'), - async cmd(...args: unknown[]) { - gitSpy.call('cmd', args); - return ''; - }, - async push(...args: unknown[]) { - gitSpy.call('push', args); - gitSpy.mayFailPush(); // For testing retry - return ''; - }, - async pull(...args: unknown[]) { - gitSpy.call('pull', args); - return ''; - }, - async fetch(...args: unknown[]) { - gitSpy.call('fetch', args); - return ''; - }, - async clone(...args: unknown[]) { - gitSpy.call('clone', args); - return ''; - }, - async checkout(...args: unknown[]) { - gitSpy.call('checkout', args); - return ''; - }, -})); - -describe.each(['https://github.com', 'https://github.enterprise.corp'])('writeBenchmark() - %s', function (serverUrl) { - const savedCwd = process.cwd(); - - beforeAll(function () { - process.chdir(path.join(__dirname, 'data', 'write')); - }); - - afterAll(function () { - jest.unmock('@actions/core'); - jest.unmock('@actions/github'); - jest.unmock('../src/git'); - process.chdir(savedCwd); - }); - - afterEach(function () { - fakedRepos.clear(); - }); - - // Utilities for test data - const lastUpdate = Date.now() - 10000; - const user = { - email: 'dummy@example.com', - name: 'User', - username: 'user', - }; - const repoUrl = `${serverUrl}/user/repo`; - - function commit(id = 'commit id', message = 'dummy message', u = user) { - return { - author: u, - committer: u, - distinct: false, - id, - message, - timestamp: 'dummy stamp', - tree_id: 'dummy tree id', - url: `${serverUrl}/user/repo/commit/` + id, - }; - } - - function bench(name: string, value: number, range = '± 20', unit = 'ns/iter') { - return { - name, - range, - unit, - value, - }; - } - - describe('with external json file', function () { - const dataJson = 'data.json'; - const defaultCfg: Config = { - name: 'Test benchmark', - tool: 'cargo', - outputFilePath: 'dummy', // Should not affect - ghPagesBranch: 'dummy', // Should not affect - ghRepository: undefined, - benchmarkDataDirPath: 'dummy', // Should not affect - githubToken: undefined, - autoPush: false, - skipFetchGhPages: false, // Should not affect - summaryAlways: false, - commentAlways: false, - saveDataFile: true, - commentOnAlert: false, - alertThreshold: 2.0, - failOnAlert: true, - alertCommentCcUsers: ['@user'], - externalDataJsonPath: dataJson, - maxItemsInChart: null, - failThreshold: 2.0, - ref: undefined, - }; - - const savedRepository = { - private: false, - html_url: `${serverUrl}/user/repo`, - } as RepositoryPayloadSubset | null; - - afterEach(async function () { - try { - await fs.unlink(dataJson); - } catch (_) { - // Ignore - } - gitHubContext.payload.repository = savedRepository; - }); - - const md2html = markdownit(); - - const normalCases: Array<{ - it: string; - config: Config; - data: DataJson | null; - added: Benchmark; - error?: string[]; - commitComment?: string; - repoPayload?: null | RepositoryPayloadSubset; - gitServerUrl?: string; - }> = [ - { - it: 'appends new result to existing data', - config: defaultCfg, - data: { - lastUpdate, - repoUrl, - entries: { - 'Test benchmark': [ - { - commit: commit('prev commit id'), - date: lastUpdate - 1000, - tool: 'cargo', - benches: [bench('bench_fib_10', 100)], - }, - ], - }, - }, - added: { - commit: commit('current commit id'), - date: lastUpdate, - tool: 'cargo', - benches: [bench('bench_fib_10', 135)], - }, - gitServerUrl: serverUrl, - }, - { - it: 'creates new data file', - config: defaultCfg, - data: null, - added: { - commit: commit('current commit id'), - date: lastUpdate, - tool: 'cargo', - benches: [bench('bench_fib_10', 135)], - }, - }, - { - it: 'creates new result suite to existing data file', - config: defaultCfg, - data: { - lastUpdate, - repoUrl, - entries: { - 'Other benchmark': [ - { - commit: commit('prev commit id'), - date: lastUpdate - 1000, - tool: 'cargo', - benches: [bench('bench_fib_10', 10)], - }, - ], - }, - }, - added: { - commit: commit('current commit id'), - date: lastUpdate, - tool: 'cargo', - benches: [bench('bench_fib_10', 135)], - }, - }, - { - it: 'appends new result to existing multiple benchmarks data', - config: defaultCfg, - data: { - lastUpdate, - repoUrl, - entries: { - 'Test benchmark': [ - { - commit: commit('prev commit id'), - date: lastUpdate - 1000, - tool: 'pytest', - benches: [bench('bench_fib_10', 100)], - }, - ], - 'Other benchmark': [ - { - commit: commit('prev commit id'), - date: lastUpdate - 1000, - tool: 'cargo', - benches: [bench('bench_fib_10', 10)], - }, - ], - }, - }, - added: { - commit: commit('current commit id'), - date: lastUpdate, - tool: 'pytest', - benches: [bench('bench_fib_10', 135)], - }, - }, - { - it: 'raises an alert when exceeding threshold 2.0', - config: defaultCfg, - data: { - lastUpdate, - repoUrl, - entries: { - 'Test benchmark': [ - { - commit: commit('prev commit id'), - date: lastUpdate - 1000, - tool: 'go', - benches: [bench('bench_fib_10', 100), bench('bench_fib_20', 10000)], - }, - ], - }, - }, - added: { - commit: commit('current commit id'), - date: lastUpdate, - tool: 'go', - benches: [bench('bench_fib_10', 210), bench('bench_fib_20', 25000)], // Exceeds 2.0 threshold - }, - error: [ - '# :warning: **Performance Alert** :warning:', - '', - "Possible performance regression was detected for benchmark **'Test benchmark'**.", - 'Benchmark result of this commit is worse than the previous benchmark result exceeding threshold `2`.', - '', - '| Benchmark suite | Current: current commit id | Previous: prev commit id | Ratio |', - '|-|-|-|-|', - '| `bench_fib_10` | `210` ns/iter (`± 20`) | `100` ns/iter (`± 20`) | `2.10` |', - '| `bench_fib_20` | `25000` ns/iter (`± 20`) | `10000` ns/iter (`± 20`) | `2.50` |', - '', - `This comment was automatically generated by [workflow](${serverUrl}/user/repo/actions?query=workflow%3AWorkflow%20name) using [github-action-benchmark](https://github.com/marketplace/actions/continuous-benchmark).`, - '', - 'CC: @user', - ], - }, - { - it: 'raises an alert with tool whose result value is bigger-is-better', - config: defaultCfg, - data: { - lastUpdate, - repoUrl, - entries: { - 'Test benchmark': [ - { - commit: commit('prev commit id'), - date: lastUpdate - 1000, - tool: 'benchmarkjs', - benches: [bench('benchFib10', 100, '+-20', 'ops/sec')], - }, - ], - }, - }, - added: { - commit: commit('current commit id'), - date: lastUpdate, - tool: 'benchmarkjs', - benches: [bench('benchFib10', 20, '+-20', 'ops/sec')], // ops/sec so bigger is better - }, - error: [ - '# :warning: **Performance Alert** :warning:', - '', - "Possible performance regression was detected for benchmark **'Test benchmark'**.", - 'Benchmark result of this commit is worse than the previous benchmark result exceeding threshold `2`.', - '', - '| Benchmark suite | Current: current commit id | Previous: prev commit id | Ratio |', - '|-|-|-|-|', - '| `benchFib10` | `20` ops/sec (`+-20`) | `100` ops/sec (`+-20`) | `5` |', - '', - `This comment was automatically generated by [workflow](${serverUrl}/user/repo/actions?query=workflow%3AWorkflow%20name) using [github-action-benchmark](https://github.com/marketplace/actions/continuous-benchmark).`, - '', - 'CC: @user', - ], - }, - { - it: 'raises an alert without benchmark name with default benchmark name', - config: { ...defaultCfg, name: 'Benchmark' }, - data: { - lastUpdate, - repoUrl, - entries: { - Benchmark: [ - { - commit: commit('prev commit id'), - date: lastUpdate - 1000, - tool: 'cargo', - benches: [bench('bench_fib_10', 100)], - }, - ], - }, - }, - added: { - commit: commit('current commit id'), - date: lastUpdate, - tool: 'cargo', - benches: [bench('bench_fib_10', 210)], // Exceeds 2.0 threshold - }, - error: [ - '# :warning: **Performance Alert** :warning:', - '', - 'Possible performance regression was detected for benchmark.', - 'Benchmark result of this commit is worse than the previous benchmark result exceeding threshold `2`.', - '', - '| Benchmark suite | Current: current commit id | Previous: prev commit id | Ratio |', - '|-|-|-|-|', - '| `bench_fib_10` | `210` ns/iter (`± 20`) | `100` ns/iter (`± 20`) | `2.10` |', - '', - `This comment was automatically generated by [workflow](${serverUrl}/user/repo/actions?query=workflow%3AWorkflow%20name) using [github-action-benchmark](https://github.com/marketplace/actions/continuous-benchmark).`, - '', - 'CC: @user', - ], - }, - { - it: 'raises an alert without CC names', - config: { ...defaultCfg, alertCommentCcUsers: [] }, - data: { - lastUpdate, - repoUrl, - entries: { - 'Test benchmark': [ - { - commit: commit('prev commit id'), - date: lastUpdate - 1000, - tool: 'googlecpp', - benches: [bench('bench_fib_10', 100)], - }, - ], - }, - }, - added: { - commit: commit('current commit id'), - date: lastUpdate, - tool: 'googlecpp', - benches: [bench('bench_fib_10', 210)], // Exceeds 2.0 threshold - }, - error: [ - '# :warning: **Performance Alert** :warning:', - '', - "Possible performance regression was detected for benchmark **'Test benchmark'**.", - 'Benchmark result of this commit is worse than the previous benchmark result exceeding threshold `2`.', - '', - '| Benchmark suite | Current: current commit id | Previous: prev commit id | Ratio |', - '|-|-|-|-|', - '| `bench_fib_10` | `210` ns/iter (`± 20`) | `100` ns/iter (`± 20`) | `2.10` |', - '', - `This comment was automatically generated by [workflow](${serverUrl}/user/repo/actions?query=workflow%3AWorkflow%20name) using [github-action-benchmark](https://github.com/marketplace/actions/continuous-benchmark).`, - ], - }, - { - it: 'sends commit comment on alert with GitHub API', - config: { ...defaultCfg, commentOnAlert: true, githubToken: 'dummy token' }, - data: { - lastUpdate, - repoUrl, - entries: { - 'Test benchmark': [ - { - commit: commit('prev commit id'), - date: lastUpdate - 1000, - tool: 'cargo', - benches: [bench('bench_fib_10', 100)], - }, - ], - }, - }, - added: { - commit: commit('current commit id'), - date: lastUpdate, - tool: 'cargo', - benches: [bench('bench_fib_10', 210)], // Exceeds 2.0 threshold - }, - commitComment: 'Comment was generated at https://dummy-comment-url', - }, - { - it: 'does not raise an alert when both comment-on-alert and fail-on-alert are disabled', - config: { ...defaultCfg, commentOnAlert: false, failOnAlert: false }, - data: { - lastUpdate, - repoUrl, - entries: { - 'Test benchmark': [ - { - commit: commit('prev commit id'), - date: lastUpdate - 1000, - tool: 'cargo', - benches: [bench('bench_fib_10', 100)], - }, - ], - }, - }, - added: { - commit: commit('current commit id'), - date: lastUpdate, - tool: 'cargo', - benches: [bench('bench_fib_10', 210)], // Exceeds 2.0 threshold - }, - error: undefined, - commitComment: undefined, - }, - { - it: 'ignores other bench case on detecting alerts', - config: defaultCfg, - data: { - lastUpdate, - repoUrl, - entries: { - 'Test benchmark': [ - { - commit: commit('prev commit id'), - date: lastUpdate - 1000, - tool: 'cargo', - benches: [bench('another_bench', 100)], - }, - ], - }, - }, - added: { - commit: commit('current commit id'), - date: lastUpdate, - tool: 'cargo', - benches: [bench('bench_fib_10', 210)], // Exceeds 2.0 threshold - }, - error: undefined, - commitComment: undefined, - }, - { - it: 'throws an error when GitHub token is not set (though this case should not happen in favor of validation)', - config: { ...defaultCfg, commentOnAlert: true }, - data: { - lastUpdate, - repoUrl, - entries: { - 'Test benchmark': [ - { - commit: commit('prev commit id'), - date: lastUpdate - 1000, - tool: 'cargo', - benches: [bench('bench_fib_10', 100)], - }, - ], - }, - }, - added: { - commit: commit('current commit id'), - date: lastUpdate, - tool: 'cargo', - benches: [bench('bench_fib_10', 210)], // Exceeds 2.0 threshold - }, - error: ["'comment-on-alert' input is set but 'github-token' input is not set"], - commitComment: undefined, - }, - { - it: 'truncates data items if it exceeds max-items-in-chart', - config: { ...defaultCfg, maxItemsInChart: 1 }, - data: { - lastUpdate, - repoUrl, - entries: { - 'Test benchmark': [ - { - commit: commit('prev commit id'), - date: lastUpdate - 1000, - tool: 'go', - benches: [bench('bench_fib_10', 100), bench('bench_fib_20', 10000)], - }, - ], - }, - }, - added: { - commit: commit('current commit id'), - date: lastUpdate, - tool: 'go', - benches: [bench('bench_fib_10', 210), bench('bench_fib_20', 25000)], // Exceeds 2.0 threshold - }, - // Though first item is truncated due to maxItemsInChart, alert still can be raised since previous data - // is obtained before truncating an array of data items. - error: [ - '# :warning: **Performance Alert** :warning:', - '', - "Possible performance regression was detected for benchmark **'Test benchmark'**.", - 'Benchmark result of this commit is worse than the previous benchmark result exceeding threshold `2`.', - '', - '| Benchmark suite | Current: current commit id | Previous: prev commit id | Ratio |', - '|-|-|-|-|', - '| `bench_fib_10` | `210` ns/iter (`± 20`) | `100` ns/iter (`± 20`) | `2.10` |', - '| `bench_fib_20` | `25000` ns/iter (`± 20`) | `10000` ns/iter (`± 20`) | `2.50` |', - '', - `This comment was automatically generated by [workflow](${serverUrl}/user/repo/actions?query=workflow%3AWorkflow%20name) using [github-action-benchmark](https://github.com/marketplace/actions/continuous-benchmark).`, - '', - 'CC: @user', - ], - }, - { - it: 'changes title when threshold is zero which means comment always happens', - config: { ...defaultCfg, alertThreshold: 0, failThreshold: 0 }, - data: { - lastUpdate, - repoUrl, - entries: { - 'Test benchmark': [ - { - commit: commit('prev commit id'), - date: lastUpdate - 1000, - tool: 'benchmarkjs', - benches: [bench('benchFib10', 100, '+-20', 'ops/sec')], - }, - ], - }, - }, - added: { - commit: commit('current commit id'), - date: lastUpdate, - tool: 'benchmarkjs', - benches: [bench('benchFib10', 100, '+-20', 'ops/sec')], - }, - error: [ - '# Performance Report', - '', - "Possible performance regression was detected for benchmark **'Test benchmark'**.", - 'Benchmark result of this commit is worse than the previous benchmark result exceeding threshold `0`.', - '', - '| Benchmark suite | Current: current commit id | Previous: prev commit id | Ratio |', - '|-|-|-|-|', - '| `benchFib10` | `100` ops/sec (`+-20`) | `100` ops/sec (`+-20`) | `1` |', - '', - `This comment was automatically generated by [workflow](${serverUrl}/user/repo/actions?query=workflow%3AWorkflow%20name) using [github-action-benchmark](https://github.com/marketplace/actions/continuous-benchmark).`, - '', - 'CC: @user', - ], - }, - { - it: 'raises an alert with different failure threshold from alert threshold', - config: { ...defaultCfg, failThreshold: 3 }, - data: { - lastUpdate, - repoUrl, - entries: { - 'Test benchmark': [ - { - commit: commit('prev commit id'), - date: lastUpdate - 1000, - tool: 'go', - benches: [bench('bench_fib_10', 100)], - }, - ], - }, - }, - added: { - commit: commit('current commit id'), - date: lastUpdate, - tool: 'go', - benches: [bench('bench_fib_10', 350)], // Exceeds 3.0 failure threshold - }, - error: [ - '1 of 1 alerts exceeded the failure threshold `3` specified by fail-threshold input:', - '', - '# :warning: **Performance Alert** :warning:', - '', - "Possible performance regression was detected for benchmark **'Test benchmark'**.", - 'Benchmark result of this commit is worse than the previous benchmark result exceeding threshold `2`.', - '', - '| Benchmark suite | Current: current commit id | Previous: prev commit id | Ratio |', - '|-|-|-|-|', - '| `bench_fib_10` | `350` ns/iter (`± 20`) | `100` ns/iter (`± 20`) | `3.50` |', - '', - `This comment was automatically generated by [workflow](${serverUrl}/user/repo/actions?query=workflow%3AWorkflow%20name) using [github-action-benchmark](https://github.com/marketplace/actions/continuous-benchmark).`, - '', - 'CC: @user', - ], - }, - { - it: 'does not raise an alert when not exceeding failure threshold', - config: { ...defaultCfg, failThreshold: 3 }, - data: { - lastUpdate, - repoUrl, - entries: { - 'Test benchmark': [ - { - commit: commit('prev commit id'), - date: lastUpdate - 1000, - tool: 'go', - benches: [bench('bench_fib_10', 100)], - }, - ], - }, - }, - added: { - commit: commit('current commit id'), - date: lastUpdate, - tool: 'go', - benches: [bench('bench_fib_10', 210)], // Exceeds 2.0 threshold - }, - error: undefined, - }, - ]; - - it.each(normalCases)('$it', async function (t) { - gitHubContext.payload.repository = { - private: false, - html_url: `${serverUrl}/user/repo`, - } as RepositoryPayloadSubset | null; - - if (t.repoPayload !== undefined) { - gitHubContext.payload.repository = t.repoPayload; - } - if (t.data !== null) { - await fs.writeFile(dataJson, JSON.stringify(t.data), 'utf8'); - } - - let caughtError: Error | null = null; - try { - await writeBenchmark(t.added, t.config); - } catch (err: any) { - if (!t.error && !t.commitComment) { - throw err; - } - caughtError = err; - } - - const json: DataJson = JSON.parse(await fs.readFile(dataJson, 'utf8')); - - expect('number').toEqual(typeof json.lastUpdate); - expect(json.entries[t.config.name]).toBeTruthy(); - const len = json.entries[t.config.name].length; - ok(len > 0); - expect(t.added).toEqual(json.entries[t.config.name][len - 1]); // Check last item is the newest - - if (t.data !== null) { - ok(json.lastUpdate > t.data.lastUpdate); - expect(t.data.repoUrl).toEqual(json.repoUrl); - for (const name of Object.keys(t.data.entries)) { - const entries = t.data.entries[name]; - if (name === t.config.name) { - if (t.config.maxItemsInChart === null || len < t.config.maxItemsInChart) { - expect(entries.length + 1).toEqual(len); - // Check benchmark data except for the last appended one are not modified - expect(entries).toEqual(json.entries[name].slice(0, -1)); - } else { - // When data items was truncated due to max-items-in-chart - expect(entries.length).toEqual(len); // Number of items did not change because first item was shifted - expect(entries.slice(1)).toEqual(json.entries[name].slice(0, -1)); - } - } else { - expect(entries).toEqual(json.entries[name]); // eq(json.entries[name], entries, name); - } - } - } - - if (t.error) { - ok(caughtError); - const expected = t.error.join('\n'); - expect(caughtError.message).toEqual(expected); - } - - if (t.commitComment !== undefined) { - ok(caughtError); - // Last line is appended only for failure message - const messageLines = caughtError.message.split('\n'); - ok(messageLines.length > 0); - const expectedMessage = wrapBodyWithBenchmarkTags( - 'Test benchmark Alert', - messageLines.slice(0, -1).join('\n'), - ); - ok(fakedRepos.spyOpts.length > 0, `len: ${fakedRepos.spyOpts.length}, caught: ${caughtError.message}`); - const opts = fakedRepos.lastCall(); - expect('user').toEqual(opts.owner); - expect('repo').toEqual(opts.repo); - expect('current commit id').toEqual(opts.commit_sha); - expect(expectedMessage).toEqual(opts.body); - const commentLine = messageLines[messageLines.length - 1]; - expect(t.commitComment).toEqual(commentLine); - - // Check the body is a correct markdown document by markdown parser - // Validate markdown content via HTML - // TODO: Use Markdown AST instead of DOM API - const html = md2html.render(opts.body); - const query = cheerio.load(html); - - const h1 = query('h1'); - expect(1).toEqual(h1.length); - expect(':warning: Performance Alert :warning:').toEqual(h1.text()); - - const tr = query('tbody tr'); - expect(t.added.benches.length).toEqual(tr.length); - - const a = query('a'); - expect(2).toEqual(a.length); - - const workflowLink = a.first(); - expect('workflow').toEqual(workflowLink.text()); - const workflowUrl = workflowLink.attr('href'); - ok(workflowUrl?.startsWith(json.repoUrl), workflowUrl); - - const actionLink = a.last(); - expect('github-action-benchmark').toEqual(actionLink.text()); - expect('https://github.com/marketplace/actions/continuous-benchmark').toEqual(actionLink.attr('href')); - } - }); - }); - - // Tests for updating GitHub Pages branch - describe('with gh-pages branch', function () { - beforeEach(async function () { - (global as any).window = {}; // Fake window object on browser - }); - afterEach(async function () { - gitSpy.clear(); - delete (global as any).window; - for (const p of [ - path.join('data-dir', 'data.js'), - path.join('data-dir', 'index.html'), - 'new-data-dir', - path.join('with-index-html', 'data.js'), - path.join('benchmark-data-repository', 'data-dir', 'data.js'), - path.join('benchmark-data-repository', 'data-dir', 'index.html'), - path.join('benchmark-data-repository', 'new-data-dir'), - ]) { - // Ignore exception - await new Promise((resolve) => rimraf(p, resolve)); - } - }); - - async function isFile(p: string) { - try { - const s = await fs.stat(p); - return s.isFile(); - } catch (_) { - return false; - } - } - - async function isDir(p: string) { - try { - const s = await fs.stat(p); - return s.isDirectory(); - } catch (_) { - return false; - } - } - - async function loadDataJs(dataDir: string, serverUrl: string) { - const dataJs = path.join(dataDir, 'data.js'); - if (!(await isDir(dataDir)) || !(await isFile(dataJs))) { - return null; - } - let dataSource = await fs.readFile(dataJs, 'utf8'); - if (serverUrl !== 'https://github.com') { - dataSource = dataSource.replace(/https:\/\/github.com/gm, serverUrl); - } - eval(dataSource); - return (global as any).window.BENCHMARK_DATA as DataJson; - } - - const defaultCfg: Config = { - name: 'Test benchmark', - tool: 'cargo', - outputFilePath: 'dummy', // Should not affect - ghPagesBranch: 'gh-pages', - ghRepository: undefined, - benchmarkDataDirPath: 'data-dir', // Should not affect - githubToken: 'dummy token', - autoPush: true, - skipFetchGhPages: false, // Should not affect - commentAlways: false, - summaryAlways: false, - saveDataFile: true, - commentOnAlert: false, - alertThreshold: 2.0, - failOnAlert: true, - alertCommentCcUsers: [], - externalDataJsonPath: undefined, - maxItemsInChart: null, - failThreshold: 2.0, - ref: undefined, - }; - - function gitHistory( - cfg: { - dir?: string; - addIndexHtml?: boolean; - autoPush?: boolean; - token?: string | undefined; - fetch?: boolean; - skipFetch?: boolean; - } = {}, - ): [GitFunc, unknown[]][] { - const dir = cfg.dir ?? 'data-dir'; - const token = 'token' in cfg ? cfg.token : 'dummy token'; - const fetch = cfg.fetch ?? true; - const addIndexHtml = cfg.addIndexHtml ?? true; - const autoPush = cfg.autoPush ?? true; - const skipFetch = cfg.skipFetch ?? false; - const hist: Array<[GitFunc, unknown[]] | undefined> = [ - skipFetch ? undefined : ['fetch', [token, 'gh-pages']], - ['cmd', [[], 'switch', 'gh-pages']], - fetch ? ['pull', [token, 'gh-pages']] : undefined, - ['cmd', [[], 'add', path.join(dir, 'data.js')]], - addIndexHtml ? ['cmd', [[], 'add', path.join(dir, 'index.html')]] : undefined, - ['cmd', [[], 'commit', '-m', 'add Test benchmark (cargo) benchmark result for current commit id']], - autoPush ? ['push', [token, undefined, 'gh-pages', []]] : undefined, - ['cmd', [[], 'checkout', '-']], // Return from gh-pages - ]; - return hist.filter((x: [GitFunc, unknown[]] | undefined): x is [GitFunc, unknown[]] => x !== undefined); - } - - const normalCases: Array<{ - it: string; - config: Config; - added: Benchmark; - gitServerUrl: string; - gitHistory: [GitFunc, unknown[]][]; - privateRepo?: boolean; - error?: string[]; - expectedDataBaseDirectory?: string; - }> = [ - { - it: 'appends new data', - config: defaultCfg, - added: { - commit: commit('current commit id'), - date: lastUpdate, - tool: 'cargo', - benches: [bench('bench_fib_10', 135)], - }, - gitServerUrl: serverUrl, - gitHistory: gitHistory(), - }, - { - it: 'creates new data file', - config: { ...defaultCfg, benchmarkDataDirPath: 'new-data-dir' }, - added: { - commit: commit('current commit id'), - date: lastUpdate, - tool: 'cargo', - benches: [bench('bench_fib_10', 135)], - }, - gitServerUrl: serverUrl, - gitHistory: gitHistory({ dir: 'new-data-dir' }), - }, - { - it: 'appends new data in other repository', - config: { - ...defaultCfg, - ghRepository: 'https://github.com/user/other-repo', - benchmarkDataDirPath: 'data-dir', - }, - added: { - commit: commit('current commit id'), - date: lastUpdate, - tool: 'cargo', - benches: [bench('bench_fib_10', 135)], - }, - gitServerUrl: serverUrl, - gitHistory: [ - ['clone', ['dummy token', 'https://github.com/user/other-repo', './benchmark-data-repository']], - [ - 'checkout', - [ - 'gh-pages', - ['--work-tree=./benchmark-data-repository', '--git-dir=./benchmark-data-repository/.git'], - ], - ], - [ - 'cmd', - [ - ['--work-tree=./benchmark-data-repository', '--git-dir=./benchmark-data-repository/.git'], - 'add', - path.join('data-dir', 'data.js'), - ], - ], - [ - 'cmd', - [ - ['--work-tree=./benchmark-data-repository', '--git-dir=./benchmark-data-repository/.git'], - 'add', - path.join('data-dir', 'index.html'), - ], - ], - [ - 'cmd', - [ - ['--work-tree=./benchmark-data-repository', '--git-dir=./benchmark-data-repository/.git'], - 'commit', - '-m', - 'add Test benchmark (cargo) benchmark result for current commit id', - ], - ], - [ - 'push', - [ - 'dummy token', - 'https://github.com/user/other-repo', - 'gh-pages', - ['--work-tree=./benchmark-data-repository', '--git-dir=./benchmark-data-repository/.git'], - ], - ], - ], - expectedDataBaseDirectory: 'benchmark-data-repository', - }, - { - it: 'creates new data file in other repository', - config: { - ...defaultCfg, - ghRepository: 'https://github.com/user/other-repo', - }, - added: { - commit: commit('current commit id'), - date: lastUpdate, - tool: 'cargo', - benches: [bench('bench_fib_10', 135)], - }, - gitServerUrl: serverUrl, - gitHistory: [ - ['clone', ['dummy token', 'https://github.com/user/other-repo', './benchmark-data-repository']], - [ - 'checkout', - [ - 'gh-pages', - ['--work-tree=./benchmark-data-repository', '--git-dir=./benchmark-data-repository/.git'], - ], - ], - [ - 'cmd', - [ - ['--work-tree=./benchmark-data-repository', '--git-dir=./benchmark-data-repository/.git'], - 'add', - path.join('data-dir', 'data.js'), - ], - ], - [ - 'cmd', - [ - ['--work-tree=./benchmark-data-repository', '--git-dir=./benchmark-data-repository/.git'], - 'add', - path.join('data-dir', 'index.html'), - ], - ], - [ - 'cmd', - [ - ['--work-tree=./benchmark-data-repository', '--git-dir=./benchmark-data-repository/.git'], - 'commit', - '-m', - 'add Test benchmark (cargo) benchmark result for current commit id', - ], - ], - [ - 'push', - [ - 'dummy token', - 'https://github.com/user/other-repo', - 'gh-pages', - ['--work-tree=./benchmark-data-repository', '--git-dir=./benchmark-data-repository/.git'], - ], - ], - ], - expectedDataBaseDirectory: 'benchmark-data-repository', - }, - { - it: 'creates new suite in data', - config: defaultCfg, - added: { - commit: commit('current commit id'), - date: lastUpdate, - tool: 'cargo', - benches: [bench('other_bench_foo', 100)], - }, - gitServerUrl: serverUrl, - gitHistory: gitHistory(), - }, - { - it: 'does not create index.html if it already exists', - config: { ...defaultCfg, benchmarkDataDirPath: 'with-index-html' }, - added: { - commit: commit('current commit id'), - date: lastUpdate, - tool: 'cargo', - benches: [bench('bench_fib_10', 100)], - }, - gitServerUrl: serverUrl, - gitHistory: gitHistory({ dir: 'with-index-html', addIndexHtml: false }), - }, - { - it: 'does not push to remote when auto-push is off', - config: { ...defaultCfg, autoPush: false }, - added: { - commit: commit('current commit id'), - date: lastUpdate, - tool: 'cargo', - benches: [bench('bench_fib_10', 135)], - }, - gitServerUrl: serverUrl, - gitHistory: gitHistory({ autoPush: false }), - }, - { - it: 'does not push to remote when auto-push is off without token', - config: { ...defaultCfg, autoPush: false, githubToken: undefined }, - added: { - commit: commit('current commit id'), - date: lastUpdate, - tool: 'cargo', - benches: [bench('bench_fib_10', 135)], - }, - gitServerUrl: serverUrl, - gitHistory: gitHistory({ autoPush: false, token: undefined }), - }, - { - it: 'does not fetch remote when github-token is not set for private repo', - config: { ...defaultCfg, autoPush: false, githubToken: undefined }, - added: { - commit: commit('current commit id'), - date: lastUpdate, - tool: 'cargo', - benches: [bench('bench_fib_10', 135)], - }, - gitServerUrl: serverUrl, - gitHistory: gitHistory({ autoPush: false, token: undefined, fetch: false }), - privateRepo: true, - }, - { - it: 'does not fetch remote when skip-fetch-gh-pages is enabled', - config: { ...defaultCfg, skipFetchGhPages: true }, - added: { - commit: commit('current commit id'), - date: lastUpdate, - tool: 'cargo', - benches: [bench('bench_fib_10', 135)], - }, - gitServerUrl: serverUrl, - gitHistory: gitHistory({ fetch: false, skipFetch: true }), - }, - { - it: 'fails when exceeding the threshold', - config: defaultCfg, - added: { - commit: commit('current commit id'), - date: lastUpdate, - tool: 'cargo', - benches: [bench('bench_fib_10', 210)], // Exceeds 2.0 threshold - }, - gitServerUrl: serverUrl, - gitHistory: gitHistory(), - error: [ - '# :warning: **Performance Alert** :warning:', - '', - "Possible performance regression was detected for benchmark **'Test benchmark'**.", - 'Benchmark result of this commit is worse than the previous benchmark result exceeding threshold `2`.', - '', - '| Benchmark suite | Current: current commit id | Previous: prev commit id | Ratio |', - '|-|-|-|-|', - '| `bench_fib_10` | `210` ns/iter (`± 20`) | `100` ns/iter (`± 20`) | `2.10` |', - '', - `This comment was automatically generated by [workflow](${serverUrl}/user/repo/actions?query=workflow%3AWorkflow%20name) using [github-action-benchmark](https://github.com/marketplace/actions/continuous-benchmark).`, - ], - }, - { - it: 'sends commit message but does not raise an error when exceeding alert threshold but not exceeding failure threshold', - config: { - ...defaultCfg, - commentOnAlert: true, - githubToken: 'dummy token', - alertThreshold: 2, - failThreshold: 3, - }, - added: { - commit: commit('current commit id'), - date: lastUpdate, - tool: 'cargo', - benches: [bench('bench_fib_10', 210)], // Exceeds 2.0 threshold but not exceed 3.0 threshold - }, - gitServerUrl: serverUrl, - gitHistory: gitHistory(), - error: undefined, - }, - ]; - for (const t of normalCases) { - // FIXME: can't use `it.each` currently as tests running in parallel interfere with each other - it(t.it, async function () { - if (t.privateRepo) { - gitHubContext.payload.repository = gitHubContext.payload.repository - ? { ...gitHubContext.payload.repository, private: true } - : null; - } - - const dataDirPath = path.join(t.expectedDataBaseDirectory ?? '', t.config.benchmarkDataDirPath); - const originalDataJs = path.join(dataDirPath, 'original_data.js'); - const dataJs = path.join(dataDirPath, 'data.js'); - const indexHtml = path.join(dataDirPath, 'index.html'); - - if (await isFile(originalDataJs)) { - await fs.copyFile(originalDataJs, dataJs); - } - - let indexHtmlBefore = null; - try { - indexHtmlBefore = await fs.readFile(indexHtml); - } catch (_) { - // Ignore - } - - let caughtError: Error | null = null; - const beforeData = await loadDataJs(dataDirPath, t.gitServerUrl); - const beforeDate = Date.now(); - try { - await writeBenchmark(t.added, t.config); - } catch (err: any) { - if (t.error === undefined) { - throw err; - } - caughtError = err; - } - - if (t.error) { - ok(caughtError); - const expected = t.error.join('\n'); - expect(caughtError.message).toEqual(expected); - return; - } - - // Post condition checks for success cases - - const afterDate = Date.now(); - - expect(gitSpy.history).toEqual(t.gitHistory); - - ok(await isDir(dataDirPath)); - ok(await isFile(path.join(dataDirPath, 'index.html'))); - ok(await isFile(dataJs)); - - const data = await loadDataJs(dataDirPath, t.gitServerUrl); - ok(data); - - expect('number').toEqual(typeof data.lastUpdate); - ok( - beforeDate <= data.lastUpdate && data.lastUpdate <= afterDate, - `Should be ${beforeDate} <= ${data.lastUpdate} <= ${afterDate}`, - ); - ok(data.entries[t.config.name]); - const len = data.entries[t.config.name].length; - ok(len > 0); - expect(t.added).toEqual(data.entries[t.config.name][len - 1]); // Check last item is the newest - - if (beforeData !== null) { - expect(data.repoUrl).toEqual(beforeData.repoUrl); - for (const name of Object.keys(beforeData.entries)) { - if (name === t.config.name) { - expect(data.entries[name].slice(0, -1)).toEqual(beforeData.entries[name]); // New data was appended - } else { - expect(data.entries[name]).toEqual(beforeData.entries[name]); - } - } - } - - if (indexHtmlBefore !== null) { - const indexHtmlAfter = await fs.readFile(indexHtml); - expect(indexHtmlAfter).toEqual(indexHtmlBefore); // If index.html is already existing, do not touch it - } - }); - } - - const maxRetries = 10; - const retryCases: Array<{ - it: string; - error?: RegExp; - pushErrorMessage: string; - pushErrorCount: number; - }> = [ - ...[1, 2].map((retries) => ({ - it: `updates data successfully after ${retries} retries`, - pushErrorMessage: '... [remote rejected] ...', - pushErrorCount: retries, - })), - { - it: `gives up updating data after ${maxRetries} retries with an error`, - pushErrorMessage: '... [remote rejected] ...', - pushErrorCount: maxRetries, - error: /Auto-push failed 3 times since the remote branch gh-pages rejected pushing all the time/, - }, - { - it: `gives up updating data after ${maxRetries} retries with an error containing "[rejected]" in message`, - pushErrorMessage: '... [rejected] ...', - pushErrorCount: maxRetries, - error: /Auto-push failed 3 times since the remote branch gh-pages rejected pushing all the time/, - }, - { - it: 'handles an unexpected error without retry', - pushErrorMessage: 'Some fatal error', - pushErrorCount: 1, - error: /Some fatal error/, - }, - ]; - - it.each(retryCases)('$it', async function (t) { - gitSpy.pushFailure = t.pushErrorMessage; - gitSpy.pushFailureCount = t.pushErrorCount; - const config = { ...defaultCfg, benchmarkDataDirPath: 'with-index-html' }; - const added: Benchmark = { - commit: commit('current commit id'), - date: lastUpdate, - tool: 'cargo', - benches: [bench('bench_fib_10', 110)], - }; - - const originalDataJs = path.join(config.benchmarkDataDirPath, 'original_data.js'); - const dataJs = path.join(config.benchmarkDataDirPath, 'data.js'); - await fs.copyFile(originalDataJs, dataJs); - - const history = gitHistory({ dir: 'with-index-html', addIndexHtml: false }); - if (t.pushErrorCount > 0) { - // First 2 commands are fetch and switch. They are not repeated on retry - const retryHistory = history.slice(2, -1); - retryHistory.push(['cmd', [[], 'reset', '--hard', 'HEAD~1']]); - - const retries = Math.min(t.pushErrorCount, maxRetries); - for (let i = 0; i < retries; i++) { - history.splice(2, 0, ...retryHistory); - } - } - - try { - await writeBenchmark(added, config); - expect(gitSpy.history).toEqual(history); - } catch (err: any) { - if (t.error === undefined) { - throw err; - } - ok(t.error.test(err.message), `'${err.message}' did not match to ${t.error}`); - } - }); - }); -});