Skip to content
CI

fix: Catch2 v.3.5.0 changed output format #893

Sign in to view logs

fix: Catch2 v.3.5.0 changed output format

fix: Catch2 v.3.5.0 changed output format #893

Workflow file for this run

.github/workflows/ci.yml at e32befd
name: CI
on:
push:
branches:
- master
pull_request:
branches:
- master
jobs:
benchmarkdotnet-framework:
name: Run Benchmark.Net .Net Benchmark Framework example
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 20
cache: 'npm'
- uses: actions/setup-dotnet@v3
with:
dotnet-version: '6.0.101' # SDK Version to use. keep in line with examples/benchmarkdotnet/global.json
- run: npm ci
- run: npm run build
- name: Run benchmark
run: cd examples/benchmarkdotnet && dotnet run --exporters json --filter '*'
- name: Save previous data.js
run: |
git fetch origin gh-pages
git checkout gh-pages
cp ./dev/bench/data.js before_data.js
git checkout -
- name: Store benchmark result
uses: ./
with:
name: Benchmark.Net Benchmark
tool: 'benchmarkdotnet'
output-file-path: examples/benchmarkdotnet/BenchmarkDotNet.Artifacts/results/Sample.Benchmarks-report-full-compressed.json
skip-fetch-gh-pages: true
fail-on-alert: true
summary-always: true
github-token: ${{ secrets.GITHUB_TOKEN }}
comment-on-alert: true
- run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.Net Benchmark'
benchmarkjs:
name: Run JavaScript benchmark example
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 20
cache: 'npm'
- run: npm ci
- run: npm run build
- name: Run benchmark
run: cd examples/benchmarkjs && npm install && node bench.js | tee output.txt
- name: Save previous data.js
run: |
git fetch origin gh-pages
git checkout gh-pages
cp ./dev/bench/data.js before_data.js
git checkout -
- name: Store benchmark result
uses: ./
with:
name: Benchmark.js Benchmark
tool: 'benchmarkjs'
output-file-path: examples/benchmarkjs/output.txt
skip-fetch-gh-pages: true
fail-on-alert: true
summary-always: true
github-token: ${{ secrets.GITHUB_TOKEN }}
comment-on-alert: true
- run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.js Benchmark'
catch2-framework:
name: Run Catch2 C++ Benchmark Framework example
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 20
cache: 'npm'
- run: npm ci
- run: npm run build
- name: Run benchmark
run: |
cd examples/catch2
mkdir build && cd build
cmake -DCMAKE_BUILD_TYPE=Release ..
cmake --build . --config Release
./Catch2_bench > ../benchmark_result.txt
- name: Save previous data.js
run: |
git fetch origin gh-pages
git checkout gh-pages
cp ./dev/bench/data.js before_data.js
git checkout -
- name: Store benchmark result
uses: ./
with:
name: Catch2 Benchmark
tool: 'catch2'
output-file-path: examples/catch2/benchmark_result.txt
skip-fetch-gh-pages: true
fail-on-alert: true
summary-always: true
github-token: ${{ secrets.GITHUB_TOKEN }}
comment-on-alert: true
- run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark'
cpp-framework:
name: Run Google C++ Benchmark Framework example
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 20
cache: 'npm'
- run: npm ci
- run: npm run build
- name: Cache Benchmark library
uses: actions/cache@v4
with:
path: examples/cpp/benchmark
key: ${{ runner.os }}-googlebenchmark-v1.5.0
- name: Run benchmark
run: |
cd examples/cpp
make json
- name: Save previous data.js
run: |
git fetch origin gh-pages
git checkout gh-pages
cp ./dev/bench/data.js before_data.js
git checkout -
- name: Store benchmark result
uses: ./
with:
name: C++ Benchmark
tool: 'googlecpp'
output-file-path: examples/cpp/benchmark_result.json
skip-fetch-gh-pages: true
fail-on-alert: true
summary-always: true
github-token: ${{ secrets.GITHUB_TOKEN }}
comment-on-alert: true
- run: node ./dist/scripts/ci_validate_modification.js before_data.js 'C++ Benchmark'
go:
name: Run Go benchmark example
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 20
cache: 'npm'
- uses: actions/setup-go@v4
with:
go-version: "stable"
- run: npm ci
- run: npm run build
- name: Run benchmark
run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt
- name: Save previous data.js
run: |
git fetch origin gh-pages
git checkout gh-pages
cp ./dev/bench/data.js before_data.js
git checkout -
- name: Store benchmark result
uses: ./
with:
name: Go Benchmark
tool: 'go'
output-file-path: examples/go/output.txt
skip-fetch-gh-pages: true
fail-on-alert: true
summary-always: true
github-token: ${{ secrets.GITHUB_TOKEN }}
comment-on-alert: true
- run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Go Benchmark'
java-jmh:
name: Run JMH Java Benchmark Framework example
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 20
cache: 'npm'
- uses: actions/setup-java@v4
with:
distribution: 'adopt'
java-version: '11'
- run: npm ci
- run: npm run build
- name: Run benchmark
run: |
cd examples/java
mvn clean verify
java -jar target/benchmarks.jar -wi 1 -i 3 -f 1 -rf json
- name: Save previous data.js
run: |
git fetch origin gh-pages
git checkout gh-pages
cp ./dev/bench/data.js before_data.js
git checkout -
- name: Store benchmark result
uses: ./
with:
name: JMH Benchmark
tool: "jmh"
output-file-path: examples/java/jmh-result.json
skip-fetch-gh-pages: true
fail-on-alert: true
summary-always: true
github-token: ${{ secrets.GITHUB_TOKEN }}
comment-on-alert: true
- run: node ./dist/scripts/ci_validate_modification.js before_data.js 'JMH Benchmark'
julia-benchmark:
name: Run Julia benchmark example
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 20
cache: 'npm'
- run: npm ci
- run: npm run build
- uses: julia-actions/setup-julia@v1
with:
version: '1'
arch: x64
- name: Run benchmark
run: |
cd examples/julia
julia --project --color=yes -e '
using Pkg;
Pkg.instantiate();
include("fib.jl")'
- name: Save previous data.js
run: |
git fetch origin gh-pages
git checkout gh-pages
cp ./dev/bench/data.js before_data.js
git checkout -
- name: Store benchmark result
uses: ./
with:
name: Julia benchmark
tool: 'julia'
output-file-path: examples/julia/output.json
skip-fetch-gh-pages: true
fail-on-alert: true
summary-always: true
github-token: ${{ secrets.GITHUB_TOKEN }}
comment-on-alert: true
- run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Julia benchmark'
pytest-benchmark:
name: Run Pytest benchmark example
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 20
cache: 'npm'
- uses: actions/setup-python@v5
with:
python-version: 3.9
- run: npm ci
- run: npm run build
- name: Run benchmark
run: |
cd examples/pytest
pip install -r requirements.txt
pytest bench.py --benchmark-json output.json
- name: Save previous data.js
run: |
git fetch origin gh-pages
git checkout gh-pages
cp ./dev/bench/data.js before_data.js
git checkout -
- name: Store benchmark result
uses: ./
with:
name: Python Benchmark with pytest-benchmark
tool: 'pytest'
output-file-path: examples/pytest/output.json
skip-fetch-gh-pages: true
fail-on-alert: true
summary-always: true
github-token: ${{ secrets.GITHUB_TOKEN }}
comment-on-alert: true
- run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Python Benchmark with pytest-benchmark'
rust:
name: Run Rust benchmark example
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 20
cache: 'npm'
- run: npm ci
- run: npm run build
- run: rustup toolchain update nightly && rustup default nightly
- name: Run benchmark
run: cd examples/rust && cargo +nightly bench | tee output.txt
- name: Save previous data.js
run: |
git fetch origin gh-pages
git checkout gh-pages
cp ./dev/bench/data.js before_data.js
git checkout -
- name: Store benchmark result
uses: ./
with:
name: Rust Benchmark
tool: 'cargo'
output-file-path: examples/rust/output.txt
fail-on-alert: true
summary-always: true
github-token: ${{ secrets.GITHUB_TOKEN }}
comment-on-alert: true
- run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Rust Benchmark'
rust-criterion-rs-framework:
name: Run Criterion.rs benchmark example
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 20
cache: 'npm'
- run: npm ci
- run: npm run build
- run: rustup toolchain update nightly && rustup default nightly
- name: Run benchmark
run: cd examples/criterion-rs && cargo +nightly bench -- --output-format bencher | tee output.txt
- name: Save previous data.js
run: |
git fetch origin gh-pages
git checkout gh-pages
cp ./dev/bench/data.js before_data.js
git checkout -
- name: Store benchmark result
uses: ./
with:
name: Criterion.rs Benchmark
tool: 'cargo'
output-file-path: examples/criterion-rs/output.txt
fail-on-alert: true
summary-always: true
github-token: ${{ secrets.GITHUB_TOKEN }}
comment-on-alert: true
- run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Criterion.rs Benchmark'
only-alert-with-cache:
name: Run alert check with actions/cache
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 20
cache: 'npm'
- run: npm ci
- run: npm run build
- uses: actions/setup-go@v4
with:
go-version: "stable"
- name: Run benchmark
run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt
- name: Download previous benchmark data
uses: actions/cache@v4
with:
path: ./cache
key: ${{ runner.os }}-ci-cache-test
- name: Store benchmark result
uses: ./
with:
name: Alert setup example with cache
tool: 'go'
output-file-path: examples/go/output.txt
external-data-json-path: ./cache/benchmark-data.json
github-token: ${{ secrets.GITHUB_TOKEN }}
alert-threshold: '200%'
comment-on-alert: true
fail-on-alert: true
alert-comment-cc-users: '@ktrz'
unit-tests:
name: Run unit tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 20
cache: 'npm'
- run: npm ci
- run: npm run coverage
- name: Apply yamllint
run: |
sudo pip install yamllint
yamllint --strict .github/workflows
- uses: codecov/codecov-action@v2
lint:
name: Run linting and formatting check
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 20
cache: 'npm'
- run: npm ci
- run: npm run lint
- run: npm run format:check