diff --git a/.github/workflows/c-cpp.yml b/.github/workflows/c-cpp.yml index 27519b76..abdcb8d9 100644 --- a/.github/workflows/c-cpp.yml +++ b/.github/workflows/c-cpp.yml @@ -94,4 +94,4 @@ jobs: env: CXX: g++-12 GBENCH: ${{ github.workspace }}/benchmark - run: bash -x bench-compare.sh + run: bash -x scripts/branch-compare.sh avx diff --git a/run-bench.py b/run-bench.py new file mode 100644 index 00000000..d7b697b8 --- /dev/null +++ b/run-bench.py @@ -0,0 +1,26 @@ +import sys +import argparse +import subprocess + +parser = argparse.ArgumentParser() +parser.add_argument('--branchcompare', action='store_true', help='Compare benchmarks of current branch with main. Provide an optional --filter') +parser.add_argument('--benchcompare', action='store_true', help='Compare across benchmarks. Requires --baseline and --contender') +parser.add_argument("-f", '--filter', type=str, required=False) +parser.add_argument("-b", '--baseline', type=str, required=False) +parser.add_argument("-c", '--contender', type=str, required=False) +args = parser.parse_args() + +if len(sys.argv) == 1: + parser.error("requires one of --benchcompare or --branchcompare") + +if args.benchcompare: + if (args.baseline is None or args.contender is None): + parser.error("--benchcompare requires --baseline and --contender") + else: + rc = subprocess.check_call("./scripts/bench-compare.sh '%s %s'" % (args.baseline, args.contender), shell=True) + +if args.branchcompare: + if args.filter is None: + rc = subprocess.call("./scripts/branch-compare.sh") + else: + rc = subprocess.check_call("./scripts/branch-compare.sh '%s'" % args.filter, shell=True) diff --git a/scripts/bench-compare.sh b/scripts/bench-compare.sh new file mode 100755 index 00000000..65b82b5e --- /dev/null +++ b/scripts/bench-compare.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -e +branch=$(git rev-parse --abbrev-ref HEAD) +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +cd $SCRIPT_DIR/.. +if [[ -z "${GBENCH}" ]]; then + echo "Please set env variable GBENCH and re run" + exit 1 +fi + +compare=$GBENCH/tools/compare.py +if [ ! -f $compare ]; then + echo "Unable to locate $GBENCH/tools/compare.py" + exit 1 +fi + +meson setup --warnlevel 0 --buildtype plain builddir-${branch} +cd builddir-${branch} +ninja +$compare filters ./benchexe $1 $2 diff --git a/bench-compare.sh b/scripts/branch-compare.sh old mode 100644 new mode 100755 similarity index 70% rename from bench-compare.sh rename to scripts/branch-compare.sh index 7b8da319..8d929e1d --- a/bench-compare.sh +++ b/scripts/branch-compare.sh @@ -1,7 +1,7 @@ -#~/bin/bash +#!/bin/bash set -e SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -cd $SCRIPT_DIR +BASE_DIR=$(dirname $SCRIPT_DIR) branch=$(git rev-parse --abbrev-ref HEAD) echo "Comparing main branch with $branch" @@ -20,7 +20,7 @@ rm -rf .bench-compare mkdir .bench-compare cd .bench-compare echo "Fetching and build $branch .." -git clone ${SCRIPT_DIR} -b $branch . +git clone ${BASE_DIR} -b $branch . git fetch origin meson setup --warnlevel 0 --buildtype plain builddir-${branch} cd builddir-${branch} @@ -34,5 +34,10 @@ meson setup --warnlevel 0 --buildtype plain builddir-main cd builddir-main ninja cd .. -echo "Running benchmarks .." -$compare benchmarks ./builddir-main/benchexe ./builddir-${branch}/benchexe +if [ -z "$1" ]; then + echo "Comparing all benchmarks .." + $compare benchmarks ./builddir-main/benchexe ./builddir-${branch}/benchexe +else + echo "Comparing benchmark $1 .." + $compare benchmarksfiltered ./builddir-main/benchexe $1 ./builddir-${branch}/benchexe $1 +fi