Add scripts to help run benchmarks

.github/workflows/c-cpp.yml

@@ -94,4 +94,4 @@ jobs:
       env:
         CXX: g++-12
         GBENCH: ${{ github.workspace }}/benchmark
-      run: bash -x bench-compare.sh
+      run: bash -x scripts/branch-compare.sh avx

run-bench.py

@@ -0,0 +1,26 @@
+import sys
+import argparse
+import subprocess
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--branchcompare', action='store_true', help='Compare benchmarks of current branch with main. Provide an optional --filter')
+parser.add_argument('--benchcompare', action='store_true', help='Compare across benchmarks. Requires --baseline and --contender')
+parser.add_argument("-f", '--filter', type=str, required=False)
+parser.add_argument("-b", '--baseline', type=str, required=False)
+parser.add_argument("-c", '--contender', type=str, required=False)
+args = parser.parse_args()
+
+if len(sys.argv) == 1:
+        parser.error("requires one of --benchcompare or --branchcompare")
+
+if args.benchcompare:
+    if (args.baseline is None or args.contender is None):
+        parser.error("--benchcompare requires --baseline and --contender")
+    else:
+        rc = subprocess.check_call("./scripts/bench-compare.sh '%s %s'" % (args.baseline, args.contender), shell=True)
+
+if args.branchcompare:
+    if args.filter is None:
+        rc = subprocess.call("./scripts/branch-compare.sh")
+    else:
+        rc = subprocess.check_call("./scripts/branch-compare.sh '%s'" % args.filter, shell=True)

scripts/bench-compare.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+set -e
+branch=$(git rev-parse --abbrev-ref HEAD)
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+cd $SCRIPT_DIR/..
+if [[ -z "${GBENCH}" ]]; then
+    echo "Please set env variable GBENCH and re run"
+    exit 1
+fi
+
+compare=$GBENCH/tools/compare.py
+if [ ! -f $compare ]; then
+    echo "Unable to locate $GBENCH/tools/compare.py"
+    exit 1
+fi
+
+meson setup --warnlevel 0 --buildtype plain builddir-${branch}
+cd builddir-${branch}
+ninja
+$compare filters ./benchexe $1 $2

bench-compare.sh → scripts/branch-compare.sh

@@ -1,7 +1,7 @@
-#~/bin/bash
+#!/bin/bash
 set -e
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-cd $SCRIPT_DIR
+BASE_DIR=$(dirname $SCRIPT_DIR)
 branch=$(git rev-parse --abbrev-ref HEAD)
 echo "Comparing main branch with $branch"
 
@@ -20,7 +20,7 @@ rm -rf .bench-compare
 mkdir .bench-compare
 cd .bench-compare
 echo "Fetching and build $branch .."
-git clone ${SCRIPT_DIR} -b $branch .
+git clone ${BASE_DIR} -b $branch .
 git fetch origin
 meson setup --warnlevel 0 --buildtype plain builddir-${branch}
 cd builddir-${branch}
@@ -34,5 +34,10 @@ meson setup --warnlevel 0 --buildtype plain builddir-main
 cd builddir-main
 ninja
 cd ..
-echo "Running benchmarks .."
-$compare benchmarks ./builddir-main/benchexe ./builddir-${branch}/benchexe
+if [ -z "$1" ]; then
+    echo "Comparing all benchmarks .."
+    $compare benchmarks ./builddir-main/benchexe ./builddir-${branch}/benchexe
+else
+    echo "Comparing benchmark $1 .."
+    $compare benchmarksfiltered ./builddir-main/benchexe $1 ./builddir-${branch}/benchexe $1
+fi