From 72a54693146788dcce3295d05132480a3cb7ce34 Mon Sep 17 00:00:00 2001 From: Daniel Morris Date: Thu, 21 May 2026 22:52:55 +0100 Subject: [PATCH] feat: restore cross-language benchmark suite (ILO-65) Add bench/ directory with five canonical benchmarks (fib, hof, listproc, pattern-match, sum-loop) each implemented in ilo + Python / Node.js / Rust. bench/run.sh runs all impls, verifies correctness, and emits bench/results.json. CI nightly workflow (.github/workflows/bench.yml) runs the suite and gates on >10% regression vs previous run. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/bench.yml | 84 +++++++++ bench/fib/fib.ilo | 1 + bench/fib/fib.js | 17 ++ bench/fib/fib.py | 20 ++ bench/fib/fib.rs | 22 +++ bench/hof/hof.ilo | 1 + bench/hof/hof.js | 16 ++ bench/hof/hof.py | 23 +++ bench/hof/hof.rs | 20 ++ bench/listproc/listproc.ilo | 2 + bench/listproc/listproc.js | 31 +++ bench/listproc/listproc.py | 31 +++ bench/listproc/listproc.rs | 37 ++++ bench/pattern-match/pattern-match.ilo | 4 + bench/pattern-match/pattern-match.js | 42 +++++ bench/pattern-match/pattern-match.py | 48 +++++ bench/pattern-match/pattern-match.rs | 50 +++++ bench/results.json | 35 ++++ bench/run.sh | 259 ++++++++++++++++++++++++++ bench/sum-loop/sum-loop.ilo | 2 + bench/sum-loop/sum-loop.js | 26 +++ bench/sum-loop/sum-loop.py | 26 +++ bench/sum-loop/sum-loop.rs | 31 +++ 23 files changed, 828 insertions(+) create mode 100644 .github/workflows/bench.yml create mode 100644 bench/fib/fib.ilo create mode 100644 bench/fib/fib.js create mode 100644 bench/fib/fib.py create mode 100644 bench/fib/fib.rs create mode 100644 bench/hof/hof.ilo create mode 100644 bench/hof/hof.js create mode 100644 bench/hof/hof.py create mode 100644 bench/hof/hof.rs create mode 100644 bench/listproc/listproc.ilo create mode 100644 bench/listproc/listproc.js create mode 100644 bench/listproc/listproc.py create mode 100644 bench/listproc/listproc.rs create mode 100644 bench/pattern-match/pattern-match.ilo create mode 100644 bench/pattern-match/pattern-match.js create mode 100644 bench/pattern-match/pattern-match.py create mode 100644 bench/pattern-match/pattern-match.rs create mode 100644 bench/results.json create mode 100755 bench/run.sh create mode 100644 bench/sum-loop/sum-loop.ilo create mode 100644 bench/sum-loop/sum-loop.js create mode 100644 bench/sum-loop/sum-loop.py create mode 100644 bench/sum-loop/sum-loop.rs diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml new file mode 100644 index 00000000..cf2b6fe1 --- /dev/null +++ b/.github/workflows/bench.yml @@ -0,0 +1,84 @@ +name: Benchmark suite + +on: + schedule: + - cron: "0 3 * * *" # nightly at 03:00 UTC + workflow_dispatch: # manual trigger + +permissions: + contents: write # to push results.json + +jobs: + bench: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-bench-${{ hashFiles('Cargo.lock') }} + restore-keys: ${{ runner.os }}-bench- + + - name: Install Node.js + uses: actions/setup-node@v4 + with: + node-version: "20" + + - name: Run benchmark suite + run: bash bench/run.sh --no-rust + + - name: Compile Rust baselines and re-run + run: | + mkdir -p bench/.build + for d in bench/*/; do + b=$(basename "$d") + rs="$d$b.rs" + if [ -f "$rs" ]; then + rustc -O -o "bench/.build/${b}_rs" "$rs" || true + fi + done + bash bench/run.sh + + - name: Check for regression (>10% vs previous) + run: | + if [ -f bench/results-prev.json ]; then + python3 - bench/results.json bench/results-prev.json << 'PYEOF' + import sys, json + cur = json.load(open(sys.argv[1]))["benchmarks"] + prev = json.load(open(sys.argv[2]))["benchmarks"] + failures = [] + for bench, langs in cur.items(): + for lang, ns in langs.items(): + prev_ns = prev.get(bench, {}).get(lang) + if prev_ns and ns > prev_ns * 1.10: + pct = (ns - prev_ns) / prev_ns * 100 + failures.append(f" {bench}/{lang}: {prev_ns}ns -> {ns}ns (+{pct:.1f}%)") + if failures: + print("REGRESSION DETECTED (>10%):") + for f in failures: + print(f) + sys.exit(1) + else: + print("No regressions detected.") + PYEOF + fi + + - name: Rotate results + run: | + cp bench/results.json bench/results-prev.json 2>/dev/null || true + + - name: Commit updated results + uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_message: "chore(bench): update nightly results [skip ci]" + file_pattern: "bench/results.json bench/results-prev.json" + branch: main diff --git a/bench/fib/fib.ilo b/bench/fib/fib.ilo new file mode 100644 index 00000000..441453fc --- /dev/null +++ b/bench/fib/fib.ilo @@ -0,0 +1 @@ +fib n:n>n;<=n 1 n;a=fib -n 1;b=fib -n 2;+a b diff --git a/bench/fib/fib.js b/bench/fib/fib.js new file mode 100644 index 00000000..99914b07 --- /dev/null +++ b/bench/fib/fib.js @@ -0,0 +1,17 @@ +function fib(n) { + if (n <= 1) return n; + return fib(n - 1) + fib(n - 2); +} + +const N = parseInt(process.argv[2]) || 25; +// warmup +for (let i = 0; i < 100; i++) fib(N); +const iters = 1000; +const start = process.hrtime.bigint(); +let r; +for (let i = 0; i < iters; i++) r = fib(N); +const elapsed = Number(process.hrtime.bigint() - start); +console.log(`result: ${r}`); +console.log(`iterations: ${iters}`); +console.log(`total: ${(elapsed / 1e6).toFixed(2)}ms`); +console.log(`per call: ${Math.floor(elapsed / iters)}ns`); diff --git a/bench/fib/fib.py b/bench/fib/fib.py new file mode 100644 index 00000000..c951ed10 --- /dev/null +++ b/bench/fib/fib.py @@ -0,0 +1,20 @@ +import sys, time + +def fib(n): + if n <= 1: + return n + return fib(n - 1) + fib(n - 2) + +N = int(sys.argv[1]) if len(sys.argv) > 1 else 25 +# warmup +for _ in range(5): + fib(N) +iters = 100 +start = time.monotonic_ns() +for _ in range(iters): + r = fib(N) +elapsed = time.monotonic_ns() - start +print(f"result: {r}") +print(f"iterations: {iters}") +print(f"total: {elapsed / 1e6:.2f}ms") +print(f"per call: {elapsed // iters}ns") diff --git a/bench/fib/fib.rs b/bench/fib/fib.rs new file mode 100644 index 00000000..b1904433 --- /dev/null +++ b/bench/fib/fib.rs @@ -0,0 +1,22 @@ +use std::env; +use std::time::Instant; + +fn fib(n: i64) -> i64 { + if n <= 1 { return n; } + fib(n - 1) + fib(n - 2) +} + +fn main() { + let n: i64 = env::args().nth(1).and_then(|s| s.parse().ok()).unwrap_or(25); + // warmup + for _ in 0..100 { let _ = fib(n); } + let iters = 1000; + let start = Instant::now(); + let mut r = 0i64; + for _ in 0..iters { r = fib(n); } + let elapsed = start.elapsed().as_nanos(); + println!("result: {}", r); + println!("iterations: {}", iters); + println!("total: {:.2}ms", elapsed as f64 / 1e6); + println!("per call: {}ns", elapsed / iters); +} diff --git a/bench/hof/hof.ilo b/bench/hof/hof.ilo new file mode 100644 index 00000000..5669f06c --- /dev/null +++ b/bench/hof/hof.ilo @@ -0,0 +1 @@ +bench n:n>n;xs=[];i=0;wh n;+a b) (flt (x:n>b;>x 0) (map (x:n>n;*x x) xs)) 0 diff --git a/bench/hof/hof.js b/bench/hof/hof.js new file mode 100644 index 00000000..25a837c9 --- /dev/null +++ b/bench/hof/hof.js @@ -0,0 +1,16 @@ +function bench(n) { + const xs = Array.from({length: n}, (_, i) => i); + return xs.map(x => x * x).filter(x => x > 0).reduce((a, b) => a + b, 0); +} + +const N = parseInt(process.argv[2]) || 500; +for (let i = 0; i < 1000; i++) bench(N); +const iters = 10000; +const start = process.hrtime.bigint(); +let r; +for (let i = 0; i < iters; i++) r = bench(N); +const elapsed = Number(process.hrtime.bigint() - start); +console.log(`result: ${r}`); +console.log(`iterations: ${iters}`); +console.log(`total: ${(elapsed / 1e6).toFixed(2)}ms`); +console.log(`per call: ${Math.floor(elapsed / iters)}ns`); diff --git a/bench/hof/hof.py b/bench/hof/hof.py new file mode 100644 index 00000000..54e39571 --- /dev/null +++ b/bench/hof/hof.py @@ -0,0 +1,23 @@ +import sys, time +from functools import reduce + +def sq(x): return x * x +def pos(x): return x > 0 +def add(a, b): return a + b + +def bench(n): + xs = list(range(n)) + return reduce(add, filter(pos, map(sq, xs)), 0) + +N = int(sys.argv[1]) if len(sys.argv) > 1 else 500 +for _ in range(100): + bench(N) +iters = 10000 +start = time.monotonic_ns() +for _ in range(iters): + r = bench(N) +elapsed = time.monotonic_ns() - start +print(f"result: {r}") +print(f"iterations: {iters}") +print(f"total: {elapsed / 1e6:.2f}ms") +print(f"per call: {elapsed // iters}ns") diff --git a/bench/hof/hof.rs b/bench/hof/hof.rs new file mode 100644 index 00000000..5d233e7e --- /dev/null +++ b/bench/hof/hof.rs @@ -0,0 +1,20 @@ +use std::env; +use std::time::Instant; + +fn bench(n: i64) -> i64 { + (0..n).map(|x| x * x).filter(|&x| x > 0).sum() +} + +fn main() { + let n: i64 = env::args().nth(1).and_then(|s| s.parse().ok()).unwrap_or(500); + for _ in 0..1000 { let _ = bench(n); } + let iters: u128 = 10000; + let start = Instant::now(); + let mut r = 0i64; + for _ in 0..iters { r = bench(n); } + let elapsed = start.elapsed().as_nanos(); + println!("result: {}", r); + println!("iterations: {}", iters); + println!("total: {:.2}ms", elapsed as f64 / 1e6); + println!("per call: {}ns", elapsed / iters); +} diff --git a/bench/listproc/listproc.ilo b/bench/listproc/listproc.ilo new file mode 100644 index 00000000..84107bc8 --- /dev/null +++ b/bench/listproc/listproc.ilo @@ -0,0 +1,2 @@ +lev x:n>n;>=x 5000 5;>=x 3000 4;>=x 1000 3;>=x 500 2;1 +bench n:n>n;s=0;i=0;wh = 5000) return 5; + if (x >= 3000) return 4; + if (x >= 1000) return 3; + if (x >= 500) return 2; + return 1; +} + +function bench(n) { + let s = 0; + for (let i = 0; i < n; i++) { + const a = i * 3; + const b = a + 1; + const c = b * 2; + const d = lev(c); + s += d; + } + return s; +} + +const N = parseInt(process.argv[2]) || 1000; +for (let i = 0; i < 1000; i++) bench(N); +const iters = 10000; +const start = process.hrtime.bigint(); +let r; +for (let i = 0; i < iters; i++) r = bench(N); +const elapsed = Number(process.hrtime.bigint() - start); +console.log(`result: ${r}`); +console.log(`iterations: ${iters}`); +console.log(`total: ${(elapsed / 1e6).toFixed(2)}ms`); +console.log(`per call: ${Math.floor(elapsed / iters)}ns`); diff --git a/bench/listproc/listproc.py b/bench/listproc/listproc.py new file mode 100644 index 00000000..37c13dc3 --- /dev/null +++ b/bench/listproc/listproc.py @@ -0,0 +1,31 @@ +import sys, time + +def lev(x): + if x >= 5000: return 5 + if x >= 3000: return 4 + if x >= 1000: return 3 + if x >= 500: return 2 + return 1 + +def bench(n): + s = 0 + for i in range(n): + a = i * 3 + b = a + 1 + c = b * 2 + d = lev(c) + s += d + return s + +N = int(sys.argv[1]) if len(sys.argv) > 1 else 1000 +for _ in range(100): + bench(N) +iters = 10000 +start = time.monotonic_ns() +for _ in range(iters): + r = bench(N) +elapsed = time.monotonic_ns() - start +print(f"result: {r}") +print(f"iterations: {iters}") +print(f"total: {elapsed / 1e6:.2f}ms") +print(f"per call: {elapsed // iters}ns") diff --git a/bench/listproc/listproc.rs b/bench/listproc/listproc.rs new file mode 100644 index 00000000..75b42246 --- /dev/null +++ b/bench/listproc/listproc.rs @@ -0,0 +1,37 @@ +use std::env; +use std::time::Instant; + +#[inline(never)] +fn lev(x: i64) -> i64 { + if x >= 5000 { return 5; } + if x >= 3000 { return 4; } + if x >= 1000 { return 3; } + if x >= 500 { return 2; } + 1 +} + +fn bench(n: i64) -> i64 { + let mut s: i64 = 0; + for i in 0..n { + let a = i * 3; + let b = a + 1; + let c = b * 2; + let d = lev(c); + s += d; + } + s +} + +fn main() { + let n: i64 = env::args().nth(1).and_then(|s| s.parse().ok()).unwrap_or(1000); + for _ in 0..1000 { let _ = bench(n); } + let iters: u128 = 10000; + let start = Instant::now(); + let mut r = 0i64; + for _ in 0..iters { r = bench(n); } + let elapsed = start.elapsed().as_nanos(); + println!("result: {}", r); + println!("iterations: {}", iters); + println!("total: {:.2}ms", elapsed as f64 / 1e6); + println!("per call: {}ns", elapsed / iters); +} diff --git a/bench/pattern-match/pattern-match.ilo b/bench/pattern-match/pattern-match.ilo new file mode 100644 index 00000000..35fa5083 --- /dev/null +++ b/bench/pattern-match/pattern-match.ilo @@ -0,0 +1,4 @@ +cata x:n>n;>=x 900 9;>=x 800 8;>=x 700 7;>=x 600 6;>=x 500 5;>=x 400 4;>=x 300 3;>=x 200 2;1 +catb x:n>n;>=x 500 *x 3;>=x 200 *x 2;+x 0 +combine a:n b:n>n;>=a 7 +b *a 10;>=a 4 +b *a 5;+b a +bench n:n>n;s=0;i=0;wh = 800) { if (x >= 900) return 9; return 8; } + if (x >= 600) { if (x >= 700) return 7; return 6; } + if (x >= 400) { if (x >= 500) return 5; return 4; } + if (x >= 200) { if (x >= 300) return 3; return 2; } + return 1; +} + +function catb(x) { + if (x >= 500) return x * 3; + if (x >= 200) return x * 2; + return x; +} + +function combine(a, b) { + if (a >= 7) return b + a * 10; + if (a >= 4) return b + a * 5; + return b + a; +} + +function bench(n) { + let s = 0; + for (let i = 0; i < n; i++) { + const a = cata(i); + const b = catb(i); + const c = combine(a, b); + s += c; + } + return s; +} + +const N = parseInt(process.argv[2]) || 1000; +for (let i = 0; i < 1000; i++) bench(N); +const iters = 10000; +const start = process.hrtime.bigint(); +let r; +for (let i = 0; i < iters; i++) r = bench(N); +const elapsed = Number(process.hrtime.bigint() - start); +console.log(`result: ${r}`); +console.log(`iterations: ${iters}`); +console.log(`total: ${(elapsed / 1e6).toFixed(2)}ms`); +console.log(`per call: ${Math.floor(elapsed / iters)}ns`); diff --git a/bench/pattern-match/pattern-match.py b/bench/pattern-match/pattern-match.py new file mode 100644 index 00000000..61e1e1a1 --- /dev/null +++ b/bench/pattern-match/pattern-match.py @@ -0,0 +1,48 @@ +import sys, time + +def cata(x): + if x >= 800: + if x >= 900: return 9 + return 8 + if x >= 600: + if x >= 700: return 7 + return 6 + if x >= 400: + if x >= 500: return 5 + return 4 + if x >= 200: + if x >= 300: return 3 + return 2 + return 1 + +def catb(x): + if x >= 500: return x * 3 + if x >= 200: return x * 2 + return x + +def combine(a, b): + if a >= 7: return b + a * 10 + if a >= 4: return b + a * 5 + return b + a + +def bench(n): + s = 0 + for i in range(n): + a = cata(i) + b = catb(i) + c = combine(a, b) + s += c + return s + +N = int(sys.argv[1]) if len(sys.argv) > 1 else 1000 +for _ in range(100): + bench(N) +iters = 10000 +start = time.monotonic_ns() +for _ in range(iters): + r = bench(N) +elapsed = time.monotonic_ns() - start +print(f"result: {r}") +print(f"iterations: {iters}") +print(f"total: {elapsed / 1e6:.2f}ms") +print(f"per call: {elapsed // iters}ns") diff --git a/bench/pattern-match/pattern-match.rs b/bench/pattern-match/pattern-match.rs new file mode 100644 index 00000000..4f04c030 --- /dev/null +++ b/bench/pattern-match/pattern-match.rs @@ -0,0 +1,50 @@ +use std::env; +use std::time::Instant; + +#[inline(never)] +fn cata(x: i64) -> i64 { + if x >= 800 { if x >= 900 { return 9; } return 8; } + if x >= 600 { if x >= 700 { return 7; } return 6; } + if x >= 400 { if x >= 500 { return 5; } return 4; } + if x >= 200 { if x >= 300 { return 3; } return 2; } + 1 +} + +#[inline(never)] +fn catb(x: i64) -> i64 { + if x >= 500 { return x * 3; } + if x >= 200 { return x * 2; } + x +} + +#[inline(never)] +fn combine(a: i64, b: i64) -> i64 { + if a >= 7 { return b + a * 10; } + if a >= 4 { return b + a * 5; } + b + a +} + +fn bench(n: i64) -> i64 { + let mut s: i64 = 0; + for i in 0..n { + let a = cata(i); + let b = catb(i); + let c = combine(a, b); + s += c; + } + s +} + +fn main() { + let n: i64 = env::args().nth(1).and_then(|s| s.parse().ok()).unwrap_or(1000); + for _ in 0..1000 { let _ = bench(n); } + let iters: u128 = 10000; + let start = Instant::now(); + let mut r = 0i64; + for _ in 0..iters { r = bench(n); } + let elapsed = start.elapsed().as_nanos(); + println!("result: {}", r); + println!("iterations: {}", iters); + println!("total: {:.2}ms", elapsed as f64 / 1e6); + println!("per call: {}ns", elapsed / iters); +} diff --git a/bench/results.json b/bench/results.json new file mode 100644 index 00000000..5e6f2ee0 --- /dev/null +++ b/bench/results.json @@ -0,0 +1,35 @@ +{ + "generated": "2026-05-21T21:52:31Z", + "benchmarks": { + "fib": { + "ilo-vm": 70703, + "ilo-jit": 25402, + "Node": 5354, + "Python": 56873 + }, + "hof": { + "ilo-vm": 160728, + "ilo-jit": 455373, + "Node": 70309, + "Python": 125605 + }, + "listproc": { + "ilo-vm": 65686, + "ilo-jit": 1466, + "Node": 1415, + "Python": 118849 + }, + "pattern-match": { + "ilo-vm": 146273, + "ilo-jit": 3147, + "Node": 2623, + "Python": 151531 + }, + "sum-loop": { + "ilo-vm": 51469, + "ilo-jit": 1364, + "Node": 978, + "Python": 62956 + } + } +} \ No newline at end of file diff --git a/bench/run.sh b/bench/run.sh new file mode 100755 index 00000000..b1e410c3 --- /dev/null +++ b/bench/run.sh @@ -0,0 +1,259 @@ +#!/usr/bin/env bash +# ilo cross-language benchmark suite +# Benchmarks: fib, hof, listproc, pattern-match, sum-loop +# Languages: ilo (VM + JIT), Python 3, Node.js (V8), Rust (native) +# Output: bench/results.json +# +# Usage: ./bench/run.sh [--quick] [--no-rust] +# --quick Fewer iterations (faster, less precise) +# --no-rust Skip Rust (avoid compile time) +set -euo pipefail + +cd "$(dirname "$0")/.." + +# ── Parse flags ────────────────────────────────────────────────────────────── +QUICK=false +SKIP_RUST=false +for arg in "$@"; do + case "$arg" in + --quick) QUICK=true ;; + --no-rust) SKIP_RUST=true ;; + esac +done + +# ── Config ─────────────────────────────────────────────────────────────────── +BENCH_DIR="bench" +RESULTS_FILE="$BENCH_DIR/results.json" +BUILD_DIR="$BENCH_DIR/.build" +ILO="./target/release/ilo" + +BENCHMARKS=(fib hof listproc pattern-match sum-loop) + +# Argument passed to each benchmark program +bench_arg() { + case "$1" in + fib) echo "15" ;; + *) echo "1000" ;; + esac +} + +# Function name for ilo --bench and direct invocation +bench_func() { + case "$1" in + fib) echo "fib" ;; + *) echo "bench" ;; + esac +} + +# Expected correct output for correctness checks +expected_for() { + case "$1" in + fib) echo "610" ;; + hof) echo "332833500" ;; + listproc) echo "3417" ;; + pattern-match) echo "1386050" ;; + sum-loop) echo "1353850" ;; + esac +} + +# ── Helpers ────────────────────────────────────────────────────────────────── +check_cmd() { command -v "$1" >/dev/null 2>&1; } + +section() { + echo "" + echo "═══════════════════════════════════════════════════════════" + echo " $1" + echo "═══════════════════════════════════════════════════════════" +} + +# Extract "per call: NNNns" from output +extract_ns() { + echo "$1" | sed -n 's/.*per call:[[:space:]]*\([0-9]*\)ns/\1/p' | tail -1 +} + +# ── Results store ───────────────────────────────────────────────────────────── +RESULTS_TMP=$(mktemp) +trap "rm -f $RESULTS_TMP" EXIT + +record() { + local bench="$1" lang="$2" ns="$3" + echo "${bench}|${lang}|${ns}" >> "$RESULTS_TMP" +} + +# ── Build ilo ──────────────────────────────────────────────────────────────── +section "Building ilo (release)" +if check_cmd cargo; then + if cargo build --release --features cranelift 2>/dev/null; then + echo " Built with Cranelift JIT" + else + cargo build --release + echo " Built without Cranelift JIT" + fi +fi +if [[ ! -x "$ILO" ]]; then + echo " ERROR: $ILO not found. Run: cargo build --release" >&2 + exit 1 +fi + +# ── Verify ilo correctness ─────────────────────────────────────────────────── +section "Verifying ilo programs" +all_ok=true +for bench in "${BENCHMARKS[@]}"; do + ilo_file="$BENCH_DIR/$bench/$bench.ilo" + arg=$(bench_arg "$bench") + func=$(bench_func "$bench") + result=$("$ILO" "$ilo_file" "$func" "$arg" 2>/dev/null || echo "ERROR") + expected=$(expected_for "$bench") + if [[ "$result" == "$expected" ]]; then + echo " $bench: OK ($result)" + else + echo " $bench: FAIL — expected $expected, got '$result'" >&2 + all_ok=false + fi +done +if [[ "$all_ok" != "true" ]]; then + echo "" >&2 + echo "ERROR: Correctness check failed. Fix ilo programs before benchmarking." >&2 + exit 1 +fi + +# ── Compile Rust baselines ─────────────────────────────────────────────────── +if [[ "$SKIP_RUST" == "false" ]] && check_cmd rustc; then + section "Compiling Rust baselines" + mkdir -p "$BUILD_DIR" + for bench in "${BENCHMARKS[@]}"; do + rs="$BENCH_DIR/$bench/$bench.rs" + out="$BUILD_DIR/${bench}_rs" + if [[ -f "$rs" ]]; then + if rustc -O -o "$out" "$rs" 2>/dev/null; then + echo " rustc: $bench OK" + else + echo " rustc: $bench FAIL (skipping)" + fi + fi + done +else + [[ "$SKIP_RUST" == "true" ]] || true +fi + +# ── Run benchmarks ─────────────────────────────────────────────────────────── +for bench in "${BENCHMARKS[@]}"; do + arg=$(bench_arg "$bench") + func=$(bench_func "$bench") + ilo_file="$BENCH_DIR/$bench/$bench.ilo" + + section "$bench (arg=$arg)" + + # ilo + echo "--- ilo ---" + ilo_out=$("$ILO" "$ilo_file" --bench "$func" "$arg" 2>&1 || true) + echo "$ilo_out" + + # Parse JSON-lines output: {"engine":"vm","variant":"reusable","perCallNs":N,...} + vm_ns=$(echo "$ilo_out" | python3 -c " +import sys,json +for line in sys.stdin: + try: + d=json.loads(line) + if d.get('engine')=='vm' and d.get('variant')=='reusable': + print(d['perCallNs']) + except: pass +" 2>/dev/null || true) + jit_ns=$(echo "$ilo_out" | python3 -c " +import sys,json +for line in sys.stdin: + try: + d=json.loads(line) + if d.get('engine')=='jit': + print(d['perCallNs']) + except: pass +" 2>/dev/null || true) + [[ -n "$vm_ns" ]] && record "$bench" "ilo-vm" "$vm_ns" + [[ -n "$jit_ns" ]] && record "$bench" "ilo-jit" "$jit_ns" + + # Rust + if [[ "$SKIP_RUST" == "false" ]] && [[ -x "$BUILD_DIR/${bench}_rs" ]]; then + echo "--- Rust ---" + rs_out=$("$BUILD_DIR/${bench}_rs" "$arg" 2>&1 || true) + echo "$rs_out" + ns=$(extract_ns "$rs_out") + [[ -n "$ns" ]] && record "$bench" "Rust" "$ns" + fi + + # Node.js + js_file="$BENCH_DIR/$bench/$bench.js" + if check_cmd node && [[ -f "$js_file" ]]; then + echo "--- Node.js ---" + node_out=$(node "$js_file" "$arg" 2>&1 || true) + echo "$node_out" + ns=$(extract_ns "$node_out") + [[ -n "$ns" ]] && record "$bench" "Node" "$ns" + fi + + # Python + py_file="$BENCH_DIR/$bench/$bench.py" + if check_cmd python3 && [[ -f "$py_file" ]]; then + echo "--- Python 3 ---" + py_out=$(python3 "$py_file" "$arg" 2>&1 || true) + echo "$py_out" + ns=$(extract_ns "$py_out") + [[ -n "$ns" ]] && record "$bench" "Python" "$ns" + fi +done + +# ── Summary table ──────────────────────────────────────────────────────────── +section "Summary: per-call time (ns)" + +LANGS="ilo-jit ilo-vm Rust Node Python" + +printf "\n%-16s" "Benchmark" +for lang in $LANGS; do + printf " %10s" "$lang" +done +printf "\n%-16s" "----------------" +for lang in $LANGS; do + printf " %10s" "----------" +done +printf "\n" + +for bench in "${BENCHMARKS[@]}"; do + printf "%-16s" "$bench" + for lang in $LANGS; do + val=$(awk -F'|' -v b="$bench" -v l="$lang" '$1==b && $2==l {print $3}' "$RESULTS_TMP") + printf " %10s" "${val:-"-"}" + done + printf "\n" +done + +# ── Emit results.json ───────────────────────────────────────────────────────── +section "Writing $RESULTS_FILE" + +python3 - "$RESULTS_TMP" "$RESULTS_FILE" << 'PYEOF' +import sys, json, datetime + +results_tmp = sys.argv[1] +out_path = sys.argv[2] + +data = {} +with open(results_tmp) as f: + for line in f: + line = line.strip() + if not line: + continue + bench, lang, ns = line.split("|") + data.setdefault(bench, {})[lang] = int(ns) + +output = { + "generated": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "benchmarks": data, +} + +with open(out_path, "w") as f: + json.dump(output, f, indent=2) + +print(f" Written {out_path}") +PYEOF + +section "Done" +echo " Results saved to $RESULTS_FILE" +echo "" diff --git a/bench/sum-loop/sum-loop.ilo b/bench/sum-loop/sum-loop.ilo new file mode 100644 index 00000000..6100aa2e --- /dev/null +++ b/bench/sum-loop/sum-loop.ilo @@ -0,0 +1,2 @@ +tier x:n>n;>=x 500 3;>=x 200 2;1 +bench n:n>n;s=0;i=0;wh = 500) return 3; + if (x >= 200) return 2; + return 1; +} + +function bench(n) { + let s = 0; + for (let i = 0; i < n; i++) { + s += i * tier(i); + } + return s; +} + +const N = parseInt(process.argv[2]) || 1000; +// warmup +for (let i = 0; i < 1000; i++) bench(N); +const iters = 10000; +const start = process.hrtime.bigint(); +let r; +for (let i = 0; i < iters; i++) r = bench(N); +const elapsed = Number(process.hrtime.bigint() - start); +console.log(`result: ${r}`); +console.log(`iterations: ${iters}`); +console.log(`total: ${(elapsed / 1e6).toFixed(2)}ms`); +console.log(`per call: ${Math.floor(elapsed / iters)}ns`); diff --git a/bench/sum-loop/sum-loop.py b/bench/sum-loop/sum-loop.py new file mode 100644 index 00000000..ee7e08f5 --- /dev/null +++ b/bench/sum-loop/sum-loop.py @@ -0,0 +1,26 @@ +import sys, time + +def tier(x): + if x >= 500: return 3 + if x >= 200: return 2 + return 1 + +def bench(n): + s = 0 + for i in range(n): + s += i * tier(i) + return s + +N = int(sys.argv[1]) if len(sys.argv) > 1 else 1000 +# warmup +for _ in range(100): + bench(N) +iters = 10000 +start = time.monotonic_ns() +for _ in range(iters): + r = bench(N) +elapsed = time.monotonic_ns() - start +print(f"result: {r}") +print(f"iterations: {iters}") +print(f"total: {elapsed / 1e6:.2f}ms") +print(f"per call: {elapsed // iters}ns") diff --git a/bench/sum-loop/sum-loop.rs b/bench/sum-loop/sum-loop.rs new file mode 100644 index 00000000..69dcc24e --- /dev/null +++ b/bench/sum-loop/sum-loop.rs @@ -0,0 +1,31 @@ +use std::env; +use std::time::Instant; + +#[inline(never)] +fn tier(x: i64) -> i64 { + if x >= 500 { return 3; } + if x >= 200 { return 2; } + 1 +} + +fn bench(n: i64) -> i64 { + let mut s: i64 = 0; + for i in 0..n { + s += i * tier(i); + } + s +} + +fn main() { + let n: i64 = env::args().nth(1).and_then(|s| s.parse().ok()).unwrap_or(1000); + for _ in 0..1000 { let _ = bench(n); } + let iters: u128 = 10000; + let start = Instant::now(); + let mut r = 0i64; + for _ in 0..iters { r = bench(n); } + let elapsed = start.elapsed().as_nanos(); + println!("result: {}", r); + println!("iterations: {}", iters); + println!("total: {:.2}ms", elapsed as f64 / 1e6); + println!("per call: {}ns", elapsed / iters); +}