diff --git a/bench/perf-table.md b/bench/perf-table.md new file mode 100644 index 00000000..4c33f9fc --- /dev/null +++ b/bench/perf-table.md @@ -0,0 +1,13 @@ + + +## Performance + +Wall-clock time (lower is better). Baseline = Node. + +| Benchmark | ilo-jit | ilo-vm | Node | Python | +| --- | --- | --- | --- | --- | +| fib | 25.4 ms (×4.7) | 70.7 ms (×13.2) | 5.4 ms (×1) | 56.9 ms (×10.6) | +| hof | 455.4 ms (×6.5) | 160.7 ms (×2.3) | 70.3 ms (×1) | 125.6 ms (×1.8) | +| listproc | 1.5 ms (×1) | 65.7 ms (×46.4) | 1.4 ms (×1) | 118.8 ms (×84) | +| pattern-match | 3.1 ms (×1.2) | 146.3 ms (×55.8) | 2.6 ms (×1) | 151.5 ms (×57.8) | +| sum-loop | 1.4 ms (×1.4) | 51.5 ms (×52.6) | 978 µs (×1) | 63.0 ms (×64.4) | diff --git a/scripts/gen-perf-table.py b/scripts/gen-perf-table.py new file mode 100755 index 00000000..2ba21116 --- /dev/null +++ b/scripts/gen-perf-table.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Generate bench/perf-table.md from bench/results.json. + +Usage: + python3 scripts/gen-perf-table.py + +Reads bench/results.json relative to the repo root and writes +bench/perf-table.md. Run this script whenever bench/results.json is +updated (e.g. in CI after bench/run.sh). + +Website integration +------------------- +If a static-site generator is added under site/, include the generated +file with a snippet such as: + + --8<-- "bench/perf-table.md" (mkdocs-material) + {% include "../../bench/perf-table.md" %} (Jinja / Jekyll) + +The file contains only the table and a one-line header comment so it +can be embedded directly without further processing. +""" + +import json +import pathlib +import sys + +REPO_ROOT = pathlib.Path(__file__).parent.parent +RESULTS = REPO_ROOT / "bench" / "results.json" +OUT = REPO_ROOT / "bench" / "perf-table.md" + + +def fmt_us(value_us: int) -> str: + """Format microseconds: show ms when >= 1000, otherwise us.""" + if value_us >= 1000: + ms = value_us / 1000 + return f"{ms:.1f} ms" + return f"{value_us} µs" + + +def ratio(value: int, baseline: int) -> str: + """Return a ×N.N relative-to-baseline string.""" + r = value / baseline + if abs(r - round(r)) < 0.05: + return f"×{r:.0f}" + return f"×{r:.1f}" + + +def main() -> None: + if not RESULTS.exists(): + print(f"ERROR: {RESULTS} not found", file=sys.stderr) + sys.exit(1) + + data = json.loads(RESULTS.read_text()) + generated = data.get("generated", "unknown") + benchmarks = data["benchmarks"] + + # Determine column order: ilo-jit first, then others alphabetically + all_impls: list[str] = [] + for times in benchmarks.values(): + for impl in times: + if impl not in all_impls: + all_impls.append(impl) + + preferred = ["ilo-jit", "ilo-vm"] + ordered = [i for i in preferred if i in all_impls] + sorted( + i for i in all_impls if i not in preferred + ) + + lines: list[str] = [ + f"", + "", + "## Performance", + "", + "Wall-clock time (lower is better). Baseline = Node.", + "", + ] + + # Header row + header = "| Benchmark | " + " | ".join(ordered) + " |" + separator = "| --- | " + " | ".join(["---"] * len(ordered)) + " |" + lines += [header, separator] + + for bench, times in sorted(benchmarks.items()): + baseline = times.get("Node") or next(iter(times.values())) + cells: list[str] = [] + for impl in ordered: + if impl not in times: + cells.append("—") + else: + t = times[impl] + r = ratio(t, baseline) + cells.append(f"{fmt_us(t)} ({r})") + lines.append("| " + bench + " | " + " | ".join(cells) + " |") + + lines += [""] + OUT.write_text("\n".join(lines)) + print(f"Wrote {OUT}") + + +if __name__ == "__main__": + main()