fix(ci): save all benchmark reports and use git-based dev versioning (#196)

* fix(cli): remove branch-compare references to non-existent module

The branch-compare command was registered in cli.js and its exports
added to index.js, but the implementation file src/branch-compare.js
was never created. This caused two issues:

1. `codegraph branch-compare` crashed with ERR_MODULE_NOT_FOUND
2. `import('@optave/codegraph')` crashed entirely because index.js
   has a top-level re-export from the missing file, making the
   programmatic API completely unusable

Remove the dead references until an implementation exists.

Closes #166

* fix: recover branch-compare implementation from lost worktree files

The branch-compare command was registered in cli.js and index.js but
src/branch-compare.js was never committed — it was lost as untracked
files in the fix/complexity-sql-sanitize worktree. Recovered the full
implementation (568 lines) and integration test (192 lines, 7 tests)
from git object 22c8185.

This restores the cli.js command and index.js exports that were removed
in 746aa65, now that the implementation file exists. Updated the dogfood
report to reflect the recovery (no bugs found, rating 8.5/10).

Closes #166

Impact: 13 functions changed, 4 affected

* fix(ci): use commit-count versioning for dev builds and benchmarks

Dev builds always produced 2.4.1-dev.<sha> because PATCH was hardcoded
to +1 from package.json. Now uses git rev-list to count commits since
the release tag, giving monotonically increasing versions like 2.4.5,
2.4.15, etc.

Benchmark scripts now use getBenchmarkVersion() (via git describe) to
detect dev vs release state, preventing dev runs from overwriting
release benchmark data in historical reports.

Impact: 2 functions changed, 6 affected

* fix(bench): integrate getBenchmarkVersion into shared bench-config

The bench-config.js local mode returned 'dev' as the version string.
Now uses getBenchmarkVersion() to derive a proper semver from git state,
so all 4 benchmark scripts that use bench-config get dev-aware versions.

Impact: 1 functions changed, 4 affected

* fix(ci): align version strategy between publish.yml and bench-version.js

Both now use git describe --tags --match "v*" to find the nearest
release tag and count commits from it. Previously publish.yml
constructed the tag from package.json (v${CURRENT}) while
bench-version.js used git describe, which would diverge if
package.json was bumped but not yet tagged.

Impact: 1 functions changed, 0 affected

* fix(bench): align no-tags fallback with publish.yml behavior

When no git tags exist, bench-version.js now produces PATCH+1 (e.g.
"2.5.1-dev") matching publish.yml's fallback, instead of the bare
"2.5.0-dev" which diverged.

Impact: 1 functions changed, 0 affected

* fix(bench): use same two-step git strategy as publish.yml

Refactor getBenchmarkVersion to use --abbrev=0 + git rev-list (same
as publish.yml) instead of parsing full git describe output. Both
now use identical steps: find tag, count commits, get short SHA.

Impact: 1 functions changed, 0 affected

* fix(bench): include SHA in no-tags fallback to match publish.yml

Impact: 1 functions changed, 0 affected

* fix(ci): save all benchmark reports and use git-based dev versioning

Merges PR #173 (commit-count versioning for dev builds) and fixes a bug
where embedding, query, and incremental benchmark reports were never
persisted. The "Check for changes" step used `git diff` which only
detects modifications to tracked files — newly created report files
(untracked) were invisible, so the PR was never created. Now also checks
for untracked files via `git ls-files --others`.

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

Author: carlos-alm

.github/workflows/benchmark.yml

@@ -92,11 +92,16 @@ jobs:
       - name: Check for changes
         id: changes
         run: |
-          if git diff --quiet HEAD -- generated/BUILD-BENCHMARKS.md README.md; then
-            echo "changed=false" >> "$GITHUB_OUTPUT"
-          else
-            echo "changed=true" >> "$GITHUB_OUTPUT"
+          CHANGED=false
+          # Detect modified tracked files
+          if ! git diff --quiet HEAD -- generated/BUILD-BENCHMARKS.md README.md 2>/dev/null; then
+            CHANGED=true
+          fi
+          # Detect newly created (untracked) files
+          if [ -n "$(git ls-files --others --exclude-standard generated/BUILD-BENCHMARKS.md)" ]; then
+            CHANGED=true
           fi
+          echo "changed=$CHANGED" >> "$GITHUB_OUTPUT"
 
       - name: Commit and push via PR
         if: steps.changes.outputs.changed == 'true'
@@ -212,11 +217,16 @@ jobs:
       - name: Check for changes
         id: changes
         run: |
-          if git diff --quiet HEAD -- generated/EMBEDDING-BENCHMARKS.md; then
-            echo "changed=false" >> "$GITHUB_OUTPUT"
-          else
-            echo "changed=true" >> "$GITHUB_OUTPUT"
+          CHANGED=false
+          # Detect modified tracked files
+          if ! git diff --quiet HEAD -- generated/EMBEDDING-BENCHMARKS.md 2>/dev/null; then
+            CHANGED=true
+          fi
+          # Detect newly created (untracked) files
+          if [ -n "$(git ls-files --others --exclude-standard generated/EMBEDDING-BENCHMARKS.md)" ]; then
+            CHANGED=true
           fi
+          echo "changed=$CHANGED" >> "$GITHUB_OUTPUT"
 
       - name: Commit and push via PR
         if: steps.changes.outputs.changed == 'true'
@@ -320,11 +330,16 @@ jobs:
       - name: Check for changes
         id: changes
         run: |
-          if git diff --quiet HEAD -- generated/QUERY-BENCHMARKS.md; then
-            echo "changed=false" >> "$GITHUB_OUTPUT"
-          else
-            echo "changed=true" >> "$GITHUB_OUTPUT"
+          CHANGED=false
+          # Detect modified tracked files
+          if ! git diff --quiet HEAD -- generated/QUERY-BENCHMARKS.md 2>/dev/null; then
+            CHANGED=true
+          fi
+          # Detect newly created (untracked) files
+          if [ -n "$(git ls-files --others --exclude-standard generated/QUERY-BENCHMARKS.md)" ]; then
+            CHANGED=true
           fi
+          echo "changed=$CHANGED" >> "$GITHUB_OUTPUT"
 
       - name: Commit and push via PR
         if: steps.changes.outputs.changed == 'true'
@@ -428,11 +443,16 @@ jobs:
       - name: Check for changes
         id: changes
         run: |
-          if git diff --quiet HEAD -- generated/INCREMENTAL-BENCHMARKS.md; then
-            echo "changed=false" >> "$GITHUB_OUTPUT"
-          else
-            echo "changed=true" >> "$GITHUB_OUTPUT"
+          CHANGED=false
+          # Detect modified tracked files
+          if ! git diff --quiet HEAD -- generated/INCREMENTAL-BENCHMARKS.md 2>/dev/null; then
+            CHANGED=true
+          fi
+          # Detect newly created (untracked) files
+          if [ -n "$(git ls-files --others --exclude-standard generated/INCREMENTAL-BENCHMARKS.md)" ]; then
+            CHANGED=true
           fi
+          echo "changed=$CHANGED" >> "$GITHUB_OUTPUT"
 
       - name: Commit and push via PR
         if: steps.changes.outputs.changed == 'true'

.github/workflows/publish.yml

@@ -44,6 +44,8 @@ jobs:
       npm_tag: ${{ steps.compute.outputs.npm_tag }}
     steps:
       - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
 
       - name: Compute version
         id: compute
@@ -65,12 +67,21 @@ jobs:
             NPM_TAG="latest"
             echo "Stable release (manual retry): $VERSION"
           else
-            IFS='.' read -r MAJOR MINOR PATCH <<< "$CURRENT"
-            DEV_PATCH=$((PATCH + 1))
+            # Use git describe to find the nearest release tag — same strategy
+            # as scripts/bench-version.js so versions are always consistent.
+            RELEASE_TAG=$(git describe --tags --match "v*" --abbrev=0 2>/dev/null || echo "")
+            if [ -n "$RELEASE_TAG" ]; then
+              COMMITS=$(git rev-list "${RELEASE_TAG}..HEAD" --count)
+              IFS='.' read -r MAJOR MINOR PATCH <<< "${RELEASE_TAG#v}"
+            else
+              COMMITS=1
+              IFS='.' read -r MAJOR MINOR PATCH <<< "$CURRENT"
+            fi
+            DEV_PATCH=$((PATCH + COMMITS))
             SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7)
             VERSION="${MAJOR}.${MINOR}.${DEV_PATCH}-dev.${SHORT_SHA}"
             NPM_TAG="dev"
-            echo "Dev release: $VERSION"
+            echo "Dev release: $VERSION (${COMMITS} commits since ${RELEASE_TAG:-none})"
           fi
 
           echo "version=$VERSION" >> "$GITHUB_OUTPUT"

scripts/bench-version.js

@@ -0,0 +1,61 @@
+/**
+ * Compute the benchmark version string from git state.
+ *
+ * Uses the same two-step strategy as publish.yml's compute-version job:
+ *   1. `git describe --tags --match "v*" --abbrev=0` → find nearest release tag
+ *   2. `git rev-list <tag>..HEAD --count` → count commits since that tag
+ *
+ * - If HEAD is exactly tagged (0 commits): returns "2.5.0"
+ * - Otherwise: returns "2.5.N-dev.hash" (e.g. "2.5.3-dev.c50f7f5")
+ *   where N = PATCH + commits since tag, hash = short commit SHA
+ *
+ * This prevents dev/dogfood benchmark runs from overwriting release data
+ * in the historical benchmark reports (which deduplicate by version).
+ */
+
+import { execFileSync } from 'node:child_process';
+
+const GIT_OPTS = { encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'] };
+
+export function getBenchmarkVersion(pkgVersion, cwd) {
+	try {
+		// Step 1: find the nearest release tag (mirrors publish.yml --abbrev=0)
+		const tag = execFileSync('git', ['describe', '--tags', '--match', 'v*', '--abbrev=0'], {
+			cwd,
+			...GIT_OPTS,
+		}).trim();
+
+		// Step 2: count commits since that tag (mirrors publish.yml git rev-list)
+		const commits = Number(
+			execFileSync('git', ['rev-list', `${tag}..HEAD`, '--count'], { cwd, ...GIT_OPTS }).trim(),
+		);
+
+		const m = tag.match(/^v(\d+)\.(\d+)\.(\d+)$/);
+		if (!m) return `${pkgVersion}-dev`;
+
+		const [, major, minor, patch] = m;
+
+		// Exact tag (0 commits since tag): return clean release version
+		if (commits === 0) return `${major}.${minor}.${patch}`;
+
+		// Dev build: MAJOR.MINOR.(PATCH+COMMITS)-dev.SHORT_SHA
+		const hash = execFileSync('git', ['rev-parse', '--short', 'HEAD'], { cwd, ...GIT_OPTS }).trim();
+		const devPatch = Number(patch) + commits;
+		return `${major}.${minor}.${devPatch}-dev.${hash}`;
+	} catch {
+		/* git not available or no tags */
+	}
+
+	// Fallback: no git or no tags — match publish.yml's no-tags behavior (PATCH+1-dev.SHA)
+	const parts = pkgVersion.split('.');
+	if (parts.length === 3) {
+		const [major, minor, patch] = parts;
+		try {
+			const hash = execFileSync('git', ['rev-parse', '--short', 'HEAD'], { cwd, ...GIT_OPTS }).trim();
+			return `${major}.${minor}.${Number(patch) + 1}-dev.${hash}`;
+		} catch {
+			return `${major}.${minor}.${Number(patch) + 1}-dev`;
+		}
+	}
+	return `${pkgVersion}-dev`;
+}

scripts/lib/bench-config.js

@@ -13,6 +13,8 @@ import os from 'node:os';
 import path from 'node:path';
 import { pathToFileURL } from 'node:url';
 
+import { getBenchmarkVersion } from '../bench-version.js';
+
 /**
  * Parse `--version <v>` and `--npm` from process.argv.
  */
@@ -44,10 +46,11 @@ export async function resolveBenchmarkSource() {
 	const { version: cliVersion, npm } = parseArgs();
 
 	if (!npm) {
-		// Local mode — use repo src/, label as "dev" unless overridden
+		// Local mode — use repo src/, version derived from git state
 		const root = path.resolve(path.dirname(new URL(import.meta.url).pathname.replace(/^\/([A-Z]:)/, '$1')), '..', '..');
+		const pkg = JSON.parse(fs.readFileSync(path.join(root, 'package.json'), 'utf8'));
 		return {
-			version: cliVersion || 'dev',
+			version: cliVersion || getBenchmarkVersion(pkg.version, root),
 			srcDir: path.join(root, 'src'),
 			cleanup() {},
 		};

scripts/token-benchmark.js

@@ -26,10 +26,12 @@ import { fileURLToPath } from 'node:url';
 import { parseArgs } from 'node:util';
 
 import { ISSUES, extractAgentOutput, validateResult } from './token-benchmark-issues.js';
+import { getBenchmarkVersion } from './bench-version.js';
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const root = path.resolve(__dirname, '..');
 const pkg = JSON.parse(fs.readFileSync(path.join(root, 'package.json'), 'utf8'));
+const benchVersion = getBenchmarkVersion(pkg.version, root);
 
 // Redirect console.log to stderr so only JSON goes to stdout
 const origLog = console.log;
@@ -590,7 +592,7 @@ async function main() {
 	console.log = origLog;
 
 	const output = {
-		version: pkg.version,
+		version: benchVersion,
 		date: new Date().toISOString().slice(0, 10),
 		model: MODEL,
 		runsPerIssue: RUNS,