feat: add embedding regression test with real ML model validation

Add integration test that runs the embed+search pipeline with the real
minilm model to catch regressions from model updates, embedding text
format changes, or pipeline bugs. Includes a dedicated CI workflow
(weekly + on relevant PR changes) with HuggingFace model caching.

Author: github-actions[bot]

.github/workflows/embedding-regression.yml

@@ -0,0 +1,41 @@
+name: Embedding Regression
+
+on:
+  schedule:
+    - cron: '0 6 * * 1' # Monday 6am UTC
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - 'src/embedder.js'
+      - 'tests/search/**'
+      - 'package.json'
+
+concurrency:
+  group: embedding-regression-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  embedding-regression:
+    runs-on: ubuntu-latest
+    name: Embedding regression tests
+    timeout-minutes: 15
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 22
+
+      - name: Install dependencies
+        run: npm install
+
+      - name: Cache HuggingFace models
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/huggingface
+          key: hf-models-minilm-v1
+
+      - name: Run embedding regression tests
+        run: npx vitest run tests/search/embedding-regression.test.js

src/embedder.js

@@ -173,10 +173,10 @@ function initEmbeddingsSchema(db) {
 /**
  * Build embeddings for all functions/methods/classes in the graph.
  */
-export async function buildEmbeddings(rootDir, modelKey) {
+export async function buildEmbeddings(rootDir, modelKey, customDbPath) {
   // path already imported at top
   // fs already imported at top
-  const dbPath = findDbPath(null);
+  const dbPath = customDbPath || findDbPath(null);
 
   const db = new Database(dbPath);
   initEmbeddingsSchema(db);

tests/search/embedding-regression.test.js

@@ -0,0 +1,134 @@
+/**
+ * Embedding regression test — validates the embed+search pipeline
+ * produces semantically meaningful results with a real ML model.
+ *
+ * Skips automatically when @huggingface/transformers is not installed.
+ * Run explicitly: npx vitest run tests/search/embedding-regression.test.js
+ */
+
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import Database from 'better-sqlite3';
+import { afterAll, beforeAll, describe, expect, test } from 'vitest';
+
+// Detect whether transformers is available (optional dep)
+let hasTransformers = false;
+try {
+  await import('@huggingface/transformers');
+  hasTransformers = true;
+} catch {
+  // not installed — tests will skip
+}
+
+// Lazy-import to avoid top-level errors when transformers is missing
+const { buildGraph } = await import('../../src/builder.js');
+const { buildEmbeddings, searchData } = await import('../../src/embedder.js');
+
+// Same ES-module fixture files used by build.test.js
+const FIXTURE_FILES = {
+  'math.js': `
+export function add(a, b) { return a + b; }
+export function multiply(a, b) { return a * b; }
+export function square(x) { return multiply(x, x); }
+`.trimStart(),
+  'utils.js': `
+import { add, square } from './math.js';
+export function sumOfSquares(a, b) { return add(square(a), square(b)); }
+export class Calculator {
+  compute(x, y) { return sumOfSquares(x, y); }
+}
+`.trimStart(),
+  'index.js': `
+import { sumOfSquares, Calculator } from './utils.js';
+import { add } from './math.js';
+export function main() {
+  console.log(add(1, 2));
+  console.log(sumOfSquares(3, 4));
+  const calc = new Calculator();
+  console.log(calc.compute(5, 6));
+}
+`.trimStart(),
+};
+
+let tmpDir, dbPath;
+
+describe.skipIf(!hasTransformers)('embedding regression (real model)', () => {
+  beforeAll(async () => {
+    tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-embed-regression-'));
+    for (const [name, content] of Object.entries(FIXTURE_FILES)) {
+      fs.writeFileSync(path.join(tmpDir, name), content);
+    }
+
+    // Build the dependency graph
+    await buildGraph(tmpDir, { skipRegistry: true });
+    dbPath = path.join(tmpDir, '.codegraph', 'graph.db');
+
+    // Build embeddings with the smallest/fastest model
+    await buildEmbeddings(tmpDir, 'minilm', dbPath);
+  }, 120_000);
+
+  afterAll(() => {
+    if (tmpDir) fs.rmSync(tmpDir, { recursive: true, force: true });
+  });
+
+  describe('smoke tests', () => {
+    test('stored at least 6 embeddings', () => {
+      const db = new Database(dbPath, { readonly: true });
+      const count = db.prepare('SELECT COUNT(*) as c FROM embeddings').get().c;
+      db.close();
+      expect(count).toBeGreaterThanOrEqual(6);
+    });
+
+    test('metadata records correct model and dimension', () => {
+      const db = new Database(dbPath, { readonly: true });
+      const model = db.prepare("SELECT value FROM embedding_meta WHERE key = 'model'").get().value;
+      const dim = db.prepare("SELECT value FROM embedding_meta WHERE key = 'dim'").get().value;
+      db.close();
+      expect(model).toBe('Xenova/all-MiniLM-L6-v2');
+      expect(Number(dim)).toBe(384);
+    });
+
+    test('search returns results with positive similarity', async () => {
+      const data = await searchData('add numbers', dbPath, { minScore: 0.01 });
+      expect(data).not.toBeNull();
+      expect(data.results.length).toBeGreaterThan(0);
+      for (const r of data.results) {
+        expect(r.similarity).toBeGreaterThan(0);
+      }
+    });
+  });
+
+  describe('regression queries', () => {
+    /**
+     * Helper: search for a query and assert that a given function name
+     * appears within the top N results.
+     */
+    async function expectInTopN(query, expectedName, topN) {
+      const data = await searchData(query, dbPath, { minScore: 0.01, limit: topN });
+      expect(data).not.toBeNull();
+      const names = data.results.map((r) => r.name);
+      expect(names).toContain(expectedName);
+    }
+
+    test('"add two numbers together" finds add in top 3', async () => {
+      await expectInTopN('add two numbers together', 'add', 3);
+    });
+
+    test('"multiply values" finds multiply in top 3', async () => {
+      await expectInTopN('multiply values', 'multiply', 3);
+    });
+
+    test('"compute the square of a number" finds square in top 3', async () => {
+      await expectInTopN('compute the square of a number', 'square', 3);
+    });
+
+    test('"sum of squares calculation" finds sumOfSquares in top 3', async () => {
+      await expectInTopN('sum of squares calculation', 'sumOfSquares', 3);
+    });
+
+    test('"main entry point function" finds main in top 5', async () => {
+      await expectInTopN('main entry point function', 'main', 5);
+    });
+  });
+});