feat(foraging): add examples_list, examples_query, examples_integrate_plan MCP tools

.changeset/foraging-mcp-tools.md

@@ -0,0 +1,37 @@
+---
+"@colony/foraging": minor
+"@colony/core": minor
+"@colony/storage": minor
+"@colony/mcp-server": minor
+---
+
+Expose foraged food sources to MCP clients through three new tools and
+wire `MemoryStore.search` with an optional kind/metadata filter so
+scoped queries don't pollute the main search.
+
+New MCP tools (registered alongside spec in `apps/mcp-server`):
+
+- `examples_list({ repo_root })` — compact list of indexed example
+  names, manifest kinds, and cached observation counts.
+- `examples_query({ query, example_name?, limit? })` — BM25 hits
+  scoped to `kind = 'foraged-pattern'` and optionally to a specific
+  example. Returns compact snippets — fetch full bodies via
+  `get_observations`.
+- `examples_integrate_plan({ repo_root, example_name, target_hint? })`
+  — deterministic plan: npm dependency delta between the example and
+  the target `package.json`, files to copy (derived from indexed
+  entrypoints), `config_steps` (npm scripts), and an
+  `uncertainty_notes` list for everything the planner couldn't
+  resolve. No LLM in the loop.
+
+`@colony/foraging` adds `buildIntegrationPlan(storage, opts)`. The
+function reads manifests fresh from disk to avoid round-tripping
+structured JSON through the compressor.
+
+`@colony/core` extends `MemoryStore.search(query, limit?, embedder?, filter?)`
+with `{ kind?: string; metadata?: Record<string, string> }`. When a
+filter is set the method skips vector ranking — the embedding index has
+no kind column, so mixing vector hits would require a second pass to
+drop them. `@colony/storage`'s `searchFts(query, limit, filter?)`
+applies the filter in SQL via `json_extract` so the LIMIT still bounds
+the scan.

apps/mcp-server/package.json

@@ -19,6 +19,7 @@
     "@colony/config": "workspace:*",
     "@colony/core": "workspace:*",
     "@colony/embedding": "workspace:*",
+    "@colony/foraging": "workspace:*",
     "@colony/hooks": "workspace:*",
     "@colony/process": "workspace:*",
     "@colony/spec": "workspace:*",

apps/mcp-server/src/server.ts

@@ -7,6 +7,7 @@ import { isMainEntry } from '@colony/process';
 import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
 import type { ToolContext } from './tools/context.js';
+import * as foraging from './tools/foraging.js';
 import * as handoff from './tools/handoff.js';
 import { installActiveSessionHeartbeat } from './tools/heartbeat.js';
 import * as hivemind from './tools/hivemind.js';
@@ -77,6 +78,11 @@ export function buildServer(store: MemoryStore, settings: Settings): McpServer {
   // core tool first.
   spec.register(server, ctx);
 
+  // Foraging lane (@colony/foraging). Adds examples_list, examples_query,
+  // examples_integrate_plan. Registered after spec so the heartbeat has
+  // wrapped the earlier tools before we bind these three.
+  foraging.register(server, ctx);
+
   return server;
 }
 

apps/mcp-server/src/tools/foraging.ts

@@ -0,0 +1,70 @@
+import { buildIntegrationPlan } from '@colony/foraging';
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import { z } from 'zod';
+import type { ToolContext } from './context.js';
+
+/**
+ * Foraging surface exposed to MCP clients.
+ *
+ * Progressive disclosure: `examples_list` and `examples_query` return
+ * compact shapes. Full observation bodies are fetched by
+ * `get_observations(ids[])`, which already exists in search.ts. Keeps
+ * the contract tight enough that a single `examples_query` call stays
+ * under the MCP response-size budget even on large example sets.
+ */
+export function register(server: McpServer, ctx: ToolContext): void {
+  const { store, resolveEmbedder } = ctx;
+
+  server.tool(
+    'examples_list',
+    'List indexed example projects (food sources) for a repo root.',
+    { repo_root: z.string().min(1) },
+    async ({ repo_root }) => {
+      const rows = store.storage.listExamples(repo_root);
+      const compact = rows.map((r) => ({
+        example_name: r.example_name,
+        manifest_kind: r.manifest_kind,
+        observation_count: r.observation_count,
+        last_scanned_at: r.last_scanned_at,
+      }));
+      return { content: [{ type: 'text', text: JSON.stringify(compact) }] };
+    },
+  );
+
+  server.tool(
+    'examples_query',
+    'Search indexed example patterns. Compact hits — fetch bodies via get_observations.',
+    {
+      query: z.string().min(1),
+      example_name: z.string().optional(),
+      limit: z.number().int().positive().max(20).optional(),
+    },
+    async ({ query, example_name, limit }) => {
+      const e = (await resolveEmbedder()) ?? undefined;
+      const filter: { kind: string; metadata?: Record<string, string> } = {
+        kind: 'foraged-pattern',
+      };
+      if (example_name) filter.metadata = { example_name };
+      const hits = await store.search(query, limit ?? 10, e, filter);
+      return { content: [{ type: 'text', text: JSON.stringify(hits) }] };
+    },
+  );
+
+  server.tool(
+    'examples_integrate_plan',
+    'Build an integration plan: dependency delta + files to copy + config steps.',
+    {
+      example_name: z.string().min(1),
+      repo_root: z.string().min(1),
+      target_hint: z.string().optional(),
+    },
+    async ({ example_name, repo_root, target_hint }) => {
+      const plan = buildIntegrationPlan(store.storage, {
+        example_name,
+        repo_root,
+        ...(target_hint !== undefined ? { target_hint } : {}),
+      });
+      return { content: [{ type: 'text', text: JSON.stringify(plan) }] };
+    },
+  );
+}

apps/mcp-server/test/foraging.test.ts

@@ -0,0 +1,133 @@
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { defaultSettings } from '@colony/config';
+import { MemoryStore } from '@colony/core';
+import { scanExamples } from '@colony/foraging';
+import { Client } from '@modelcontextprotocol/sdk/client/index.js';
+import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js';
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import { buildServer } from '../src/server.js';
+
+let dir: string;
+let repoRoot: string;
+let store: MemoryStore;
+let client: Client;
+
+function write(rel: string, contents: string): void {
+  const abs = join(repoRoot, rel);
+  mkdirSync(join(abs, '..'), { recursive: true });
+  writeFileSync(abs, contents);
+}
+
+beforeEach(async () => {
+  dir = mkdtempSync(join(tmpdir(), 'colony-mcp-forage-'));
+  repoRoot = join(dir, 'repo');
+  mkdirSync(repoRoot, { recursive: true });
+  store = new MemoryStore({ dbPath: join(dir, 'data.db'), settings: defaultSettings });
+  store.startSession({ id: 'mcp-session', ide: 'test', cwd: repoRoot });
+
+  const server = buildServer(store, defaultSettings);
+  const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair();
+  client = new Client({ name: 'forage-test', version: '0.0.0' });
+  await Promise.all([server.connect(serverTransport), client.connect(clientTransport)]);
+});
+
+afterEach(async () => {
+  await client.close();
+  store.close();
+  rmSync(dir, { recursive: true, force: true });
+});
+
+async function callJson<T>(name: string, args: Record<string, unknown>): Promise<T> {
+  const res = await client.callTool({ name, arguments: args });
+  const content = (res.content as Array<{ type: string; text: string }>)[0];
+  if (!content || content.type !== 'text') throw new Error(`unexpected MCP reply for ${name}`);
+  return JSON.parse(content.text) as T;
+}
+
+describe('MCP foraging tools', () => {
+  it('examples_list returns the compact rows for a scanned repo', async () => {
+    write('package.json', JSON.stringify({ name: 'target' }));
+    write('examples/stripe/package.json', JSON.stringify({ name: 'stripe' }));
+    write('examples/stripe/src/index.ts', 'export {}');
+    scanExamples({ repo_root: repoRoot, store, session_id: 'mcp-session' });
+
+    const rows = await callJson<
+      Array<{ example_name: string; manifest_kind: string | null; observation_count: number }>
+    >('examples_list', { repo_root: repoRoot });
+
+    expect(rows).toHaveLength(1);
+    expect(rows[0]).toMatchObject({ example_name: 'stripe', manifest_kind: 'npm' });
+    expect(rows[0]?.observation_count).toBeGreaterThan(0);
+  });
+
+  it('examples_query returns compact hits scoped to foraged-pattern rows', async () => {
+    write('package.json', JSON.stringify({ name: 'target' }));
+    write(
+      'examples/stripe/package.json',
+      JSON.stringify({ name: 'stripe', dependencies: { stripe: '^14.0.0' } }),
+    );
+    scanExamples({ repo_root: repoRoot, store, session_id: 'mcp-session' });
+    // Add a *non*-foraged observation with the same keyword — it must not
+    // show up in the scoped query.
+    store.addObservation({
+      session_id: 'mcp-session',
+      kind: 'note',
+      content: 'A random mention of stripe that should not match a foraged query.',
+    });
+
+    const hits = await callJson<Array<{ id: number; snippet: string }>>('examples_query', {
+      query: 'stripe',
+    });
+    expect(hits.length).toBeGreaterThan(0);
+
+    // Every hit id must be a foraged-pattern row.
+    for (const h of hits) {
+      const row = store.storage.getObservation(h.id);
+      expect(row?.kind).toBe('foraged-pattern');
+    }
+  });
+
+  it('examples_query honors the example_name filter', async () => {
+    write('examples/alpha/package.json', JSON.stringify({ name: 'alpha' }));
+    write('examples/beta/package.json', JSON.stringify({ name: 'beta' }));
+    scanExamples({ repo_root: repoRoot, store, session_id: 'mcp-session' });
+
+    const hits = await callJson<Array<{ id: number }>>('examples_query', {
+      query: 'alpha',
+      example_name: 'alpha',
+    });
+
+    expect(hits.length).toBeGreaterThan(0);
+    for (const h of hits) {
+      const row = store.storage.getObservation(h.id);
+      const md = row?.metadata ? (JSON.parse(row.metadata) as { example_name: string }) : null;
+      expect(md?.example_name).toBe('alpha');
+    }
+  });
+
+  it('examples_integrate_plan returns a deterministic plan', async () => {
+    write('package.json', JSON.stringify({ name: 'target', dependencies: { zod: '^3.23.0' } }));
+    write(
+      'examples/stripe/package.json',
+      JSON.stringify({
+        name: 'stripe',
+        dependencies: { zod: '^3.23.0', stripe: '^14.0.0' },
+        scripts: { build: 'tsc' },
+      }),
+    );
+    scanExamples({ repo_root: repoRoot, store, session_id: 'mcp-session' });
+
+    const plan = await callJson<{
+      example_name: string;
+      dependency_delta: { add: Record<string, string>; remove: string[] };
+      config_steps: string[];
+    }>('examples_integrate_plan', { repo_root: repoRoot, example_name: 'stripe' });
+
+    expect(plan.example_name).toBe('stripe');
+    expect(plan.dependency_delta.add.stripe).toBe('^14.0.0');
+    expect(plan.dependency_delta.add.zod).toBeUndefined();
+    expect(plan.config_steps).toContain('npm run build');
+  });
+});

apps/mcp-server/test/server.test.ts

@@ -49,6 +49,9 @@ describe('MCP server', () => {
       'agent_get_profile',
       'agent_upsert_profile',
       'attention_inbox',
+      'examples_integrate_plan',
+      'examples_list',
+      'examples_query',
       'get_observations',
       'hivemind',
       'hivemind_context',
@@ -395,13 +398,7 @@ describe('MCP server', () => {
         isolatedClient.connect(clientTransport),
       ]);
 
-      const sessionFile = join(
-        repoRoot,
-        '.omx',
-        'state',
-        'active-sessions',
-        'hb-session-1.json',
-      );
+      const sessionFile = join(repoRoot, '.omx', 'state', 'active-sessions', 'hb-session-1.json');
       const afterConnect = JSON.parse(readFileSync(sessionFile, 'utf8'));
       expect(afterConnect.sessionKey).toBe('hb-session-1');
       expect(afterConnect.branch).toBe('hb-branch');

packages/core/src/memory-store.ts

@@ -117,10 +117,23 @@ export class MemoryStore {
 
   // --- search ---
 
-  async search(query: string, limit?: number, embedder?: Embedder): Promise<SearchResult[]> {
+  async search(
+    query: string,
+    limit?: number,
+    embedder?: Embedder,
+    filter?: { kind?: string; metadata?: Record<string, string> },
+  ): Promise<SearchResult[]> {
     const cap = limit ?? this.settings.search.defaultLimit;
     const alpha = this.settings.search.alpha;
-    const keyword = this.storage.searchFts(query, cap * 2);
+    const keyword = this.storage.searchFts(query, cap * 2, filter);
+    // When the caller scopes the result to a `kind` / `metadata` pair,
+    // skip vector ranking: the embedding index has no kind filter, so
+    // mixing vector hits would bring back observations from other kinds
+    // and force a second pass to drop them. The filtered FTS output is
+    // already scoped correctly — keyword-only is faster and cleaner.
+    if (filter && (filter.kind || (filter.metadata && Object.keys(filter.metadata).length > 0))) {
+      return keyword.slice(0, cap);
+    }
     if (!embedder || this.settings.embedding.provider === 'none') {
       return keyword.slice(0, cap);
     }

packages/foraging/src/index.ts

@@ -4,6 +4,8 @@ export { extract, readCapped } from './extractor.js';
 export type { ExtractedShape } from './extractor.js';
 export { indexFoodSource } from './indexer.js';
 export type { IndexFoodSourceOptions } from './indexer.js';
+export { buildIntegrationPlan } from './integration-plan.js';
+export type { BuildIntegrationPlanOptions } from './integration-plan.js';
 export { redact } from './redact.js';
 export type {
   ExampleManifestKind,