feat(memory): add default MemoryHydeRetriever, auto-attach when LLM available

Memory-specific HyDE retriever generates hypothetical stored traces for
improved recall on vague queries. Auto-attached in CognitiveMemoryManager
when any LLM invoker is configured. Remains opt-in per query via
retrieve({ hyde: true }). Based on the generation effect from cognitive
psychology.

Author: jddunn

src/memory/CognitiveMemoryManager.ts

@@ -342,6 +342,19 @@ export class CognitiveMemoryManager implements ICognitiveMemoryManager {
       }
     }
 
+    // --- HyDE Retriever (auto-attached when any LLM invoker is available) ---
+    // Generates hypothetical memory traces for improved recall on vague queries.
+    // Opt-in per query via retrieve({ hyde: true }). Based on the "generation
+    // effect" — generating what a memory WOULD look like activates retrieval
+    // pathways more effectively than raw query embedding.
+    const anyLlmInvoker = config.reflector?.llmInvoker
+      ?? config.observer?.llmInvoker
+      ?? config.featureDetectionLlmInvoker;
+    if (anyLlmInvoker && !this.hydeRetriever) {
+      const { MemoryHydeRetriever } = await import('./retrieval/hyde/MemoryHydeRetriever.js');
+      this.hydeRetriever = new MemoryHydeRetriever(anyLlmInvoker) as unknown as HydeRetriever;
+    }
+
     this.initialized = true;
   }
 

src/memory/retrieval/hyde/MemoryHydeRetriever.ts

@@ -0,0 +1,103 @@
+/**
+ * @fileoverview Memory-specific HyDE (Hypothetical Document Embedding) retriever.
+ *
+ * Improves memory recall for vague or abstract queries by generating a
+ * hypothetical memory trace BEFORE embedding. The hypothesis is closer
+ * in embedding space to actual stored traces than the raw query.
+ *
+ * Cognitive science grounding: this mirrors the "generation effect" —
+ * generating information about a topic activates related neural pathways
+ * more strongly than passive recognition. By generating what a memory
+ * WOULD look like, we activate the right retrieval pathways.
+ *
+ * Effective for:
+ * - Abstract queries ("that deployment discussion")
+ * - Emotional recall ("when they were upset")
+ * - Temporal queries ("something from last week")
+ * - Vague references ("the thing about cats")
+ *
+ * Auto-attached by CognitiveMemoryManager when any LLM invoker is available.
+ * Remains opt-in per query via `options.hyde: true` on `retrieve()`.
+ *
+ * @module agentos/memory/retrieval/hyde/MemoryHydeRetriever
+ * @see {@link CognitiveMemoryManager.retrieve} — consumes the hypothesis
+ */
+
+/** LLM invoker function signature matching AgentOS observer/reflector convention. */
+type LlmInvoker = (systemPrompt: string, userPrompt: string) => Promise<string>;
+
+/**
+ * System prompt for hypothesis generation.
+ *
+ * Instructs the LLM to output what a STORED memory trace would look like,
+ * not to answer the query. This produces embeddings that are semantically
+ * closer to actual stored traces than raw recall queries.
+ */
+const HYDE_SYSTEM_PROMPT = `You are a memory system. Given a recall query, generate what a stored memory trace about this topic would look like. Write it as a first-person observation note, 1-2 sentences, as if you recorded it when it happened.
+
+Do NOT answer the query. Generate what the STORED MEMORY would say.
+
+Examples:
+Query: "what does the user do for work?"
+Hypothesis: "User mentioned they are a software engineer working on backend systems at a startup."
+
+Query: "when were they upset?"
+Hypothesis: "User expressed frustration and stress about a missed deadline. Emotional tone was tense."
+
+Query: "that thing about cats"
+Hypothesis: "User talked about having two cats named Luna and Mochi. They seem important to the user."`;
+
+/**
+ * Memory-specific HyDE retriever that generates hypothetical memory traces.
+ *
+ * Implements the same `generateHypothesis()` interface expected by
+ * CognitiveMemoryManager so it can be assigned via `setHydeRetriever()`.
+ *
+ * Lightweight: uses `maxTokens: 150` with no chain-of-thought. Target
+ * latency is under 500ms with a fast model.
+ *
+ * @example
+ * ```ts
+ * const retriever = new MemoryHydeRetriever(llmInvoker);
+ * const result = await retriever.generateHypothesis('what does the user like?');
+ * // result.hypothesis = "User mentioned they enjoy hiking and cooking..."
+ * ```
+ */
+export class MemoryHydeRetriever {
+  private readonly llmInvoker: LlmInvoker;
+
+  /**
+   * @param llmInvoker - Function that calls an LLM with (systemPrompt, userPrompt).
+   *   Typically reused from the observer, reflector, or feature detection config.
+   */
+  constructor(llmInvoker: LlmInvoker) {
+    this.llmInvoker = llmInvoker;
+  }
+
+  /**
+   * Generate a hypothetical memory trace for a recall query.
+   *
+   * The generated hypothesis is used as the embedding input for vector
+   * search, producing results that are more semantically aligned with
+   * actual stored traces.
+   *
+   * Returns the same shape as `HydeRetriever.generateHypothesis()` so
+   * CognitiveMemoryManager can use it interchangeably.
+   *
+   * @param query - The recall query (e.g., "what does the user do for work?")
+   * @returns Object with `hypothesis` text and `latencyMs` timing
+   */
+  async generateHypothesis(query: string): Promise<{ hypothesis: string; latencyMs: number }> {
+    const start = Date.now();
+    try {
+      const hypothesis = await this.llmInvoker(HYDE_SYSTEM_PROMPT, `Query: "${query}"`);
+      return {
+        hypothesis: hypothesis.trim(),
+        latencyMs: Date.now() - start,
+      };
+    } catch {
+      // HyDE is non-critical — return empty hypothesis to fall through to raw query
+      return { hypothesis: '', latencyMs: Date.now() - start };
+    }
+  }
+}

src/memory/retrieval/hyde/__tests__/MemoryHydeRetriever.test.ts

@@ -0,0 +1,54 @@
+import { describe, it, expect, vi } from 'vitest';
+import { MemoryHydeRetriever } from '../MemoryHydeRetriever.js';
+
+describe('MemoryHydeRetriever', () => {
+  it('generates a hypothetical memory trace for a recall query', async () => {
+    const llmInvoker = vi.fn().mockResolvedValue(
+      'User mentioned they are a software engineer working on backend systems.'
+    );
+
+    const retriever = new MemoryHydeRetriever(llmInvoker);
+    const result = await retriever.generateHypothesis('what does the user do for work?');
+
+    expect(result.hypothesis).toBe(
+      'User mentioned they are a software engineer working on backend systems.'
+    );
+    expect(result.latencyMs).toBeGreaterThanOrEqual(0);
+    expect(llmInvoker).toHaveBeenCalledTimes(1);
+
+    // System prompt should instruct to generate a stored memory, not answer the query
+    const systemPrompt = llmInvoker.mock.calls[0][0] as string;
+    expect(systemPrompt).toContain('STORED MEMORY');
+    expect(systemPrompt).toContain('Do NOT answer the query');
+  });
+
+  it('returns empty hypothesis when LLM fails', async () => {
+    const llmInvoker = vi.fn().mockRejectedValue(new Error('LLM unavailable'));
+    const retriever = new MemoryHydeRetriever(llmInvoker);
+    const result = await retriever.generateHypothesis('test query');
+
+    expect(result.hypothesis).toBe('');
+    expect(result.latencyMs).toBeGreaterThanOrEqual(0);
+  });
+
+  it('trims whitespace from hypothesis', async () => {
+    const llmInvoker = vi.fn().mockResolvedValue(
+      '  User likes hiking and cooking.  \n'
+    );
+
+    const retriever = new MemoryHydeRetriever(llmInvoker);
+    const result = await retriever.generateHypothesis('what does the user like?');
+
+    expect(result.hypothesis).toBe('User likes hiking and cooking.');
+  });
+
+  it('passes the query in the user prompt', async () => {
+    const llmInvoker = vi.fn().mockResolvedValue('hypothesis');
+    const retriever = new MemoryHydeRetriever(llmInvoker);
+
+    await retriever.generateHypothesis('tell me about their family');
+
+    const userPrompt = llmInvoker.mock.calls[0][1] as string;
+    expect(userPrompt).toContain('tell me about their family');
+  });
+});