diff --git a/packages/core/src/agents-file.ts b/packages/core/src/agents-file.ts index bd11790..2dc6c90 100644 --- a/packages/core/src/agents-file.ts +++ b/packages/core/src/agents-file.ts @@ -559,6 +559,10 @@ export function shouldImportLoreFile(projectPath: string): boolean { /** * Import knowledge entries from `.lore.md` into the local DB. * Parses the full file content (no section markers to split on). + * + * After a successful import, updates the file cache so that + * `shouldImportLoreFile()` fast-paths on the next check — the file + * content hasn't changed, only the DB was updated to match it. */ export function importLoreFile(projectPath: string): void { const fp = join(projectPath, LORE_FILE); @@ -569,4 +573,14 @@ export function importLoreFile(projectPath: string): void { if (!fileEntries.length) return; _importEntries(fileEntries, projectPath); + + // Update cache: DB now matches the file, so shouldImportLoreFile() can + // fast-path on the next check. We re-stat after import because the file + // hasn't changed — only the DB was updated to match it. + try { + const { mtimeMs } = statSync(fp); + setCache(fp, { mtimeMs, hash: hashSection(fileContent) }); + } catch { + // stat failure is non-fatal — worst case we re-import next time + } } diff --git a/packages/core/test/agents-file.test.ts b/packages/core/test/agents-file.test.ts index a0671bb..78d458f 100644 --- a/packages/core/test/agents-file.test.ts +++ b/packages/core/test/agents-file.test.ts @@ -1243,6 +1243,28 @@ describe("importLoreFile", () => { expect(second).toBe(first); }); + + test("importLoreFile updates cache so shouldImportLoreFile fast-paths afterwards", () => { + // Simulate a .lore.md from another machine (entries not in DB yet). + const fp = join(PROJECT, LORE_FILE); + writeFileSync( + fp, + `\n\n## Long-term Knowledge\n\n### Decision\n\n\n* **Auth**: OAuth2\n`, + "utf8", + ); + + // DB is empty, file has entries — should need import. + expect(shouldImportLoreFile(PROJECT)).toBe(true); + + // Import the entries. + importLoreFile(PROJECT); + const entry = ltm.get(TEST_UUIDS[0]); + expect(entry).not.toBeNull(); + + // After import, shouldImportLoreFile should return false WITHOUT needing + // an export cycle — importLoreFile itself updates the cache. + expect(shouldImportLoreFile(PROJECT)).toBe(false); + }); }); // --------------------------------------------------------------------------- diff --git a/packages/gateway/src/pipeline.ts b/packages/gateway/src/pipeline.ts index 1742dd4..557e0be 100644 --- a/packages/gateway/src/pipeline.ts +++ b/packages/gateway/src/pipeline.ts @@ -42,6 +42,7 @@ import { loreFileExists, shouldImport, importFromFile, + LORE_FILE, latReader, embedding, } from "@loreai/core"; @@ -194,6 +195,10 @@ export async function resetPipelineState(): Promise { } llmClient = null; activeInterceptor = undefined; + if (stopFileWatcher) { + stopFileWatcher(); + stopFileWatcher = null; + } if (stopIdleScheduler) { stopIdleScheduler(); stopIdleScheduler = null; @@ -277,6 +282,9 @@ let batchQueueEnabled = false; /** Cleanup function for the idle scheduler timer. */ let stopIdleScheduler: (() => void) | null = null; +/** Cleanup function for the .lore.md / agents-file watcher. */ +let stopFileWatcher: (() => void) | null = null; + /** Last seen session model ID — used for worker model discovery context. */ let lastSeenSessionModel: string | null = null; @@ -369,6 +377,115 @@ export function computeMaxTokens( return Math.min(headroom, Math.max(adaptive, MAX_TOKENS_FLOOR), ceiling); } +// --------------------------------------------------------------------------- +// Knowledge file import — shared by startup + file watcher + new-session check +// --------------------------------------------------------------------------- + +/** + * Attempt to import knowledge from `.lore.md` (preferred) or the agents file + * (AGENTS.md/CLAUDE.md, backward compat). Safe to call frequently — the + * underlying `shouldImportLoreFile()` / `shouldImport()` do mtime + content-hash + * checks and short-circuit when nothing changed. + * + * Returns true if entries were actually imported. + */ +function tryImportKnowledge(projectPath: string): boolean { + const cfg = loreConfig(); + if (!cfg.knowledge.enabled) return false; + + try { + if (loreFileExists(projectPath)) { + if (shouldImportLoreFile(projectPath)) { + importLoreFile(projectPath); + log.info("imported knowledge from .lore.md"); + return true; + } + } else if (cfg.agentsFile.enabled) { + const { join } = require("node:path") as typeof import("node:path"); + const filePath = join(projectPath, cfg.agentsFile.path); + if (shouldImport({ projectPath, filePath })) { + importFromFile({ projectPath, filePath }); + log.info("imported knowledge from", cfg.agentsFile.path); + return true; + } + } + } catch (e) { + log.error("knowledge import error:", e); + } + + return false; +} + +// --------------------------------------------------------------------------- +// File watcher for .lore.md / agents file — picks up external edits live +// --------------------------------------------------------------------------- + +/** + * Start watching `.lore.md` (and the agents file as fallback) for changes. + * Uses `fs.watch()` with a debounce to avoid rapid-fire triggers from + * editors that do atomic write-rename sequences. + * + * Safe against import-after-export loops: `shouldImportLoreFile()` compares + * the file content hash against what the DB would produce, so our own + * exports are recognized as no-ops. + */ +function startKnowledgeFileWatcher(projectPath: string): () => void { + const { join } = require("node:path") as typeof import("node:path"); + const { watch, existsSync } = require("node:fs") as typeof import("node:fs"); + + const watchers: import("node:fs").FSWatcher[] = []; + let debounceTimer: ReturnType | null = null; + const DEBOUNCE_MS = 500; + + const onFileChange = () => { + // Debounce: editors often write-rename-delete in rapid succession. + if (debounceTimer) clearTimeout(debounceTimer); + debounceTimer = setTimeout(() => { + debounceTimer = null; + tryImportKnowledge(projectPath); + }, DEBOUNCE_MS); + }; + + // Watch .lore.md + const loreFilePath = join(projectPath, LORE_FILE); + if (existsSync(loreFilePath)) { + try { + const w = watch(loreFilePath, onFileChange); + w.on("error", () => {}); // suppress — file may be deleted + watchers.push(w); + } catch { + // watch not supported (rare) — fall back to session-start checks only + } + } + + // Watch agents file (AGENTS.md etc.) as fallback + const cfg = loreConfig(); + if (cfg.agentsFile.enabled) { + const agentsFilePath = join(projectPath, cfg.agentsFile.path); + if (existsSync(agentsFilePath)) { + try { + const w = watch(agentsFilePath, onFileChange); + w.on("error", () => {}); + watchers.push(w); + } catch { + // watch not supported + } + } + } + + if (watchers.length > 0) { + log.info(`watching ${watchers.length} knowledge file(s) for changes`); + } + + return () => { + if (debounceTimer) clearTimeout(debounceTimer); + for (const w of watchers) { + try { w.close(); } catch { /* already closed */ } + } + watchers.length = 0; + }; +} + // --------------------------------------------------------------------------- // Initialization // --------------------------------------------------------------------------- @@ -389,29 +506,18 @@ async function initIfNeeded(projectPath: string, config?: GatewayConfig): Promis // since last session). Falls back to agents file for backward compat. const cfg = loreConfig(); if (cfg.knowledge.enabled) { - try { - const { join } = await import("node:path"); - if (loreFileExists(projectPath)) { - if (shouldImportLoreFile(projectPath)) { - importLoreFile(projectPath); - log.info("imported knowledge from .lore.md"); - } - } else if (cfg.agentsFile.enabled) { - const filePath = join(projectPath, cfg.agentsFile.path); - if (shouldImport({ projectPath, filePath })) { - importFromFile({ projectPath, filePath }); - log.info("imported knowledge from", cfg.agentsFile.path); - } - } - } catch (e) { - log.error("startup knowledge import error:", e); - } + tryImportKnowledge(projectPath); // Prune corrupted/oversized knowledge entries (safety net for past bugs). const pruned = ltm.pruneOversized(1200); if (pruned > 0) { log.info(`pruned ${pruned} oversized knowledge entries (confidence set to 0)`); } + + // Watch knowledge files for live changes (git pull, manual edits, etc.) + if (!stopFileWatcher) { + stopFileWatcher = startKnowledgeFileWatcher(projectPath); + } } // Startup backfills — idempotent, run once per process. @@ -1655,6 +1761,13 @@ async function handleConversationTurn( const result = learnHeaders(sessionState.candidateHeaders, req.rawHeaders); sessionState.candidateHeaders = result.updatedCandidates; } + + // Re-check knowledge files on new session start. The file watcher + // covers live edits, but this catches cases where: + // - The watcher wasn't set up (file didn't exist at startup) + // - The watcher missed an event (e.g. network-mounted fs) + // - The file was created after gateway startup (first export from another machine) + tryImportKnowledge(projectPath); } // --- Compaction anomaly detection ---