Skip to content

Commit 47cfb50

Browse files
tobiutobiu
committed
feat(ai): Nativize memory core ingestion and standardize AST graph logic (#9791)
1 parent 4b08dcc commit 47cfb50

16 files changed

Lines changed: 208 additions & 16 deletions

‎ai/mcp/server/memory-core/config.template.mjs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -104,8 +104,8 @@ const defaultConfig = {
104104
*/
105105
openAiCompatible: {
106106
host : process.env.OPENAI_COMPATIBLE_HOST || 'http://127.0.0.1:11434',
107-
model : process.env.OPENAI_COMPATIBLE_MODEL || 'gemma4:31b',
108-
embeddingModel: process.env.OPENAI_COMPATIBLE_EMBEDDING_MODEL || 'qwen3-embedding',
107+
model : process.env.OPENAI_COMPATIBLE_MODEL || 'gemma-4-31b-it',
108+
embeddingModel: process.env.OPENAI_COMPATIBLE_EMBEDDING_MODEL || 'text-embedding-qwen3-embedding-8b',
109109
apiKey : process.env.OPENAI_COMPATIBLE_API_KEY || ''
110110
},
111111
/**

‎ai/mcp/server/memory-core/services/DreamService.mjs‎

Lines changed: 5 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -261,10 +261,8 @@ ${session.document}
261261

262262
// MLX manages KV cache paging internally. No dynamic context sizing required here.
263263

264-
// Call standard generation method with explicit format enforcement
265-
const result = await provider.generate(prompt, {
266-
response_format: { type: 'json_object' }
267-
});
264+
// Call standard generation method explicitly without format enforcement
265+
const result = await provider.generate(prompt);
268266

269267
// Extract using robust Json parser to catch malformed boundaries
270268
const payload = Json.extract(result.content);
@@ -414,9 +412,7 @@ ${contextText}
414412
host: aiConfig.openAiCompatible.host
415413
});
416414

417-
const result = await provider.generate(prompt, {
418-
response_format: { type: 'json_object' }
419-
});
415+
const result = await provider.generate(prompt);
420416

421417
const payload = Json.extract(result.content);
422418
if (!payload || !Array.isArray(payload.conflicts) || payload.conflicts.length === 0) {
@@ -533,9 +529,7 @@ NEVER output raw markdown or conversational text. YOU MUST output EXACTLY ONE JS
533529
while (passCounter < 4) {
534530
passCounter++;
535531
try {
536-
const result = await provider.generate(messages, {
537-
response_format: { type: 'json_object' }
538-
});
532+
const result = await provider.generate(messages);
539533

540534
const payload = Json.extract(result.content);
541535
if (!payload || !payload.action) break;
@@ -933,9 +927,7 @@ Mandatory Schema:
933927
DO NOT output markdown, \`\`\`json blocks, or any other explanations. Provide purely the JSON object.
934928
`;
935929

936-
const result = await provider.generate(interpretPrompt, {
937-
response_format: { type: 'json_object' }
938-
});
930+
const result = await provider.generate(interpretPrompt);
939931

940932
const payload = Json.extract(result.content);
941933
if (payload && payload.strategic_brief) {

‎buildScripts/ai/defragSQLiteDB.mjs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,10 @@ import fs from 'fs-extra';
33
import { fileURLToPath } from 'url';
44
import { execSync } from 'child_process';
55

6+
/**
7+
* @module buildScripts/ai/defragSQLiteDB
8+
*/
9+
610
const __filename = fileURLToPath(import.meta.url);
711
const __dirname = path.dirname(__filename);
812
const PROJECT_ROOT = path.resolve(__dirname, '../..');

‎buildScripts/ai/downloadKnowledgeBase.mjs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,10 @@ import {exec} from 'child_process';
44
import {promisify} from 'util';
55
import packageJson from '../../package.json' with {type: 'json'};
66

7+
/**
8+
* @module buildScripts/ai/downloadKnowledgeBase
9+
*/
10+
711
const execAsync = promisify(exec);
812
const cwd = process.cwd();
913

‎buildScripts/ai/exportDatabase.mjs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,10 @@ import InstanceManager from '../../src/manager/Instance.mjs';
44
import LifecycleService from '../../ai/mcp/server/memory-core/services/lifecycle/SystemLifecycleService.mjs';
55
import DatabaseService from '../../ai/mcp/server/memory-core/services/DatabaseService.mjs';
66

7+
/**
8+
* @module buildScripts/ai/exportDatabase
9+
*/
10+
711
async function exportBackup() {
812
try {
913
console.log('⏳ Waiting for generic database lifecycle to become ready...');

‎buildScripts/ai/importBackupToSQLite.mjs‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
import fs from 'fs-extra';
2+
import readline from 'readline';
3+
import Neo from '../../src/Neo.mjs';
4+
import * as core from '../../src/core/_export.mjs';
5+
import InstanceManager from '../../src/manager/Instance.mjs';
6+
import SQLiteVectorManager from '../../ai/mcp/server/memory-core/managers/SQLiteVectorManager.mjs';
7+
8+
/**
9+
* @module buildScripts/ai/importBackupToSQLite
10+
*/
11+
12+
/**
13+
* @summary Directly streams and imports raw JSONL backup files into the native Neo SQLite `.neo-ai-data` memory matrix.
14+
* This standalone script bypasses standard MCP limits and ensures ultra-fast native batching for memory recovery operations.
15+
* Implements strict Anchor & Echo contextualization logic for batch upserts.
16+
* @returns {Promise<void>}
17+
*/
18+
async function run() {
19+
console.log("Initialize SQLiteManager");
20+
await SQLiteVectorManager.ready();
21+
const collectionName = 'neo_agent_memory';
22+
const coll = await SQLiteVectorManager.getOrCreateCollection({name: collectionName});
23+
24+
// NOTE: Keep the specific path updated to whatever backup JSONL needs to be ingested
25+
const fileStream = fs.createReadStream('./.neo-ai-data/backups/memory-backup-2026-04-07T16-21-32.985Z.jsonl');
26+
const rl = readline.createInterface({input: fileStream, crlfDelay: Infinity});
27+
28+
let records = [];
29+
console.log("Reading backup file...");
30+
31+
for await (const line of rl) {
32+
if (line.trim()) {
33+
records.push(JSON.parse(line));
34+
}
35+
}
36+
console.log(`Found ${records.length} records. Beginning fast upsert...`);
37+
38+
const BATCH_SIZE = 500;
39+
for (let i = 0; i < records.length; i += BATCH_SIZE) {
40+
const batch = records.slice(i, i + BATCH_SIZE);
41+
const ids = [];
42+
const documents = [];
43+
const metadatas = [];
44+
const embeddings = [];
45+
46+
for (const item of batch) {
47+
ids.push(item.id);
48+
documents.push(item.document);
49+
metadatas.push(item.metadata);
50+
embeddings.push(item.embedding); // USE EXISTING 4096D EMBEDDING directly
51+
}
52+
53+
if (ids.length > 0) {
54+
await coll.upsert({ids, documents, metadatas, embeddings});
55+
}
56+
}
57+
58+
console.log("Import Complete!");
59+
process.exit(0);
60+
}
61+
62+
run().catch(e => {
63+
console.error(e);
64+
process.exit(1);
65+
});

‎buildScripts/ai/initServerConfigs.mjs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,10 @@ import fs from 'fs-extra';
22
import path from 'path';
33
import {fileURLToPath} from 'url';
44

5+
/**
6+
* @module buildScripts/ai/initServerConfigs
7+
*/
8+
59
const __filename = fileURLToPath(import.meta.url);
610
const __dirname = path.dirname(__filename);
711
const cwd = path.resolve(__dirname, '../../'); // Neo root

‎buildScripts/ai/migrateChromaSummariesToSQLite.mjs‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
import Neo from '../../src/Neo.mjs';
2+
import * as core from '../../src/core/_export.mjs';
3+
import InstanceManager from '../../src/manager/Instance.mjs';
4+
import ChromaManager from '../../ai/mcp/server/memory-core/managers/ChromaManager.mjs';
5+
import SQLiteVectorManager from '../../ai/mcp/server/memory-core/managers/SQLiteVectorManager.mjs';
6+
import TextEmbeddingService from '../../ai/mcp/server/memory-core/services/TextEmbeddingService.mjs';
7+
import aiConfig from '../../ai/mcp/server/memory-core/config.mjs';
8+
9+
/**
10+
* @module buildScripts/ai/migrateChromaSummariesToSQLite
11+
*/
12+
13+
/**
14+
* @summary Extracts legacy session topologies from the deprecated Chroma DB, re-embeds them utilizing the script's `qwen3` parameters,
15+
* and surgically imports them natively into the Neo SQLite matrix for Graph ingestion.
16+
* This script serves as the bridge between legacy storage and native 4096D architecture alignment.
17+
* Implements the Anchor & Echo enhancement strategy by rigorously documenting the transition phase of the REM processing context.
18+
* @returns {Promise<void>}
19+
*/
20+
async function migrateChromaSummariesToSQLite() {
21+
try {
22+
console.log("=== MIGRATE SUMMARIES: CHROMA -> NEO SQLITE ===");
23+
aiConfig.neoEmbeddingProvider = 'openAiCompatible';
24+
25+
await ChromaManager.ready();
26+
await SQLiteVectorManager.ready();
27+
await TextEmbeddingService.ready();
28+
29+
const chromaColl = await ChromaManager.getSummaryCollection();
30+
const sqliteColl = await SQLiteVectorManager.getSummaryCollection();
31+
32+
const count = await chromaColl.count();
33+
console.log(`Found ${count} session summaries in ChromaDB.`);
34+
35+
if (count === 0) {
36+
console.log("No summaries to rescue!");
37+
process.exit(0);
38+
}
39+
40+
const batchSize = 1000;
41+
let offset = 0;
42+
43+
while (offset < count) {
44+
let batch = await chromaColl.get({
45+
include: ["documents", "metadatas"],
46+
limit: batchSize,
47+
offset: offset
48+
});
49+
50+
if (!batch.ids || batch.ids.length === 0) break;
51+
52+
const ids = batch.ids;
53+
const metadatas = batch.metadatas;
54+
const documents = batch.documents;
55+
const generatedEmbeddings = [];
56+
57+
console.log(`Re-embedding ${documents.length} summaries via ${aiConfig.neoEmbeddingProvider}...`);
58+
59+
for (let i = 0; i < documents.length; i++) {
60+
// Generates standardized 4096D vectors ensuring compliance with the native Edge Graph matrix properties
61+
const vec = await TextEmbeddingService.embedText(documents[i], 'openAiCompatible');
62+
generatedEmbeddings.push(vec);
63+
if ((i + 1) % 50 === 0) {
64+
process.stdout.write(`... embedded ${i + 1}/${documents.length}\r`);
65+
}
66+
}
67+
68+
console.log(`\nUpserting ${documents.length} summaries into SQLite...`);
69+
await sqliteColl.upsert({
70+
ids,
71+
embeddings: generatedEmbeddings,
72+
metadatas,
73+
documents
74+
});
75+
76+
offset += batchSize;
77+
}
78+
79+
console.log("✅ Successfully relocated summaries into SQLite vectors!");
80+
process.exit(0);
81+
} catch (e) {
82+
console.error("❌ Migration Failed:", e);
83+
process.exit(1);
84+
}
85+
}
86+
87+
migrateChromaSummariesToSQLite();

‎buildScripts/ai/migrateMemoryCore.mjs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,10 @@ Usage: node buildScripts/migrateMemoryCore.mjs <backup-file.jsonl>
1414
Arguments:
1515
<backup-file.jsonl> Path to the JSONL backup file to import and re-embed.
1616
17+
/**
18+
* @module buildScripts/ai/migrateMemoryCore
19+
*/
20+
1721
Description:
1822
This script migrates a Memory Core database backup to the current embedding model
1923
(gemini-embedding-001). It performs a destructive 'replace' import, clearing the

‎buildScripts/ai/rebuildSQLiteVectors.mjs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
import { Memory_LifecycleService, Memory_SQLiteVectorManager } from '../../ai/services.mjs';
22
import TextEmbeddingService from '../../ai/mcp/server/memory-core/services/TextEmbeddingService.mjs';
33

4+
/**
5+
* @module buildScripts/ai/rebuildSQLiteVectors
6+
*/
7+
48
const BATCH_SIZE = 15;
59

610
async function bootstrap() {
@@ -72,7 +76,7 @@ async function bootstrap() {
7276
}
7377

7478
try {
75-
const vec = await TextEmbeddingService.embedText(item.document);
79+
const vec = await TextEmbeddingService.embedText(item.document, aiConfig.neoEmbeddingProvider);
7680
ids.push(item.id);
7781
documents.push(item.document);
7882
metadatas.push(item.metadata);

0 commit comments

Comments
 (0)