Skip to content

Commit ecda364

Browse files
tobiutobiu
committed
Refactor Memory Core to use numeric timestamps for ChromaDB compatibility #7865 
1 parent 082e769 commit ecda364

4 files changed

Lines changed: 330 additions & 4 deletions

File tree

‎ai/examples/debug_session_state.mjs‎

Lines changed: 208 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,208 @@
1+
import {
2+
Memory_SessionService,
3+
Memory_ChromaManager,
4+
Memory_Config
5+
} from '../services.mjs';
6+
7+
async function debugSessionState() {
8+
console.log('🔍 Starting Session State Debugger...');
9+
10+
// 1. Initialize
11+
console.log('⏳ Waiting for Memory Core readiness...');
12+
try {
13+
await Memory_ChromaManager.ready();
14+
await Memory_SessionService.ready();
15+
console.log('✅ Memory Core Services Ready.');
16+
} catch (e) {
17+
console.error('❌ Failed to initialize services:', e);
18+
process.exit(1);
19+
}
20+
21+
// 2. Access Collections directly
22+
// The service exposes these as properties (getters)
23+
const memCol = Memory_SessionService.memoryCollection;
24+
const sumCol = Memory_SessionService.sessionsCollection;
25+
26+
if (!memCol || !sumCol) {
27+
console.error('❌ Collections not initialized in SessionService.');
28+
process.exit(1);
29+
}
30+
31+
console.log(`📂 Connected to collections: ${memCol.name}, ${sumCol.name}`);
32+
33+
// 3. Scan Data (Replicating logic with logging)
34+
const includeAll = true; // Scan everything to be safe
35+
const ONE_MONTH_MS = 30 * 24 * 60 * 60 * 1000;
36+
const minTimestamp = Date.now() - ONE_MONTH_MS;
37+
38+
console.log(`
39+
📊 Scanning Memories (Include All: ${includeAll})...`);
40+
41+
const limit = 2000;
42+
let offset = 0;
43+
let allMemories = [];
44+
let hasMore = true;
45+
46+
const memQuery = {
47+
include: ['metadatas'],
48+
limit
49+
};
50+
51+
if (!includeAll) {
52+
memQuery.where = { timestamp: { '$gt': minTimestamp } };
53+
}
54+
55+
while (hasMore) {
56+
memQuery.offset = offset;
57+
const batch = await memCol.get(memQuery);
58+
59+
if (batch.ids.length === 0) {
60+
hasMore = false;
61+
} else {
62+
allMemories = allMemories.concat(batch.metadatas);
63+
offset += limit;
64+
process.stdout.write(`\r Fetched ${allMemories.length} records...`);
65+
if (batch.ids.length < limit) hasMore = false;
66+
}
67+
}
68+
console.log(`\n ✅ Total Memories Found: ${allMemories.length}`);
69+
70+
// 4. Group Memories
71+
const sessions = {};
72+
allMemories.forEach(m => {
73+
if (!m.sessionId) return;
74+
if (!sessions[m.sessionId]) sessions[m.sessionId] = { count: 0, lastActive: m.timestamp };
75+
sessions[m.sessionId].count++;
76+
if (m.timestamp > sessions[m.sessionId].lastActive) sessions[m.sessionId].lastActive = m.timestamp;
77+
});
78+
79+
const sessionIds = Object.keys(sessions);
80+
console.log(` found ${sessionIds.length} unique sessions.`);
81+
82+
// 5. Scan Summaries
83+
console.log(`
84+
📊 Scanning Summaries...
85+
`);
86+
87+
offset = 0;
88+
hasMore = true;
89+
let allSummaries = [];
90+
const sumQuery = {
91+
include: ['metadatas'],
92+
limit
93+
};
94+
95+
if (!includeAll) {
96+
sumQuery.where = { timestamp: { '$gt': minTimestamp } };
97+
}
98+
99+
while (hasMore) {
100+
sumQuery.offset = offset;
101+
const batch = await sumCol.get(sumQuery);
102+
103+
if (batch.ids.length === 0) {
104+
hasMore = false;
105+
} else {
106+
allSummaries = allSummaries.concat(batch.metadatas);
107+
offset += limit;
108+
process.stdout.write(`\r Fetched ${allSummaries.length} records...`);
109+
if (batch.ids.length < limit) hasMore = false;
110+
}
111+
}
112+
console.log(`\n ✅ Total Summaries Found: ${allSummaries.length}`);
113+
114+
const summaryMap = {};
115+
allSummaries.forEach(m => {
116+
if (m.sessionId) summaryMap[m.sessionId] = m;
117+
});
118+
119+
// 6. Compare and Diagnose
120+
console.log(`
121+
🕵️ Diagnosing Session Status:`);
122+
console.log('------------------------------------------------------------------------------------------------------------------');
123+
console.log('| Session ID | Mem (DB) | Sum (DB) | Drift? | Status | Last Active |');
124+
console.log('------------------------------------------------------------------------------------------------------------------');
125+
126+
let candidates = 0;
127+
128+
// Sort sessions by last active (newest first)
129+
sessionIds.sort((a, b) => new Date(sessions[b].lastActive) - new Date(sessions[a].lastActive));
130+
131+
sessionIds.forEach(id => {
132+
const memCount = sessions[id].count;
133+
const sumData = summaryMap[id];
134+
const sumCount = sumData ? (sumData.memoryCount || 0) : 'N/A';
135+
const lastActive = sessions[id].lastActive;
136+
137+
let status = '';
138+
let drift = false;
139+
140+
if (sumCount === 'N/A') {
141+
status = 'MISSING SUMMARY';
142+
drift = true;
143+
} else if (memCount !== sumCount) {
144+
status = 'COUNT MISMATCH';
145+
drift = true;
146+
} else {
147+
status = 'SYNCED';
148+
}
149+
150+
if (drift) candidates++;
151+
152+
// Truncate ID for display if needed, or keep full
153+
const dispId = id.length > 36 ? id.substring(0, 33) + '...' : id.padEnd(36);
154+
155+
// Safely handle lastActive display
156+
let activeDisplay = String(lastActive);
157+
if (typeof lastActive === 'number') {
158+
activeDisplay = new Date(lastActive).toISOString();
159+
}
160+
161+
console.log(`| ${dispId} | ${String(memCount).padStart(8)} | ${String(sumCount).padStart(8)} | ${drift ? 'YES' : 'NO '} | ${status.padEnd(15)} | ${activeDisplay.padEnd(27)} |`);
162+
});
163+
console.log('------------------------------------------------------------------------------------------------------------------');
164+
console.log(`\nDiagnosis complete. Found ${candidates} sessions needing summarization.`);
165+
166+
console.log(`\n🧪 Verifying Service Logic (Memory_SessionService.findSessionsToSummarize(false))...`);
167+
let serviceCandidates = [];
168+
try {
169+
serviceCandidates = await Memory_SessionService.findSessionsToSummarize(false);
170+
console.log(` Service returned ${serviceCandidates.length} candidates:`, serviceCandidates);
171+
172+
const missing = sessionIds.filter(id => {
173+
// Logic: if my diagnosis said it needs update (drift=true) but service didn't find it
174+
const memCount = sessions[id].count;
175+
const sumData = summaryMap[id];
176+
const sumCount = sumData ? (sumData.memoryCount || 0) : undefined;
177+
const needsUpdate = (sumCount === undefined || memCount !== sumCount);
178+
return needsUpdate && !serviceCandidates.includes(id);
179+
});
180+
181+
if (missing.length > 0) {
182+
console.warn(` ⚠️ Service MISSED these candidates:`, missing);
183+
} else {
184+
console.log(` ✅ Service logic matches diagnosis.`);
185+
}
186+
187+
} catch (e) {
188+
console.error(' ❌ Service call failed:', e);
189+
}
190+
191+
if (serviceCandidates.length > 0) {
192+
console.log('\n🚀 Executing Summarization for Candidates...');
193+
for (const sessionId of serviceCandidates) {
194+
process.stdout.write(` Summarizing ${sessionId}... `);
195+
try {
196+
await Memory_SessionService.summarizeSession(sessionId);
197+
console.log('✅ Done');
198+
} catch(err) {
199+
console.log('❌ Failed', err.message);
200+
}
201+
}
202+
console.log('\n✨ Batch Summarization Complete.');
203+
}
204+
205+
process.exit(0);
206+
}
207+
208+
debugSessionState();

‎ai/examples/migrate_timestamps.mjs‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
import {
2+
Memory_ChromaManager,
3+
Memory_SessionService
4+
} from '../services.mjs';
5+
6+
async function migrateTimestamps() {
7+
console.log('🚀 Starting Timestamp Migration (String -> Number)...');
8+
9+
// 1. Initialize
10+
console.log('⏳ Waiting for Memory Core readiness...');
11+
try {
12+
await Memory_ChromaManager.ready();
13+
// SessionService initialization also ensures collections are ready
14+
await Memory_SessionService.ready();
15+
console.log('✅ Memory Core Services Ready.');
16+
} catch (e) {
17+
console.error('❌ Failed to initialize services:', e);
18+
process.exit(1);
19+
}
20+
21+
const memCol = Memory_SessionService.memoryCollection;
22+
const sumCol = Memory_SessionService.sessionsCollection;
23+
24+
if (!memCol || !sumCol) {
25+
console.error('❌ Collections not initialized.');
26+
process.exit(1);
27+
}
28+
29+
// Helper function to migrate a collection
30+
async function migrateCollection(collection, name) {
31+
console.log(`
32+
📂 Processing Collection: ${name}`);
33+
34+
let offset = 0;
35+
const limit = 2000;
36+
let hasMore = true;
37+
let totalProcessed = 0;
38+
let totalUpdated = 0;
39+
40+
while (hasMore) {
41+
// Fetch batch
42+
const batch = await collection.get({
43+
include: ['metadatas', 'embeddings', 'documents'], // Need everything to upsert/update safely
44+
limit,
45+
offset
46+
});
47+
48+
if (batch.ids.length === 0) {
49+
hasMore = false;
50+
break;
51+
}
52+
53+
const updates = {
54+
ids: [],
55+
embeddings: [],
56+
metadatas: [],
57+
documents: []
58+
};
59+
60+
// Process batch
61+
for (let i = 0; i < batch.ids.length; i++) {
62+
const id = batch.ids[i];
63+
const metadata = batch.metadatas[i];
64+
const currentTimestamp = metadata.timestamp;
65+
66+
// Check if migration is needed (is it a string?)
67+
if (typeof currentTimestamp === 'string') {
68+
const numericTimestamp = Date.parse(currentTimestamp);
69+
70+
if (!isNaN(numericTimestamp)) {
71+
// Create updated metadata
72+
const newMetadata = { ...metadata, timestamp: numericTimestamp };
73+
74+
updates.ids.push(id);
75+
// If embeddings/documents are missing in fetch, we can't strictly "update" just metadata
76+
// easily without potential issues depending on Chroma version,
77+
// but 'update' method usually takes ids and metadatas.
78+
// Let's try using collection.update which is designed for this.
79+
updates.metadatas.push(newMetadata);
80+
} else {
81+
console.warn(` ⚠️ Skipping ID ${id}: Invalid date string "${currentTimestamp}"`);
82+
}
83+
}
84+
}
85+
86+
// Perform updates for this batch
87+
if (updates.ids.length > 0) {
88+
// Use collection.update to only modify metadata
89+
// Note: ChromaDB JS client 'update' signature: { ids, embeddings?, metadatas?, documents? }
90+
await collection.update({
91+
ids: updates.ids,
92+
metadatas: updates.metadatas
93+
});
94+
95+
totalUpdated += updates.ids.length;
96+
process.stdout.write(`\r Migrated ${totalUpdated} records...`);
97+
}
98+
99+
totalProcessed += batch.ids.length;
100+
offset += limit;
101+
102+
if (batch.ids.length < limit) {
103+
hasMore = false;
104+
}
105+
}
106+
console.log(`\n ✅ ${name} complete. Scanned: ${totalProcessed}, Updated: ${totalUpdated}`);
107+
}
108+
109+
// Run migration for both collections
110+
await migrateCollection(memCol, 'Memories');
111+
await migrateCollection(sumCol, 'Summaries');
112+
113+
console.log('\n✨ Migration finished successfully.');
114+
process.exit(0);
115+
}
116+
117+
migrateTimestamps();

‎ai/mcp/server/memory-core/services/MemoryService.mjs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,8 @@ class MemoryService extends Base {
3535
async addMemory({ prompt, response, thought, sessionId }) {
3636
const collection = await ChromaManager.getMemoryCollection();
3737
const combinedText = `User Prompt: ${prompt}\nAgent Thought: ${thought}\nAgent Response: ${response}`;
38-
const timestamp = new Date().toISOString();
38+
const now = Date.now();
39+
const timestamp = new Date(now).toISOString();
3940
const memoryId = `mem_${timestamp}`;
4041
const embedding = await TextEmbeddingService.embedText(combinedText);
4142

@@ -51,7 +52,7 @@ class MemoryService extends Base {
5152
response,
5253
thought,
5354
sessionId,
54-
timestamp,
55+
timestamp: now,
5556
type: 'agent-interaction'
5657
}],
5758
documents: [combinedText]

‎ai/mcp/server/memory-core/services/SessionService.mjs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,7 @@ class SessionService extends Base {
167167
// Default: Last 30 Days only (limits scope to recent active work).
168168
// Override: All time (if includeAll is true).
169169
const ONE_MONTH_MS = 30 * 24 * 60 * 60 * 1000;
170-
const minTimestamp = new Date(Date.now() - ONE_MONTH_MS).toISOString();
170+
const minTimestamp = Date.now() - ONE_MONTH_MS;
171171
const limit = aiConfig.summarizationBatchLimit || 2000;
172172
const maxIterations= 1000; // Safety break: max 2M records (2000 * 1000)
173173

@@ -354,7 +354,7 @@ ${aggregatedContent}
354354
ids: [summaryId],
355355
embeddings: [embedding],
356356
metadatas: [{
357-
sessionId, timestamp: new Date().toISOString(), memoryCount: memories.ids.length,
357+
sessionId, timestamp: Date.now(), memoryCount: memories.ids.length,
358358
title, category, quality, productivity, impact, complexity, technologies: technologies.join(',')
359359
}],
360360
documents: [summary]

0 commit comments

Comments
 (0)