Skip to content

Commit 2599576

Browse files
authored
feat(memory-core): backfill memory miniSummary nodes (#12673) (#12676)
1 parent fdfd386 commit 2599576

12 files changed

Lines changed: 398 additions & 5 deletions

File tree

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
/**
2+
* Reads pending `AGENT_MEMORY` rows that still lack `miniSummary`.
3+
*
4+
* @summary Cheap scheduler-side existence check; the supervised lifecycle child owns the actual
5+
* Chroma join, model call, and graph update.
6+
* @param {Object} db SQLite database handle.
7+
* @param {Object} [options]
8+
* @param {Number} [options.limit=50] Maximum ids to return.
9+
* @returns {String[]}
10+
*/
11+
export function getPendingMemorySummaryBackfillJobs(db, {limit = 50} = {}) {
12+
if (!db?.prepare) {
13+
return [];
14+
}
15+
16+
const numericLimit = Number.isInteger(limit) && limit > 0 ? limit : 50;
17+
18+
try {
19+
return db.prepare(`
20+
SELECT memory.id AS id
21+
FROM Nodes memory
22+
WHERE json_extract(memory.data, '$.label') = 'AGENT_MEMORY'
23+
AND json_extract(memory.data, '$.properties.miniSummary') IS NULL
24+
ORDER BY json_extract(memory.data, '$.properties.timestamp') DESC, memory.id DESC
25+
LIMIT ?
26+
`).all(numericLimit).map(row => row.id).filter(Boolean);
27+
} catch {
28+
return [];
29+
}
30+
}
31+
32+
/**
33+
* Builds the task trigger for memory miniSummary backfill.
34+
*
35+
* @param {Object} options
36+
* @param {String[]} [options.pendingJobs=[]] Pending AGENT_MEMORY ids.
37+
* @returns {Object|null}
38+
*/
39+
export function buildMemorySummaryBackfillTrigger({pendingJobs = []} = {}) {
40+
if (pendingJobs.length > 0) {
41+
return {
42+
taskName : 'memory-summary-backfill',
43+
source : 'pending-memory-minisummary',
44+
reason : `pending-memory-minisummary:${pendingJobs.length}`,
45+
pendingCount: pendingJobs.length
46+
};
47+
}
48+
49+
return null;
50+
}
51+
52+
/**
53+
* Resolves the next memory miniSummary backfill trigger.
54+
*
55+
* @param {Object} options
56+
* @param {Object} options.db SQLite database handle.
57+
* @param {Function} [options.getPendingMemorySummaryBackfillJobsFn] Test seam.
58+
* @returns {Object|null}
59+
*/
60+
export function getDueTask({
61+
db,
62+
getPendingMemorySummaryBackfillJobsFn = getPendingMemorySummaryBackfillJobs
63+
}) {
64+
return buildMemorySummaryBackfillTrigger({
65+
pendingJobs: getPendingMemorySummaryBackfillJobsFn(db)
66+
});
67+
}

ai/daemons/orchestrator/scheduling/registry.mjs

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import {getDueTask as getBackupDueTask} from './backup.mjs';
33
import {getDueTask as getDreamDueTask} from './dream.mjs';
44
import {getDueTask as getGraphLogCompactionDueTask} from './graphLogCompaction.mjs';
55
import {getDueTask as getGoldenPathDueTask} from './goldenPath.mjs';
6+
import {getDueTask as getMemorySummaryBackfillDueTask} from './memorySummaryBackfill.mjs';
67
import {getDueTask as getPrimaryDevSyncDueTask} from './primaryDevSync.mjs';
78
import {getDueTask as getSwarmHeartbeatDueTask} from './swarmHeartbeat.mjs';
89
import {getDueTask as getTenantRepoSyncDueTask} from './tenantRepoSync.mjs';
@@ -49,6 +50,16 @@ export const TASK_REGISTRY = Object.freeze([
4950
});
5051
}
5152
},
53+
{
54+
taskName : 'memory-summary-backfill',
55+
executionKind : 'supervised-child-process',
56+
maintenanceClass: 'heavy',
57+
backpressure : 'exclusive-heavy',
58+
dependencies : [],
59+
getDueTask({db}) {
60+
return getMemorySummaryBackfillDueTask({db});
61+
}
62+
},
5263
{
5364
taskName : 'kbSync',
5465
executionKind : 'supervised-child-process',

ai/daemons/orchestrator/services/MaintenanceBackpressureService.mjs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import {DEFAULT_DATA_DIR} from '../taskDefinitions.mjs';
1717
*/
1818
export const DEFAULT_HEAVY_MAINTENANCE_TASK_NAMES = Object.freeze([
1919
'summary',
20+
'memory-summary-backfill',
2021
'kbSync',
2122
'backup',
2223
'graphlog-compaction',

ai/daemons/orchestrator/taskDefinitions.mjs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,13 @@ export function buildTaskDefinitions({
8787
pidFileName : 'summarization.pid',
8888
expectedCommand: 'summarize-sessions.mjs'
8989
},
90+
'memory-summary-backfill': {
91+
label : 'memory miniSummary backfill',
92+
command : nodeBin,
93+
args : [path.join(scriptDir, 'lifecycle', 'backfill-memory-summaries.mjs')],
94+
pidFileName : 'memory-summary-backfill.pid',
95+
expectedCommand: 'backfill-memory-summaries.mjs'
96+
},
9097
kbSync: {
9198
label : 'knowledge base sync',
9299
command : nodeBin,
Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
#!/usr/bin/env node
2+
/**
3+
* @summary Lifecycle child that backfills `AGENT_MEMORY.miniSummary` in bounded batches.
4+
*
5+
* The Orchestrator schedules this as a supervised child process, mirroring
6+
* `summarize-sessions.mjs`. The business logic remains in `MemoryService` so this entry point is
7+
* a thin bootstrap + observability wrapper and can also be run manually for recovery.
8+
*
9+
* Exit codes:
10+
* - 0: batch completed (including successful no-op or fail-soft deferred rows)
11+
* - 1: lifecycle or storage failure prevented the batch from running
12+
*
13+
* @see ai/services/memory-core/MemoryService.backfillMiniSummaries
14+
* @see ai/scripts/lifecycle/summarize-sessions.mjs
15+
*/
16+
import Neo from '../../../src/Neo.mjs';
17+
import * as core from '../../../src/core/_export.mjs';
18+
import {withHeavyMaintenanceLease} from '../../daemons/orchestrator/services/HeavyMaintenanceLeaseService.mjs';
19+
import LifecycleService from '../../services/memory-core/lifecycle/SystemLifecycleService.mjs';
20+
import MemoryService from '../../services/memory-core/MemoryService.mjs';
21+
22+
async function main() {
23+
const outcome = await withHeavyMaintenanceLease(
24+
async () => {
25+
await LifecycleService.initAsync();
26+
return MemoryService.backfillMiniSummaries();
27+
},
28+
{
29+
owner : 'memory-summary-backfill',
30+
reason : 'manual-cli',
31+
metadata: {script: 'ai/scripts/lifecycle/backfill-memory-summaries.mjs'}
32+
}
33+
);
34+
35+
if (outcome.status === 'held') {
36+
const held = outcome.lease;
37+
console.log(JSON.stringify({
38+
success : true,
39+
deferred: true,
40+
reason : 'heavy-maintenance-lease-held',
41+
holder : held ? {
42+
owner : held.owner,
43+
reason : held.reason,
44+
pid : held.pid,
45+
acquiredAt: held.acquiredAt
46+
} : null
47+
}));
48+
process.exit(0)
49+
}
50+
51+
console.log(JSON.stringify({success: true, ...(outcome.result || {})}));
52+
process.exit(0)
53+
}
54+
55+
main().catch(error => {
56+
console.error('backfill-memory-summaries failed:', error.message);
57+
process.exit(1)
58+
});

ai/services/memory-core/MemoryService.mjs

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -703,6 +703,130 @@ class MemoryService extends Base {
703703
}
704704
}
705705

706+
/**
707+
* @summary Merges a generated `miniSummary` into an existing `AGENT_MEMORY` node without changing tenant ownership.
708+
*
709+
* Backfill runs as daemon/system work, usually without a request-bound tenant. Plain
710+
* {@link GraphService#upsertNode} is intentionally RLS-aware and may not see private
711+
* tenant rows from that context, so it can treat an existing tenant-owned memory as a
712+
* new node. The backfill path must instead update the full persisted node JSON in
713+
* place, preserving `userId`, `agentIdentity`, `sessionId`, timestamps, and every
714+
* other property while adding only `miniSummary`.
715+
*
716+
* @param {Object} options
717+
* @param {String} options.id Existing `AGENT_MEMORY` node id.
718+
* @param {String} options.miniSummary Generated compact summary to merge.
719+
* @returns {Boolean} `true` when the node was updated, `false` when the row is missing.
720+
* @private
721+
*/
722+
updateMemoryMiniSummary({id, miniSummary}) {
723+
const graph = GraphService.db,
724+
sqlite = graph?.storage?.db;
725+
726+
if (!sqlite) {
727+
return false;
728+
}
729+
730+
const row = sqlite.prepare('SELECT data FROM Nodes WHERE id = ? LIMIT 1').get(id);
731+
if (!row?.data) {
732+
return false;
733+
}
734+
735+
const existing = JSON.parse(row.data),
736+
properties = {...(existing.properties || {}), miniSummary},
737+
nodeData = {
738+
id : existing.id || id,
739+
label : existing.label || 'AGENT_MEMORY',
740+
properties
741+
};
742+
743+
graph.storage.addNodes([nodeData]);
744+
745+
return true;
746+
}
747+
748+
/**
749+
* @summary Backfills compact per-turn summaries for existing `AGENT_MEMORY` graph rows.
750+
*
751+
* Mirrors the inline {@link addMemory} enrichment path for pre-existing memories and for
752+
* turns written while the summarizer was unavailable. The scan is graph-first and
753+
* most-recent-first; Chroma is only joined by the selected node ids to fetch that memory's own
754+
* prompt/response. Updates merge `miniSummary` into the same graph node through a
755+
* tenant-preserving storage-layer merge, preserving tenant attribution (`userId`, `agentIdentity`)
756+
* and every other property already present on the row.
757+
*
758+
* Fail-soft by construction: model/provider failures leave the row unmodified so a later batch
759+
* can retry it. A failure for one row never aborts the batch.
760+
*
761+
* @param {Object} [options]
762+
* @param {Number} [options.limit] Maximum rows to process. Defaults to
763+
* `aiConfig.summarizationBatchLimit`.
764+
* @param {Function} [options.buildMiniSummary] Optional summarizer seam for deterministic tests.
765+
* @returns {Promise<{processed: Number, updated: Number, deferred: Number, missingContent: Number}>}
766+
*/
767+
async backfillMiniSummaries({limit, buildMiniSummary} = {}) {
768+
const sqlite = GraphService.db?.storage?.db;
769+
if (!sqlite) {
770+
return {processed: 0, updated: 0, deferred: 0, missingContent: 0};
771+
}
772+
773+
const defaultLimit = Number(aiConfig.summarizationBatchLimit) || 50;
774+
const numericLimit = Number(limit) || defaultLimit;
775+
const boundedLimit = Math.max(1, Math.min(numericLimit, defaultLimit));
776+
const summarize = buildMiniSummary || (options => this.buildMiniSummary(options));
777+
778+
const rows = sqlite.prepare(`
779+
SELECT memory.id AS id,
780+
json_extract(memory.data, '$.properties.timestamp') AS timestamp
781+
FROM Nodes memory
782+
WHERE json_extract(memory.data, '$.label') = 'AGENT_MEMORY'
783+
AND json_extract(memory.data, '$.properties.miniSummary') IS NULL
784+
ORDER BY json_extract(memory.data, '$.properties.timestamp') DESC, memory.id DESC
785+
LIMIT ?
786+
`).all(boundedLimit);
787+
788+
if (rows.length === 0) {
789+
return {processed: 0, updated: 0, deferred: 0, missingContent: 0};
790+
}
791+
792+
const collection = await StorageRouter.getMemoryCollection();
793+
const fetched = await collection.get({ids: rows.map(row => row.id), include: ['metadatas']});
794+
const byId = new Map((fetched.ids || []).map((id, index) => [id, fetched.metadatas?.[index] || {}]));
795+
796+
let updated = 0, deferred = 0, missingContent = 0;
797+
798+
for (const row of rows) {
799+
const metadata = byId.get(row.id);
800+
if (!metadata || (!metadata.prompt && !metadata.response)) {
801+
missingContent++;
802+
continue;
803+
}
804+
805+
try {
806+
const miniSummary = await summarize({
807+
prompt : metadata.prompt,
808+
response: metadata.response
809+
});
810+
811+
if (!miniSummary) {
812+
deferred++;
813+
continue;
814+
}
815+
816+
if (this.updateMemoryMiniSummary({id: row.id, miniSummary})) {
817+
updated++;
818+
} else {
819+
missingContent++;
820+
}
821+
} catch (error) {
822+
logger.warn(`[MemoryService] miniSummary backfill deferred for ${row.id} (fail-soft): ${error.message}`);
823+
deferred++;
824+
}
825+
}
826+
827+
return {processed: rows.length, updated, deferred, missingContent};
828+
}
829+
706830
/**
707831
* Executes a semantic search against the memory collection.
708832
* @param {Object} options

test/playwright/unit/ai/daemons/orchestrator/Orchestrator.spec.mjs

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,7 @@ test.describe('Neo.ai.daemons.Orchestrator (#11009)', () => {
167167
nodeBin : '/node'
168168
}));
169169

170-
expect(Object.keys(state)).toEqual(['chroma', 'bridgeDaemon', 'summary', 'kbSync', 'backup', 'graphlog-compaction', 'chromaDefrag', 'primary-dev-sync', 'tenant-repo-sync', 'dream', 'golden-path', 'swarm-heartbeat']);
170+
expect(Object.keys(state)).toEqual(['chroma', 'bridgeDaemon', 'summary', 'memory-summary-backfill', 'kbSync', 'backup', 'graphlog-compaction', 'chromaDefrag', 'primary-dev-sync', 'tenant-repo-sync', 'dream', 'golden-path', 'swarm-heartbeat']);
171171
expect(state.mlx).toBeUndefined();
172172
expect(state.memoryCoreChroma).toBeUndefined();
173173
expect(state.summary).toMatchObject({
@@ -729,9 +729,11 @@ test.describe('Neo.ai.daemons.Orchestrator (#11009)', () => {
729729
expect(orchestrator.stateFile).toBe(path.join(dataDir, 'orchestrator-state.json'));
730730

731731
const expectedSummaryScript = path.resolve(repoRoot, 'ai/scripts/lifecycle/summarize-sessions.mjs');
732-
const expectedKbSyncScript = path.resolve(repoRoot, 'ai/scripts/maintenance/syncKnowledgeBase.mjs');
732+
const expectedBackfillScript = path.resolve(repoRoot, 'ai/scripts/lifecycle/backfill-memory-summaries.mjs');
733+
const expectedKbSyncScript = path.resolve(repoRoot, 'ai/scripts/maintenance/syncKnowledgeBase.mjs');
733734

734735
expect(orchestrator.taskDefinitions.summary.args[0]).toBe(expectedSummaryScript);
736+
expect(orchestrator.taskDefinitions['memory-summary-backfill'].args[0]).toBe(expectedBackfillScript);
735737
expect(orchestrator.taskDefinitions.kbSync.args[0]).toBe(expectedKbSyncScript);
736738
expect(orchestrator.taskDefinitions.mlx).toBeUndefined();
737739
});
@@ -906,6 +908,7 @@ test.describe('Neo.ai.daemons.Orchestrator (#11009)', () => {
906908
'backup',
907909
'graphlog-compaction',
908910
'kbSync',
911+
'memory-summary-backfill',
909912
'primary-dev-sync',
910913
'dream',
911914
'summary'

test/playwright/unit/ai/daemons/orchestrator/daemon.spec.mjs

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,10 @@ test.describe('ai/daemons/orchestrator/daemon.mjs (#11006/#11009)', () => {
2626
expect(tasks.summary.args).toEqual([path.join(scriptDir, 'lifecycle', 'summarize-sessions.mjs')]);
2727
expect(tasks.summary.expectedCommand).toBe('summarize-sessions.mjs');
2828

29+
expect(tasks['memory-summary-backfill'].command).toBe('/test/node');
30+
expect(tasks['memory-summary-backfill'].args).toEqual([path.join(scriptDir, 'lifecycle', 'backfill-memory-summaries.mjs')]);
31+
expect(tasks['memory-summary-backfill'].expectedCommand).toBe('backfill-memory-summaries.mjs');
32+
2933
expect(tasks.kbSync.command).toBe('/test/node');
3034
expect(tasks.kbSync.args).toEqual([path.join(scriptDir, 'maintenance', 'syncKnowledgeBase.mjs')]);
3135
expect(tasks.kbSync.expectedCommand).toBe('syncKnowledgeBase.mjs');
@@ -43,6 +47,17 @@ test.describe('ai/daemons/orchestrator/daemon.mjs (#11006/#11009)', () => {
4347
expect(tasks['graphlog-compaction'].expectedCommand).toBe('compactGraphLog.mjs');
4448
});
4549

50+
test('memory-summary backfill CLI is guarded by the shared heavy-maintenance lease', () => {
51+
const source = fs.readFileSync(
52+
path.resolve(process.cwd(), 'ai/scripts/lifecycle/backfill-memory-summaries.mjs'),
53+
'utf8'
54+
);
55+
56+
expect(source).toContain('withHeavyMaintenanceLease');
57+
expect(source).toContain("owner : 'memory-summary-backfill'");
58+
expect(source).toContain("reason : 'manual-cli'");
59+
});
60+
4661
test('buildTaskDefinitions is pure: tasks.mlx is omitted when mlxEnabled is false', () => {
4762
const scriptDir = path.resolve(process.cwd(), 'ai/scripts');
4863
const tasks = buildTaskDefinitions({scriptDir, nodeBin: '/test/node'});

0 commit comments

Comments
 (0)