Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion packages/core/src/distillation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -623,6 +623,10 @@ export async function run(input: {
skipMeta?: boolean;
urgent?: boolean;
callType?: "batch" | "direct";
/** Override the meta-distillation gen-0 threshold. When set, meta-distillation
* triggers at this count instead of `cfg.distillation.metaThreshold`.
* Used by the urgent-distillation path to consolidate earlier under bust pressure. */
metaThresholdOverride?: number;
}): Promise<{ rounds: number; distilled: number }> {
return distillLimiter.get(input.sessionID)(() => runInner(input));
}
Expand All @@ -648,6 +652,8 @@ async function runInner(input: {
/** Whether the LLM call will use batch or direct pricing. Recorded on the
* distillation row for accurate historical cost estimates. */
callType?: "batch" | "direct";
/** Override the meta-distillation gen-0 threshold (see run()). */
metaThresholdOverride?: number;
}): Promise<{ rounds: number; distilled: number }> {
// Reset orphaned messages (marked distilled by a deleted/migrated distillation)
const orphans = resetOrphans(input.projectPath, input.sessionID);
Expand Down Expand Up @@ -708,10 +714,11 @@ async function runInner(input: {
// Check if meta-distillation is needed (skip when cache is warm to avoid
// prefix cache invalidation — row IDs change after meta-distill, busting
// the prompt cache on the next turn).
const effectiveMetaThreshold = input.metaThresholdOverride ?? cfg.distillation.metaThreshold;
if (
!input.skipMeta &&
gen0Count(input.projectPath, input.sessionID) >=
cfg.distillation.metaThreshold
effectiveMetaThreshold
) {
// Call inner directly — we're already under the per-session limiter.
await metaDistillInner({
Expand Down
20 changes: 19 additions & 1 deletion packages/core/src/gradient.ts
Original file line number Diff line number Diff line change
Expand Up @@ -603,6 +603,15 @@ export function inspectSessionState(sessionID: string): {
};
}

/**
* Return the consecutive-bust counter for a session.
* Used by the gateway idle handler and urgent-distillation scheduler to
* lower the meta-distillation threshold under bust pressure.
*/
export function getConsecutiveBusts(sessionID: string): number {
return getSessionState(sessionID).consecutiveBusts;
}

/**
* For testing only — set the session's lastTurnAt field. Used to simulate
* idle gaps without sleeping. Creates the session state if not present so
Expand Down Expand Up @@ -1703,6 +1712,7 @@ function transformInner(input: {
distilledBudget,
rawBudget,
refreshLtm: false,
unsustainable: sid ? getSessionState(sid).consecutiveBusts >= 5 : false,
};
}

Expand Down Expand Up @@ -1830,7 +1840,15 @@ function transformInner(input: {
if (sid && (s > 0 || cached.tokens === 0)) {
urgentDistillationMap.set(sid, true);
}
return { ...result!, layer: stageLayer, usable, distilledBudget, rawBudget, refreshLtm: false };
return {
...result!,
layer: stageLayer,
usable,
distilledBudget,
rawBudget,
refreshLtm: false,
unsustainable: sid ? getSessionState(sid).consecutiveBusts >= 5 : false,
};
}
}

Expand Down
1 change: 1 addition & 0 deletions packages/core/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,7 @@ export {
// gaps without sleeping. Not part of the public API.
setLastTurnAtForTest,
inspectSessionState,
getConsecutiveBusts,
} from "./gradient";
export {
formatKnowledge,
Expand Down
10 changes: 9 additions & 1 deletion packages/gateway/src/idle.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import {
saveSessionCosts,
saveSessionTracking,
saveGradientState,
getConsecutiveBusts,
} from "@loreai/core";
import type { LLMClient } from "@loreai/core";
import type { GatewayConfig } from "./config";
Expand Down Expand Up @@ -229,8 +230,15 @@ export function buildIdleWorkHandler(
// Meta consolidation: safe on idle because cache is already cold.
// Run as a separate step so gen-0 segments from the force-distill
// above are counted toward the threshold.
// Under bust pressure (3+ consecutive busts), lower the threshold
// to consolidate earlier — shrinks the distilled prefix before the
// session becomes unsustainable.
const busts = getConsecutiveBusts(sessionID);
const effectiveMetaThreshold = busts >= 3
? Math.max(3, Math.floor(cfg.distillation.metaThreshold / 4))
: cfg.distillation.metaThreshold;
const g0 = distillation.gen0Count(projectPath, sessionID);
if (g0 >= cfg.distillation.metaThreshold) {
if (g0 >= effectiveMetaThreshold) {
await distillation.metaDistill({ llm, projectPath, sessionID, model, callType });
}
} catch (e) {
Expand Down
9 changes: 9 additions & 0 deletions packages/gateway/src/pipeline.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ import {
onIdleResume,
consumeCameOutOfIdle,
needsUrgentDistillation,
getConsecutiveBusts,
formatKnowledge,
buildCompactPrompt,
shouldImportLoreFile,
Expand Down Expand Up @@ -1994,7 +1995,14 @@ function scheduleBackgroundWork(
// Check if urgent distillation is needed (gradient flagged it).
// Mark urgent: true so these bypass the batch queue — the gradient is
// in overflow and needs the result before the next user turn.
// Under bust pressure (3+ consecutive busts), lower the meta-distillation
// threshold to consolidate gen-0 segments earlier — shrinks the distilled
// prefix before the session becomes unsustainable.
if (needsUrgentDistillation(sessionState.sessionID)) {
const busts = getConsecutiveBusts(sessionState.sessionID);
const metaThresholdOverride = busts >= 3
? Math.max(3, Math.floor(cfg.distillation.metaThreshold / 4))
: undefined;
distillation
.run({
llm,
Expand All @@ -2004,6 +2012,7 @@ function scheduleBackgroundWork(
force: true,
urgent: true,
callType: "direct",
metaThresholdOverride,
})
.catch((e) => log.error("background distillation failed:", e));
} else {
Expand Down
1 change: 1 addition & 0 deletions packages/gateway/test/helpers/idle-worker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ mock.module("@loreai/core", () => ({
saveSessionCosts: () => {},
saveSessionTracking: () => {},
saveGradientState: () => {},
getConsecutiveBusts: () => 0,
loadSessionTracking: () => null,
getKV: () => null,
setKV: () => {},
Expand Down
Loading