Replies: 3 comments
|
补充：复现排查时写的最小补丁（协调器在日志消失后从 seq 0 重建整条 live log）与回归测试，供参考： diff --git a/packages/session/session-persistence-jsonl/tests/jsonl.spec.ts b/packages/session/session-persistence-jsonl/tests/jsonl.spec.ts
index 70ae6d1..e8fbc1e 100644
--- a/packages/session/session-persistence-jsonl/tests/jsonl.spec.ts
+++ b/packages/session/session-persistence-jsonl/tests/jsonl.spec.ts
@@ -287,6 +287,39 @@ describe('JsonlSessionPersistence: durability and crash semantics', () => {
expect((await ctx.sessionPersistence.list()).map(h => h.id)).toContain(m.id)
})
+
+ it('re-materializes the log when its directory is removed at runtime', async () => {
+ // A live session loses its persisted log file (e.g. the user or another
+ // process removed the session directory while dsh still holds it open).
+ // The next write must not crash with ENOENT: the coordinator re-materializes
+ // the WHOLE live log from seq 0, so the file stays readable instead of
+ // starting mid-sequence.
+ const session = ctx.sessions.create(SessionId('rm-runtime'), { meta: { cwd: '/work' } })
+ appendLog(session, oneTurnLog())
+ await ctx.sessions.flush(session)
+ const path = rawLogPath(root, '/work', session.id)
+ const dir = sessionDir(root, '/work', session.id)
+ expect(await stat(path).then(() => true)).toBe(true)
+
+ await rm(dir, { recursive: true, force: true })
+
+ session.append('turn/start', { turn: 2 })
+ session.append('turn/end', { turn: 2, reason: { kind: 'completed' } })
+ await expect(ctx.sessions.flush(session)).resolves.toBeTruthy()
+
+ const log = await readFile(path, 'utf8')
+ const headerLine = JSON.parse(log.split('\n')[0] as string) as { id?: string; type?: string }
+ expect(headerLine.type).toBe('session')
+ expect(headerLine.id).toBe(String(session.id))
+ const expectedTypes = [...oneTurnLog().map(event => event.type), 'turn/start', 'turn/end']
+ const scanned = scanLog(Buffer.from(log))
+ expect(scanned.events.map(event => event.type)).toEqual(expectedTypes)
+
+ const loaded = await ctx.sessionPersistence.load(session.id)
+ expect(loaded.meta.id).toBe(session.id)
+ expect(loaded.events.map(event => event.type)).toEqual(expectedTypes)
+ })
+
it('readRaw returns the stored artifact text verbatim with its original filename', async () => {
const m = meta('raw-read', '/work')
await ctx.sessionPersistence.create(m)
diff --git a/packages/session/session-persistence/src/coordinator.ts b/packages/session/session-persistence/src/coordinator.ts
index eb5f971..29c4984 100644
--- a/packages/session/session-persistence/src/coordinator.ts
+++ b/packages/session/session-persistence/src/coordinator.ts
@@ -32,6 +32,11 @@ export const DEFAULT_WRITE_BATCH_MAX_DELAY_MS = 200
/** Largest write batching delay accepted by Node's timer implementation. */
export const MAX_WRITE_BATCH_DELAY_MS = MAX_TIMER_DELAY_MS
+/** Whether a durable-write failure means the backend's log path vanished. */
+function isMissingLog(error: unknown): boolean {
+ return (error as NodeJS.ErrnoException | null)?.code === 'ENOENT'
+}
+
/** Durable session contents failed validation after a successful backend read. */
export class SessionPersistenceCorruptionError extends Error {
/**
@@ -1342,7 +1347,7 @@ export class PersistenceCoordinator<TornMarker = unknown> {
maxDelayMs: this.writeBatchMaxDelayMs,
write: async (batch) => {
await ready()
- await this.serialize(session.header.id, () => this.appendLiveBatch(session.header.id, batch))
+ await this.serialize(session.header.id, () => this.appendLiveBatch(session, batch))
},
reportBackgroundFailure: (error) => {
this.ctx.logger.warn(`${this.backend.name}: background write for session "${session.id}" failed (buffered events retained): ${String(error)}`)
@@ -1351,11 +1356,31 @@ export class PersistenceCoordinator<TornMarker = unknown> {
}
/** Append one controller-owned prefix after filtering events initialization already stored. */
- private async appendLiveBatch(id: SessionId, batch: readonly SessionEvent[]): Promise<void> {
+ private async appendLiveBatch(session: Session, batch: readonly SessionEvent[]): Promise<void> {
+ const id = session.header.id
const state = this.states.get(id)
/* v8 ignore next -- state is always set by the awaited initialization */
const cursor = state?.cursor ?? 0
const fresh = batch.filter(e => e.seq >= cursor)
- await this.appendCore(id, fresh)
+ try {
+ await this.appendCore(id, fresh)
+ } catch (error: unknown) {
+ if (!isMissingLog(error) || state === undefined) throw error
+ // The durable log (or its directory) was removed externally while this
+ // session stayed live. Re-materialize the WHOLE live log from seq 0:
+ // appending just this batch would start the log at a non-zero seq and
+ // every dsh reader would reject it as a seq gap.
+ const wasMaterialized = state.materialized
+ const wasCursor = state.cursor
+ state.materialized = false
+ state.cursor = 0
+ try {
+ await this.appendCore(id, session.events.map(event => structuredClone(event)))
+ } catch (recoveryError: unknown) {
+ state.materialized = wasMaterialized
+ state.cursor = wasCursor
+ throw new AggregateError([error, recoveryError], `failed to re-materialize the live log of "${id}" after it disappeared`)
+ }
+ }
}
}
|
|
Root-caused and staged a patch. Root cause: Staged fix ? https://github.com/zoahdev/deepseek-harness/tree/fix/session-persistence-recreate-on-enoent
Verified against the official suite (full monorepo @ 47f9438, Windows / Node 24): |
|
One durability invariant is important when evaluating the two proposed recovery shapes here. Recreating the missing directory and retrying A history-preserving runtime recovery therefore needs one of these:
For operators, the safest current response is to stop sending turns after the first persistence error, stop the writer, preserve every surviving artifact, and diagnose copies only. I documented the live-memory, write-behind, durable-log, and cold-replay boundary here: https://sandbaseai.github.io/deepseek-harness-handbook/session-log-durability.html Canonical source-backed runbook: sandbaseai/deepseek-harness-handbook#22 |
Uh oh!
There was an error while loading. Please reload this page.
现象
正在运行的会话，其持久化日志（或整个会话目录）被外部删除后（例如清理脚本误删
$DSH_HOME/sessions/**、其他工具/进程删除，或手动rm -rf），下一个回合结束时dsh 崩溃退出：
会话本身一直在运行（对话在内存里继续），一次持久化失败就让整个进程退出。
compression: 'none'（session.jsonl）与zstd（session.jsonl.zstd）都观察到同样表现。复现办法
（日志已落盘）。
rm -rf $DSH_HOME/sessions/<workspace 目录>/<session-id>（或直接
rm -rf ~/.dsh/sessions/*）。fatal load failure ... ENOENT崩溃退出。环境：dsh 0.1.0-rc.6，Linux（x64），TUI surface（dsh-pi-tui）与 web 同属一类路径。
相关讨论
All reactions