Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 26 additions & 5 deletions apps/cli/src/commands/health.ts
Original file line number Diff line number Diff line change
Expand Up @@ -109,10 +109,13 @@ interface ClaimBeforeEditPayload extends ClaimBeforeEditStats {
* PreToolUse hook is firing somewhere; zero with edits > 0 strongly
* suggests the hook is not wired into the active editor session. */
pre_tool_use_signals: number;
/** PreToolUse fired, but the hook session id was not present in Colony
* storage, so telemetry was recorded under the diagnostics fallback. */
session_binding_missing: number;
/** True when edits happened but no PreToolUse telemetry was recorded —
* diagnostic that points at hook wiring rather than agent discipline. */
likely_missing_hook: boolean;
/** User-facing remediation when likely_missing_hook is true. */
/** User-facing remediation for hook wiring or session-binding failures. */
install_hint: string | null;
}

Expand Down Expand Up @@ -616,6 +619,7 @@ function claimBeforeEditPayload(
const editsWithoutClaimBefore = stats.edits_with_file_path - stats.edits_claimed_before;
const autoClaimedBeforeEdit = stats.auto_claimed_before_edit ?? 0;
const preToolUseSignals = stats.pre_tool_use_signals ?? 0;
const sessionBindingMissing = stats.session_binding_missing ?? 0;
const status =
stats.edit_tool_calls === 0
? 'no_data'
Expand All @@ -625,9 +629,13 @@ function claimBeforeEditPayload(
// If edits landed but no claim-before-edit observation was ever written,
// PreToolUse is almost certainly not firing for the active editor.
const likelyMissingHook = stats.edit_tool_calls > 0 && preToolUseSignals === 0;
const sessionBindingHint =
sessionBindingMissing > 0
? 'PreToolUse is firing, but Colony session binding is missing. Restart the editor session so SessionStart binds the session id; keep calling task_claim_file manually until binding is restored.'
: null;
const installHint = likelyMissingHook
? 'PreToolUse auto-claim is not covering edits in this window. Run colony install --ide <ide>, restart the editor session, and ensure an active task is bound for the session.'
: null;
: sessionBindingHint;
return {
...stats,
status,
Expand All @@ -639,6 +647,7 @@ function claimBeforeEditPayload(
claim_before_edit_ratio:
status === 'available' ? ratio(stats.edits_claimed_before, stats.edits_with_file_path) : null,
pre_tool_use_signals: preToolUseSignals,
session_binding_missing: sessionBindingMissing,
likely_missing_hook: likelyMissingHook,
install_hint: installHint,
};
Expand Down Expand Up @@ -785,8 +794,13 @@ function formatClaimBeforeEdit(payload: ClaimBeforeEditPayload): string[] {
lines.push(
` telemetry: edits_with_claim=${payload.edits_with_claim}, edits_missing_claim=${payload.edits_missing_claim}, auto_claimed_before_edit=${payload.auto_claimed_before_edit}, pre_tool_use_signals=${payload.pre_tool_use_signals}`,
);
if (payload.session_binding_missing > 0) {
lines.push(kleur.yellow(` session binding missing: ${payload.session_binding_missing}`));
}
if (payload.likely_missing_hook && payload.install_hint) {
lines.push(kleur.yellow(` ${payload.install_hint}`));
} else if (payload.session_binding_missing > 0 && payload.install_hint) {
lines.push(kleur.yellow(` ${payload.install_hint}`));
}
return lines;
}
Expand Down Expand Up @@ -848,22 +862,29 @@ function healthActionHints(payload: ColonyHealthPayloadWithoutHints): ActionHint
)
) {
const missingHook = payload.task_claim_file_before_edits.likely_missing_hook;
const sessionBindingMissing = payload.task_claim_file_before_edits.session_binding_missing > 0;
hints.push({
metric: 'claim-before-edit',
status: 'bad',
current: formatPercent(payload.task_claim_file_before_edits.claim_before_edit_ratio),
target: `${formatPercent(TARGET_CLAIM_BEFORE_EDIT)}+`,
action: missingHook
? 'PreToolUse auto-claim hook is not firing for these edits. Reinstall and restart the editor; PreToolUse will auto-claim before edits.'
: 'Call task_claim_file for touched files before Edit or Write tool use.',
: sessionBindingMissing
? 'PreToolUse is firing, but session binding is missing. Restart the editor so SessionStart binds the active session before relying on auto-claim.'
: 'Call task_claim_file for touched files before Edit or Write tool use.',
tool_call:
'mcp__colony__task_claim_file({ task_id: <task_id>, session_id: "<session_id>", file_path: "<file>", note: "pre-edit claim" })',
command: missingHook
? 'colony install --ide <ide> # then restart the editor session'
: 'colony install --ide <ide> # enables pre-edit auto-claim hooks',
: sessionBindingMissing
? 'colony install --ide <ide> # then restart the editor session to refresh SessionStart binding'
: 'colony install --ide <ide> # enables pre-edit auto-claim hooks',
prompt: missingHook
? 'PreToolUse auto-claim is not covering edits — run colony install --ide <ide>, restart the editor session, and ensure an active task is bound. Until the hook fires, call mcp__colony__task_claim_file before each edit.'
: 'Before editing, call mcp__colony__task_claim_file for each touched path; if agents keep missing this, run colony install --ide <ide> to enable pre-edit auto-claim hooks.',
: sessionBindingMissing
? 'PreToolUse is firing, but Colony session binding is missing. Restart the editor session so SessionStart binds the active session id; until then, call mcp__colony__task_claim_file before each edit.'
: 'Before editing, call mcp__colony__task_claim_file for each touched path; if agents keep missing this, run colony install --ide <ide> to enable pre-edit auto-claim hooks.',
});
}

Expand Down
41 changes: 41 additions & 0 deletions apps/cli/test/health.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -620,6 +620,44 @@ describe('colony health payload', () => {
expect(text).toContain('top recorded tools: Bash (3), Edit (1), Read (1)');
});

it('diagnoses missing session binding separately from a missing PreToolUse hook', () => {
const payload = buildColonyHealthPayload(
fakeStorage({
calls: [call(1, 'codex-alpha-session', 'Edit', NOW - 1_000)],
claimBeforeEdit: {
edit_tool_calls: 1,
edits_with_file_path: 1,
edits_claimed_before: 0,
pre_tool_use_signals: 1,
session_binding_missing: 1,
},
}),
{
since: SINCE,
window_hours: 24,
now: NOW,
codex_sessions_root: NO_CODEX_ROOT,
},
);

expect(payload.task_claim_file_before_edits).toMatchObject({
likely_missing_hook: false,
pre_tool_use_signals: 1,
session_binding_missing: 1,
install_hint: expect.stringContaining('session binding is missing'),
});
const claimHint = payload.action_hints.find((hint) => hint.metric === 'claim-before-edit');
expect(claimHint).toMatchObject({
action: expect.stringContaining('session binding is missing'),
prompt: expect.stringContaining('SessionStart binds the active session id'),
});
expect(claimHint?.action).not.toContain('hook is not firing');

const text = formatColonyHealthOutput(payload);
expect(text).toContain('session binding missing: 1');
expect(text).not.toContain('PreToolUse auto-claim hook is not firing');
});

it('omits the zero-mcp diagnostic when the window is genuinely empty', () => {
const payload = buildColonyHealthPayload(
fakeStorage({
Expand Down Expand Up @@ -747,6 +785,9 @@ function fakeStorage(args: {
edit_tool_calls: number;
edits_with_file_path: number;
edits_claimed_before: number;
auto_claimed_before_edit?: number;
session_binding_missing?: number;
pre_tool_use_signals?: number;
};
tasks?: TestTask[];
observationsByTask?: Record<number, TestObservation[]>;
Expand Down
20 changes: 18 additions & 2 deletions packages/hooks/src/handlers/pre-tool-use.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import type { HookInput } from '../types.js';
import { extractTouchedFiles } from './post-tool-use.js';

const CLAIM_WARNING_DEBOUNCE_MS = 60_000;
const CLAIM_BEFORE_EDIT_FALLBACK_SESSION_ID = 'colony-pre-tool-use-diagnostics';
const claimWarningDebounceByStore = new WeakMap<MemoryStore, Map<string, number>>();

export interface ClaimBeforeEditFallbackWarning {
Expand Down Expand Up @@ -158,10 +159,22 @@ function recordClaimBeforeEditFailure(
candidates: AutoClaimFailure['candidates'];
},
): void {
if (metadata.code === 'SESSION_NOT_FOUND' || metadata.code === 'COLONY_UNAVAILABLE') return;
if (metadata.code === 'COLONY_UNAVAILABLE') return;
const sessionBindingMissing = metadata.code === 'SESSION_NOT_FOUND';
const observationSessionId = sessionBindingMissing
? CLAIM_BEFORE_EDIT_FALLBACK_SESSION_ID
: session_id;
try {
if (sessionBindingMissing) {
store.startSession({
id: CLAIM_BEFORE_EDIT_FALLBACK_SESSION_ID,
ide: 'colony-hook',
cwd: null,
metadata: { source: 'pre-tool-use', purpose: 'session-binding-diagnostics' },
});
}
store.addObservation({
session_id,
session_id: observationSessionId,
kind: 'claim-before-edit',
content: `edits_missing_claim: ${metadata.file_path}`,
metadata: {
Expand All @@ -172,6 +185,9 @@ function recordClaimBeforeEditFailure(
tool: metadata.tool,
code: metadata.code,
error: metadata.error,
...(sessionBindingMissing
? { session_binding_missing: true, original_session_id: session_id }
: {}),
candidates: compactCandidates(metadata.candidates),
},
});
Expand Down
20 changes: 20 additions & 0 deletions packages/hooks/test/auto-claim.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -535,6 +535,26 @@ describe('claimBeforeEditFromToolUse', () => {
},
},
]);
expect(store.storage.getSession('missing')).toBeUndefined();
expect(store.storage.getSession('colony-pre-tool-use-diagnostics')).toMatchObject({
ide: 'colony-hook',
});
const fallbackTelemetry = store
.timeline('colony-pre-tool-use-diagnostics')
.filter((row) => row.kind === 'claim-before-edit');
expect(fallbackTelemetry).toHaveLength(1);
expect(metadataOf(fallbackTelemetry[0])).toMatchObject({
source: 'pre-tool-use',
outcome: 'edits_missing_claim',
file_path: 'src/x.ts',
code: 'SESSION_NOT_FOUND',
session_binding_missing: true,
original_session_id: 'missing',
});
expect(store.storage.claimBeforeEditStats(0)).toMatchObject({
pre_tool_use_signals: 1,
session_binding_missing: 1,
});
});

it('debounces repeated warning output for the same session, file, and code', () => {
Expand Down
13 changes: 12 additions & 1 deletion packages/storage/src/storage.ts
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,9 @@ export interface ClaimBeforeEditStats {
edits_with_file_path: number;
edits_claimed_before: number;
auto_claimed_before_edit?: number;
/** Count of PreToolUse claim-before-edit rows that had to be recorded under
* a fallback diagnostics session because the hook session row was missing. */
session_binding_missing?: number;
/** Count of `claim-before-edit` telemetry observations in the window — any
* outcome (success, conflict, failure). Authoritative signal that the
* PreToolUse hook is firing at all in the active editor sessions. */
Expand Down Expand Up @@ -1534,25 +1537,33 @@ export class Storage {
AND json_extract(c.metadata, '$.source') = 'pre-tool-use'
AND json_extract(c.metadata, '$.auto_claimed_before_edit') = 1
) AS auto_claimed_before_edit,
(
SELECT COUNT(*) FROM observations c
WHERE c.ts > ?
AND c.kind = 'claim-before-edit'
AND json_extract(c.metadata, '$.session_binding_missing') = 1
) AS session_binding_missing,
(
SELECT COUNT(*) FROM observations c
WHERE c.ts > ?
AND c.kind = 'claim-before-edit'
) AS pre_tool_use_signals
FROM edit_rows`,
)
.get(FILE_EDIT_TOOLS_JSON, since_ts, since_ts, since_ts) as {
.get(FILE_EDIT_TOOLS_JSON, since_ts, since_ts, since_ts, since_ts) as {
edit_tool_calls: number;
edits_with_file_path: number | null;
edits_claimed_before: number | null;
auto_claimed_before_edit: number | null;
session_binding_missing: number | null;
pre_tool_use_signals: number | null;
};
return {
edit_tool_calls: row.edit_tool_calls,
edits_with_file_path: row.edits_with_file_path ?? 0,
edits_claimed_before: row.edits_claimed_before ?? 0,
auto_claimed_before_edit: row.auto_claimed_before_edit ?? 0,
session_binding_missing: row.session_binding_missing ?? 0,
pre_tool_use_signals: row.pre_tool_use_signals ?? 0,
};
}
Expand Down
17 changes: 16 additions & 1 deletion packages/storage/test/coordination-activity.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,20 @@ describe('colony health read queries', () => {
toolUse('codex@health', 'Edit', 3_000, 'src/claimed.ts');
toolUse('claude@health', 'Edit', 4_000, 'src/unclaimed.ts');
toolUse('claude@health', 'Edit', 5_000);
session('colony-pre-tool-use-diagnostics');
storage.insertObservation({
session_id: 'colony-pre-tool-use-diagnostics',
kind: 'claim-before-edit',
content: 'session binding missing',
compressed: false,
intensity: null,
ts: 5_500,
metadata: {
source: 'pre-tool-use',
outcome: 'edits_missing_claim',
session_binding_missing: true,
},
});

expect(storage.toolCallsSince(0).map((row) => row.tool)).toEqual([
'mcp__colony__task_list',
Expand All @@ -151,7 +165,8 @@ describe('colony health read queries', () => {
edits_with_file_path: 2,
edits_claimed_before: 1,
auto_claimed_before_edit: 0,
pre_tool_use_signals: 0,
session_binding_missing: 1,
pre_tool_use_signals: 1,
});
});
});
Expand Down
Loading