feat: Transmit remarks about integration docs defects on conclusion

src/lib/agent-interface.ts

@@ -8,7 +8,10 @@ import clack from '../utils/clack';
 import { debug, logToFile, initLogFile, LOG_FILE_PATH } from '../utils/debug';
 import type { WizardOptions } from '../utils/types';
 import { analytics } from '../utils/analytics';
-import { WIZARD_INTERACTION_EVENT_NAME } from './constants';
+import {
+  WIZARD_INTERACTION_EVENT_NAME,
+  WIZARD_REMARK_EVENT_NAME,
+} from './constants';
 import { getLlmGatewayUrlFromHost } from '../utils/urls';
 import { LINTING_TOOLS } from './safe-tools';
 
@@ -43,6 +46,8 @@ export const AgentSignals = {
   ERROR_MCP_MISSING: '[ERROR-MCP-MISSING]',
   /** Signal emitted when the agent cannot access the setup resource */
   ERROR_RESOURCE_MISSING: '[ERROR-RESOURCE-MISSING]',
+  /** Signal emitted when the agent provides a remark about its run */
+  WIZARD_REMARK: '[WIZARD-REMARK]',
 } as const;
 
 export type AgentSignal = (typeof AgentSignals)[keyof typeof AgentSignals];
@@ -431,6 +436,23 @@ export async function runAgent(
       logToFile(`Agent run completed in ${durationSeconds}s`);
     }
 
+    // Extract and capture the agent's reflection on the run
+    const outputText = collectedText.join('\n');
+    const remarkRegex = new RegExp(
+      `${AgentSignals.WIZARD_REMARK.replace(
+        /[.*+?^${}()|[\]\\]/g,
+        '\\$&',
+      )}\\s*(.+?)(?:\\n|$)`,
+      's',
+    );
+    const remarkMatch = outputText.match(remarkRegex);
+    if (remarkMatch && remarkMatch[1]) {
+      const remark = remarkMatch[1].trim();
+      if (remark) {
+        analytics.capture(WIZARD_REMARK_EVENT_NAME, { remark });
+      }
+    }
+
     analytics.capture(WIZARD_INTERACTION_EVENT_NAME, {
       action: 'agent integration completed',
       duration_ms: durationMs,
@@ -492,6 +514,33 @@ export async function runAgent(
             debug('CLI stderr:', data);
           }
         },
+        // Stop hook to have the agent reflect on its run
+        hooks: {
+          Stop: [
+            {
+              hooks: [
+                (input: { stop_hook_active: boolean }) => {
+                  logToFile('Stop hook triggered', {
+                    stop_hook_active: input.stop_hook_active,
+                  });
+
+                  // Only ask for reflection on first stop (not after reflection is provided)
+                  if (input.stop_hook_active) {
+                    logToFile('Stop hook: allowing stop (already reflected)');
+                    return {}; // Allow stopping
+                  }
+
+                  logToFile('Stop hook: requesting reflection');
+                  return {
+                    decision: 'block',
+                    reason: `Before concluding, provide a brief remark about what information or guidance would have been useful to have in the integration prompt or documentation for this run. Specifically cite anything that would have prevented tool failures, erroneous edits, or other wasted turns. Format your response exactly as: ${AgentSignals.WIZARD_REMARK} Your remark here`,
+                  };
+                },
+              ],
+              timeout: 30,
+            },
+          ],
+        },
       },
     });
 

src/lib/constants.ts

@@ -80,3 +80,4 @@ export const POSTHOG_DEV_CLIENT_ID = 'DC5uRLVbGI02YQ82grxgnK6Qn12SXWpCqdPb60oZ';
 export const OAUTH_PORT = 8239;
 
 export const WIZARD_INTERACTION_EVENT_NAME = 'wizard interaction';
+export const WIZARD_REMARK_EVENT_NAME = 'wizard remark';