fix(emergent): pre-parse syntax validation + actionable hints + classifier category

Forge attempts that produced ill-formed JavaScript (e.g. an LLM emitting
`() => const x = 1` or sneaking in TypeScript `interface` / `: type`
annotations) previously failed with an opaque 'Correctness FAIL: all test
cases did not run' message because parsing failed before any test input
touched the sandbox. Three fixes:

1. SandboxedToolForge.execute now parses the wrapped source via
   AsyncFunction before runInContext. SyntaxErrors short-circuit with a
   'SyntaxError before execution' result plus a hint that maps common
   V8 error shapes (Unexpected token const/let/interface/:,
   Unexpected end of input, Unexpected reserved word) to concrete
   rewrite guidance.

2. ForgeToolMetaTool.sandbox-code description now spells out: pure
   JS (no TypeScript), block braces on every if/for/while, arrow fns
   with declarations use { ... return }, and outputs must match the
   declared schema without extra fields.

3. ForgeRejectionClassifier gets a new `syntax_error` category,
   evaluated before parse_error, so dashboards / retry loops can see
   syntax failures as a first-class bucket instead of bleeding into
   'judge_correctness' or 'other'.

Targeted suites: sandboxed-forge (24 \u2192 27) + forge-rejection-classifier
(13 \u2192 17), full emergent suite green (290/290).

Author: jddunn

src/emergent/ForgeRejectionClassifier.ts

@@ -40,6 +40,15 @@ export type ForgeRejectionCategory =
    * non-deterministic behavior).
    */
   | 'judge_correctness'
+  /**
+   * Implementation could not be parsed as JavaScript — typically an
+   * LLM forge emitting TypeScript syntax (`interface`, `: type`),
+   * declarations in the wrong position (`() => const x = ...`), or
+   * single-line `if`/`for`/`while` without braces. Caught by the
+   * sandbox pre-parse step or by a failing `execute` call that could
+   * not even instantiate the function.
+   */
+  | 'syntax_error'
   /**
    * Everything else. A non-zero `other` bucket is a signal to inspect
    * the raw reasons and consider adding a new category.
@@ -95,6 +104,17 @@ const PARSE_ERROR_PATTERNS = [
   'judge response was not valid json',
 ];
 
+const SYNTAX_ERROR_PATTERNS = [
+  'syntaxerror',
+  'syntax error',
+  'unexpected token',
+  'unexpected identifier',
+  'unexpected end of input',
+  'unexpected reserved word',
+  'did not run successfully due to a syntax',
+  'syntaxerror before execution',
+];
+
 const JUDGE_CORRECTNESS_PATTERNS = [
   'logic error',
   'threshold ordering',
@@ -147,6 +167,9 @@ export function classifyForgeRejection(errorReason: string | undefined): ForgeRe
   for (const p of SHAPE_CHECK_PATTERNS) {
     if (lower.includes(p)) return 'shape_check';
   }
+  for (const p of SYNTAX_ERROR_PATTERNS) {
+    if (lower.includes(p)) return 'syntax_error';
+  }
   for (const p of PARSE_ERROR_PATTERNS) {
     if (lower.includes(p)) return 'parse_error';
   }

src/emergent/ForgeToolMetaTool.ts

@@ -165,7 +165,20 @@ export class ForgeToolMetaTool implements ITool<ForgeToolInput, ForgeResult> {
             type: 'object',
             properties: {
               mode: { type: 'string', const: 'sandbox' },
-              code: { type: 'string' },
+              code: {
+                type: 'string',
+                description:
+                  'Pure JavaScript (ES2020). MUST define `function execute(input) { ... }` ' +
+                  '(or an async equivalent) at the top level. RULES that block the judge from ' +
+                  'approving otherwise-correct tools: ' +
+                  '(1) NO TypeScript: no `: type` annotations, no `interface`, no `type` aliases, no `<T>` generics, no `enum`. ' +
+                  '(2) Every `if`/`for`/`while`/`else` body uses BLOCK BRACES `{ ... }` — ' +
+                  'single-line bodies with declarations are forbidden. ' +
+                  '(3) Arrow functions that declare variables must use `(...) => { const x = ...; return x; }` — ' +
+                  'never `(...) => const x = ...` (a SyntaxError). ' +
+                  '(4) Output MUST conform exactly to `outputSchema` — no extra fields. ' +
+                  'If the schema has `additionalProperties: false`, return ONLY the declared keys.',
+              },
               allowlist: {
                 type: 'array',
                 items: { type: 'string', enum: ['fetch', 'fs.readFile', 'crypto'] },

src/emergent/SandboxedToolForge.ts

@@ -189,6 +189,46 @@ export class SandboxedToolForge {
    * // result.violations === ['eval() is forbidden']
    * ```
    */
+  /**
+   * Translate a raw V8 SyntaxError message into a concrete hint that
+   * points the LLM (or the retry loop) at the likely cause. Returns
+   * empty string when no known pattern matches — callers still surface
+   * the raw message in that case. The hints map to the 4 most common
+   * LLM forge mistakes observed in production.
+   */
+  private describeSyntaxError(message: string): string {
+    const m = message || '';
+    if (/Unexpected token 'const'/.test(m) || /Unexpected token 'let'/.test(m)) {
+      return (
+        'A `const` or `let` appeared in a position JavaScript does not allow. ' +
+        'Common causes: arrow function without braces (`() => const x = 1` — wrap in `{}` and add `return`), ' +
+        '`if (x) const y = 1` without a block, or a declaration used as an expression. ' +
+        'Every `if`/`for`/`while` body must use block braces; every arrow fn that declares variables must use `{ ... }` and `return`.'
+      );
+    }
+    if (/Unexpected token '?:'?/.test(m)) {
+      return (
+        'Unexpected `:` — most likely a TypeScript type annotation leaked into the output. ' +
+        'Output must be pure JavaScript. Remove all `: type` annotations, `interface` blocks, and generic brackets `<T>`.'
+      );
+    }
+    if (/Unexpected reserved word/.test(m) && /interface|type/.test(m)) {
+      return (
+        'TypeScript-only keyword leaked into the output. Remove `interface`, `type`, `enum`, and `implements` — emit pure JavaScript only.'
+      );
+    }
+    if (/Unexpected identifier/.test(m)) {
+      return (
+        'Two identifiers appeared adjacent without a binding keyword. ' +
+        'Commonly caused by missing commas in objects or missing semicolons between statements.'
+      );
+    }
+    if (/Unexpected end of input/.test(m)) {
+      return 'Code is missing a closing `}`, `)`, or `]` somewhere. Check all brace/paren pairs.';
+    }
+    return '';
+  }
+
   validateCode(code: string, allowlist: SandboxAPI[]): { valid: boolean; violations: string[] } {
     const violations: string[] = [];
 
@@ -295,6 +335,34 @@ export class SandboxedToolForge {
       })();
     `;
 
+    // Step 3.5: Pre-parse to isolate syntax errors with line-level hints
+    // BEFORE running test cases. Without this, LLM-generated code that
+    // has a `SyntaxError: Unexpected token 'const'` (common when the LLM
+    // writes `() => const x = 1` or sneaks in TypeScript types) would
+    // cause every test case to fail with the same opaque message, and
+    // the judge's retry prompt wouldn't have enough context to fix it.
+    // Throwing an AsyncFunction is the cheapest parse-only check that
+    // covers the exact same async-IIFE shape we run in the VM — so any
+    // syntax issue here is guaranteed to also fail in runInContext.
+    try {
+      // The wrappedCode is an expression (an IIFE), not statements,
+      // so wrap it once more in a function body. `new AsyncFunction`
+      // parses + instantiates without executing.
+      const AsyncFunctionCtor = Object.getPrototypeOf(async function () {
+        /* pre-parse probe */
+      }).constructor as typeof Function;
+      new AsyncFunctionCtor('return (async () => {' + request.code + '})();');
+    } catch (err: unknown) {
+      const msg = err instanceof Error ? err.message : String(err);
+      const hint = this.describeSyntaxError(msg);
+      return {
+        success: false,
+        error: `SyntaxError before execution: ${msg}${hint ? ` | Hint: ${hint}` : ''}`,
+        executionTimeMs: Math.round(performance.now() - startTime),
+        memoryUsedBytes: 0,
+      };
+    }
+
     // Step 4: Execute in VM with timeout.
     try {
       const ctx = createContext(sandboxGlobals);

src/emergent/__tests__/forge-rejection-classifier.spec.ts

@@ -79,6 +79,28 @@ const CASES: Array<{ input: string; expected: ForgeRejectionCategory; label: str
     input: 'Sandbox timeout exceeded after 10000ms',
     expected: 'other',
   },
+  {
+    label: 'syntax error — "Unexpected token const" from production',
+    input:
+      "Correctness FAIL: all provided test cases did not run successfully due to a syntax error in the implementation: 'SyntaxError: Unexpected token \\'const\\''",
+    expected: 'syntax_error',
+  },
+  {
+    label: 'syntax error — pre-parse message',
+    input: 'SyntaxError before execution: Unexpected identifier | Hint: missing commas in objects',
+    expected: 'syntax_error',
+  },
+  {
+    label: 'syntax error — unexpected end of input (missing brace)',
+    input:
+      'Correctness FAIL: SyntaxError: Unexpected end of input — implementation is missing a closing brace',
+    expected: 'syntax_error',
+  },
+  {
+    label: 'syntax error — reserved word (TypeScript leaked)',
+    input: 'SyntaxError: Unexpected reserved word interface in implementation',
+    expected: 'syntax_error',
+  },
 ];
 
 describe('classifyForgeRejection', () => {

src/emergent/__tests__/sandboxed-forge.spec.ts

@@ -402,4 +402,47 @@ describe('SandboxedToolForge', () => {
     expect(result.success).toBe(true);
     expect(result.output).toEqual({ total: 55, count: 3 });
   });
+
+  // -------------------------------------------------------------------------
+  // Pre-parse: syntax errors surface with actionable hints (not generic
+  // "test cases failed" opaqueness that the judge gets otherwise).
+  // -------------------------------------------------------------------------
+  it('pre-parse catches arrow-fn with const in expression position', async () => {
+    const request = makeRequest(
+      // Invalid: const in arrow body without braces
+      'const calc = (x) => const doubled = x * 2;\nfunction execute(input) { return { out: calc(input.n) }; }',
+      { n: 5 },
+    );
+
+    const result = await forge.execute(request);
+
+    expect(result.success).toBe(false);
+    expect(result.error).toMatch(/SyntaxError before execution/);
+    expect(result.error).toMatch(/arrow function without braces|wrap in `{}`/i);
+  });
+
+  it('pre-parse catches TypeScript syntax leaks', async () => {
+    const request = makeRequest(
+      // Invalid JS: interface keyword
+      'interface Input { n: number }\nfunction execute(input) { return { n: input.n }; }',
+      { n: 1 },
+    );
+
+    const result = await forge.execute(request);
+
+    expect(result.success).toBe(false);
+    expect(result.error).toMatch(/SyntaxError before execution/);
+  });
+
+  it('pre-parse passes valid ES2020 code through to execution', async () => {
+    const request = makeRequest(
+      'async function execute(input) { return { doubled: input.n * 2 }; }',
+      { n: 7 },
+    );
+
+    const result = await forge.execute(request);
+
+    expect(result.success).toBe(true);
+    expect(result.output).toEqual({ doubled: 14 });
+  });
 });