fix: keep unusable DOCX numeric references as literal text

sdk/typescript/src/knowledge-base.ts

@@ -12,6 +12,7 @@ import { tmpdir } from "node:os";
 import { basename, extname, join, resolve } from "node:path";
 import { unzipSync } from "fflate";
 
+const MAX_CODE_POINT = 0x10ffff;
 const SUPPORTED_EXTENSIONS = new Set([
   ".md",
   ".markdown",
@@ -216,9 +217,28 @@ function decodeXml(value: string): string {
     (entity, name: string) => {
       if (!name.startsWith("#")) return entities[name.toLowerCase()] ?? entity;
       const hexadecimal = name[1]?.toLowerCase() === "x";
-      return String.fromCodePoint(
-        Number.parseInt(name.slice(hexadecimal ? 2 : 1), hexadecimal ? 16 : 10),
+      const codePoint = Number.parseInt(
+        name.slice(hexadecimal ? 2 : 1),
+        hexadecimal ? 16 : 10,
       );
+      // A reference that cannot name a Unicode scalar value is left as literal
+      // text, matching the unrecognized-named-entity fallback above. Without the
+      // bound String.fromCodePoint throws RangeError, which surfaced as an
+      // unextractable document and failed the whole knowledge base. Surrogates
+      // are excluded too: XML forbids them and writing one would silently encode
+      // as U+FFFD.
+      return isUnicodeScalarValue(codePoint)
+        ? String.fromCodePoint(codePoint)
+        : entity;
     },
   );
 }
+
+function isUnicodeScalarValue(codePoint: number): boolean {
+  return (
+    Number.isInteger(codePoint) &&
+    codePoint >= 0 &&
+    codePoint <= MAX_CODE_POINT &&
+    !(codePoint >= 0xd800 && codePoint <= 0xdfff)
+  );
+}

sdk/typescript/tests-ts/knowledge-base.test.ts

@@ -121,6 +121,63 @@ describe("scan knowledge bases", () => {
     expect(documents).toContain("SSRF & IDOR\n");
   });
 
+  test("keeps DOCX numeric references that cannot name a code point as literal text", async () => {
+    // https://github.com/openai/codex-security/issues/40 -- an out-of-range
+    // reference used to raise RangeError out of String.fromCodePoint, which
+    // surfaced as an unextractable document and failed the whole knowledge base.
+    const cases: Array<[string, string, string]> = [
+      [
+        "above-max-hex",
+        "Boundary &#x110000; case.",
+        "Boundary &#x110000; case.",
+      ],
+      [
+        "above-max-decimal",
+        "Boundary &#1114112; case.",
+        "Boundary &#1114112; case.",
+      ],
+      [
+        "huge-decimal",
+        "Huge &#99999999999999; case.",
+        "Huge &#99999999999999; case.",
+      ],
+      [
+        "lone-surrogate",
+        "Surrogate &#xD800; case.",
+        "Surrogate &#xD800; case.",
+      ],
+      ["valid-ascii", "Valid &#65; case.", "Valid A case."],
+      ["valid-astral", "Valid &#128512; case.", "Valid \u{1F600} case."],
+      ["max-code-point", "Valid &#x10FFFF; case.", "Valid \u{10FFFF} case."],
+    ];
+
+    for (const [name, body, expected] of cases) {
+      const root = await temporaryDirectory();
+      await writeFile(join(root, `${name}.docx`), docx(body));
+      const knowledgeBase = await prepareKnowledgeBase([root]);
+      temporaryDirectories.push(knowledgeBase.path);
+      const documents = await extractedDocuments(knowledgeBase.path);
+      expect(documents).toEqual([`${expected}\n`]);
+    }
+  });
+
+  test("keeps one unusable reference from failing the other knowledge-base documents", async () => {
+    const root = await temporaryDirectory();
+    await writeFile(join(root, "notes.md"), "Authentication boundary notes");
+    await writeFile(
+      join(root, "threat-model.docx"),
+      docx("Boundary &#x110000; case."),
+    );
+
+    const knowledgeBase = await prepareKnowledgeBase([root]);
+    temporaryDirectories.push(knowledgeBase.path);
+    const documents = await extractedDocuments(knowledgeBase.path);
+
+    expect(documents).toHaveLength(2);
+    expect(documents).toContain("Authentication boundary notes");
+    expect(documents).toContain("Boundary &#x110000; case.\n");
+  });
+
   test("cleans up documents and rediscovers directory contents on later runs", async () => {
     const root = await temporaryDirectory();
     const source = join(root, "scope.md");