fix(toolkit-docs-generator): preserve previous summary when regen fails

toolkit-docs-generator/scripts/check-stale-summaries.ts

@@ -0,0 +1,82 @@
+#!/usr/bin/env tsx
+
+/**
+ * Check stale summaries
+ *
+ * Scans toolkit-docs-generator/data/toolkits/*.json and reports any toolkit
+ * whose summary was carried forward from a previous run (summaryStale: true)
+ * because regeneration was skipped or failed. Exits 1 if any are found so
+ * the script can gate CI or a local sanity check.
+ *
+ * Usage:
+ *   pnpm dlx tsx toolkit-docs-generator/scripts/check-stale-summaries.ts
+ */
+
+import { readdirSync, readFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const here = dirname(fileURLToPath(import.meta.url));
+const TOOLKITS_DIR = join(here, "..", "data", "toolkits");
+
+type ToolkitShape = {
+  id?: unknown;
+  summary?: unknown;
+  summaryStale?: unknown;
+  summaryStaleReason?: unknown;
+};
+
+const listToolkitFiles = (): string[] => {
+  try {
+    return readdirSync(TOOLKITS_DIR).filter(
+      (name) => name.endsWith(".json") && name !== "index.json"
+    );
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    console.error(
+      `✗ Cannot read toolkit directory ${TOOLKITS_DIR}: ${message}`
+    );
+    process.exit(1);
+  }
+};
+
+const main = (): number => {
+  const stale: Array<{ file: string; id: string; reason: string }> = [];
+  for (const file of listToolkitFiles()) {
+    let toolkit: ToolkitShape;
+    try {
+      const raw = readFileSync(join(TOOLKITS_DIR, file), "utf8");
+      toolkit = JSON.parse(raw) as ToolkitShape;
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      console.error(`✗ Cannot parse ${file}: ${message}`);
+      return 1;
+    }
+    if (toolkit.summaryStale === true) {
+      stale.push({
+        file,
+        id: typeof toolkit.id === "string" ? toolkit.id : file,
+        reason:
+          typeof toolkit.summaryStaleReason === "string"
+            ? toolkit.summaryStaleReason
+            : "unknown",
+      });
+    }
+  }
+
+  if (stale.length === 0) {
+    console.log(`✓ No stale summaries in ${TOOLKITS_DIR}`);
+    return 0;
+  }
+
+  console.error(`✗ ${stale.length} toolkit(s) have a stale summary:\n`);
+  for (const finding of stale) {
+    console.error(`  - ${finding.id} (${finding.file}): ${finding.reason}`);
+  }
+  console.error(
+    "\nRe-run the generator with a working LLM, or refresh the summary by hand and clear the `summaryStale` / `summaryStaleReason` fields."
+  );
+  return 1;
+};
+
+process.exit(main());

toolkit-docs-generator/src/diff/previous-output.ts

@@ -276,6 +276,12 @@ const buildFallbackToolkit = (
   const authResult = MergedToolkitAuthSchema.safeParse(record.auth);
   const summary =
     typeof record.summary === "string" ? record.summary : undefined;
+  const summaryStale =
+    typeof record.summaryStale === "boolean" ? record.summaryStale : undefined;
+  const summaryStaleReason =
+    typeof record.summaryStaleReason === "string"
+      ? record.summaryStaleReason
+      : undefined;
   const generatedAt =
     typeof record.generatedAt === "string" ? record.generatedAt : undefined;
 
@@ -299,6 +305,8 @@ const buildFallbackToolkit = (
       version,
       description,
       ...(summary ? { summary } : {}),
+      ...(summaryStale !== undefined ? { summaryStale } : {}),
+      ...(summaryStaleReason !== undefined ? { summaryStaleReason } : {}),
       metadata: metadataResult.success
         ? metadataResult.data
         : DEFAULT_PREVIOUS_TOOLKIT_METADATA,

toolkit-docs-generator/src/merger/data-merger.ts

@@ -467,6 +467,31 @@ const getToolDocumentationChunks = (
   return fromPrevious;
 };
 
+/**
+ * Mark the toolkit's summary as stale — the summary is being carried forward
+ * from a previous run even though the toolkit signature changed (regen was
+ * skipped or failed). The CI check in tests/stale-summaries.test.ts surfaces
+ * these toolkits so a human rerun or fix can follow.
+ */
+export const STALE_SUMMARY_WARNING_PREFIX = "Summary is stale for";
+
+const markSummaryStale = (
+  toolkit: MergedToolkit,
+  reason: string,
+  warnings: string[]
+): void => {
+  toolkit.summaryStale = true;
+  toolkit.summaryStaleReason = reason;
+  warnings.push(
+    `${STALE_SUMMARY_WARNING_PREFIX} ${toolkit.id}: ${reason}. Previous summary carried forward.`
+  );
+};
+
+const clearStaleSummaryFlags = (toolkit: MergedToolkit): void => {
+  toolkit.summaryStale = undefined;
+  toolkit.summaryStaleReason = undefined;
+};
+
 const isOverviewChunk = (chunk: DocumentationChunk): boolean =>
   chunk.location === "header" &&
   chunk.position === "before" &&
@@ -950,23 +975,49 @@ export class DataMerger {
     if (hasToolkitOverviewChunk(result.toolkit)) {
       // Keep overview as the canonical toolkit-level narrative.
       result.toolkit.summary = undefined;
+      clearStaleSummaryFlags(result.toolkit);
+      return;
+    }
+
+    // Defensive guard: if something upstream already set a summary on
+    // `result.toolkit`, respect it. buildMergedToolkit does not set one
+    // today, but we don't want the reuse / fallback paths below to
+    // silently replace a pre-populated value.
+    if (result.toolkit.summary) {
+      clearStaleSummaryFlags(result.toolkit);
       return;
     }
 
     if (previousToolkit?.summary) {
       const currentSignature = buildToolkitSummarySignature(result.toolkit);
       const previousSignature = buildToolkitSummarySignature(previousToolkit);
-      if (currentSignature === previousSignature) {
+      // Signature-match reuse is only safe when the PREVIOUS summary itself
+      // was fresh. If the previous run already flagged the summary stale,
+      // a matching signature does not prove freshness — the stale summary
+      // was carried forward from an even earlier toolset and will stay
+      // wrong until an LLM actually regenerates it.
+      if (
+        currentSignature === previousSignature &&
+        !previousToolkit.summaryStale
+      ) {
         result.toolkit.summary = previousToolkit.summary;
+        clearStaleSummaryFlags(result.toolkit);
         return;
       }
     }
 
     if (!this.toolkitSummaryGenerator) {
-      return;
-    }
-
-    if (result.toolkit.summary) {
+      // Preserve previous summary when no LLM is available to regenerate.
+      // A slightly stale summary is better than silently wiping hand-authored
+      // or previously generated content on every run that disables the LLM.
+      if (previousToolkit?.summary) {
+        result.toolkit.summary = previousToolkit.summary;
+        markSummaryStale(
+          result.toolkit,
+          "llm_generator_unavailable",
+          result.warnings
+        );
+      }
       return;
     }
 
@@ -975,11 +1026,24 @@ export class DataMerger {
         result.toolkit
       );
       result.toolkit.summary = summary;
+      clearStaleSummaryFlags(result.toolkit);
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);
       result.warnings.push(
         `Summary generation failed for ${result.toolkit.id}: ${message}`
       );
+      // Preserve previous summary on LLM failure. Losing the summary on a
+      // transient API error would require waiting for another run — and if
+      // the failure is persistent we keep showing the last known-good text
+      // instead of a blank overview.
+      if (previousToolkit?.summary) {
+        result.toolkit.summary = previousToolkit.summary;
+        markSummaryStale(
+          result.toolkit,
+          "llm_generation_failed",
+          result.warnings
+        );
+      }
     }
   }
 

toolkit-docs-generator/src/types/index.ts

@@ -399,6 +399,20 @@ export const MergedToolkitSchema = z.object({
   description: z.string().nullable(),
   /** LLM-generated summary (optional) */
   summary: z.string().optional(),
+  /**
+   * True when the current `summary` is known to be out of date with the
+   * toolkit's current tools (the signature changed but regeneration was
+   * skipped or failed, so the previous summary was carried forward as a
+   * fallback). Cleared whenever a fresh summary is successfully generated
+   * or when the summary is verified against an unchanged signature.
+   */
+  summaryStale: z.boolean().optional(),
+  /**
+   * Machine-readable reason the summary is stale (e.g.
+   * "llm_generator_unavailable", "llm_generation_failed"). Always set
+   * together with `summaryStale: true`. Cleared together with it.
+   */
+  summaryStaleReason: z.string().optional(),
   /** Metadata from Design System */
   metadata: MergedToolkitMetadataSchema,
   /** Authentication requirements */