LLMs-code.txt

.github/workflows/deploy-production.yml

@@ -30,12 +30,18 @@ jobs:
 
       - name: 🤖 Generate LLMs files
         working-directory: ./docusaurus
-        run: node scripts/generate-llms.js
+        run: npm run generate-llms
+
+      - name: 🤖 Generate LLMs code file
+        working-directory: ./docusaurus
+        run: |
+          node scripts/llms/generate-llms-code.js --anchors --all --output static/llms-code.txt
+          node scripts/llms/validate-llms-code.js --path static/llms-code.txt --strict --verify-anchors --project-root ..
 
       - name: 🔍 Check for changes
         id: check-changes
         run: |
-          if git diff --quiet HEAD -- docusaurus/static/llms*.txt; then
+          if git diff --quiet HEAD -- docusaurus/static/llms.txt docusaurus/static/llms-full.txt docusaurus/static/llms-code.txt; then
             echo "changed=false" >> $GITHUB_OUTPUT
             echo "🔄 No changes in LLMs files"
           else
@@ -48,7 +54,7 @@ jobs:
         run: |
           git config --local user.email "noreply@github.com"
           git config --local user.name "GitHub Actions"
-          git add docusaurus/static/llms*.txt
+          git add docusaurus/static/llms.txt docusaurus/static/llms-full.txt docusaurus/static/llms-code.txt
           git commit -m "🤖 Update LLMs files [skip ci]"
           git push
 

docusaurus/package.json

@@ -4,8 +4,8 @@
   "private": true,
   "scripts": {
     "docusaurus": "docusaurus",
-    "dev": "docusaurus start --port 8080 --no-open",
-    "build": "docusaurus build",
+    "dev": "yarn generate-llms && node scripts/llms/generate-llms-code.js --anchors --all --output static/llms-code.txt && node scripts/llms/validate-llms-code.js --path static/llms-code.txt --strict --verify-anchors --project-root .. && docusaurus start --port 8080 --no-open",
+    "build": "yarn generate-llms && node scripts/llms/generate-llms-code.js --anchors --all --output static/llms-code.txt && node scripts/llms/validate-llms-code.js --path static/llms-code.txt --strict --verify-anchors --project-root .. && docusaurus build",
     "swizzle": "docusaurus swizzle",
     "deploy": "docusaurus deploy",
     "clear": "docusaurus clear",
@@ -14,9 +14,14 @@
     "write-heading-ids": "docusaurus write-heading-ids",
     "release-notes": "bash ./scripts/release-notes-script.sh",
     "redirections-analysis": "node ./scripts/redirection-analysis/redirect-analyzer.js",
-    "generate-llms": "node scripts/generate-llms.js",
-    "dev:with-llms": "yarn generate-llms && docusaurus start --port 8080 --no-open",
-    "build:with-llms": "yarn generate-llms && docusaurus build",
+    "generate-llms": "node scripts/llms/generate-llms.js",
+    "dev:with-llms": "yarn generate-llms && node scripts/llms/generate-llms-code.js --anchors --all --output static/llms-code.txt && node scripts/llms/validate-llms-code.js --path static/llms-code.txt --strict --verify-anchors --project-root .. && docusaurus start --port 8080 --no-open",
+    "build:with-llms": "yarn generate-llms && node scripts/llms/generate-llms-code.js --anchors --all --output static/llms-code.txt && node scripts/llms/validate-llms-code.js --path static/llms-code.txt --strict --verify-anchors --project-root .. && docusaurus build",
+    "llms:generate-and-validate": "yarn generate-llms && node scripts/llms/generate-llms-code.js --anchors --all --output static/llms-code.txt && node scripts/llms/validate-llms-code.js --path static/llms-code.txt --strict --verify-anchors --project-root ..",
+    "llms:generate-verbose": "yarn generate-llms && node scripts/llms/generate-llms-code.js --anchors --all --verbose --output static/llms-code.txt && echo 'Skip log (if any): static/llms-code-skip.log'",
+    "llms:generate-and-validate:verbose": "yarn generate-llms && node scripts/llms/generate-llms-code.js --anchors --all --verbose --output static/llms-code.txt && echo 'Skip log (if any): static/llms-code-skip.log' && node scripts/llms/validate-llms-code.js --path static/llms-code.txt --strict --verify-anchors --project-root ..",
+    "validate:llms-code": "node scripts/llms/validate-llms-code.js --path static/llms-code.txt --strict --verify-anchors --project-root ..",
+    "validate:llms-code:strict": "node scripts/llms/validate-llms-code.js --path static/llms-code.txt --strict --check-files --verify-anchors --project-root ..",
     "meilisearch:update-order": "node -r dotenv/config scripts/meilisearch/add-category-order.js"
   },
   "dependencies": {

docusaurus/scripts/README-llms-code.md

@@ -0,0 +1,79 @@
+# llms-code: generator and validator
+
+This tooling extracts code examples from docs and emits a single consumable file for LLMs, plus an optional validation pass to catch structural issues early.
+
+## What it generates
+
+- `static/llms-code.txt` with blocks grouped by page and section:
+  - `## Section`
+  - `Description: ...` (optional)
+  - `(Source: https://docs.strapi.io/...#anchor)` when `--anchors` is set
+  - For each variant: `Language: ...`, `File path: ...` (or `N/A`), fenced code, `---` divider between variants
+- In verbose runs a skip log is written to `static/llms-code-skip.log` listing pages with no code snippets.
+
+## npm/yarn scripts
+
+Run from `docusaurus/`:
+
+- `yarn llms:generate-and-validate`
+  - Generate for all docs and validate (quiet output; no file existence checks)
+
+- `yarn llms:generate-verbose`
+  - Generate with `--verbose`; prints each skipped page and writes `static/llms-code-skip.log`
+
+- `yarn llms:generate-and-validate:verbose`
+  - Verbose generate (writes skip log) then validate (quiet)
+
+- `yarn validate:llms-code`
+  - Validate an existing `static/llms-code.txt` (quiet; no file existence checks)
+
+- `yarn validate:llms-code:strict`
+  - Validate with file existence checks (use only if paths point to a real project), plus anchor verification
+
+## Generator flags (`scripts/generate-llms-code.js`)
+
+- `--all` Scan all docs (restricted to `cms/` and `cloud/` trees)
+- `--include a,b` / `--exclude x,y` Filter discovered doc IDs by substring
+- `--anchors` Include section anchors in Source lines
+- `--line-numbers` Emit `Lines: start-end` for each variant
+- `--verbose` Print per-file skip messages; auto-writes `static/llms-code-skip.log`
+- `--log-file path` Custom path for the skip log
+- `--output path` Destination for generated text (use `-` for stdout)
+
+Notes:
+- Discovery intentionally excludes `snippets/` and other non-doc trees.
+- When file path cannot be inferred it is emitted as `N/A`.
+
+## Validator flags (`scripts/validate-llms-code.js`)
+
+- `--path path` Input file (use `/dev/stdin` to validate from a pipe)
+- `--strict` Exit with non-zero on any errors (warnings do not fail)
+- `--verify-anchors` Check that section anchors exist in the source doc
+- `--check-files` Check referenced files exist (use with `--project-root ..` when appropriate)
+- `--project-root dir` Base path for file checks and anchor lookups
+- `--report json|text` Output diagnostics as JSON or text (default)
+
+Heuristics and niceties:
+- Language aliases normalized (e.g., `js` ≡ `JavaScript`, `ts` ≡ `TypeScript`, `sh` treated as `Bash`, `graphql`, `html`, `dotenv`, `text` recognized)
+- Fence-first blocks (without a `Language:` line) are accepted by inferring language from the fence
+- Section Description and Source are optional and do not fail validation
+- File path line may be `File path:` or legacy `File:`; missing is treated as `N/A`
+
+## Examples
+
+- Generate + validate (quiet):
+```
+yarn llms:generate-and-validate
+```
+
+- Verbose generate + validate, with skip log:
+```
+yarn llms:generate-and-validate:verbose
+```
+
+- Validate a streamed output without writing a file:
+```
+node scripts/generate-llms-code.js --anchors --all --output - \
+  | node scripts/validate-llms-code.js --path /dev/stdin --strict --verify-anchors --project-root ..
+```
+