feat: Add llms.txt support for AI-friendly documentation

package.json

@@ -6,7 +6,7 @@
   "scripts": {
     "analyze": "ANALYZE=true next build",
     "dev": "next-remote-watch ./src/content",
-    "build": "next build && node --experimental-modules ./scripts/downloadFonts.mjs",
+    "build": "next build && node --experimental-modules ./scripts/downloadFonts.mjs && node scripts/generate-llms-txt.js",
     "lint": "next lint && eslint \"src/content/**/*.md\"",
     "lint:fix": "next lint --fix && eslint \"src/content/**/*.md\" --fix",
     "format:source": "prettier --config .prettierrc --write \"{plugins,src}/**/*.{js,ts,jsx,tsx,css}\"",
@@ -21,6 +21,7 @@
     "postinstall": "yarn --cwd eslint-local-rules install && is-ci || husky install .husky",
     "check-all": "npm-run-all prettier lint:fix tsc rss",
     "rss": "node scripts/generateRss.js",
+    "llms": "node scripts/generate-llms-txt.js",
     "deadlinks": "node scripts/deadLinkChecker.js",
     "copyright": "node scripts/copyright.js",
     "test:eslint-local-rules": "yarn --cwd eslint-local-rules test"

public/llms.txt

@@ -0,0 +1,176 @@
+# React
+
+> The library for web and native user interfaces
+
+## Learn React
+
+- [Quick Start](https://react.dev/learn)
+  - [Tutorial: Tic-Tac-Toe](https://react.dev/learn/tutorial-tic-tac-toe)
+  - [Thinking in React](https://react.dev/learn/thinking-in-react)
+- [Installation](https://react.dev/learn/installation)
+  - [Creating a React App](https://react.dev/learn/creating-a-react-app)
+  - [Build a React App from Scratch](https://react.dev/learn/build-a-react-app-from-scratch)
+  - [Add React to an Existing Project](https://react.dev/learn/add-react-to-an-existing-project)
+- [Setup](https://react.dev/learn/setup)
+  - [Editor Setup](https://react.dev/learn/editor-setup)
+  - [Using TypeScript](https://react.dev/learn/typescript)
+  - [React Developer Tools](https://react.dev/learn/react-developer-tools)
+  - [Introduction](https://react.dev/learn/react-compiler/introduction)
+  - [Installation](https://react.dev/learn/react-compiler/installation)
+  - [Incremental Adoption](https://react.dev/learn/react-compiler/incremental-adoption)
+  - [Debugging and Troubleshooting](https://react.dev/learn/react-compiler/debugging)
+- [Describing the UI](https://react.dev/learn/describing-the-ui)
+  - [Your First Component](https://react.dev/learn/your-first-component)
+  - [Importing and Exporting Components](https://react.dev/learn/importing-and-exporting-components)
+  - [Writing Markup with JSX](https://react.dev/learn/writing-markup-with-jsx)
+  - [JavaScript in JSX with Curly Braces](https://react.dev/learn/javascript-in-jsx-with-curly-braces)
+  - [Passing Props to a Component](https://react.dev/learn/passing-props-to-a-component)
+  - [Conditional Rendering](https://react.dev/learn/conditional-rendering)
+  - [Rendering Lists](https://react.dev/learn/rendering-lists)
+  - [Keeping Components Pure](https://react.dev/learn/keeping-components-pure)
+  - [Your UI as a Tree](https://react.dev/learn/understanding-your-ui-as-a-tree)
+- [Adding Interactivity](https://react.dev/learn/adding-interactivity)
+  - [Responding to Events](https://react.dev/learn/responding-to-events)
+  - [State: A Component's Memory](https://react.dev/learn/state-a-components-memory)
+  - [Render and Commit](https://react.dev/learn/render-and-commit)
+  - [State as a Snapshot](https://react.dev/learn/state-as-a-snapshot)
+  - [Queueing a Series of State Updates](https://react.dev/learn/queueing-a-series-of-state-updates)
+  - [Updating Objects in State](https://react.dev/learn/updating-objects-in-state)
+  - [Updating Arrays in State](https://react.dev/learn/updating-arrays-in-state)
+- [Managing State](https://react.dev/learn/managing-state)
+  - [Reacting to Input with State](https://react.dev/learn/reacting-to-input-with-state)
+  - [Choosing the State Structure](https://react.dev/learn/choosing-the-state-structure)
+  - [Sharing State Between Components](https://react.dev/learn/sharing-state-between-components)
+  - [Preserving and Resetting State](https://react.dev/learn/preserving-and-resetting-state)
+  - [Extracting State Logic into a Reducer](https://react.dev/learn/extracting-state-logic-into-a-reducer)
+  - [Passing Data Deeply with Context](https://react.dev/learn/passing-data-deeply-with-context)
+  - [Scaling Up with Reducer and Context](https://react.dev/learn/scaling-up-with-reducer-and-context)
+- [Escape Hatches](https://react.dev/learn/escape-hatches)
+  - [Referencing Values with Refs](https://react.dev/learn/referencing-values-with-refs)
+  - [Manipulating the DOM with Refs](https://react.dev/learn/manipulating-the-dom-with-refs)
+  - [Synchronizing with Effects](https://react.dev/learn/synchronizing-with-effects)
+  - [You Might Not Need an Effect](https://react.dev/learn/you-might-not-need-an-effect)
+  - [Lifecycle of Reactive Effects](https://react.dev/learn/lifecycle-of-reactive-effects)
+  - [Separating Events from Effects](https://react.dev/learn/separating-events-from-effects)
+  - [Removing Effect Dependencies](https://react.dev/learn/removing-effect-dependencies)
+  - [Reusing Logic with Custom Hooks](https://react.dev/learn/reusing-logic-with-custom-hooks)
+
+## API Reference
+
+- [Hooks](https://react.dev/reference/react/hooks)
+  - [useActionState](https://react.dev/reference/react/useActionState)
+  - [useCallback](https://react.dev/reference/react/useCallback)
+  - [useContext](https://react.dev/reference/react/useContext)
+  - [useDebugValue](https://react.dev/reference/react/useDebugValue)
+  - [useDeferredValue](https://react.dev/reference/react/useDeferredValue)
+  - [useEffect](https://react.dev/reference/react/useEffect)
+  - [useEffectEvent](https://react.dev/reference/react/useEffectEvent)
+  - [useId](https://react.dev/reference/react/useId)
+  - [useImperativeHandle](https://react.dev/reference/react/useImperativeHandle)
+  - [useInsertionEffect](https://react.dev/reference/react/useInsertionEffect)
+  - [useLayoutEffect](https://react.dev/reference/react/useLayoutEffect)
+  - [useMemo](https://react.dev/reference/react/useMemo)
+  - [useOptimistic](https://react.dev/reference/react/useOptimistic)
+  - [useReducer](https://react.dev/reference/react/useReducer)
+  - [useRef](https://react.dev/reference/react/useRef)
+  - [useState](https://react.dev/reference/react/useState)
+  - [useSyncExternalStore](https://react.dev/reference/react/useSyncExternalStore)
+  - [useTransition](https://react.dev/reference/react/useTransition)
+- [Components](https://react.dev/reference/react/components)
+  - [<Fragment> (<>)](https://react.dev/reference/react/Fragment)
+  - [<Profiler>](https://react.dev/reference/react/Profiler)
+  - [<StrictMode>](https://react.dev/reference/react/StrictMode)
+  - [<Suspense>](https://react.dev/reference/react/Suspense)
+  - [<Activity>](https://react.dev/reference/react/Activity)
+  - [<ViewTransition>](https://react.dev/reference/react/ViewTransition)
+- [APIs](https://react.dev/reference/react/apis)
+  - [act](https://react.dev/reference/react/act)
+  - [addTransitionType](https://react.dev/reference/react/addTransitionType)
+  - [cache](https://react.dev/reference/react/cache)
+  - [cacheSignal](https://react.dev/reference/react/cacheSignal)
+  - [captureOwnerStack](https://react.dev/reference/react/captureOwnerStack)
+  - [createContext](https://react.dev/reference/react/createContext)
+  - [lazy](https://react.dev/reference/react/lazy)
+  - [memo](https://react.dev/reference/react/memo)
+  - [startTransition](https://react.dev/reference/react/startTransition)
+  - [use](https://react.dev/reference/react/use)
+  - [experimental_taintObjectReference](https://react.dev/reference/react/experimental_taintObjectReference)
+  - [experimental_taintUniqueValue](https://react.dev/reference/react/experimental_taintUniqueValue)
+  - [useFormStatus](https://react.dev/reference/react-dom/hooks/useFormStatus)
+  - [Common (e.g. <div>)](https://react.dev/reference/react-dom/components/common)
+  - [<form>](https://react.dev/reference/react-dom/components/form)
+  - [<input>](https://react.dev/reference/react-dom/components/input)
+  - [<option>](https://react.dev/reference/react-dom/components/option)
+  - [<progress>](https://react.dev/reference/react-dom/components/progress)
+  - [<select>](https://react.dev/reference/react-dom/components/select)
+  - [<textarea>](https://react.dev/reference/react-dom/components/textarea)
+  - [<link>](https://react.dev/reference/react-dom/components/link)
+  - [<meta>](https://react.dev/reference/react-dom/components/meta)
+  - [<script>](https://react.dev/reference/react-dom/components/script)
+  - [<style>](https://react.dev/reference/react-dom/components/style)
+  - [<title>](https://react.dev/reference/react-dom/components/title)
+  - [createPortal](https://react.dev/reference/react-dom/createPortal)
+  - [flushSync](https://react.dev/reference/react-dom/flushSync)
+  - [preconnect](https://react.dev/reference/react-dom/preconnect)
+  - [prefetchDNS](https://react.dev/reference/react-dom/prefetchDNS)
+  - [preinit](https://react.dev/reference/react-dom/preinit)
+  - [preinitModule](https://react.dev/reference/react-dom/preinitModule)
+  - [preload](https://react.dev/reference/react-dom/preload)
+  - [preloadModule](https://react.dev/reference/react-dom/preloadModule)
+  - [createRoot](https://react.dev/reference/react-dom/client/createRoot)
+  - [hydrateRoot](https://react.dev/reference/react-dom/client/hydrateRoot)
+  - [renderToPipeableStream](https://react.dev/reference/react-dom/server/renderToPipeableStream)
+  - [renderToReadableStream](https://react.dev/reference/react-dom/server/renderToReadableStream)
+  - [renderToStaticMarkup](https://react.dev/reference/react-dom/server/renderToStaticMarkup)
+  - [renderToString](https://react.dev/reference/react-dom/server/renderToString)
+  - [resume](https://react.dev/reference/react-dom/server/resume)
+  - [resumeToPipeableStream](https://react.dev/reference/react-dom/server/resumeToPipeableStream)
+  - [prerender](https://react.dev/reference/react-dom/static/prerender)
+  - [prerenderToNodeStream](https://react.dev/reference/react-dom/static/prerenderToNodeStream)
+  - [resumeAndPrerender](https://react.dev/reference/react-dom/static/resumeAndPrerender)
+  - [resumeAndPrerenderToNodeStream](https://react.dev/reference/react-dom/static/resumeAndPrerenderToNodeStream)
+- [Configuration](https://react.dev/reference/react-compiler/configuration)
+  - [compilationMode](https://react.dev/reference/react-compiler/compilationMode)
+  - [gating](https://react.dev/reference/react-compiler/gating)
+  - [logger](https://react.dev/reference/react-compiler/logger)
+  - [panicThreshold](https://react.dev/reference/react-compiler/panicThreshold)
+  - [target](https://react.dev/reference/react-compiler/target)
+- [Directives](https://react.dev/reference/react-compiler/directives)
+  - ["use memo"](https://react.dev/reference/react-compiler/directives/use-memo)
+  - ["use no memo"](https://react.dev/reference/react-compiler/directives/use-no-memo)
+- [Compiling Libraries](https://react.dev/reference/react-compiler/compiling-libraries)
+- [React Performance tracks](https://react.dev/reference/dev-tools/react-performance-tracks)
+  - [exhaustive-deps](https://react.dev/reference/eslint-plugin-react-hooks/lints/exhaustive-deps)
+  - [rules-of-hooks](https://react.dev/reference/eslint-plugin-react-hooks/lints/rules-of-hooks)
+  - [component-hook-factories](https://react.dev/reference/eslint-plugin-react-hooks/lints/component-hook-factories)
+  - [config](https://react.dev/reference/eslint-plugin-react-hooks/lints/config)
+  - [error-boundaries](https://react.dev/reference/eslint-plugin-react-hooks/lints/error-boundaries)
+  - [gating](https://react.dev/reference/eslint-plugin-react-hooks/lints/gating)
+  - [globals](https://react.dev/reference/eslint-plugin-react-hooks/lints/globals)
+  - [immutability](https://react.dev/reference/eslint-plugin-react-hooks/lints/immutability)
+  - [incompatible-library](https://react.dev/reference/eslint-plugin-react-hooks/lints/incompatible-library)
+  - [preserve-manual-memoization](https://react.dev/reference/eslint-plugin-react-hooks/lints/preserve-manual-memoization)
+  - [purity](https://react.dev/reference/eslint-plugin-react-hooks/lints/purity)
+  - [refs](https://react.dev/reference/eslint-plugin-react-hooks/lints/refs)
+  - [set-state-in-effect](https://react.dev/reference/eslint-plugin-react-hooks/lints/set-state-in-effect)
+  - [set-state-in-render](https://react.dev/reference/eslint-plugin-react-hooks/lints/set-state-in-render)
+  - [static-components](https://react.dev/reference/eslint-plugin-react-hooks/lints/static-components)
+  - [unsupported-syntax](https://react.dev/reference/eslint-plugin-react-hooks/lints/unsupported-syntax)
+  - [use-memo](https://react.dev/reference/eslint-plugin-react-hooks/lints/use-memo)
+  - [Components and Hooks must be pure](https://react.dev/reference/rules/components-and-hooks-must-be-pure)
+  - [React calls Components and Hooks](https://react.dev/reference/rules/react-calls-components-and-hooks)
+  - [Rules of Hooks](https://react.dev/reference/rules/rules-of-hooks)
+- [Server Components](https://react.dev/reference/rsc/server-components)
+- [Server Functions](https://react.dev/reference/rsc/server-functions)
+- [Directives](https://react.dev/reference/rsc/directives)
+  - ['use client'](https://react.dev/reference/rsc/use-client)
+  - ['use server'](https://react.dev/reference/rsc/use-server)
+- [Legacy React APIs](https://react.dev/reference/react/legacy)
+  - [Children](https://react.dev/reference/react/Children)
+  - [cloneElement](https://react.dev/reference/react/cloneElement)
+  - [Component](https://react.dev/reference/react/Component)
+  - [createElement](https://react.dev/reference/react/createElement)
+  - [createRef](https://react.dev/reference/react/createRef)
+  - [forwardRef](https://react.dev/reference/react/forwardRef)
+  - [isValidElement](https://react.dev/reference/react/isValidElement)
+  - [PureComponent](https://react.dev/reference/react/PureComponent)

scripts/generate-llms-txt.js

@@ -0,0 +1,28 @@
+#!/usr/bin/env node
+
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/**
+ * Generates llms.txt for react.dev
+ * Following spec: https://llmstxt.org/
+ *
+ * Usage:
+ *   yarn llms                    # Generate manually
+ *   yarn build                   # Automatically runs during build
+ *
+ * The file is generated from:
+ *   - Sidebar configs: src/sidebarLearn.json, src/sidebarReference.json
+ *   - Markdown content: src/content directory
+ *
+ * Output:
+ *   - public/llms.txt (format with links and descriptions)
+ */
+
+const {generateLlmsTxt} = require('../src/utils/llms');
+
+generateLlmsTxt();

src/utils/llms.js

@@ -0,0 +1,127 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ */
+const fs = require('fs');
+const path = require('path');
+const matter = require('gray-matter');
+
+const BASE_URL = 'https://react.dev';
+const CONTENT_DIR = path.join(process.cwd(), 'src/content');
+const PUBLIC_DIR = path.join(process.cwd(), 'public');
+
+/**
+ * Extract frontmatter from markdown file
+ */
+function getMarkdownContent(filePath) {
+  try {
+    const fullPath = path.join(CONTENT_DIR, filePath);
+    const fileContent = fs.readFileSync(fullPath, 'utf8');
+    const {data, content} = matter(fileContent);
+    return {frontmatter: data, content, raw: fileContent};
+  } catch (err) {
+    console.warn(`Could not read ${filePath}:`, err.message);
+    return null;
+  }
+}
+
+/**
+ * Convert sidebar path to file path
+ */
+function pathToFile(urlPath) {
+  // /learn -> learn/index.md
+  // /learn/hooks -> learn/hooks.md
+  const parts = urlPath.split('/').filter(Boolean);
+
+  if (parts.length === 1) {
+    return `${parts[0]}/index.md`;
+  }
+
+  return `${parts.join('/')}.md`;
+}
+
+/**
+ * Recursively process routes from sidebar config
+ */
+function processRoutes(routes, indent = 0, output = []) {
+  for (const route of routes) {
+    // Skip section headers
+    if (route.hasSectionHeader) continue;
+
+    const {title, path: urlPath, routes: children} = route;
+
+    if (urlPath) {
+      const filePath = pathToFile(urlPath);
+      const mdContent = getMarkdownContent(filePath);
+
+      if (mdContent) {
+        const {frontmatter} = mdContent;
+        const description = frontmatter.description || '';
+
+        // For llms.txt
+        const prefix = '  '.repeat(indent);
+        const link = `[${title}](${BASE_URL}${urlPath})`;
+        const line = description
+          ? `${prefix}- ${link}: ${description}`
+          : `${prefix}- ${link}`;
+        output.push(line);
+      }
+    }
+
+    // Process children
+    if (children && children.length > 0) {
+      processRoutes(children, indent + 1, output);
+    }
+  }
+
+  return output;
+}
+
+/**
+ * Generate llms.txt content
+ */
+function generate(sidebars) {
+  const lines = ['# React'];
+  lines.push('');
+  lines.push('> The library for web and native user interfaces');
+  lines.push('');
+
+  // Learn section
+  lines.push('## Learn React');
+  lines.push('');
+  const learnResult = processRoutes(sidebars.learn.routes || [], 0, []);
+  lines.push(...learnResult);
+  lines.push('');
+
+  // Reference section
+  lines.push('## API Reference');
+  lines.push('');
+  const refResult = processRoutes(sidebars.reference.routes || [], 0, []);
+  lines.push(...refResult);
+
+  return lines.join('\n');
+}
+
+/**
+ * Generate llms.txt file
+ */
+exports.generateLlmsTxt = function () {
+  console.log('Generating llms.txt...');
+
+  // Read sidebar configs
+  const sidebars = {
+    learn: require(path.join(process.cwd(), 'src/sidebarLearn.json')),
+    reference: require(path.join(process.cwd(), 'src/sidebarReference.json')),
+    // community and blog are less critical for LLMs
+  };
+
+  const content = generate(sidebars);
+  fs.writeFileSync(path.join(PUBLIC_DIR, 'llms.txt'), content);
+  console.log(`✓ Generated llms.txt (${(content.length / 1024).toFixed(1)}KB)`);
+};