feat(seo): noindex LLM text endpoints

app/llms-full.txt/route.ts

@@ -13,6 +13,10 @@ export async function GET() {
   const scanned = await Promise.all(scan);
 
   return new Response(AGENT_INSTRUCTIONS + scanned.join('\n\n'), {
-    headers: { 'Content-Type': 'text/markdown; charset=utf-8' },
+    headers: {
+      'Content-Type': 'text/markdown; charset=utf-8',
+      // A dump of every docs page: fetchable by agents, but not a search result.
+      'X-Robots-Tag': 'noindex',
+    },
   });
 }

app/llms.mdx/[[...slug]]/route.ts

@@ -20,7 +20,12 @@ export async function GET(_req: NextRequest, { params }: { params: Promise<{ slu
   const page = getPage(slug);
   if (!page) notFound();
 
-  const headers = new Headers({ 'Content-Type': 'text/markdown; charset=utf-8' });
+  // This markdown duplicates the canonical HTML page, so keep it out of search
+  // results. Crawlers may still fetch it: noindex only suppresses indexing.
+  const headers = new Headers({
+    'Content-Type': 'text/markdown; charset=utf-8',
+    'X-Robots-Tag': 'noindex',
+  });
   appendMarkdownVaryHeader(headers);
 
   return new NextResponse(await getLLMText(page, { indexPointer: true }), {

next.config.mjs

@@ -220,6 +220,28 @@ const config = {
           },
         ],
       },
+      {
+        // Plain-text mirrors of the docs. Agents should fetch them; Google
+        // should not list them next to the HTML pages they duplicate. Matching
+        // happens on the requested path, so the homepage keeps its own headers
+        // when middleware rewrites it to /AGENTS.md for markdown clients.
+        source: "/:path*/llms.txt",
+        headers: [
+          {
+            key: "X-Robots-Tag",
+            value: "noindex",
+          },
+        ],
+      },
+      {
+        source: "/AGENTS.md",
+        headers: [
+          {
+            key: "X-Robots-Tag",
+            value: "noindex",
+          },
+        ],
+      },
       {
         source: "/(.*)",
         headers: [

tests/e2e/llm-endpoints.test.ts

@@ -95,6 +95,34 @@ describe('.md suffix end-to-end', () => {
     expect(response.headers.get('location')).toBe('/AGENTS.md');
   });
 
+  test('marks a .md docs URL noindex while its canonical HTML page stays indexable', async () => {
+    const markdown = await fetch(`${BASE_URL}/overview/sessions-api/quickstart.md`, {
+      headers: BROWSER_HEADERS,
+    });
+    expect(markdown.headers.get('x-robots-tag')).toContain('noindex');
+
+    const html = await fetch(`${BASE_URL}/overview/sessions-api/quickstart`, {
+      headers: BROWSER_HEADERS,
+    });
+    expect(html.headers.get('x-robots-tag')).toBeNull();
+  });
+
+  test('marks the plain-text agent endpoints noindex', async () => {
+    // Status is not asserted: /llms.txt is generated into public/ at build
+    // time, so it is absent when tests run. The header comes from the route
+    // config either way, which is what matters for crawlers.
+    for (const path of ['/llms.txt', '/llms-full.txt', '/AGENTS.md']) {
+      const response = await fetch(`${BASE_URL}${path}`, { headers: BROWSER_HEADERS });
+      expect(response.headers.get('x-robots-tag')).toContain('noindex');
+    }
+  });
+
+  test('keeps the homepage indexable when it negotiates markdown', async () => {
+    const response = await fetch(BASE_URL, { headers: { accept: 'text/markdown' } });
+    expect(response.headers.get('content-type')).toStartWith('text/markdown');
+    expect(response.headers.get('x-robots-tag')).toBeNull();
+  });
+
   test('llms-full.txt does not repeat the index pointer', async () => {
     const response = await fetch(`${BASE_URL}/llms-full.txt`, { headers: BROWSER_HEADERS });
     expect(response.status).toBe(200);