Skip to content

feat(site): split llms.txt into per-framework and blog sub-indexes #697

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
decepulis merged 1 commit into from
Mar 3, 2026
Merged

feat(site): split llms.txt into per-framework and blog sub-indexes #697

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion site/astro.config.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,14 @@ export default defineConfig({
}),
mdx({ extendMarkdownConfig: true }),
sitemap({
customPages: [`${SITE_URL}/llms.txt`],
// llms-markdown.ts auto-generates per-framework sub-indexes, but sitemap
// entries are hardcoded here. Add a new line when adding a framework.
customPages: [
`${SITE_URL}/llms.txt`,
`${SITE_URL}/blog/llms.txt`,
`${SITE_URL}/docs/framework/html/llms.txt`,
`${SITE_URL}/docs/framework/react/llms.txt`,
],
}),
pagefind(),
llmsMarkdown(),
Expand Down
190 changes: 100 additions & 90 deletions site/integrations/llms-markdown.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,14 @@ import type { AstroIntegration } from 'astro';
import { JSDOM } from 'jsdom';
import TurndownService from 'turndown';

interface PageEntry {
pathname: string;
title: string;
description?: string;
sort?: string;
framework?: string;
}

export default function llmsMarkdown(): AstroIntegration {
return {
name: 'llms-markdown',
Expand All @@ -18,9 +26,9 @@ export default function llmsMarkdown(): AstroIntegration {
});

// Track all docs and blog pages for llms.txt index
const docsPages: Array<{ pathname: string; title: string; description?: string; sort?: string }> = [];
const blogPages: Array<{ pathname: string; title: string; description?: string; sort?: string }> = [];
const otherPages: Array<{ pathname: string; title: string; description?: string; sort?: string }> = [];
const docsPages: PageEntry[] = [];
const blogPages: PageEntry[] = [];
const otherPages: PageEntry[] = [];

logger.info('Generating LLM-optimized markdown files...');

Expand Down Expand Up @@ -71,15 +79,18 @@ export default function llmsMarkdown(): AstroIntegration {
const sortAttr = contentElements[0]?.getAttribute('data-llms-sort');
const sort = sortAttr || undefined;

const frameworkAttr = contentElements[0]?.getAttribute('data-framework');
const framework = frameworkAttr || undefined;

// Write markdown file as sibling to the directory
// docs/framework/html/style/css/slug -> docs/framework/html/style/css/slug.md
// docs/framework/html/how-to/slug -> docs/framework/html/how-to/slug.md
const mdPath = join(siteDir, `${pathname}.md`);
await mkdir(dirname(mdPath), { recursive: true });
await writeFile(mdPath, markdown, 'utf-8');

// Track for llms.txt index (with leading slash for URLs)
if (pathname.startsWith('docs/')) {
docsPages.push({ pathname: `/${pathname}`, title, description, sort });
docsPages.push({ pathname: `/${pathname}`, title, description, sort, framework });
} else if (pathname.startsWith('blog/')) {
blogPages.push({ pathname: `/${pathname}`, title, description, sort });
} else {
Expand All @@ -90,117 +101,116 @@ export default function llmsMarkdown(): AstroIntegration {
}
}

// Generate llms.txt index file
const llmsTxt = generateLlmsTxt(docsPages, blogPages, otherPages);
const llmsTxtPath = join(siteDir, 'llms.txt');
await writeFile(llmsTxtPath, llmsTxt, 'utf-8');
// Group docs by framework
const docsByFramework = new Map<string, PageEntry[]>();
for (const doc of docsPages) {
const fw = doc.framework ?? 'unknown';
if (!docsByFramework.has(fw)) {
docsByFramework.set(fw, []);
}
docsByFramework.get(fw)!.push(doc);
}

// Write per-framework docs sub-indexes
const frameworks: string[] = [];
for (const [fw, fwPages] of docsByFramework) {
frameworks.push(fw);
const subIndex = generateDocsIndex(fw, fwPages);
const subIndexPath = join(siteDir, 'docs', 'framework', fw, 'llms.txt');
await mkdir(dirname(subIndexPath), { recursive: true });
await writeFile(subIndexPath, subIndex, 'utf-8');
}

// Write blog sub-index
if (blogPages.length > 0) {
const blogIndex = generateBlogIndex(blogPages);
const blogIndexPath = join(siteDir, 'blog', 'llms.txt');
await mkdir(dirname(blogIndexPath), { recursive: true });
await writeFile(blogIndexPath, blogIndex, 'utf-8');
}

// Write root llms.txt index
const rootIndex = generateRootIndex(frameworks, blogPages.length > 0, otherPages);
const rootIndexPath = join(siteDir, 'llms.txt');
await writeFile(rootIndexPath, rootIndex, 'utf-8');

const subIndexCount = frameworks.length + (blogPages.length > 0 ? 1 : 0);
logger.info(
`Generated ${docsPages.length + blogPages.length + otherPages.length} markdown files and llms.txt index`
`Generated ${docsPages.length + blogPages.length + otherPages.length} markdown files, llms.txt root index, and ${subIndexCount} sub-indexes`
);
},
},
};
}

function generateLlmsTxt(
docsPages: Array<{ pathname: string; title: string; description?: string; sort?: string }>,
blogPages: Array<{ pathname: string; title: string; description?: string; sort?: string }>,
otherPages: Array<{ pathname: string; title: string; description?: string; sort?: string }>
): string {
// Group docs by framework and style
const docsByFrameworkStyle = new Map<
string,
Array<{ pathname: string; title: string; description?: string; sort?: string }>
>();

for (const doc of docsPages) {
// Extract framework and style from pathname
// Pattern: /docs/framework/{framework}/style/{style}/{...slug}
const match = doc.pathname.match(/^\/docs\/framework\/([^/]+)\/style\/([^/]+)\//);
if (match) {
const [, framework, style] = match;
const key = `${framework}/${style}`;
if (!docsByFrameworkStyle.has(key)) {
docsByFrameworkStyle.set(key, []);
}
docsByFrameworkStyle.get(key)!.push(doc);
}
}

// Build llms.txt content
function generateRootIndex(frameworks: string[], hasBlog: boolean, otherPages: PageEntry[]): string {
let content = `# Video.js v10\n\n`;
content += `> Modern video player framework with multi-platform support\n\n`;

// Add documentation sections grouped by framework/style
if (docsByFrameworkStyle.size > 0) {
content += `## Documentation\n\n`;

// Sort by framework/style for consistent output
const sortedKeys = Array.from(docsByFrameworkStyle.keys()).sort();

for (const key of sortedKeys) {
const [framework, style] = key.split('/');
const frameworkLabel = framework.charAt(0).toUpperCase() + framework.slice(1);
const styleLabel = style.toUpperCase();

content += `### ${frameworkLabel} + ${styleLabel}\n\n`;
content += `## Documentation\n\n`;
for (const fw of [...frameworks].sort()) {
const label = fw.charAt(0).toUpperCase() + fw.slice(1);
content += `- [${label} Docs](/docs/framework/${fw}/llms.txt)\n`;
}
content += `\n`;

const docs = docsByFrameworkStyle.get(key)!;
// Sort docs by pathname for consistent output
docs.sort((a, b) => a.pathname.localeCompare(b.pathname));
if (hasBlog) {
content += `## Blog\n\n`;
content += `- [Blog Posts](/blog/llms.txt)\n\n`;
}

for (const doc of docs) {
if (doc.description) {
content += `- [${doc.title}](${doc.pathname}): ${doc.description}\n`;
} else {
content += `- [${doc.title}](${doc.pathname})\n`;
}
if (otherPages.length > 0) {
content += `## Other\n\n`;
const sorted = [...otherPages].sort((a, b) => a.pathname.localeCompare(b.pathname));
for (const page of sorted) {
if (page.description) {
content += `- [${page.title}](${page.pathname}.md): ${page.description}\n`;
} else {
content += `- [${page.title}](${page.pathname}.md)\n`;
}
content += `\n`;
}
content += `\n`;
}

// Add blog posts section
if (blogPages.length > 0) {
content += `## Blog Posts\n\n`;
return content;
}

// Sort by date using data-llms-sort attribute in reverse order (newest first)
const sortedBlogPages = [...blogPages].sort((a, b) => {
// If both have sort attributes, compare them (reverse for newest first)
if (a.sort && b.sort) {
return b.sort.localeCompare(a.sort);
}
// Fallback to pathname comparison if sort is missing
return b.pathname.localeCompare(a.pathname);
});
function generateDocsIndex(framework: string, pages: PageEntry[]): string {
const label = framework.charAt(0).toUpperCase() + framework.slice(1);
let content = `# Video.js v10 — ${label} Documentation\n\n`;

for (const post of sortedBlogPages) {
if (post.description) {
content += `- [${post.title}](${post.pathname}): ${post.description}\n`;
} else {
content += `- [${post.title}](${post.pathname})\n`;
}
const sorted = [...pages].sort((a, b) => a.pathname.localeCompare(b.pathname));
for (const page of sorted) {
if (page.description) {
content += `- [${page.title}](${page.pathname}.md): ${page.description}\n`;
} else {
content += `- [${page.title}](${page.pathname}.md)\n`;
}
content += `\n`;
}
content += `\n`;

// Add other pages section
if (otherPages.length > 0) {
content += `## Other\n\n`;
return content;
}

// Sort by pathname
const sortedOtherPages = [...otherPages].sort((a, b) => a.pathname.localeCompare(b.pathname));
function generateBlogIndex(pages: PageEntry[]): string {
let content = `# Video.js v10 — Blog\n\n`;

for (const page of sortedOtherPages) {
if (page.description) {
content += `- [${page.title}](${page.pathname}): ${page.description}\n`;
} else {
content += `- [${page.title}](${page.pathname})\n`;
}
// Newest first
const sorted = [...pages].sort((a, b) => {
if (a.sort && b.sort) {
return b.sort.localeCompare(a.sort);
}
return b.pathname.localeCompare(a.pathname);
});

for (const post of sorted) {
if (post.description) {
content += `- [${post.title}](${post.pathname}.md): ${post.description}\n`;
} else {
content += `- [${post.title}](${post.pathname}.md)\n`;
}
content += `\n`;
}
content += `\n`;

return content;
}