From 07b8ce97f4d365545e7796deaa17d5d94fc2fd95 Mon Sep 17 00:00:00 2001
From: twlite <46562212+twlite@users.noreply.github.com>
Date: Fri, 13 Jun 2025 22:49:33 +0545
Subject: [PATCH] fix: llms.txt generator

---
 apps/website/package.json            |  1 +
 apps/website/src/plugins/llms-txt.js | 80 +++++++++++++---------------
 pnpm-lock.yaml                       |  3 ++
 3 files changed, 41 insertions(+), 43 deletions(-)

diff --git a/apps/website/package.json b/apps/website/package.json
index 9f4d7841..be87de4c 100644
--- a/apps/website/package.json
+++ b/apps/website/package.json
@@ -23,6 +23,7 @@
     "@docusaurus/theme-search-algolia": "^3.7.0",
     "@mdx-js/react": "^3.0.0",
     "clsx": "^2.0.0",
+    "gray-matter": "^4.0.3",
     "prism-react-renderer": "^2.3.0",
     "react": "^19.0.0",
     "react-dom": "^19.0.0"
diff --git a/apps/website/src/plugins/llms-txt.js b/apps/website/src/plugins/llms-txt.js
index 913b85c6..a82f272d 100644
--- a/apps/website/src/plugins/llms-txt.js
+++ b/apps/website/src/plugins/llms-txt.js
@@ -1,5 +1,8 @@
 const path = require('path');
 const fs = require('fs');
+const matter = require('gray-matter');
+
+const DOCS_URL = 'https://commandkit.dev/docs/next/guide';
 
 module.exports = function (context) {
   return {
@@ -7,75 +10,66 @@ module.exports = function (context) {
     loadContent: async () => {
       const { siteDir } = context;
       const docsDir = path.join(siteDir, 'docs');
+      const guideDir = path.join(docsDir, 'guide');
       const versionedDocsDir = path.join(siteDir, 'versioned_docs');
-      const allMdx = [];
+      const allDocs = [];
 
-      // Recursive function to get all mdx and md files (excluding API reference)
       const getMdxFiles = async (dir, versionPrefix = '') => {
         if (!fs.existsSync(dir)) return;
 
+        // skip versioned docs
+        if (dir.includes(versionedDocsDir)) return;
+
         const entries = await fs.promises.readdir(dir, { withFileTypes: true });
 
         for (const entry of entries) {
           const fullPath = path.join(dir, entry.name);
           if (entry.isDirectory()) {
-            // Skip api-reference directories
-            if (entry.name === 'api-reference') {
-              continue;
-            }
             await getMdxFiles(fullPath, versionPrefix);
-          } else if (
-            entry.name.endsWith('.mdx') ||
-            entry.name.endsWith('.md')
-          ) {
-            // Skip files in api-reference paths
-            if (fullPath.includes('api-reference')) {
-              continue;
-            }
+          } else if (entry.name.endsWith('.mdx') || entry.name.endsWith('.md')) {
             const content = await fs.promises.readFile(fullPath, 'utf8');
-            allMdx.push({
-              content,
-              path: fullPath,
+            const { data: frontmatter } = matter(content);
+
+            // Get relative path from guide directory
+            const relativePath = path.relative(guideDir, fullPath)
+              .replace(/\.mdx?$/, '')
+              .replace(/\\/g, '/');
+
+            allDocs.push({
+              title: frontmatter.title,
+              description: frontmatter.description,
+              path: relativePath,
               version: versionPrefix,
             });
           }
         }
       };
 
-      // Get current/latest docs
-      await getMdxFiles(docsDir, 'latest');
-
-      // Get versioned docs
-      if (fs.existsSync(versionedDocsDir)) {
-        const versionDirs = await fs.promises.readdir(versionedDocsDir, {
-          withFileTypes: true,
-        });
-        for (const versionDir of versionDirs) {
-          if (versionDir.isDirectory()) {
-            const versionName = versionDir.name.replace('version-', '');
-            await getMdxFiles(
-              path.join(versionedDocsDir, versionDir.name),
-              versionName,
-            );
-          }
-        }
+      // Only process guide directory
+      if (fs.existsSync(guideDir)) {
+        await getMdxFiles(guideDir, 'latest');
       }
 
-      return { allMdx };
+      return { allDocs };
     },
     postBuild: async ({ content, outDir }) => {
-      const { allMdx } = content;
+      const { allDocs } = content;
 
-      // Filter out any remaining API reference content and concatenate
-      const filteredContent = allMdx
-        .filter((item) => !item.path.includes('api-reference'))
-        .map((item) => item.content)
-        .join('\n\n---\n\n');
+      // Generate markdown content
+      const markdownContent = [
+        '# CommandKit Docs\n',
+        ...allDocs
+          .filter(doc => doc.title && doc.description)
+          .map(doc => {
+            const url = `${DOCS_URL}/${doc.path}`;
+            return `- [${doc.title}](${url}): ${doc.description || doc.title}`;
+          })
+      ].join('\n');
 
-      // Write concatenated content as llms.txt
+      // Write markdown content
       const llmsTxtPath = path.join(outDir, 'llms.txt');
       try {
-        await fs.promises.writeFile(llmsTxtPath, filteredContent);
+        await fs.promises.writeFile(llmsTxtPath, markdownContent);
         console.log('✅ llms.txt generated successfully');
       } catch (err) {
         console.error('❌ Error generating llms.txt:', err);
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 70833811..6f1770f2 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -114,6 +114,9 @@ importers:
       clsx:
         specifier: ^2.0.0
         version: 2.1.1
+      gray-matter:
+        specifier: ^4.0.3
+        version: 4.0.3
       prism-react-renderer:
         specifier: ^2.3.0
         version: 2.4.1(react@19.1.0)