diff --git a/docusaurus.config.ts b/docusaurus.config.ts index 5bc27295e3..6807372575 100644 --- a/docusaurus.config.ts +++ b/docusaurus.config.ts @@ -4,6 +4,7 @@ import { themes as prismThemes } from "prism-react-renderer"; import type { Config, RouteConfig } from "@docusaurus/types"; import type * as Preset from "@docusaurus/preset-classic"; +import type { PluginOptions } from "@signalwire/docusaurus-plugin-llms-txt/public"; const DOCUSAURUS_BASE_URL = process.env.DOCUSAURUS_BASE_URL ?? "/"; @@ -91,6 +92,21 @@ const config: Config = { enableInDevelopment: false, }, ], + [ + "@signalwire/docusaurus-plugin-llms-txt", + { + // Enable with defaults + generate: { + enableMarkdownFiles: true, + enableLlmsFullTxt: false, + }, + include: { + includeBlog: false, + includePages: false, + includeDocs: true, + }, + } satisfies PluginOptions, + ], async function pluginLlmsTxt(context) { return { name: "llms-txt-plugin", @@ -297,7 +313,7 @@ const config: Config = { to: "/postgres/database/prisma-studio", label: "Studio", sub: "Explore and manipulate your data", - icon: "fa-regular fa-table" + icon: "fa-regular fa-table", }, { type: "docSidebar", @@ -305,7 +321,7 @@ const config: Config = { className: "teal", label: "Optimize", sub: "AI-driven query analysis", - icon: "fa-regular fa-magnifying-glass-chart" + icon: "fa-regular fa-magnifying-glass-chart", }, { type: "docSidebar", @@ -313,7 +329,7 @@ const config: Config = { className: "teal", label: "Accelerate", sub: "Make your database global", - icon: "fa-regular fa-bolt" + icon: "fa-regular fa-bolt", }, { type: "docSidebar", @@ -321,7 +337,7 @@ const config: Config = { sidebarId: "aiSidebar", label: "Prisma + AI", sub: "Build faster with Prisma + AI", - icon: "fa-regular fa-robot" + icon: "fa-regular fa-robot", }, ], }, diff --git a/functions/_middleware.ts b/functions/_middleware.ts new file mode 100644 index 0000000000..cfd258bc87 --- /dev/null +++ b/functions/_middleware.ts @@ -0,0 +1,113 @@ +interface Env { + ASSETS: { + fetch: typeof fetch; + }; +} + +const AI_CRAWLER_PATTERNS = [ + 'GPTBot', + 'OAI-SearchBot', + 'ChatGPT-User', + 'ChatGPT-User 2.0', + 'anthropic-ai', + 'OpenAI', + 'ClaudeBot', + 'claude-web', + 'Claude/', + 'Claude-User', + 'anthropic', + 'PerplexityBot', + 'Perplexity-User', + 'Google-Extended', + 'BingBot', + 'Amazonbot', + 'Applebot', + 'Applebot-Extended', + 'FacebookBot', + 'meta-externalagent', + 'LinkedInBot', + 'Bytespider', + 'AI2Bot', + 'CCBot', + 'Diffbot', + 'DuckAssistBot', + 'DuckDuckGo', + 'DeepSeekBot', + 'cohere-ai', + 'omgili', + 'TimpiBot', + 'YouBot', + 'MistralAI-User', + 'ImagesiftBot', + 'Scrapy', + + 'chatgpt', + 'openai', + 'gpt', + 'claude', + 'anthropic', + 'cursor', + 'windsurf', + 'perplexity', + 'copilot', + 'ai-agent', + 'llm-agent', +]; + +function isAICrawler(request: Request): boolean { + // if (!userAgent) return false; + + const userAgent = request.headers.get('user-agent') || ''; + const accept = request.headers.get('accept') || ''; + + const hasHtml = accept.includes('text/html'); + + const prefersNonHtml = + !hasHtml && + (accept.includes('application/json') || + accept.includes('text/plain') || + accept.includes('application/xml')); + + const hasAiUserAgent = AI_CRAWLER_PATTERNS.some((pattern) => userAgent.toLowerCase().includes(pattern.toLowerCase())); + + return prefersNonHtml || hasAiUserAgent + + // const normalizedUA = userAgent.toLowerCase(); + // return AI_CRAWLER_PATTERNS.some((pattern) => normalizedUA.includes(pattern.toLowerCase())); +} + +export const onRequest: PagesFunction = async (context) => { + const { request, next } = context; + + if (isAICrawler(request)) { + const url = new URL(request.url); + const urlPath = url.pathname; + + const markdownPath = `${urlPath}.md`; + + const markdownRequest = new Request(new URL(markdownPath, url.origin), { + method: request.method, + headers: request.headers, + }); + + const response = await context.env.ASSETS.fetch(markdownRequest); + + if (response.ok) { + // Check what content type we actually got + const actualContentType = response.headers.get('content-type'); + console.log(`Fetched ${markdownPath}, got content-type: ${actualContentType}`); + + return new Response(response.body, { + status: 200, + headers: { + 'Content-Type': 'text/markdown; charset=utf-8', + 'Cache-Control': 'public, max-age=3600', + }, + }); + } + + return next(); + } + + return next(); +}; diff --git a/functions/types.d.ts b/functions/types.d.ts new file mode 100644 index 0000000000..86ad944860 --- /dev/null +++ b/functions/types.d.ts @@ -0,0 +1,12 @@ +// TypeScript definitions for Cloudflare Pages Functions +interface EventContext { + request: Request; + env: Env; + params: Record; + data: Record; + next: () => Promise; + waitUntil: (promise: Promise) => void; + passThroughOnException: () => void; +} + +type PagesFunction = (context: EventContext) => Response | Promise; diff --git a/package-lock.json b/package-lock.json index b57e49bc85..128d01fdb5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,6 +16,7 @@ "@react-aria/overlays": "^3.29.1", "@react-aria/tooltip": "^3.8.7", "@react-aria/utils": "^3.24.1", + "@signalwire/docusaurus-plugin-llms-txt": "^2.0.0-alpha.2", "clsx": "^2.1.1", "copy-text-to-clipboard": "^3.1.0", "docusaurus-plugin-sass": "^0.2.6", @@ -6511,6 +6512,44 @@ "integrity": "sha512-RNiOoTPkptFtSVzQevY/yWtZwf/RxyVnPy/OcA9HBM3MlGDnBEYL5B41H0MTn0Uec8Hi+2qUtTfG2WWZBmMejQ==", "license": "BSD-3-Clause" }, + "node_modules/@signalwire/docusaurus-plugin-llms-txt": { + "version": "2.0.0-alpha.2", + "resolved": "https://registry.npmjs.org/@signalwire/docusaurus-plugin-llms-txt/-/docusaurus-plugin-llms-txt-2.0.0-alpha.2.tgz", + "integrity": "sha512-jLATcj60UE4/iYtrB4xbBqki9/+WSf2t/BQ2hcszzTnm1Rafl+kW7ykh1TXQSeq97nQaL6zTDzsJYrIdcFig1A==", + "license": "MIT", + "dependencies": { + "fs-extra": "^11.0.0", + "hast-util-select": "^6.0.4", + "hast-util-to-html": "^9.0.5", + "hast-util-to-string": "^3.0.1", + "p-map": "^7.0.2", + "rehype-parse": "^9", + "rehype-remark": "^10", + "remark-gfm": "^4", + "remark-stringify": "^11", + "string-width": "^5.0.0", + "unified": "^11", + "unist-util-visit": "^5" + }, + "engines": { + "node": ">=18.0.0" + }, + "peerDependencies": { + "@docusaurus/core": "^3.0.0" + } + }, + "node_modules/@signalwire/docusaurus-plugin-llms-txt/node_modules/p-map": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-7.0.3.tgz", + "integrity": "sha512-VkndIv2fIB99swvQoA65bm+fsmt6UNdGeIB0oxBs+WhAhdh08QA04JXpI7rbB9r08/nkbysKoya9rtDERYOYMA==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/@sinclair/typebox": { "version": "0.27.8", "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz", @@ -11667,6 +11706,20 @@ "node": ">= 0.4" } }, + "node_modules/hast-util-embedded": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/hast-util-embedded/-/hast-util-embedded-3.0.0.tgz", + "integrity": "sha512-naH8sld4Pe2ep03qqULEtvYr7EjrLK2QHY8KJR6RJkTUjPGObe1vnx585uzem2hGra+s1q08DZZpfgDVYRbaXA==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "hast-util-is-element": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/hast-util-from-dom": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/hast-util-from-dom/-/hast-util-from-dom-5.0.1.tgz", @@ -11749,6 +11802,19 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/hast-util-is-body-ok-link": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/hast-util-is-body-ok-link/-/hast-util-is-body-ok-link-3.0.1.tgz", + "integrity": "sha512-0qpnzOBLztXHbHQenVB8uNuxTnm/QBFUOmdOSsEn7GnBtyY07+ENTWVFBAnXd/zEgd9/SUG3lRY7hSIBWRgGpQ==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/hast-util-is-element": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/hast-util-is-element/-/hast-util-is-element-3.0.0.tgz", @@ -11762,6 +11828,23 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/hast-util-minify-whitespace": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/hast-util-minify-whitespace/-/hast-util-minify-whitespace-1.0.1.tgz", + "integrity": "sha512-L96fPOVpnclQE0xzdWb/D12VT5FabA7SnZOUMtL1DbXmYiHJMXZvFkIZfiMmTCNJHUeO2K9UYNXoVyfz+QHuOw==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "hast-util-embedded": "^3.0.0", + "hast-util-is-element": "^3.0.0", + "hast-util-whitespace": "^3.0.0", + "unist-util-is": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/hast-util-parse-selector": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-4.0.0.tgz", @@ -11775,6 +11858,23 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/hast-util-phrasing": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/hast-util-phrasing/-/hast-util-phrasing-3.0.1.tgz", + "integrity": "sha512-6h60VfI3uBQUxHqTyMymMZnEbNl1XmEGtOxxKYL7stY2o601COo62AWAYBQR9lZbYXYSBoxag8UpPRXK+9fqSQ==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "hast-util-embedded": "^3.0.0", + "hast-util-has-property": "^3.0.0", + "hast-util-is-body-ok-link": "^3.0.0", + "hast-util-is-element": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/hast-util-raw": { "version": "9.1.0", "resolved": "https://registry.npmjs.org/hast-util-raw/-/hast-util-raw-9.1.0.tgz", @@ -11855,6 +11955,29 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/hast-util-to-html": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/hast-util-to-html/-/hast-util-to-html-9.0.5.tgz", + "integrity": "sha512-OguPdidb+fbHQSU4Q4ZiLKnzWo8Wwsf5bZfbvu7//a9oTYoqD/fWpe96NuHkoS9h0ccGOTe0C4NGXdtS0iObOw==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/unist": "^3.0.0", + "ccount": "^2.0.0", + "comma-separated-tokens": "^2.0.0", + "hast-util-whitespace": "^3.0.0", + "html-void-elements": "^3.0.0", + "mdast-util-to-hast": "^13.0.0", + "property-information": "^7.0.0", + "space-separated-tokens": "^2.0.0", + "stringify-entities": "^4.0.0", + "zwitch": "^2.0.4" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/hast-util-to-jsx-runtime": { "version": "2.3.6", "resolved": "https://registry.npmjs.org/hast-util-to-jsx-runtime/-/hast-util-to-jsx-runtime-2.3.6.tgz", @@ -11882,6 +12005,32 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/hast-util-to-mdast": { + "version": "10.1.2", + "resolved": "https://registry.npmjs.org/hast-util-to-mdast/-/hast-util-to-mdast-10.1.2.tgz", + "integrity": "sha512-FiCRI7NmOvM4y+f5w32jPRzcxDIz+PUqDwEqn1A+1q2cdp3B8Gx7aVrXORdOKjMNDQsD1ogOr896+0jJHW1EFQ==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "@ungap/structured-clone": "^1.0.0", + "hast-util-phrasing": "^3.0.0", + "hast-util-to-html": "^9.0.0", + "hast-util-to-text": "^4.0.0", + "hast-util-whitespace": "^3.0.0", + "mdast-util-phrasing": "^4.0.0", + "mdast-util-to-hast": "^13.0.0", + "mdast-util-to-string": "^4.0.0", + "rehype-minify-whitespace": "^6.0.0", + "trim-trailing-lines": "^2.0.0", + "unist-util-position": "^5.0.0", + "unist-util-visit": "^5.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/hast-util-to-parse5": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/hast-util-to-parse5/-/hast-util-to-parse5-8.0.0.tgz", @@ -11924,6 +12073,22 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/hast-util-to-text": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/hast-util-to-text/-/hast-util-to-text-4.0.2.tgz", + "integrity": "sha512-KK6y/BN8lbaq654j7JgBydev7wuNMcID54lkRav1P0CaE1e47P72AWWPiGKXTJU271ooYzcvTAn/Zt0REnvc7A==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/unist": "^3.0.0", + "hast-util-is-element": "^3.0.0", + "unist-util-find-after": "^5.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/hast-util-whitespace": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/hast-util-whitespace/-/hast-util-whitespace-3.0.0.tgz", @@ -18919,6 +19084,35 @@ "regjsparser": "bin/parser" } }, + "node_modules/rehype-minify-whitespace": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/rehype-minify-whitespace/-/rehype-minify-whitespace-6.0.2.tgz", + "integrity": "sha512-Zk0pyQ06A3Lyxhe9vGtOtzz3Z0+qZ5+7icZ/PL/2x1SHPbKao5oB/g/rlc6BCTajqBb33JcOe71Ye1oFsuYbnw==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "hast-util-minify-whitespace": "^1.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/rehype-parse": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/rehype-parse/-/rehype-parse-9.0.1.tgz", + "integrity": "sha512-ksCzCD0Fgfh7trPDxr2rSylbwq9iYDkSn8TCDmEJ49ljEUBxDVCzCHv7QNzZOfODanX4+bWQ4WZqLCRWYLfhag==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "hast-util-from-html": "^2.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/rehype-raw": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/rehype-raw/-/rehype-raw-7.0.0.tgz", @@ -18949,6 +19143,23 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/rehype-remark": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/rehype-remark/-/rehype-remark-10.0.1.tgz", + "integrity": "sha512-EmDndlb5NVwXGfUa4c9GPK+lXeItTilLhE6ADSaQuHr4JUlKw9MidzGzx4HpqZrNCt6vnHmEifXQiiA+CEnjYQ==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "hast-util-to-mdast": "^10.0.0", + "unified": "^11.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/relateurl": { "version": "0.2.7", "resolved": "https://registry.npmjs.org/relateurl/-/relateurl-0.2.7.tgz", @@ -20702,6 +20913,16 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/trim-trailing-lines": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/trim-trailing-lines/-/trim-trailing-lines-2.1.0.tgz", + "integrity": "sha512-5UR5Biq4VlVOtzqkm2AZlgvSlDJtME46uV0br0gENbwN4l5+mMKT4b9gJKqWtuL2zAIqajGJGuvbCbcAJUZqBg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/trough": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/trough/-/trough-2.2.0.tgz", @@ -20922,6 +21143,20 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/unist-util-find-after": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/unist-util-find-after/-/unist-util-find-after-5.0.0.tgz", + "integrity": "sha512-amQa0Ep2m6hE2g72AugUItjbuM8X8cGQnFoHk0pGfrFeT9GZhzN5SW8nRsiGKK7Aif4CrACPENkA6P/Lw6fHGQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-is": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/unist-util-is": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-6.0.0.tgz", diff --git a/package.json b/package.json index f9989b6807..93af9e443b 100644 --- a/package.json +++ b/package.json @@ -24,9 +24,10 @@ "@react-aria/overlays": "^3.29.1", "@react-aria/tooltip": "^3.8.7", "@react-aria/utils": "^3.24.1", + "@signalwire/docusaurus-plugin-llms-txt": "^2.0.0-alpha.2", "clsx": "^2.1.1", - "docusaurus-plugin-sass": "^0.2.6", "copy-text-to-clipboard": "^3.1.0", + "docusaurus-plugin-sass": "^0.2.6", "posthog-docusaurus": "^2.0.4", "prism-react-renderer": "^2.4.1", "react": "^19.1.1",