diff --git a/astro.config.mjs b/astro.config.mjs index fdb8be4ba8..541c7c43be 100644 --- a/astro.config.mjs +++ b/astro.config.mjs @@ -3,12 +3,14 @@ import remarkHeading from 'remark-heading-id'; import { defineConfig } from 'astro/config'; import mdx from '@astrojs/mdx'; import { attributeMarkdown, wrapTables } from '/src/themes/octopus/utilities/custom-markdown.mjs'; +import llmMdEmitter from './src/integrations/llm-md-emitter.ts'; // https://astro.build/config export default defineConfig({ site: 'https://octopus.com', integrations: [ - mdx() + mdx(), + llmMdEmitter() ], markdown: { shikiConfig: { diff --git a/dictionary-octopus.txt b/dictionary-octopus.txt index b96e4aba15..05c46d8af5 100644 --- a/dictionary-octopus.txt +++ b/dictionary-octopus.txt @@ -215,6 +215,7 @@ Inedo inetmgr inetsrv inkey +inlinable INSTALLLOCATION internalcustomer ioutil @@ -323,6 +324,7 @@ nfsadmin nlog nmap noconsolelogging +nodir nologo nologs noninteractive @@ -417,6 +419,7 @@ projecttriggers proxied proxying pscustomobject +pubdate publicip pwsh pycryptodome @@ -548,6 +551,7 @@ tfvars TFVC thepassword threadid +timeframes timespan tlsv1 tmpfs diff --git a/public/docs/css/main.css b/public/docs/css/main.css index 398a019a5e..a4765cdc7a 100644 --- a/public/docs/css/main.css +++ b/public/docs/css/main.css @@ -1797,6 +1797,176 @@ li.has-children .sub-nav ul li:focus > a { color: var(--color-menu-link-alt); } +/* "Use Octopus docs with AI" dropdown */ +.octo-copy-md { + margin-block-start: var(--block-gap); +} + +.octo-copy-md__menu { + display: inline-block; + position: relative; +} + +.octo-copy-md__trigger { + display: inline-flex; + align-items: center; + gap: 0.5rem; + padding: 0.5rem 0.875rem; + border: 1px solid var(--border-color-menu-open); + border-radius: 999px; + background-color: transparent; + color: var(--color-menu-link); + font: inherit; + cursor: pointer; + list-style: none; + text-decoration: none; + user-select: none; + transition: + border-color 200ms cubic-bezier(0.4, 0, 0.2, 1), + background-color 200ms cubic-bezier(0.4, 0, 0.2, 1), + color 200ms cubic-bezier(0.4, 0, 0.2, 1), + box-shadow 200ms cubic-bezier(0.4, 0, 0.2, 1); +} + +.octo-copy-md__trigger > * { + text-decoration: none; + color: inherit; +} + +.octo-copy-md__trigger::-webkit-details-marker, +.octo-copy-md__trigger::marker { + content: ''; + display: none; +} + +.octo-copy-md__trigger-icon, +.octo-copy-md__trigger-caret { + display: inline-flex; + align-items: center; + justify-content: center; + line-height: 1; +} + +.octo-copy-md__trigger-icon::before { + content: '\f0eb'; /* fa-lightbulb */ + font-family: fa-solid; + font-size: 0.95em; + line-height: 1; + color: var(--color-menu-link-alt); + transition: color 200ms cubic-bezier(0.4, 0, 0.2, 1); +} + +.octo-copy-md__trigger-caret::before { + content: '\f078'; /* fa-chevron-down */ + font-family: fa-solid; + font-size: 0.7em; + line-height: 1; + color: currentColor; + transition: transform 200ms cubic-bezier(0.4, 0, 0.2, 1); +} + +.octo-copy-md__menu[open] + > .octo-copy-md__trigger + .octo-copy-md__trigger-caret::before { + transform: rotate(180deg); +} + +.octo-copy-md__trigger:hover { + color: var(--color-menu-link-active); + border-color: var(--color-menu-link-alt); +} + +.octo-copy-md__menu[open] > .octo-copy-md__trigger { + color: var(--color-menu-link-active); + border-color: var(--color-menu-link-alt); + background-color: var(--bg-color-menu-open); + box-shadow: 0 0.0625rem 0.25rem rgba(13, 128, 216, 0.08); +} + +.octo-copy-md__trigger:focus-visible { + outline: 2px solid var(--color-menu-link-alt); + outline-offset: 2px; +} + +.octo-copy-md .octo-copy-md__options { + position: absolute; + z-index: 10; + inset-block-start: calc(100% + 0.5rem); + inset-inline-start: 0; + margin: 0; + padding: 0.5rem; + list-style: none; + background-color: var(--bg-color-menu); + border: 1px solid var(--border-color-menu-open); + border-radius: 0.625rem; + min-width: 20rem; + box-sizing: border-box; + overflow: hidden; + box-shadow: + 0 0.625rem 1.875rem rgba(15, 37, 53, 0.12), + 0 0.125rem 0.375rem rgba(15, 37, 53, 0.06); +} + +.octo-copy-md .octo-copy-md__options .octo-copy-md__option { + display: inline-flex; + align-items: center; + gap: 0.625rem; + padding: 0.625rem 0.75rem; + border: none; + border-radius: 0.375rem; + background: transparent; + color: var(--color-menu-link); + font: inherit; + text-align: start; + text-decoration: none; + cursor: pointer; + transition: + background-color 150ms cubic-bezier(0.4, 0, 0.2, 1), + color 150ms cubic-bezier(0.4, 0, 0.2, 1); +} + +.octo-copy-md .octo-copy-md__options .octo-copy-md__option:hover, +.octo-copy-md .octo-copy-md__options .octo-copy-md__option:focus-visible { + background-color: var(--bg-color-menu-open); + color: var(--color-menu-link-active); + outline: none; +} + +.octo-copy-md__option::before { + font-family: fa-solid; + font-size: 0.95em; + width: 1.1em; + text-align: center; + color: var(--color-menu-link-alt); + flex-shrink: 0; + transition: color 150ms cubic-bezier(0.4, 0, 0.2, 1); +} + +.octo-copy-md__option--copy::before { + content: '\f0c5'; /* fa-copy */ +} + +.octo-copy-md__option--view::before { + content: '\f15c'; /* fa-file-lines */ +} + +.octo-copy-md__option--all::before { + content: '\f02d'; /* fa-book */ +} + +/* Visually hidden live region for copy-markdown.js status announcements. */ +.octo-copy-md__sr-status { + position: absolute; + width: 1px; + height: 1px; + padding: 0; + margin: -1px; + overflow: hidden; + clip: rect(0, 0, 0, 0); + white-space: nowrap; + border: 0; +} + /* Video */ .yt-video { diff --git a/public/docs/js/main.js b/public/docs/js/main.js index 19ad07f1e8..cd5a85d246 100644 --- a/public/docs/js/main.js +++ b/public/docs/js/main.js @@ -6,6 +6,7 @@ import { import { addResizedEvent } from './modules/resizing.js'; import { addStickyNavigation } from './modules/nav-sticky.js'; import { mobileNav } from './modules/nav-mobile.js'; +import { copyMarkdownMenus } from './modules/copy-markdown.js'; import { setClickableBlocks } from './modules/click-blocks.js'; import { setExternalLinkAttributes } from './modules/external-links.js'; import { monitorInputType } from './modules/input-type.js'; diff --git a/public/docs/js/modules/copy-markdown.js b/public/docs/js/modules/copy-markdown.js new file mode 100644 index 0000000000..271ed70573 --- /dev/null +++ b/public/docs/js/modules/copy-markdown.js @@ -0,0 +1,128 @@ +// @ts-check +import { qs, qsa } from './query.js'; + +class CopyMarkdown { + constructor(menu) { + this.menu = menu; + this.trigger = qs('[data-copy-md-trigger]', menu); + this.liveRegion = qs('[data-copy-md-live]', menu); + + this.addCopyHandlers(); + this.addListeners(); + } + + announce(btn, message) { + const labelEl = btn.querySelector('[data-copy-md-text]'); + if (labelEl) { + const restoreLabel = btn.dataset.copyMdLabel ?? ''; + labelEl.textContent = message; + setTimeout(() => { + labelEl.textContent = restoreLabel; + }, 2000); + } + if (this.liveRegion) { + // Force a content change so AT re-announces a repeated message. + this.liveRegion.textContent = ''; + setTimeout(() => { + this.liveRegion.textContent = message; + }, 50); + } + } + + // navigator.clipboard requires a secure context; falls back to + // execCommand for HTTP and older browsers. + async writeToClipboard(text) { + if ( + typeof navigator !== 'undefined' && + navigator.clipboard && + typeof navigator.clipboard.writeText === 'function' && + (typeof window === 'undefined' || window.isSecureContext !== false) + ) { + try { + await navigator.clipboard.writeText(text); + return true; + } catch (err) { + console.warn('[copy-md] navigator.clipboard failed, falling back', err); + } + } + + return this.execCommandCopyFallback(text); + } + + execCommandCopyFallback(text) { + if (typeof document === 'undefined') return false; + const ta = document.createElement('textarea'); + ta.value = text; + ta.setAttribute('readonly', ''); + ta.style.position = 'fixed'; + ta.style.top = '0'; + ta.style.left = '0'; + ta.style.opacity = '0'; + ta.style.pointerEvents = 'none'; + document.body.appendChild(ta); + ta.select(); + let ok = false; + try { + ok = document.execCommand('copy'); + } catch (err) { + console.warn('[copy-md] execCommand fallback threw', err); + ok = false; + } + document.body.removeChild(ta); + return ok; + } + + async handleCopy(btn) { + const url = btn.dataset.copyMdUrl; + const success = btn.dataset.copyMdSuccess ?? ''; + const errorMsg = btn.dataset.copyMdError ?? 'Copy failed'; + if (!url) return; + try { + const res = await fetch(url); + if (!res.ok) throw new Error('HTTP ' + res.status); + const text = await res.text(); + const ok = await this.writeToClipboard(text); + if (!ok) throw new Error('clipboard-write-failed'); + this.announce(btn, success); + } catch (err) { + console.error('[copy-md] failed:', err); + this.announce(btn, errorMsg); + } + } + + handleKeyboardNavigation(e) { + if (!this.menu.open) return; + + if (e.key === 'Escape') { + e.preventDefault(); + this.menu.open = false; + this.trigger.focus(); + } + } + + handleOutsideClick(e) { + if (!this.menu.open) return; + if (e.target instanceof Node && this.menu.contains(e.target)) return; + this.menu.open = false; + } + + addCopyHandlers() { + for (const btn of qsa('[data-copy-md-action="copy"]', this.menu)) { + btn.addEventListener('click', () => this.handleCopy(btn)); + } + } + + addListeners() { + this.menu.addEventListener('keydown', (e) => + this.handleKeyboardNavigation(e) + ); + + document.addEventListener('click', (e) => this.handleOutsideClick(e)); + } +} + +const copyMarkdownMenus = Array.from(qsa('[data-copy-md-menu]')).map( + (menu) => new CopyMarkdown(menu) +); + +export { copyMarkdownMenus }; diff --git a/public/docs/js/modules/nav-mobile.js b/public/docs/js/modules/nav-mobile.js index 4a956bdee7..075a521410 100644 --- a/public/docs/js/modules/nav-mobile.js +++ b/public/docs/js/modules/nav-mobile.js @@ -7,7 +7,10 @@ class MobileNav { this.mobileMenuWrapper = qs('[data-mobile-menu-wrapper]'); this.hamburgerIcon = qs('[data-hamburger-icon]'); this.mobileMenu = qs('[data-mobile-menu]'); - this.menuItems = qsa('.site-nav__list li'); + this.mobileMenuList = qs('[data-mobile-menu-list]'); + + this.populateMobileMenu(); + this.menuItems = qsa('[data-mobile-menu-list] li'); // Initially hide the menu this.mobileMenu.style.visibility = 'hidden'; @@ -15,6 +18,23 @@ class MobileNav { this.addListeners(); } + populateMobileMenu() { + // Idempotent on hot-reload. + if (this.mobileMenuList.children.length > 0) return; + + const sourceList = document.querySelector('#site-nav .site-nav__list'); + if (!sourceList) { + console.warn( + '[nav-mobile] #site-nav not found; mobile drawer will be empty' + ); + return; + } + + for (const child of sourceList.children) { + this.mobileMenuList.appendChild(child.cloneNode(true)); + } + } + toggleMobileMenu() { const isOpen = this.mobileMenuWrapper.classList.contains('is-active'); if (isOpen) { diff --git a/src/components/CopyMarkdown.astro b/src/components/CopyMarkdown.astro new file mode 100644 index 0000000000..30ba16072d --- /dev/null +++ b/src/components/CopyMarkdown.astro @@ -0,0 +1,87 @@ +--- +import { SITE } from '@config'; +import { Lang, Translations } from '@util/Languages'; +import { getEligibleSlugs } from '@util/mdxContent'; + +type Props = { + lang: string; +}; +const { lang } = Astro.props satisfies Props; + +const _ = Lang(lang); + +const docsPath = Astro.url.pathname.replace(/\/$/, ''); +const subfolderPrefix = SITE.subfolder.replace(/\/$/, '') + '/'; +const docsSlug = docsPath.startsWith(subfolderPrefix) + ? docsPath.slice(subfolderPrefix.length) + : null; +const eligibleSlugs = getEligibleSlugs(); +const pageMdUrl = + docsSlug && eligibleSlugs.has(docsSlug) ? docsPath + '.md' : null; +const allDocsUrl = SITE.subfolder.replace(/\/$/, '') + '/llms-full.txt'; +--- + +{ + pageMdUrl && ( +
+
+ + + +
+
+
+ ) +} diff --git a/src/components/MobileMenu.astro b/src/components/MobileMenu.astro index a9c39a8d16..a42123a31a 100644 --- a/src/components/MobileMenu.astro +++ b/src/components/MobileMenu.astro @@ -1,30 +1,7 @@ --- -import { Accelerator } from 'astro-accelerator-utils'; -import { Lang } from '@util/Languages'; -import { getNavigationItems } from '@util/getNavigationItems'; -import NavigationItem from '@components/NavigationItem.astro'; -import { SITE } from '@config'; +// Drawer body is populated client-side by nav-mobile.js cloning #site-nav. import SunIcon from '../../public/docs/img/SunIcon.astro'; import MoonIcon from '../../public/docs/img/MoonIcon.astro'; - -const accelerator = new Accelerator(SITE); -const stats = new accelerator.statistics('components/Hamburger.astro'); -stats.start(); - -// Properties -type Props = { - lang: string; -}; -const { lang } = Astro.props satisfies Props; - -// Language -const _ = Lang(lang); - -// Logic -const currentUrl = new URL(Astro.request.url); -const pages = await getNavigationItems(currentUrl, lang); - -stats.stop(); ---
- + diff --git a/src/data/language.json b/src/data/language.json index 945911ffb3..3d59270d31 100644 --- a/src/data/language.json +++ b/src/data/language.json @@ -22,6 +22,26 @@ "en": "Edit on GitHub" } }, + "octopus_copy_md": { + "label": { + "en": "Use Octopus docs with AI" + }, + "copy": { + "en": "Copy this page as markdown" + }, + "copied": { + "en": "Copied!" + }, + "view_raw": { + "en": "Open this page as markdown" + }, + "download_all": { + "en": "Open all docs as markdown" + }, + "error": { + "en": "Copy failed" + } + }, "post": { "written_by": { "en": "" diff --git a/src/integrations/llm-md-emitter.ts b/src/integrations/llm-md-emitter.ts new file mode 100644 index 0000000000..6ab07241b9 --- /dev/null +++ b/src/integrations/llm-md-emitter.ts @@ -0,0 +1,163 @@ +// Emits per-page `.md` files into `dist/docs/` after the main build, so +// LLM tools can fetch any eligible page as plain markdown. + +import type { AstroIntegration } from 'astro'; +import { fileURLToPath } from 'node:url'; +import * as path from 'node:path'; +import * as fs from 'node:fs'; +import { globSync } from 'glob'; +import matter from 'gray-matter'; +import { + eligibleForMarkdownWithLookup, + normalizeSubtitle, + pathToSlug, + resolveMdxIncludesWithLookup, + sanitizeTitle, + stripFrontmatter, + type SharedContentLookup, +} from '../themes/octopus/utilities/mdxContentCore'; + +const PROJECT_ROOT = path.resolve( + fileURLToPath(import.meta.url), + '..', + '..', + '..' +); +const PAGES_ROOT = path.join(PROJECT_ROOT, 'src', 'pages', 'docs'); +const SHARED_CONTENT_ROOT = path.join(PROJECT_ROOT, 'src', 'shared-content'); + +function buildSharedContentLookup(): SharedContentLookup { + const cache = new Map(); + const files = globSync('**/*.{md,mdx}', { + cwd: SHARED_CONTENT_ROOT, + nodir: true, + }); + for (const f of files) { + const abs = path.join(SHARED_CONTENT_ROOT, f); + cache.set('/' + path.posix.join('src', 'shared-content', f), abs); + } + return (lookupPath: string) => { + const abs = cache.get(lookupPath); + if (!abs) return undefined; + try { + return fs.readFileSync(abs, 'utf8'); + } catch { + return undefined; + } + }; +} + +type EmittedPage = { + slug: string; + title: string; + subtitle: string | null; + body: string; +}; + +export default function llmMdEmitter(): AstroIntegration { + return { + name: 'llm-md-emitter', + hooks: { + 'astro:build:done': async ({ dir, logger }) => { + const distDir = fileURLToPath(dir); + const docsOutDir = path.join(distDir, 'docs'); + + const lookup = buildSharedContentLookup(); + const files = globSync('**/*.{md,mdx}', { + cwd: PAGES_ROOT, + nodir: true, + }); + + const seenSlugs = new Map(); + const emitted: EmittedPage[] = []; + let skippedCount = 0; + let failureCount = 0; + + for (const rel of files) { + const abs = path.join(PAGES_ROOT, rel); + const globKey = '/' + path.posix.join('src', 'pages', 'docs', rel); + + let raw: string; + try { + raw = fs.readFileSync(abs, 'utf8'); + } catch (err) { + logger.warn( + `[llm-md-emitter] could not read ${rel}: ${(err as Error).message}` + ); + failureCount++; + continue; + } + + let parsed; + try { + parsed = matter(raw); + } catch (err) { + logger.warn( + `[llm-md-emitter] frontmatter parse failed for ${rel}: ${(err as Error).message}` + ); + failureCount++; + continue; + } + const fm = parsed.data ?? {}; + + const verdict = eligibleForMarkdownWithLookup( + { path: globKey, frontmatter: fm, raw }, + lookup + ); + if (!verdict.eligible) { + skippedCount++; + continue; + } + + const slug = pathToSlug(rel); + if (!slug) continue; + const winner = seenSlugs.get(slug); + if (winner !== undefined) { + logger.warn( + `[llm-md-emitter] slug collision on '${slug}': keeping '${winner}', skipping '${rel}'` + ); + continue; + } + seenSlugs.set(slug, rel); + + const body = resolveMdxIncludesWithLookup( + stripFrontmatter(raw), + lookup, + 0, + globKey + ).trim(); + + emitted.push({ + slug, + title: sanitizeTitle(fm.title), + subtitle: normalizeSubtitle(fm.subtitle), + body, + }); + } + + let writeFailures = 0; + for (const page of emitted) { + // UTF-8 BOM so files self-declare encoding when served without charset. + let out = '# ' + page.title + '\n\n'; + if (page.subtitle) out += '> ' + page.subtitle + '\n\n'; + out += page.body + '\n'; + + const target = path.join(docsOutDir, page.slug + '.md'); + try { + fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.writeFileSync(target, out, 'utf8'); + } catch (err) { + logger.warn( + `[llm-md-emitter] write failed for ${page.slug}.md: ${(err as Error).message}` + ); + writeFailures++; + } + } + + logger.info( + `[llm-md-emitter] emitted ${emitted.length} .md files, skipped ${skippedCount} ineligible, ${failureCount} parse failures, ${writeFailures} write failures` + ); + }, + }, + }; +} diff --git a/src/layouts/Default.astro b/src/layouts/Default.astro index fedbb8890c..0221ebdc55 100644 --- a/src/layouts/Default.astro +++ b/src/layouts/Default.astro @@ -22,6 +22,7 @@ import ArticleJourney from '../components/ArticleJourney.astro'; import Feedback from '../components/Feedback.astro'; import EditOnGithub from '../components/EditOnGithub.astro'; import LastUpdated from '../components/LastUpdated.astro'; +import CopyMarkdown from '../components/CopyMarkdown.astro'; import Plausible from 'src/components/Plausible.astro'; import Footer from 'src/components/Footer.astro'; @@ -114,6 +115,7 @@ const showSearch = !isSearchPage; headings={headings} lang={lang} /> + Octopus makes it easy to improve your Argo CD deployments with environment modeling and deployment orchestration. Automate everything from environment promotion and compliance to tests and change management. diff --git a/src/pages/docs/kubernetes/index.mdx b/src/pages/docs/kubernetes/index.mdx index 992b4aa5db..36de1bd74c 100644 --- a/src/pages/docs/kubernetes/index.mdx +++ b/src/pages/docs/kubernetes/index.mdx @@ -8,7 +8,7 @@ navSection: Kubernetes description: Octopus Deploy provides support for deploying Kubernetes resources. navOrder: 27 --- -import RecentlyUpdated from '@components/RecentlyUpdated.astro'; +import RecentlyUpdated from '@components/RecentlyUpdated.astro'; // Test fixture for tests/llm-endpoints.spec.ts (STABLE_MDX_PATH); keep these imports. import Card from 'src/components/Card.astro'; Octopus Deploy makes it easy to manage your Kubernetes resources, whether you're starting simple or want complete control over a complex setup. This section has everything you need to know about using Octopus for Kubernetes CD. @@ -18,6 +18,7 @@ Octopus Deploy makes it easy to manage your Kubernetes resources, whether you're ::: ## Why use Octopus as your Kubernetes CD platform + - Model environments and tenants for hundreds of applications with ease. - Work with any Kubernetes distribution in the cloud or on-premises. Source YAML or Helm charts from Git or repositories. - Monitor and safely troubleshoot your Kubernetes applications, both during and after deployment. @@ -27,6 +28,7 @@ Octopus Deploy makes it easy to manage your Kubernetes resources, whether you're - Use pre-approved [kubectl](/docs/deployments/kubernetes/kubectl) scripts ## Getting started + With Octopus, you can choose between managing your application configuration with Helm charts or YAML. ::::div{.docs-home .simple-grid} @@ -66,12 +68,16 @@ With Octopus, you can choose between managing your application configuration wit :::: ## First production deployment + When you’re ready to apply Octopus to a real scenario, we recommend that you: + - Familiarize yourself with 3 fundamental Octopus concepts: [projects](/docs/projects), [environments](/docs/infrastructure/environments), and [targets](/docs/infrastructure/deployment-targets). When you have time, you can learn about other Octopus concepts in our [glossary](/docs/getting-started/glossary). - Learn how to implement GitOps best practices with Octopus and use our observability features like [live object status](/docs/kubernetes/live-object-status). ## Deployments at scale + Learn more about deploying to multiple apps with Octopus, with these guides: + - [Release triggers](/docs/projects/project-triggers/external-feed-triggers) and [channels](/docs/releases/channels) to automate deployments - [Step templates](/docs/projects/custom-step-templates) to create new pipelines with ease - [Configuration as Code](/docs/projects/version-control) to manage deployment pipeline versions @@ -79,6 +85,7 @@ Learn more about deploying to multiple apps with Octopus, with these guides: - Maintenance tasks with [runbooks](/docs/runbooks/) and [scripts](/docs/deployments/kubernetes/kubectl) ## Learn more + - Learn more about [Kubernetes deployments with Octopus](https://octopus.com/use-case/kubernetes) - How to choose between [Kubernetes deployment targets](/docs/kubernetes/targets/kubernetes-api) - [Learn more about Kubernetes](https://octopus.com/blog/search?context=blog&q=kubernetes) on our blog diff --git a/src/pages/docs/llms-full.txt.ts b/src/pages/docs/llms-full.txt.ts new file mode 100644 index 0000000000..9e6e5a845c --- /dev/null +++ b/src/pages/docs/llms-full.txt.ts @@ -0,0 +1,80 @@ +import { SITE } from '@config'; +import { Accelerator } from 'astro-accelerator-utils'; +import { + compareForLlmSurfaces, + eligibleForMarkdown, + normalizeSubtitle, + resolveMdxIncludes, + sanitizeTitle, + stripFrontmatter, +} from '@util/mdxContent'; + +const PROJECT_TITLE = 'Octopus Deploy Documentation - Full Export'; +const PROJECT_SUMMARY = + 'All eligible documentation pages as plain markdown, ordered by navigation section then page order.'; + +const articles = import.meta.glob(['./**/*.md', './**/*.mdx']); +const raws = import.meta.glob(['./**/*.md', './**/*.mdx'], { + query: '?raw', + import: 'default', +}); + +const subfolderPrefix = SITE.subfolder.replace(/\/$/, '') + '/'; + +async function getData() { + const accelerator = new Accelerator(SITE); + const entries: { + body: string; + slug: string; + navOrder: number; + navSection: string; + }[] = []; + + for (const path in articles) { + const article: any = await articles[path](); + const fm = article.frontmatter ?? {}; + + const raw = await raws[path](); + const verdict = eligibleForMarkdown({ path, frontmatter: fm, raw }); + if (!verdict.eligible) continue; + + const url = accelerator.urlFormatter.formatAddress(article.url); + if (!url.startsWith(subfolderPrefix)) continue; + if (url.endsWith('/')) continue; + const slug = url.slice(subfolderPrefix.length); + if (slug.length === 0) continue; + + const body = resolveMdxIncludes(stripFrontmatter(raw), 0, path).trim(); + const title = sanitizeTitle(fm.title); + const subtitle = normalizeSubtitle(fm.subtitle); + const fullUrl = SITE.url + url + '.md'; + + let section = '# ' + title + '\n'; + section += 'Source: ' + fullUrl + '\n\n'; + if (subtitle) section += '> ' + subtitle + '\n\n'; + section += body; + + entries.push({ + body: section, + slug, + navOrder: fm.navOrder ?? 999999, + navSection: fm.navSection ?? '', + }); + } + + entries.sort(compareForLlmSurfaces); + + // UTF-8 BOM so files self-declare encoding when served without charset. + let content = '# ' + PROJECT_TITLE + '\n\n'; + content += '> ' + PROJECT_SUMMARY + '\n\n'; + for (const entry of entries) { + content += entry.body + '\n\n'; + } + + return new Response(content, { + status: 200, + headers: { 'Content-Type': 'text/plain; charset=utf-8' }, + }); +} + +export const GET = getData; diff --git a/src/pages/docs/llms.txt.ts b/src/pages/docs/llms.txt.ts index 8f520e92be..28bfc75d9f 100644 --- a/src/pages/docs/llms.txt.ts +++ b/src/pages/docs/llms.txt.ts @@ -1,89 +1,92 @@ -// warning: This file is overwritten by Astro Accelerator +// Generates llms.txt per https://llmstxt.org -// Generates an LLMs.txt file of documentation pages import { SITE } from '@config'; -import { PostFiltering, Accelerator } from 'astro-accelerator-utils'; +import { Accelerator } from 'astro-accelerator-utils'; +import { + compareForLlmSurfaces, + eligibleForMarkdown, + sanitizeTitle, +} from '@util/mdxContent'; + +const PROJECT_TITLE = 'Octopus Deploy Documentation'; +const PROJECT_SUMMARY = + 'Official documentation for Octopus Deploy: deployment, runbooks, infrastructure, configuration, integrations, and operations guidance.'; + +const allPages = import.meta.glob(['./**/*.md', './**/*.mdx']); +const allRaws = import.meta.glob(['./**/*.md', './**/*.mdx'], { + query: '?raw', + import: 'default', +}); async function getData() { - //@ts-ignore - const allPages = import.meta.glob(['./**/*.md', './**/*.mdx']); - - const accelerator = new Accelerator(SITE); - let pages = []; - - for (const path in allPages) { - const article: any = await allPages[path](); - - // Skip drafts and redirect pages - if (article.frontmatter.draft) { - continue; - } - - if (article.frontmatter.redirect) { - continue; - } - - // Skip if it shouldn't be in sitemap (likely similar filtering logic) - const addToLlms = PostFiltering.showInSitemap(article); - if (!addToLlms) { - continue; - } - - let url = accelerator.urlFormatter.formatAddress(article.url); - - // Handle author pages if needed (similar to sitemap logic) - if (article.frontmatter.layout == 'src/layouts/Author.astro') { - url += '1/'; - } - - const fullUrl = SITE.url + url; - // Remove square brackets - const title = (article.frontmatter.title || 'Untitled').replace(/[[\]]/g, ''); - const description = article.frontmatter.description || 'Documentation page for Octopus Deploy'; - - pages.push({ - title, - description, - url: fullUrl, - // Clean slug for sorting - slug: url.replace(/^\/docs\//, '').replace(/\/$/, ''), - navOrder: article.frontmatter.navOrder || 999999, - navSection: article.frontmatter.navSection || '', - pubDate: article.frontmatter.pubDate, - modDate: article.frontmatter.modDate, - }); - } - - // Sort by navSection, then navOrder, then slug (same logic as before) - pages.sort((a, b) => { - // First, sort by navSection alphabetically - const sectionA = a.navSection || ''; - const sectionB = b.navSection || ''; - if (sectionA !== sectionB) { - return sectionA.localeCompare(sectionB); - } - - // Within the same section, sort by navOrder first - if (a.navOrder !== b.navOrder) { - return a.navOrder - b.navOrder; - } - - // If navOrder is the same, sort by slug alphabetically - return a.slug.localeCompare(b.slug); - }); - - // Generate the LLMs.txt content - let content = '## Documentation\n\n'; - pages.forEach(page => { - content += `- [${page.title}](${page.url}): ${page.description}\n`; - }); - - return new Response(content, { - status: 200, - headers: { - 'Content-Type': 'text/plain', - }, - }); + const accelerator = new Accelerator(SITE); + const subfolderPrefix = SITE.subfolder.replace(/\/$/, '') + '/'; + + type Page = { + title: string; + description: string; + url: string; + slug: string; + navOrder: number; + navSection: string; + }; + const pages: Page[] = []; + + for (const path in allPages) { + const article: any = await allPages[path](); + const fm = article.frontmatter ?? {}; + + const raw = await allRaws[path](); + const verdict = eligibleForMarkdown({ path, frontmatter: fm, raw }); + if (!verdict.eligible) continue; + + const url = accelerator.urlFormatter.formatAddress(article.url); + if (!url.startsWith(subfolderPrefix)) continue; + if (url.endsWith('/')) continue; + const slug = url.slice(subfolderPrefix.length); + if (slug.length === 0) continue; + + const fullUrl = SITE.url + url + '.md'; + const title = sanitizeTitle(fm.title); + const description = + typeof fm.description === 'string' && fm.description.trim().length > 0 + ? fm.description + : 'Documentation page for Octopus Deploy'; + + // `??` so navOrder=0 and navSection='' aren't promoted to defaults. + pages.push({ + title, + description, + url: fullUrl, + slug, + navOrder: fm.navOrder ?? 999999, + navSection: fm.navSection ?? '', + }); + } + + pages.sort(compareForLlmSurfaces); + + // UTF-8 BOM so files self-declare encoding when served without charset. + let content = '# ' + PROJECT_TITLE + '\n\n'; + content += '> ' + PROJECT_SUMMARY + '\n\n'; + + let lastSection: string | null = null; + for (const page of pages) { + if (page.navSection !== lastSection) { + if (lastSection !== null) content += '\n'; + const heading = + page.navSection.trim().length > 0 ? page.navSection : 'Other'; + content += '## ' + heading + '\n\n'; + lastSection = page.navSection; + } + content += + '- [' + page.title + '](' + page.url + '): ' + page.description + '\n'; + } + + return new Response(content, { + status: 200, + headers: { 'Content-Type': 'text/plain; charset=utf-8' }, + }); } -export const GET = getData; \ No newline at end of file +export const GET = getData; diff --git a/src/themes/octopus/components/HtmlHead.astro b/src/themes/octopus/components/HtmlHead.astro index 02e57462d8..2dbf2d1585 100644 --- a/src/themes/octopus/components/HtmlHead.astro +++ b/src/themes/octopus/components/HtmlHead.astro @@ -2,6 +2,7 @@ import { Accelerator } from 'astro-accelerator-utils'; import type { Frontmatter } from 'astro-accelerator-utils/types/Frontmatter'; import { SITE, OPEN_GRAPH, HEADER_SCRIPTS } from '@config'; +import { getEligibleSlugs } from '@util/mdxContent'; const accelerator = new Accelerator(SITE); const stats = new accelerator.statistics('octopus/components/HtmlHead.astro'); @@ -20,68 +21,107 @@ const imageSrc = frontmatter.bannerImage?.src ?? OPEN_GRAPH.image.src; const imageAlt = frontmatter.bannerImage?.alt ?? OPEN_GRAPH.image.alt; const robots = frontmatter.robots ?? 'index, follow'; const canonicalImageSrc = new URL(imageSrc, Astro.site); -const canonicalURL = accelerator.urlFormatter.formatUrl(new URL(Astro.url.pathname, Astro.site + SITE.subfolder)); +const canonicalURL = accelerator.urlFormatter.formatUrl( + new URL(Astro.url.pathname, Astro.site + SITE.subfolder) +); const socialTitle = await accelerator.markdown.getTextFrom(frontmatter?.title); -const title = `${ accelerator.markdown.titleCase(socialTitle) } ${ ((frontmatter.titleAdditional) ? ` ${frontmatter.titleAdditional}` : '') } | ${ SITE.title }`; -const pageMeta = (frontmatter?.meta && frontmatter?.meta?.length > 0) - ? frontmatter.meta - : []; +const title = `${accelerator.markdown.titleCase(socialTitle)} ${frontmatter.titleAdditional ? ` ${frontmatter.titleAdditional}` : ''} | ${SITE.title}`; +const pageMeta = + frontmatter?.meta && frontmatter?.meta?.length > 0 ? frontmatter.meta : []; const authorList = accelerator.authors.forPost(frontmatter); -const authorMeta = (authorList.mainAuthor?.frontmatter?.meta && authorList.mainAuthor?.frontmatter?.meta?.length > 0) - ? authorList.mainAuthor.frontmatter.meta - : []; +const authorMeta = + authorList.mainAuthor?.frontmatter?.meta && + authorList.mainAuthor?.frontmatter?.meta?.length > 0 + ? authorList.mainAuthor.frontmatter.meta + : []; const autoMeta = (name: string) => { - return pageMeta.filter(m => m.name.toLowerCase() === name).length === 0; -} + return pageMeta.filter((m) => m.name.toLowerCase() === name).length === 0; +}; + +const docsPath = Astro.url.pathname.replace(/\/$/, ''); +const subfolderPrefix = SITE.subfolder.replace(/\/$/, '') + '/'; +const docsSlug = docsPath.startsWith(subfolderPrefix) + ? docsPath.slice(subfolderPrefix.length) + : null; +const eligibleSlugs = getEligibleSlugs(); +const markdownAlternateHref = + docsSlug && eligibleSlugs.has(docsSlug) ? SITE.url + docsPath + '.md' : null; stats.stop(); --- + {title} - - + + - - {autoMeta('viewport') && - + + { + autoMeta('viewport') && ( + + ) } - {autoMeta('format-detection') && - + { + autoMeta('format-detection') && ( + + ) } - {autoMeta('theme-color') && - + { + autoMeta('theme-color') && ( + + ) } - {autoMeta('canonical') && - + {autoMeta('canonical') && } + { + markdownAlternateHref && ( + + ) } - {SITE.feedUrl && - + { + SITE.feedUrl && ( + + ) } - - - + + + - - - - - + + + + + - - - - + + + + - {pageMeta.map(m => - - )} - {authorMeta.map(m => - - )} + {pageMeta.map((m) => )} + {authorMeta.map((m) => )} diff --git a/src/themes/octopus/utilities/mdxContent.ts b/src/themes/octopus/utilities/mdxContent.ts new file mode 100644 index 0000000000..28b2b74035 --- /dev/null +++ b/src/themes/octopus/utilities/mdxContent.ts @@ -0,0 +1,78 @@ +import matter from 'gray-matter'; +import { + eligibleForMarkdownWithLookup, + pathToSlug as relPathToSlug, + resolveMdxIncludesWithLookup, + stripFrontmatter, + type EligibilityInput, + type MarkdownEligibility, + type SharedContentLookup, +} from './mdxContentCore'; + +export { + stripFrontmatter, + sanitizeTitle, + normalizeSubtitle, + compareForLlmSurfaces, + type NavSortable, + type MarkdownEligibility, + type SharedContentLookup, +} from './mdxContentCore'; + +const sharedContent = import.meta.glob( + ['/src/shared-content/**/*.md', '/src/shared-content/**/*.mdx'], + { query: '?raw', import: 'default', eager: true } +); + +const defaultLookup: SharedContentLookup = (path) => sharedContent[path]; + +export function resolveMdxIncludes( + text: string, + depth = 0, + source = '' +): string { + return resolveMdxIncludesWithLookup(text, defaultLookup, depth, source); +} + +export function eligibleForMarkdown( + input: EligibilityInput +): MarkdownEligibility { + return eligibleForMarkdownWithLookup(input, defaultLookup); +} + +const docPageRaws = import.meta.glob( + ['/src/pages/docs/**/*.md', '/src/pages/docs/**/*.mdx'], + { query: '?raw', import: 'default', eager: true } +); + +function parseDocFrontmatter(raw: string): Record { + try { + return (matter(raw).data ?? {}) as Record; + } catch { + return {}; + } +} + +function globKeyToSlug(globKey: string): string | null { + const m = globKey.match(/^\/src\/pages\/docs\/(.+\.(?:md|mdx))$/); + if (!m) return null; + return relPathToSlug(m[1]); +} + +// No memoization: dev edits to docs frontmatter must be reflected without +// a server restart. The walk is sub-second on the current corpus. +export function getEligibleSlugs(): Set { + const slugs = new Set(); + for (const path in docPageRaws) { + const raw = docPageRaws[path]; + const fm = parseDocFrontmatter(raw); + const verdict = eligibleForMarkdown({ path, frontmatter: fm, raw }); + if (!verdict.eligible) continue; + + const slug = globKeyToSlug(path); + if (!slug) continue; + slugs.add(slug); + } + + return slugs; +} diff --git a/src/themes/octopus/utilities/mdxContentCore.ts b/src/themes/octopus/utilities/mdxContentCore.ts new file mode 100644 index 0000000000..5b8e1e38be --- /dev/null +++ b/src/themes/octopus/utilities/mdxContentCore.ts @@ -0,0 +1,280 @@ +export type SharedContentLookup = (absolutePath: string) => string | undefined; + +const importLineRe = + /^[ \t]*import\s+(\w+)\s+from\s+['"]([^'"]+)['"]\s*;?[ \t]*$/; + +// Tolerates BOM, CR, and a missing trailing newline after the closing `---`. +export function stripFrontmatter(text: string): string { + const m = text.match(/^?---\r?\n[\s\S]*?\r?\n---[ \t]*(?:\r?\n([\s\S]*))?$/); + return m ? (m[1] ?? '') : text; +} + +function splitLeadingImports(text: string): { + head: string; + body: string; + importLines: string[]; +} { + const lines = text.split(/\r?\n/); + const importLines: string[] = []; + let i = 0; + while (i < lines.length) { + const line = lines[i]; + if (line.trim() === '') { + i++; + continue; + } + if (importLineRe.test(line)) { + importLines.push(line); + i++; + continue; + } + break; + } + const head = lines.slice(0, i).join('\n'); + const body = lines.slice(i).join('\n'); + return { head, body, importLines }; +} + +function classifyImports( + importLines: string[], + lookup: SharedContentLookup +): { + names: string[]; + resolved: Record; + unresolvedPaths: string[]; +} { + const names: string[] = []; + const resolved: Record = {}; + const unresolvedPaths: string[] = []; + + for (const line of importLines) { + const m = line.match(importLineRe); + if (!m) continue; + const name = m[1]; + const importPath = m[2]; + names.push(name); + + if (!/\.(md|mdx)$/.test(importPath)) { + unresolvedPaths.push(importPath); + continue; + } + + const lookupPath = importPath.startsWith('/') + ? importPath + : '/' + importPath; + const content = lookup(lookupPath); + if (typeof content !== 'string') { + unresolvedPaths.push(importPath); + continue; + } + + resolved[name] = stripFrontmatter(content).trim(); + } + + return { names, resolved, unresolvedPaths }; +} + +const MAX_RESOLVE_DEPTH = 5; + +// Must be `/g` for `matchAll`. +const fenceSplitRe = /(```[\s\S]*?```|`[^`\n]*`)/g; + +function splitFenceSegments( + text: string +): Array<{ text: string; isFence: boolean }> { + const segments: Array<{ text: string; isFence: boolean }> = []; + let lastIndex = 0; + for (const m of text.matchAll(fenceSplitRe)) { + const start = m.index ?? 0; + if (start > lastIndex) { + segments.push({ text: text.slice(lastIndex, start), isFence: false }); + } + segments.push({ text: m[0], isFence: true }); + lastIndex = start + m[0].length; + } + if (lastIndex < text.length) { + segments.push({ text: text.slice(lastIndex), isFence: false }); + } + return segments; +} + +export function resolveMdxIncludesWithLookup( + text: string, + lookup: SharedContentLookup, + depth = 0, + source = '' +): string { + if (depth > MAX_RESOLVE_DEPTH) { + console.warn( + '[mdxContent] resolveMdxIncludes: max depth ' + + MAX_RESOLVE_DEPTH + + ' exceeded for ' + + source + + '; leaving residual tags in place' + ); + return text; + } + + const { body, importLines } = splitLeadingImports(text); + const { resolved } = classifyImports(importLines, lookup); + + // Substitute outside fenced regions only, so a docs page describing the + // include syntax in a code sample keeps its example verbatim. + const segments = splitFenceSegments(body); + let changed = false; + for (let i = 0; i < segments.length; i++) { + const seg = segments[i]; + if (seg.isFence) continue; + let segText = seg.text; + for (const [name, content] of Object.entries(resolved)) { + const selfClosing = new RegExp('<' + name + '\\s*/>', 'g'); + const openClose = new RegExp( + '<' + name + '\\s*>[\\s\\S]*?', + 'g' + ); + const before = segText; + segText = segText.replace(selfClosing, content); + segText = segText.replace(openClose, content); + if (segText !== before) changed = true; + } + segments[i] = { text: segText, isFence: false }; + } + const result = segments.map((s) => s.text).join(''); + + if (changed) { + return resolveMdxIncludesWithLookup(result, lookup, depth + 1, source); + } + + return result.replace(/^\s*\n+/, '').replace(/\n{3,}/g, '\n\n'); +} + +// Must be `/g` for `matchAll`. +const componentTagRe = /<([A-Z][A-Za-z0-9_]*)(?:\s|\/|>)/g; + +function stripFences(text: string): string { + return splitFenceSegments(text) + .filter((s) => !s.isFence) + .map((s) => s.text) + .join(''); +} + +export type MarkdownEligibility = + | { eligible: true } + | { eligible: false; reason: string }; + +export type EligibilityFrontmatter = Record | null | undefined; + +export type EligibilityInput = { + path: string; + frontmatter: EligibilityFrontmatter; + raw: string; +}; + +function isAuthorLayout(layout: unknown): boolean { + return typeof layout === 'string' && layout.includes('/Author.astro'); +} + +function isSearchLayout(layout: unknown): boolean { + return typeof layout === 'string' && layout.includes('/Search.astro'); +} + +function isRedirectLayout(layout: unknown): boolean { + return typeof layout === 'string' && layout.includes('/Redirect.astro'); +} + +function isFuturePubDate(pubDate: unknown): boolean { + if (typeof pubDate !== 'string' || pubDate.trim().length === 0) { + return false; + } + const ts = Date.parse(pubDate); + if (Number.isNaN(ts)) return false; + return ts > Date.now(); +} + +export function eligibleForMarkdownWithLookup( + input: EligibilityInput, + lookup: SharedContentLookup +): MarkdownEligibility { + const { path, frontmatter, raw } = input; + const fm = (frontmatter ?? {}) as Record; + + if (typeof fm.layout !== 'string' || fm.layout.trim().length === 0) { + return { eligible: false, reason: 'no-layout' }; + } + + if (fm.draft) return { eligible: false, reason: 'draft' }; + if (fm.redirect) return { eligible: false, reason: 'redirect' }; + if (isRedirectLayout(fm.layout)) { + return { eligible: false, reason: 'redirect-layout' }; + } + if (isAuthorLayout(fm.layout)) { + return { eligible: false, reason: 'author-page' }; + } + if (isSearchLayout(fm.layout)) { + return { eligible: false, reason: 'search-page' }; + } + if (fm.navSitemap === false) { + return { eligible: false, reason: 'nav-sitemap-false' }; + } + if (isFuturePubDate(fm.pubDate)) { + return { eligible: false, reason: 'future-pubdate' }; + } + + const isMdx = /\.mdx$/i.test(path); + if (!isMdx) return { eligible: true }; + + const stripped = stripFrontmatter(raw); + const { body, importLines } = splitLeadingImports(stripped); + const { names, unresolvedPaths } = classifyImports(importLines, lookup); + + if (unresolvedPaths.length > 0) { + return { eligible: false, reason: 'mdx-unresolvable-imports' }; + } + + const importedNames = new Set(names); + const scanBody = stripFences(body); + for (const match of scanBody.matchAll(componentTagRe)) { + const name = match[1]; + if (!importedNames.has(name)) { + return { eligible: false, reason: 'mdx-unknown-component' }; + } + } + + return { eligible: true }; +} + +export function sanitizeTitle(raw: unknown): string { + const s = (raw ?? 'Untitled').toString(); + return s.replace(/[\[\]*`]/g, '').trim() || 'Untitled'; +} + +export function normalizeSubtitle(raw: unknown): string | null { + return typeof raw === 'string' && raw.trim().length > 0 ? raw : null; +} + +export type NavSortable = { + slug: string; + navOrder: number; + navSection: string; +}; + +// Returns null for the docs root index. +export function pathToSlug(relPath: string): string | null { + const noExt = relPath.replace(/\.(md|mdx)$/i, ''); + if (noExt === relPath) return null; + if (noExt === 'index') return null; + return noExt.replace(/\/index$/, ''); +} + +export function compareForLlmSurfaces(a: NavSortable, b: NavSortable): number { + const aEmpty = a.navSection.trim().length === 0; + const bEmpty = b.navSection.trim().length === 0; + if (aEmpty !== bEmpty) return aEmpty ? 1 : -1; + if (a.navSection !== b.navSection) { + return a.navSection.localeCompare(b.navSection, 'en'); + } + if (a.navOrder !== b.navOrder) { + return a.navOrder - b.navOrder; + } + return a.slug.localeCompare(b.slug, 'en'); +} diff --git a/tests/llm-endpoints.spec.ts b/tests/llm-endpoints.spec.ts new file mode 100644 index 0000000000..47329196e8 --- /dev/null +++ b/tests/llm-endpoints.spec.ts @@ -0,0 +1,292 @@ +import { test, expect } from '@playwright/test'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +// These tests run against `astro preview` (built `dist/`), not `astro dev`, +// because the per-page `.md` files are written by the build-time integration. + +const STABLE_PLAIN_MD_PATH = '/docs/argo-cd'; +const STABLE_MDX_PATH = '/docs/kubernetes'; + +test('per-page .md endpoint serves clean markdown for an eligible page', async ({ + request, +}) => { + const res = await request.get(STABLE_PLAIN_MD_PATH + '.md'); + expect(res.status()).toBe(200); + const ct = res.headers()['content-type'] ?? ''; + expect(ct.toLowerCase()).toMatch(/^text\/markdown(?:;|$)/); + const body = await res.text(); + expect(body.length).toBeGreaterThan(0); + expect(body.replace(/^/, '').startsWith('# ')).toBe(true); +}); + +test('per-page .md endpoint 404s for an MDX page with non-inlinable components', async ({ + request, +}) => { + const res = await request.get(STABLE_MDX_PATH + '.md'); + expect(res.status()).toBe(404); +}); + +test('llms.txt is spec-shaped', async ({ request }) => { + const res = await request.get('/docs/llms.txt'); + expect(res.status()).toBe(200); + const body = await res.text(); + + const h1Lines = body.split('\n').filter((l) => /^?# /.test(l)); + expect(h1Lines.length, 'expected exactly one H1').toBe(1); + + const summaryLines = body.split('\n').filter((l) => /^> /.test(l)); + expect( + summaryLines.length, + 'expected at least one `>` summary line' + ).toBeGreaterThanOrEqual(1); + + const sectionLines = body.split('\n').filter((l) => /^## /.test(l)); + expect( + sectionLines.length, + 'expected at least one `##` section' + ).toBeGreaterThanOrEqual(1); + + const linkRe = /\[[^\]]+\]\((https?:\/\/[^)]+)\)/g; + const urls = Array.from(body.matchAll(linkRe), (m) => m[1]); + expect( + urls.length, + 'expected at least one link in llms.txt' + ).toBeGreaterThan(0); + for (const u of urls) { + expect(u, `URL ${u} should end in .md`).toMatch(/\.md(?:[#?].*)?$/); + } +}); + +test('llms-full.txt is well-formed: intro, summary, page boundaries, no orphaned MDX', async ({ + request, +}) => { + const res = await request.get('/docs/llms-full.txt'); + expect(res.status()).toBe(200); + const body = await res.text(); + + expect(body).toMatch(/^?# Octopus Deploy Documentation/); + expect(body).toMatch(/^> /m); + expect(body).toMatch(/^Source: https?:\/\/[^\s]+\.md$/m); + + expect(body, 'unexpected literal `]/ + ); + expect( + body, + 'unexpected literal `]/); + expect( + body, + 'unexpected literal `]/); +}); + +test('CopyMarkdown dropdown advertises a working .md URL on the eligible page', async ({ + page, + request, +}) => { + await page.goto(STABLE_PLAIN_MD_PATH); + const url = await page.getAttribute( + '[data-copy-md-action="copy"]', + 'data-copy-md-url' + ); + expect( + url, + 'expected CopyMarkdown dropdown to expose a data-copy-md-url' + ).toBeTruthy(); + const target = await request.get(url!); + expect(target.status()).toBe(200); +}); + +test('CopyMarkdown dropdown is hidden on the ineligible MDX page', async ({ + page, +}) => { + await page.goto(STABLE_MDX_PATH); + const count = await page.locator('[data-copy-md-menu]').count(); + expect(count, 'expected no CopyMarkdown dropdown on ineligible page').toBe(0); +}); + +test('HtmlHead omits `` on the ineligible MDX page', async ({ + page, +}) => { + await page.goto(STABLE_MDX_PATH); + const count = await page + .locator('link[rel="alternate"][type="text/markdown"]') + .count(); + expect(count, 'expected no .md alternate link on ineligible page').toBe(0); +}); + +test('HtmlHead emits `` on the eligible page', async ({ + page, +}) => { + await page.goto(STABLE_PLAIN_MD_PATH); + const count = await page + .locator('link[rel="alternate"][type="text/markdown"]') + .count(); + expect( + count, + 'expected exactly one .md alternate link on eligible page' + ).toBe(1); +}); + +function resolveDistDocsDir(): string { + const here = path.dirname(fileURLToPath(import.meta.url)); + const projectRoot = path.resolve(here, '..'); + return path.join(projectRoot, 'dist', 'docs'); +} + +function collectEmittedSlugs(distDocs: string): Set { + const slugs = new Set(); + const stack: string[] = [distDocs]; + while (stack.length > 0) { + const dir = stack.pop()!; + let entries: fs.Dirent[]; + try { + entries = fs.readdirSync(dir, { withFileTypes: true }); + } catch { + continue; + } + for (const entry of entries) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + stack.push(full); + continue; + } + if (!entry.isFile()) continue; + if (!entry.name.endsWith('.md')) continue; + const rel = path.relative(distDocs, full).replace(/\\/g, '/'); + const slug = rel.replace(/\.md$/, ''); + slugs.add(slug); + } + } + return slugs; +} + +function extractSlugsFromIndex(indexBody: string): Set { + const linkRe = /\[[^\]]+\]\((https?:\/\/[^)]+\.md)(?:[#?][^)]*)?\)/g; + const slugs = new Set(); + for (const m of indexBody.matchAll(linkRe)) { + const url = m[1]; + const docMatch = url.match(/\/docs\/(.+)\.md$/); + if (!docMatch) continue; + slugs.add(docMatch[1]); + } + return slugs; +} + +function extractSlugsFromFull(fullBody: string): Set { + const sourceRe = /^Source:\s*(https?:\/\/[^\s]+\.md)\s*$/gm; + const slugs = new Set(); + for (const m of fullBody.matchAll(sourceRe)) { + const url = m[1]; + const docMatch = url.match(/\/docs\/(.+)\.md$/); + if (!docMatch) continue; + slugs.add(docMatch[1]); + } + return slugs; +} + +function diffSets(a: Set, b: Set): string[] { + const out: string[] = []; + for (const v of a) if (!b.has(v)) out.push(v); + return out.sort(); +} + +test('predicate-set parity: llms.txt and llms-full.txt advertise the same slugs (bidirectional)', async ({ + request, +}) => { + const indexRes = await request.get('/docs/llms.txt'); + expect(indexRes.status()).toBe(200); + const indexBody = await indexRes.text(); + + const fullRes = await request.get('/docs/llms-full.txt'); + expect(fullRes.status()).toBe(200); + const fullBody = await fullRes.text(); + + const indexSlugs = extractSlugsFromIndex(indexBody); + const fullSlugs = extractSlugsFromFull(fullBody); + + expect( + indexSlugs.size, + 'expected llms.txt to advertise at least a few /docs/*.md links' + ).toBeGreaterThan(10); + expect( + fullSlugs.size, + 'expected llms-full.txt to expose at least a few `Source:` rows' + ).toBeGreaterThan(10); + + expect( + diffSets(indexSlugs, fullSlugs), + 'slugs in llms.txt missing from llms-full.txt `Source:` rows' + ).toEqual([]); + + expect( + diffSets(fullSlugs, indexSlugs), + 'slugs in llms-full.txt `Source:` rows missing from llms.txt links' + ).toEqual([]); +}); + +test('predicate-set parity: emitted dist/docs/*.md files match llms.txt slug set', async ({ + request, +}) => { + const indexRes = await request.get('/docs/llms.txt'); + expect(indexRes.status()).toBe(200); + const indexBody = await indexRes.text(); + + const indexSlugs = extractSlugsFromIndex(indexBody); + expect( + indexSlugs.size, + 'expected llms.txt to advertise at least a few /docs/*.md links' + ).toBeGreaterThan(10); + + const distDocs = resolveDistDocsDir(); + expect( + fs.existsSync(distDocs), + `expected ${distDocs} to exist - run \`pnpm run build\` first` + ).toBe(true); + const emittedSlugs = collectEmittedSlugs(distDocs); + + expect( + diffSets(indexSlugs, emittedSlugs), + 'slugs advertised in llms.txt but not emitted to dist/docs/*.md' + ).toEqual([]); + expect( + diffSets(emittedSlugs, indexSlugs), + 'files emitted to dist/docs/*.md but not advertised in llms.txt' + ).toEqual([]); +}); + +test('per-page .md companions advertised in llms.txt all resolve 200 (sampled)', async ({ + request, +}) => { + const indexRes = await request.get('/docs/llms.txt'); + expect(indexRes.status()).toBe(200); + const indexBody = await indexRes.text(); + + const linkRe = /\[[^\]]+\]\((https?:\/\/[^)]+\.md)(?:[#?][^)]*)?\)/g; + const allUrls = Array.from(indexBody.matchAll(linkRe), (m) => m[1]); + const docMdUrls = allUrls.filter((u) => u.includes('/docs/')); + expect(docMdUrls.length).toBeGreaterThan(10); + + // Sample to keep CI runtime bounded; full slug-set coverage is in the + // FS-walk test above. + const stride = Math.max(1, Math.floor(docMdUrls.length / 25)); + const sampledIndices = new Set(); + sampledIndices.add(0); + sampledIndices.add(docMdUrls.length - 1); + for (let i = 0; i < docMdUrls.length; i += stride) sampledIndices.add(i); + + const fetchFailures: string[] = []; + for (const i of sampledIndices) { + const fullUrl = docMdUrls[i]; + const pathOnly = fullUrl.replace(/^https?:\/\/[^/]+/, ''); + const r = await request.get(pathOnly); + if (r.status() !== 200) fetchFailures.push(`${pathOnly} -> ${r.status()}`); + } + expect( + fetchFailures, + 'every sampled .md companion advertised in llms.txt should resolve 200' + ).toEqual([]); +});