diff --git a/app/docs/[section]/[[...slug]]/page.tsx b/app/docs/[section]/[[...slug]]/page.tsx index 6170f50..0b3e04f 100644 --- a/app/docs/[section]/[[...slug]]/page.tsx +++ b/app/docs/[section]/[[...slug]]/page.tsx @@ -83,6 +83,12 @@ export default async function ArticlePage({ params }: PageProps) { // Use title from frontmatter if available, otherwise fall back to navigation title or section name const pageTitle = frontmatter.title || selectedNavItem?.title || section; const showInstallPrimer = section === "getting-started" && file === "index"; + const isComingSoon = frontmatter.layout === 'coming-soon'; + // coming-soon pages carry placeholder prose with no markdown heading, so + // the h1 has to come from the frontmatter title - otherwise the page ships + // with no heading element at all. Guarded in case a future coming-soon page + // does start with one. + const showComingSoonHeading = isComingSoon && !/^#\s/m.test(content); const sectionTitle = capitalCase(section) .replace(/documentdb/i, 'DocumentDB') .replace(/api/i, 'API'); @@ -184,7 +190,12 @@ export default async function ArticlePage({ params }: PageProps) { {/* Coming Soon Component for coming-soon layout */} - {frontmatter.layout === 'coming-soon' && } + {showComingSoonHeading && ( +

+ {pageTitle} +

+ )} + {isComingSoon && } {showInstallPrimer && (
diff --git a/app/not-found.tsx b/app/not-found.tsx index cfe6ef2..3c1f427 100644 --- a/app/not-found.tsx +++ b/app/not-found.tsx @@ -1,5 +1,19 @@ +import type { Metadata } from "next"; import Link from "next/link"; +// Without this the route inherits the root layout's metadata, which hardcodes +// robots index/follow - so the page emitted both that tag and the noindex +// Next.js injects for not-found, and carried the homepage title verbatim. +export const metadata: Metadata = { + title: "Page not found - DocumentDB", + description: + "The page you're looking for doesn't exist. Find documentation, downloads, and samples for DocumentDB.", + robots: { + index: false, + follow: true, + }, +}; + const suggestions = [ { title: "Documentation", diff --git a/articles/content.yml b/articles/content.yml index b78dbeb..5765777 100644 --- a/articles/content.yml +++ b/articles/content.yml @@ -12,7 +12,7 @@ landing: link: /docs/postgres-api - title: DocumentDB Local link: /docs/documentdb-local - - title: Architecture under the hood + - title: Architecture under the hood (Coming soon) link: /docs/architecture - title: Samples & Demos link: /samples diff --git a/scripts/generate-sitemap.mjs b/scripts/generate-sitemap.mjs index 50aa062..b4e4f3e 100644 --- a/scripts/generate-sitemap.mjs +++ b/scripts/generate-sitemap.mjs @@ -11,21 +11,41 @@ import path from 'node:path'; const siteUrl = 'https://documentdb.io'; const outDir = path.join(process.cwd(), 'out'); -// Top-level build outputs that are not HTML pages: Next.js assets, the APT/RPM -// package repositories, images, and the not-found page (with trailingSlash the -// export emits out/404/index.html alongside out/404.html). The packages -// workflow adds deb/ and rpm/ after this script runs in the deploy job, but -// they are excluded here too so local full builds behave identically. Note -// that out/packages/ is NOT excluded: it is the exported /packages download -// page; the workflow only adds release-info.json (not a page) next to it. +// Top-level build outputs that are not pages at all: Next.js assets, the +// APT/RPM package repositories, and images. The packages workflow adds deb/ +// and rpm/ after this script runs in the deploy job, but they are excluded +// here too so local full builds behave identically. Note that out/packages/ is +// NOT excluded: it is the exported /packages download page; the workflow only +// adds release-info.json (not a page) next to it. +// +// Pages that exist but must not be advertised are handled by isNoindex() +// rather than by name. That covers both forms the not-found route takes - with +// trailingSlash the export writes out/404/index.html alongside out/404.html, +// and the App Router adds out/_not-found/index.html - and anything else the +// framework starts emitting later. Listing a noindex URL is what earns the +// "submitted URL marked noindex" warning in Search Console, and a page already +// states that about itself, so there is no second list to keep in sync. const excludedTopLevelDirectories = new Set([ '_next', 'deb', 'rpm', 'images', - '404', ]); +let noindexPagesSkipped = 0; + +/** + * True when the page asks crawlers not to index it. Reads the meta tag in + * either attribute order and tolerates content lists such as "noindex,nofollow". + */ +function isNoindex(html) { + return (html.match(/]*>/gi) ?? []).some( + (tag) => + /\bname=["']?robots["']?/i.test(tag) && + /\bcontent=["'][^"']*\bnoindex\b/i.test(tag), + ); +} + function xmlEscape(value) { return value .replace(/&/g, '&') @@ -45,10 +65,14 @@ function collectPages(directory, relativePath = '') { const indexFile = path.join(directory, 'index.html'); if (fs.existsSync(indexFile)) { - pages.push({ - url: relativePath === '' ? '/' : `/${relativePath}/`, - lastModified: fs.statSync(indexFile).mtime, - }); + if (isNoindex(fs.readFileSync(indexFile, 'utf8'))) { + noindexPagesSkipped += 1; + } else { + pages.push({ + url: relativePath === '' ? '/' : `/${relativePath}/`, + lastModified: fs.statSync(indexFile).mtime, + }); + } } for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { @@ -113,4 +137,8 @@ if (fs.existsSync(robotsPath)) { fs.writeFileSync(robotsPath, `User-agent: *\nAllow: /\n\n${sitemapLine}\n`); } -console.log(`Wrote out/sitemap.xml with ${pages.length} URLs and advertised it in out/robots.txt.`); +console.log( + `Wrote out/sitemap.xml with ${pages.length} URLs (skipped ${noindexPagesSkipped} noindex ${ + noindexPagesSkipped === 1 ? 'page' : 'pages' + }) and advertised it in out/robots.txt.`, +);