From 3913fae9f30e481f58665caa486509af15c851d5 Mon Sep 17 00:00:00 2001 From: Guanzhou Song Date: Mon, 3 Aug 2026 07:45:37 -0400 Subject: [PATCH 1/2] SEO cleanup: sitemap, 404 metadata, coming-soon heading Three small fixes found while validating the live site. Sitemap advertised /_not-found/. collectPages() emits a URL for any directory holding an index.html and filters only top-level names in an explicit deny-list. With trailingSlash the export writes out/_not-found/index.html, and _not-found was not in the list. #123 fixed exactly this class for 404 but only added that one name. The page also serves noindex, so the sitemap was submitting a URL that tells crawlers not to index it. Added _not-found to the exclusion set. 404 page emitted two conflicting robots tags. not-found.tsx exported no metadata, so it inherited the root layout's, which hardcodes index/follow in metadataService - and Next.js separately injects noindex for the not-found route. It also carried the homepage title byte for byte. Added a metadata export with a distinct title and index: false. /docs/architecture had no heading element at all - not just no h1, zero h1-h6. The coming-soon layout renders the graphic plus placeholder prose, and the markdown carries no heading, while the sidebar section label is deliberately a

because "the article's h1 comes from the markdown content". That contract is unmet for coming-soon pages, so the h1 now comes from the frontmatter title, guarded in case a future coming-soon page does start with a markdown heading. The docs index also showed the Architecture card with no marker, styled identically to the six finished ones. Marked it the same way the Kubernetes Operator card marks preview status, via a title suffix. Fixes #129 --- app/docs/[section]/[[...slug]]/page.tsx | 13 ++++++++++++- app/not-found.tsx | 14 ++++++++++++++ articles/content.yml | 2 +- scripts/generate-sitemap.mjs | 16 ++++++++++------ 4 files changed, 37 insertions(+), 8 deletions(-) diff --git a/app/docs/[section]/[[...slug]]/page.tsx b/app/docs/[section]/[[...slug]]/page.tsx index 6170f50..0b3e04f 100644 --- a/app/docs/[section]/[[...slug]]/page.tsx +++ b/app/docs/[section]/[[...slug]]/page.tsx @@ -83,6 +83,12 @@ export default async function ArticlePage({ params }: PageProps) { // Use title from frontmatter if available, otherwise fall back to navigation title or section name const pageTitle = frontmatter.title || selectedNavItem?.title || section; const showInstallPrimer = section === "getting-started" && file === "index"; + const isComingSoon = frontmatter.layout === 'coming-soon'; + // coming-soon pages carry placeholder prose with no markdown heading, so + // the h1 has to come from the frontmatter title - otherwise the page ships + // with no heading element at all. Guarded in case a future coming-soon page + // does start with one. + const showComingSoonHeading = isComingSoon && !/^#\s/m.test(content); const sectionTitle = capitalCase(section) .replace(/documentdb/i, 'DocumentDB') .replace(/api/i, 'API'); @@ -184,7 +190,12 @@ export default async function ArticlePage({ params }: PageProps) { {/* Coming Soon Component for coming-soon layout */} - {frontmatter.layout === 'coming-soon' && } + {showComingSoonHeading && ( +

+ {pageTitle} +

+ )} + {isComingSoon && } {showInstallPrimer && (
diff --git a/app/not-found.tsx b/app/not-found.tsx index cfe6ef2..3c1f427 100644 --- a/app/not-found.tsx +++ b/app/not-found.tsx @@ -1,5 +1,19 @@ +import type { Metadata } from "next"; import Link from "next/link"; +// Without this the route inherits the root layout's metadata, which hardcodes +// robots index/follow - so the page emitted both that tag and the noindex +// Next.js injects for not-found, and carried the homepage title verbatim. +export const metadata: Metadata = { + title: "Page not found - DocumentDB", + description: + "The page you're looking for doesn't exist. Find documentation, downloads, and samples for DocumentDB.", + robots: { + index: false, + follow: true, + }, +}; + const suggestions = [ { title: "Documentation", diff --git a/articles/content.yml b/articles/content.yml index b78dbeb..5765777 100644 --- a/articles/content.yml +++ b/articles/content.yml @@ -12,7 +12,7 @@ landing: link: /docs/postgres-api - title: DocumentDB Local link: /docs/documentdb-local - - title: Architecture under the hood + - title: Architecture under the hood (Coming soon) link: /docs/architecture - title: Samples & Demos link: /samples diff --git a/scripts/generate-sitemap.mjs b/scripts/generate-sitemap.mjs index 50aa062..4bf4263 100644 --- a/scripts/generate-sitemap.mjs +++ b/scripts/generate-sitemap.mjs @@ -12,18 +12,22 @@ const siteUrl = 'https://documentdb.io'; const outDir = path.join(process.cwd(), 'out'); // Top-level build outputs that are not HTML pages: Next.js assets, the APT/RPM -// package repositories, images, and the not-found page (with trailingSlash the -// export emits out/404/index.html alongside out/404.html). The packages -// workflow adds deb/ and rpm/ after this script runs in the deploy job, but -// they are excluded here too so local full builds behave identically. Note -// that out/packages/ is NOT excluded: it is the exported /packages download -// page; the workflow only adds release-info.json (not a page) next to it. +// package repositories, images, and the two forms the not-found page takes +// (with trailingSlash the export emits out/404/index.html alongside +// out/404.html, plus out/_not-found/index.html for the App Router's not-found +// route - both serve noindex, so listing either one submits a URL that tells +// crawlers not to index it). The packages workflow adds deb/ and rpm/ after +// this script runs in the deploy job, but they are excluded here too so local +// full builds behave identically. Note that out/packages/ is NOT excluded: it +// is the exported /packages download page; the workflow only adds +// release-info.json (not a page) next to it. const excludedTopLevelDirectories = new Set([ '_next', 'deb', 'rpm', 'images', '404', + '_not-found', ]); function xmlEscape(value) { From 718dfd1418bbfeb66d8a7c3282af7e378d9502cd Mon Sep 17 00:00:00 2001 From: Guanzhou Song Date: Mon, 3 Aug 2026 15:40:56 -0400 Subject: [PATCH 2/2] Filter the sitemap on noindex rather than on directory name The exclusion list was growing one framework route at a time. #123 added 404 after it appeared in the sitemap; this branch added _not-found for the same reason; and the versioned-docs work in progress adds a third hand-written skip for the archived version directories. Each is the same rule discovered again: do not advertise a page that tells crawlers not to index it. That property is readable from the page itself, so collectPages now reads each index.html and skips the ones carrying a noindex robots meta. The name list keeps only the directories that hold no pages at all - _next, deb, rpm, images - where it is a traversal concern rather than an indexing decision. The tag is matched in either attribute order and tolerates content lists such as "noindex,nofollow", since Next.js and hand-written metadata do not agree on either. The final log line now reports how many pages were skipped, so a filter that starts matching too much is visible in the build output instead of silently shrinking the sitemap. Verified against a synthetic export carrying an indexable root, /docs, /docs/versions, /packages and /samples, plus three noindex pages written in three different tag forms and an excluded _next directory: five URLs emitted, three skipped, and out/404.html left alone as the file it is. Note that the archived-version case is now covered by this rule, so the docs/versions skip on the versioned-docs branch can go when it lands. --- scripts/generate-sitemap.mjs | 58 +++++++++++++++++++++++++----------- 1 file changed, 41 insertions(+), 17 deletions(-) diff --git a/scripts/generate-sitemap.mjs b/scripts/generate-sitemap.mjs index 4bf4263..b4e4f3e 100644 --- a/scripts/generate-sitemap.mjs +++ b/scripts/generate-sitemap.mjs @@ -11,25 +11,41 @@ import path from 'node:path'; const siteUrl = 'https://documentdb.io'; const outDir = path.join(process.cwd(), 'out'); -// Top-level build outputs that are not HTML pages: Next.js assets, the APT/RPM -// package repositories, images, and the two forms the not-found page takes -// (with trailingSlash the export emits out/404/index.html alongside -// out/404.html, plus out/_not-found/index.html for the App Router's not-found -// route - both serve noindex, so listing either one submits a URL that tells -// crawlers not to index it). The packages workflow adds deb/ and rpm/ after -// this script runs in the deploy job, but they are excluded here too so local -// full builds behave identically. Note that out/packages/ is NOT excluded: it -// is the exported /packages download page; the workflow only adds -// release-info.json (not a page) next to it. +// Top-level build outputs that are not pages at all: Next.js assets, the +// APT/RPM package repositories, and images. The packages workflow adds deb/ +// and rpm/ after this script runs in the deploy job, but they are excluded +// here too so local full builds behave identically. Note that out/packages/ is +// NOT excluded: it is the exported /packages download page; the workflow only +// adds release-info.json (not a page) next to it. +// +// Pages that exist but must not be advertised are handled by isNoindex() +// rather than by name. That covers both forms the not-found route takes - with +// trailingSlash the export writes out/404/index.html alongside out/404.html, +// and the App Router adds out/_not-found/index.html - and anything else the +// framework starts emitting later. Listing a noindex URL is what earns the +// "submitted URL marked noindex" warning in Search Console, and a page already +// states that about itself, so there is no second list to keep in sync. const excludedTopLevelDirectories = new Set([ '_next', 'deb', 'rpm', 'images', - '404', - '_not-found', ]); +let noindexPagesSkipped = 0; + +/** + * True when the page asks crawlers not to index it. Reads the meta tag in + * either attribute order and tolerates content lists such as "noindex,nofollow". + */ +function isNoindex(html) { + return (html.match(/]*>/gi) ?? []).some( + (tag) => + /\bname=["']?robots["']?/i.test(tag) && + /\bcontent=["'][^"']*\bnoindex\b/i.test(tag), + ); +} + function xmlEscape(value) { return value .replace(/&/g, '&') @@ -49,10 +65,14 @@ function collectPages(directory, relativePath = '') { const indexFile = path.join(directory, 'index.html'); if (fs.existsSync(indexFile)) { - pages.push({ - url: relativePath === '' ? '/' : `/${relativePath}/`, - lastModified: fs.statSync(indexFile).mtime, - }); + if (isNoindex(fs.readFileSync(indexFile, 'utf8'))) { + noindexPagesSkipped += 1; + } else { + pages.push({ + url: relativePath === '' ? '/' : `/${relativePath}/`, + lastModified: fs.statSync(indexFile).mtime, + }); + } } for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { @@ -117,4 +137,8 @@ if (fs.existsSync(robotsPath)) { fs.writeFileSync(robotsPath, `User-agent: *\nAllow: /\n\n${sitemapLine}\n`); } -console.log(`Wrote out/sitemap.xml with ${pages.length} URLs and advertised it in out/robots.txt.`); +console.log( + `Wrote out/sitemap.xml with ${pages.length} URLs (skipped ${noindexPagesSkipped} noindex ${ + noindexPagesSkipped === 1 ? 'page' : 'pages' + }) and advertised it in out/robots.txt.`, +);