DOCS-2996: Add all-versions sitemap and full-coverage link check

.github/workflows/link-check-weekly.yml

@@ -44,6 +44,10 @@ jobs:
         run: make test 2>&1 | tee link-check.log
         env:
           CI: 'true'
+          # Crawl every built version (via sitemap-full.xml), not just latest. This job is
+          # non-blocking and files an issue on failure, so it is the safe place to surface the
+          # full-version backlog. PR CI stays latest-only until the backlog is cleared.
+          FULL_COVERAGE: 'true'
 
       - name: Upload the link-check log
         if: always()

__tests__/crawler.test.js

@@ -22,7 +22,10 @@ test('Crawl the docs and execute tests', async () => {
   const varSkipList = ['{{end}}'];
   const liquidRegex = /\{%.*?%}/gs;
   const liquidSkipList = [];
-  const SITEMAP = 'sitemap.xml';
+  // FULL_COVERAGE seeds the crawl from the all-versions sitemap (sitemap-full.xml) instead of the
+  // latest-only sitemap.xml. Used by the weekly job; leave off for PR CI until the backlog is fixed.
+  const fullCoverage = process.env.FULL_COVERAGE === 'true';
+  const SITEMAP = fullCoverage ? 'sitemap-full.xml' : 'sitemap.xml';
   const SITEMAP_URL = `${DOCS}/${SITEMAP}`;
   const USE_LC = [
     { regex: inspectFilesRegExp(), processContent: true },
@@ -612,6 +615,7 @@ test('Crawl the docs and execute tests', async () => {
   await crawler.addRequests(urls);
 
   console.log(`Crawling the docs (${DOCS}) and executing tests.`);
+  console.log(`Coverage: ${fullCoverage ? 'FULL (all versions)' : 'latest only'} via ${SITEMAP}.`);
   console.log(`Localhost mode is ${isLocalHost ? 'ON' : 'OFF'}.`);
   console.log(`Validity tests on code blocks: ${validityTest.length ? validityTest.join(',') : 'none'}`);
   console.log(`Validity tests on files: ${validityTestFiles.length ? validityTestFiles.join(',') : 'none'}`);

docusaurus.config.js

@@ -411,6 +411,19 @@ export default async function createAsyncConfig() {
         },
       },
     plugins: [
+      // Second sitemap instance for the internal link checker only. The preset sitemap above
+      // stays latest-only for SEO (sitemap.xml, the one advertised in robots.txt). This one has
+      // no version ignorePatterns, so it lists every built version at sitemap-full.xml. The
+      // crawler seeds from it when FULL_COVERAGE=true; it is not referenced in robots.txt.
+      [
+        '@docusaurus/plugin-sitemap',
+        {
+          id: 'full',
+          filename: 'sitemap-full.xml',
+          lastmod: 'date',
+          ignorePatterns: [],
+        },
+      ],
       'docusaurus-plugin-sass',
       [
         '@docusaurus/plugin-content-docs',

package.json

@@ -40,6 +40,7 @@
     "@docusaurus/core": "^3.10.1",
     "@docusaurus/faster": "^3.10.1",
     "@docusaurus/plugin-content-docs": "^3.10.1",
+    "@docusaurus/plugin-sitemap": "3.10.1",
     "@docusaurus/preset-classic": "^3.10.1",
     "@docusaurus/theme-common": "^3.10.1",
     "@docusaurus/theme-search-algolia": "^3.10.1",

yarn.lock

@@ -20109,6 +20109,7 @@ __metadata:
     "@docusaurus/faster": "npm:^3.10.1"
     "@docusaurus/module-type-aliases": "npm:^3.10.1"
     "@docusaurus/plugin-content-docs": "npm:^3.10.1"
+    "@docusaurus/plugin-sitemap": "npm:3.10.1"
     "@docusaurus/preset-classic": "npm:^3.10.1"
     "@docusaurus/theme-common": "npm:^3.10.1"
     "@docusaurus/theme-search-algolia": "npm:^3.10.1"