fix(sitemap): exclude pages with robots noindex from sitemap (#7143)

facebook · Apr 14, 2022 · 03516dc · 03516dc
1 parent 6306cbc
commit 03516dc
Show file tree

Hide file tree

Showing 13 changed files with 105 additions and 41 deletions.
diff --git a/packages/docusaurus-plugin-debug/src/index.ts b/packages/docusaurus-plugin-debug/src/index.ts
@@ -9,8 +9,6 @@ import type {LoadContext, Plugin} from '@docusaurus/types';
 import {docuHash, normalizeUrl, posixPath} from '@docusaurus/utils';
 import path from 'path';
 
-export const routeBasePath = '__docusaurus/debug';
-
 export default function pluginDebug({
   siteConfig: {baseUrl},
   generatedFilesDir,
@@ -42,37 +40,37 @@ export default function pluginDebug({
 
       // Home is config (duplicate for now)
       addRoute({
-        path: normalizeUrl([baseUrl, routeBasePath]),
+        path: normalizeUrl([baseUrl, '__docusaurus/debug']),
         component: '@theme/DebugConfig',
         exact: true,
       });
 
       addRoute({
-        path: normalizeUrl([baseUrl, routeBasePath, 'config']),
+        path: normalizeUrl([baseUrl, '__docusaurus/debug/config']),
         component: '@theme/DebugConfig',
         exact: true,
       });
 
       addRoute({
-        path: normalizeUrl([baseUrl, routeBasePath, 'metadata']),
+        path: normalizeUrl([baseUrl, '__docusaurus/debug/metadata']),
         component: '@theme/DebugSiteMetadata',
         exact: true,
       });
 
       addRoute({
-        path: normalizeUrl([baseUrl, routeBasePath, 'registry']),
+        path: normalizeUrl([baseUrl, '__docusaurus/debug/registry']),
         component: '@theme/DebugRegistry',
         exact: true,
       });
 
       addRoute({
-        path: normalizeUrl([baseUrl, routeBasePath, 'routes']),
+        path: normalizeUrl([baseUrl, '__docusaurus/debug/routes']),
         component: '@theme/DebugRoutes',
         exact: true,
       });
 
       addRoute({
-        path: normalizeUrl([baseUrl, routeBasePath, 'content']),
+        path: normalizeUrl([baseUrl, '__docusaurus/debug/content']),
         component: '@theme/DebugContent',
         exact: true,
         modules: {
@@ -81,7 +79,7 @@ export default function pluginDebug({
       });
 
       addRoute({
-        path: normalizeUrl([baseUrl, routeBasePath, 'globalData']),
+        path: normalizeUrl([baseUrl, '__docusaurus/debug/globalData']),
         component: '@theme/DebugGlobalData',
         exact: true,
       });

diff --git a/packages/docusaurus-plugin-debug/src/plugin-debug.d.ts b/packages/docusaurus-plugin-debug/src/plugin-debug.d.ts
@@ -7,10 +7,6 @@
 
 /// <reference types="@docusaurus/module-type-aliases" />
 
-declare module '@docusaurus/plugin-debug' {
-  export const routeBasePath: string;
-}
-
 declare module '@theme/DebugConfig' {
   export default function DebugMetadata(): JSX.Element;
 }

diff --git a/packages/docusaurus-plugin-debug/src/theme/DebugLayout/index.tsx b/packages/docusaurus-plugin-debug/src/theme/DebugLayout/index.tsx
@@ -35,6 +35,7 @@ export default function DebugLayout({
       <Head>
         <html lang="en" />
         <title>Docusaurus debug panel</title>
+        <meta name="robots" content="noindex" />
       </Head>
 
       <div>

diff --git a/packages/docusaurus-plugin-sitemap/src/__tests__/createSitemap.test.ts b/packages/docusaurus-plugin-sitemap/src/__tests__/createSitemap.test.ts
@@ -5,6 +5,7 @@
  * LICENSE file in the root directory of this source tree.
  */
 
+import React from 'react';
 import createSitemap from '../createSitemap';
 import type {DocusaurusConfig} from '@docusaurus/types';
 import {EnumChangefreq} from 'sitemap';
@@ -16,6 +17,7 @@ describe('createSitemap', () => {
         url: 'https://example.com',
       } as DocusaurusConfig,
       ['/', '/test'],
+      {},
       {
         changefreq: EnumChangefreq.DAILY,
         priority: 0.7,
@@ -29,7 +31,7 @@ describe('createSitemap', () => {
 
   it('empty site', () =>
     expect(async () => {
-      await createSitemap({} as DocusaurusConfig, [], {});
+      await createSitemap({} as DocusaurusConfig, [], {}, {});
     }).rejects.toThrow(
       'URL in docusaurus.config.js cannot be empty/undefined.',
     ));
@@ -40,6 +42,7 @@ describe('createSitemap', () => {
         url: 'https://example.com',
       } as DocusaurusConfig,
       ['/', '/404.html', '/my-page'],
+      {},
       {
         changefreq: EnumChangefreq.DAILY,
         priority: 0.7,
@@ -55,6 +58,7 @@ describe('createSitemap', () => {
         url: 'https://example.com',
       } as DocusaurusConfig,
       ['/', '/search/', '/tags/', '/search/foo', '/tags/foo/bar'],
+      {},
       {
         changefreq: EnumChangefreq.DAILY,
         priority: 0.7,
@@ -78,6 +82,7 @@ describe('createSitemap', () => {
         trailingSlash: undefined,
       } as DocusaurusConfig,
       ['/', '/test', '/nested/test', '/nested/test2/'],
+      {},
       {
         changefreq: EnumChangefreq.DAILY,
         priority: 0.7,
@@ -98,6 +103,7 @@ describe('createSitemap', () => {
         trailingSlash: true,
       } as DocusaurusConfig,
       ['/', '/test', '/nested/test', '/nested/test2/'],
+      {},
       {
         changefreq: EnumChangefreq.DAILY,
         priority: 0.7,
@@ -118,6 +124,7 @@ describe('createSitemap', () => {
         trailingSlash: false,
       } as DocusaurusConfig,
       ['/', '/test', '/nested/test', '/nested/test2/'],
+      {},
       {
         changefreq: EnumChangefreq.DAILY,
         priority: 0.7,
@@ -130,4 +137,30 @@ describe('createSitemap', () => {
     expect(sitemap).toContain('<loc>https://example.com/nested/test</loc>');
     expect(sitemap).toContain('<loc>https://example.com/nested/test2</loc>');
   });
+
+  it('filters pages with noindex', async () => {
+    const sitemap = await createSitemap(
+      {
+        url: 'https://example.com',
+        trailingSlash: false,
+      } as DocusaurusConfig,
+      ['/', '/noindex', '/nested/test', '/nested/test2/'],
+      {
+        '/noindex': {
+          meta: {
+            toComponent: () => [
+              React.createElement('meta', {name: 'robots', content: 'noindex'}),
+            ],
+          },
+        },
+      },
+      {
+        changefreq: EnumChangefreq.DAILY,
+        priority: 0.7,
+        ignorePatterns: [],
+      },
+    );
+
+    expect(sitemap).not.toContain('/noindex');
+  });
 });
diff --git a/packages/docusaurus-plugin-sitemap/src/createSitemap.ts b/packages/docusaurus-plugin-sitemap/src/createSitemap.ts
@@ -10,10 +10,13 @@ import type {PluginOptions} from '@docusaurus/plugin-sitemap';
 import type {DocusaurusConfig} from '@docusaurus/types';
 import {applyTrailingSlash} from '@docusaurus/utils-common';
 import {createMatcher} from '@docusaurus/utils';
+import type {HelmetServerState} from 'react-helmet-async';
+import type {ReactElement} from 'react';
 
 export default async function createSitemap(
   siteConfig: DocusaurusConfig,
   routesPaths: string[],
+  head: {[location: string]: HelmetServerState},
   options: PluginOptions,
 ): Promise<string> {
   const {url: hostname} = siteConfig;
@@ -26,18 +29,29 @@ export default async function createSitemap(
 
   const sitemapStream = new SitemapStream({hostname});
 
-  routesPaths
-    .filter((route) => !route.endsWith('404.html') && !ignoreMatcher(route))
-    .forEach((routePath) =>
-      sitemapStream.write({
-        url: applyTrailingSlash(routePath, {
-          trailingSlash: siteConfig.trailingSlash,
-          baseUrl: siteConfig.baseUrl,
-        }),
-        changefreq,
-        priority,
-      }),
+  function routeShouldBeIncluded(route: string) {
+    if (route.endsWith('404.html') || ignoreMatcher(route)) {
+      return false;
+    }
+    // https://github.com/staylor/react-helmet-async/pull/167
+    const meta = head[route]?.meta.toComponent() as unknown as
+      | ReactElement[]
+      | undefined;
+    return !meta?.some(
+      (tag) => tag.props.name === 'robots' && tag.props.content === 'noindex',
     );
+  }
+
+  routesPaths.filter(routeShouldBeIncluded).forEach((routePath) =>
+    sitemapStream.write({
+      url: applyTrailingSlash(routePath, {
+        trailingSlash: siteConfig.trailingSlash,
+        baseUrl: siteConfig.baseUrl,
+      }),
+      changefreq,
+      priority,
+    }),
+  );
 
   sitemapStream.end();
 

diff --git a/packages/docusaurus-plugin-sitemap/src/index.ts b/packages/docusaurus-plugin-sitemap/src/index.ts
@@ -18,14 +18,15 @@ export default function pluginSitemap(
   return {
     name: 'docusaurus-plugin-sitemap',
 
-    async postBuild({siteConfig, routesPaths, outDir}) {
+    async postBuild({siteConfig, routesPaths, outDir, head}) {
       if (siteConfig.noIndex) {
         return;
       }
       // Generate sitemap.
       const generatedSitemap = await createSitemap(
         siteConfig,
         routesPaths,
+        head,
         options,
       );
 

diff --git a/packages/docusaurus-preset-classic/src/index.ts b/packages/docusaurus-preset-classic/src/index.ts
@@ -5,7 +5,6 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-import {routeBasePath as debugPluginRouteBasePath} from '@docusaurus/plugin-debug';
 import type {
   Preset,
   LoadContext,
@@ -29,21 +28,20 @@ export default function preset(
   opts: Options = {},
 ): Preset {
   const {siteConfig} = context;
-  const {themeConfig, baseUrl} = siteConfig;
+  const {themeConfig} = siteConfig;
   const {algolia} = themeConfig as Partial<ThemeConfig>;
   const isProd = process.env.NODE_ENV === 'production';
   const {
     debug,
     docs,
     blog,
     pages,
-    sitemap = {},
+    sitemap,
     theme,
     googleAnalytics,
     gtag,
     ...rest
   } = opts;
-  const isDebugEnabled = debug || (debug === undefined && !isProd);
 
   const themes: PluginConfig[] = [];
   themes.push(makePluginConfig('@docusaurus/theme-classic', theme));
@@ -76,17 +74,13 @@ export default function preset(
       makePluginConfig('@docusaurus/plugin-google-analytics', googleAnalytics),
     );
   }
-  if (isDebugEnabled) {
+  if (debug || (debug === undefined && !isProd)) {
     plugins.push(require.resolve('@docusaurus/plugin-debug'));
   }
   if (gtag) {
     plugins.push(makePluginConfig('@docusaurus/plugin-google-gtag', gtag));
   }
   if (isProd && sitemap !== false) {
-    if (isDebugEnabled) {
-      sitemap.ignorePatterns ??= [];
-      sitemap.ignorePatterns.push(`${baseUrl}${debugPluginRouteBasePath}/**`);
-    }
     plugins.push(makePluginConfig('@docusaurus/plugin-sitemap', sitemap));
   }
   if (Object.keys(rest).length > 0) {

diff --git a/packages/docusaurus-types/package.json b/packages/docusaurus-types/package.json
@@ -19,6 +19,7 @@
     "commander": "^5.1.0",
     "history": "^4.9.0",
     "joi": "^17.6.0",
+    "react-helmet-async": "^1.2.3",
     "utility-types": "^3.10.0",
     "webpack": "^5.72.0",
     "webpack-merge": "^5.8.0"

diff --git a/packages/docusaurus-types/src/index.d.ts b/packages/docusaurus-types/src/index.d.ts
@@ -10,6 +10,7 @@ import type {CustomizeRuleString} from 'webpack-merge/dist/types';
 import type {CommanderStatic} from 'commander';
 import type {ParsedUrlQueryInput} from 'querystring';
 import type Joi from 'joi';
+import type {HelmetServerState} from 'react-helmet-async';
 import type {
   DeepRequired,
   Required as RequireKeys,
@@ -319,7 +320,12 @@ export type Plugin<Content = unknown> = {
     actions: PluginContentLoadedActions;
   }) => Promise<void> | void;
   routesLoaded?: (routes: RouteConfig[]) => void; // TODO remove soon, deprecated (alpha-60)
-  postBuild?: (props: Props & {content: Content}) => Promise<void> | void;
+  postBuild?: (
+    props: Props & {
+      content: Content;
+      head: {[location: string]: HelmetServerState};
+    },
+  ) => Promise<void> | void;
   // TODO refactor the configureWebpack API surface: use an object instead of
   // multiple params (requires breaking change)
   configureWebpack?: (

diff --git a/packages/docusaurus/src/client/serverEntry.tsx b/packages/docusaurus/src/client/serverEntry.tsx
@@ -70,6 +70,7 @@ async function doRender(locals: Locals & {path: string}) {
     preBodyTags,
     postBodyTags,
     onLinksCollected,
+    onHeadTagsCollected,
     baseUrl,
     ssrTemplate,
     noIndex,
@@ -105,6 +106,7 @@ async function doRender(locals: Locals & {path: string}) {
     helmet.link.toString(),
     helmet.script.toString(),
   ];
+  onHeadTagsCollected(location, helmet);
   const metaAttributes = metaStrings.filter(Boolean);
 
   const {generatedFilesDir} = locals;

diff --git a/packages/docusaurus/src/commands/build.ts b/packages/docusaurus/src/commands/build.ts
@@ -27,6 +27,7 @@ import {
 import CleanWebpackPlugin from '../webpack/plugins/CleanWebpackPlugin';
 import {loadI18n} from '../server/i18n';
 import {mapAsyncSequential} from '@docusaurus/utils';
+import type {HelmetServerState} from 'react-helmet-async';
 
 export async function build(
   siteDir: string,
@@ -149,12 +150,16 @@ async function buildLocale({
   );
 
   const allCollectedLinks: {[location: string]: string[]} = {};
+  const headTags: {[location: string]: HelmetServerState} = {};
 
   let serverConfig: Configuration = await createServerConfig({
     props,
     onLinksCollected: (staticPagePath, links) => {
       allCollectedLinks[staticPagePath] = links;
     },
+    onHeadTagsCollected: (staticPagePath, tags) => {
+      headTags[staticPagePath] = tags;
+    },
   });
 
   if (staticDirectories.length > 0) {
@@ -224,7 +229,11 @@ async function buildLocale({
       if (!plugin.postBuild) {
         return;
       }
-      await plugin.postBuild({...props, content: plugin.content});
+      await plugin.postBuild({
+        ...props,
+        head: headTags,
+        content: plugin.content,
+      });
     }),
   );