Merge pull request #374 from easyops-cn/steve/unlisted-content

feat: do not index unlisted content
easyops-cn · Nov 24, 2023 · d45732f · d45732f
2 parents 683687d + b0c425d
commit d45732f
Show file tree

Hide file tree

Showing 6 changed files with 53 additions and 9 deletions.
diff --git a/.eslintrc b/.eslintrc
@@ -35,6 +35,7 @@
     }
   ],
   "rules": {
+    "@typescript-eslint/no-explicit-any": "warn",
     "react-hooks/rules-of-hooks": "error",
     "react-hooks/exhaustive-deps": "warn"
   }

diff --git a/docusaurus-search-local/src/server/utils/parse.spec.ts b/docusaurus-search-local/src/server/utils/parse.spec.ts
@@ -5,7 +5,7 @@ import {
 import { parse } from "./parse";
 
 describe("parse", () => {
-  test.each<[string, "docs" | "blog" | "page", ParsedDocument]>([
+  test.each<[string, "docs" | "blog" | "page", ParsedDocument | null]>([
     [
       `<body>
         <article>
@@ -84,6 +84,24 @@ describe("parse", () => {
         breadcrumb: [],
       },
     ],
+    [
+      `<head>
+        <meta name="robots" content="noindex, nofollow" />
+      </head>
+      <body>
+        <article>
+          <header>
+            <h1>Hello World</h1>
+          </header>
+          <main>
+            <span class="ignore">Test</span>
+            Peace.
+          </main>
+        </article>
+      </body>`,
+      "docs",
+      null,
+    ],
   ])("parse(...) should work", (html, type, doc) => {
     expect(
       parse(html, type, "", {

diff --git a/docusaurus-search-local/src/server/utils/parse.ts b/docusaurus-search-local/src/server/utils/parse.ts
@@ -11,8 +11,15 @@ export function parse(
   type: "docs" | "blog" | "page",
   url: string,
   { ignoreCssSelectors }: ProcessedPluginOptions
-): ParsedDocument {
+): ParsedDocument | null {
   const $ = cheerio.load(html);
+
+  const robotsMeta = $('meta[name="robots"]');
+  if (robotsMeta.attr("content")?.includes("noindex")) {
+    // Unlisted content
+    return null;
+  }
+
   // Remove copy buttons from code boxes
   $('div[class^="mdxCodeBlock_"] button').remove();
 

diff --git a/docusaurus-search-local/src/server/utils/scanDocuments.spec.ts b/docusaurus-search-local/src/server/utils/scanDocuments.spec.ts
@@ -33,6 +33,12 @@ describe("scanDocuments", () => {
         url: "/2",
         type: "page",
       },
+      {
+        // Unlisted
+        filePath: "/tmp/3",
+        url: "/3",
+        type: "docs",
+      },
     ];
     mockParse.mockImplementation((html) => {
       if (html.includes("1")) {
@@ -52,7 +58,7 @@ describe("scanDocuments", () => {
           ],
           breadcrumb: ["Docs"],
         };
-      } else {
+      } else if (html.includes("2")) {
         return {
           pageTitle: "Hello First Page",
           sections: [
@@ -64,6 +70,8 @@ describe("scanDocuments", () => {
           ],
           breadcrumb: [],
         };
+      } else {
+        return null;
       }
     });
     const allDocuments = await scanDocuments(

diff --git a/docusaurus-search-local/src/server/utils/scanDocuments.ts b/docusaurus-search-local/src/server/utils/scanDocuments.ts
@@ -35,12 +35,13 @@ export async function scanDocuments(
       );
 
       const html = await readFileAsync(filePath, { encoding: "utf8" });
-      const { pageTitle, sections, breadcrumb } = parse(
-        html,
-        type,
-        url,
-        config
-      );
+
+      const parsed = parse(html, type, url, config);
+      if (!parsed) {
+        // Unlisted content
+        return;
+      }
+      const { pageTitle, sections, breadcrumb } = parsed;
 
       const titleId = getNextDocId();
 

diff --git a/website/docs/unlisted-post.md b/website/docs/unlisted-post.md
@@ -0,0 +1,9 @@
+---
+unlisted: true
+---
+
+# Unlisted Post
+
+This unlisted blog post should be "hidden" in production, but remain accessible.
+
+It is filtered from the sidebar, sitemap, SEO indexation...