Allow to exclude robots.txt file from generation, add tests, fix typo…

…s. (#73) * Allow to exclude robots.txt file from generation, add tests, fix typos. * chore: update comment --------- Co-authored-by: mikezotov <mike@fluid.ch> Co-authored-by: Jean-Baptiste AUBRÉE <50230578+jbaubree@users.noreply.github.com>
jbaubree · Apr 29, 2023 · d1e2a98 · d1e2a98
1 parent 176633c
commit d1e2a98
Show file tree

Hide file tree

Showing 6 changed files with 58 additions and 6 deletions.
diff --git a/README.md b/README.md
@@ -102,7 +102,14 @@ Last modification option for sitemap.
 - **Type:** `boolean`
 - **Default:** `false`
 
-Converts XML into a human readable format
+Converts XML into a human-readable format
+
+### generateRobotsTxt
+
+- **Type:** `boolean`
+- **Default:** `true`
+
+Enables robots.txt file generation
 
 ### robots
 

diff --git a/src/index.ts b/src/index.ts
@@ -10,10 +10,13 @@ import type { ResolvedOptions, UserOptions } from './types'
 
 export function generateSitemap(options: UserOptions = {}) {
   const resolvedOptions: ResolvedOptions = resolveOptions(options)
+
   // robots.txt
-  const robotRules = getRules(resolvedOptions.robots)
-  const robotContent = getContent(robotRules, resolvedOptions.hostname)
-  writeFileSync(getResolvedPath('robots.txt', resolvedOptions), robotContent)
+  if (resolvedOptions.generateRobotsTxt) {
+    const robotRules = getRules(resolvedOptions.robots)
+    const robotContent = getContent(robotRules, resolvedOptions.hostname)
+    writeFileSync(getResolvedPath('robots.txt', resolvedOptions), robotContent)
+  }
 
   // sitemap.xml
   const routes = getRoutes(resolvedOptions)

diff --git a/src/options.ts b/src/options.ts
@@ -13,6 +13,7 @@ export function resolveOptions(userOptions: UserOptions): ResolvedOptions {
       priority: 1,
       lastmod: new Date(),
       readable: false,
+      generateRobotsTxt: true,
       robots: [{
         userAgent: '*',
         allow: '/',

diff --git a/src/types.ts b/src/types.ts
@@ -80,10 +80,15 @@ interface Options {
    */
   lastmod: Date
   /**
-   * Converts XML into a human readable format
+   * Converts XML into a human-readable format
    * @default false
    */
   readable: boolean
+  /**
+   * Enables robots.txt file generation
+   * @default true
+   */
+  generateRobotsTxt: boolean
   /**
    * Robots policy
    * @default [{ userAgent: '*', allow: '/' }]

diff --git a/test/index.test.ts b/test/index.test.ts
@@ -10,8 +10,12 @@ describe('Index', () => {
     expect(existsSync(SITEMAP_FILE)).toBe(false)
     expect(existsSync(ROBOTS_FILE)).toBe(false)
 
-    generateSitemap()
+    generateSitemap({ generateRobotsTxt: false })
 
+    expect(existsSync(SITEMAP_FILE)).toBe(false)
+    expect(existsSync(ROBOTS_FILE)).toBe(false)
+
+    generateSitemap()
     expect(existsSync(SITEMAP_FILE)).toBe(false)
     expect(existsSync(ROBOTS_FILE)).toBe(true)
 

diff --git a/test/options.test.ts b/test/options.test.ts
@@ -12,6 +12,7 @@ describe('Options', () => {
         "dynamicRoutes": [],
         "exclude": [],
         "extensions": "html",
+        "generateRobotsTxt": true,
         "hostname": "http://localhost/",
         "lastmod": Any<Date>,
         "outDir": "dist",
@@ -42,6 +43,7 @@ describe('Options', () => {
           "/route2/sub-route",
         ],
         "extensions": "html",
+        "generateRobotsTxt": true,
         "hostname": "http://localhost/",
         "lastmod": Any<Date>,
         "outDir": "dist",
@@ -72,6 +74,7 @@ describe('Options', () => {
           "html",
           "md",
         ],
+        "generateRobotsTxt": true,
         "hostname": "http://localhost/",
         "lastmod": Any<Date>,
         "outDir": "dist",
@@ -110,6 +113,7 @@ describe('Options', () => {
         "dynamicRoutes": [],
         "exclude": [],
         "extensions": "html",
+        "generateRobotsTxt": true,
         "hostname": "http://localhost/",
         "lastmod": Any<Date>,
         "outDir": "dist",
@@ -138,4 +142,32 @@ describe('Options', () => {
       }
     `)
   })
+
+  test('resolve options with disabled robots.txt file', () => {
+    expect(resolveOptions({
+      generateRobotsTxt: false,
+    })).toMatchInlineSnapshot({
+      lastmod: expect.any(Date),
+    }, `
+      {
+        "basePath": "",
+        "changefreq": "daily",
+        "dynamicRoutes": [],
+        "exclude": [],
+        "extensions": "html",
+        "generateRobotsTxt": false,
+        "hostname": "http://localhost/",
+        "lastmod": Any<Date>,
+        "outDir": "dist",
+        "priority": 1,
+        "readable": false,
+        "robots": [
+          {
+            "allow": "/",
+            "userAgent": "*",
+          },
+        ],
+      }
+    `)
+  })
 })