Merge pull request #4 from iamvishnusankar/development

Added robots.txt option

README.md

@@ -14,8 +14,9 @@ yarn add next-sitemap -D
 
 ```js
 module.exports = {
-  siteUrl: 'https://example.com'
-  // other options
+  siteUrl: 'https://example.com',
+  generateRobotsTxt: true // (optional)
+  // ...other options
 }
 ```
 
@@ -35,23 +36,60 @@ Define the `sitemapSize` property in `next-sitemap.js` to split large sitemap in
 ```js
 module.exports = {
   siteUrl: 'https://example.com',
-  sitemapSize: 5000
+  generateRobotsTxt: true
 }
 ```
 
 Above is the minimal configuration to split a large sitemap. When the number of URLs in a sitemap is more than 5000, `next-sitemap` will create sitemap (e.g. sitemap-1.xml, sitemap-2.xml) and index (e.g. sitemap.xml) files.
 
 ## `next-sitemap.js` Options
 
-| property              | description                                                                   |
-| --------------------- | ----------------------------------------------------------------------------- |
-| siteUrl               | Base url of your website                                                      |
-| changefreq (optional) | Change frequency. Default to `daily`                                          |
-| priority (optional)   | Priority. Default to `0.7`                                                    |
-| path (optional)       | Sitemap export path. Default `public/sitemap.xml`                             |
-| sitemapSize(optional) | Split large sitemap into multiple files by specifying sitemap size (eg: 5000) |
+| property                            | description                                                                        | type     |
+| ----------------------------------- | ---------------------------------------------------------------------------------- | -------- |
+| siteUrl                             | Base url of your website                                                           | string   |
+| changefreq (optional)               | Change frequency. Default `daily`                                                  | string   |
+| priority (optional)                 | Priority. Default `0.7`                                                            | number   |
+| sitemapSize(optional)               | Split large sitemap into multiple files by specifying sitemap size. Default `5000` | number   |
+| generateRobotsTxt                   | Generate a `robots.txt` file and list the generated sitemaps. Default `false`      | boolean  |
+| robotsTxtOptions.policies           | Policies for generating `robots.txt`. Default to `[{ userAgent: '*', allow: '/' }` | []       |
+| robotsTxtOptions.additionalSitemaps | Options to add addition sitemap to `robots.txt` host entry                         | string[] |
+
+## Full configuration
+
+Here's an example `next-sitemap.js` configuration with all options
+
+```js
+module.exports = {
+  siteUrl: 'https://example.com',
+  changefreq: 'daily',
+  priority: 0.7,
+  sitemapSize: 5000,
+  generateRobotsTxt: true,
+  robotsTxtOptions: {
+    policies: [
+      {
+        userAgent: '*',
+        allow: '/'
+      },
+      {
+        userAgent: 'test-bot',
+        allow: ['/path', '/path-2']
+      },
+      {
+        userAgent: 'black-listed-bot',
+        disallow: ['/sub-path-1', '/path-2']
+      }
+    ],
+    additionalSitemaps: [
+      'https://example.com/my-custom-sitemap-1.xml',
+      'https://example.com/my-custom-sitemap-2.xml',
+      'https://example.com/my-custom-sitemap-3.xml'
+    ]
+  }
+}
+```
 
 ## TODO
 
 - <s>Add support for splitting sitemap</s>
-- Add support for `robots.txt`
+- <s>Add support for `robots.txt`</s>

example/README.md

@@ -0,0 +1 @@
+[Documentation](https://github.com/iamvishnusankar/next-sitemap)

example/next-sitemap.js

@@ -1,4 +1,12 @@
 module.exports = {
   siteUrl: 'https://example.com',
-  sitemapSize: 3000
+  generateRobotsTxt: true,
+  // optional
+  robotsTxtOptions: {
+    additionalSitemaps: [
+      'https://example.com/my-custom-sitemap-1.xml',
+      'https://example.com/my-custom-sitemap-2.xml',
+      'https://example.com/my-custom-sitemap-3.xml'
+    ]
+  }
 }

packages/next-sitemap/package.json

@@ -14,5 +14,8 @@
   },
   "scripts": {
     "build": "tsc"
+  },
+  "dependencies": {
+    "deepmerge": "^4.2.2"
   }
 }

packages/next-sitemap/src/array/index.test.ts

@@ -1,4 +1,4 @@
-import { toChunks } from '.'
+import { toChunks, toArray } from './index'
 
 describe('next-sitemap/array', () => {
   test('toChunks', () => {
@@ -10,4 +10,9 @@ describe('next-sitemap/array', () => {
     expect(chunks).toMatchSnapshot()
     expect(chunks.length).toBe(Math.ceil(inputArray.length / chunkSize))
   })
+
+  test('toArray', () => {
+    expect(toArray('hello')).toStrictEqual(['hello'])
+    expect(toArray(['hello', 'world'])).toStrictEqual(['hello', 'world'])
+  })
 })

packages/next-sitemap/src/array/index.ts

@@ -4,3 +4,11 @@ export const toChunks = <T>(arr: T[], chunkSize: number) => {
     []
   )
 }
+
+/**
+ * simple method to normalize any string to array
+ * @param inp
+ */
+export const toArray = (inp: string | string[]) => {
+  return typeof inp === 'string' ? [inp] : inp
+}

packages/next-sitemap/src/buildSitemapXml/index.test.ts

@@ -8,8 +8,8 @@ describe('generateSitemap', () => {
           siteUrl: 'https://example.com',
           priority: 0.7,
           changefreq: 'daily',
-          path: 'sitemap'
-        },
+          rootDir: 'public'
+        } as any,
         ['/', '/another', '/example']
       )
     ).toMatchSnapshot()

packages/next-sitemap/src/config/index.test.ts

@@ -0,0 +1,50 @@
+import { defaultConfig, withDefaultConfig } from '.'
+
+describe('next-sitemap/config', () => {
+  test('defaultConfig', () => {
+    expect(defaultConfig).toStrictEqual({
+      rootDir: 'public',
+      priority: 0.7,
+      changefreq: 'daily',
+      sitemapSize: 5000,
+      robotsTxtOptions: {
+        policies: [
+          {
+            userAgent: '*',
+            allow: '/'
+          }
+        ],
+        additionalSitemaps: []
+      }
+    })
+  })
+
+  test('withDefaultConfig', () => {
+    const myConfig = withDefaultConfig({
+      generateRobotsTxt: true,
+      sitemapSize: 50000,
+      robotsTxtOptions: {
+        policies: [],
+        additionalSitemaps: [
+          'https://example.com/awesome-sitemap.xml',
+          'https://example.com/awesome-sitemap-2.xml'
+        ]
+      }
+    })
+
+    expect(myConfig).toStrictEqual({
+      rootDir: 'public',
+      priority: 0.7,
+      changefreq: 'daily',
+      sitemapSize: 50000,
+      generateRobotsTxt: true,
+      robotsTxtOptions: {
+        policies: [],
+        additionalSitemaps: [
+          'https://example.com/awesome-sitemap.xml',
+          'https://example.com/awesome-sitemap-2.xml'
+        ]
+      }
+    })
+  })
+})

packages/next-sitemap/src/config/index.ts

@@ -1,16 +1,31 @@
 import fs from 'fs'
 import allPath from '../path'
 import { IConfig } from '../interface'
+import deepmerge from 'deepmerge'
 
-export const withDefaultConfig = (config: IConfig) => {
-  return {
-    path: './public/sitemap.xml',
-    priority: 0.7,
-    changefreq: 'daily',
-    ...(config as any)
-  } as IConfig
+export const defaultConfig: Partial<IConfig> = {
+  rootDir: 'public',
+  priority: 0.7,
+  changefreq: 'daily',
+  sitemapSize: 5000,
+  robotsTxtOptions: {
+    policies: [
+      {
+        userAgent: '*',
+        allow: '/'
+      }
+    ],
+    additionalSitemaps: []
+  }
 }
 
+const overwriteMerge = (_: any[], sourceArray: any[], __: any) => sourceArray
+
+export const withDefaultConfig = (config: Partial<IConfig>) =>
+  deepmerge(defaultConfig, config, {
+    arrayMerge: overwriteMerge
+  })
+
 export const loadConfig = (): IConfig => {
   if (fs.existsSync(allPath.CONFIG_FILE)) {
     const config = require(allPath.CONFIG_FILE)

packages/next-sitemap/src/export/index.ts

@@ -1,11 +1,11 @@
 import fs from 'fs'
 import path from 'path'
 
-export const exportSitemap = (filePath: string, xml: string) => {
+export const exportFile = (filePath: string, content: string) => {
   const folder = path.dirname(filePath)
   if (!fs.existsSync(folder)) {
     fs.mkdirSync(folder)
   }
 
-  fs.writeFileSync(filePath, xml)
+  fs.writeFileSync(filePath, content)
 }

packages/next-sitemap/src/index.ts

@@ -1,34 +1,43 @@
 import { loadConfig } from './config'
 import { loadManifest } from './manifest'
-import { createUrlSet } from './url'
+import { createUrlSet, generateUrl } from './url'
 import { buildSitemapXml } from './buildSitemapXml'
-import { exportSitemap } from './export'
+import { exportFile } from './export'
 import { toChunks } from './array'
 import { resolveSitemapChunks } from './path'
+import { generateRobotsTxt } from './robotsTxt'
 
 const config = loadConfig()
 const manifest = loadManifest()
 const urlSet = createUrlSet(config, manifest)
-const sitemapPath = config.path
+const sitemapPath = `${config.rootDir}/sitemap.xml`
+const robotsTxtFile = `${config.rootDir}/robots.txt`
 
-if (!!!config.sitemapSize && urlSet.length > 5000) {
-  console.warn(
-    `WARN: Looks like you have too many links. Consider splitting your sitemap into multiple files by specifying 'sitemapSize' property in next-sitemap.js`
-  )
-}
-
-export const generateBasicSitemap = (path: string, urls: string[]) => {
+export const generateSitemap = (path: string, urls: string[]) => {
   const sitemapXml = buildSitemapXml(config, urls)
-  exportSitemap(path, sitemapXml)
+  exportFile(path, sitemapXml)
 }
 
-// Generate Basic sitemap if the chunk size is not specified
-if (!!!config.sitemapSize) {
-  generateBasicSitemap(sitemapPath, urlSet)
-} else {
-  // Spile sitemap into multiple files
-  const chunks = toChunks(urlSet, config.sitemapSize)
-  const sitemapChunks = resolveSitemapChunks(sitemapPath, chunks)
+const allSitemaps: string[] = []
+
+// Split sitemap into multiple files
+const chunks = toChunks(urlSet, config.sitemapSize!)
+const sitemapChunks = resolveSitemapChunks(sitemapPath, chunks)
+sitemapChunks.forEach((chunk) => {
+  generateSitemap(chunk.path, chunk.urls)
+  allSitemaps.push(generateUrl(config.siteUrl, `/${chunk.filename}`))
+})
+
+if (config.generateRobotsTxt) {
+  // Push the known sitemaps to the additionalSitemapList
+  config.robotsTxtOptions!.additionalSitemaps = [
+    ...allSitemaps,
+    ...config.robotsTxtOptions!.additionalSitemaps!
+  ]
+
+  const robotsTxt = generateRobotsTxt(config)
 
-  sitemapChunks.forEach((chunk) => generateBasicSitemap(chunk.path, chunk.urls))
+  if (robotsTxt) {
+    exportFile(robotsTxtFile, robotsTxt)
+  }
 }

packages/next-sitemap/src/interface.ts

@@ -1,9 +1,22 @@
+export interface IRobotPolicy {
+  userAgent: string
+  disallow?: string | string[]
+  allow?: string | string[]
+}
+
+export interface IRobotsTxt {
+  policies?: IRobotPolicy[]
+  additionalSitemaps?: string[]
+}
+
 export interface IConfig {
   siteUrl: string
   changefreq: string
   priority: any
-  path: string
+  rootDir: string
   sitemapSize?: number
+  generateRobotsTxt: boolean
+  robotsTxtOptions?: IRobotsTxt
 }
 
 export interface IBuildManifest {

packages/next-sitemap/src/path/index.ts

@@ -6,10 +6,15 @@ export const getPath = (rel: string) => {
 
 export const resolveSitemapChunks = (baseSitemapPath: string, chunks: string[][]) => {
   const folder = path.dirname(baseSitemapPath)
-  return chunks.map((chunk, index) => ({
-    path: `${folder}/sitemap${index > 0 ? `-${index}` : ''}.xml`,
-    urls: chunk
-  }))
+  return chunks.map((chunk, index) => {
+    const filename = `sitemap${index > 0 ? `-${index}` : ''}.xml`
+
+    return {
+      path: `${folder}/${filename}`,
+      urls: chunk,
+      filename
+    }
+  })
 }
 
 const allPath = {

packages/next-sitemap/src/robotsTxt/__snapshots__/index.test.ts.snap

@@ -0,0 +1,14 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`next-sitemap/generateRobotsTxt generateRobotsTxt: additionalSitemap 1`] = `
+"User-agent: *
+Allow: /
+User-agent: black-listed-bot
+Disallow: /sub-path-1
+Disallow: /path-2
+Host: https://example.com
+Sitemap: https://example.com/my-custom-sitemap-1.xml
+Sitemap: https://example.com/my-custom-sitemap-2.xml
+Sitemap: https://example.com/my-custom-sitemap-3.xml
+"
+`;