feat: read static/robots.txt (#28)

README.md

@@ -84,12 +84,6 @@ UserAgent: Bingbot
 Disallow: /admin
 ```
 
-## Development
-
-1. Clone this repository
-2. Install dependencies using `yarn install` or `npm install`
-3. Start development server using `npm run dev`
-
 ## License
 
 [MIT License](./LICENSE)

lib/module.js

@@ -1,23 +1,44 @@
 const { resolve } = require('path')
-const { writeFileSync } = require('fs')
+const { existsSync, readFileSync, writeFileSync } = require('fs')
 
 module.exports = function (moduleOptions) {
+  const { rootDir, srcDir, dir: { static: staticDir }, generate: { dir: generateDir } } = this.options
   const options = getOptions.call(this, moduleOptions)
+  const fileName = 'robots.txt'
+  let staticRules = []
+
+  // read static robots.txt
+  this.nuxt.hook('build:before', async () => {
+    const staticFilePath = resolve(srcDir, staticDir, fileName)
+
+    if (existsSync(staticFilePath)) {
+      const content = readFileSync(staticFilePath).toString()
+
+      staticRules = await getRules.call(this, parseFile(content))
+    }
+  })
 
   // generate robots.txt in dist
   this.nuxt.hook('generate:done', async () => {
-    const robotsFilePath = resolve(this.options.rootDir, this.options.generate.dir, 'robots.txt')
+    const generateFilePath = resolve(rootDir, generateDir, fileName)
+    const rules = await getRules.call(this, options)
 
-    writeFileSync(robotsFilePath, await render(options))
+    writeFileSync(generateFilePath, render([...staticRules, ...rules]))
   })
 
   // render robots.txt via SSR
-  this.addServerMiddleware({
-    path: 'robots.txt',
-    async handler (req, res) {
-      res.setHeader('Content-Type', 'text/plain')
-      res.end(await render(options, req))
-    }
+  this.nuxt.hook('render:setupMiddleware', () => {
+    const moduleContainer = this
+
+    this.nuxt.server.useMiddleware({
+      path: fileName,
+      async handler (req, res) {
+        const rules = await getRules.call(moduleContainer, options, req)
+
+        res.setHeader('Content-Type', 'text/plain')
+        res.end(render([...staticRules, ...rules]))
+      }
+    })
   })
 }
 
@@ -43,37 +64,60 @@ function getOptions (moduleOptions) {
   return options
 }
 
-async function render (options, req = null) {
+async function getRules (options, req = null) {
   const correspondances = {
     UserAgent: 'User-agent',
     CrawlDelay: 'Crawl-delay',
     Disallow: 'Disallow',
     Allow: 'Allow',
     Host: 'Host',
     Sitemap: 'Sitemap',
-    CleanParam: 'Clean-param'
+    CleanParam: 'Clean-param',
+    'User-agent': 'User-agent',
+    'Crawl-delay': 'Crawl-delay',
+    'Clean-param': 'Clean-param'
   }
 
-  const content = []
-  const robots = Array.isArray(options) ? options : [options]
+  const rules = []
+  const items = Array.isArray(options) ? options : [options]
 
-  for (const robot of robots) {
-    const rules = []
-    const keys = Object.keys(correspondances).filter(key => typeof robot[key] !== 'undefined')
+  for (const item of items) {
+    const keys = Object.keys(correspondances).filter(key => typeof item[key] !== 'undefined')
 
     for (const key of keys) {
-      let values = typeof robot[key] === 'function' ? await robot[key](req) : robot[key]
+      let values = typeof item[key] === 'function' ? await item[key].call(this, req) : item[key]
       values = (Array.isArray(values)) ? values : [values]
 
       for (const value of values) {
-        rules.push(`${correspondances[key]}: ${typeof value === 'function' ? await value(req) : value}`)
+        rules.push({
+          key: correspondances[key],
+          value: typeof value === 'function' ? await value.call(this, req) : value
+        })
       }
     }
-
-    content.push(rules.join('\n'))
   }
 
-  return content.join('\n')
+  return rules
+}
+
+function render (rules) {
+  return rules.map(rule => `${rule.key}: ${String(rule.value).trim()}`).join('\n')
+}
+
+function parseFile (content) {
+  const rules = []
+
+  content.split('\n').forEach((item) => {
+    const ar = item.split(':')
+
+    if (ar[0]) {
+      rules.push({
+        [ar[0]]: ar[1]
+      })
+    }
+  })
+
+  return rules
 }
 
 module.exports.meta = require('../package.json')

package.json

@@ -14,7 +14,6 @@
     "access": "public"
   },
   "scripts": {
-    "dev": "nuxt test/fixture",
     "lint": "eslint --ext .js,.vue lib test",
     "test": "yarn lint && jest",
     "release": "yarn test && standard-version && git push --follow-tags && npm publish"

test/fixture/nuxt.config.js → test/fixture/empty/nuxt.config.js

@@ -1,13 +1,13 @@
 const { resolve } = require('path')
 
 module.exports = {
-  rootDir: resolve(__dirname, '../..'),
+  rootDir: resolve(__dirname, '../../..'),
   buildDir: resolve(__dirname, '.nuxt'),
   srcDir: __dirname,
   render: {
     resourceHints: false
   },
   modules: [
-    { handler: require('../../') }
+    [require('../../../'), []]
   ]
 }

test/fixture/generate/nuxt.config.js

@@ -0,0 +1,13 @@
+const { resolve } = require('path')
+
+module.exports = {
+  rootDir: resolve(__dirname, '../../..'),
+  buildDir: resolve(__dirname, '.nuxt'),
+  srcDir: __dirname,
+  render: {
+    resourceHints: false
+  },
+  modules: [
+    { handler: require('../../../') }
+  ]
+}

test/fixture/key-as-function/nuxt.config.js

@@ -0,0 +1,19 @@
+const { resolve } = require('path')
+
+module.exports = {
+  rootDir: resolve(__dirname, '../../..'),
+  buildDir: resolve(__dirname, '.nuxt'),
+  srcDir: __dirname,
+  render: {
+    resourceHints: false
+  },
+  modules: [
+    { handler: require('../../../') }
+  ],
+  robots: [
+    {
+      UserAgent: () => ['Googlebot', 'Bingbot'],
+      Disallow: '/admin'
+    }
+  ]
+}

test/fixture/spa/nuxt.config.js

@@ -0,0 +1,14 @@
+const { resolve } = require('path')
+
+module.exports = {
+  mode: 'spa',
+  rootDir: resolve(__dirname, '../../..'),
+  buildDir: resolve(__dirname, '.nuxt'),
+  srcDir: __dirname,
+  render: {
+    resourceHints: false
+  },
+  modules: [
+    { handler: require('../../../') }
+  ]
+}

test/fixture/ssr/nuxt.config.js

@@ -0,0 +1,13 @@
+const { resolve } = require('path')
+
+module.exports = {
+  rootDir: resolve(__dirname, '../../..'),
+  buildDir: resolve(__dirname, '.nuxt'),
+  srcDir: __dirname,
+  render: {
+    resourceHints: false
+  },
+  modules: [
+    { handler: require('../../../') }
+  ]
+}

test/fixture/static/nuxt.config.js

@@ -0,0 +1,13 @@
+const { resolve } = require('path')
+
+module.exports = {
+  rootDir: resolve(__dirname, '../../..'),
+  buildDir: resolve(__dirname, '.nuxt'),
+  srcDir: __dirname,
+  render: {
+    resourceHints: false
+  },
+  modules: [
+    { handler: require('../../../') }
+  ]
+}

test/fixture/static/static/robots.txt

@@ -0,0 +1,2 @@
+foo: bar // unrecognized
+Disallow: /foo

test/fixture/with-array/nuxt.config.js

@@ -0,0 +1,19 @@
+const { resolve } = require('path')
+
+module.exports = {
+  rootDir: resolve(__dirname, '../../..'),
+  buildDir: resolve(__dirname, '.nuxt'),
+  srcDir: __dirname,
+  render: {
+    resourceHints: false
+  },
+  modules: [
+    { handler: require('../../../') }
+  ],
+  robots: [
+    {
+      UserAgent: ['Googlebot', () => 'Bingbot'],
+      Disallow: '/admin'
+    }
+  ]
+}

test/fixture/with-object/nuxt.config.js

@@ -0,0 +1,17 @@
+const { resolve } = require('path')
+
+module.exports = {
+  rootDir: resolve(__dirname, '../../..'),
+  buildDir: resolve(__dirname, '.nuxt'),
+  srcDir: __dirname,
+  render: {
+    resourceHints: false
+  },
+  modules: [
+    { handler: require('../../../') }
+  ],
+  robots: {
+    UserAgent: 'Googlebot',
+    Disallow: () => '/'
+  }
+}