Generate a sitemap + robots.txt

src/cmds/manifest.js

@@ -3,7 +3,8 @@ const { hydrateTree } = require('../core/hydrate')
 
 module.exports = async (args, config) => {
   const tree = await dirTree(config.root)
-  const manifest = await hydrateTree(tree, config)
+  const { manifest } = await hydrateTree(tree, config)
+
   const manifestStr = JSON.stringify(manifest, null, 2)
 
   console.log(manifestStr)

src/core/hydrate.js

@@ -4,6 +4,7 @@ const ourpath = require('../utils/path')
 const { getFrontmatterOnly } = require('../utils/frontmatter')
 const { mergeLeftByKey } = require('../utils/merge')
 const { walkSource } = require('./source')
+const Sitemap = require('./sitemap')
 
 async function getMetaData (item, parentItems) {
   const data = item.type === 'file'
@@ -49,6 +50,7 @@ function normalizeItems (data) {
 
 async function hydrateTree (tree, config, onRegenerate) {
   const urls = {}
+  const sitemap = new Sitemap()
 
   if (tree.childrenIndex === undefined) {
     throw new Error('No index file was found! Create a `readme.md` at the root of your project.')
@@ -104,6 +106,10 @@ async function hydrateTree (tree, config, onRegenerate) {
       // url is now taken, like most women
       urls[hydratedItem.url] = hoistedItem.path
 
+      // add url to the sitemap
+      const fullUrl = `${config.domain}${hydratedItem.url}`
+      sitemap.addUrl(fullUrl, metaData.sitemap)
+
       hydratedItem.input = metaData.input || hoistedItem.path
       hydratedItem.outputDir = syspath.join(
         config.output,
@@ -156,7 +162,10 @@ async function hydrateTree (tree, config, onRegenerate) {
     return hydratedItem
   }
 
-  return _recursive(tree)
+  return {
+    manifest: await _recursive(tree),
+    sitemap: sitemap.generate(),
+  }
 }
 
 // async function hydrateContent (manifest) {

src/core/index.js

@@ -2,6 +2,7 @@ const loadSyntax = require('./syntax')
 const staticAssets = require('./static')
 const { dirTree } = require('./filesystem')
 const { hydrateTree } = require('./hydrate')
+const { getRobotsTxt } = require('./robots')
 const { getCompiler } = require('./compiler')
 const { log } = require('../utils/emit')
 
@@ -14,12 +15,17 @@ module.exports = async (env, localConfig) => {
 
   // generate and hydrate the manifest
   const tree = await dirTree(localConfig.root)
-  const manifest = await hydrateTree(tree, localConfig)
+  const hydrated = await hydrateTree(tree, localConfig)
 
   log('Generated and hydrated manifest')
 
   // this gets passed to the theme
-  const props = { manifest, config: localConfig }
+  const props = {
+    config: localConfig,
+    manifest: hydrated.manifest,
+    sitemap: hydrated.sitemap,
+    robots: getRobotsTxt(localConfig),
+  }
 
   // setup webpack compiler so we can build (or watch)
   const compiler = await getCompiler(env, props)

src/core/output.js

@@ -1,14 +1,21 @@
 const fs = require('fs-extra')
 const syspath = require('path')
+const { warn } = require('../utils/emit')
 const { generateDatabase } = require('./database')
 const { templateForProduction } = require('./template')
 const { getContent, getTableOfContents } = require('./filesystem')
 
 module.exports = async (entrypoints, props) => {
-  const db = await generateDatabase(props.manifest)
   const outputDB = syspath.join(props.config.output, 'db.json')
+  const outputSitemap = syspath.join(props.config.output, 'sitemap.xml')
+  const outputRobots = syspath.join(props.config.output, 'robots.txt')
 
-  await fs.outputJson(outputDB, db)
+  await fs.outputJson(outputDB, await generateDatabase(props.manifest))
+  await fs.outputFile(outputSitemap, props.sitemap)
+
+  await fs.pathExists(outputRobots)
+    ? warn('You have a custom robots.txt file, so one was not generated for you!')
+    : await fs.outputFile(outputRobots, props.robots)
 
   const _recursive = async ({ items, ...item }) => {
     if (item.outputDir) {

src/core/robots.js

@@ -0,0 +1,13 @@
+function getRobotsTxt (config = {}) {
+  const lines = [
+    'User-agent: *',
+    `Sitemap: ${config.domain || ''}/sitemap.xml`,
+    `Disallow: ${config.crawlable ? '' : '/'}`,
+  ]
+
+  return lines.join('\n')
+}
+
+module.exports = {
+  getRobotsTxt,
+}

src/core/robots.spec.js

@@ -0,0 +1,12 @@
+const { expect } = require('code')
+const robots = require('./robots')
+
+describe('unit: core/robots', () => {
+  it('getRobotsTxt()', async () => {
+    expect(robots.getRobotsTxt()).to.equal('User-agent: *\nSitemap: /sitemap.xml\nDisallow: /')
+    expect(robots.getRobotsTxt({
+      domain: 'https://foo.bar',
+      crawlable: true,
+    })).to.equal('User-agent: *\nSitemap: https://foo.bar/sitemap.xml\nDisallow: ')
+  })
+})

src/core/server.js

@@ -35,11 +35,21 @@ module.exports = (props, compiler) => {
     index: false,
   }))
 
+  app.use('/robots.txt', (req, res, next) => {
+    res.setHeader('Content-Type', 'text/plain; charset=utf-8')
+    res.end(props.robots)
+  })
+
+  app.use('/sitemap.xml', (req, res, next) => {
+    res.setHeader('Content-Type', 'application/xml; charset=utf-8')
+    res.end(props.sitemap)
+  })
+
   app.use('/db.json', async (req, res, next) => {
     try {
       const db = await generateDatabase(props.manifest)
 
-      res.setHeader('Content-Type', 'application/json')
+      res.setHeader('Content-Type', 'application/json; charset=utf-8')
       res.end(JSON.stringify(db))
     } catch (err) {
       next(err)

src/core/sitemap.js

@@ -0,0 +1,38 @@
+const { minify } = require('html-minifier')
+
+class Sitemap {
+  constructor () {
+    this.urls = []
+  }
+
+  addUrl (url, data = {}) {
+    const merged = Object.assign({}, {
+      loc: url,
+      priority: 0.5,
+      changefreq: null,
+      lastmod: null,
+    }, data)
+
+    this.urls.push(merged)
+  }
+
+  generate () {
+    return minify(`
+      <?xml version="1.0" encoding="UTF-8"?>
+      <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+        ${this.urls.map(data => `
+          <url>
+            ${data.loc ? `<loc>${data.loc}</loc>` : ''}
+            ${data.lastmod ? `<lastmod>${data.lastmod}</lastmod>` : ''}
+            ${data.changefreq ? `<changefreq>${data.changefreq}</changefreq>` : ''}
+            ${data.priority ? `<priority>${data.priority}</priority>` : ''}
+          </url>
+        `).join('\n')}
+      </urlset>
+    `, {
+      collapseWhitespace: true,
+    })
+  }
+}
+
+module.exports = Sitemap

src/core/sitemap.spec.js

@@ -0,0 +1,12 @@
+const { expect } = require('code')
+const Sitemap = require('./sitemap')
+
+describe('unit: core/sitemap', () => {
+  it('Sitemap()', async () => {
+    const sitemap = new Sitemap()
+    sitemap.addUrl('/foo')
+    sitemap.addUrl('/bar', { priority: 1 })
+    sitemap.addUrl('/baz', { changefreq: 'daily' })
+    expect(sitemap.generate()).to.equal('<?xml version="1.0" encoding="UTF-8"?> <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"><url><loc>/foo</loc><priority>0.5</priority></url><url><loc>/bar</loc><priority>1</priority></url><url><loc>/baz</loc><changefreq>daily</changefreq><priority>0.5</priority></url></urlset>')
+  })
+})

src/utils/config.js

@@ -21,6 +21,8 @@ const DEFAULT_CONFIG = {
   static: '.static',
   temp: tempDir(),
   baseURL: '/',
+  domain: '',
+  crawlable: true,
   host: 'localhost',
   port: 8000,
   languages: ['bash', 'json'],