Feature: Generate sitemaps

example/package.json

@@ -23,10 +23,11 @@
   },
   "scripts": {
     "unit-test": "",
-    "build": "webdoc --tutorials ./tutorials --site-root example-documentation",
+    "build": "webdoc --tutorials ./tutorials --site-root example-documentation --site-domain https://webdoc-labs.github.io",
     "build-next": "cd .. && webdoc && cd example",
     "build-pixi-api": "cd ../../pixi-api && webdoc --site-root docs && cd ../webdoc/example",
-    "build-pixi-api-prod": "cd ../../pixi-api && webdoc --site-root pixi-api && cd ../webdoc/example"
+    "build-pixi-api-prod": "cd ../../pixi-api && webdoc --site-root pixi-api && cd ../webdoc/example",
+    "build-pixi-api-gcp": "cd ../../pixi-api && webdoc && cd ../webdoc/example"
   },
   "bugs": {
     "url": "https://github.com/SukantPal/webdoc/issues"

packages/webdoc-cli/README.md

@@ -14,6 +14,8 @@ npm install --save-dev @webdoc/cli
 
 ### Command-line arguments
 
+* `--site-domain <path>`: (optional) The domain of the website where you'll publish the documentation is used to
+generate the `sitemap.xml`. This is useful if you want to integrate with [Algolia and use its crawler](https://www.algolia.com/products/crawler/). You must include the protocol for this to work currently, e.g. `http://pixijs.webdoclabs.com`.
 * `--site-root <path>`: If using absolute links in a template, this will set the basepath. The basepath should the directory in which the documentation is being stored relative to where the server is running. The site root is "/" by default - which means that you'll need to serve the documentation directory as top-level. Note that @webdoc/default-template uses absolute links.
 * `-c <config-path>`: This sets the path of the configuration file webdoc uses.
 

packages/webdoc-cli/src/config.js

@@ -31,6 +31,7 @@ type ConfigSchema = {
     template?: string,
   },
   template: {
+    siteDomain?: string,
     siteRoot: string,
     mainPage?: {
       title?: string

packages/webdoc-cli/src/index.js

@@ -66,6 +66,9 @@ async function main(argv: yargs.Argv) {
   if (argv.siteRoot) {
     config.template.siteRoot = argv.siteRoot;
   }
+  if (argv.siteDomain) {
+    config.template.siteDomain = argv.siteDomain;
+  }
   if (config.template.siteRoot[0] === "/") {
     config.template.siteRoot = config.template.siteRoot.slice(1);
   }
@@ -137,7 +140,11 @@ async function main(argv: yargs.Argv) {
   };
 
   if (template.publish && typeof template.publish === "function") {
-    template.publish(publishOptions);
+    const resolve = template.publish(publishOptions);
+
+    if (resolve) {
+      await resolve;
+    }
   } else {
     console.error("[Config]: ", `${getTemplate(config)} not found.`);
   }
@@ -146,7 +153,9 @@ async function main(argv: yargs.Argv) {
 }
 
 const argv = yargs.scriptName("@webdoc/cli")
-  .usage("$0 -c <configFile> -u <tutorialDir> --verbose --site-root <siteRoot>")
+  .usage("$0 -c <configFile> -u <tutorialDir> --verbose " +
+    "--site-root <siteRoot> " +
+    "--site-domain <siteDomain>")
   .default("config", path.join(process.cwd(), "webdoc.conf.json"), "webdoc config file")
   .alias("c", "config")
   .alias("u", "tutorials")

packages/webdoc-default-template/publish.js

@@ -7,6 +7,7 @@ const {traverse} = require("@webdoc/model");
 const {
   FlushToFile,
   RelationsPlugin,
+  Sitemap,
   TemplateRenderer,
   TemplatePipeline,
   TemplateTagsResolver,
@@ -45,7 +46,7 @@ const PRETTIFIER_SCRIPT_FILES = [
 
 let idToDoc/*: Map<string, Doc> */;
 
-exports.publish = (options /*: PublishOptions */) => {
+exports.publish = async function publish(options /*: PublishOptions */) {
   const config = options.config;
 
   linker.siteRoot = config.template.siteRoot;
@@ -65,9 +66,17 @@ exports.publish = (options /*: PublishOptions */) => {
     .installPlugin("signature", signaturePlugin)
     .installPlugin("categoryFilter", categoryFilterPlugin)
     .installPlugin("relations", RelationsPlugin);
-  const pipeline = new TemplatePipeline(renderer)
-    .pipe(new TemplateTagsResolver())
-    .pipe(new FlushToFile({skipNullFile: false}));
+
+  const pipeline = new TemplatePipeline(renderer).pipe(new TemplateTagsResolver());
+
+  if (config.template.siteDomain) {
+    pipeline.pipe(new Sitemap(
+      outDir,
+      config.template.siteDomain,
+      config.template.siteRoot));
+  }
+
+  pipeline.pipe(new FlushToFile({skipNullFile: false}));
 
   renderer.getPlugin("relations").buildRelations();
 
@@ -87,6 +96,8 @@ exports.publish = (options /*: PublishOptions */) => {
   outMainPage(path.join(outDir, indexRelative), pipeline, options.config);
   outIndexes(outDir, pipeline, options.config, crawlData.index);
   outReference(outDir, pipeline, options.config, docTree);
+
+  pipeline.close();
 };
 
 // Copy the contents of ./static to the output directory

packages/webdoc-template-library/src/TemplatePipeline.js

@@ -26,6 +26,11 @@ export interface TemplatePipelineElement<T> {
    * Clones the element, less any pipeline-related state.
    */
   clone(): TemplatePipelineElement<T>;
+
+  /**
+   * Flush and close the pipeline's state.
+   */
+  close(): void;
 }
 
 /**
@@ -68,6 +73,15 @@ export class TemplatePipeline {
     return output;
   }
 
+  /**
+   * Close the template pipeline. Pipes can flush files at this stage.
+   */
+  close() {
+    for (const pe of this.elements) {
+      pe.close();
+    }
+  }
+
   /**
    * Adds the pipeline-element to run after the last pipeline element.
    *

packages/webdoc-template-library/src/pipeline-elements/FlushToFile.js

@@ -55,4 +55,6 @@ export class FlushToFile implements TemplatePipelineElement<FlushData> {
       skipNullFile: this.skipNullFile,
     });
   }
+
+  close() {}
 }

packages/webdoc-template-library/src/pipeline-elements/Sitemap.js

@@ -0,0 +1,67 @@
+// @flow
+
+import * as fse from "fs-extra";
+import type {TemplatePipeline, TemplatePipelineElement} from "../TemplatePipeline";
+import path from "path";
+
+type SitemapData = {
+  outputFile: string;
+};
+
+export class Sitemap implements TemplatePipelineElement<SitemapData> {
+  urls: string[] = [];
+  dir: string;
+  domain: string;
+  root: string;
+
+  /**
+   * @param {string} dir - Directory to save sitemap.xml
+   * @param {string} domain - The site domain with protocol
+   * @param {string} root - the siteroot
+   */
+  constructor(dir: string, domain: string, root: string) {
+    this.dir = dir;
+    this.domain = domain;
+    this.root = root;
+
+    if (this.domain.charAt(this.domain.length - 1) !== "/") {
+      this.domain += "/";
+    }
+  }
+
+  attachTo(pipeline: TemplatePipeline) {
+    // noop
+  }
+
+  run(input: string, pipelineData: SitemapData = {}) {
+    if (pipelineData.outputFile) {
+      this.urls.push(pipelineData.outputFile);
+    }
+  }
+
+  close() {
+    /* eslint-disable max-len */
+    const xml =
+`<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
+   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+   xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd">
+${this.urls.map((url) =>
+    `  <url>
+    <loc>${this.domain + path.join(this.root, path.relative(this.dir, url))}</loc>
+  </url>`).join("\n")}
+</urlset>`;
+    /* eslint-enable max-len */
+
+    fse.outputFile(path.join(this.dir, "sitemap.xml"), xml, (err) => {
+      if (err) throw err;
+    });
+  }
+
+  clone() {
+    const clone = new Sitemap(this.dir, this.domain, this.root);
+
+    clone.urls = [...this.urls];
+
+    return clone;
+  }
+}

packages/webdoc-template-library/src/pipeline-elements/TemplateTagsResolver.js

@@ -94,6 +94,8 @@ export class TemplateTagsResolver implements TemplatePipelineElement<{}> {
       linkClass: this.linkClass,
     });
   }
+
+  close() {}
 }
 
 // Helper function to check if link content is just a URL

packages/webdoc-template-library/src/pipeline-elements/index.js

@@ -1,2 +1,3 @@
 export {FlushToFile} from "./FlushToFile";
+export {Sitemap} from "./Sitemap";
 export {TemplateTagsResolver} from "./TemplateTagsResolver";