build(aio): implement prerendering

The current implementation is based on @IgorMinar's [angular-io-v42][1]. It is
using Protractor to request all docs URLs, let them fallback to `/index.html`
and save the rendered page.

[1]: https://github.com/IgorMinar/angular-io-v42/tree/05508ab3/tools/prerenderer

Fixes #15104

aio/.gitignore

@@ -7,5 +7,6 @@ yarn-error.log
 
 # Ignore generated content
 /dist
+/tmp
 /src/content/docs
 /.sass-cache

aio/aio-builds-setup/dockerbuild/nginx/aio-builds.conf

@@ -35,7 +35,7 @@ server {
   }
 
   location / {
-    try_files $uri $uri/ /index.html =404;
+    try_files $uri /content/docs-prerendered/$uri.html $uri/ =404;
   }
 }
 

aio/firebase.json

@@ -4,11 +4,10 @@
   },
   "hosting": {
     "public": "dist",
-    "cleanUrls": true,
     "rewrites": [
       {
-        "source": "**",
-        "destination": "/index.html"
+        "source": "/:path*",
+        "destination": "/content/docs-prerendered/:path*.html"
       }
     ]
   }

aio/package.json

@@ -14,13 +14,20 @@
     "lint": "yarn check-env && ng lint",
     "pree2e": "yarn ~~update-webdriver",
     "e2e": "yarn check-env && ng e2e --no-webdriver-update",
+    "predeploy-preview": "yarn prerender",
     "deploy-preview": "scripts/deploy-preview.sh",
+    "predeploy-staging": "yarn prerender",
     "deploy-staging": "firebase use staging --token \"$FIREBASE_TOKEN\" && yarn ~~deploy",
     "check-env": "node ../tools/check-environment.js",
     "docs": "dgeni ./transforms/angular.io-package",
     "docs-test": "node ../dist/tools/cjs-jasmine/index-tools ../../transforms/**/*.spec.js",
+    "preprerender": "yarn build",
+    "prerender": "concurrently --kill-others --raw --success first \"yarn ~~prerender-serve\" \"yarn ~~prerender\"",
+    "postprerender": "node tools/prerender/copy-to-dist",
     "~~update-webdriver": "webdriver-manager update --standalone false --gecko false",
-    "pre~~deploy": "yarn build",
+    "pre~~prerender": "yarn ~~update-webdriver",
+    "~~prerender": "node tools/prerender/create-specs && protractor tools/prerender/protractor.conf.js",
+    "~~prerender-serve": "node tools/prerender/serve",
     "~~deploy": "firebase deploy --message \"Commit: $TRAVIS_COMMIT\" --non-interactive --token \"$FIREBASE_TOKEN\""
   },
   "private": true,
@@ -49,6 +56,7 @@
     "@types/node": "~6.0.60",
     "canonical-path": "^0.0.2",
     "codelyzer": "~2.0.0-beta.4",
+    "concurrently": "^3.4.0",
     "dgeni": "^0.4.7",
     "dgeni-packages": "^0.16.8",
     "entities": "^1.1.1",
@@ -65,6 +73,7 @@
     "lodash": "^4.17.4",
     "protractor": "~5.1.0",
     "rho": "^0.3.0",
+    "shelljs": "^0.7.7",
     "ts-node": "~2.0.0",
     "tslint": "~4.4.2",
     "typescript": "2.1.6"

aio/protractor.conf.js

@@ -1,5 +1,6 @@
 // Protractor configuration file, see link for more information
 // https://github.com/angular/protractor/blob/master/lib/config.ts
+'use strict';
 
 /*global jasmine */
 const { SpecReporter } = require('jasmine-spec-reporter');

aio/scripts/deploy-preview.sh

@@ -11,7 +11,7 @@ UPLOAD_URL=$AIO_BUILDS_HOST/create-build/$TRAVIS_PULL_REQUEST/$TRAVIS_PULL_REQUE
 
 cd "`dirname $0`/.."
 
-yarn run build
+# Assumes the build step has already run
 tar --create --gzip --directory "$INPUT_DIR" --file "$OUTPUT_FILE" .
 
 exec 3>&1

aio/tools/prerender/constants.js

@@ -0,0 +1,30 @@
+'use strict';
+
+// Imports
+const path = require('path');
+
+// Constants
+const BROWSER_INSTANCES = 7;
+
+const PORT = 4201;
+const BASE_URL = `http://localhost:${PORT}`;
+
+const ROOT_DIR = path.join(__dirname, '../..');
+const DIST_DIR = path.join(ROOT_DIR, 'dist');
+const CONTENT_DIR = path.join(DIST_DIR, 'content');
+const INPUT_DIR = path.join(CONTENT_DIR, 'docs');
+const TMP_SPECS_DIR = path.join(ROOT_DIR, 'tmp/docs-prerender-specs');
+const TMP_OUTPUT_DIR = path.join(ROOT_DIR, 'tmp/docs-prerendered');
+
+// Exports
+module.exports = {
+  BASE_URL,
+  BROWSER_INSTANCES,
+  CONTENT_DIR,
+  DIST_DIR,
+  INPUT_DIR,
+  PORT,
+  ROOT_DIR,
+  TMP_OUTPUT_DIR,
+  TMP_SPECS_DIR
+};

aio/tools/prerender/copy-to-dist.js

@@ -0,0 +1,10 @@
+'use strict';
+
+// Imports
+const sh = require('shelljs');
+const { CONTENT_DIR, TMP_OUTPUT_DIR } = require('./constants');
+
+sh.config.fatal = true;
+
+// Run
+sh.cp('-r', TMP_OUTPUT_DIR, CONTENT_DIR);

aio/tools/prerender/create-specs.js

@@ -0,0 +1,89 @@
+'use strict';
+
+// Imports
+const fs = require('fs');
+const path = require('path');
+const sh = require('shelljs');
+const { BASE_URL, BROWSER_INSTANCES, INPUT_DIR, PORT, TMP_OUTPUT_DIR, TMP_SPECS_DIR } = require('./constants');
+
+sh.config.fatal = true;
+
+// Helpers
+const chunkArray = (items, numChunks) => {
+  numChunks = Math.min(numChunks, items.length);
+  const itemsPerChunk = Math.ceil(items.length / numChunks);
+  const chunks = new Array(numChunks);
+
+  console.log(`Chunking ${items.length} items into ${numChunks} chunks.`);
+
+  for (let i = 0; i < numChunks; i++) {
+    chunks[i] = items.slice(i * itemsPerChunk, (i + 1) * itemsPerChunk);
+  }
+
+  return chunks;
+};
+
+const getAllFiles = rootDir => fs.readdirSync(rootDir).reduce((files, file) => {
+  const absolutePath = path.join(rootDir, file);
+  const isFile = fs.lstatSync(absolutePath).isFile();
+
+  return files.concat(isFile ? absolutePath : getAllFiles(absolutePath));
+}, []);
+
+const getAllUrls = rootDir => getAllFiles(rootDir).
+  filter(absolutePath => path.extname(absolutePath) === '.json').
+  map(absolutePath => absolutePath.slice(0, -5)).
+  map(absolutePath => path.relative(INPUT_DIR, absolutePath)).
+  map(relativePath => `${BASE_URL}/${relativePath}`);
+
+const getTestForChunk = (chunk, idx) => `
+  'use strict';
+
+  const fs = require('fs');
+  const path = require('path');
+  const protractor = require('protractor');
+  const sh = require('shelljs');
+  const url = require('url');
+
+  const browser = protractor.browser;
+  sh.config.fatal = true;
+
+  describe('chunk ${idx}', () => ${JSON.stringify(chunk)}.forEach(urlToPage => {
+    const parsedUrl = url.parse(urlToPage);
+
+    it(\`should render \${parsedUrl.path}\`, done => {
+      browser.get(urlToPage);
+      browser.getPageSource()
+        .then(source => {
+          if (/document not found/i.test(source) && !/file-not-found/i.test(urlToPage)) {
+            return Promise.reject(\`404 for \${urlToPage}\`);
+          }
+
+          const relativeFilePath = parsedUrl.path.replace(/\\/$/, '/index').replace(/^\\//, '') + '.html';
+          const absoluteFilePath = path.resolve('${TMP_OUTPUT_DIR}', relativeFilePath);
+          const absoluteDirPath = path.dirname(absoluteFilePath);
+
+          console.log(\`Writing to \${absoluteFilePath}...\`);
+
+          sh.mkdir('-p', absoluteDirPath);
+          fs.writeFileSync(absoluteFilePath, source);
+        })
+        .then(done, done.fail);
+    });
+  }));
+`;
+
+// Run
+const docsUrls = getAllUrls(INPUT_DIR);
+const chunked = chunkArray(docsUrls, BROWSER_INSTANCES);
+
+sh.rm('-rf', TMP_OUTPUT_DIR);
+sh.rm('-rf', TMP_SPECS_DIR);
+sh.mkdir('-p', TMP_SPECS_DIR);
+
+chunked.forEach((chunk, idx) => {
+  const outputFile = path.join(TMP_SPECS_DIR, `chunk${idx}.spec.js`);
+  const testContent = getTestForChunk(chunk, idx);
+
+  fs.writeFileSync(outputFile, testContent);
+});

aio/tools/prerender/protractor.conf.js

@@ -0,0 +1,35 @@
+// Protractor configuration file, see link for more information
+// https://github.com/angular/protractor/blob/master/lib/config.ts
+'use strict';
+
+/*global jasmine */
+const { SpecReporter } = require('jasmine-spec-reporter');
+const path = require('path');
+const { BASE_URL, BROWSER_INSTANCES, TMP_SPECS_DIR } = require('./constants');
+
+exports.config = {
+  allScriptsTimeout: 11000,
+  specs: [
+    path.join(TMP_SPECS_DIR, 'chunk*.spec.js')
+  ],
+  capabilities: {
+    browserName: 'chrome',
+    shardTestFiles: true,
+    maxInstances: BROWSER_INSTANCES,
+    // For Travis
+    chromeOptions: {
+      binary: process.env.CHROME_BIN
+    }
+  },
+  directConnect: true,
+  baseUrl: BASE_URL,
+  framework: 'jasmine',
+  jasmineNodeOpts: {
+    showColors: true,
+    defaultTimeoutInterval: 30000,
+    print: function() {}
+  },
+  onPrepare() {
+    jasmine.getEnv().addReporter(new SpecReporter({spec: {displayStacktrace: true}}));
+  }
+};

aio/tools/prerender/serve.js

@@ -0,0 +1,63 @@
+'use strict';
+
+// Imports
+const fs = require('fs');
+const http = require('http');
+const path = require('path');
+const { BASE_URL, DIST_DIR, PORT } = require('./constants');
+
+// Constants
+const CONTENT_TYPES = {
+  '.css': 'text/css',
+  '.html': 'text/html',
+  '.ico': 'image/x-icon',
+  '.jpg': 'image/jpeg',
+  '.js': 'text/javascript',
+  '.json': 'application/json',
+  '.png': 'image/png',
+  '.svg': 'image/svg+xml'
+};
+const CACHE = {};
+const VERBOSE = process.argv.includes('--verbose');
+
+// Helpers
+const urlToFile = url => path.join(DIST_DIR, url);
+
+const getFile = filePath => new Promise((resolve, reject) => CACHE.hasOwnProperty(filePath) ?
+  resolve(CACHE[filePath]) :
+  fs.readFile(filePath, 'utf-8', (err, content) => err ? reject(err) : resolve(CACHE[filePath] = content)));
+
+const middleware = (req, res) => {
+  const method = req.method;
+  let url = req.url;
+
+  if (VERBOSE) console.log(`Request: ${method} ${url}`);
+  if (method !== 'GET') return;
+
+  if (url.endsWith('/')) url += 'index';
+  if (!url.includes('.')) url += '.html';
+
+  let filePath = urlToFile(url);
+  if (!fs.existsSync(filePath)) filePath = urlToFile('index.html');
+
+  getFile(filePath).
+    then(content => {
+      const contentType = CONTENT_TYPES[path.extname(filePath)] || 'application/octet-stream';
+      res.setHeader('Content-Type', contentType);
+      res.end(content);
+    }).
+    catch(err => {
+      console.error(err);
+      res.statusCode = 500;
+      res.end(http.STATUS_CODES[500]);
+    });
+};
+
+// Run
+const server = http.
+  createServer(middleware).
+  on('error', err => console.error(err)).
+  on('listening', () => console.log(`Server listening at ${BASE_URL}.`)).
+  listen(PORT);
+
+