Add Puppeteer to /web/tools/ (#5644)

* Add Puppeteer to /web/tools

* Make tests pass

* Support {% includecode github_path="..." regex="..."}

* Use real path

* better nav

* Build scripts for puppeteer docs

* make eslint happy

* fix build error

* Add ppptr docs build to travis

* Fix travis?

* update desc

* feedback

* remove _project.yaml for now

* remove footer

.eslintrc

@@ -1,7 +1,7 @@
 {
   "extends": ["eslint:recommended", "google"],
   "parserOptions": {
-    "ecmaVersion": 6
+    "ecmaVersion": 2017
   },
   "env": {
     "node": true,

.gitignore

@@ -36,6 +36,7 @@ src/content/*/showcase/_index.yaml
 src/content/*/showcase/*/_toc.yaml
 src/content/*/showcase/*/index.md
 src/content/*/showcase/tags/*
+src/content/*/tools/puppeteer/_src/*
 src/content/*/rss.xml
 src/content/*/atom.xml
 src/data/codelabs/*/img/*

.travis.yml

@@ -18,6 +18,7 @@ install:
   - npm run clean
 script:
   - npm run build
+  - gulp puppeteer:build
   - npm run test
   - node ./tools/travis/addCommitComment.js
 after_success:

devsiteHelper.py

@@ -108,7 +108,7 @@ def readFile(requestedFile, lang='en'):
     return None
 
 
-def fetchGithubFile(path):
+def fetchGithubFile(path, revision = None):
   """Fetchs a file in a repository hosted on github.com.
 
   Retrieves rows pertaining to the given keys from the Table instance
@@ -118,11 +118,15 @@ def fetchGithubFile(path):
   Args:
       path: The path to a file on Github in the form
           github-user/repository/path/to/file.
+      revision: Optional git revision string of the form
+          "refs/tags/v0.10.0". See https://goo.gl/DBLwBb for other options.
 
   Returns:
       A string from the HTTP response.
   """
   path = path.replace('/blob', '')
+  if revision:
+    path = path.replace('master', revision.replace('refs/tags/', ''))
   url = 'https://raw.githubusercontent.com/%s' % path
   try:
     response = urllib2.urlopen(url)
@@ -419,12 +423,14 @@ def getIncludeCode(include_tag, lang='en'):
   region_regex = re.search(r"region_tag=\"(.+?)\"", include_tag)
   dedent_regex = re.search(r"adjust_indentation=\"(.+?)\"", include_tag)
   github_regex = re.search(r"github_path=\"(.+?)\"", include_tag)
+  git_revision_regex = re.search(r"git_revision=\"(.+?)\"", include_tag)
+  file_content_regex = re.search(r"regexp=\"(.+?)\"", include_tag)
 
   # TODO: support these arguments
   as_downloadable_regex = re.search(r"as_downloadable=\"(.+?)\"", include_tag)
   github_link_regex = re.search(r"github_link=\"(.+?)\"", include_tag)
-  git_revision_regex = re.search(r"git_revision=\"(.+?)\"", include_tag)
-  if as_downloadable_regex or github_link_regex or git_revision_regex:
+
+  if as_downloadable_regex or github_link_regex:
     msg = 'Error: as_downloadable, github_link, and git_revision args are not supported'
     logging.error(' - ' + msg)
     return msg
@@ -441,9 +447,12 @@ def getIncludeCode(include_tag, lang='en'):
     if result is None:
       return 'Warning: Unable to find includecode <code>%s</code>' % file_name
   elif github_regex:
+    git_revision = None
+    if git_revision_regex:
+      git_revision = git_revision_regex.group(1)
 
     file_url = github_regex.group(1)
-    result = fetchGithubFile(file_url)
+    result = fetchGithubFile(file_url, git_revision)
     if result is None:
       return 'Warning: Unable to includecode from github_path="<code>%s</code>"' % file_url
 
@@ -456,6 +465,13 @@ def getIncludeCode(include_tag, lang='en'):
       end_at = result.rfind('\n', start_at, end_at)
       result = result[start_at:end_at]
 
+  if file_content_regex:
+    print file_content_regex.groups()
+    r = re.compile(file_content_regex.group(1), re.DOTALL)
+    m = re.search(r, result)
+    if m:
+      result = m.group(1)
+
   if dedent_regex and dedent_regex.group(1) == 'auto':
     result = textwrap.dedent(result)
 

gulp-tasks/puppeteer/build_docs.js

@@ -0,0 +1,128 @@
+/**
+ * @fileoverview Generates the Puppeteer docs at /web/tools/puppeteer/
+ *
+ * @author Eric Bidelman <e.bidelman@google.com>
+ */
+
+'use strict';
+
+const gulp = require('gulp');
+const os = require('os');
+const path = require('path');
+const fs = require('fs-extra');
+const jsYaml = require('js-yaml');
+const {promisify} = require('util');
+const semver = require('semver');
+
+const checkoutRepoAtTag = require('../reference-docs/get-source-code');
+const wfHelper = require('../wfHelper');
+
+const readFile = promisify(fs.readFile);
+const ensureDir = promisify(fs.ensureDir);
+const writeFile = promisify(fs.writeFile);
+
+const ORG = 'GoogleChrome';
+const USER = 'puppeteer';
+const GITHUB_PROJECT_URL = `https://github.com/${ORG}/${USER}`;
+const GIT_URL = `${GITHUB_PROJECT_URL}.git`;
+const NUM_RELEASES_TO_INCLUDE = 3;
+
+const DOC_FILES = [
+  'README.md',
+  'docs/troubleshooting.md',
+  'examples/README.md',
+];
+
+const PPTR_FOLDER_PATH = path.join(
+  __dirname, '..', '..', 'src', 'content', 'en', 'tools', 'puppeteer'
+);
+
+/**
+ * Writes a file to disk.
+ * @param {string} outputFile Path to output file
+ * @param {string} content File content to write.
+ */
+async function saveFile(outputFile, content) {
+  const filePath = path.join(PPTR_FOLDER_PATH, '_src', outputFile);
+  await ensureDir(path.dirname(filePath));
+  await writeFile(filePath, content);
+}
+
+/**
+ * Updates _toc.yaml file with latest releases and saves file to disk.
+ * @param {string} tocYaml Path to toc yaml file.
+ * @param {!Array<string>} tags
+ */
+async function generateTOC(tocYaml, tags) {
+  const items = tags.map((tag, i) => {
+    const obj = {
+      title: tag,
+      section: [{
+        title: 'Docs',
+        path: `${GITHUB_PROJECT_URL}/blob/${tag}/docs/api.md`,
+        status: 'external',
+      }, {
+        title: 'Release Notes',
+        path: `${GITHUB_PROJECT_URL}/releases/tag/${tag}`,
+        status: 'external',
+      }],
+    };
+    // Latest release.
+    if (i === 0) {
+      obj.status = 'new';
+    }
+    return obj;
+  });
+
+  items.push({
+    title: 'Old releases',
+    path: `${GITHUB_PROJECT_URL}/releases`,
+    status: 'external',
+  });
+
+  const filePath = path.join(PPTR_FOLDER_PATH, '_src', 'api_section.yaml');
+  await writeFile(filePath, jsYaml.safeDump({toc: items}));
+}
+
+/**
+ * Extracts a devsite START/END region from file.
+ * @param {string} content
+ * @param {string} region Name of region to pull out.
+ * @return {string|null}
+ */
+function extractRegion(content, region) {
+  const re = new RegExp(
+      `<!-- \\[START ${region}\\] -->([\\s\\S]*)<!-- \\[END ${region}\\] -->`);
+  const m = content.match(re);
+  return m && m[0] || null;
+}
+
+gulp.task('puppeteer:build', async () => {
+  const result = await wfHelper.promisedExec(
+      `git ls-remote --tags ${GITHUB_PROJECT_URL}`, '.');
+  const refs = result.split('\n');
+  const tags = refs.map((ref) => ref.split('refs/tags/')[1])
+      .sort(semver.rcompare).slice(0, NUM_RELEASES_TO_INCLUDE);
+  const latestTag = tags[0];
+
+  const tmpDir = path.join(os.tmpdir(), Date.now().toString(), latestTag);
+  await checkoutRepoAtTag(GIT_URL, latestTag, tmpDir);
+
+  // Extract sections from README and create separate .md files for them.
+  const readme = DOC_FILES.shift();
+  let content = await readFile(path.join(tmpDir, readme), {encoding: 'utf-8'});
+  await saveFile('_index/badges.md', extractRegion(content, 'badges'));
+  await saveFile('_index/usecases.md', extractRegion(content, 'usecases'));
+  await saveFile('_index/getstarted.md', extractRegion(content, 'getstarted'));
+  await saveFile('_index/debugging.md', extractRegion(content, 'debugging'));
+  await saveFile(
+      '_index/runtimesettings.md', extractRegion(content, 'runtimesettings'));
+  await saveFile('_index/faq.md', extractRegion(content, 'faq'));
+
+  for (const iFile of DOC_FILES) {
+    let content = await readFile(path.join(tmpDir, iFile), {encoding: 'utf-8'});
+    await saveFile(iFile, content);
+  }
+
+  await generateTOC(path.join(PPTR_FOLDER_PATH, '_toc.yaml'), tags);
+});

gulp-tasks/test.js

@@ -679,16 +679,26 @@ function testMarkdown(filename, contents, options) {
 
     // Verify all {% includecode %} elements work properly
     matched = wfRegEx.getMatches(wfRegEx.RE_INCLUDE_CODE, contents);
-    matched.forEach(function(match) {
+    matched.forEach((match) => {
       const msg = 'IncludeCode widget -';
       const widget = match[0];
       const position = {line: getLineNumber(contents, match.index)};
+
       const inclFile = wfRegEx.getMatch(wfRegEx.RE_INCLUDE_CODE_PATH, widget);
-      if (inclFile.indexOf('web/') !== 0) {
-        logError(filename, position, `${msg} path must start with 'web/'`);
+      if (inclFile) {
+        if (inclFile.indexOf('web/') !== 0) {
+          logError(filename, position, `${msg} path must start with 'web/'`);
+        }
+        if (doesFileExist(inclFile) !== true) {
+          logError(filename, position, `${msg} file not found: '${inclFile}'`);
+        }
       }
-      if (doesFileExist(inclFile) !== true) {
-        logError(filename, position, `${msg} file not found: '${inclFile}'`);
+
+      const githubFile = wfRegEx.getMatch(
+          wfRegEx.RE_INCLUDE_CODE_GITHUB_PATH, widget);
+      if (githubFile && githubFile.includes('web/')) {
+        logError(filename, position,
+            `${msg} github_path must reference a file on github`);
       }
     });
 

gulp-tasks/wfRegEx.js

@@ -42,6 +42,7 @@ const RE_INCLUDES = /{%\s?include (["|']?)(.+?)(["|']?)\s?%}/gm;
 
 const RE_INCLUDE_CODE = /{% includecode .*?%}/gm;
 const RE_INCLUDE_CODE_PATH = /content_path=["']?(.*?)["' ]/;
+const RE_INCLUDE_CODE_GITHUB_PATH = /github_path=["']?(.*?)["' ]/;
 
 const RE_SINGLE_LINE_COMMENT = /^{%\s?comment\s?%}.*{%\s?endcomment\s?%}$/gm;
 
@@ -111,6 +112,7 @@ exports.RE_INCLUDE_MD = RE_INCLUDE_MD;
 exports.RE_INCLUDE_FILE = RE_INCLUDE_FILE;
 exports.RE_INCLUDE_CODE = RE_INCLUDE_CODE;
 exports.RE_INCLUDE_CODE_PATH = RE_INCLUDE_CODE_PATH;
+exports.RE_INCLUDE_CODE_GITHUB_PATH = RE_INCLUDE_CODE_GITHUB_PATH;
 exports.RE_SINGLE_LINE_COMMENT = RE_SINGLE_LINE_COMMENT;
 exports.RE_AUTO_GENERATED = RE_AUTO_GENERATED;
 exports.RE_DEVSITE_TRANSLATION = RE_DEVSITE_TRANSLATION;

gulpfile.js

@@ -13,6 +13,7 @@ const requireDir = require('require-dir');
 
 requireDir('./gulp-tasks');
 requireDir('./gulp-tasks/workbox');
+requireDir('./gulp-tasks/puppeteer');
 
 gutil.log('---------------------------------');
 gutil.log(`${chalk.dim('Web')}${chalk.bold('Fundamentals')} Gulp`);
@@ -126,6 +127,7 @@ gulp.task('clean', function() {
     'src/content/*/shows/**/feed.xml',
     'src/content/*/shows/http203/podcast/index.md',
     'src/content/*/shows/designer-vs-developer/podcast/index.md',
+    'src/content/*/tools/puppeteer/_src/**/*',
     'src/content/*/updates/_index.yaml',
     'src/content/*/updates/*/index.md',
     'src/content/*/updates/tags/*',

src/content/en/showcase/2017/eleme.md

@@ -10,6 +10,7 @@ book_path: /web/showcase/_book.yaml
 {# wf_region: asia #}
 {# wf_vertical: retail #}
 {# wf_featured_date: 2017-09-27 #}
+{# wf_blink_components: N/A #}
 
 # Ele.me improves performance load times with a multi-page Progressive Web App {: .page-title }
 

src/content/en/tools/_book.yaml

@@ -19,6 +19,9 @@ upper_tabs:
       - name: Lighthouse
         contents:
         - include: /web/tools/lighthouse/_toc.yaml
+      - name: Puppeteer
+        contents:
+        - include: /web/tools/puppeteer/_toc.yaml
       - name: Workbox
         contents:
         - include: /web/tools/workbox/_toc.yaml

src/content/en/tools/_index.yaml

@@ -21,12 +21,24 @@ landing_page:
     - heading: Lighthouse
       description: >
         Lighthouse is an open-source, automated tool for improving the quality
-        of your web apps. Run it as a Chrome Extension or from the command line.
+        of your web apps. It is integrated directly into the Chrome DevTools
+        Audits panel. You can also run Lighthouse from the command line or
+        install the Chrome Extension.
       image_path: images/lighthouse-16x9.png
       path: /web/tools/lighthouse/
       buttons:
       - label: Get Started
         path: /web/tools/lighthouse/
+    - heading: Puppeteer
+      description: >
+        Puppeteer is a Node library which provides a high-level API to control
+        headless Chrome over the DevTools Protocol. It can also be configured
+        to use full (non-headless) Chrome or Chromium.
+      image_path: images/puppeteer-16x9.png
+      path: /web/tools/puppeteer/
+      buttons:
+      - label: Learn more
+        path: /web/tools/puppeteer/
     - heading: Workbox
       description: >
         Workbox is a set of service worker libraries and tools that make it

src/content/en/tools/puppeteer/_shared/styles.html

@@ -0,0 +1,5 @@
+<style>
+  .page-title {
+    display: none;
+  }
+</style>

src/content/en/tools/puppeteer/_toc.yaml

@@ -0,0 +1,50 @@
+toc:
+  - heading: Get Started
+  - title: Overview
+    path: /web/tools/puppeteer/
+  - title: Get Started
+    path: /web/tools/puppeteer/get-started
+  - title: API Reference
+    style: accordion
+    section:
+     - include: /web/tools/puppeteer/_src/api_section.yaml
+  - title: Examples
+    path: /web/tools/puppeteer/examples
+
+  - heading: Resources
+  - title: Debugging tips
+    path: /web/tools/puppeteer/debugging
+  - title: Troubleshooting
+    path: /web/tools/puppeteer/troubleshooting
+  - title: Headless Chrome
+    style: accordian
+    section:
+      - title: Getting Started with Headless Chrome
+        path: /web/updates/2017/04/headless-chrome
+      - title: Automated testing with Headless Chrome
+        path: /web/updates/2017/06/headless-karma-mocha-chai
+  - title: DevTools Protocol APIs
+    path: 'https://chromedevtools.github.io/devtools-protocol/'
+    status: external
+  - title: Try Puppeteer!
+    path: 'https://try-puppeteer.appspot.com/'
+    status: external
+
+  - heading: Project
+  - title: NPM
+    path: 'https://www.npmjs.com/package/puppeteer'
+    status: external
+  - title: Github
+    path: 'https://github.com/GoogleChrome/puppeteer'
+    status: external
+  - title: Report an issue
+    path: 'https://github.com/GoogleChrome/puppeteer/issues/new'
+    status: external
+  - title: Releases
+    path: 'https://github.com/GoogleChrome/puppeteer/releases'
+    status: external
+  - title: How to Contribute
+    path: 'https://github.com/GoogleChrome/puppeteer/blob/master/CONTRIBUTING.md'
+    status: external
+  - title: FAQ
+    path: /web/tools/puppeteer/faq