diff --git a/.eslintrc b/.eslintrc index b7f66ff4ff0..09c099e0cfd 100644 --- a/.eslintrc +++ b/.eslintrc @@ -1,7 +1,7 @@ { "extends": ["eslint:recommended", "google"], "parserOptions": { - "ecmaVersion": 6 + "ecmaVersion": 2017 }, "env": { "node": true, diff --git a/.gitignore b/.gitignore index 2099f5fe633..e4513ee99b5 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,7 @@ src/content/*/showcase/_index.yaml src/content/*/showcase/*/_toc.yaml src/content/*/showcase/*/index.md src/content/*/showcase/tags/* +src/content/*/tools/puppeteer/_src/* src/content/*/rss.xml src/content/*/atom.xml src/data/codelabs/*/img/* diff --git a/.travis.yml b/.travis.yml index c5c0d5b1e92..8e7521d451a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,6 +18,7 @@ install: - npm run clean script: - npm run build + - gulp puppeteer:build - npm run test - node ./tools/travis/addCommitComment.js after_success: diff --git a/devsiteHelper.py b/devsiteHelper.py index f4fc8ca3b73..1c422c3fd27 100644 --- a/devsiteHelper.py +++ b/devsiteHelper.py @@ -108,7 +108,7 @@ def readFile(requestedFile, lang='en'): return None -def fetchGithubFile(path): +def fetchGithubFile(path, revision = None): """Fetchs a file in a repository hosted on github.com. Retrieves rows pertaining to the given keys from the Table instance @@ -118,11 +118,15 @@ def fetchGithubFile(path): Args: path: The path to a file on Github in the form github-user/repository/path/to/file. + revision: Optional git revision string of the form + "refs/tags/v0.10.0". See https://goo.gl/DBLwBb for other options. Returns: A string from the HTTP response. """ path = path.replace('/blob', '') + if revision: + path = path.replace('master', revision.replace('refs/tags/', '')) url = 'https://raw.githubusercontent.com/%s' % path try: response = urllib2.urlopen(url) @@ -419,12 +423,14 @@ def getIncludeCode(include_tag, lang='en'): region_regex = re.search(r"region_tag=\"(.+?)\"", include_tag) dedent_regex = re.search(r"adjust_indentation=\"(.+?)\"", include_tag) github_regex = re.search(r"github_path=\"(.+?)\"", include_tag) + git_revision_regex = re.search(r"git_revision=\"(.+?)\"", include_tag) + file_content_regex = re.search(r"regexp=\"(.+?)\"", include_tag) # TODO: support these arguments as_downloadable_regex = re.search(r"as_downloadable=\"(.+?)\"", include_tag) github_link_regex = re.search(r"github_link=\"(.+?)\"", include_tag) - git_revision_regex = re.search(r"git_revision=\"(.+?)\"", include_tag) - if as_downloadable_regex or github_link_regex or git_revision_regex: + + if as_downloadable_regex or github_link_regex: msg = 'Error: as_downloadable, github_link, and git_revision args are not supported' logging.error(' - ' + msg) return msg @@ -441,9 +447,12 @@ def getIncludeCode(include_tag, lang='en'): if result is None: return 'Warning: Unable to find includecode %s' % file_name elif github_regex: + git_revision = None + if git_revision_regex: + git_revision = git_revision_regex.group(1) file_url = github_regex.group(1) - result = fetchGithubFile(file_url) + result = fetchGithubFile(file_url, git_revision) if result is None: return 'Warning: Unable to includecode from github_path="%s"' % file_url @@ -456,6 +465,13 @@ def getIncludeCode(include_tag, lang='en'): end_at = result.rfind('\n', start_at, end_at) result = result[start_at:end_at] + if file_content_regex: + print file_content_regex.groups() + r = re.compile(file_content_regex.group(1), re.DOTALL) + m = re.search(r, result) + if m: + result = m.group(1) + if dedent_regex and dedent_regex.group(1) == 'auto': result = textwrap.dedent(result) diff --git a/gulp-tasks/puppeteer/build_docs.js b/gulp-tasks/puppeteer/build_docs.js new file mode 100644 index 00000000000..0d6bdcd8425 --- /dev/null +++ b/gulp-tasks/puppeteer/build_docs.js @@ -0,0 +1,128 @@ +/** + * @fileoverview Generates the Puppeteer docs at /web/tools/puppeteer/ + * + * @author Eric Bidelman + */ + +'use strict'; + +const gulp = require('gulp'); +const os = require('os'); +const path = require('path'); +const fs = require('fs-extra'); +const jsYaml = require('js-yaml'); +const {promisify} = require('util'); +const semver = require('semver'); + +const checkoutRepoAtTag = require('../reference-docs/get-source-code'); +const wfHelper = require('../wfHelper'); + +const readFile = promisify(fs.readFile); +const ensureDir = promisify(fs.ensureDir); +const writeFile = promisify(fs.writeFile); + +const ORG = 'GoogleChrome'; +const USER = 'puppeteer'; +const GITHUB_PROJECT_URL = `https://github.com/${ORG}/${USER}`; +const GIT_URL = `${GITHUB_PROJECT_URL}.git`; +const NUM_RELEASES_TO_INCLUDE = 3; + +const DOC_FILES = [ + 'README.md', + 'docs/troubleshooting.md', + 'examples/README.md', +]; + +const PPTR_FOLDER_PATH = path.join( + __dirname, '..', '..', 'src', 'content', 'en', 'tools', 'puppeteer' +); + +/** + * Writes a file to disk. + * @param {string} outputFile Path to output file + * @param {string} content File content to write. + */ +async function saveFile(outputFile, content) { + const filePath = path.join(PPTR_FOLDER_PATH, '_src', outputFile); + await ensureDir(path.dirname(filePath)); + await writeFile(filePath, content); +} + +/** + * Updates _toc.yaml file with latest releases and saves file to disk. + * @param {string} tocYaml Path to toc yaml file. + * @param {!Array} tags + */ +async function generateTOC(tocYaml, tags) { + const items = tags.map((tag, i) => { + const obj = { + title: tag, + section: [{ + title: 'Docs', + path: `${GITHUB_PROJECT_URL}/blob/${tag}/docs/api.md`, + status: 'external', + }, { + title: 'Release Notes', + path: `${GITHUB_PROJECT_URL}/releases/tag/${tag}`, + status: 'external', + }], + }; + // Latest release. + if (i === 0) { + obj.status = 'new'; + } + return obj; + }); + + items.push({ + title: 'Old releases', + path: `${GITHUB_PROJECT_URL}/releases`, + status: 'external', + }); + + const filePath = path.join(PPTR_FOLDER_PATH, '_src', 'api_section.yaml'); + await writeFile(filePath, jsYaml.safeDump({toc: items})); +} + +/** + * Extracts a devsite START/END region from file. + * @param {string} content + * @param {string} region Name of region to pull out. + * @return {string|null} + */ +function extractRegion(content, region) { + const re = new RegExp( + `([\\s\\S]*)`); + const m = content.match(re); + return m && m[0] || null; +} + +gulp.task('puppeteer:build', async () => { + const result = await wfHelper.promisedExec( + `git ls-remote --tags ${GITHUB_PROJECT_URL}`, '.'); + const refs = result.split('\n'); + const tags = refs.map((ref) => ref.split('refs/tags/')[1]) + .sort(semver.rcompare).slice(0, NUM_RELEASES_TO_INCLUDE); + const latestTag = tags[0]; + + const tmpDir = path.join(os.tmpdir(), Date.now().toString(), latestTag); + await checkoutRepoAtTag(GIT_URL, latestTag, tmpDir); + + // Extract sections from README and create separate .md files for them. + const readme = DOC_FILES.shift(); + let content = await readFile(path.join(tmpDir, readme), {encoding: 'utf-8'}); + await saveFile('_index/badges.md', extractRegion(content, 'badges')); + await saveFile('_index/usecases.md', extractRegion(content, 'usecases')); + await saveFile('_index/getstarted.md', extractRegion(content, 'getstarted')); + await saveFile('_index/debugging.md', extractRegion(content, 'debugging')); + await saveFile( + '_index/runtimesettings.md', extractRegion(content, 'runtimesettings')); + await saveFile('_index/faq.md', extractRegion(content, 'faq')); + + for (const iFile of DOC_FILES) { + let content = await readFile(path.join(tmpDir, iFile), {encoding: 'utf-8'}); + await saveFile(iFile, content); + } + + await generateTOC(path.join(PPTR_FOLDER_PATH, '_toc.yaml'), tags); +}); diff --git a/gulp-tasks/test.js b/gulp-tasks/test.js index 6642e453e0f..b7ef1fed01f 100644 --- a/gulp-tasks/test.js +++ b/gulp-tasks/test.js @@ -679,16 +679,26 @@ function testMarkdown(filename, contents, options) { // Verify all {% includecode %} elements work properly matched = wfRegEx.getMatches(wfRegEx.RE_INCLUDE_CODE, contents); - matched.forEach(function(match) { + matched.forEach((match) => { const msg = 'IncludeCode widget -'; const widget = match[0]; const position = {line: getLineNumber(contents, match.index)}; + const inclFile = wfRegEx.getMatch(wfRegEx.RE_INCLUDE_CODE_PATH, widget); - if (inclFile.indexOf('web/') !== 0) { - logError(filename, position, `${msg} path must start with 'web/'`); + if (inclFile) { + if (inclFile.indexOf('web/') !== 0) { + logError(filename, position, `${msg} path must start with 'web/'`); + } + if (doesFileExist(inclFile) !== true) { + logError(filename, position, `${msg} file not found: '${inclFile}'`); + } } - if (doesFileExist(inclFile) !== true) { - logError(filename, position, `${msg} file not found: '${inclFile}'`); + + const githubFile = wfRegEx.getMatch( + wfRegEx.RE_INCLUDE_CODE_GITHUB_PATH, widget); + if (githubFile && githubFile.includes('web/')) { + logError(filename, position, + `${msg} github_path must reference a file on github`); } }); diff --git a/gulp-tasks/wfRegEx.js b/gulp-tasks/wfRegEx.js index 96ff987d1bb..fc0ceaca3e2 100644 --- a/gulp-tasks/wfRegEx.js +++ b/gulp-tasks/wfRegEx.js @@ -42,6 +42,7 @@ const RE_INCLUDES = /{%\s?include (["|']?)(.+?)(["|']?)\s?%}/gm; const RE_INCLUDE_CODE = /{% includecode .*?%}/gm; const RE_INCLUDE_CODE_PATH = /content_path=["']?(.*?)["' ]/; +const RE_INCLUDE_CODE_GITHUB_PATH = /github_path=["']?(.*?)["' ]/; const RE_SINGLE_LINE_COMMENT = /^{%\s?comment\s?%}.*{%\s?endcomment\s?%}$/gm; @@ -111,6 +112,7 @@ exports.RE_INCLUDE_MD = RE_INCLUDE_MD; exports.RE_INCLUDE_FILE = RE_INCLUDE_FILE; exports.RE_INCLUDE_CODE = RE_INCLUDE_CODE; exports.RE_INCLUDE_CODE_PATH = RE_INCLUDE_CODE_PATH; +exports.RE_INCLUDE_CODE_GITHUB_PATH = RE_INCLUDE_CODE_GITHUB_PATH; exports.RE_SINGLE_LINE_COMMENT = RE_SINGLE_LINE_COMMENT; exports.RE_AUTO_GENERATED = RE_AUTO_GENERATED; exports.RE_DEVSITE_TRANSLATION = RE_DEVSITE_TRANSLATION; diff --git a/gulpfile.js b/gulpfile.js index 962cb7be4ec..979a07e1d57 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -13,6 +13,7 @@ const requireDir = require('require-dir'); requireDir('./gulp-tasks'); requireDir('./gulp-tasks/workbox'); +requireDir('./gulp-tasks/puppeteer'); gutil.log('---------------------------------'); gutil.log(`${chalk.dim('Web')}${chalk.bold('Fundamentals')} Gulp`); @@ -126,6 +127,7 @@ gulp.task('clean', function() { 'src/content/*/shows/**/feed.xml', 'src/content/*/shows/http203/podcast/index.md', 'src/content/*/shows/designer-vs-developer/podcast/index.md', + 'src/content/*/tools/puppeteer/_src/**/*', 'src/content/*/updates/_index.yaml', 'src/content/*/updates/*/index.md', 'src/content/*/updates/tags/*', diff --git a/src/content/en/showcase/2017/eleme.md b/src/content/en/showcase/2017/eleme.md index 9ab918d4cac..715d58162ff 100644 --- a/src/content/en/showcase/2017/eleme.md +++ b/src/content/en/showcase/2017/eleme.md @@ -10,6 +10,7 @@ book_path: /web/showcase/_book.yaml {# wf_region: asia #} {# wf_vertical: retail #} {# wf_featured_date: 2017-09-27 #} +{# wf_blink_components: N/A #} # Ele.me improves performance load times with a multi-page Progressive Web App {: .page-title } diff --git a/src/content/en/tools/_book.yaml b/src/content/en/tools/_book.yaml index 0035874c28b..a0a246329a5 100644 --- a/src/content/en/tools/_book.yaml +++ b/src/content/en/tools/_book.yaml @@ -19,6 +19,9 @@ upper_tabs: - name: Lighthouse contents: - include: /web/tools/lighthouse/_toc.yaml + - name: Puppeteer + contents: + - include: /web/tools/puppeteer/_toc.yaml - name: Workbox contents: - include: /web/tools/workbox/_toc.yaml diff --git a/src/content/en/tools/_index.yaml b/src/content/en/tools/_index.yaml index a1ba6a38fe8..b2122d0d343 100644 --- a/src/content/en/tools/_index.yaml +++ b/src/content/en/tools/_index.yaml @@ -21,12 +21,24 @@ landing_page: - heading: Lighthouse description: > Lighthouse is an open-source, automated tool for improving the quality - of your web apps. Run it as a Chrome Extension or from the command line. + of your web apps. It is integrated directly into the Chrome DevTools + Audits panel. You can also run Lighthouse from the command line or + install the Chrome Extension. image_path: images/lighthouse-16x9.png path: /web/tools/lighthouse/ buttons: - label: Get Started path: /web/tools/lighthouse/ + - heading: Puppeteer + description: > + Puppeteer is a Node library which provides a high-level API to control + headless Chrome over the DevTools Protocol. It can also be configured + to use full (non-headless) Chrome or Chromium. + image_path: images/puppeteer-16x9.png + path: /web/tools/puppeteer/ + buttons: + - label: Learn more + path: /web/tools/puppeteer/ - heading: Workbox description: > Workbox is a set of service worker libraries and tools that make it diff --git a/src/content/en/tools/images/puppeteer-16x9.png b/src/content/en/tools/images/puppeteer-16x9.png new file mode 100644 index 00000000000..79df239ac88 Binary files /dev/null and b/src/content/en/tools/images/puppeteer-16x9.png differ diff --git a/src/content/en/tools/images/puppeteer.png b/src/content/en/tools/images/puppeteer.png new file mode 100644 index 00000000000..0163931af33 Binary files /dev/null and b/src/content/en/tools/images/puppeteer.png differ diff --git a/src/content/en/tools/puppeteer/_shared/styles.html b/src/content/en/tools/puppeteer/_shared/styles.html new file mode 100644 index 00000000000..32cd47cc000 --- /dev/null +++ b/src/content/en/tools/puppeteer/_shared/styles.html @@ -0,0 +1,5 @@ + diff --git a/src/content/en/tools/puppeteer/_toc.yaml b/src/content/en/tools/puppeteer/_toc.yaml new file mode 100644 index 00000000000..fa2f1a88b50 --- /dev/null +++ b/src/content/en/tools/puppeteer/_toc.yaml @@ -0,0 +1,50 @@ +toc: + - heading: Get Started + - title: Overview + path: /web/tools/puppeteer/ + - title: Get Started + path: /web/tools/puppeteer/get-started + - title: API Reference + style: accordion + section: + - include: /web/tools/puppeteer/_src/api_section.yaml + - title: Examples + path: /web/tools/puppeteer/examples + + - heading: Resources + - title: Debugging tips + path: /web/tools/puppeteer/debugging + - title: Troubleshooting + path: /web/tools/puppeteer/troubleshooting + - title: Headless Chrome + style: accordian + section: + - title: Getting Started with Headless Chrome + path: /web/updates/2017/04/headless-chrome + - title: Automated testing with Headless Chrome + path: /web/updates/2017/06/headless-karma-mocha-chai + - title: DevTools Protocol APIs + path: 'https://chromedevtools.github.io/devtools-protocol/' + status: external + - title: Try Puppeteer! + path: 'https://try-puppeteer.appspot.com/' + status: external + + - heading: Project + - title: NPM + path: 'https://www.npmjs.com/package/puppeteer' + status: external + - title: Github + path: 'https://github.com/GoogleChrome/puppeteer' + status: external + - title: Report an issue + path: 'https://github.com/GoogleChrome/puppeteer/issues/new' + status: external + - title: Releases + path: 'https://github.com/GoogleChrome/puppeteer/releases' + status: external + - title: How to Contribute + path: 'https://github.com/GoogleChrome/puppeteer/blob/master/CONTRIBUTING.md' + status: external + - title: FAQ + path: /web/tools/puppeteer/faq diff --git a/src/content/en/tools/puppeteer/debugging.md b/src/content/en/tools/puppeteer/debugging.md new file mode 100644 index 00000000000..61d0238144f --- /dev/null +++ b/src/content/en/tools/puppeteer/debugging.md @@ -0,0 +1,13 @@ +project_path: /web/tools/_project.yaml +book_path: /web/tools/_book.yaml +description: Helpful tips for debugging Puppeteer + +{# wf_updated_on: 2018-01-10 #} +{# wf_published_on: 2018-01-10 #} +{# wf_blink_components: Internals>Headless #} + +{% include "web/tools/puppeteer/_shared/styles.html" %} + +# Debugging {: .page-title } + +<<_src/_index/debugging.md>> diff --git a/src/content/en/tools/puppeteer/examples.md b/src/content/en/tools/puppeteer/examples.md new file mode 100644 index 00000000000..824c238efb3 --- /dev/null +++ b/src/content/en/tools/puppeteer/examples.md @@ -0,0 +1,13 @@ +project_path: /web/tools/_project.yaml +book_path: /web/tools/_book.yaml +description: Puppeteer examples + +{# wf_updated_on: 2018-01-10 #} +{# wf_published_on: 2018-01-10 #} +{# wf_blink_components: Internals>Headless #} + +{% include "web/tools/puppeteer/_shared/styles.html" %} + +# Examples {: .page-title } + +<<_src/examples/README.md>> \ No newline at end of file diff --git a/src/content/en/tools/puppeteer/faq.md b/src/content/en/tools/puppeteer/faq.md new file mode 100644 index 00000000000..3cddca5c36f --- /dev/null +++ b/src/content/en/tools/puppeteer/faq.md @@ -0,0 +1,13 @@ +project_path: /web/tools/_project.yaml +book_path: /web/tools/_book.yaml +description: FAQ + +{# wf_updated_on: 2018-01-10 #} +{# wf_published_on: 2018-01-10 #} +{# wf_blink_components: Internals>Headless #} + +{% include "web/tools/puppeteer/_shared/styles.html" %} + +# FAQ {: .page-title } + +<<_src/_index/faq.md>> \ No newline at end of file diff --git a/src/content/en/tools/puppeteer/get-started.md b/src/content/en/tools/puppeteer/get-started.md new file mode 100644 index 00000000000..04c5c4d662b --- /dev/null +++ b/src/content/en/tools/puppeteer/get-started.md @@ -0,0 +1,19 @@ +project_path: /web/tools/_project.yaml +book_path: /web/tools/_book.yaml +description: Learn how to use Puppeteer to automate and test apps in headless Chrome. + +{# wf_updated_on: 2018-01-10 #} +{# wf_published_on: 2018-01-10 #} +{# wf_blink_components: Internals>Headless #} + +# Quick start {: .page-title } + +<<_src/_index/getstarted.md>> + +<<_src/_index/runtimesettings.md>> + +## Next steps {: #next-steps } + +* Learn more about [headless Chrome](/web/updates/2017/04/headless-chrome) +* Look over the [examples](./examples). + diff --git a/src/content/en/tools/puppeteer/index.md b/src/content/en/tools/puppeteer/index.md new file mode 100644 index 00000000000..d24a21ef9f4 --- /dev/null +++ b/src/content/en/tools/puppeteer/index.md @@ -0,0 +1,35 @@ +project_path: /web/tools/_project.yaml +book_path: /web/tools/_book.yaml +description: Overview of Puppeteer + +{# wf_updated_on: 2018-01-11 #} +{# wf_published_on: 2018-01-11 #} +{# wf_blink_components: Internals>Headless #} + +# Puppeteer {: .page-title } + +<<_src/_index/badges.md>> + + + +Puppeteer is a Node library which provides a high-level API to control +[headless](/web/updates/2017/04/headless-chrome) Chrome or Chromium over the +[DevTools Protocol](https://chromedevtools.github.io/devtools-protocol/). It +can also be configured to use full (non-headless) Chrome or Chromium. +{: .objective } + + + +<<_src/_index/usecases.md>> + +## Next steps {: #next-steps } + +* Install Puppeteer and [Get Started](./get-started)! +* Look over the [examples](./examples). + diff --git a/src/content/en/tools/puppeteer/troubleshooting.md b/src/content/en/tools/puppeteer/troubleshooting.md new file mode 100644 index 00000000000..8bc81709efc --- /dev/null +++ b/src/content/en/tools/puppeteer/troubleshooting.md @@ -0,0 +1,13 @@ +project_path: /web/tools/_project.yaml +book_path: /web/tools/_book.yaml +description: Troubleshooting Guide + +{# wf_updated_on: 2018-01-10 #} +{# wf_published_on: 2018-01-10 #} +{# wf_blink_components: Internals>Headless #} + +{% include "web/tools/puppeteer/_shared/styles.html" %} + +# Troubleshooting {: .page-title } + +<<_src/docs/troubleshooting.md>> \ No newline at end of file diff --git a/src/content/en/updates/2017/04/headless-chrome.md b/src/content/en/updates/2017/04/headless-chrome.md index 6c225a52c0b..2f6271403dc 100644 --- a/src/content/en/updates/2017/04/headless-chrome.md +++ b/src/content/en/updates/2017/04/headless-chrome.md @@ -142,7 +142,7 @@ commands going across the wire, communicating with the browser. ### The Puppeteer API {: #puppeteer } -[Puppeteer](https://github.com/GoogleChrome/puppeteer) is a Node library +[Puppeteer](/web/tools/puppeteer/) is a Node library developed by the Chrome team. It provides a high-level API to control headless (or full) Chrome. It's similar to other automated testing libraries like Phantom and NightmareJS, but it only works with the latest versions of Chrome. @@ -165,7 +165,7 @@ const puppeteer = require('puppeteer'); (async() => { const browser = await puppeteer.launch(); console.log(await browser.version()); - browser.close(); + await browser.close(); })(); ``` @@ -175,13 +175,12 @@ const puppeteer = require('puppeteer'); const puppeteer = require('puppeteer'); (async() => { - const browser = await puppeteer.launch(); const page = await browser.newPage(); -await page.goto('https://www.chromestatus.com', {waitUntil: 'networkidle'}); +await page.goto('https://www.chromestatus.com', {waitUntil: 'networkidle2'}); await page.pdf({path: 'page.pdf', format: 'A4'}); -browser.close(); +await browser.close(); })(); ```