feat(frame): introduce Frame.goto and Frame.waitForNavigation

This patch introduces API to manage frame navigations.
As a drive-by, the `response.frame()` method is added as a shortcut
for `response.request().frame()`.

Fixes puppeteer#2918.

docs/api.md

@@ -194,6 +194,7 @@
   * [frame.evaluateHandle(pageFunction, ...args)](#frameevaluatehandlepagefunction-args)
   * [frame.executionContext()](#frameexecutioncontext)
   * [frame.focus(selector)](#framefocusselector)
+  * [frame.goto(url, options)](#framegotourl-options)
   * [frame.hover(selector)](#framehoverselector)
   * [frame.isDetached()](#frameisdetached)
   * [frame.name()](#framename)
@@ -206,6 +207,7 @@
   * [frame.url()](#frameurl)
   * [frame.waitFor(selectorOrFunctionOrTimeout[, options[, ...args]])](#framewaitforselectororfunctionortimeout-options-args)
   * [frame.waitForFunction(pageFunction[, options[, ...args]])](#framewaitforfunctionpagefunction-options-args)
+  * [frame.waitForNavigation(options)](#framewaitfornavigationoptions)
   * [frame.waitForSelector(selector[, options])](#framewaitforselectorselector-options)
   * [frame.waitForXPath(xpath[, options])](#framewaitforxpathxpath-options)
 - [class: ExecutionContext](#class-executioncontext)
@@ -261,6 +263,7 @@
   * [request.url()](#requesturl)
 - [class: Response](#class-response)
   * [response.buffer()](#responsebuffer)
+  * [response.frame()](#responseframe)
   * [response.fromCache()](#responsefromcache)
   * [response.fromServiceWorker()](#responsefromserviceworker)
   * [response.headers()](#responseheaders)
@@ -1801,9 +1804,10 @@ This resolves when the page navigates to a new URL or reloads. It is useful for
 which will indirectly cause the page to navigate. Consider this example:
 
 ```js
-const navigationPromise = page.waitForNavigation();
-await page.click('a.my-link'); // Clicking the link will indirectly cause a navigation
-await navigationPromise; // The navigationPromise resolves after navigation has finished
+const [response] = await Promise.all([
+  page.waitForNavigation(), // The promise resolves after navigation has finished
+  page.click('a.my-link'), // Clicking the link will indirectly cause a navigation
+]);
 ```
 
 **NOTE** Usage of the [History API](https://developer.mozilla.org/en-US/docs/Web/API/History_API) to change the URL is considered a navigation.
@@ -2368,6 +2372,29 @@ Returns promise that resolves to the frame's default execution context.
 This method fetches an element with `selector` and focuses it.
 If there's no element matching `selector`, the method throws an error.
 
+#### frame.goto(url, options)
+- `url` <[string]> URL to navigate frame to. The url should include scheme, e.g. `https://`.
+- `options` <[Object]> Navigation parameters which might have the following properties:
+  - `timeout` <[number]> Maximum navigation time in milliseconds, defaults to 30 seconds, pass `0` to disable timeout. The default value can be changed by using the [page.setDefaultNavigationTimeout(timeout)](#pagesetdefaultnavigationtimeouttimeout) method.
+  - `waitUntil` <[string]|[Array]<[string]>> When to consider navigation succeeded, defaults to `load`. Given an array of event strings, navigation is considered to be successful after all events have been fired. Events can be either:
+    - `load` - consider navigation to be finished when the `load` event is fired.
+    - `domcontentloaded` - consider navigation to be finished when the `DOMContentLoaded` event is fired.
+    - `networkidle0` - consider navigation to be finished when there are no more than 0 network connections for at least `500` ms.
+    - `networkidle2` - consider navigation to be finished when there are no more than 2 network connections for at least `500` ms.
+  - `referer` <[string]> Referer header value. If provided it will take preference over the referer header value set by [page.setExtraHTTPHeaders()](#pagesetextrahttpheadersheaders).
+- returns: <[Promise]<?[Response]>> Promise which resolves to the main resource response. In case of multiple redirects, the navigation will resolve with the response of the last redirect.
+
+The `frame.goto` will throw an error if:
+- there's an SSL error (e.g. in case of self-signed certificates).
+- target URL is invalid.
+- the `timeout` is exceeded during navigation.
+- the main resource failed to load.
+
+> **NOTE** `frame.goto` either throw or return a main resource response. The only exceptions are navigation to `about:blank` or navigation to the same URL with a different hash, which would succeed and return `null`.
+
+> **NOTE** Headless mode doesn't match any nodest navigating to a PDF document. See the [upstream issue](https://bugs.chromium.org/p/chromium/issues/detail?id=761295).
+
+
 #### frame.hover(selector)
 - `selector` <[string]> A [selector] to search for element to hover. If there are multiple elements satisfying the selector, the first will be hovered.
 - returns: <[Promise]> Promise which resolves when the element matching `selector` is successfully hovered. Promise gets rejected if there's no element matching `selector`.
@@ -2499,6 +2526,29 @@ const selector = '.foo';
 await page.waitForFunction(selector => !!document.querySelector(selector), {}, selector);
 ```
 
+#### frame.waitForNavigation(options)
+- `options` <[Object]> Navigation parameters which might have the following properties:
+  - `timeout` <[number]> Maximum navigation time in milliseconds, defaults to 30 seconds, pass `0` to disable timeout. The default value can be changed by using the [page.setDefaultNavigationTimeout(timeout)](#pagesetdefaultnavigationtimeouttimeout) method.
+  - `waitUntil` <[string]|[Array]<[string]>> When to consider navigation succeeded, defaults to `load`. Given an array of event strings, navigation is considered to be successful after all events have been fired. Events can be either:
+    - `load` - consider navigation to be finished when the `load` event is fired.
+    - `domcontentloaded` - consider navigation to be finished when the `DOMContentLoaded` event is fired.
+    - `networkidle0` - consider navigation to be finished when there are no more than 0 network connections for at least `500` ms.
+    - `networkidle2` - consider navigation to be finished when there are no more than 2 network connections for at least `500` ms.
+- returns: <[Promise]<[?Response]>> Promise which resolves to the main resource response. In case of multiple redirects, the navigation will resolve with the response of the last redirect. In case of navigation to a different anchor or navigation due to History API usage, the navigation will resolve with `null`.
+
+This resolves when the frame navigates to a new URL. It is useful for when you run code
+which will indirectly cause the frame to navigate. Consider this example:
+
+```js
+const [response] = await Promise.all([
+  frame.waitForNavigation(), // The navigation promise resolves after navigation has finished
+  frame.click('a.my-link'), // Clicking the link will indirectly cause a navigation
+]);
+```
+
+**NOTE** Usage of the [History API](https://developer.mozilla.org/en-US/docs/Web/API/History_API) to change the URL is considered a navigation.
+
+
 #### frame.waitForSelector(selector[, options])
 - `selector` <[string]> A [selector] of an element to wait for
 - `options` <[Object]> Optional waiting parameters
@@ -2989,7 +3039,7 @@ page.on('requestfailed', request => {
 ```
 
 #### request.frame()
-- returns: <?[Frame]> A matching [Frame] object, or `null` if navigating to error pages.
+- returns: <?[Frame]> A [Frame] that initiated this request, or `null` if navigating to error pages.
 
 #### request.headers()
 - returns: <[Object]> An object with HTTP headers associated with the request. All header names are lower-case.
@@ -3079,6 +3129,9 @@ page.on('request', request => {
 #### response.buffer()
 - returns: <Promise<[Buffer]>> Promise which resolves to a buffer with response body.
 
+#### response.frame()
+- returns: <?[Frame]> A [Frame] that initiated this response, or `null` if navigating to error pages.
+
 #### response.fromCache()
 - returns: <[boolean]>
 

lib/FrameManager.js

@@ -75,7 +75,7 @@ class FrameManager extends EventEmitter {
     const watcher = new NavigatorWatcher(this._client, this, this._networkManager, frame, timeout, options);
     let ensureNewDocumentNavigation = false;
     let error = await Promise.race([
-      navigate(this._client, url, referrer),
+      navigate(this._client, url, referrer, frame._id),
       watcher.timeoutOrTerminationPromise(),
     ]);
     if (!error) {
@@ -93,11 +93,12 @@ class FrameManager extends EventEmitter {
      * @param {!Puppeteer.CDPSession} client
      * @param {string} url
      * @param {string} referrer
+     * @param {string} frameId
      * @return {!Promise<?Error>}
      */
-    async function navigate(client, url, referrer) {
+    async function navigate(client, url, referrer, frameId) {
       try {
-        const response = await client.send('Page.navigate', {url, referrer});
+        const response = await client.send('Page.navigate', {url, referrer, frameId});
         ensureNewDocumentNavigation = !!response.loaderId;
         return response.errorText ? new Error(`${response.errorText} at ${url}`) : null;
       } catch (error) {
@@ -394,6 +395,23 @@ class Frame {
     }
   }
 
+  /**
+   * @param {string} url
+   * @param {!Object=} options
+   * @return {!Promise<?Puppeteer.Response>}
+   */
+  async goto(url, options = {}) {
+    return await this._frameManager.navigateFrame(this, url, options);
+  }
+
+  /**
+   * @param {!Object=} options
+   * @return {!Promise<?Puppeteer.Response>}
+   */
+  async waitForNavigation(options = {}) {
+    return await this._frameManager.waitForFrameNavigation(this, options);
+  }
+
   /**
    * @return {!Promise<!ExecutionContext>}
    */
@@ -1128,7 +1146,7 @@ class NavigatorWatcher {
       helper.addEventListener(Connection.fromSession(client), Connection.Events.Disconnected, () => this._terminate(new Error('Navigation failed because browser has disconnected!'))),
       helper.addEventListener(this._frameManager, FrameManager.Events.LifecycleEvent, this._checkLifecycleComplete.bind(this)),
       helper.addEventListener(this._frameManager, FrameManager.Events.FrameNavigatedWithinDocument, this._navigatedWithinDocument.bind(this)),
-      helper.addEventListener(this._frameManager, FrameManager.Events.FrameDetached, this._checkLifecycleComplete.bind(this)),
+      helper.addEventListener(this._frameManager, FrameManager.Events.FrameDetached, this._onFrameDetached.bind(this)),
       helper.addEventListener(this._networkManager, NetworkManager.Events.Request, this._onRequest.bind(this)),
     ];
 
@@ -1155,6 +1173,17 @@ class NavigatorWatcher {
     this._navigationRequest = request;
   }
 
+  /**
+   * @param {!Puppeteer.Frame} frame
+   */
+  _onFrameDetached(frame) {
+    if (this._frame === frame) {
+      this._terminationCallback.call(null, new Error('Navigating frame is detached'));
+      return;
+    }
+    this._checkLifecycleComplete();
+  }
+
   /**
    * @return {?Puppeteer.Response}
    */

lib/NetworkManager.js

@@ -629,6 +629,13 @@ class Response {
   fromServiceWorker() {
     return this._fromServiceWorker;
   }
+
+  /**
+   * @return {?Puppeteer.Frame}
+   */
+  frame() {
+    return this._request.frame();
+  }
 }
 helper.tracePublicAPI(Response);
 

lib/Page.js

@@ -576,8 +576,7 @@ class Page extends EventEmitter {
    * @return {!Promise<?Puppeteer.Response>}
    */
   async goto(url, options = {}) {
-    const mainFrame = this._frameManager.mainFrame();
-    return await this._frameManager.navigateFrame(mainFrame, url, options);
+    return await this._frameManager.mainFrame().goto(url, options);
   }
 
   /**
@@ -597,8 +596,7 @@ class Page extends EventEmitter {
    * @return {!Promise<?Puppeteer.Response>}
    */
   async waitForNavigation(options = {}) {
-    const mainFrame = this._frameManager.mainFrame();
-    return await this._frameManager.waitForFrameNavigation(mainFrame, options);
+    return await this._frameManager.mainFrame().waitForNavigation(options);
   }
 
   /**

test/frame.spec.js

@@ -22,7 +22,7 @@ module.exports.addTests = function({testRunner, expect}) {
   const {it, fit, xit} = testRunner;
   const {beforeAll, beforeEach, afterAll, afterEach} = testRunner;
 
-  describe('Frame.context', function() {
+  describe('Frame.executionContext', function() {
     it('should work', async({page, server}) => {
       await page.goto(server.EMPTY_PAGE);
       await utils.attachFrame(page, 'frame1', server.EMPTY_PAGE);
@@ -49,6 +49,84 @@ module.exports.addTests = function({testRunner, expect}) {
     });
   });
 
+  describe('Frame.goto', function() {
+    it('should navigate subframes', async({page, server}) => {
+      await page.goto(server.PREFIX + '/frames/one-frame.html');
+      expect(page.frames()[0].url()).toContain('/frames/one-frame.html');
+      expect(page.frames()[1].url()).toContain('/frames/frame.html');
+
+      const response = await page.frames()[1].goto(server.EMPTY_PAGE);
+      expect(response.ok()).toBe(true);
+      expect(response.frame()).toBe(page.frames()[1]);
+    });
+    it('should reject when frame detaches', async({page, server}) => {
+      await page.goto(server.PREFIX + '/frames/one-frame.html');
+
+      server.setRoute('/empty.html', () => {});
+      const navigationPromise = page.frames()[1].goto(server.EMPTY_PAGE).catch(e => e);
+      await server.waitForRequest('/empty.html');
+
+      await page.$eval('iframe', frame => frame.remove());
+      const error = await navigationPromise;
+      expect(error.message).toBe('Navigating frame is detached');
+    });
+    it('should return matching responses', async({page, server}) => {
+      // Disable cache: otherwise, chromium will cache similar requests.
+      await page.setCacheEnabled(false);
+      await page.goto(server.EMPTY_PAGE);
+      // Attach three frames.
+      const frames = await Promise.all([
+        utils.attachFrame(page, 'frame1', server.EMPTY_PAGE),
+        utils.attachFrame(page, 'frame2', server.EMPTY_PAGE),
+        utils.attachFrame(page, 'frame3', server.EMPTY_PAGE),
+      ]);
+      // Navigate all frames to the same URL.
+      const serverResponses = [];
+      server.setRoute('/one-style.html', (req, res) => serverResponses.push(res));
+      const navigations = [];
+      for (let i = 0; i < 3; ++i) {
+        navigations.push(frames[i].goto(server.PREFIX + '/one-style.html'));
+        await server.waitForRequest('/one-style.html');
+      }
+      // Respond from server out-of-order.
+      const serverResponseTexts = ['AAA', 'BBB', 'CCC'];
+      for (const i of [1, 2, 0]) {
+        serverResponses[i].end(serverResponseTexts[i]);
+        const response = await navigations[i];
+        expect(response.frame()).toBe(frames[i]);
+        expect(await response.text()).toBe(serverResponseTexts[i]);
+      }
+    });
+  });
+
+  describe('Frame.waitForNavigation', function() {
+    it('should work', async({page, server}) => {
+      await page.goto(server.PREFIX + '/frames/one-frame.html');
+      const frame = page.frames()[1];
+      const [response] = await Promise.all([
+        frame.waitForNavigation(),
+        frame.evaluate(url => window.location.href = url, server.PREFIX + '/grid.html')
+      ]);
+      expect(response.ok()).toBe(true);
+      expect(response.url()).toContain('grid.html');
+      expect(response.frame()).toBe(frame);
+      expect(page.url()).toContain('/frames/one-frame.html');
+    });
+    it('should reject when frame detaches', async({page, server}) => {
+      await page.goto(server.PREFIX + '/frames/one-frame.html');
+      const frame = page.frames()[1];
+
+      server.setRoute('/empty.html', () => {});
+      const navigationPromise = frame.waitForNavigation();
+      await Promise.all([
+        server.waitForRequest('/empty.html'),
+        frame.evaluate(() => window.location = '/empty.html')
+      ]);
+      await page.$eval('iframe', frame => frame.remove());
+      await navigationPromise;
+    });
+  });
+
   describe('Frame.evaluateHandle', function() {
     it('should work', async({page, server}) => {
       await page.goto(server.EMPTY_PAGE);

test/page.spec.js

@@ -790,11 +790,10 @@ module.exports.addTests = function({testRunner, expect, headless}) {
   describe('Page.waitForNavigation', function() {
     it('should work', async({page, server}) => {
       await page.goto(server.EMPTY_PAGE);
-      const [result] = await Promise.all([
+      const [response] = await Promise.all([
         page.waitForNavigation(),
         page.evaluate(url => window.location.href = url, server.PREFIX + '/grid.html')
       ]);
-      const response = await result;
       expect(response.ok()).toBe(true);
       expect(response.url()).toContain('grid.html');
     });

test/server/SimpleServer.js

@@ -193,8 +193,10 @@ class SimpleServer {
       }
     }
     // Notify request subscriber.
-    if (this._requestSubscribers.has(pathName))
+    if (this._requestSubscribers.has(pathName)) {
       this._requestSubscribers.get(pathName)[fulfillSymbol].call(null, request);
+      this._requestSubscribers.delete(pathName);
+    }
     const handler = this._routes.get(pathName);
     if (handler) {
       handler.call(null, request, response);

test/utils.js

@@ -37,16 +37,19 @@ const utils = module.exports = {
    * @param {!Page} page
    * @param {string} frameId
    * @param {string} url
+   * @return {!Puppeteer.Frame}
    */
   attachFrame: async function(page, frameId, url) {
-    await page.evaluate(attachFrame, frameId, url);
+    const handle = await page.evaluateHandle(attachFrame, frameId, url);
+    return await handle.asElement().contentFrame();
 
-    function attachFrame(frameId, url) {
+    async function attachFrame(frameId, url) {
       const frame = document.createElement('iframe');
       frame.src = url;
       frame.id = frameId;
       document.body.appendChild(frame);
-      return new Promise(x => frame.onload = x);
+      await new Promise(x => frame.onload = x);
+      return frame;
     }
   },