From 733a7472d988c7fe66d6d747fc6e6382a9c38c0b Mon Sep 17 00:00:00 2001 From: Randolf Date: Wed, 23 Aug 2023 18:06:10 +0200 Subject: [PATCH] chore: implement `ElementHandle.prototype.clickablePoint` --- .../puppeteer-core/src/api/ElementHandle.ts | 137 ++++++++++------ .../src/common/ElementHandle.ts | 149 ------------------ .../src/common/IsolatedWorld.ts | 14 +- test/TestExpectations.json | 30 ++-- test/src/elementhandle.spec.ts | 6 +- 5 files changed, 120 insertions(+), 216 deletions(-) diff --git a/packages/puppeteer-core/src/api/ElementHandle.ts b/packages/puppeteer-core/src/api/ElementHandle.ts index 571aa8d43a57b..d9495e8f3a436 100644 --- a/packages/puppeteer-core/src/api/ElementHandle.ts +++ b/packages/puppeteer-core/src/api/ElementHandle.ts @@ -630,9 +630,21 @@ export abstract class ElementHandle< /** * Returns the middle point within an element unless a specific offset is provided. */ - async clickablePoint(offset?: Offset): Promise; - async clickablePoint(): Promise { - throw new Error('Not implemented'); + async clickablePoint(offset?: Offset): Promise { + const box = await this.#clickableBox(); + if (!box) { + throw new Error('Node is either not clickable or not an Element'); + } + if (offset !== undefined) { + return { + x: box.x + offset.x, + y: box.y + offset.y, + }; + } + return { + x: box.x + box.width / 2, + y: box.y + box.height / 2, + }; } /** @@ -861,6 +873,35 @@ export abstract class ElementHandle< options?: Readonly ): Promise; + async #clickableBox(): Promise { + const adoptedThis = await this.frame.isolatedRealm().adoptHandle(this); + const box = await adoptedThis.evaluate(element => { + if (!(element instanceof Element)) { + return null; + } + return element.getClientRects()[0]?.toJSON() as DOMRect; + }); + void adoptedThis.dispose().catch(debugError); + if (!box) { + return null; + } + if (box.width * box.height <= 1) { + return null; + } + const offset = await this.#getTopLeftCornerOfFrame(); + if (!offset) { + return null; + } + box.x += offset.x; + box.y += offset.y; + return { + x: box.x, + y: box.y, + height: box.height, + width: box.width, + }; + } + /** * This method returns the bounding box of the element (relative to the main frame), * or `null` if the element is not visible. @@ -876,12 +917,7 @@ export abstract class ElementHandle< return null; } const rect = element.getBoundingClientRect(); - return { - x: rect.left, - y: rect.top, - width: rect.width, - height: rect.height, - }; + return rect.toJSON() as DOMRect; }); void adoptedThis.dispose().catch(debugError); if (!box) { @@ -893,45 +929,12 @@ export abstract class ElementHandle< } box.x += offset.x; box.y += offset.y; - return box; - } - - async #getTopLeftCornerOfFrame() { - const point = {x: 0, y: 0}; - let frame: Frame | null | undefined = this.frame; - let element: HandleFor | null | undefined; - while ((element = await frame?.frameElement())) { - try { - element = await element.frame.isolatedRealm().transferHandle(element); - const parentBox = await element.evaluate(element => { - // Element is not visible. - if (element.getClientRects().length === 0) { - return null; - } - const rect = element.getBoundingClientRect(); - const style = window.getComputedStyle(element); - return { - left: - rect.left + - parseInt(style.paddingLeft, 10) + - parseInt(style.borderLeftWidth, 10), - top: - rect.top + - parseInt(style.paddingTop, 10) + - parseInt(style.borderTopWidth, 10), - }; - }); - if (!parentBox) { - return null; - } - point.x += parentBox.left; - point.y += parentBox.top; - frame = frame?.parentFrame(); - } finally { - void element.dispose().catch(debugError); - } - } - return point; + return { + x: box.x, + y: box.y, + height: box.height, + width: box.width, + }; } /** @@ -1038,6 +1041,44 @@ export abstract class ElementHandle< return model; } + async #getTopLeftCornerOfFrame() { + const point = {x: 0, y: 0}; + let frame: Frame | null | undefined = this.frame; + let element: HandleFor | null | undefined; + while ((element = await frame?.frameElement())) { + try { + element = await element.frame.isolatedRealm().transferHandle(element); + const parentBox = await element.evaluate(element => { + // Element is not visible. + if (element.getClientRects().length === 0) { + return null; + } + const rect = element.getBoundingClientRect(); + const style = window.getComputedStyle(element); + return { + left: + rect.left + + parseInt(style.paddingLeft, 10) + + parseInt(style.borderLeftWidth, 10), + top: + rect.top + + parseInt(style.paddingTop, 10) + + parseInt(style.borderTopWidth, 10), + }; + }); + if (!parentBox) { + return null; + } + point.x += parentBox.left; + point.y += parentBox.top; + frame = frame?.parentFrame(); + } finally { + void element.dispose().catch(debugError); + } + } + return point; + } + /** * This method scrolls element into view if needed, and then uses * {@link Page.(screenshot:3) } to take a screenshot of the element. diff --git a/packages/puppeteer-core/src/common/ElementHandle.ts b/packages/puppeteer-core/src/common/ElementHandle.ts index ba8414442f273..5b8f2c945a26b 100644 --- a/packages/puppeteer-core/src/common/ElementHandle.ts +++ b/packages/puppeteer-core/src/common/ElementHandle.ts @@ -20,9 +20,7 @@ import { AutofillData, ClickOptions, ElementHandle, - Offset, Point, - Quad, } from '../api/ElementHandle.js'; import {KeyboardTypeOptions, KeyPressOptions} from '../api/Input.js'; import {Page, ScreenshotOptions} from '../api/Page.js'; @@ -34,23 +32,10 @@ import {Frame} from './Frame.js'; import {FrameManager} from './FrameManager.js'; import {WaitForSelectorOptions} from './IsolatedWorld.js'; import {CDPJSHandle} from './JSHandle.js'; -import {CDPPage} from './Page.js'; import {NodeFor} from './types.js'; import {KeyInput} from './USKeyboardLayout.js'; import {debugError} from './util.js'; -const applyOffsetsToQuad = ( - quad: Point[], - offsetX: number, - offsetY: number -) => { - assert(quad.length === 4); - return quad.map(part => { - return {x: part.x + offsetX, y: part.y + offsetY}; - // SAFETY: We know this is a quad from the length check. - }) as Quad; -}; - /** * The CDPElementHandle extends ElementHandle now to keep compatibility * with `instanceof` because of that we need to have methods for @@ -156,127 +141,6 @@ export class CDPElementHandle< } } - async #getOOPIFOffsets( - frame: Frame - ): Promise<{offsetX: number; offsetY: number}> { - let offsetX = 0; - let offsetY = 0; - let currentFrame: Frame | null = frame; - while (currentFrame && currentFrame.parentFrame()) { - const parent = currentFrame.parentFrame(); - if (!currentFrame.isOOPFrame() || !parent) { - currentFrame = parent; - continue; - } - const {backendNodeId} = await parent._client().send('DOM.getFrameOwner', { - frameId: currentFrame._id, - }); - const result = await parent._client().send('DOM.getBoxModel', { - backendNodeId: backendNodeId, - }); - if (!result) { - break; - } - const contentBoxQuad = result.model.content; - const topLeftCorner = this.#fromProtocolQuad(contentBoxQuad)[0]; - offsetX += topLeftCorner!.x; - offsetY += topLeftCorner!.y; - currentFrame = parent; - } - return {offsetX, offsetY}; - } - - override async clickablePoint(offset?: Offset): Promise { - const [result, layoutMetrics] = await Promise.all([ - this.client - .send('DOM.getContentQuads', { - objectId: this.id, - }) - .catch(debugError), - (this.#page as CDPPage)._client().send('Page.getLayoutMetrics'), - ]); - if (!result || !result.quads.length) { - throw new Error('Node is either not clickable or not an HTMLElement'); - } - // Filter out quads that have too small area to click into. - // Fallback to `layoutViewport` in case of using Firefox. - const {clientWidth, clientHeight} = - layoutMetrics.cssLayoutViewport || layoutMetrics.layoutViewport; - const {offsetX, offsetY} = await this.#getOOPIFOffsets(this.#frame); - const quads = result.quads - .map(quad => { - return this.#fromProtocolQuad(quad); - }) - .map(quad => { - return applyOffsetsToQuad(quad, offsetX, offsetY); - }) - .map(quad => { - return this.#intersectQuadWithViewport(quad, clientWidth, clientHeight); - }) - .filter(quad => { - return computeQuadArea(quad) > 1; - }); - if (!quads.length) { - throw new Error('Node is either not clickable or not an HTMLElement'); - } - const quad = quads[0]!; - if (offset) { - // Return the point of the first quad identified by offset. - let minX = Number.MAX_SAFE_INTEGER; - let minY = Number.MAX_SAFE_INTEGER; - for (const point of quad) { - if (point.x < minX) { - minX = point.x; - } - if (point.y < minY) { - minY = point.y; - } - } - if ( - minX !== Number.MAX_SAFE_INTEGER && - minY !== Number.MAX_SAFE_INTEGER - ) { - return { - x: minX + offset.x, - y: minY + offset.y, - }; - } - } - // Return the middle point of the first quad. - let x = 0; - let y = 0; - for (const point of quad) { - x += point.x; - y += point.y; - } - return { - x: x / 4, - y: y / 4, - }; - } - - #fromProtocolQuad(quad: number[]): Point[] { - return [ - {x: quad[0]!, y: quad[1]!}, - {x: quad[2]!, y: quad[3]!}, - {x: quad[4]!, y: quad[5]!}, - {x: quad[6]!, y: quad[7]!}, - ]; - } - - #intersectQuadWithViewport( - quad: Point[], - width: number, - height: number - ): Point[] { - return quad.map(point => { - return { - x: Math.min(Math.max(point.x, 0), width), - y: Math.min(Math.max(point.y, 0), height), - }; - }); - } - /** * This method scrolls element into view if needed, and then * uses {@link Page.mouse} to hover over the center of the element. @@ -532,16 +396,3 @@ export class CDPElementHandle< assert(this.executionContext()._world); } } - -function computeQuadArea(quad: Point[]): number { - /* Compute sum of all directed areas of adjacent triangles - https://en.wikipedia.org/wiki/Polygon#Simple_polygons - */ - let area = 0; - for (let i = 0; i < quad.length; ++i) { - const p1 = quad[i]!; - const p2 = quad[(i + 1) % quad.length]!; - area += (p1.x * p2.y - p2.x * p1.y) / 2; - } - return Math.abs(area); -} diff --git a/packages/puppeteer-core/src/common/IsolatedWorld.ts b/packages/puppeteer-core/src/common/IsolatedWorld.ts index f9c0c1ce58f2c..1ab695524a9c1 100644 --- a/packages/puppeteer-core/src/common/IsolatedWorld.ts +++ b/packages/puppeteer-core/src/common/IsolatedWorld.ts @@ -500,10 +500,16 @@ export class IsolatedWorld implements Realm { async adoptHandle>(handle: T): Promise { const context = await this.executionContext(); - assert( - (handle as unknown as CDPJSHandle).executionContext() !== context, - 'Cannot adopt handle that already belongs to this execution context' - ); + if ( + (handle as unknown as CDPJSHandle).executionContext() === context + ) { + // If the context has already adopted this handle, clone it so downstream + // disposal doesn't become an issue. + return (await handle.evaluateHandle(value => { + return value; + // SAFETY: We know the + })) as unknown as T; + } const nodeInfo = await this.#client.send('DOM.describeNode', { objectId: handle.id, }); diff --git a/test/TestExpectations.json b/test/TestExpectations.json index 76b5429349235..495409d329304 100644 --- a/test/TestExpectations.json +++ b/test/TestExpectations.json @@ -623,6 +623,12 @@ "parameters": ["webDriverBiDi"], "expectations": ["PASS"] }, + { + "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.boundingBox should work", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["FAIL", "PASS"] + }, { "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.boundingBox should work with SVG nodes", "platforms": ["darwin", "linux", "win32"], @@ -665,6 +671,12 @@ "parameters": ["webDriverBiDi"], "expectations": ["PASS"] }, + { + "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.clickablePoint should work", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["webDriverBiDi"], + "expectations": ["PASS"] + }, { "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.contentFrame should work", "platforms": ["darwin", "linux", "win32"], @@ -2075,18 +2087,6 @@ "parameters": ["cdp", "firefox"], "expectations": ["FAIL"] }, - { - "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.boundingBox should work", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["FAIL"] - }, - { - "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.boundingBox should work", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["cdp", "firefox"], - "expectations": ["FAIL", "PASS"] - }, { "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.boxModel should work", "platforms": ["darwin", "linux", "win32"], @@ -2111,6 +2111,12 @@ "parameters": ["chrome", "webDriverBiDi"], "expectations": ["PASS"] }, + { + "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.clickablePoint should work for iframes", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["chrome", "webDriverBiDi"], + "expectations": ["PASS"] + }, { "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.contentFrame should work", "platforms": ["darwin", "linux", "win32"], diff --git a/test/src/elementhandle.spec.ts b/test/src/elementhandle.spec.ts index 60e95a790dc38..b0dcc5b578065 100644 --- a/test/src/elementhandle.spec.ts +++ b/test/src/elementhandle.spec.ts @@ -294,7 +294,7 @@ describe('ElementHandle specs', function () { return error_; }); expect(error.message).atLeastOneToContain([ - 'Node is either not clickable or not an HTMLElement', + 'Node is either not clickable or not an Element', 'no such element', ]); }); @@ -310,7 +310,7 @@ describe('ElementHandle specs', function () { return error_; }); expect(error.message).atLeastOneToContain([ - 'Node is either not clickable or not an HTMLElement', + 'Node is either not clickable or not an Element', 'no such element', ]); }); @@ -323,7 +323,7 @@ describe('ElementHandle specs', function () { return error_; }); expect(error.message).atLeastOneToContain([ - 'Node is either not clickable or not an HTMLElement', + 'Node is either not clickable or not an Element', 'no such node', ]); });