Skip to content

Commit

Permalink
chore: implement ElementHandle.prototype.clickablePoint
Browse files Browse the repository at this point in the history
  • Loading branch information
@jrandolf
jrandolf committed Aug 24, 2023
1 parent 5f8d2a4 commit 733a747
Show file tree
Hide file tree
Showing 5 changed files with 120 additions and 216 deletions.
137 changes: 89 additions & 48 deletions packages/puppeteer-core/src/api/ElementHandle.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -630,9 +630,21 @@ export abstract class ElementHandle<
/**
* Returns the middle point within an element unless a specific offset is provided.
*/
async clickablePoint(offset?: Offset): Promise<Point>;
async clickablePoint(): Promise<Point> {
throw new Error('Not implemented');
async clickablePoint(offset?: Offset): Promise<Point> {
const box = await this.#clickableBox();
if (!box) {
throw new Error('Node is either not clickable or not an Element');
}
if (offset !== undefined) {
return {
x: box.x + offset.x,
y: box.y + offset.y,
};
}
return {
x: box.x + box.width / 2,
y: box.y + box.height / 2,
};
}

/**
Expand Down Expand Up @@ -861,6 +873,35 @@ export abstract class ElementHandle<
options?: Readonly<KeyPressOptions>
): Promise<void>;

async #clickableBox(): Promise<BoundingBox | null> {
const adoptedThis = await this.frame.isolatedRealm().adoptHandle(this);
const box = await adoptedThis.evaluate(element => {
if (!(element instanceof Element)) {
return null;
}
return element.getClientRects()[0]?.toJSON() as DOMRect;
});
void adoptedThis.dispose().catch(debugError);
if (!box) {
return null;
}
if (box.width * box.height <= 1) {
return null;
}
const offset = await this.#getTopLeftCornerOfFrame();
if (!offset) {
return null;
}
box.x += offset.x;
box.y += offset.y;
return {
x: box.x,
y: box.y,
height: box.height,
width: box.width,
};
}

/**
* This method returns the bounding box of the element (relative to the main frame),
* or `null` if the element is not visible.
Expand All @@ -876,12 +917,7 @@ export abstract class ElementHandle<
return null;
}
const rect = element.getBoundingClientRect();
return {
x: rect.left,
y: rect.top,
width: rect.width,
height: rect.height,
};
return rect.toJSON() as DOMRect;
});
void adoptedThis.dispose().catch(debugError);
if (!box) {
Expand All @@ -893,45 +929,12 @@ export abstract class ElementHandle<
}
box.x += offset.x;
box.y += offset.y;
return box;
}

async #getTopLeftCornerOfFrame() {
const point = {x: 0, y: 0};
let frame: Frame | null | undefined = this.frame;
let element: HandleFor<HTMLIFrameElement> | null | undefined;
while ((element = await frame?.frameElement())) {
try {
element = await element.frame.isolatedRealm().transferHandle(element);
const parentBox = await element.evaluate(element => {
// Element is not visible.
if (element.getClientRects().length === 0) {
return null;
}
const rect = element.getBoundingClientRect();
const style = window.getComputedStyle(element);
return {
left:
rect.left +
parseInt(style.paddingLeft, 10) +
parseInt(style.borderLeftWidth, 10),
top:
rect.top +
parseInt(style.paddingTop, 10) +
parseInt(style.borderTopWidth, 10),
};
});
if (!parentBox) {
return null;
}
point.x += parentBox.left;
point.y += parentBox.top;
frame = frame?.parentFrame();
} finally {
void element.dispose().catch(debugError);
}
}
return point;
return {
x: box.x,
y: box.y,
height: box.height,
width: box.width,
};
}

/**
Expand Down Expand Up @@ -1038,6 +1041,44 @@ export abstract class ElementHandle<
return model;
}

async #getTopLeftCornerOfFrame() {
const point = {x: 0, y: 0};
let frame: Frame | null | undefined = this.frame;
let element: HandleFor<HTMLIFrameElement> | null | undefined;
while ((element = await frame?.frameElement())) {
try {
element = await element.frame.isolatedRealm().transferHandle(element);
const parentBox = await element.evaluate(element => {
// Element is not visible.
if (element.getClientRects().length === 0) {
return null;
}
const rect = element.getBoundingClientRect();
const style = window.getComputedStyle(element);
return {
left:
rect.left +
parseInt(style.paddingLeft, 10) +
parseInt(style.borderLeftWidth, 10),
top:
rect.top +
parseInt(style.paddingTop, 10) +
parseInt(style.borderTopWidth, 10),
};
});
if (!parentBox) {
return null;
}
point.x += parentBox.left;
point.y += parentBox.top;
frame = frame?.parentFrame();
} finally {
void element.dispose().catch(debugError);
}
}
return point;
}

/**
* This method scrolls element into view if needed, and then uses
* {@link Page.(screenshot:3) } to take a screenshot of the element.
Expand Down
149 changes: 0 additions & 149 deletions packages/puppeteer-core/src/common/ElementHandle.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,7 @@ import {
AutofillData,
ClickOptions,
ElementHandle,
Offset,
Point,
Quad,
} from '../api/ElementHandle.js';
import {KeyboardTypeOptions, KeyPressOptions} from '../api/Input.js';
import {Page, ScreenshotOptions} from '../api/Page.js';
Expand All @@ -34,23 +32,10 @@ import {Frame} from './Frame.js';
import {FrameManager} from './FrameManager.js';
import {WaitForSelectorOptions} from './IsolatedWorld.js';
import {CDPJSHandle} from './JSHandle.js';
import {CDPPage} from './Page.js';
import {NodeFor} from './types.js';
import {KeyInput} from './USKeyboardLayout.js';
import {debugError} from './util.js';

const applyOffsetsToQuad = (
quad: Point[],
offsetX: number,
offsetY: number
) => {
assert(quad.length === 4);
return quad.map(part => {
return {x: part.x + offsetX, y: part.y + offsetY};
// SAFETY: We know this is a quad from the length check.
}) as Quad;
};

/**
* The CDPElementHandle extends ElementHandle now to keep compatibility
* with `instanceof` because of that we need to have methods for
Expand Down Expand Up @@ -156,127 +141,6 @@ export class CDPElementHandle<
}
}

async #getOOPIFOffsets(
frame: Frame
): Promise<{offsetX: number; offsetY: number}> {
let offsetX = 0;
let offsetY = 0;
let currentFrame: Frame | null = frame;
while (currentFrame && currentFrame.parentFrame()) {
const parent = currentFrame.parentFrame();
if (!currentFrame.isOOPFrame() || !parent) {
currentFrame = parent;
continue;
}
const {backendNodeId} = await parent._client().send('DOM.getFrameOwner', {
frameId: currentFrame._id,
});
const result = await parent._client().send('DOM.getBoxModel', {
backendNodeId: backendNodeId,
});
if (!result) {
break;
}
const contentBoxQuad = result.model.content;
const topLeftCorner = this.#fromProtocolQuad(contentBoxQuad)[0];
offsetX += topLeftCorner!.x;
offsetY += topLeftCorner!.y;
currentFrame = parent;
}
return {offsetX, offsetY};
}

override async clickablePoint(offset?: Offset): Promise<Point> {
const [result, layoutMetrics] = await Promise.all([
this.client
.send('DOM.getContentQuads', {
objectId: this.id,
})
.catch(debugError),
(this.#page as CDPPage)._client().send('Page.getLayoutMetrics'),
]);
if (!result || !result.quads.length) {
throw new Error('Node is either not clickable or not an HTMLElement');
}
// Filter out quads that have too small area to click into.
// Fallback to `layoutViewport` in case of using Firefox.
const {clientWidth, clientHeight} =
layoutMetrics.cssLayoutViewport || layoutMetrics.layoutViewport;
const {offsetX, offsetY} = await this.#getOOPIFOffsets(this.#frame);
const quads = result.quads
.map(quad => {
return this.#fromProtocolQuad(quad);
})
.map(quad => {
return applyOffsetsToQuad(quad, offsetX, offsetY);
})
.map(quad => {
return this.#intersectQuadWithViewport(quad, clientWidth, clientHeight);
})
.filter(quad => {
return computeQuadArea(quad) > 1;
});
if (!quads.length) {
throw new Error('Node is either not clickable or not an HTMLElement');
}
const quad = quads[0]!;
if (offset) {
// Return the point of the first quad identified by offset.
let minX = Number.MAX_SAFE_INTEGER;
let minY = Number.MAX_SAFE_INTEGER;
for (const point of quad) {
if (point.x < minX) {
minX = point.x;
}
if (point.y < minY) {
minY = point.y;
}
}
if (
minX !== Number.MAX_SAFE_INTEGER &&
minY !== Number.MAX_SAFE_INTEGER
) {
return {
x: minX + offset.x,
y: minY + offset.y,
};
}
}
// Return the middle point of the first quad.
let x = 0;
let y = 0;
for (const point of quad) {
x += point.x;
y += point.y;
}
return {
x: x / 4,
y: y / 4,
};
}

#fromProtocolQuad(quad: number[]): Point[] {
return [
{x: quad[0]!, y: quad[1]!},
{x: quad[2]!, y: quad[3]!},
{x: quad[4]!, y: quad[5]!},
{x: quad[6]!, y: quad[7]!},
];
}

#intersectQuadWithViewport(
quad: Point[],
width: number,
height: number
): Point[] {
return quad.map(point => {
return {
x: Math.min(Math.max(point.x, 0), width),
y: Math.min(Math.max(point.y, 0), height),
};
});
}

/**
* This method scrolls element into view if needed, and then
* uses {@link Page.mouse} to hover over the center of the element.
Expand Down Expand Up @@ -532,16 +396,3 @@ export class CDPElementHandle<
assert(this.executionContext()._world);
}
}

function computeQuadArea(quad: Point[]): number {
/* Compute sum of all directed areas of adjacent triangles
https://en.wikipedia.org/wiki/Polygon#Simple_polygons
*/
let area = 0;
for (let i = 0; i < quad.length; ++i) {
const p1 = quad[i]!;
const p2 = quad[(i + 1) % quad.length]!;
area += (p1.x * p2.y - p2.x * p1.y) / 2;
}
return Math.abs(area);
}
14 changes: 10 additions & 4 deletions packages/puppeteer-core/src/common/IsolatedWorld.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -500,10 +500,16 @@ export class IsolatedWorld implements Realm {

async adoptHandle<T extends JSHandle<Node>>(handle: T): Promise<T> {
const context = await this.executionContext();
assert(
(handle as unknown as CDPJSHandle<Node>).executionContext() !== context,
'Cannot adopt handle that already belongs to this execution context'
);
if (
(handle as unknown as CDPJSHandle<Node>).executionContext() === context
) {
// If the context has already adopted this handle, clone it so downstream
// disposal doesn't become an issue.
return (await handle.evaluateHandle(value => {
return value;
// SAFETY: We know the
})) as unknown as T;
}
const nodeInfo = await this.#client.send('DOM.describeNode', {
objectId: handle.id,
});
Expand Down

0 comments on commit 733a747

Please sign in to comment.