feat: add iframe expansion to parseWithCheerio in browsers (#2542)

Replaces the `iframe` elements with their contents in `<div class="crawlee-iframe-replacement"></div>` element. Closes #2507
apify · Jun 20, 2024 · 328d085 · 328d085
1 parent 39cf673
commit 328d085
Show file tree

Hide file tree

Showing 6 changed files with 112 additions and 0 deletions.
diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.ts
@@ -342,6 +342,7 @@ export abstract class BrowserCrawler<
         persistCookiesPerSession: ow.optional.boolean,
         useSessionPool: ow.optional.boolean,
         proxyConfiguration: ow.optional.object.validate(validators.proxyConfiguration),
+        ignoreShadowRoots: ow.optional.boolean,
     };
 
     /**
@@ -370,6 +371,7 @@ export abstract class BrowserCrawler<
             failedRequestHandler,
             handleFailedRequestFunction,
             headless,
+            ignoreShadowRoots,
             ...basicCrawlerOptions
         } = options;
 

diff --git a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts b/packages/playwright-crawler/src/internals/utils/playwright-utils.ts
@@ -600,6 +600,28 @@ export async function saveSnapshot(page: Page, options: SaveSnapshotOptions = {}
 export async function parseWithCheerio(page: Page, ignoreShadowRoots = false): Promise<CheerioRoot> {
     ow(page, ow.object.validate(validators.browserPage));
 
+    if (page.frames().length > 1) {
+        const frames = await page.$$('iframe');
+
+        await Promise.all(
+            frames.map(async (frame) => {
+                const iframe = await frame.contentFrame();
+
+                if (iframe) {
+                    const contents = await iframe.content();
+
+                    await frame.evaluate((f, c) => {
+                        const replacementNode = document.createElement('div');
+                        replacementNode.innerHTML = c;
+                        replacementNode.className = 'crawlee-iframe-replacement';
+
+                        f.replaceWith(replacementNode);
+                    }, contents);
+                }
+            }),
+        );
+    }
+
     const html = ignoreShadowRoots
         ? null
         : ((await page.evaluate(`(${expandShadowRoots.toString()})(document)`)) as string);

diff --git a/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts b/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts
@@ -191,6 +191,28 @@ export async function injectJQuery(page: Page, options?: { surviveNavigations?:
 export async function parseWithCheerio(page: Page, ignoreShadowRoots = false): Promise<CheerioRoot> {
     ow(page, ow.object.validate(validators.browserPage));
 
+    if (page.frames().length > 1) {
+        const frames = await page.$$('iframe');
+
+        await Promise.all(
+            frames.map(async (frame) => {
+                const iframe = await frame.contentFrame();
+
+                if (iframe) {
+                    const contents = await iframe.content();
+
+                    await frame.evaluate((f, c) => {
+                        const replacementNode = document.createElement('div');
+                        replacementNode.innerHTML = c;
+                        replacementNode.className = 'crawlee-iframe-replacement';
+
+                        f.replaceWith(replacementNode);
+                    }, contents);
+                }
+            }),
+        );
+    }
+
     const html = ignoreShadowRoots
         ? null
         : ((await page.evaluate(`(${expandShadowRoots.toString()})(document)`)) as string);

diff --git a/test/core/playwright_utils.test.ts b/test/core/playwright_utils.test.ts
@@ -159,6 +159,24 @@ describe('playwrightUtils', () => {
         }
     });
 
+    test('parseWithCheerio() iframe expansion works', async () => {
+        const browser = await launchPlaywright(launchContext);
+
+        try {
+            const page = await browser.newPage();
+            await page.goto(new URL('/special/outside-iframe', serverAddress).toString());
+
+            const $ = await playwrightUtils.parseWithCheerio(page);
+
+            const headings = $('h1')
+                .map((i, el) => $(el).text())
+                .get();
+            expect(headings).toEqual(['Outside iframe', 'In iframe']);
+        } finally {
+            await browser.close();
+        }
+    });
+
     describe('blockRequests()', () => {
         let browser: Browser = null;
         beforeAll(async () => {

diff --git a/test/core/puppeteer_utils.test.ts b/test/core/puppeteer_utils.test.ts
@@ -160,6 +160,24 @@ describe('puppeteerUtils', () => {
             }
         });
 
+        test('parseWithCheerio() iframe expansion works', async () => {
+            const browser = await launchPuppeteer(launchContext);
+
+            try {
+                const page = await browser.newPage();
+                await page.goto(new URL('/special/outside-iframe', serverAddress).toString());
+
+                const $ = await puppeteerUtils.parseWithCheerio(page);
+
+                const headings = $('h1')
+                    .map((i, el) => $(el).text())
+                    .get();
+                expect(headings).toEqual(['Outside iframe', 'In iframe']);
+            } finally {
+                await browser.close();
+            }
+        });
+
         describe('blockRequests()', () => {
             let browser: Browser = null;
             beforeAll(async () => {

diff --git a/test/shared/_helper.ts b/test/shared/_helper.ts
@@ -172,6 +172,28 @@ console.log('Hello world!');
     </div>
 </body>
 </html>`,
+    outsideIframe: `
+    <!DOCTYPE html>
+    <html>
+        <head>
+            <title>Outside iframe</title>
+        </head>
+        <body>
+            <h1>Outside iframe</h1>
+            <iframe src="./inside-iframe"></iframe>
+        </body>
+    </html>`,
+    insideIframe: `
+    <!DOCTYPE html>
+    <html>
+        <head>
+            <title>In iframe</title>
+        </head>
+        <body>
+            <h1>In iframe</h1>
+            <p>Some content from inside of an iframe.</p>
+        </body>
+    </html>`,
 };
 
 export async function runExampleComServer(): Promise<[Server, number]> {
@@ -268,6 +290,14 @@ export async function runExampleComServer(): Promise<[Server, number]> {
         special.get('/cloudflareBlocking', async (_req, res) => {
             res.type('html').status(403).send(responseSamples.cloudflareBlocking);
         });
+
+        special.get('/outside-iframe', (_req, res) => {
+            res.type('html').send(responseSamples.outsideIframe);
+        });
+
+        special.get('/inside-iframe', (_req, res) => {
+            res.type('html').send(responseSamples.insideIframe);
+        });
     })();
 
     // "cacheable" site with one page, scripts and stylesheets