From 25327b5392fc86033452ab7383578a754045690e Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Fri, 12 Aug 2022 11:04:43 +0200 Subject: [PATCH 1/3] fix(goto): bypass CSP when is necessary --- packages/goto/src/index.js | 10 ++++++++ packages/goto/test/index.js | 50 +++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 packages/goto/test/index.js diff --git a/packages/goto/src/index.js b/packages/goto/src/index.js index 391c1ace33..e6dae2e64c 100644 --- a/packages/goto/src/index.js +++ b/packages/goto/src/index.js @@ -238,6 +238,16 @@ module.exports = ({ }) } + if (modules || scripts || styles) { + prePromises.push( + run({ + fn: page.setBypassCSP(true), + timeout: actionTimeout, + debug: 'bypassCSP' + }) + ) + } + if (adblock) { prePromises.push( run({ diff --git a/packages/goto/test/index.js b/packages/goto/test/index.js new file mode 100644 index 0000000000..6a74eb088c --- /dev/null +++ b/packages/goto/test/index.js @@ -0,0 +1,50 @@ +'use strict' + +const { initBrowserless } = require('@browserless/test/util') +const test = require('ava') + +const browserlessFactory = initBrowserless({ evasions: false }) + +test('setup `scripts`', async t => { + const browserless = await browserlessFactory.createContext() + + t.teardown(browserless.destroyContext) + + const getVersion = browserless.evaluate(async page => page.evaluate('jQuery.fn.jquery')) + + const version = await getVersion('https://github.com', { + scripts: ['https://code.jquery.com/jquery-3.5.0.min.js'] + }) + + t.is(version, '3.5.0') +}) + +test('setup `modules`', async t => { + const browserless = await browserlessFactory.createContext() + + t.teardown(browserless.destroyContext) + + const getVersion = browserless.evaluate(async page => page.evaluate('jQuery.fn.jquery')) + + const version = await getVersion('https://github.com', { + modules: ['https://code.jquery.com/jquery-3.5.0.min.js'] + }) + + t.is(version, '3.5.0') +}) + +test('setup `styles`', async t => { + const browserless = await browserlessFactory.createContext() + + t.teardown(browserless.destroyContext) + + const getStyle = browserless.evaluate(async page => + page.evaluate('window.getComputedStyle(document.body).fontFamily') + ) + + const style = await getStyle('https://github.com', { + styles: ['https://cdn.jsdelivr.net/npm/bootstrap@3.4.1/dist/css/bootstrap.min.css'] + }) + + t.is(style, '"Helvetica Neue", Helvetica, Arial, sans-serif') +}) From b94d4e4c28bfc1d3c2c0ee73cef6f19a3a3fe476 Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Fri, 12 Aug 2022 11:09:03 +0200 Subject: [PATCH 2/3] test: run in serial --- packages/goto/test/e2e/evasions.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/goto/test/e2e/evasions.js b/packages/goto/test/e2e/evasions.js index 941ea511a1..a3ef8c8e84 100644 --- a/packages/goto/test/e2e/evasions.js +++ b/packages/goto/test/e2e/evasions.js @@ -15,7 +15,7 @@ test.skip('arh.antoinevastel.com/bots/areyouheadless', async t => { t.true(content.includes('You are not Chrome headless')) }) -test('fingerprintjs', async t => { +test.serial('fingerprintjs', async t => { const getFingerprint = async userAgent => { const browserless = await browserlessFactory.createContext() const fingerprint = await browserless.evaluate(page => From 645665e1b8972fa551ad2c8adfc6cdb512f5a714 Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Fri, 12 Aug 2022 11:40:05 +0200 Subject: [PATCH 3/3] test(goto): give a chance to resolve evasions --- packages/goto/package.json | 2 +- packages/goto/src/index.js | 20 ++++---- packages/goto/test/e2e/evasions.js | 78 ++++++++++++++++++------------ packages/goto/test/index.js | 4 +- 4 files changed, 61 insertions(+), 43 deletions(-) diff --git a/packages/goto/package.json b/packages/goto/package.json index 6d47eb523a..d241e46deb 100644 --- a/packages/goto/package.json +++ b/packages/goto/package.json @@ -45,9 +45,9 @@ }, "devDependencies": { "@browserless/test": "latest", + "p-wait-for": "3", "ava": "3", "browserless": "latest", - "exit-hook": "2", "fpscanner": "latest" }, "engines": { diff --git a/packages/goto/src/index.js b/packages/goto/src/index.js index e6dae2e64c..46283d7cbd 100644 --- a/packages/goto/src/index.js +++ b/packages/goto/src/index.js @@ -222,6 +222,16 @@ module.exports = ({ const prePromises = [] + if (modules || scripts || styles) { + prePromises.push( + run({ + fn: page.setBypassCSP(true), + timeout: actionTimeout, + debug: 'bypassCSP' + }) + ) + } + if (abortTypes.length > 0) { await page.setRequestInterception(true) page.on('request', req => { @@ -238,16 +248,6 @@ module.exports = ({ }) } - if (modules || scripts || styles) { - prePromises.push( - run({ - fn: page.setBypassCSP(true), - timeout: actionTimeout, - debug: 'bypassCSP' - }) - ) - } - if (adblock) { prePromises.push( run({ diff --git a/packages/goto/test/e2e/evasions.js b/packages/goto/test/e2e/evasions.js index a3ef8c8e84..7f81713f43 100644 --- a/packages/goto/test/e2e/evasions.js +++ b/packages/goto/test/e2e/evasions.js @@ -1,45 +1,63 @@ 'use strict' +const { getBrowserless } = require('@browserless/test/util') const fpscanner = require('fpscanner') +const pWaitFor = require('p-wait-for') const test = require('ava') -const browserlessFactory = require('browserless')() -const exitHook = require('exit-hook') +const browserlessFactory = getBrowserless() -exitHook(browserlessFactory.close) +test('arh.antoinevastel.com/bots/areyouheadless', async t => { + let assertion = false -test.skip('arh.antoinevastel.com/bots/areyouheadless', async t => { - const browserless = await browserlessFactory.createContext() - t.teardown(() => browserless.destroyContext()) - const content = await browserless.text('https://arh.antoinevastel.com/bots/areyouheadless') - t.true(content.includes('You are not Chrome headless')) -}) - -test.serial('fingerprintjs', async t => { - const getFingerprint = async userAgent => { + const fn = async () => { const browserless = await browserlessFactory.createContext() - const fingerprint = await browserless.evaluate(page => - page.evaluate("document.querySelector('.giant').innerText") - ) + const content = await browserless.text('https://arh.antoinevastel.com/bots/areyouheadless') + await browserless.destroyContext() + assertion = content.includes('You are not Chrome headless') + return assertion + } - const hash = await fingerprint('https://fingerprintjs.github.io/fingerprintjs/', { - headers: { - 'user-agent': userAgent - } - }) + await pWaitFor(fn) - await browserless.destroyContext() - return hash + t.true(assertion) +}) + +test('fingerprintjs', async t => { + let assertion = false + + const fn = async () => { + const getFingerprint = async userAgent => { + const browserless = await browserlessFactory.createContext() + const fingerprint = await browserless.evaluate(page => + page.evaluate("document.querySelector('.giant').innerText") + ) + + const hash = await fingerprint('https://fingerprintjs.github.io/fingerprintjs/', { + headers: { + 'user-agent': userAgent + } + }) + + await browserless.destroyContext() + return hash + } + + const [fingerprintOne, fingerprintTwo] = await Promise.all([ + await getFingerprint( + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Safari/605.1.15' + ), + await getFingerprint( + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:102.0) Gecko/20100101 Firefox/102.0' + ) + ]) + + assertion = fingerprintOne !== fingerprintTwo + return assertion } - t.not( - await getFingerprint( - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Safari/605.1.15' - ), - await getFingerprint( - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:102.0) Gecko/20100101 Firefox/102.0' - ) - ) + await pWaitFor(fn) + t.true(assertion) }) test('fpscanner', async t => { diff --git a/packages/goto/test/index.js b/packages/goto/test/index.js index 6a74eb088c..5eefda7562 100644 --- a/packages/goto/test/index.js +++ b/packages/goto/test/index.js @@ -1,9 +1,9 @@ 'use strict' -const { initBrowserless } = require('@browserless/test/util') +const { getBrowserless } = require('@browserless/test/util') const test = require('ava') -const browserlessFactory = initBrowserless({ evasions: false }) +const browserlessFactory = getBrowserless() test('setup `scripts`', async t => { const browserless = await browserlessFactory.createContext()