Skip to content

Commit

Permalink
refactor(goto): wrap & run
Browse files Browse the repository at this point in the history
  • Loading branch information
@Kikobeats
Kikobeats committed Oct 17, 2021
1 parent 980f39f commit a95f2a8
Show file tree
Hide file tree
Showing 5 changed files with 136 additions and 90 deletions.
2 changes: 1 addition & 1 deletion packages/browserless/src/driver.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ const defaultArgs = [
'--disable-features=AudioServiceOutOfProcess,IsolateOrigins,site-per-process', // https://source.chromium.org/search?q=file:content_features.cc&ss=chromium
'--disable-gesture-typing',
'--disable-infobars',
'--disable-notifications',
// '--disable-notifications', // BUG: headless cause weird behavrior https://bugs.chromium.org/p/chromium/issues/detail?id=1052332
'--disable-offer-store-unmasked-wallet-cards',
'--disable-offer-upload-credit-cards',
'--disable-print-preview', // https://source.chromium.org/search?q=lang:cpp+symbol:kDisablePrintPreview&ss=chromium
Expand Down
204 changes: 124 additions & 80 deletions packages/goto/src/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,17 @@ const isUrl = require('is-url-http')
const path = require('path')
const fs = require('fs')

const wrap = (fn, timeout) => pReflect(pTimeout(fn, timeout))

const run = async ({ fn, timeout, debug: props }) => {
const debugProps = { duration: timeSpan() }
const result = await wrap(fn, timeout)
debugProps.duration = prettyMs(debugProps.duration())
if (result.isRejected) debugProps.error = result.reason.message || result.reason
debug(props, debugProps)
return result
}

const EVASIONS = require('./evasions')

const ALL_EVASIONS_KEYS = Object.keys(EVASIONS)
Expand All @@ -29,16 +40,6 @@ const isEmpty = val => val == null || !(Object.keys(val) || val).length

const castArray = value => [].concat(value).filter(Boolean)

const getInjectKey = (ext, value) =>
isUrl(value) ? 'url' : value.endsWith(`.${ext}`) ? 'path' : 'content'

const injectCSS = (page, css) =>
pReflect(
page.addStyleTag({
content: css
})
)

const parseCookies = (url, str) =>
str.split(';').reduce((acc, cookieStr) => {
const jar = new toughCookie.CookieJar(undefined, { rejectPublicSuffixes: false })
Expand Down Expand Up @@ -66,28 +67,19 @@ const getMediaFeatures = ({ animations, colorScheme }) => {
return prefers
}

const injectScripts = (page, values, attributes) =>
Promise.all(
castArray(values).map(value =>
pReflect(
page.addScriptTag({
[getInjectKey('js', value)]: value,
...attributes
})
)
)
)
const injectScript = (page, value, attributes) =>
page.addScriptTag({
[getInjectKey('js', value)]: value,
...attributes
})

const injectStyles = (page, styles) =>
Promise.all(
castArray(styles).map(style =>
pReflect(
page.addStyleTag({
[getInjectKey('css', style)]: style
})
)
)
)
const injectStyle = (page, style) =>
page.addStyleTag({
[getInjectKey('css', style)]: style
})

const getInjectKey = (ext, value) =>
isUrl(value) ? 'url' : value.endsWith(`.${ext}`) ? 'path' : 'content'

const disableAnimations = `
*,
Expand All @@ -101,29 +93,22 @@ const disableAnimations = `
}
`.trim()

const run = async ({ fn, debug: props }) => {
const debugProps = { duration: timeSpan() }
const result = await pReflect(fn)
debugProps.duration = prettyMs(debugProps.duration())
if (result.isRejected) debugProps.error = result.reason.message || result.reason
debug(props, debugProps)
return result
}

// related https://github.com/puppeteer/puppeteer/issues/1353
const createWaitUntilAuto = defaultOpts => (page, opts) => {
const { timeout } = { ...defaultOpts, ...opts }

return run({
fn: pTimeout(
Promise.all([
page.waitForNavigation({ waitUntil: 'networkidle2' }),
page.evaluate(() => window.history.pushState(null, null, null))
]),
timeout * (1 / 8)
),
debug: { isWaitUntilAuto: true }
})
return Promise.all(
[
{
fn: () => page.waitForNavigation({ waitUntil: 'networkidle2' }),
debug: 'waitUntilAuto:waitForNavigation'
},
{
fn: () => page.evaluate(() => window.history.pushState(null, null, null)),
debug: 'waitUntilAuto:pushState'
}
].map(({ fn, ...opts }) => run({ fn: fn(), timeout, ...opts }))
)
}

module.exports = ({
Expand All @@ -133,14 +118,12 @@ module.exports = ({
...deviceOpts
}) => {
const baseTimeout = globalTimeout * (1 / 2)
const actionTimeout = baseTimeout * (1 / 8)

const getDevice = createDevices(deviceOpts)
const { viewport: defaultViewport } = getDevice.findDevice(defaultDevice)

const applyEvasions = castArray(evasions)
.filter(Boolean)
.reduce((acc, key) => [...acc, EVASIONS[key]], [])

const _waitUntilAuto = createWaitUntilAuto({ timeout: baseTimeout })
const _waitUntilAuto = createWaitUntilAuto({ timeout: actionTimeout })

const goto = async (
page,
Expand Down Expand Up @@ -178,13 +161,20 @@ module.exports = ({
const prePromises = []

if (adblock) {
prePromises.push(engine.enableBlockingInPage(page))
prePromises.push(
run({
fn: engine.enableBlockingInPage(page),
timeout: actionTimeout,
debug: 'adblock'
})
)
}

if (javascript === false) {
prePromises.push(
run({
fn: page.setJavaScriptEnabled(false),
timeout: actionTimeout,
debug: { javascript }
})
)
Expand All @@ -200,6 +190,7 @@ module.exports = ({
prePromises.push(
run({
fn: page.setViewport(device.viewport),
timeout: actionTimeout,
debug: 'viewport'
})
)
Expand All @@ -213,6 +204,7 @@ module.exports = ({
prePromises.push(
run({
fn: page.setCookie(...cookies),
timeout: actionTimeout,
debug: ['cookies', ...cookies.map(({ name }) => name)]
})
)
Expand All @@ -222,6 +214,7 @@ module.exports = ({
prePromises.push(
run({
fn: page.setUserAgent(headers['user-agent']),
timeout: actionTimeout,
debug: { 'user-agent': headers['user-agent'] }
})
)
Expand All @@ -230,6 +223,7 @@ module.exports = ({
prePromises.push(
run({
fn: page.setExtraHTTPHeaders(headers),
timeout: actionTimeout,
debug: { headers: headersKeys }
})
)
Expand All @@ -239,6 +233,7 @@ module.exports = ({
prePromises.push(
run({
fn: page.emulateMediaType(mediaType),
timeout: actionTimeout,
debug: { mediaType }
})
)
Expand All @@ -248,6 +243,7 @@ module.exports = ({
prePromises.push(
run({
fn: page.emulateTimezone(timezone),
timeout: actionTimeout,
debug: { timezone }
})
)
Expand All @@ -259,12 +255,23 @@ module.exports = ({
prePromises.push(
run({
fn: page.emulateMediaFeatures(mediaFeatures),
debug: { mediaFeatures: mediaFeatures.length }
timeout: actionTimeout,
debug: { mediaFeatures: mediaFeatures.map(({ name }) => name) }
})
)
}

await Promise.all(prePromises.concat(applyEvasions.map(fn => fn(page))))
const applyEvasions = castArray(evasions)
.filter(Boolean)
.map(key =>
run({
fn: EVASIONS[key](page),
timeout: actionTimeout,
debug: key
})
)

await Promise.all(prePromises.concat(applyEvasions))

if (abortTypes.length > 0) {
await page.setRequestInterception(true)
Expand All @@ -277,7 +284,8 @@ module.exports = ({
}

const { value } = await run({
fn: pTimeout(html ? page.setContent(html, args) : page.goto(url, args), timeout),
fn: html ? page.setContent(html, args) : page.goto(url, args),
timeout,
debug: html ? 'html' : 'url'
})

Expand All @@ -287,40 +295,71 @@ module.exports = ({
waitForFunction,
waitForTimeout
})) {
if (value) await run({ fn: page[key](value), debug: { [key]: value } })
if (value) {
await run({ fn: page[key](value), timeout: actionTimeout, debug: { [key]: value } })
}
}

const postPromises = []

if (animations === false) {
postPromises.push(injectCSS(page, disableAnimations))
postPromises.push(
run({
fn: injectStyle(page, disableAnimations),
timeout: actionTimeout,
debug: 'disableAnimations'
})
)
}

const hideOrRemove = [
hide && injectCSS(page, `${castArray(hide).join(', ')} { visibility: hidden !important; }`),
remove && injectCSS(page, `${castArray(remove).join(', ')} { display: none !important; }`)
].filter(Boolean)
if (hide) {
postPromises.push(
run({
fn: injectStyle(page, `${castArray(hide).join(', ')} { visibility: hidden !important; }`),
timeout: actionTimeout,
debug: 'hide'
})
)
}

if (remove) {
postPromises.push(
run({
fn: injectStyle(page, `${castArray(remove).join(', ')} { display: none !important; }`),
timeout: actionTimeout,
debug: 'remove'
})
)
}

if (hideOrRemove.length > 0) {
if (modules) {
postPromises.push(
run({
fn: Promise.all(hideOrRemove),
debug: { hideOrRemove: hideOrRemove.length }
fn: Promise.all(
castArray(modules).map(value => injectScript(page, value, { type: 'modules' }))
),
timeout: actionTimeout,
debug: 'modules'
})
)
}

const injections = [
modules && injectScripts(page, modules, { type: 'modules' }),
scripts && injectScripts(page, scripts),
styles && injectStyles(page, styles)
].filter(Boolean)
if (scripts) {
postPromises.push(
run({
fn: Promise.all(castArray(scripts).map(value => injectScript(page, value))),
timeout: actionTimeout,
debug: 'scripts'
})
)
}

if (injections.length > 0) {
if (styles) {
postPromises.push(
run({
fn: Promise.all(injections),
debug: { injections: injections.length }
fn: Promise.all(castArray(styles).map(style => injectStyle(page, style))),
timeout: actionTimeout,
debug: 'styles'
})
)
}
Expand All @@ -329,18 +368,21 @@ module.exports = ({

if (click) {
for (const selector of castArray(click)) {
await run({ fn: page.click(selector), debug: { click: selector } })
await run({ fn: page.click(selector), timeout: actionTimeout, debug: { click: selector } })
}
}

if (scroll) {
await run({
fn: page.$eval(scroll, el => el.scrollIntoView()),
timeout: actionTimeout,
debug: { scroll }
})
}

if (isWaitUntilAuto) await waitUntilAuto(page, { response: value, timeout })
if (isWaitUntilAuto) {
await waitUntilAuto(page, { response: value, timeout: actionTimeout * 2 })
}

return { response: value, device }
}
Expand All @@ -352,11 +394,13 @@ module.exports = ({
goto.defaultViewport = defaultViewport
goto.waitUntilAuto = _waitUntilAuto
goto.timeout = baseTimeout
goto.actionTimeout = actionTimeout
goto.run = run

return goto
}

module.exports.parseCookies = parseCookies
module.exports.injectScripts = injectScripts
module.exports.injectStyles = injectStyles
module.exports.injectScript = injectScript
module.exports.injectStyle = injectStyle
module.exports.evasions = ALL_EVASIONS_KEYS
2 changes: 1 addition & 1 deletion packages/goto/test/e2e/evasions.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

const test = require('ava')

const browserlessFactory = require('browserless')({ timeout: 300000 })
const browserlessFactory = require('browserless')()
const onExit = require('signal-exit')

onExit(browserlessFactory.close)
Expand Down
1 change: 0 additions & 1 deletion packages/screenshot/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@
"map-values-deep": "~1.0.2",
"mime-types": "~2.1.32",
"p-reflect": "~2.1.0",
"p-timeout": "~4.1.0",
"pretty-ms": "~7.0.1",
"prism-themes": "~1.9.0",
"sharp": "~0.29.1",
Expand Down
Loading

0 comments on commit a95f2a8

Please sign in to comment.