diff --git a/packages/goto/package.json b/packages/goto/package.json index fc49764be..a74abb5eb 100644 --- a/packages/goto/package.json +++ b/packages/goto/package.json @@ -31,7 +31,6 @@ "dependencies": { "@browserless/devices": "^10.7.13", "@ghostery/adblocker-puppeteer": "~2.11.3", - "@kikobeats/time-span": "~1.0.8", "debug-logfmt": "~1.4.0", "got": "~11.8.6", "is-url-http": "~2.3.10", diff --git a/packages/goto/src/index.js b/packages/goto/src/index.js index 356cd81cb..a2defc6ed 100644 --- a/packages/goto/src/index.js +++ b/packages/goto/src/index.js @@ -11,8 +11,6 @@ const isUrl = require('is-url-http') const path = require('path') const fs = require('fs') -const timeSpan = require('@kikobeats/time-span')({ format: require('pretty-ms') }) - const { DEFAULT_INTERCEPT_RESOLUTION_PRIORITY } = require('puppeteer') const debug = require('debug-logfmt')('browserless:goto') @@ -34,11 +32,10 @@ const isEmpty = val => val == null || !(Object.keys(val) || val).length const castArray = value => [].concat(value).filter(Boolean) const run = async ({ fn, timeout, debug: props }) => { - const debugProps = { duration: timeSpan() } + const duration = debug.duration() const result = await pReflect(timeout ? pTimeout(fn, timeout) : fn) - debugProps.duration = debugProps.duration() - if (result.isRejected) debugProps.error = result.reason.message || result.reason - debug(props, debugProps) + const errorProps = result.isRejected ? { error: result.reason.message || result.reason } : {} + duration(props, errorProps) return result } diff --git a/packages/screenshot/src/index.js b/packages/screenshot/src/index.js index 8b20a6d11..514ca8643 100644 --- a/packages/screenshot/src/index.js +++ b/packages/screenshot/src/index.js @@ -32,16 +32,80 @@ const waitForImagesOnViewport = page => ) ) +const waitForDomStability = ({ idle, timeout } = {}) => + new Promise(resolve => { + if (!document.body) return resolve({ status: 'no-body' }) + + let lastChange = performance.now() + const observer = new window.MutationObserver(() => { + lastChange = performance.now() + }) + observer.observe(document.body, { + childList: true, + subtree: true, + attributes: false, + characterData: false + }) + + const deadline = performance.now() + timeout + + ;(function check () { + const now = performance.now() + if (now - lastChange >= idle) { + observer.disconnect() + return resolve({ status: 'idle' }) + } + if (now >= deadline) { + observer.disconnect() + return resolve({ status: 'timeout' }) + } + window.requestAnimationFrame(check) + })() + }) + +const scrollFullPageToLoadContent = async (page, timeout) => { + const debug = require('debug-logfmt')('browserless:goto') + + const duration = debug.duration() + const result = await page.evaluate(waitForDomStability, { + idle: timeout / 2 / 2, + timeout: timeout / 2 + }) + + duration('waitForDomStability', result) + + await page.evaluate( + timeout => + new Promise(resolve => { + let currentScrollPosition = 0 + const scrollStep = Math.floor(window.innerHeight) + const pageHeight = document.body.scrollHeight + const totalSteps = Math.ceil(pageHeight / scrollStep) + const stepDelay = timeout / 2 / totalSteps + const scrollNext = async () => { + if (currentScrollPosition >= pageHeight) { + resolve() + return + } + window.scrollBy(0, scrollStep) + currentScrollPosition += scrollStep + setTimeout(scrollNext, stepDelay) + } + scrollNext() + }), + timeout + ) + await page.evaluate(() => window.scrollTo(0, 0)) +} + const waitForElement = async (page, element) => { const screenshotOpts = {} - if (element) { await page.waitForSelector(element, { visible: true }) screenshotOpts.clip = await page.$eval(element, getBoundingClientRect) screenshotOpts.fullPage = false return screenshotOpts } - return screenshotOpts } @@ -51,35 +115,58 @@ module.exports = ({ goto, ...gotoOpts }) => { return function screenshot (page) { return async ( url, - { - element, - codeScheme = 'atom-dark', - overlay: overlayOpts = {}, - waitUntil = 'auto', - ...opts - } = {} + { codeScheme = 'atom-dark', overlay: overlayOpts = {}, waitUntil = 'auto', ...opts } = {} ) => { let screenshot let response - const beforeScreenshot = response => { - const timeout = goto.timeouts.action(goto.timeouts.base(opts.timeout)) - return Promise.all( - [ - { - fn: () => page.evaluate('document.fonts.ready'), - debug: 'beforeScreenshot:fontsReady' - }, - { - fn: () => waitForPrism(page, response, { codeScheme, ...opts }), - debug: 'beforeScreenshot:waitForPrism' + const beforeScreenshot = async (page, response, { element, fullPage = false } = {}) => { + const timeout = goto.timeouts.action(opts.timeout) + + let screenshotOpts = {} + const tasks = [ + { + fn: () => page.evaluate('document.fonts.ready'), + debug: 'beforeScreenshot:fontsReady' + }, + { + fn: () => waitForImagesOnViewport(page), + debug: 'beforeScreenshot:waitForImagesOnViewport' + } + ] + + if (codeScheme && response) { + tasks.push({ + fn: () => waitForPrism(page, response, { codeScheme, ...opts }), + debug: 'beforeScreenshot:waitForPrism' + }) + } + + if (fullPage) { + tasks.push({ + fn: () => scrollFullPageToLoadContent(page, timeout, goto), + debug: 'beforeScreenshot:scrollFullPageToLoadContent' + }) + } else if (element) { + tasks.push({ + fn: async () => { + screenshotOpts = await waitForElement(page, element) }, - { - fn: () => waitForImagesOnViewport(page), - debug: 'beforeScreenshot:waitForImagesOnViewport' - } - ].map(({ fn, ...opts }) => goto.run({ fn: fn(), ...opts, timeout })) + debug: 'beforeScreenshot:waitForElement' + }) + } + + await Promise.all( + tasks.map(({ fn, ...opts }) => + goto.run({ + fn: fn(), + ...opts, + timeout: fullPage ? timeout * 2 : timeout + }) + ) ) + + return screenshotOpts } const takeScreenshot = async opts => { @@ -98,19 +185,13 @@ module.exports = ({ goto, ...gotoOpts }) => { if (waitUntil !== 'auto') { ;({ response } = await goto(page, { ...opts, url, waitUntil })) - const [screenshotOpts] = await Promise.all([ - waitForElement(page, element), - beforeScreenshot(response) - ]) + const screenshotOpts = await beforeScreenshot(page, response, opts) screenshot = await page.screenshot({ ...opts, ...screenshotOpts }) debug('screenshot', { waitUntil, duration: timeScreenshot() }) } else { ;({ response } = await goto(page, { ...opts, url, waitUntil, waitUntilAuto })) async function waitUntilAuto (page, { response }) { - const [screenshotOpts] = await Promise.all([ - waitForElement(page, element), - beforeScreenshot(response) - ]) + const screenshotOpts = await beforeScreenshot(page, response, opts) const { isWhite } = await takeScreenshot({ ...opts, ...screenshotOpts }) debug('screenshot', { waitUntil, isWhite, duration: timeScreenshot() }) } diff --git a/packages/screenshot/src/pretty/index.js b/packages/screenshot/src/pretty/index.js index 3387a12d2..870a2423b 100644 --- a/packages/screenshot/src/pretty/index.js +++ b/packages/screenshot/src/pretty/index.js @@ -32,8 +32,6 @@ const JSONParse = input => { } module.exports = async (page, response, { timeout, codeScheme, styles, scripts, modules }) => { - if (!response || !codeScheme) return - let [theme, content, prism] = await Promise.all([getTheme(codeScheme), response.text(), getPrism]) if (isHtmlContent(content)) return