feat: authenticate hook for easier programmatic authentication

Author: harlan-zw

docs/content/1.guide/guides/authentication.md

@@ -103,6 +103,19 @@ Alternatively, you can provide the `--default-query-params` flag to the CLI.
 unlighthouse --site <your-site> --default-query-params auth=<token>,foo=bar
 ```
 
+## Local Storage
+
+If you can configure your authentication using local storage,
+then you can provide them using the `localStorage` option in your configuration file:
+
+```ts
+// unlighthouse.config.ts
+export default {
+  localStorage: {
+    auth: '<token>'
+  }
+}
+```
 
 ## Programmatic Usage
 
@@ -115,16 +128,8 @@ You can see an example here:
 ```ts
 // unlighthouse.config.ts
 export default {
-  puppeteerOptions: {
-    // slow down slightly so input is not missed
-    slowMo: 50,
-  },
-  lighthouseOptions: {
-    // allow storage to persist between pages
-    disableStorageReset: true,
-  },
   hooks: {
-    'puppeteer:before-goto': async (page) => {
+    'authenticate': async (page) => {
       // login to the page
       await page.goto('https://example.com/login')
       const emailInput = await page.$('input[type="email"]')
@@ -139,3 +144,24 @@ export default {
   },
 }
 ```
+
+## Troubleshooting
+
+If you're having trouble authenticating,
+you can use the `debug: true` and `headless: false`,
+flags to see what's happening. 
+
+```bash
+// unlighthouse.config.ts
+export default {
+  debug: true,
+  // show the browser window
+  puppeteerOptions: {
+    headless: false,
+  },
+  // only run a single scan at a time
+  puppeteerClusterOptions: {
+    maxConcurrency: 1,
+  },
+}
+```

packages/core/src/puppeteer/tasks/html.ts

@@ -2,17 +2,18 @@ import { join } from 'node:path'
 import fs from 'fs-extra'
 import type { CheerioAPI } from 'cheerio'
 import cheerio from 'cheerio'
-import type { Page } from 'puppeteer-core'
+import type { Page } from 'puppeteer'
 import { $URL, withoutTrailingSlash } from 'ufo'
 import chalk from 'chalk'
 import type { HTMLExtractPayload, PuppeteerTask } from '../../types'
 import { useUnlighthouse } from '../../unlighthouse'
 import { useLogger } from '../../logger'
 import { ReportArtifacts, fetchUrlRaw, formatBytes, trimSlashes } from '../../util'
 import { normaliseRoute } from '../../router'
+import { setupPage } from '../util'
 
 export const extractHtmlPayload: (page: Page, route: string) => Promise<{ success: boolean; redirected?: false | string; message?: string; payload?: string }> = async (page, route) => {
-  const { worker, resolvedConfig, hooks } = useUnlighthouse()
+  const { worker, resolvedConfig } = useUnlighthouse()
 
   // if we don't need to execute any javascript we can do a less expensive fetch of the URL
   if (resolvedConfig.scanner.skipJavascript) {
@@ -41,9 +42,11 @@ export const extractHtmlPayload: (page: Page, route: string) => Promise<{ succes
         request.continue()
     })
 
-    await hooks.callHook('puppeteer:before-goto', page)
+    await setupPage(page)
 
     const pageVisit = await page.goto(route, { waitUntil: resolvedConfig.scanner.skipJavascript ? 'domcontentloaded' : 'networkidle0' })
+    if (!pageVisit)
+      return { success: false, message: `Failed to go to route ${route}.` }
 
     // only 2xx we'll consider valid
     const { 'content-type': contentType, location } = pageVisit.headers()

packages/core/src/puppeteer/tasks/lighthouse.ts

@@ -10,6 +10,7 @@ import type { LighthouseReport, PuppeteerTask, UnlighthouseRouteReport } from '.
 import { useUnlighthouse } from '../../unlighthouse'
 import { useLogger } from '../../logger'
 import { ReportArtifacts, base64ToBuffer } from '../../util'
+import { setupPage } from '../util'
 
 export function normaliseLighthouseResult(route: UnlighthouseRouteReport, result: LH.Result): LighthouseReport {
   const { resolvedConfig, runtimeSettings } = useUnlighthouse()
@@ -85,7 +86,7 @@ export function normaliseLighthouseResult(route: UnlighthouseRouteReport, result
 
 export const runLighthouseTask: PuppeteerTask = async (props) => {
   const logger = useLogger()
-  const { resolvedConfig, runtimeSettings, worker, hooks } = useUnlighthouse()
+  const { resolvedConfig, runtimeSettings, worker } = useUnlighthouse()
   const { page, data: routeReport } = props
 
   // if the report doesn't exist, we're going to run a new lighthouse process to generate it
@@ -96,32 +97,9 @@ export const runLighthouseTask: PuppeteerTask = async (props) => {
     return routeReport
   }
 
-  const browser = page.browser()
-  const port = new URL(browser.wsEndpoint()).port
-  // ignore csp errors
-  await page.setBypassCSP(true)
-
-  if (resolvedConfig.auth)
-    await page.authenticate(resolvedConfig.auth)
-
-  if (resolvedConfig.cookies)
-    await page.setCookie(...resolvedConfig.cookies)
-  if (resolvedConfig.extraHeaders)
-    await page.setExtraHTTPHeaders(resolvedConfig.extraHeaders)
-
-  // Wait for Lighthouse to open url, then allow hook to run
-  browser.on('targetchanged', async (target) => {
-    const page = await target.page()
-    if (page) {
-      // in case they get reset
-      if (resolvedConfig.cookies)
-        await page.setCookie(...resolvedConfig.cookies)
-      if (resolvedConfig.extraHeaders)
-        await page.setExtraHTTPHeaders(resolvedConfig.extraHeaders)
-      await hooks.callHook('puppeteer:before-goto', page)
-    }
-  })
+  await setupPage(page)
 
+  const port = new URL(page.browser().wsEndpoint()).port
   // allow changing behavior of the page
   const clonedRouteReport = { ...routeReport }
   // just modify the url for the unlighthouse request

packages/core/src/puppeteer/util.ts

@@ -0,0 +1,41 @@
+import type { Page } from 'puppeteer'
+import { useUnlighthouse } from '../unlighthouse'
+
+export async function setupPage(page: Page) {
+  const { resolvedConfig, hooks } = useUnlighthouse()
+
+  const browser = page.browser()
+  // ignore csp errors
+  await page.setBypassCSP(true)
+
+  if (resolvedConfig.auth)
+    await page.authenticate(resolvedConfig.auth)
+
+  // set local storage
+  if (resolvedConfig.localStorage) {
+    await page.evaluateOnNewDocument(
+      (data) => {
+        localStorage.clear()
+        for (const key in data)
+          localStorage.setItem(key, data[key])
+      }, resolvedConfig.localStorage)
+  }
+  if (resolvedConfig.cookies)
+    await page.setCookie(...resolvedConfig.cookies.map(cookie => ({ domain: resolvedConfig.site, ...cookie })))
+  if (resolvedConfig.extraHeaders)
+    await page.setExtraHTTPHeaders(resolvedConfig.extraHeaders)
+
+  // Wait for Lighthouse to open url, then allow hook to run
+  browser.on('targetchanged', async (target) => {
+    const page = await target.page()
+    if (page) {
+      // in case they get reset
+      if (resolvedConfig.cookies)
+        await page.setCookie(...resolvedConfig.cookies.map(cookie => ({ domain: resolvedConfig.site, ...cookie })))
+      // set local storage
+      if (resolvedConfig.extraHeaders)
+        await page.setExtraHTTPHeaders(resolvedConfig.extraHeaders)
+      await hooks.callHook('puppeteer:before-goto', page)
+    }
+  })
+}

packages/core/src/types.ts

@@ -263,6 +263,12 @@ export interface ResolvedUserConfig {
    * @default false
    */
   cookies: false | { name: string; value: string; [v: string]: string }[]
+  /**
+   * Local storage to add to the browser context.
+   *
+   * @default {}
+   */
+  localStorage: Record<string, any>
   /**
    * Extra headers to provide for any HTTP requests.
    *
@@ -699,6 +705,11 @@ export interface UnlighthouseHooks {
    * @param page
    */
   'puppeteer:before-goto': (page: Page) => HookResult
+  /**
+   * Authenticate a page before it's visited.
+   * @param page
+   */
+  'authenticate': (page: Page) => HookResult
 }
 
 /**
@@ -800,7 +811,7 @@ export interface UnlighthouseContext {
   mockRouter?: MockRouter
   /**
    * Settings that are computed from runtime data.
-   */
+   */localStorage
   runtimeSettings: RuntimeSettings
   /**
    * Access the hook system, either calling a hook or listening to one.

packages/core/src/unlighthouse.ts

@@ -15,7 +15,7 @@ import { version } from '../package.json'
 import { WS, createApi, createBroadcastingEvents, createMockRouter } from './router'
 import { createUnlighthouseWorker, inspectHtmlTask, runLighthouseTask } from './puppeteer'
 import type {
-  Provider, RuntimeSettings,
+  Provider, ResolvedUserConfig, RuntimeSettings,
   UnlighthouseContext,
   UnlighthouseHooks,
   UserConfig,
@@ -136,6 +136,24 @@ export async function createUnlighthouse(userConfig: UserConfig, provider?: Prov
 
   const worker = await createUnlighthouseWorker(tasks)
 
+  // do an authentication step
+  await worker.cluster.execute({}, async (taskCtx) => {
+    await hooks.callHook('authenticate', taskCtx.page)
+    // collect page authentication, either cookie or localStorage tokens
+    const localStorageData = await taskCtx.page.evaluate(() => {
+      const json = {}
+      for (let i = 0; i < localStorage.length; i++) {
+        const key = localStorage.key(i)
+        json[key] = localStorage.getItem(key)
+      }
+      return json
+    })
+    const cookies = await taskCtx.page.cookies()
+    // merge this into the config
+    ctx.resolvedConfig.cookies = [...(ctx.resolvedConfig.cookies || []), ...cookies as any as ResolvedUserConfig['cookies']]
+    ctx.resolvedConfig.localStorage = { ...ctx.resolvedConfig.localStorage, ...localStorageData }
+  })
+
   ctx.worker = worker
 
   ctx.setCiContext = async () => {
@@ -248,11 +266,6 @@ export async function createUnlighthouse(userConfig: UserConfig, provider?: Prov
   }
 
   ctx.start = async () => {
-    if (worker.hasStarted()) {
-      logger.debug('Attempted to start Unlighthouse, has already started.')
-      return ctx
-    }
-
     logger.debug(`Starting Unlighthouse [Server: ${provider?.name === 'ci' ? 'N/A' : ctx.runtimeSettings.clientUrl} Site: ${ctx.resolvedConfig.site} Debug: \`${ctx.resolvedConfig.debug}\`]`)
 
     if (typeof provider?.routeDefinitions === 'function')