From 087f1d9056ba5cf74c4f3149c30d89eceed82d03 Mon Sep 17 00:00:00 2001 From: EGOIST <0x142857@gmail.com> Date: Sun, 13 Mar 2022 16:37:55 +0000 Subject: [PATCH] feat: add `htmlSelector` option BREAKING CHANGE: now the signature of `request` has been changed to: ```js require(options) // vs. previous // require(options | options[], browserOptions) ``` --- .devcontainer/Dockerfile | 10 +++++ .devcontainer/devcontainer.json | 35 ++++++++++++++++++ README.md | 17 +++++---- src/index.ts | 59 +++++++++++++++--------------- test/index.test.ts | 16 +++++++- test/server/html-selector.html | 12 ++++++ test/snapshots/index.test.ts.md | 6 +++ test/snapshots/index.test.ts.snap | Bin 430 -> 456 bytes tsconfig.json | 3 +- 9 files changed, 120 insertions(+), 38 deletions(-) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/devcontainer.json create mode 100644 test/server/html-selector.html diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..768d8bc --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,10 @@ +# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.222.0/containers/typescript-node/.devcontainer/base.Dockerfile + +# [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 16, 14, 12, 16-bullseye, 14-bullseye, 12-bullseye, 16-buster, 14-buster, 12-buster +ARG VARIANT="16-bullseye" +FROM mcr.microsoft.com/vscode/devcontainers/typescript-node:0-${VARIANT} + +# https://github.com/microsoft/vscode-dev-containers/blob/e5d1e95407309e972a37914cfd18a53c480bc1fa/script-library/docs/desktop-lite.md#installing-a-browser +RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ + && curl -sSL https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb -o /tmp/chrome.deb \ + && apt-get -y install /tmp/chrome.deb \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..3a18a5a --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,35 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: +// https://github.com/microsoft/vscode-dev-containers/tree/v0.222.0/containers/typescript-node +{ + "name": "Node.js & TypeScript", + "build": { + "dockerfile": "Dockerfile", + // Update 'VARIANT' to pick a Node version: 16, 14, 12. + // Append -bullseye or -buster to pin to an OS version. + // Use -bullseye variants on local on arm64/Apple Silicon. + "args": { + "VARIANT": "16-bullseye" + } + }, + + // Set *default* container specific settings.json values on container create. + "settings": {}, + + + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "dbaeumer.vscode-eslint" + ], + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + + // Use 'postCreateCommand' to run commands after the container is created. + // "postCreateCommand": "yarn install", + + // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. + "remoteUser": "node", + "features": { + "git": "latest" + } +} diff --git a/README.md b/README.md index e5faee1..d6bb5f0 100644 --- a/README.md +++ b/README.md @@ -38,15 +38,18 @@ import { cleanup } from 'taki' cleanup() ``` -### Multiplate URLs +### Custom html selector + +By default it returns the html for the entire document, but you can specify a selector to get the html for a specific element. ```js -request([ - { url: 'https://sao.js.org' }, - { url: 'https://sao.js.org/#/create' }, -]).then((result) => { - // Then the result will an array of html string -}) +const { request } = require('taki') + +request({ url: 'https://example.com', htmlSelector: '.some-element' }).then( + (html) => { + console.log(html) + } +) ``` ### Manually take snapshot diff --git a/src/index.ts b/src/index.ts index d783ffc..811a016 100644 --- a/src/index.ts +++ b/src/index.ts @@ -24,6 +24,10 @@ export type RequestOptions = { minify?: boolean resourceFilter?: (ctx: ResourceFilterCtx) => boolean blockCrossOrigin?: boolean + proxy?: string + headless?: boolean + sandbox?: boolean + htmlSelector?: string } async function getHTML(browser: Browser, options: RequestOptions) { @@ -64,15 +68,17 @@ async function getHTML(browser: Browser, options: RequestOptions) { } return next() }) - let resolveFunction: Function | undefined + let resolveResult: Function | undefined let content: string = '' type Result = { content: string } - const promise = new Promise((resolve) => (resolveFunction = resolve)) + const resultPromise = new Promise( + (resolve) => (resolveResult = resolve) + ) if (options.manually) { const functionName = typeof options.manually === 'string' ? options.manually : 'snapshot' await page.exposeFunction(functionName, (result: any) => { - resolveFunction!(result) + resolveResult!(result) }) } await page.goto(options.url, { @@ -80,13 +86,20 @@ async function getHTML(browser: Browser, options: RequestOptions) { }) let result: Result | undefined if (options.manually) { - result = await promise + result = await resultPromise } else if (typeof options.wait === 'number') { await page.waitForTimeout(options.wait) } else if (typeof options.wait === 'string') { await page.waitForSelector(options.wait) } - content = result ? result.content : await page.content() + content = result + ? result.content + : options.htmlSelector + ? await page.evaluate( + (selector) => document.querySelector(selector).innerHTML, + [options.htmlSelector] + ) + : await page.content() options.onBeforeClosingPage && (await options.onBeforeClosingPage(page)) await page.close() options.onAfterRequest && options.onAfterRequest(options.url) @@ -109,37 +122,25 @@ async function getHTML(browser: Browser, options: RequestOptions) { let browser: Browser | undefined -export type BrowserOptions = { proxy?: string; headless?: boolean } - -export async function request( - options: RequestOptions, - browserOptions?: BrowserOptions -): Promise -export async function request( - options: RequestOptions[], - browserOptions?: BrowserOptions -): Promise +export type BrowserOptions = { + proxy?: string + headless?: boolean + sandbox?: boolean +} -export async function request( - options: RequestOptions | RequestOptions[], - { proxy, headless }: BrowserOptions = {} -) { +export async function request(options: RequestOptions) { if (!browser) { browser = await pptr.launch({ executablePath: findChrome(), - args: [proxy && `--proxy-server=${proxy}`].filter(truthy), - headless, + args: [ + options.proxy && `--proxy-server=${options.proxy}`, + options.sandbox === false && '--no-sandbox', + ].filter(truthy), + headless: options.headless, }) } - try { - const result = Array.isArray(options) - ? await Promise.all(options.map((option) => getHTML(browser!, option))) - : await getHTML(browser, options) - return result - } catch (error) { - throw error - } + return getHTML(browser, options) } export async function cleanup() { diff --git a/test/index.test.ts b/test/index.test.ts index 373bf58..b8a780a 100644 --- a/test/index.test.ts +++ b/test/index.test.ts @@ -26,9 +26,10 @@ test.after.always(async () => { await cleanup() }) -test('basic', async t => { +test('basic', async (t) => { const html = await request({ url: `http://localhost:${port}/basic.html`, + sandbox: false, }) t.snapshot(html) }) @@ -37,6 +38,7 @@ test('minify', async (t) => { const html = await request({ url: `http://localhost:${port}/basic.html`, minify: true, + sandbox: false, }) t.snapshot(html) }) @@ -45,6 +47,7 @@ test('wait for selector', async (t) => { const html = await request({ url: `http://localhost:${port}/wait-for-selector.html`, wait: '#bar', + sandbox: false, }) t.snapshot(html) }) @@ -53,6 +56,17 @@ test('manually', async (t) => { const html = await request({ url: `http://localhost:${port}/manually.html`, manually: true, + sandbox: false, + }) + + t.snapshot(html) +}) + +test('custom html selector', async (t) => { + const html = await request({ + url: `http://localhost:${port}/html-selector.html`, + htmlSelector: '#app', + sandbox: false, }) t.snapshot(html) diff --git a/test/server/html-selector.html b/test/server/html-selector.html new file mode 100644 index 0000000..1a80c82 --- /dev/null +++ b/test/server/html-selector.html @@ -0,0 +1,12 @@ + + + + + + + Document + + +
app
+ + diff --git a/test/snapshots/index.test.ts.md b/test/snapshots/index.test.ts.md index 38e6fb7..702b84f 100644 --- a/test/snapshots/index.test.ts.md +++ b/test/snapshots/index.test.ts.md @@ -25,6 +25,12 @@ Generated by [AVA](https://avajs.dev). ␊ ` +## custom html selector + +> Snapshot 1 + + 'app' + ## manually > Snapshot 1 diff --git a/test/snapshots/index.test.ts.snap b/test/snapshots/index.test.ts.snap index 8650a79439a089231f2f44688c3654c58cc267df..c3a6a2881c27c893e3f65dfd1531b298794973e4 100644 GIT binary patch literal 456 zcmV;(0XP0ZRzVw=ca9PO6e!Mruw@zCw9^QBDd_nV}sQmrZeUQD#Aj z9hZVaT4`Q#NoIbYLT+Mao`$AEHBd}JAtgV#G&eP`L@zzH#5E@s#B{3kOwmvW8>z0T zSCU#$;+&sX0+dkDwpCDv7_Sc0T+0O{Ack?-=))`mT3J$=lM1p?8EgjBqU8LX{30ub zqSO>?uwI}Jh&ntzKSa>yU>h*}4f8dcm#cAk78-sip4GNhf|#gO3(*btBGiKr-&I5W y1@u-e%tyFA#K@djP=F%fh&N57WR`)WHz~15$qq=^=mSO2%m)BXaCckl0ssJwZpxzo literal 430 zcmV;f0a5-zRzV(QTs-HPxoLq!MVUGFVe} za(+&JkyTM@N-ax=sI|Xf9 z1$Bt=>Ojr4TtEV17#G~1AS=Nh0?I)=3$+O1Wd)$0t-*R>9_PaCb4EwJNgyS&4D9)& Y#3Cg-AYr2q6hR9}07Nc0M8E<703OV;OaK4? diff --git a/tsconfig.json b/tsconfig.json index c151452..0cdbe75 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -5,7 +5,8 @@ "target": "es2019" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */, "module": "commonjs" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */, "lib": [ - "ES2019" + "ES2019", + "DOM" ] /* Specify library files to be included in the compilation. */, // "allowJs": true, /* Allow javascript files to be compiled. */ // "checkJs": true, /* Report errors in .js files. */