Skip to content

Commit

Permalink
Merge 70faceb into cb40f0c
Browse files Browse the repository at this point in the history
  • Loading branch information
Kikobeats committed Jul 15, 2023
2 parents cb40f0c + 70faceb commit 0cceaac
Show file tree
Hide file tree
Showing 42 changed files with 798 additions and 1,260 deletions.
18 changes: 18 additions & 0 deletions .github/workflows/get-matrix.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
import { readdir, readFile } from 'fs/promises'
import { dirname, join } from 'path'
import { fileURLToPath } from 'url'

const __dirname = dirname(fileURLToPath(import.meta.url))

const packages = []

const packagesPath = join(__dirname, '../../packages')

for (const packagePath of await readdir(packagesPath)) {
const pkg = join(packagesPath, packagePath, 'package.json')
const { name } = JSON.parse(await readFile(pkg))
packages.push(name)
}

console.log(`{"package":${JSON.stringify(packages)}}`)

47 changes: 39 additions & 8 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,15 +28,30 @@ jobs:
run: |
git config --global user.email ${{ secrets.GIT_EMAIL }}
git config --global user.name ${{ secrets.GIT_USERNAME }}
npm run contributors
pnpm run contributors
- name: Push changes
run: |
git push origin ${{ github.head_ref }}
matrix:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
- id: matrix
run: echo "matrix=$(node .github/workflows/get-matrix.mjs)" >> $GITHUB_OUTPUT
- run: echo ${{ steps.matrix.outputs.matrix }}
outputs:
matrix: ${{ steps.matrix.outputs.matrix }}
test:
if: ${{ !startsWith(github.event.head_commit.message, 'chore(release):') && !startsWith(github.event.head_commit.message, 'docs:') && !startsWith(github.event.head_commit.message, 'ci:') }}
needs: [contributors]
if: |
!startsWith(github.event.head_commit.message, 'chore(release):') &&
!startsWith(github.event.head_commit.message, 'docs:') &&
!startsWith(github.event.head_commit.message, 'ci:')
needs: [contributors, matrix]
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix: ${{fromJson(needs.matrix.outputs.matrix)}}
steps:
- name: Checkout
uses: actions/checkout@v3
Expand All @@ -52,10 +67,26 @@ jobs:
version: latest
run_install: true
- name: Test
run: npm test
- name: Report
run: mkdir -p coverage && npx c8 report --reporter=text-lcov > coverage/lcov.info
run: pnpm run test --scope "${{ matrix.package }}"
env:
PROXY_USERNAME: ${{ secrets.PROXY_USERNAME }}
PROXY_PASSWORD: ${{ secrets.PROXY_PASSWORD }}
PROXY_HOST: ${{ secrets.PROXY_HOST }}
- name: Coverage
uses: coverallsapp/github-action@master
run: pnpm run coverage
- name: Upload
uses: coverallsapp/github-action@main
with:
flag-name: ${{ matrix.package }}
parallel: true
allow-empty: true
github-token: ${{ secrets.GITHUB_TOKEN }}
finish:
needs: test
runs-on: ubuntu-latest
steps:
- name: Coveralls Finished
uses: coverallsapp/github-action@main
with:
parallel-finished: true

39 changes: 35 additions & 4 deletions .github/workflows/pull_request.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,30 @@ on:
- master

jobs:
matrix:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
- id: matrix
run: echo "matrix=$(node .github/workflows/get-matrix.mjs)" >> $GITHUB_OUTPUT
- run: echo ${{ steps.matrix.outputs.matrix }}
outputs:
matrix: ${{ steps.matrix.outputs.matrix }}
test:
if: github.ref != 'refs/heads/master'
needs: matrix
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix: ${{fromJson(needs.matrix.outputs.matrix)}}
steps:
- name: Checkout
uses: actions/checkout@v3
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup ffmpeg
uses: FedericoCarboni/setup-ffmpeg@v2
- name: Setup Node.js
uses: actions/setup-node@v3
with:
Expand All @@ -27,10 +43,25 @@ jobs:
version: latest
run_install: true
- name: Test
run: npm test
- name: Report
run: mkdir -p coverage && npx c8 report --reporter=text-lcov > coverage/lcov.info
run: pnpm run test --scope "${{ matrix.package }}"
env:
PROXY_USERNAME: ${{ secrets.PROXY_USERNAME }}
PROXY_PASSWORD: ${{ secrets.PROXY_PASSWORD }}
PROXY_HOST: ${{ secrets.PROXY_HOST }}
- name: Coverage
uses: coverallsapp/github-action@master
run: pnpm run coverage
- name: Upload
uses: coverallsapp/github-action@main
with:
flag-name: ${{ matrix.package }}
parallel: true
allow-empty: true
github-token: ${{ secrets.GITHUB_TOKEN }}
finish:
needs: test
runs-on: ubuntu-latest
steps:
- name: Coveralls Finished
uses: coverallsapp/github-action@main
with:
parallel-finished: true
79 changes: 20 additions & 59 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
## Highlights

- Compatible with Puppeteer API ([text](texturl-options), [screenshot](#screenshoturl-options), [html](#htmlurl-options), [pdf](#pdfurl-options)).
- Built-in [evasion](#evasions) techniques to prevent being blocked.
- Built-in [adblocker](#adblock) for canceling unnecessary requests.
- Shell interaction via [Browserless CLI](command-line-interface).
- Easy [Google Lighthouse](#lighthouse) integration.
Expand Down Expand Up @@ -566,40 +565,6 @@ default: `'macbook pro 13'`

It specifies the [device](#devices) descriptor to use in order to retrieve `userAgent` and `viewport`.

##### evasions

type: `string[]`</br>
default: `require('@browserless/goto').evasions`

It makes your Headless undetectable, preventing to being blocked.

![](/static/evasions.png)

These techniques are used by [antibot](https://news.ycombinator.com/item?id=20479015) systems to check if you are a real browser and block any kind of automated access. All the evasion techniques implemented are:

| Evasion | Description |
|--------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| [chromeRuntime](https://github.com/microlinkhq/browserless/blob/master/packages/goto/src/evasions/chrome-runtime.js) | Ensure `window.chrome` is defined. |
| [stackTraces](https://github.com/microlinkhq/browserless/blob/master/packages/goto/src/evasions/error-stack-trace.js) | Prevent detect Puppeteer via variable name. |
| [mediaCodecs](https://github.com/microlinkhq/browserless/blob/master/packages/goto/src/evasions/media-codecs.js) | Ensure media codedcs are defined. |
| [navigatorPermissions](https://github.com/microlinkhq/browserless/blob/master/packages/goto/src/evasions/navigator-permissions.js) | Mock over [Notification.permissions](https://developer.mozilla.org/en-US/docs/Web/API/Notification/permission). |
| [navigatorPlugins](https://github.com/microlinkhq/browserless/blob/master/packages/goto/src/evasions/navigator-plugins.js) | Ensure your browser has [NavigatorPlugins](https://developer.mozilla.org/en-US/docs/Web/API/NavigatorPlugins) defined. |
| [navigatorWebdriver](https://github.com/microlinkhq/browserless/blob/master/packages/goto/src/evasions/navigator-webdriver.js) | Ensure [Navigator.webdriver](https://developer.mozilla.org/en-US/docs/Web/API/Navigator/webdriver) exists. |
| [randomizeUserAgent](https://github.com/microlinkhq/browserless/blob/master/packages/goto/src/evasions/randomize-user-agent.js) | Use a different [User-Agent](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent) every time. |
| [webglVendor](https://github.com/microlinkhq/browserless/blob/master/packages/goto/src/evasions/webgl-vendor.js) | Ensure [WebGLRenderingContext](https://developer.mozilla.org/en-US/docs/Web/API/WebGLRenderingContext) & [WebGL2RenderingContext](https://developer.mozilla.org/en-US/docs/Web/API/WebGL2RenderingContext) are defined. |

The evasion techniques are enabled by default. You can omit techniques just filtering them:

```js
const createBrowser = require('browserless')

const evasions = require('@browserless/goto').evasions.filter(
evasion => evasion !== 'randomizeUserAgent'
)

const browser = createBrowser({ evasions })
```

##### headers

type: `object`
Expand Down Expand Up @@ -906,64 +871,60 @@ const screencast = require('@browserless/screencast')

const buffer = await screencast({
getBrowserless: () => browserless,
videoFormat: 'webm',
format: 'webm',
ffmpegPath: await execa.command('which ffmpeg').then(({ stdout }) => stdout),
gotoOpts: {
url: 'https://vercel.com',
animations: true,
abortTypes: [],
waitUntil: 'load'
},
withPage: async page => {
await page.waitForTimeout(3000)
await page.waitForTimeout(7000)
}
})
```

#### options

##### everyNthFrame
##### ffmpegPath

type: `number`</br>
default: `1`</br>
type: `string`

Sends every n-th frame.
The path for using `ffmpeg` binary.

##### format

type: `string`</br>
default: `'video/webm;codecs=vp9'`</br>
values: `'mp4'` | `'gif' | 'webm'`</br>
default: `'webm'`

The video output format.

The MIME media type that will be used for creating the `MediaRecorder` object.
##### frames

These options will be passed to [Page.startScreencast](https://chromedevtools.github.io/devtools-protocol/tot/Page/#method-startScreencast)

##### gotoOpts

type: `object`

These options will be passed to [goto#options](/#options-6) in order to resolve, prior to starting the recording.

##### imageFormat

type: `string`</br>
default: `'png'`</br>
values: `'jpeg'` | `'png'`

The image compression format.

##### quality

type: `number`</br>
default: `100`</br>
values: `'0' to '100'

Compression quality range for JPEG image format.

##### timeout

type: `number`</br>
default: `30000`

Sets the maximum navigation time.

##### tmpPath

type: `string`</br>
default: `os.tmpdir()`

The temporary directory for writing the video. This is necessary for ffmpeg, will be cleaned before the function finished.

##### withPage(page)

type: `function`
Expand Down
46 changes: 46 additions & 0 deletions packages/benchmark/src/screenshot/fixtures/example.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
<!DOCTYPE html>
<html><head>
<title>Example Domain</title>

<meta charset="utf-8">
<meta http-equiv="Content-type" content="text/html; charset=utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<style type="text/css">
body {
background-color: #f0f0f2;
margin: 0;
padding: 0;
font-family: -apple-system, system-ui, BlinkMacSystemFont, "Segoe UI", "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;

}
div {
width: 600px;
margin: 5em auto;
padding: 2em;
background-color: #fdfdff;
border-radius: 0.5em;
box-shadow: 2px 3px 7px 2px rgba(0,0,0,0.02);
}
a:link, a:visited {
color: #38488f;
text-decoration: none;
}
@media (max-width: 700px) {
div {
margin: 0 auto;
width: auto;
}
}
</style>
</head>

<body>
<div>
<h1>Example Domain</h1>
<p>This domain is for use in illustrative examples in documents. You may use this
domain in literature without prior coordination or asking for permission.</p>
<p><a href="https://www.iana.org/domains/example">More information...</a></p>
</div>


</body></html>
28 changes: 28 additions & 0 deletions packages/benchmark/src/screenshot/speed.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
'use strict'

const createBrowser = require('browserless')
const path = require('path')

const remoteBrowser = createBrowser()

const optimizeForSpeed = process.argv[2] === 'true'

const fileUrl = `file://${path.join(__dirname, './fixtures/example.html')}`

/**
* Run:
* $ hyperfine -L optimizeForSpeed false,true 'node src/screenshot/speed.js {optimizeForSpeed}'
*/
const main = async () => {
const browserless = await remoteBrowser.createContext()
await browserless.screenshot(fileUrl, { optimizeForSpeed })
await browserless.destroyContext()
await remoteBrowser.close()
}

main()
.then(remoteBrowser.close)
.catch(error => {
console.log(error)
process.exit(1)
})
3 changes: 2 additions & 1 deletion packages/browserless/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
"devDependencies": {
"@browserless/test": "^9.11.0",
"ava": "latest",
"is-ci": "latest",
"ps-list": "7"
},
"engines": {
Expand All @@ -63,4 +64,4 @@
"timeout": "30s",
"workerThreads": false
}
}
}
Loading

0 comments on commit 0cceaac

Please sign in to comment.