feat: add builtin adblock

Author: Kikobeats

README.md

@@ -19,7 +19,7 @@ Although you can think [puppeteer](https://github.com/GoogleChrome/puppeteer) co
 - Sensible good defaults, aborting unnecessary requests based of what you are doing (e.g, aborting image request if you just want to get [`.html`](#htmlurl-options) content).
 - Privacy by default, blocking tracker requests.
 - Easily create a pool of instance (via [`@browserless/pool`](#pool-of-instances)).
-- Built-in AdBlocker ([soon](https://github.com/Kikobeats/browserless/issues/26)).
+- Built-in adblocker for aborting ads requests.
 
 ## Install
 
@@ -165,12 +165,12 @@ default: `['image', 'media', 'stylesheet', 'font', 'xhr']`
 
 A list of `resourceType` requests that can be aborted in order to make the process faster.
 
-##### abortTrackers
+##### adblock
 
 type: `boolean`</br>
 default: `true`
 
-It will be abort request coming for [tracking domains](https://npm.im/is-tracking-domain).
+It will be abort requests detected as ads.
 
 ### .text(url, options)
 
@@ -366,21 +366,14 @@ The target URL
 type: `string`</br>
 default: `[]`
 
-A list of `req.resourceType()` to be blocked.
+A list of `req.resourceType()` to be aborted.
 
-##### abortTrackers
+##### adblock
 
 type: `boolean`</br>
 default: `true`
 
-It will be abort request coming for [tracking domains](https://npm.im/is-tracking-domain).
-
-##### abortTrackers
-
-type: `boolean`</br>
-default: `true`
-
-It will be abort request coming for [tracking domains](https://npm.im/is-tracking-domain).
+It will be abort requests detected as ads.
 
 ##### waitFor
 

packages/browserless/src/index.js

@@ -62,14 +62,14 @@ module.exports = ({
   const evaluate = fn =>
     wrapError(page => async (url, opts = {}) => {
       const {
-        abortTrackers = true,
+        adblock = true,
         abortTypes = ['image', 'imageset', 'media', 'stylesheet', 'font', 'object', 'sub_frame'],
         ...args
       } = opts
 
       const response = await goto(page, {
         url,
-        abortTrackers,
+        adblock,
         abortTypes,
         ...args
       })
@@ -78,8 +78,8 @@ module.exports = ({
     })
 
   const screenshot = wrapError(page => async (url, opts = {}) => {
-    const { device = 'macbook pro 13', type = 'png', viewport, ...args } = opts
-    await goto(page, { url, device, ...args })
+    const { adblock = true, device = 'macbook pro 13', type = 'png', viewport, ...args } = opts
+    await goto(page, { url, device, adblock, ...args })
     return page.screenshot({ type, ...args })
   })
 

packages/goto/package.json

@@ -29,9 +29,10 @@
   ],
   "dependencies": {
     "@browserless/devices": "^5.2.3",
+    "got": "~9.6.0",
+    "tldts": "~4.0.5",
+    "@cliqz/adblocker": "~0.8.0",
     "debug": "~4.1.0",
-    "extract-domain": "~2.0.4",
-    "is-tracking-domain": "~1.1.6",
     "p-timeout": "~2.0.1",
     "require-one-of": "~1.0.2"
   },
@@ -42,7 +43,8 @@
     "src"
   ],
   "scripts": {
-    "test": "exit 0"
+    "test": "exit 0",
+    "postinstall": "node scripts/postinstall"
   },
   "license": "MIT"
 }

packages/goto/scripts/postinstall.js

@@ -0,0 +1,69 @@
+'use strict'
+
+const { promisify } = require('util')
+const { EOL } = require('os')
+const got = require('got')
+const fs = require('fs')
+
+const writeFile = promisify(fs.writeFile)
+
+const FILTERS = [
+  // uBlock Origin –  https://github.com/uBlockOrigin/uAssets/tree/master/filters
+  'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/filters.txt',
+  'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/badware.txt',
+  'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/privacy.txt',
+  'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/resource-abuse.txt',
+  'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/unbreak.txt',
+  'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/annoyances.txt',
+
+  // Easylist – https://easylist.to/
+  'https://easylist.to/easylist/easylist.txt',
+  'https://easylist.to/easylist/easyprivacy.txt',
+  'https://easylist.to/easylist/fanboy-annoyance.txt',
+  'https://easylist.to/easylist/fanboy-social.txt',
+
+  // Other
+  'http://pgl.yoyo.org/as/serverlist.php?hostformat=adblockplus;showintro=0&mimetype=plaintext',
+  'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/recipes/recipes_en.txt',
+  'https://www.i-dont-care-about-cookies.eu/abp/',
+  'https://raw.githubusercontent.com/liamja/Prebake/master/obtrusive.txt',
+  'https://gnuzilla.gnu.org/filters/blacklist.txt'
+]
+
+const rulesFromURL = async url => {
+  const { body } = await got(url)
+  return body
+}
+
+const rulesFromURLs = async urls => {
+  const lists = await Promise.all(urls.map(rulesFromURL))
+
+  // Remove duplicate rules
+  const set = lists.reduce((acc, list) => {
+    const rules = list.split(EOL).filter(rule => {
+      // remove empty lines
+      if (rule === '') return false
+      // remove comments
+      if (rule.startsWith('!')) return false
+      return true
+    })
+
+    return new Set([...acc, ...rules])
+  }, new Set())
+
+  return Array.from(set)
+}
+
+const toTxt = (filepath, data) => writeFile(filepath, data)
+
+const main = async urls => {
+  const rules = await rulesFromURLs(urls)
+  await toTxt('src/rules.txt', rules.join(EOL))
+}
+
+main(FILTERS)
+  .catch(err => {
+    console.error(err)
+    process.exit(1)
+  })
+  .then(process.exit)

packages/goto/src/index.js

@@ -1,14 +1,47 @@
 'use strict'
 
+const { FiltersEngine, makeRequest } = require('@cliqz/adblocker')
 const { getDevice } = require('@browserless/devices')
 const debug = require('debug')('browserless:goto')
-const extractDomain = require('extract-domain')
+const tldts = require('tldts')
+const path = require('path')
+const fs = require('fs')
 
-const isTracker = require('./is-tracker')
+const engine = FiltersEngine.parse(fs.readFileSync(path.resolve(__dirname, './rules.txt'), 'utf-8'))
 
 const isEmpty = val => val == null || !(Object.keys(val) || val).length
 
-const isExternalUrl = (domainOne, domainTwo) => domainOne !== domainTwo
+const types = {
+  document: 'main_frame',
+  eventsource: 'other',
+  fetch: 'xhr',
+  font: 'font',
+  image: 'image',
+  manifest: 'other',
+  media: 'media',
+  other: 'other',
+  script: 'script',
+  stylesheet: 'stylesheet',
+  texttrack: 'other',
+  websocket: 'websocket',
+  xhr: 'xhr'
+}
+
+/**
+ *
+ * Mapping from puppeteer request types to adblocker. This is needed because not all
+ * types from puppeteer have the same name as the webRequest APIs from browsers
+ * (which the adblocker expects).
+ *
+ * Related:
+ * - https://github.com/GoogleChrome/puppeteer/blob/v1.14.0/docs/api.md#requestresourcetype
+ * - https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/API/webRequest/ResourceType
+ */
+const webRequestType = resourceType => {
+  const type = types[resourceType]
+  if (!type) throw Error(`Type ${resourceType} not mapped`)
+  return type
+}
 
 const WAIT_UNTIL = ['networkidle0']
 
@@ -17,7 +50,7 @@ module.exports = async (
   {
     url,
     device,
-    abortTrackers,
+    adblock,
     abortTypes = [],
     waitFor = 0,
     waitUntil = WAIT_UNTIL,
@@ -31,22 +64,31 @@ module.exports = async (
 
   page.on('request', req => {
     const resourceUrl = req.url()
-
     const resourceType = req.resourceType()
 
     if (abortTypes.includes(resourceType)) {
       debug(`abort:${resourceType}:${++reqCount.abort}`, resourceUrl)
       return req.abort()
     }
 
-    const urlDomain = extractDomain(url)
-    const resourceDomain = extractDomain(resourceUrl)
-    const isExternal = isExternalUrl(urlDomain, resourceDomain)
+    if (adblock) {
+      const { match: isMatch } = engine.match(
+        makeRequest(
+          {
+            type: webRequestType(resourceType),
+            sourceUrl: req.frame().url(),
+            url: resourceUrl
+          },
+          url => tldts.parse(url)
+        )
+      )
 
-    if (abortTrackers && isExternal && isTracker(resourceDomain)) {
-      debug(`abort:tracker:${++reqCount.abort}`, resourceUrl)
-      return req.abort()
+      if (isMatch) {
+        debug(`abort:tracker:${++reqCount.abort}`, resourceUrl)
+        return req.abort()
+      }
     }
+
     debug(`continue:${resourceType}:${++reqCount.continue}`, resourceUrl)
     return req.continue()
   })