make it easier to use HTML filtering outside of WebExtensionBlocker (…

…#368)

* make it easier to use HTML filtering outside of `WebExtensionBlocker`
  * export `isHTMLFilteringSupported`
  * export `filterRequestHTML`
  * add `id` attribute on `Request` (takes `requestId` value in WebExtension)
* export `FilterType` from main bundle.

CHANGELOG.md

@@ -4,6 +4,16 @@
 
 *not released*
 
+## 1.3.0
+
+*2019-10-07*
+
+  * make it easier to use HTML filtering outside of `WebExtensionBlocker` [#368](https://github.com/cliqz-oss/adblocker/pull/368)
+    * export `isHTMLFilteringSupported`
+    * export `filterRequestHTML`
+    * add `id` attribute on `Request` (takes `requestId` value in WebExtension)
+  * export `FilterType` from main bundle.
+
 ## 1.2.0
 
 *2019-10-01*

lerna.json

@@ -1,5 +1,5 @@
 {
   "npmClient": "yarn",
   "useWorkspaces": true,
-  "version": "1.2.0"
+  "version": "1.3.0"
 }

packages/adblocker-benchmarks/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@cliqz/adblocker-benchmarks",
   "private": true,
-  "version": "1.2.0",
+  "version": "1.3.0",
   "description": "Content blockers benchmark",
   "author": {
     "name": "Cliqz"
@@ -25,7 +25,7 @@
     "adblock-rs": "^0.1.22"
   },
   "dependencies": {
-    "@cliqz/adblocker": "^1.2.0",
+    "@cliqz/adblocker": "^1.3.0",
     "jsdom": "^15.1.1",
     "sandboxed-module": "^2.0.3"
   },

packages/adblocker-circumvention/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cliqz/adblocker-circumvention",
-  "version": "1.2.0",
+  "version": "1.3.0",
   "description": "Cliqz adblocker circumvention for Chrome",
   "author": {
     "name": "Cliqz"
@@ -78,6 +78,6 @@
     "typescript": "^3.5.3"
   },
   "dependencies": {
-    "@cliqz/adblocker-content": "^1.2.0"
+    "@cliqz/adblocker-content": "^1.3.0"
   }
 }

packages/adblocker-content/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cliqz/adblocker-content",
-  "version": "1.2.0",
+  "version": "1.3.0",
   "description": "Cliqz adblocker library (content-scripts helpers)",
   "author": {
     "name": "Cliqz"

packages/adblocker-electron-example/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@cliqz/adblocker-electron-example",
   "private": true,
-  "version": "1.2.0",
+  "version": "1.3.0",
   "description": "Cliqz adblocker Puppeteer wrapper",
   "author": {
     "name": "Cliqz"
@@ -59,7 +59,7 @@
     }
   ],
   "dependencies": {
-    "@cliqz/adblocker-electron": "^1.2.0",
+    "@cliqz/adblocker-electron": "^1.3.0",
     "electron": "^6.0.1",
     "node-fetch": "^2.6.0",
     "ts-node": "^8.3.0"

packages/adblocker-electron/adblocker.ts

@@ -30,12 +30,14 @@ interface ElectronResponseHeaders {
  * Create an instance of `Request` from `Electron.OnBeforeRequestDetails`.
  */
 export function fromElectronDetails({
+  id,
   url,
   resourceType,
   referrer,
   webContentsId,
 }: Electron.OnBeforeRequestDetails | Electron.OnHeadersReceivedDetails): Request {
   return Request.fromRawDetails({
+    requestId: `${id}`,
     sourceUrl: referrer,
     tabId: webContentsId,
     type: (resourceType || 'other') as ElectronRequestType,

packages/adblocker-electron/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cliqz/adblocker-electron",
-  "version": "1.2.0",
+  "version": "1.3.0",
   "description": "Cliqz adblocker Electron wrapper",
   "author": {
     "name": "Cliqz"
@@ -37,8 +37,8 @@
     "electron": "^6.0.1"
   },
   "dependencies": {
-    "@cliqz/adblocker": "^1.2.0",
-    "@cliqz/adblocker-content": "^1.2.0"
+    "@cliqz/adblocker": "^1.3.0",
+    "@cliqz/adblocker-content": "^1.3.0"
   },
   "devDependencies": {
     "@types/chrome": "^0.0.89",

packages/adblocker-puppeteer-example/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@cliqz/adblocker-puppeteer-example",
   "private": true,
-  "version": "1.2.0",
+  "version": "1.3.0",
   "description": "Cliqz adblocker Puppeteer wrapper",
   "author": {
     "name": "Cliqz"
@@ -24,7 +24,7 @@
     "url": "https://github.com/cliqz-oss/adblocker/issues"
   },
   "dependencies": {
-    "@cliqz/adblocker-puppeteer": "^1.2.0",
+    "@cliqz/adblocker-puppeteer": "^1.3.0",
     "node-fetch": "^2.6.0",
     "puppeteer": "^1.18.1",
     "ts-node": "^8.3.0"

packages/adblocker-puppeteer/adblocker.ts

@@ -25,8 +25,10 @@ import {
  */
 export function fromPuppeteerDetails(details: puppeteer.Request): Request {
   const frame = details.frame();
+  const sourceUrl = frame !== null ? frame.url() : undefined;
   return Request.fromRawDetails({
-    sourceUrl: frame !== null ? frame.url() : undefined,
+    requestId: `${details.resourceType()} ${details.url()} ${sourceUrl}`,
+    sourceUrl,
     type: details.resourceType(),
     url: details.url(),
   });

packages/adblocker-puppeteer/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cliqz/adblocker-puppeteer",
-  "version": "1.2.0",
+  "version": "1.3.0",
   "description": "Cliqz adblocker Puppeteer wrapper",
   "author": {
     "name": "Cliqz"
@@ -37,8 +37,8 @@
     "puppeteer": "^1.18.1"
   },
   "dependencies": {
-    "@cliqz/adblocker": "^1.2.0",
-    "@cliqz/adblocker-content": "^1.2.0",
+    "@cliqz/adblocker": "^1.3.0",
+    "@cliqz/adblocker-content": "^1.3.0",
     "@types/puppeteer": "^1.12.4",
     "tldts-experimental": "^5.3.0"
   },

packages/adblocker-webextension-cosmetics/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cliqz/adblocker-webextension-cosmetics",
-  "version": "1.2.0",
+  "version": "1.3.0",
   "description": "Enable cosmetics in WebExtension content blocker using Cliqz adblocker",
   "author": {
     "name": "Cliqz"
@@ -84,6 +84,6 @@
     "typescript": "^3.5.3"
   },
   "dependencies": {
-    "@cliqz/adblocker-content": "^1.2.0"
+    "@cliqz/adblocker-content": "^1.3.0"
   }
 }

packages/adblocker-webextension-example/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@cliqz/adblocker-webextension-example",
   "private": true,
-  "version": "1.2.0",
+  "version": "1.3.0",
   "description": "Example of WebExtension adblocker using Cliqz",
   "author": {
     "name": "Cliqz"
@@ -29,8 +29,8 @@
     "url": "https://github.com/cliqz-oss/adblocker/issues"
   },
   "dependencies": {
-    "@cliqz/adblocker-webextension": "^1.2.0",
-    "@cliqz/adblocker-webextension-cosmetics": "^1.2.0"
+    "@cliqz/adblocker-webextension": "^1.3.0",
+    "@cliqz/adblocker-webextension-cosmetics": "^1.3.0"
   },
   "devDependencies": {
     "@types/chrome": "^0.0.89",

packages/adblocker-webextension/adblocker.ts

@@ -40,6 +40,7 @@ export function fromWebRequestDetails(
   details: WebRequestBeforeRequestDetails | WebRequestHeadersReceivedDetails,
 ): Request {
   return Request.fromRawDetails({
+    requestId: details.requestId,
     sourceUrl: details.initiator || details.originUrl || details.documentUrl,
     tabId: details.tabId,
     type: details.type,
@@ -105,7 +106,7 @@ const CHARSET_HTTP_EQUIV_RE = /<meta http-equiv="content-type" content="text\/ht
 /**
  * Check if HTML filtering is possible in this browser. Only Firefox is supported.
  */
-function isHTMLFilteringSupported(): boolean {
+export function isHTMLFilteringSupported(): boolean {
   // @ts-ignore
   const browser: any = typeof browser !== 'undefined' ? browser : chrome;
 
@@ -118,12 +119,12 @@ function isHTMLFilteringSupported(): boolean {
   );
 }
 
-function filterRequestHTML(details: { requestId: string }, rules: HTMLSelector[]): void {
+export function filterRequestHTML({ id }: { id: string }, rules: HTMLSelector[]): void {
   // @ts-ignore
   const browser: any = typeof browser !== 'undefined' ? browser : chrome;
 
   // Create filter to observe loading of resource
-  const filter: StreamFilter = browser.webRequest.filterResponseData(details.requestId);
+  const filter: StreamFilter = browser.webRequest.filterResponseData(id);
   const decoder = new TextDecoder();
   const encoder = new TextEncoder();
   const htmlFilter = new StreamingHtmlFilter(rules);
@@ -211,13 +212,7 @@ export class WebExtensionBlocker extends FiltersEngine {
     chrome.runtime.onMessage.removeListener(this.onRuntimeMessage);
   }
 
-  /**
-   * Deal with request cancellation (`{ cancel: true }`) and redirection (`{ redirectUrl: '...' }`).
-   */
-  private onBeforeRequest = (
-    details: WebRequestBeforeRequestDetails,
-  ): chrome.webRequest.BlockingResponse => {
-    const request = fromWebRequestDetails(details);
+  public performHTMLFiltering(request: Request): void {
     if (request.isMainFrame()) {
       // Here we optionally perform HTML filtering. This can only be done if:
       // 1. `enableHtmlFiltering` is set to `true`.
@@ -226,10 +221,21 @@ export class WebExtensionBlocker extends FiltersEngine {
       if (this.config.enableHtmlFiltering === true && isHTMLFilteringSupported()) {
         const htmlFilters = this.getHtmlFilters(request);
         if (htmlFilters.length !== 0) {
-          filterRequestHTML(details, htmlFilters);
+          filterRequestHTML(request, htmlFilters);
         }
       }
+    }
+  }
 
+  /**
+   * Deal with request cancellation (`{ cancel: true }`) and redirection (`{ redirectUrl: '...' }`).
+   */
+  private onBeforeRequest = (
+    details: WebRequestBeforeRequestDetails,
+  ): chrome.webRequest.BlockingResponse => {
+    const request = fromWebRequestDetails(details);
+    if (request.isMainFrame()) {
+      this.performHTMLFiltering(request);
       return {};
     }
 

packages/adblocker-webextension/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cliqz/adblocker-webextension",
-  "version": "1.2.0",
+  "version": "1.3.0",
   "description": "Cliqz adblocker WebExtension wrapper",
   "author": {
     "name": "Cliqz"
@@ -50,8 +50,8 @@
     "typescript": "^3.5.3"
   },
   "dependencies": {
-    "@cliqz/adblocker": "^1.2.0",
-    "@cliqz/adblocker-content": "^1.2.0",
+    "@cliqz/adblocker": "^1.3.0",
+    "@cliqz/adblocker-content": "^1.3.0",
     "tldts-experimental": "^5.3.0"
   },
   "contributors": [

packages/adblocker/adblocker.ts

@@ -19,6 +19,7 @@ export {
 export { HTMLSelector, default as CosmeticFilter } from './src/filters/cosmetic';
 export { default as NetworkFilter } from './src/filters/network';
 export {
+  FilterType,
   IListDiff,
   IRawDiff,
   detectFilterType,

packages/adblocker/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cliqz/adblocker",
-  "version": "1.2.0",
+  "version": "1.3.0",
   "description": "Cliqz adblocker library",
   "author": {
     "name": "Cliqz"

packages/adblocker/src/engine/engine.ts

@@ -25,7 +25,7 @@ import Resources from '../resources';
 import CosmeticFilterBucket from './bucket/cosmetic';
 import NetworkFilterBucket from './bucket/network';
 
-export const ENGINE_VERSION = 38;
+export const ENGINE_VERSION = 39;
 
 // Polyfill for `btoa`
 function btoaPolyfill(buffer: string): string {

packages/adblocker/src/lists.ts

@@ -11,7 +11,7 @@ import CosmeticFilter from './filters/cosmetic';
 import NetworkFilter from './filters/network';
 import { fastStartsWith, fastStartsWithFrom } from './utils';
 
-const enum FilterType {
+export const enum FilterType {
   NOT_SUPPORTED = 0,
   NETWORK = 1,
   COSMETIC = 2,

packages/adblocker/src/request.ts

@@ -79,6 +79,7 @@ export type RequestType = WebRequestType | ElectronRequestType | PuppeteerReques
 const TOKENS_BUFFER = new TokensBuffer(300);
 
 export interface IRequestInitialization {
+  requestId: string;
   tabId: number;
 
   url: string;
@@ -97,6 +98,7 @@ export default class Request {
    * Create an instance of `Request` from raw request details.
    */
   public static fromRawDetails({
+    requestId = '0',
     tabId = 0,
     url = '',
     hostname,
@@ -123,6 +125,7 @@ export default class Request {
 
     // source URL
     return new Request({
+      requestId,
       tabId,
 
       domain,
@@ -144,6 +147,7 @@ export default class Request {
   public readonly isFirstParty: boolean;
   public readonly isThirdParty: boolean;
 
+  public readonly id: string;
   public readonly tabId: number;
   public readonly url: string;
   public readonly hostname: string;
@@ -159,6 +163,7 @@ export default class Request {
   private fuzzySignature: Uint32Array | undefined;
 
   constructor({
+    requestId,
     tabId,
 
     type,
@@ -170,6 +175,7 @@ export default class Request {
     sourceDomain,
     sourceHostname,
   }: IRequestInitialization) {
+    this.id = requestId;
     this.tabId = tabId;
     this.type = type;