snfe.js

/*******************************************************************************

    uBlock Origin - a browser extension to block requests.
    Copyright (C) 2014-present Raymond Hill

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see {http://www.gnu.org/licenses/}.

    Home: https://github.com/gorhill/uBlock
*/

/* eslint-disable-next-line no-redeclare */
/* globals process */

'use strict';

/******************************************************************************/

import { strict as assert } from 'assert';
import { createRequire } from 'module';
import { readFile, writeFile, mkdir } from 'fs/promises';
import { dirname } from 'path';
import { fileURLToPath } from 'url';

const __dirname = dirname(fileURLToPath(import.meta.url));

import { enableWASM, StaticNetFilteringEngine } from './index.js';

/******************************************************************************/

const FLAGS = process.argv.slice(2);
const COMPARE = FLAGS.includes('compare');
const MAXCOST = FLAGS.includes('maxcost');
const MEDCOST = FLAGS.includes('medcost');
const MINCOST = FLAGS.includes('mincost');
const MODIFIERS = FLAGS.includes('modifiers');
const RECORD = FLAGS.includes('record');
const WASM = FLAGS.includes('wasm');
const NEED_RESULTS = COMPARE || MAXCOST || MEDCOST || MINCOST || RECORD;

// This maps puppeteer types to WebRequest types
const WEBREQUEST_OPTIONS = {
    // Consider document requests as sub_document. This is because the request
    // dataset does not contain sub_frame or main_frame but only 'document' and
    // different blockers have different behaviours.
    document: 'sub_frame',
    stylesheet: 'stylesheet',
    image: 'image',
    media: 'media',
    font: 'font',
    script: 'script',
    xhr: 'xmlhttprequest',
    fetch: 'xmlhttprequest',
    websocket: 'websocket',
    ping: 'ping',
    // other
    other: 'other',
    eventsource: 'other',
    manifest: 'other',
    texttrack: 'other',
};

/******************************************************************************/

function nanoToMilli(bigint) {
    return (Number(bigint) / 1000000).toFixed(2) + ' ms';
}

function nanoToMicro(bigint) {
    return (Number(bigint) / 1000).toFixed(2) + ' µs';
}

async function read(path) {
    return readFile(path, 'utf8');
}

async function write(path, data) {
    await mkdir(dirname(path), { recursive: true });
    return writeFile(path, data, 'utf8');
}

/******************************************************************************/

async function matchRequests(engine, requests) {
    const results = [];
    const details = {
        r: 0,
        f: undefined,
        type: '',
        url: '',
        originURL: '',
        t: 0,
    };

    let notBlockedCount = 0;
    let blockedCount = 0;
    let unblockedCount = 0;

    const start = process.hrtime.bigint();

    for ( let i = 0; i < requests.length; i++ ) {
        const request = requests[i];
        const reqstart = process.hrtime.bigint();
        details.type = WEBREQUEST_OPTIONS[request.cpt];
        details.url = request.url;
        details.originURL = request.frameUrl;
        const r = engine.matchRequest(details);
        if ( r === 0 ) {
            notBlockedCount += 1;
        } else if ( r === 1 ) {
            blockedCount += 1;
        } else {
            unblockedCount += 1;
        }
        if ( NEED_RESULTS !== true ) { continue; }
        const reqstop = process.hrtime.bigint();
        details.r = r;
        details.f = r !== 0 ? engine.toLogData().raw : undefined;
        details.t = Math.round(Number(reqstop - reqstart) / 10) / 100;
        results.push([ i, Object.assign({}, details) ]);
    }

    const stop = process.hrtime.bigint();

    console.log(`Matched ${requests.length} requests in ${nanoToMilli(stop - start)}`);
    console.log(`\tNot blocked: ${notBlockedCount} requests`);
    console.log(`\tBlocked: ${blockedCount} requests`);
    console.log(`\tUnblocked: ${unblockedCount} requests`);
    console.log(`\tAverage: ${nanoToMicro((stop - start) / BigInt(requests.length))} per request`);

    if ( RECORD ) {
        write('data/snfe.json', JSON.stringify(results, null, 2));
    }

    if ( COMPARE ) {
        const diffs = await compare(results);
        if ( Array.isArray(diffs) ) {
            write('data/snfe.diffs.json', JSON.stringify(diffs, null, 2));
        }
        console.log(`Compare: ${diffs.length} requests differ`);
    }

    if ( MAXCOST ) {
        const costly = results.slice().sort((a,b) => b[1].t - a[1].t).slice(0, 1000);
        write('data/snfe.maxcost.json', JSON.stringify(costly, null, 2));
    }

    if ( MEDCOST ) {
        const median = results.length >>> 1;
        const costly = results.slice().sort((a,b) => b[1].t - a[1].t).slice(median - 500, median + 500);
        write('data/snfe.medcost.json', JSON.stringify(costly, null, 2));
    }

    if ( MINCOST ) {
        const costly = results.slice().sort((a,b) => b[1].t - a[1].t).slice(-1000);
        write('data/snfe.mincost.json', JSON.stringify(costly, null, 2));
    }
}

async function compare(results) {
    let before;
    try {
        const raw = await read('data/snfe.json');
        before = new Map(JSON.parse(raw));
    } catch(ex) {
        console.log(ex);
        console.log('Nothing to compare');
        return;
    }
    const after = new Map(results);
    const diffs = [];
    for ( let i = 0; i < results.length; i++ ) {
        const a = before.get(i);
        const b = after.get(i);
        if ( a.r === b.r ) { continue; }
        diffs.push([ i, {
            type: a.type,
            url: a.url,
            originURL: a.originURL,
            before: { r: a.r, f: a.f, t: a.t },
            after: { r: b.r, f: b.f, t: b.t },
        }]);
    }
    return diffs;
}

/******************************************************************************/

async function matchRequestModifiers(engine, requests) {
    const results = {
        'csp': [],
        'redirect-rule': [],
        'removeparam': [],
    };

    const details = {
        f: undefined,
        type: '',
        url: '',
        originURL: '',
        t: 0,
    };

    let modifiedCount = 0;

    const start = process.hrtime.bigint();
    for ( let i = 0; i < requests.length; i++ ) {
        const request = requests[i];
        details.type = WEBREQUEST_OPTIONS[request.cpt];
        details.url = request.url;
        details.originURL = request.frameUrl;
        const r = engine.matchRequest(details);
        let modified = false;
        if ( r !== 1 && details.type === 'sub_frame' ) {
            const reqstart = process.hrtime.bigint();
            const directives = engine.matchAndFetchModifiers(details, 'csp');
            if ( directives !== undefined ) {
                modified = true;
                if ( NEED_RESULTS ) {
                    const reqstop = process.hrtime.bigint();
                    details.f = directives.map(a => `${a.result}:${a.logData().raw}`).sort();
                    details.t = Math.round(Number(reqstop - reqstart) / 10) / 100;
                    results['csp'].push([ i, Object.assign({}, details) ]);
                }
            }
        }
        if ( r === 1 ) {
            const reqstart = process.hrtime.bigint();
            const directives = engine.matchAndFetchModifiers(details, 'redirect-rule');
            if ( directives !== undefined ) {
                modified = true;
                if ( NEED_RESULTS ) {
                    const reqstop = process.hrtime.bigint();
                    details.f = directives.map(a => `${a.result}:${a.logData().raw}`).sort();
                    details.t = Math.round(Number(reqstop - reqstart) / 10) / 100;
                    results['redirect-rule'].push([ i, Object.assign({}, details) ]);
                }
            }
        }
        if ( r !== 1 && engine.hasQuery(details) ) {
            const reqstart = process.hrtime.bigint();
            const directives = engine.matchAndFetchModifiers(details, 'removeparam');
            if ( directives !== undefined ) {
                modified = true;
                if ( NEED_RESULTS ) {
                    const reqstop = process.hrtime.bigint();
                    details.f = directives.map(a => `${a.result}:${a.logData().raw}`).sort();
                    details.t = Math.round(Number(reqstop - reqstart) / 10) / 100;
                    results['removeparam'].push([ i, Object.assign({}, details) ]);
                }
            }
        }
        if ( modified ) {
            modifiedCount += 1;
        }
    }
    const stop = process.hrtime.bigint();

    console.log(`Matched-modified ${requests.length} requests in ${nanoToMilli(stop - start)}`);
    console.log(`\t${modifiedCount} modifiers found`);
    console.log(`\tAverage: ${nanoToMicro((stop - start) / BigInt(requests.length))} per request`);

    if ( RECORD ) {
        write('data/snfe.modifiers.json', JSON.stringify(results, null, 2));
    }

    if ( COMPARE ) {
        const diffs = await compareModifiers(results);
        if ( Array.isArray(diffs) ) {
            write('data/snfe.modifiers.diffs.json', JSON.stringify(diffs, null, 2));
        }
        console.log(`Compare: ${diffs.length} modified requests differ`);
    }
}

async function compareModifiers(afterResults) {
    let beforeResults;
    try {
        const raw = await read('data/snfe.modifiers.json');
        beforeResults = JSON.parse(raw);
    } catch(ex) {
        console.log(ex);
        console.log('Nothing to compare');
        return;
    }
    const diffs = [];
    for ( const modifier of [ 'csp', 'redirect-rule', 'removeparam' ] ) {
        const before = new Map(beforeResults[modifier]);
        const after = new Map(afterResults[modifier]);
        for ( const [ i, b ] of before ) {
            const a = after.get(i);
            if ( a !== undefined && JSON.stringify(a.f) === JSON.stringify(b.f) ) { continue; }
            diffs.push([ i, {
                type: b.type,
                url: b.url,
                originURL: b.originURL,
                before: { f: b.f, t: b.t },
                after: a !== undefined ? { f: a.f, t: a.t } : null,
            }]);
        }
        for ( const [ i, a ] of after ) {
            const b = before.get(i);
            if ( b !== undefined ) { continue; }
            diffs.push([ i, {
                type: a.type,
                url: a.url,
                originURL: a.originURL,
                before: null,
                after: { f: a.f, t: a.t },
            }]);
        }
    }
    return diffs;
}

/******************************************************************************/

async function bench() {
    if ( WASM ) {
        try {
            const result = await enableWASM();
            if ( result === true ) {
                console.log('WASM code paths enabled');
            }
        } catch(ex) {
            console.log(ex);
        }
    }

    const require = createRequire(import.meta.url); // jshint ignore:line
    const requests = await require('./node_modules/scaling-palm-tree/requests.json');
    const engine = await StaticNetFilteringEngine.create();

    let start = process.hrtime.bigint();
    await engine.useLists([
        read('assets/ublock/badware.txt')
            .then(raw => ({ name: 'badware', raw })),
        read('assets/ublock/filters.txt')
            .then(raw => ({ name: 'filters', raw })),
        read('assets/ublock/filters-2020.txt')
            .then(raw => ({ name: 'filters-2020', raw })),
        read('assets/ublock/filters-2021.txt')
            .then(raw => ({ name: 'filters-2021', raw })),
        read('assets/ublock/privacy.txt')
            .then(raw => ({ name: 'privacy', raw })),
        read('assets/ublock/resource-abuse.txt')
            .then(raw => ({ name: 'resource-abuse', raw })),
        read('assets/ublock/unbreak.txt')
            .then(raw => ({ name: 'unbreak.txt', raw })),
        read('assets/thirdparties/easylist/easylist.txt')
            .then(raw => ({ name: 'easylist', raw })),
        read('assets/thirdparties/easylist/easyprivacy.txt')
            .then(raw => ({ name: 'easyprivacy', raw })),
        read('assets/thirdparties/pgl.yoyo.org/as/serverlist')
            .then(raw => ({ name: 'PGL', raw })),
        read('assets/thirdparties/urlhaus-filter/urlhaus-filter-online.txt')
            .then(raw => ({ name: 'urlhaus', raw })),
    ]);
    let stop = process.hrtime.bigint();
    console.log(`Filter lists parsed-compiled-loaded in ${nanoToMilli(stop - start)}`);

    // Dry run to let JS engine optimize hot JS code paths
    for ( let i = 0; i < requests.length; i += 8 ) {
        const request = requests[i];
        void engine.matchRequest({
            type: WEBREQUEST_OPTIONS[request.cpt],
            url: request.url,
            originURL: request.frameUrl,
        });
    }

    if ( MODIFIERS === false ) {
        matchRequests(engine, requests);
    } else {
        matchRequestModifiers(engine, requests);
    }

    StaticNetFilteringEngine.release();
}

bench();

/******************************************************************************/