diff --git a/src/js/background.js b/src/js/background.js index 5b6a3e846560c..c08f91c81187b 100644 --- a/src/js/background.js +++ b/src/js/background.js @@ -142,7 +142,7 @@ const µBlock = (( ) => { // jshint ignore:line // Read-only systemSettings: { compiledMagic: 21, // Increase when compiled format changes - selfieMagic: 21, // Increase when selfie format changes + selfieMagic: 22, // Increase when selfie format changes }, restoreBackupSettings: { diff --git a/src/js/messaging.js b/src/js/messaging.js index cb70dcfef9668..be2f64f825ec5 100644 --- a/src/js/messaging.js +++ b/src/js/messaging.js @@ -1478,18 +1478,13 @@ const logCSPViolations = function(pageStore, request) { if ( cspData === undefined ) { cspData = new Map(); - const policies = []; - const logData = []; - µb.staticNetFilteringEngine.matchAndFetchData( - 'csp', - request.docURL, - policies, - logData - ); - for ( let i = 0; i < policies.length; i++ ) { - cspData.set(policies[i], logData[i]); + const staticDirectives = + µb.staticNetFilteringEngine.matchAndFetchData(fctxt, 'csp'); + for ( const directive of staticDirectives ) { + if ( directive.result !== 1 ) { continue; } + cspData.set(directive.data, directive.logData()); } - + fctxt.type = 'inline-script'; fctxt.filter = undefined; if ( pageStore.filterRequest(fctxt) === 1 ) { diff --git a/src/js/static-net-filtering.js b/src/js/static-net-filtering.js index 93050a430f54c..609e09a3bdc91 100644 --- a/src/js/static-net-filtering.js +++ b/src/js/static-net-filtering.js @@ -1372,9 +1372,9 @@ registerFilterClass(FilterOriginMixedSet); /******************************************************************************/ const FilterDataHolder = class { - constructor(dataType, dataStr) { + constructor(dataType, data) { this.dataType = dataType; - this.dataStr = dataStr; + this.data = data; this.wrapped = undefined; } @@ -1382,12 +1382,18 @@ const FilterDataHolder = class { return this.wrapped.match(url, tokenBeg); } + matchAndFetchData(type, url, tokenBeg, out) { + if ( this.dataType === type && this.match(url, tokenBeg) ) { + out.push(this); + } + } + logData() { const out = this.wrapped.logData(); - out.compiled = [ this.fid, this.dataType, this.dataStr, out.compiled ]; + out.compiled = [ this.fid, this.dataType, this.data, out.compiled ]; let opt = this.dataType; - if ( this.dataStr !== '' ) { - opt += `=${this.dataStr}`; + if ( this.data !== '' ) { + opt += `=${this.data}`; } if ( out.opts === undefined ) { out.opts = opt; @@ -1401,13 +1407,13 @@ const FilterDataHolder = class { return [ this.fid, this.dataType, - this.dataStr, + this.data, this.wrapped.compile(toSelfie) ]; } static compile(details) { - return [ FilterDataHolder.fid, details.dataType, details.dataStr ]; + return [ FilterDataHolder.fid, details.dataType, details.data ]; } static load(args) { @@ -1419,26 +1425,29 @@ const FilterDataHolder = class { registerFilterClass(FilterDataHolder); -// Helper class for storing instances of FilterDataHolder. +// Helper class for storing instances of FilterDataHolder which were found to +// be a match. -const FilterDataHolderEntry = class { - constructor(categoryBits, tokenHash, fdata) { - this.categoryBits = categoryBits; - this.tokenHash = tokenHash; - this.filter = filterFromCompiledData(fdata); - this.next = undefined; +const FilterDataHolderResult = class { + constructor(bits, th, f) { + this.bits = bits; + this.th = th; + this.f = f; } - logData() { - return toLogDataInternal(this.categoryBits, this.tokenHash, this.filter); + get data() { + return this.f.data; } - compile() { - return [ this.categoryBits, this.tokenHash, this.filter.compile() ]; + get result() { + return (this.bits & AllowAction) === 0 ? 1 : 2; } - static load(data) { - return new FilterDataHolderEntry(data[0], data[1], data[2]); + logData() { + const r = toLogDataInternal(this.bits, this.th, this.f); + r.source = 'static'; + r.result = this.result; + return r; } }; @@ -1625,6 +1634,11 @@ const FilterPair = class { return false; } + matchAndFetchData(type, url, tokenBeg, out) { + this.f1.matchAndFetchData(type, url, tokenBeg, out); + this.f2.matchAndFetchData(type, url, tokenBeg, out); + } + logData() { return this.f.logData(); } @@ -1744,6 +1758,12 @@ const FilterBucket = class { return false; } + matchAndFetchData(type, url, tokenBeg, out) { + for ( const f of this.filters ) { + f.matchAndFetchData(type, url, tokenBeg, out); + } + } + logData() { if ( this.f === this.plainFilter || @@ -1883,7 +1903,7 @@ FilterParser.prototype.reset = function() { this.anchor = 0; this.badFilter = false; this.dataType = undefined; - this.dataStr = undefined; + this.data = undefined; this.elemHiding = false; this.f = ''; this.firstParty = false; @@ -2012,13 +2032,13 @@ FilterParser.prototype.parseOptions = function(s) { ) { this.parseTypeOption('data', not); this.dataType = 'csp'; - this.dataStr = opt.slice(4).trim(); + this.data = opt.slice(4).trim(); continue; } if ( opt === 'csp' && this.action === AllowAction ) { this.parseTypeOption('data', not); this.dataType = 'csp'; - this.dataStr = ''; + this.data = ''; continue; } // Used by Adguard: @@ -2407,7 +2427,6 @@ FilterContainer.prototype.reset = function() { FilterContainer.prototype.freeze = function() { const filterPairId = FilterPair.fid; const filterBucketId = FilterBucket.fid; - const filterDataHolderId = FilterDataHolder.fid; const redirectTypeValue = typeNameToTypeValue.redirect; const unserialize = µb.CompiledLineIO.unserialize; @@ -2431,20 +2450,6 @@ FilterContainer.prototype.freeze = function() { const tokenHash = args[1]; const fdata = args[2]; - // Special treatment: data-holding filters are stored separately - // because they require special matching algorithm (unlike other - // filters, ALL hits must be reported). - if ( fdata[0] === filterDataHolderId ) { - let entry = new FilterDataHolderEntry(bits, tokenHash, fdata); - let bucket = this.dataFilters.get(tokenHash); - if ( bucket !== undefined ) { - entry.next = bucket; - } - this.dataFilters.set(tokenHash, entry); - this.urlTokenizer.addKnownToken(tokenHash); - continue; - } - let bucket = this.categories.get(bits); if ( bucket === undefined ) { bucket = new Map(); @@ -2543,17 +2548,6 @@ FilterContainer.prototype.toSelfie = function(path) { return selfie; }; - const dataFiltersToSelfie = function(dataFilters) { - const selfie = []; - for ( let entry of dataFilters.values() ) { - do { - selfie.push(entry.compile(true)); - entry = entry.next; - } while ( entry !== undefined ); - } - return selfie; - }; - filterOrigin.optimize(); return Promise.all([ @@ -2579,7 +2573,6 @@ FilterContainer.prototype.toSelfie = function(path) { blockFilterCount: this.blockFilterCount, discardedCount: this.discardedCount, categories: categoriesToSelfie(this.categories), - dataFilters: dataFiltersToSelfie(this.dataFilters), urlTokenizer: this.urlTokenizer.toSelfie(), filterOriginStrSlots: filterOrigin.strSlots, }) @@ -2632,14 +2625,6 @@ FilterContainer.prototype.fromSelfie = function(path) { } this.categories.set(catbits, tokenMap); } - for ( const dataEntry of selfie.dataFilters ) { - const entry = FilterDataHolderEntry.load(dataEntry); - const bucket = this.dataFilters.get(entry.tokenHash); - if ( bucket !== undefined ) { - entry.next = bucket; - } - this.dataFilters.set(entry.tokenHash, entry); - } return true; }), ]).then(results => @@ -2847,99 +2832,109 @@ FilterContainer.prototype.fromCompiledContent = function(reader) { /******************************************************************************/ -FilterContainer.prototype.matchAndFetchData = function( - dataType, - requestURL, - out, - outlog +FilterContainer.prototype.realmMatchAndFetchData = function( + realmBits, + partyBits, + type, + out ) { - if ( this.dataFilters.size === 0 ) { return; } - - const url = this.urlTokenizer.setURL(requestURL); + const bits01 = realmBits | typeNameToTypeValue.data; + const bits11 = realmBits | typeNameToTypeValue.data | partyBits; - pageHostnameRegister = requestHostnameRegister = - µb.URI.hostnameFromURI(url); + const bucket01 = this.categories.get(bits01); + const bucket11 = partyBits !== 0 + ? this.categories.get(bits11) + : undefined; - // We need to visit ALL the matching filters. - const toAddImportant = new Map(); - const toAdd = new Map(); - const toRemove = new Map(); + if ( bucket01 === undefined && bucket11 === undefined ) { return false; } + const url = urlRegister; const tokenHashes = this.urlTokenizer.getTokens(); - let i = 0; - while ( i < 32 ) { - const tokenHash = tokenHashes[i++]; - if ( tokenHash === 0 ) { break; } - const tokenOffset = tokenHashes[i++]; - let entry = this.dataFilters.get(tokenHash); - while ( entry !== undefined ) { - const f = entry.filter; - if ( f.match(url, tokenOffset) === true ) { - if ( entry.categoryBits & 0x001 ) { - toRemove.set(f.dataStr, entry); - } else if ( entry.categoryBits & 0x002 ) { - toAddImportant.set(f.dataStr, entry); - } else { - toAdd.set(f.dataStr, entry); - } + const filters = []; + let i = 0, tokenBeg = 0, f; + for (;;) { + const th = tokenHashes[i]; + if ( th === 0 ) { return; } + tokenBeg = tokenHashes[i+1]; + if ( + (bucket01 !== undefined) && + (f = bucket01.get(th)) !== undefined + ) { + filters.length = 0; + f.matchAndFetchData(type, url, tokenBeg, filters); + for ( f of filters ) { + out.set(f.data, new FilterDataHolderResult(bits01, th, f)); } - entry = entry.next; } - } - let entry = this.dataFilters.get(this.noTokenHash); - while ( entry !== undefined ) { - const f = entry.filter; - if ( f.match(url) === true ) { - if ( entry.categoryBits & 0x001 ) { - toRemove.set(f.dataStr, entry); - } else if ( entry.categoryBits & 0x002 ) { - toAddImportant.set(f.dataStr, entry); - } else { - toAdd.set(f.dataStr, entry); + if ( + (bucket11 !== undefined) && + (f = bucket11.get(th)) !== undefined + ) { + filters.length = 0; + f.matchAndFetchData(type, url, tokenBeg, filters); + for ( f of filters ) { + out.set(f.data, new FilterDataHolderResult(bits11, th, f)); } } - entry = entry.next; + i += 2; } +}; + +/******************************************************************************/ + +FilterContainer.prototype.matchAndFetchData = function(fctxt, type) { + urlRegister = this.urlTokenizer.setURL(fctxt.url); + pageHostnameRegister = fctxt.getDocHostname(); + requestHostnameRegister = fctxt.getHostname(); + + const partyBits = fctxt.is3rdPartyToDoc() ? ThirdParty : FirstParty; + + const toAddImportant = new Map(); + this.realmMatchAndFetchData(BlockImportant, partyBits, type, toAddImportant); - if ( toAddImportant.size === 0 && toAdd.size === 0 ) { return; } + const toAdd = new Map(); + this.realmMatchAndFetchData(BlockAction, partyBits, type, toAdd); + + if ( toAddImportant.size === 0 && toAdd.size === 0 ) { return []; } + + const toRemove = new Map(); + this.realmMatchAndFetchData(AllowAction, partyBits, type, toRemove); - // Remove entries overriden by other filters. + // Remove entries overriden by important block filters. for ( const key of toAddImportant.keys() ) { toAdd.delete(key); toRemove.delete(key); } - for ( const key of toRemove.keys() ) { - if ( key === '' ) { + + // Special case, except-all: + // - Except-all applies only if there is at least one normal block filters. + // - Except-all does not apply to important block filters. + if ( toRemove.has('') ) { + if ( toAdd.size !== 0 ) { toAdd.clear(); - break; + toRemove.forEach((v, k, m) => { + if ( k !== '' ) { m.delete(k); } + }); + } else { + toRemove.clear(); } - toAdd.delete(key); } - - for ( const entry of toAddImportant ) { - out.push(entry[0]); - if ( outlog === undefined ) { continue; } - let logData = entry[1].logData(); - logData.source = 'static'; - logData.result = 1; - outlog.push(logData); - } - for ( const entry of toAdd ) { - out.push(entry[0]); - if ( outlog === undefined ) { continue; } - let logData = entry[1].logData(); - logData.source = 'static'; - logData.result = 1; - outlog.push(logData); - } - if ( outlog !== undefined ) { - for ( const entry of toRemove.values()) { - const logData = entry.logData(); - logData.source = 'static'; - logData.result = 2; - outlog.push(logData); + // Remove excepted block filters and unused exception filters. + else { + for ( const key of toRemove.keys() ) { + if ( toAdd.has(key) ) { + toAdd.delete(key); + } else { + toRemove.delete(key); + } } } + + // Merge important and normal block filters + for ( const [ key, entry ] of toAddImportant ) { + toAdd.set(key, entry); + } + return Array.from(toAdd.values()).concat(Array.from(toRemove.values())); }; /******************************************************************************/ diff --git a/src/js/traffic.js b/src/js/traffic.js index ac77e0e03888f..bfadd3f696f9b 100644 --- a/src/js/traffic.js +++ b/src/js/traffic.js @@ -844,14 +844,12 @@ const injectCSP = function(fctxt, pageStore, responseHeaders) { // Static filtering. - const logDataEntries = loggerEnabled ? [] : undefined; - - µb.staticNetFilteringEngine.matchAndFetchData( - 'csp', - fctxt.url, - cspSubsets, - logDataEntries - ); + const staticDirectives = + µb.staticNetFilteringEngine.matchAndFetchData(fctxt, 'csp'); + for ( const directive of staticDirectives ) { + if ( directive.result !== 1 ) { continue; } + cspSubsets.push(directive.data); + } // URL filtering `allow` rules override static filtering. if ( @@ -893,10 +891,11 @@ const injectCSP = function(fctxt, pageStore, responseHeaders) { // <<<<<<<< All policies have been collected // Static CSP policies will be applied. - if ( logDataEntries !== undefined ) { + + if ( loggerEnabled && staticDirectives.length !== 0 ) { fctxt.setRealm('network').setType('csp'); - for ( const entry of logDataEntries ) { - fctxt.setFilter(entry).toLogger(); + for ( const directive of staticDirectives ) { + fctxt.setFilter(directive.logData()).toLogger(); } }