Skip to content
Permalink
Browse files

Add ability to uncloak CNAME records

Related issue:
- uBlockOrigin/uBlock-issues#780

New webext permission added: `dns`, which purpose is
to allow an extension to fetch the DNS record of
specific hostnames, reference documentation:

https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/API/dns

The webext API `dns` is available in Firefox 60+ only.

The new API will enable uBO to "uncloak" the actual
hostname used in network requests. The ability is
currently disabled by default for now -- this is only
a first commit related to the above issue to allow
advanced users to immediately use the new ability.

Four advanced settings have been created to control the
uncloaking of actual hostnames:

cnameAliasList: a space-separated list of hostnames.
Default value: unset => empty list.
Special value: * => all hostnames.
A space-separated list of hostnames => this tells uBO
to "uncloak" the  hostnames in the list will.

cnameIgnoreList: a space-separated list of hostnames.
Default value: unset => empty list.
Special value: * => all hostnames.
A space-separated list of hostnames => this tells uBO
to NOT re-run the network request through uBO's
filtering engine with the CNAME hostname. This is
useful to exclude commonly used actual hostnames
from being re-run through uBO's filtering engine, so
as to avoid pointless overhead.

cnameIgnore1stParty: boolean.
Default value: true.
Whether uBO should ignore to re-run a network request
through the filtering engine when the CNAME hostname
is 1st-party to the alias hostname.

cnameMaxTTL: number of minutes.
Default value: 120.
This tells uBO to clear its CNAME cache after the
specified time. For efficiency purpose, uBO will
cache alias=>CNAME associations for reuse so as
to reduce calls to `browser.dns.resolve`. All the
associations will be cleared after the specified time
to ensure the map does not grow too large and too
ensure uBO uses up to date CNAME information.
  • Loading branch information
gorhill committed Nov 19, 2019
1 parent 60816b6 commit 3a564c199260a857f3d78d5f12b8c3f1aa85b865
@@ -1164,16 +1164,17 @@ vAPI.Net = class {
browser.webRequest.onBeforeRequest.addListener(
details => {
this.normalizeDetails(details);
if ( this.suspendDepth === 0 || details.tabId < 0 ) {
if ( this.suspendableListener === undefined ) { return; }
return this.suspendableListener(details);
if ( this.suspendDepth !== 0 && details.tabId >= 0 ) {
return this.suspendOneRequest(details);
}
return this.suspendOneRequest(details);
return this.onBeforeSuspendableRequest(details);
},
this.denormalizeFilters({ urls: [ 'http://*/*', 'https://*/*' ] }),
[ 'blocking' ]
);
}
setOptions(/* options */) {
}
normalizeDetails(/* details */) {
}
denormalizeFilters(filters) {
@@ -1208,6 +1209,10 @@ vAPI.Net = class {
options
);
}
onBeforeSuspendableRequest(details) {
if ( this.suspendableListener === undefined ) { return; }
return this.suspendableListener(details);
}
setSuspendableListener(listener) {
this.suspendableListener = listener;
}
@@ -1242,7 +1247,7 @@ vAPI.Net = class {
this.suspendDepth -= 1;
}
if ( this.suspendDepth !== 0 ) { return; }
this.unsuspendAllRequests(this.suspendableListener);
this.unsuspendAllRequests();
}
canSuspend() {
return false;
@@ -116,6 +116,51 @@ vAPI.webextFlavor = {

/******************************************************************************/

{
const punycode = self.punycode;
const reCommonHostnameFromURL = /^https?:\/\/([0-9a-z_][0-9a-z._-]*[0-9a-z])\//;
const reAuthorityFromURI = /^(?:[^:\/?#]+:)?(\/\/[^\/?#]+)/;
const reHostFromNakedAuthority = /^[0-9a-z._-]+[0-9a-z]$/i;
const reHostFromAuthority = /^(?:[^@]*@)?([^:]+)(?::\d*)?$/;
const reIPv6FromAuthority = /^(?:[^@]*@)?(\[[0-9a-f:]+\])(?::\d*)?$/i;
const reMustNormalizeHostname = /[^0-9a-z._-]/;

vAPI.hostnameFromURI = function(uri) {
let matches = reCommonHostnameFromURL.exec(uri);
if ( matches !== null ) { return matches[1]; }
matches = reAuthorityFromURI.exec(uri);
if ( matches === null ) { return ''; }
const authority = matches[1].slice(2);
if ( reHostFromNakedAuthority.test(authority) ) {
return authority.toLowerCase();
}
matches = reHostFromAuthority.exec(authority);
if ( matches === null ) {
matches = reIPv6FromAuthority.exec(authority);
if ( matches === null ) { return ''; }
}
let hostname = matches[1];
while ( hostname.endsWith('.') ) {
hostname = hostname.slice(0, -1);
}
if ( reMustNormalizeHostname.test(hostname) ) {
hostname = punycode.toASCII(hostname.toLowerCase());
}
return hostname;
};

const psl = self.publicSuffixList;
const reIPAddressNaive = /^\d+\.\d+\.\d+\.\d+$|^\[[\da-zA-Z:]+\]$/;

vAPI.domainFromHostname = function(hostname) {
return reIPAddressNaive.test(hostname)
? hostname
: psl.getDomain(hostname);
};
}

/******************************************************************************/

vAPI.download = function(details) {
if ( !details.url ) { return; }
const a = document.createElement('a');
@@ -75,6 +75,7 @@
"open_in_tab": true
},
"permissions": [
"dns",
"menus",
"privacy",
"storage",
@@ -60,6 +60,20 @@
constructor() {
super();
this.pendingRequests = [];
this.cnames = new Map();
this.cnameAliasList = null;
this.cnameIgnoreList = null;
this.url = new URL(vAPI.getURL('/'));
this.cnameMaxTTL = 60;
this.cnameTimer = undefined;
}
setOptions(options) {
super.setOptions(options);
this.cnameAliasList = this.regexFromStrList(options.cnameAliasList);
this.cnameIgnoreList = this.regexFromStrList(options.cnameIgnoreList);
this.cnameIgnore1stParty = options.cnameIgnore1stParty === true;
this.cnameMaxTTL = options.cnameMaxTTL || 120;
this.cnames.clear();
}
normalizeDetails(details) {
if ( mustPunycode && !reAsciiHostname.test(details.url) ) {
@@ -109,6 +123,87 @@
}
return Array.from(out);
}
processCanonicalName(cname, details) {
this.url.href = details.url;
details.cnameOf = this.url.hostname;
this.url.hostname = cname;
details.url = this.url.href;
return super.onBeforeSuspendableRequest(details);
}
recordCanonicalName(hn, record) {
let cname =
typeof record.canonicalName === 'string' &&
record.canonicalName !== hn
? record.canonicalName
: '';
if (
cname !== '' &&
this.cnameIgnore1stParty &&
vAPI.domainFromHostname(cname) === vAPI.domainFromHostname(hn)
) {
cname = '';
}
if (
cname !== '' &&
this.cnameIgnoreList !== null &&
this.cnameIgnoreList.test(cname)
) {

cname = '';
}
this.cnames.set(hn, cname);
if ( this.cnameTimer === undefined ) {
this.cnameTimer = self.setTimeout(
( ) => {
this.cnameTimer = undefined;
this.cnames.clear();
},
this.cnameMaxTTL * 60000
);
}
return cname;
}
regexFromStrList(list) {
if (
typeof list !== 'string' ||
list.length === 0 ||
list === 'unset'
) {
return null;
}
if ( list === '*' ) {
return /^./;
}
return new RegExp(
'(?:^|\.)(?:' +
list.trim()
.split(/\s+/)
.map(a => a.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))
.join('|') +
')$'
);
}
onBeforeSuspendableRequest(details) {
let r = super.onBeforeSuspendableRequest(details);
if ( r !== undefined ) { return r; }
if ( this.cnameAliasList === null ) { return; }
const hn = vAPI.hostnameFromURI(details.url);
let cname = this.cnames.get(hn);
if ( cname === '' ) { return; }
if ( cname !== undefined ) {
return this.processCanonicalName(cname, details);
}
if ( this.cnameAliasList.test(hn) === false ) {
this.cnames.set(hn, '');
return;
}
return browser.dns.resolve(hn, [ 'canonical_name' ]).then(rec => {
const cname = this.recordCanonicalName(hn, rec);
if ( cname === '' ) { return; }
return this.processCanonicalName(cname, details);

});
}
suspendOneRequest(details) {
const pending = {
details: Object.assign({}, details),
@@ -121,11 +216,11 @@
this.pendingRequests.push(pending);
return pending.promise;
}
unsuspendAllRequests(resolver) {
unsuspendAllRequests() {
const pendingRequests = this.pendingRequests;
this.pendingRequests = [];
for ( const entry of pendingRequests ) {
entry.resolve(resolver(entry.details));
entry.resolve(this.onBeforeSuspendableRequest(entry.details));
}
}
canSuspend() {
@@ -268,6 +268,9 @@ body.colorBlind #vwRenderer .logEntry > div.cosmeticRealm,
body.colorBlind #vwRenderer .logEntry > div.redirect {
background-color: rgba(0, 19, 110, 0.1);
}
#vwRenderer .logEntry > div[data-cnameof] {
color: mediumblue;
}
#vwRenderer .logEntry > div[data-type="tabLoad"] {
background-color: #666;
color: white;
@@ -46,6 +46,10 @@ const µBlock = (( ) => { // jshint ignore:line
cacheStorageAPI: 'unset',
cacheStorageCompression: true,
cacheControlForFirefox1376932: 'no-cache, no-store, must-revalidate',
cnameAliasList: 'unset',
cnameIgnoreList: 'unset',
cnameIgnore1stParty: true,
cnameMaxTTL: 120,
consoleLogLevel: 'unset',
debugScriptlets: false,
debugScriptletInjector: false,
@@ -30,6 +30,7 @@
this.tstamp = 0;
this.realm = '';
this.type = undefined;
this.cnameOf = undefined;
this.url = undefined;
this.hostname = undefined;
this.domain = undefined;
@@ -65,6 +66,7 @@
this.realm = '';
this.type = details.type;
this.setURL(details.url);
this.cnameOf = details.cnameOf !== undefined ? details.cnameOf : '';
this.docId = details.type !== 'sub_frame'
? details.frameId
: details.parentFrameId;

13 comments on commit 3a564c1

@uBlock-user

This comment has been minimized.

Copy link
Contributor

uBlock-user replied Nov 19, 2019

image

CNAME'd network request are being shown blue in color, on purpose ?

Edit: I see it here -- 3a564c1#diff-303102799b6a1f6c9e421aff823cac08R272

@gorhill

This comment has been minimized.

Copy link
Owner Author

gorhill replied Nov 19, 2019

Highlighting them with a different visual is important, the browser did not really issue these requests, uBO synthesized them using the CNAME and replayed them through the filtering engine.

@uBlock-user

This comment has been minimized.

Copy link
Contributor

uBlock-user replied Nov 19, 2019

Okay but now how do you identify which first party domain amoung all the first-party domains is NOT having the same CNAME as the website we're visiting? I don't see any identification being made there..

@gorhill

This comment has been minimized.

Copy link
Owner Author

gorhill replied Nov 19, 2019

Click the entry for details, synthetic requests will have a field "CNAME of".

@uBlock-user

This comment has been minimized.

Copy link
Contributor

uBlock-user replied Nov 19, 2019

image

I have a suggestion regarding coloring the f7ds.liberation.fr domain in liberation.fr case if you don't mind hearing about it here.

@gorhill

This comment has been minimized.

Copy link
Owner Author

gorhill replied Nov 19, 2019

You probably want to suggest to rewind the logger to associate already processed rows to newly emitted rows -- I rather not go down that road, so for now this is what works best without ending with a mess of code.

@uBlock-user

This comment has been minimized.

Copy link
Contributor

uBlock-user replied Nov 19, 2019

rewind the logger to associate already processed rows to newly emitted rows -- I rather not go down that road

No, that's not what I was going for, my suggestion is color all the domains in red that fail CNAME test. For example, f7ds.liberation.fr, in the logger and in the popup panel, so I know that there's something suspicious about that domain when I open the popup panel or the logger, and I will start investigating it and add afilter/rule about it. Currently only people who know about f7ds.liberation.fr will do something about it, but users won't be as there's NO indication in the popup and in the logger that anything is wrong with f7ds.liberation.fr entry, so make identification of such domains easier in the popup panel/logger. Doable ?

@gorhill

This comment has been minimized.

Copy link
Owner Author

gorhill replied Nov 19, 2019

No, that's not what I was going for ... For example, f7ds.liberation.fr, in the logger

It's what you are going for.

When f7ds.liberation.fr entry is emitted to the logger, uBO does not yet know this particular hostname is an alias for atc.eulerian.net, this knowledge is acquired after.

@uBlock-user

This comment has been minimized.

Copy link
Contributor

uBlock-user replied Nov 19, 2019

Well then the identification of the CNAME hostname who fail the test will remain difficult and troublesome and users will have to be self-aware of this happening to take any action.

@gorhill

This comment has been minimized.

Copy link
Owner Author

gorhill replied Nov 19, 2019

I disagree with this assessment. And I don't understand what "fail the test" means, CDNs are often aliased and that is not a failure, it's by design.

@uBlock-user

This comment has been minimized.

Copy link
Contributor

uBlock-user replied Nov 19, 2019

I don't understand what "fail the test" means

f7ds.liberation.fr's Reverse DNS resolves to atc.eulerian.net which shouldn't be the case because it's a first party sub-domain like medias.liberation.fr and should match www.liberation.fr, like medias.liberation.fr but it doesn't. CDNs are often aliased that is true but on a third-party domain and it's easy to identify that domain even if it's first-party because they have a prefix/suffix in the sub-domain name which helps, this one doesn't, looks morally and ethically wrong and suspicious.

This change of IP address/Reverse DNS of one particular sub-domain to another third-party entity is of pure malice and doesn't fall into the category of "by-design".

@gorhill

This comment has been minimized.

Copy link
Owner Author

gorhill replied Nov 19, 2019

uBO is code and the code does not know that atc.eulerian.net is bad -- it just relays the information, so uBO's code does not see f7ds.liberation.fr => atc.eulerian.net as a failure, the same way it does not see arstechnicarp.cachefly.net => cdn.arstechnica.net as a failure. Whether the actual hostname of an aliased hostname is undesirable is for filter list maintainers to decide.

I just committed a new ability to uncloak aliased hostnames, to assist filter list maintainers in creating filters, the next step will be to give filter list maintainers the ability to specific what hostname should be uncloaked such that filters may act on the actual hostname when filters for the alias are deemed unreliable. Whoever sees a blue entry from atc.eulerian.net in the logger can just click for more details and find out that it's aliased to f7ds.liberation.fr, and thus the next step for me will be to provide filter list maintainers the ability to tell uBO to uncloak network requests to liberation.fr such that liberation.fr attempts at evading filter lists are foiled.

It seems your mindset is elsewhere and I can't understand it -- you are literally lecturing me about why atc.eulerian.net is undesirable as if I was clueless.

@uBlock-user

This comment has been minimized.

Copy link
Contributor

uBlock-user replied Nov 19, 2019

It seems your mindset is elsewhere and I can't understand it -- you are literally lecturing me about why atc.eulerian.net is undesirable as if I was clueless.

I answered the what I meant by "fail the test" as you said you don't understand that part in 3a564c1#commitcomment-36036305 . It's not a lecture. It's the issue of identification of that said sub-domain either via popup-panel or via logger, which is still difficult and not easier in any way for non-filterlist maintainers in the current way, thats all. I'm speaking from a user-centric perspective, not from a filterlist maintainer perspective.

Anyways, since you answered already in 3a564c1#commitcomment-36035885, so I rest my case.

@gorhill I just saw atc.eulerian.net appearing in the popup panel which does the job for me , shall I delete all these comments so to remove the unnecessary noise that got added ?

Please sign in to comment.
You can’t perform that action at this time.