Skip to content

Commit

Permalink
feat(mitm): dns over tls lookups
Browse files Browse the repository at this point in the history
  • Loading branch information
blakebyrnes committed Oct 19, 2020
1 parent fd69f97 commit 8797847
Show file tree
Hide file tree
Showing 11 changed files with 420 additions and 25 deletions.
2 changes: 2 additions & 0 deletions commons/interfaces/IHttpRequestModifierDelegate.ts
@@ -1,9 +1,11 @@
import ResourceType from '@secret-agent/core-interfaces/ResourceType';
import IResourceHeaders from '@secret-agent/core-interfaces/IResourceHeaders';
import { ConnectionOptions } from "tls";
import OriginType from './OriginType';
import IHttpResourceLoadDetails from './IHttpResourceLoadDetails';

export default interface IHttpRequestModifierDelegate {
dnsOverTlsConnectOptions?: ConnectionOptions;
modifyHeadersBeforeSend?: (request: IResourceToModify) => { [key: string]: string };

maxConnectionsPerOrigin?: number;
Expand Down
3 changes: 1 addition & 2 deletions core/lib/Session.ts
Expand Up @@ -81,9 +81,8 @@ export default class Session {
this.id,
this.emulator.userAgent.raw,
this.proxy.isReady(),
this.emulator.delegate,
);

this.mitmRequestSession.delegate = this.emulator.delegate;
}

public getBrowserEmulation() {
Expand Down
7 changes: 7 additions & 0 deletions emulator-plugins/emulate-chrome-80/index.ts
Expand Up @@ -14,6 +14,7 @@ import {
import { randomBytes } from 'crypto';
import { pickRandom } from '@secret-agent/emulators/lib/Utils';
import IUserAgent from '@secret-agent/emulators/interfaces/IUserAgent';
import { ConnectionOptions } from 'tls';
import navigator from './navigator.json';
import chrome from './chrome.json';
import codecs from './codecs.json';
Expand All @@ -30,6 +31,11 @@ export default class Chrome80 extends EmulatorPlugin {
public static emulatorId = pkg.name;
public static statcounterBrowser = 'Chrome 80.0';
public static engine = pkg.engine;
public static dnsOverTlsConnectOptions = <ConnectionOptions>{
host: '1.1.1.1',
servername: 'cloudflare-dns.com',
};

protected static agents = UserAgents.getList(
{
deviceCategory: 'desktop',
Expand Down Expand Up @@ -70,6 +76,7 @@ export default class Chrome80 extends EmulatorPlugin {
modifyHeadersBeforeSend: modifyHeaders.bind(this, this.userAgent, headerProfiles),
tlsProfileId: 'Chrome80',
tcpVars: tcpVars(this.userAgent.os),
dnsOverTlsConnectOptions: Chrome80.dnsOverTlsConnectOptions,
};
}

Expand Down
6 changes: 6 additions & 0 deletions emulator-plugins/emulate-chrome-83/index.ts
Expand Up @@ -14,6 +14,7 @@ import {
import { randomBytes } from 'crypto';
import { pickRandom } from '@secret-agent/emulators/lib/Utils';
import IUserAgent from '@secret-agent/emulators/interfaces/IUserAgent';
import { ConnectionOptions } from 'tls';
import defaultAgents from './user-agents.json';
import navigator from './navigator.json';
import chrome from './chrome.json';
Expand All @@ -30,6 +31,10 @@ export default class Chrome83 extends EmulatorPlugin {
public static emulatorId = pkg.name;
public static statcounterBrowser = 'Chrome 83.0';
public static engine = pkg.engine;
public static dnsOverTlsConnectOptions = <ConnectionOptions>{
host: '1.1.1.1',
servername: 'cloudflare-dns.com',
};

protected static agents = UserAgents.getList(
{
Expand Down Expand Up @@ -68,6 +73,7 @@ export default class Chrome83 extends EmulatorPlugin {
modifyHeadersBeforeSend: modifyHeaders.bind(this, this.userAgent, headerProfiles),
tlsProfileId: 'Chrome83',
tcpVars: tcpVars(this.userAgent.os),
dnsOverTlsConnectOptions: Chrome83.dnsOverTlsConnectOptions,
};
}

Expand Down
51 changes: 33 additions & 18 deletions mitm/handlers/RequestSession.ts
@@ -1,18 +1,19 @@
import * as http from 'http';
import { createPromise, IResolvablePromise } from '@secret-agent/commons/utils';
import ResourceType from '@secret-agent/core-interfaces/ResourceType';
import IHttpRequestModifierDelegate from '@secret-agent/commons/interfaces/IHttpRequestModifierDelegate';
import IHttpResourceLoadDetails from '@secret-agent/commons/interfaces/IHttpResourceLoadDetails';
import IResourceRequest from '@secret-agent/core-interfaces/IResourceRequest';
import IResourceHeaders from '@secret-agent/core-interfaces/IResourceHeaders';
import * as http2 from 'http2';
import IResourceResponse from '@secret-agent/core-interfaces/IResourceResponse';
import net from 'net';
import { TypedEventEmitter } from '@secret-agent/commons/eventUtils';
import { CanceledPromiseError } from '@secret-agent/commons/interfaces/IPendingWaitEvent';
import Log, { IBoundLog } from '@secret-agent/commons/Logger';
import MitmRequestAgent from '../lib/MitmRequestAgent';
import IMitmRequestContext from '../interfaces/IMitmRequestContext';
import * as http from "http";
import { createPromise, IResolvablePromise } from "@secret-agent/commons/utils";
import ResourceType from "@secret-agent/core-interfaces/ResourceType";
import IHttpRequestModifierDelegate from "@secret-agent/commons/interfaces/IHttpRequestModifierDelegate";
import IHttpResourceLoadDetails from "@secret-agent/commons/interfaces/IHttpResourceLoadDetails";
import IResourceRequest from "@secret-agent/core-interfaces/IResourceRequest";
import IResourceHeaders from "@secret-agent/core-interfaces/IResourceHeaders";
import * as http2 from "http2";
import IResourceResponse from "@secret-agent/core-interfaces/IResourceResponse";
import net from "net";
import { TypedEventEmitter } from "@secret-agent/commons/eventUtils";
import { CanceledPromiseError } from "@secret-agent/commons/interfaces/IPendingWaitEvent";
import Log, { IBoundLog } from "@secret-agent/commons/Logger";
import MitmRequestAgent from "../lib/MitmRequestAgent";
import IMitmRequestContext from "../interfaces/IMitmRequestContext";
import { Dns } from "../lib/Dns";

const { log } = Log(module);

Expand All @@ -24,8 +25,6 @@ export default class RequestSession extends TypedEventEmitter<IRequestSessionEve
[headersHash: string]: IResolvablePromise<string>;
} = {};

public delegate: IHttpRequestModifierDelegate = {};

public isClosing = false;
public blockedResources: {
types: ResourceType[];
Expand All @@ -51,17 +50,21 @@ export default class RequestSession extends TypedEventEmitter<IRequestSessionEve

private readonly pendingResources: IPendingResourceLoad[] = [];

private readonly dns: Dns;

constructor(
readonly sessionId: string,
readonly useragent: string,
readonly upstreamProxyUrlProvider: Promise<string>,
readonly delegate: IHttpRequestModifierDelegate = {},
) {
super();
RequestSession.sessions[sessionId] = this;
this.logger = log.createChild(module, {
sessionId,
});
this.requestAgent = new MitmRequestAgent(this);
this.dns = new Dns(this);
}

public async waitForBrowserResourceRequest(ctx: IMitmRequestContext) {
Expand Down Expand Up @@ -175,6 +178,17 @@ export default class RequestSession extends TypedEventEmitter<IRequestSessionEve
return this.upstreamProxyUrlProvider ? this.upstreamProxyUrlProvider : null;
}

public async lookupDns(host: string) {
if (this.dns) {
try {
return this.dns.lookupIp(host);
} catch (error) {
// if fails, pass through to returning host untouched
}
}
return host;
}

public getProxyCredentials() {
return `secret-agent:${this.sessionId}`;
}
Expand All @@ -185,6 +199,7 @@ export default class RequestSession extends TypedEventEmitter<IRequestSessionEve
pending.load.reject(new CanceledPromiseError('Canceling: Mitm Request Session Closing'));
}
await this.requestAgent.close();
this.dns.close();

// give it a second for lingering requests to finish
setTimeout(() => delete RequestSession.sessions[this.sessionId], 1e3).unref();
Expand Down Expand Up @@ -337,7 +352,7 @@ export interface IRequestSessionRequestEvent {
}

export interface IRequestSessionHttpErrorEvent {
request: IRequestSessionResponseEvent
request: IRequestSessionResponseEvent;
error: Error;
}

Expand Down
113 changes: 113 additions & 0 deletions mitm/lib/Dns.ts
@@ -0,0 +1,113 @@
import { createPromise, IResolvablePromise } from '@secret-agent/commons/utils';
import Log from '@secret-agent/commons/Logger';
import { ConnectionOptions } from 'tls';
import moment from 'moment';
import net from 'net';
import DnsOverTlsSocket from './DnsOverTlsSocket';
import RequestSession from '../handlers/RequestSession';

const { log } = Log(module);

export class Dns {
public socket: DnsOverTlsSocket;
public dnsEntries = new Map<string, IResolvablePromise<IDnsEntry>>();
private readonly dnsServer: ConnectionOptions;

constructor(readonly requestSession?: RequestSession) {
this.dnsServer = requestSession?.delegate?.dnsOverTlsConnectOptions;
}

public async lookupIp(host: string, retries = 3) {
if (!this.dnsServer || host === 'localhost' || net.isIP(host)) return host;

try {
// get cached (or in process resolver)
const cachedRecord = await this.getNextCachedARecord(host);
if (cachedRecord) return cachedRecord.ip;
} catch (error) {
if (retries === 0) throw error;
// if the cache lookup failed, likely because another lookup failed... try again
return this.lookupIp(host, retries - 1);
}

// if not found in cache, perform dns lookup
try {
const dnsEntry = await this.lookupDnsEntry(host);
return this.nextIp(dnsEntry);
} catch (error) {
log.error('DnsLookup.Error', {
sessionId: this.requestSession.sessionId,
error,
});
// fallback to host
return host;
}
}

public close() {
this.socket?.close();
}

private async lookupDnsEntry(host: string) {
const existing = this.dnsEntries.get(host);
if (existing && !existing.isResolved) return existing.promise;

try {
const dnsEntry = createPromise<IDnsEntry>();
this.dnsEntries.set(host, dnsEntry);

if (!this.socket?.isActive) {
this.socket = new DnsOverTlsSocket(
this.dnsServer,
this.requestSession,
() => (this.socket = null),
);
}

const response = await this.socket.lookupARecords(host);

const entry = <IDnsEntry>{
aRecords: response.answers
.filter(x => x.type === 'A') // gives non-query records sometimes
.map(x => ({
ip: x.data,
expiry: moment()
.add(x.ttl, 'seconds')
.toDate(),
})),
};
dnsEntry.resolve(entry);
return entry;
} catch (error) {
this.dnsEntries.get(host)?.reject(error);
this.dnsEntries.delete(host);
throw error;
}
}

private nextIp(dnsEntry: IDnsEntry) {
// implement rotating
for (let i = 0; i < dnsEntry.aRecords.length; i += 1) {
const record = dnsEntry.aRecords[i];
if (record.expiry > new Date()) {
// move record to back
dnsEntry.aRecords.splice(i, 1);
dnsEntry.aRecords.push(record);
return record;
}
}
return null;
}

private async getNextCachedARecord(name: string) {
const cached = await this.dnsEntries.get(name)?.promise;
if (cached?.aRecords?.length) {
return this.nextIp(cached);
}
return null;
}
}

interface IDnsEntry {
aRecords: { ip: string; expiry: Date }[];
}

0 comments on commit 8797847

Please sign in to comment.