Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Adds mikeal/tunnel-agent to replace the global http.Agent when using a #128

Closed
wants to merge 1 commit into from
Closed

Adds mikeal/tunnel-agent to replace the global http.Agent when using a #128

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Expand Up @@ -2,3 +2,4 @@
node_modules
temp
sandbox
test/keys/
2 changes: 1 addition & 1 deletion README.md
View file
Open in desktop
Expand Up @@ -202,7 +202,7 @@ For information about options that've changed, there's always [the changelog](ht
- `follow_max` : (or `follow`) Number of redirects to follow. Defaults to `0`. See below for more redirect options.
- `multipart` : Enables multipart/form-data encoding. Defaults to `false`. Use it when uploading files.
- `proxy` : Forwards request through HTTP(s) proxy. Eg. `proxy: 'http://user:pass@proxy.server.com:3128'`.
- `agent` : Uses an http.Agent of your choice, instead of the global, default one.
- `agent` : Uses an http.Agent of your choice, instead of the global, default one. *If a `proxy` is in use, the global default agent will be automatically replaced with a [tunnel-agent](https://github.com/mikeal/tunnel-agent), or a custom agent if `agent` is supplied.*
- `headers` : Object containing custom HTTP headers for request. Overrides defaults described below.
- `auth` : Determines what to do with provided username/password. Options are `auto`, `digest` or `basic` (default). `auto` will detect the type of authentication depending on the response headers.
- `json` : When `true`, sets content type to `application/json` and sends request body as JSON string, instead of a query string.
Expand Down
220 changes: 205 additions & 15 deletions lib/needle.js
View file
Open in desktop
Expand Up @@ -10,6 +10,7 @@ var fs = require('fs'),
https = require('https'),
url = require('url'),
stream = require('stream'),
tunnel = require('tunnel-agent'),
debug = require('debug')('needle'),
stringify = require('./querystring').build,
multipart = require('./multipart'),
Expand All @@ -28,6 +29,36 @@ user_agent += ' (Node.js ' + process.version + '; ' + process.platform + ' '

var tls_options = 'agent pfx key passphrase cert ca ciphers rejectUnauthorized secureProtocol';

var allowed_proxy_headers_default = [
'accept',
'accept-charset',
'accept-encoding',
'accept-language',
'accept-ranges',
'cache-control',
'content-encoding',
'content-language',
'content-length',
'content-location',
'content-md5',
'content-range',
'content-type',
'connection',
'date',
'expect',
'max-forwards',
'pragma',
'referer',
'te',
'transfer-encoding',
'user-agent',
'via'
];

var only_proxy_headers_default = [
'proxy-authorization'
];

//////////////////////////////////////////
// decompressors for gzip/deflate bodies

Expand Down Expand Up @@ -221,13 +252,141 @@ Needle.prototype.setup = function(uri, options) {

if (options.proxy_user)
config.headers['Proxy-Authorization'] = auth.basic(options.proxy_user, options.proxy_pass);

}

return config;
}

Needle.prototype.setup_tunnel = function(config) {

function make_proxy_headers(current, allowed) {

var whitelist = allowed.reduce(function(set, header) {
set[header.toLowerCase()] = true;
return set;
}, {});

return Object.keys(current)
.filter(function(header) {
return whitelist[header.toLowerCase()]
})
.reduce(function(set, header) {
set[header] = current[header];
return set;
}, {});
}

function make_proxy_host(uri) {

var port = uri.portA,
protocol = uri.protocol,
proxy_host = uri.hostname + ':';

if (port) {
proxy_host += port
} else if (protocol === 'https:') {
proxy_host += '443'
} else {
proxy_host += '80'
}

debug('proxy host is ' + proxy_host)

return proxy_host;
}

function get_tunnel_fn(self) {
var uri = url.parse(self.uri),
proxy = self.proxy,
uri_pcol = (uri.protocol === 'https:' ? 'https' : 'http'),
proxy_pcol = (proxy.protocol === 'https:' ? 'Https' : 'Http');

debug('tunnelfn is ' + [uri_pcol, proxy_pcol].join('Over'));
return tunnel[[uri_pcol, proxy_pcol].join('Over')];
}

function get_tunnel_options(self, config) {
var proxy = self.proxy;

debug('getting tunnel options for ', config);

return {
proxy : {
host : proxy.hostname,
port : +proxy.port,
proxyAuth : proxy.auth,
headers : self.proxy_headers
},
headers : config.http_opts.headers,
ca : config.http_opts.ca,
cert : config.http_opts.cert,
key : config.http_opts.key,
passphrase : config.http_opts.passphrase,
pfx : config.http_opts.pfx,
ciphers : config.http_opts.ciphers,
rejectUnauthorized : config.http_opts.rejectUnauthorized,
secureOptions : config.http_opts.secureOptions,
secureProtocol : config.http_opts.secureProtocol
}
}

var self = this;

if (typeof config.proxy === 'string') {
self.proxy = url.parse(config.proxy);
} else {
self.proxy = config.proxy;
}

if (!self.proxy || !self.tunnel) {
return false;
}

self.only_proxy_headers = self.only_proxy_headers || [];
self.allowed_proxy_headers = self.allowed_proxy_headers || allowed_proxy_headers_default;
var only_proxy_headers = self.only_proxy_headers.concat(only_proxy_headers_default);
var allowed_proxy_headers = self.allowed_proxy_headers.concat(only_proxy_headers);

self.proxy_headers = make_proxy_headers(config.headers, allowed_proxy_headers);
self.proxy_headers.host = make_proxy_host(url.parse(self.uri));
// only_proxy_headers.forEach(self.removeHeader, self);

self.agent = get_tunnel_fn(self)(get_tunnel_options(self, config));

return true;

}

Needle.prototype.start = function() {

function get_tunnel_option(self, config) {
// Tunnel HTTPS by default, or if a previous request in the redirect chain
// was tunneled. Allow the user to override this setting.

// If self.tunnel is already set (because this is a redirect), use the
// existing value.
if (typeof self.tunnel !== 'undefined') {
return self.tunnel;
}

// If options.tunnel is set (the user specified a value), use it.
if (typeof config.tunnel !== 'undefined') {
return config.tunnel;
}

// If the destination is HTTPS, tunnel.
if (url.parse(self.uri).protocol === 'https:') {
return true;
}

// Otherwise, leave tunnel unset, because if a later request in the redirect
// chain is HTTPS then that request (and any subsequent ones) should be
// tunneled.
return undefined;

}

var self = this,
out = new stream.PassThrough({ objectMode: false }),
uri = this.uri,
Expand All @@ -242,6 +401,13 @@ Needle.prototype.start = function() {

var config = this.setup(uri, options);

// setup tunnel, similar to request/request tunnel
self.tunnel = get_tunnel_option(self, config);
if (config.proxy) {
self.setup_tunnel(config);
}


if (data) {
if (method.toUpperCase() == 'GET') { // build query string and append to URI

Expand Down Expand Up @@ -289,28 +455,43 @@ Needle.prototype.start = function() {
return this.send_request(1, method, uri, config, post_data, out, callback);
}


Needle.prototype.get_request_opts = function(method, uri, config) {
var opts = config.http_opts,

function get_port_from_protocol(port, protocol) {
return port || (protocol == 'https:' ? 443 : 80)
}

var self = this,
opts = config.http_opts,
proxy = config.proxy,
remote = proxy ? url.parse(proxy) : url.parse(uri);
remote = url.parse(uri);

opts.protocol = remote.protocol;
opts.host = remote.hostname;
opts.port = remote.port || (remote.protocol == 'https:' ? 443 : 80);
opts.path = proxy ? uri : remote.pathname + (remote.search || '');
if (config.proxy && !this.tunnel) {
proxy = url.parse(config.proxy);
opts.protocol = proxy.protocol;
opts.host = proxy.hostname;
opts.port = get_port_from_protocol(proxy.port, proxy.protocol);
} else {
opts.protocol = remote.protocol;
opts.host = remote.hostname;
opts.port = get_port_from_protocol(remote.port, remote.protocol);
}

opts.path = proxy && !this.tunnel ? uri : remote.pathname + (remote.search || '');
opts.method = method;
opts.headers = config.headers;

if (!opts.headers['Host']) {
// if using proxy, make sure the host header shows the final destination
var target = proxy ? url.parse(uri) : remote;
opts.headers['Host'] = target.hostname;
// if (!opts.headers['Host']) {
// // if using proxy, make sure the host header shows the final destination
// var target = proxy ? url.parse(uri) : remote;
// opts.headers['Host'] = target.hostname;

// and if a non standard port was passed, append it to the port header
if (target.port && [80, 443].indexOf(target.port) === -1) {
opts.headers['Host'] += ':' + target.port;
}
}
// // and if a non standard port was passed, append it to the port header
// if (target.port && [80, 443].indexOf(target.port) === -1) {
// opts.headers['Host'] += ':' + target.port;
// }
// }

return opts;
}
Expand Down Expand Up @@ -367,7 +548,10 @@ Needle.prototype.send_request = function(count, method, uri, config, post_data,
timer = setTimeout(function() { request.abort() }, milisecs);
}

request_opts.agent = self.agent;

debug('Making request #' + count, request_opts);

var request = protocol.request(request_opts, function(resp) {

var headers = resp.headers;
Expand Down Expand Up @@ -401,6 +585,12 @@ Needle.prototype.send_request = function(count, method, uri, config, post_data,
if (config.follow_set_referer)
config.headers['Referer'] = uri;

config.protocol = url.parse(headers.location).protocol;

if (config.protocol !== uri.protocol) {
self.setup_tunnel(config);
}

config.headers['Host'] = null; // clear previous Host header to avoid conflicts.

debug('Redirecting to ' + url.resolve(uri, headers.location));
Expand Down
3 changes: 2 additions & 1 deletion package.json
View file
Open in desktop
Expand Up @@ -41,7 +41,8 @@
},
"dependencies": {
"debug": "^2.1.2",
"iconv-lite": "^0.4.4"
"iconv-lite": "^0.4.4",
"tunnel-agent": "^0.4.0"
},
"devDependencies": {
"mocha": "",
Expand Down