diff --git a/.gitignore b/.gitignore index 214a2ec7c..75d5271f1 100644 --- a/.gitignore +++ b/.gitignore @@ -2,5 +2,6 @@ node_modules coverage .idea npm-debug.log +.vscode package-lock.json .nyc_output \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 9c9940a2b..c2cdaa922 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,6 @@ language: node_js node_js: - - node - 10 - 8 - 6 diff --git a/Fork.md b/Fork.md new file mode 100644 index 000000000..6420b3970 --- /dev/null +++ b/Fork.md @@ -0,0 +1,28 @@ +# Fork Guidelines + +This fork is meant to always follow changes in the `upstream`. +That is, there should be no changes done in this fork which would prevent drop in replacement of `postman-request` with `request` +(and vice-versa) + +## Setting up for maintainance + +1. Clone this repository + + git clone https://github.com/postmanlabs/postman-request.git + +2. Create another git origin, so that we can easily track upstream changes, and merge them as necessary. + + git remote add upstreamrepo https://github.com/user/repo.git + +3. Fetch commits from the upstream + + git fetch upstreamrepo + +4. Checkout a local branch from the upstream master + + git checkout --track upstreamrepo/master -b upstreammaster + +You can now merge in upstream changes as required, + + git checkout master + git merge upstreammaster diff --git a/README.md b/README.md index b71ea6428..a2c0f32ca 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,30 @@ # Request - Simplified HTTP client -[![npm package](https://nodei.co/npm/request.png?downloads=true&downloadRank=true&stars=true)](https://nodei.co/npm/request/) - -[![Build status](https://img.shields.io/travis/request/request/master.svg?style=flat-square)](https://travis-ci.org/request/request) -[![Coverage](https://img.shields.io/codecov/c/github/request/request.svg?style=flat-square)](https://codecov.io/github/request/request?branch=master) -[![Coverage](https://img.shields.io/coveralls/request/request.svg?style=flat-square)](https://coveralls.io/r/request/request) -[![Dependency Status](https://img.shields.io/david/request/request.svg?style=flat-square)](https://david-dm.org/request/request) -[![Known Vulnerabilities](https://snyk.io/test/npm/request/badge.svg?style=flat-square)](https://snyk.io/test/npm/request) -[![Gitter](https://img.shields.io/badge/gitter-join_chat-blue.svg?style=flat-square)](https://gitter.im/request/request?utm_source=badge) - +This is a fork of the excellent `request` module, which is used inside Postman Runtime. It contains a few bugfixes that are not fixed in `request`: + +- Handling of old-style deflate responses: https://github.com/request/request/issues/2197 +- Correct encoding of URL Parameters: https://github.com/nodejs/node/issues/8321 +- Redirect behavior for 307 responses when Host header is set: https://github.com/request/request/issues/2666 +- Fix missing `content-length` header for streaming requests: https://github.com/request/request/issues/316 +- Exception handling for large form-data: https://github.com/request/request/issues/1561 +- Added feature to bind on stream emits via options +- Allowed sending request body with HEAD method +- Added option to retain `authorization` header when a redirect happens to a different hostname +- Reinitialize FormData stream on 307 or 308 redirects +- Respect form-data fields ordering +- Fixed authentication leak in 307 and 308 redirects +- Added `secureConnect` to timings and `secureHandshake` to timingPhases +- Fixed `Request~getNewAgent` to account for `passphrase` while generating poolKey +- Added support for extending the root CA certificates +- Added `verbose` mode to bubble up low-level request-response information ## Super simple to use Request is designed to be the simplest way possible to make http calls. It supports HTTPS and follows redirects by default. ```js -const request = require('request'); +const request = require('postman-request'); request('http://www.google.com', function (error, response, body) { console.error('error:', error); // Print the error if one occurred console.log('statusCode:', response && response.statusCode); // Print the response status code if a response was received @@ -349,12 +357,12 @@ of stars and forks for the request repository. This requires a custom `User-Agent` header as well as https. ```js -const request = require('request'); +const request = require('postman-request'); const options = { url: 'https://api.github.com/repos/request/request', headers: { - 'User-Agent': 'request' + 'User-Agent': 'postman-request' } }; @@ -612,7 +620,7 @@ const fs = require('fs') , certFile = path.resolve(__dirname, 'ssl/client.crt') , keyFile = path.resolve(__dirname, 'ssl/client.key') , caFile = path.resolve(__dirname, 'ssl/ca.cert.pem') - , request = require('request'); + , request = require('postman-request'); const options = { url: 'https://api.some-server.com/', @@ -635,7 +643,7 @@ const fs = require('fs') , path = require('path') , certFile = path.resolve(__dirname, 'ssl/client.crt') , keyFile = path.resolve(__dirname, 'ssl/client.key') - , request = require('request'); + , request = require('postman-request'); const options = { url: 'https://api.some-server.com/', @@ -679,7 +687,7 @@ request.get({ The `ca` value can be an array of certificates, in the event you have a private or internal corporate public-key infrastructure hierarchy. For example, if you want to connect to https://api.some-server.com which presents a key chain consisting of: 1. its own public key, which is signed by: -2. an intermediate "Corp Issuing Server", that is in turn signed by: +2. an intermediate "Corp Issuing Server", that is in turn signed by: 3. a root CA "Corp Root CA"; you can configure your request as follows: @@ -696,6 +704,115 @@ request.get({ }); ``` +### Using `options.verbose` + +Using this option the debug object holds low level request response information like remote address, negotiated ciphers etc. Example debug object: + +```js +request({url: 'https://www.google.com', verbose: true}, function (error, response, body, debug) { + // debug: + /* + [ + { + "request": { + "method": "GET", + "href": "https://www.google.com/", + "httpVersion": "1.1" + }, + "session": { + "id": "9a1ac0d7-b757-48ad-861c-d59d6af5f43f", + "reused": false, + "data": { + "addresses": { + "local": { + "address": "8.8.4.4", + "family": "IPv4", + "port": 61632 + }, + "remote": { + "address": "172.217.31.196", + "family": "IPv4", + "port": 443 + } + }, + "tls": { + "reused": false, + "authorized": true, + "authorizationError": null, + "cipher": { + "name": "ECDHE-ECDSA-AES128-GCM-SHA256", + "version": "TLSv1/SSLv3" + }, + "protocol": "TLSv1.2", + "ephemeralKeyInfo": { + "type": "ECDH", + "name": "X25519", + "size": 253 + }, + "peerCertificate": { + "subject": { + "country": "US", + "stateOrProvince": "California", + "locality": "Mountain View", + "organization": "Google LLC", + "commonName": "www.google.com", + "alternativeNames": "DNS:www.google.com" + }, + "issuer": { + "country": "US", + "organization": "Google Trust Services", + "commonName": "Google Internet Authority G3" + }, + "validFrom": "2019-03-01T09:46:35.000Z", + "validTo": "2019-05-24T09:25:00.000Z", + "fingerprint": "DF:6B:95:81:C6:03:EB:ED:48:EB:6C:CF:EE:FE:E6:1F:AD:01:78:34", + "serialNumber": "3A15F4C87FB4D33993D3EEB3BF4AE5E4" + } + } + } + }, + "response": { + "statusCode": 200, + "httpVersion": "1.1" + }, + "timingStart": 1552908287924, + "timingStartTimer": 805.690674, + "timings": { + "socket": 28.356426000000056, + "lookup": 210.3752320000001, + "connect": 224.57993499999998, + "secureConnect": 292.80315800000017, + "response": 380.61268100000007, + "end": 401.8332560000001 + } + } + ] + */ +}); +``` + + +### Extending root CAs + +When this feature is enabled, the root CAs can be extended using the `extraCA` option. The file should consist of one or more trusted certificates in PEM format. + +This is similar to [NODE_EXTRA_CA_CERTS](https://nodejs.org/api/cli.html#cli_node_extra_ca_certs_file). But, if `options.ca` is specified, those will be extended as well. + +```js +// enable extending CAs +request.enableNodeExtraCACerts(); + +// request with extra CA certs +request.get({ + url: 'https://api.some-server.com/', + extraCA: fs.readFileSync('Extra CA Certificates .pem') +}); + +// disable this feature +request.disableNodeExtraCACerts() + +``` + [back to top](#table-of-contents) @@ -708,7 +825,7 @@ The `options.har` property will override the values: `url`, `method`, `qs`, `hea A validation step will check if the HAR Request format matches the latest spec (v1.2) and will skip parsing if not matching. ```js - const request = require('request') + const request = require('postman-request') request({ // will be ignored method: 'GET', @@ -801,6 +918,7 @@ The first argument can be either a `url` or an `options` object. The only requir - `followRedirect` - follow HTTP 3xx responses as redirects (default: `true`). This property can also be implemented as function which gets `response` object as a single argument and should return `true` if redirects should continue or `false` otherwise. - `followAllRedirects` - follow non-GET HTTP 3xx responses as redirects (default: `false`) - `followOriginalHttpMethod` - by default we redirect to HTTP method GET. you can enable this property to redirect to the original HTTP method (default: `false`) +- `followAuthorizationHeader` - retain `authorization` header when a redirect happens to a different hostname (default: `false`) - `maxRedirects` - the maximum number of redirects to follow (default: `10`) - `removeRefererHeader` - removes the referer header when a redirect happens (default: `false`). **Note:** if true, referer header set in the initial request is preserved during redirect chain. @@ -829,6 +947,7 @@ The first argument can be either a `url` or an `options` object. The only requir [linux-timeout]: http://www.sekuda.com/overriding_the_default_linux_kernel_20_second_tcp_socket_connect_timeout +- `maxResponseSize` - Abort request if the response size exceeds this threshold (bytes). --- - `localAddress` - local interface to bind for network connections. @@ -848,6 +967,7 @@ The first argument can be either a `url` or an `options` object. The only requir --- +- `disableUrlEncoding` - if `true`, it will not use postman-url-encoder to encode URL. It means that if URL is given as object, it will be used as it is without doing any encoding. But if URL is given as string, it will be encoded by Node while converting it to object. - `time` - if `true`, the request-response cycle (including all redirects) is timed at millisecond resolution. When set, the following properties are added to the response object: - `elapsedTime` Duration of the entire request/response in milliseconds (*deprecated*). - `responseStartTime` Timestamp when the response began (in Unix Epoch milliseconds) (*deprecated*). @@ -856,13 +976,15 @@ The first argument can be either a `url` or an `options` object. The only requir - `socket` Relative timestamp when the [`http`](https://nodejs.org/api/http.html#http_event_socket) module's `socket` event fires. This happens when the socket is assigned to the request. - `lookup` Relative timestamp when the [`net`](https://nodejs.org/api/net.html#net_event_lookup) module's `lookup` event fires. This happens when the DNS has been resolved. - `connect`: Relative timestamp when the [`net`](https://nodejs.org/api/net.html#net_event_connect) module's `connect` event fires. This happens when the server acknowledges the TCP connection. + - `secureConnect`: Relative timestamp when the [`tls`](https://nodejs.org/api/tls.html#tls_event_secureconnect) module's `secureconnect` event fires. This happens when the handshaking process for a new connection has successfully completed. - `response`: Relative timestamp when the [`http`](https://nodejs.org/api/http.html#http_event_response) module's `response` event fires. This happens when the first bytes are received from the server. - `end`: Relative timestamp when the last bytes of the response are received. - `timingPhases` Contains the durations of each request phase. If there were redirects, the properties reflect the timings of the final request in the redirect chain: - `wait`: Duration of socket initialization (`timings.socket`) - `dns`: Duration of DNS lookup (`timings.lookup` - `timings.socket`) - - `tcp`: Duration of TCP connection (`timings.connect` - `timings.socket`) - - `firstByte`: Duration of HTTP server response (`timings.response` - `timings.connect`) + - `tcp`: Duration of TCP connection (`timings.connect` - `timings.lookup`) + - `secureHandshake`: Duration of SSL handshake (`timings.secureConnect` - `timings.connect`) + - `firstByte`: Duration of HTTP server response (`timings.response` - `timings.connect`|`timings.secureConnect`) - `download`: Duration of HTTP download (`timings.end` - `timings.response`) - `total`: Duration entire HTTP round-trip (`timings.end`) @@ -961,7 +1083,7 @@ There are at least three ways to debug the operation of `request`: 1. Launch the node process like `NODE_DEBUG=request node script.js` (`lib,request,otherlib` works too). -2. Set `require('request').debug = true` at any time (this does the same thing +2. Set `require('postman-request').debug = true` at any time (this does the same thing as #1). 3. Use the [request-debug module](https://github.com/request/request-debug) to @@ -1005,7 +1127,7 @@ request.get('http://10.255.255.1', {timeout: 1500}, function(err) { ## Examples: ```js - const request = require('request') + const request = require('postman-request') , rand = Math.floor(Math.random()*100000000).toString() ; request( @@ -1036,7 +1158,7 @@ while the response object is unmodified and will contain compressed data if the server sent a compressed response. ```js - const request = require('request') + const request = require('postman-request') request( { method: 'GET' , uri: 'http://www.google.com' @@ -1109,7 +1231,7 @@ request('http://www.google.com', function() { The cookie store must be a [`tough-cookie`](https://github.com/SalesforceEng/tough-cookie) -store and it must support synchronous operations; see the +store and it must support asynchronous operations; see the [`CookieStore` API docs](https://github.com/SalesforceEng/tough-cookie#api) for details. @@ -1118,8 +1240,8 @@ To inspect your cookie jar after a request: ```js const j = request.jar() request({url: 'http://www.google.com', jar: j}, function () { - const cookie_string = j.getCookieString(url); // "key1=value1; key2=value2; ..." - const cookies = j.getCookies(url); + const cookie_string = j.getCookieStringSync(url); // "key1=value1; key2=value2; ..." + const cookies = j.getCookiesSync(url); // [{key: 'key1', value: 'value1', domain: "www.google.com", ...}, ...] }) ``` diff --git a/index.js b/index.js index d50f9917b..b2846d0ff 100755 --- a/index.js +++ b/index.js @@ -14,11 +14,9 @@ 'use strict' +var tls = require('tls') var extend = require('extend') var cookies = require('./lib/cookies') -var helpers = require('./lib/helpers') - -var paramsHaveRequestBody = helpers.paramsHaveRequestBody // organize params for patch, post, put, head, del function initParams (uri, options, callback) { @@ -46,10 +44,6 @@ function request (uri, options, callback) { var params = initParams(uri, options, callback) - if (params.method === 'HEAD' && paramsHaveRequestBody(params)) { - throw new Error('HTTP HEAD requests MUST NOT include a request body.') - } - return new request.Request(params) } @@ -137,6 +131,92 @@ request.forever = function (agentOptions, optionsArg) { return request.defaults(options) } +// As of now (Node v10.x LTS), the only way to extend the well known "root" CA +// is by using an environment variable called `NODE_EXTRA_CA_CERTS`. +// This function enables the same functionality and provides a programmatic way +// to extend the CA certificates. +// Refer: https://nodejs.org/docs/latest-v10.x/api/cli.html#cli_node_extra_ca_certs_file +// +// @note Unlike NODE_EXTRA_CA_CERTS, this method extends the CA for every +// request sent and since its an expensive operation its advised to use a +// keepAlive agent(agentOptions.keepAlive: true) when this is enabled. +// +// Benchmarks using a local server: +// NODE_EXTRA_CA_CERTS (keepAlive: false) : 422 ops/sec ±1.73% (77 runs sampled) +// NODE_EXTRA_CA_CERTS (keepAlive: true) : 2,096 ops/sec ±4.23% (69 runs sampled) +// +// enableNodeExtraCACerts (keepAlive: false) : 331 ops/sec ±5.64% (77 runs sampled) +// enableNodeExtraCACerts (keepAlive: true) : 2,045 ops/sec ±5.20% (69 runs sampled) +// +// @note Enabling this will override the singleton `tls.createSecureContext` method +// which will be affected for every request sent (using native HTTPS etc.) on the +// same process. BUT, this will only be effective when `extraCA` option is +// passed to `tls.createSecureContext`, which is limited to this library. +request.enableNodeExtraCACerts = function (callback) { + // @note callback is optional to catch missing tls method + !callback && (callback = function () {}) + + // bail out if already enabled + if (tls.__createSecureContext) { + return callback() + } + + // enable only if `SecureContext.addCACert` is present + // otherwise return callback with error. + // @note try-catch is used to make sure testing this will not break + // the main process due to OpenSSL error. + try { + var testContext = tls.createSecureContext() + + if (!(testContext && testContext.context && + typeof testContext.context.addCACert === 'function')) { + return callback(new Error('SecureContext.addCACert is not a function')) + } + } catch (err) { + return callback(err) + } + + // store the original tls.createSecureContext method. + // used to extend existing functionality as well as restore later. + tls.__createSecureContext = tls.createSecureContext + + // override tls.createSecureContext with extraCA support + // @note if agent is keepAlive, same context will be reused. + tls.createSecureContext = function () { + // call original createSecureContext and store the context + var secureContext = tls.__createSecureContext.apply(this, arguments) + + // if `extraCA` is present in options, extend CA certs + // @note this request option is available here because all the + // Request properties are passed to HTTPS Agent. + if (arguments[0] && arguments[0].extraCA) { + // extend root CA with specified CA certificates + // @note `addCACert` is an undocumented API and performs an expensive operations + // Refer: https://github.com/nodejs/node/blob/v10.15.1/lib/_tls_common.js#L97 + secureContext.context.addCACert(arguments[0].extraCA) + } + + return secureContext + } + + // enabled extra CA support + return callback() +} + +// disable the extended CA certificates feature +request.disableNodeExtraCACerts = function () { + // bail out if not enabled + if (typeof tls.__createSecureContext !== 'function') { + return + } + + // reset `tls.createSecureContext` with the original method + tls.createSecureContext = tls.__createSecureContext + + // delete the reference of original method + delete tls.__createSecureContext +} + // Exports module.exports = request diff --git a/lib/auth.js b/lib/auth.js index 02f203869..7620e02a1 100644 --- a/lib/auth.js +++ b/lib/auth.js @@ -136,7 +136,7 @@ Auth.prototype.onRequest = function (user, pass, sendImmediately, bearer) { authHeader = self.basic(user, pass, sendImmediately) } if (authHeader) { - request.setHeader('authorization', authHeader) + request.setHeader('Authorization', authHeader) } } diff --git a/lib/cookies.js b/lib/cookies.js index bd5d46bea..70c5d3c8b 100644 --- a/lib/cookies.js +++ b/lib/cookies.js @@ -15,24 +15,6 @@ exports.parse = function (str) { return Cookie.parse(str, {loose: true}) } -// Adapt the sometimes-Async api of tough.CookieJar to our requirements -function RequestJar (store) { - var self = this - self._jar = new CookieJar(store, {looseMode: true}) -} -RequestJar.prototype.setCookie = function (cookieOrStr, uri, options) { - var self = this - return self._jar.setCookieSync(cookieOrStr, uri, options || {}) -} -RequestJar.prototype.getCookieString = function (uri) { - var self = this - return self._jar.getCookieStringSync(uri) -} -RequestJar.prototype.getCookies = function (uri) { - var self = this - return self._jar.getCookiesSync(uri) -} - exports.jar = function (store) { - return new RequestJar(store) + return new CookieJar(store, {looseMode: true}) } diff --git a/lib/helpers.js b/lib/helpers.js index 8b2a7e6eb..5df296ace 100644 --- a/lib/helpers.js +++ b/lib/helpers.js @@ -8,14 +8,16 @@ var defer = typeof setImmediate === 'undefined' ? process.nextTick : setImmediate -function paramsHaveRequestBody (params) { - return ( - params.body || - params.requestBodyStream || - (params.json && typeof params.json !== 'boolean') || - params.multipart - ) -} +// Reference: https://github.com/postmanlabs/postman-request/pull/23 +// +// function paramsHaveRequestBody (params) { +// return ( +// params.body || +// params.requestBodyStream || +// (params.json && typeof params.json !== 'boolean') || +// params.multipart +// ) +// } function safeStringify (obj, replacer) { var ret @@ -56,7 +58,6 @@ function version () { } } -exports.paramsHaveRequestBody = paramsHaveRequestBody exports.safeStringify = safeStringify exports.md5 = md5 exports.isReadStream = isReadStream diff --git a/lib/inflate.js b/lib/inflate.js new file mode 100644 index 000000000..d59ec8cee --- /dev/null +++ b/lib/inflate.js @@ -0,0 +1,66 @@ +'use strict' + +var zlib = require('zlib') +var stream = require('stream') +var inherit = require('util').inherits +var Buffer = require('safe-buffer').Buffer +var Inflate + +Inflate = function (options) { + this.options = options + this._stream = null + stream.Transform.call(this) +} + +inherit(Inflate, stream.Transform) + +Inflate.prototype._transform = function (chunk, encoding, callback) { + var self = this + if (!self._stream) { + // If the response stream does not have a valid deflate header, use `InflateRaw` + if ((Buffer.from(chunk, encoding)[0] & 0x0F) === 0x08) { + self._stream = zlib.createInflate(self.options) + } else { + self._stream = zlib.createInflateRaw(self.options) + } + + self._stream.on('error', function (error) { + self.emit('error', error) + }) + + self.once('finish', function () { + self._stream.end() + }) + self._stream.on('data', function (chunk) { + self.push(chunk) + }) + + self._stream.once('end', function () { + self._ended = true + self.push(null) + }) + } + + self._stream.write(chunk, encoding, callback) +} + +Inflate.prototype._flush = function (callback) { + if (this._stream && !this._ended) { + this._stream.once('end', callback) + } else { + callback() + } +} + +/** + * Creates an intelligent inflate stream, that can handle deflate responses from older servers, + * which do not send the correct GZip headers in the response. See http://stackoverflow.com/a/37528114 + * for details on why this is needed. + * + * @param {Object=} options - Are passed to the underlying `Inflate` or `InflateRaw` constructor. + * + * @returns {*} + */ +module.exports.createInflate = function (options) { + return new Inflate(options) +} diff --git a/lib/multipart.js b/lib/multipart.js index 6a009bc13..7adf16ee3 100644 --- a/lib/multipart.js +++ b/lib/multipart.js @@ -47,18 +47,18 @@ Multipart.prototype.setHeaders = function (chunked) { var self = this if (chunked && !self.request.hasHeader('transfer-encoding')) { - self.request.setHeader('transfer-encoding', 'chunked') + self.request.setHeader('Transfer-Encoding', 'chunked') } var header = self.request.getHeader('content-type') if (!header || header.indexOf('multipart') === -1) { - self.request.setHeader('content-type', 'multipart/related; boundary=' + self.boundary) + self.request.setHeader('Content-Type', 'multipart/related; boundary=' + self.boundary) } else { if (header.indexOf('boundary') !== -1) { self.boundary = header.replace(/.*boundary=([^\s;]+).*/, '$1') } else { - self.request.setHeader('content-type', header + '; boundary=' + self.boundary) + self.request.setHeader('Content-Type', header + '; boundary=' + self.boundary) } } } diff --git a/lib/redirect.js b/lib/redirect.js index b9150e77c..aa41a3a28 100644 --- a/lib/redirect.js +++ b/lib/redirect.js @@ -1,6 +1,7 @@ 'use strict' var url = require('url') +var fs = require('fs') var isUrl = /^https?:/ function Redirect (request) { @@ -9,6 +10,7 @@ function Redirect (request) { this.followRedirects = true this.followAllRedirects = false this.followOriginalHttpMethod = false + this.followAuthorizationHeader = false this.allowRedirect = function () { return true } this.maxRedirects = 10 this.redirects = [] @@ -40,6 +42,9 @@ Redirect.prototype.onRequest = function (options) { if (options.followOriginalHttpMethod !== undefined) { self.followOriginalHttpMethod = options.followOriginalHttpMethod } + if (options.followAuthorizationHeader !== undefined) { + self.followAuthorizationHeader = options.followAuthorizationHeader + } } Redirect.prototype.redirectTo = function (response) { @@ -67,9 +72,12 @@ Redirect.prototype.redirectTo = function (response) { } } } else if (response.statusCode === 401) { + // retry the request with the new Authorization header value using + // WWW-Authenticate response header. + // https://tools.ietf.org/html/rfc7235#section-3.1 var authHeader = request._auth.onResponse(response) if (authHeader) { - request.setHeader('authorization', authHeader) + request.setHeader('Authorization', authHeader) redirectTo = request.uri } } @@ -79,6 +87,7 @@ Redirect.prototype.redirectTo = function (response) { Redirect.prototype.onResponse = function (response) { var self = this var request = self.request + var options = {} var redirectTo = self.redirectTo(response) if (!redirectTo || !self.allowRedirect.call(request, response)) { @@ -114,15 +123,36 @@ Redirect.prototype.onResponse = function (response) { self.redirects.push({ statusCode: response.statusCode, redirectUri: redirectTo }) - if (self.followAllRedirects && request.method !== 'HEAD' && - response.statusCode !== 401 && response.statusCode !== 307) { - request.method = self.followOriginalHttpMethod ? request.method : 'GET' + // if the redirect hostname (not just port or protocol) is changed: + // 1. remove host header, the new host will be populated on request.init + // 2. remove authorization header, avoid authentication leak + // @note: This is done because of security reasons, irrespective of the + // status code or request method used. + if (request.headers && uriPrev.hostname !== request.uri.hostname) { + request.removeHeader('host') + + // use followAuthorizationHeader option to retain authorization header + if (!self.followAuthorizationHeader) { + request.removeHeader('authorization') + } } - // request.method = 'GET' // Force all redirects to use GET || commented out fixes #215 + delete request.src delete request.req delete request._started - if (response.statusCode !== 401 && response.statusCode !== 307) { + + // if statusCode code is 401, 307 or 308: + // 1. Switch request method to GET if followOriginalHttpMethod is not set + // 2. Remove request body on redirect + if (response.statusCode !== 401 && response.statusCode !== 307 && response.statusCode !== 308) { + // force all redirects to use GET (due to legacy shenanigans) + // use followOriginalHttpMethod option to avoid this + // @todo: figure out why its only done for HEAD method and not for similar + // request methods like OPTIONS or CONNECT. + if (!self.followOriginalHttpMethod && request.method !== 'HEAD') { + request.method = 'GET' + } + // Remove parameters from the previous response, unless this is the second request // for a server that requires digest authentication. delete request.body @@ -131,22 +161,51 @@ Redirect.prototype.onResponse = function (response) { request.removeHeader('host') request.removeHeader('content-type') request.removeHeader('content-length') - if (request.uri.hostname !== request.originalHost.split(':')[0]) { - // Remove authorization if changing hostnames (but not if just - // changing ports or protocols). This matches the behavior of curl: - // https://github.com/bagder/curl/blob/6beb0eee/lib/http.c#L710 - request.removeHeader('authorization') + } + } else if (request.formData && + // make sure _form is released and there's no pending _streams left + // which will be the case for 401 redirects. so, reuse _form on redirect + // @note: multiple form-param / file-streams may cause following issue: + // https://github.com/request/request/issues/887 + // @todo: expose stream errors as events + request._form && request._form._released && + request._form._streams && !request._form._streams.length) { + // reinitialize FormData stream for 307 or 308 redirects + delete request._form + // remove content-type header for new boundary + request.removeHeader('content-type') + // remove content-length header since FormValue may be dropped if its not a file stream + request.removeHeader('content-length') + + var formData = [] + var resetFormData = function (key, value, paramOptions) { + // if `value` is of type stream + if (typeof (value && value.pipe) === 'function') { + // bail out if not a file stream + if (!(value.hasOwnProperty('fd') && value.path)) return + // create new file stream + value = fs.createReadStream(value.path) } + + formData.push({key: key, value: value, options: paramOptions}) } + for (var i = 0, ii = request.formData.length; i < ii; i++) { + var formParam = request.formData[i] + if (!formParam) { continue } + resetFormData(formParam.key, formParam.value, formParam.options) + } + + // setting `options.formData` will reinitialize FormData in `request.init` + options.formData = formData } if (!self.removeRefererHeader) { - request.setHeader('referer', uriPrev.href) + request.setHeader('Referer', uriPrev.href) } request.emit('redirect') - request.init() + request.init(options) return true } diff --git a/lib/tunnel.js b/lib/tunnel.js index 4479003f6..2deee5dc2 100644 --- a/lib/tunnel.js +++ b/lib/tunnel.js @@ -1,7 +1,7 @@ 'use strict' var url = require('url') -var tunnel = require('tunnel-agent') +var tunnel = require('@postman/tunnel-agent') var defaultProxyHeaderWhiteList = [ 'accept', diff --git a/lib/url-parse.js b/lib/url-parse.js new file mode 100644 index 000000000..c20675731 --- /dev/null +++ b/lib/url-parse.js @@ -0,0 +1,103 @@ +var url = require('url') +var urlEncoder = require('postman-url-encoder') +var EMPTY = '' +var STRING = 'string' +var AMPERSAND = '&' +var EQUALS = '=' +var QUESTION_MARK = '?' +var stringify +var parse + +/** + * Parses a query string into an array, preserving parameter values + * + * @param string + * @returns {*} + */ +parse = function (string) { + var parts + if (typeof string === STRING) { // eslint-disable-line valid-typeof + parts = string.split(AMPERSAND) + return parts.map(function (param, idx) { + if (param === EMPTY && idx !== (parts.length - 1)) { + return { key: null, value: null } + } + + var index = (typeof param === STRING) ? param.indexOf(EQUALS) : -1 // eslint-disable-line valid-typeof + var paramObj = {} + + // this means that there was no value for this key (not even blank, so we store this info) and the value is set + // to null + if (index < 0) { + paramObj.key = param.substr(0, param.length) + paramObj.value = null + } else { + paramObj.key = param.substr(0, index) + paramObj.value = param.substr(index + 1) + } + + return paramObj + }) + } + return [] +} + +/** + * Stringifies a query string, from an array of parameters + * + * @param parameters + * @returns {string} + */ +stringify = function (parameters) { + return parameters ? parameters.map(function (param) { + var key = param.key + var value = param.value + + if (value === undefined) { + return '' + } + + if (key === null) { + key = '' + } + + if (value === null) { + return urlEncoder.encode(key) + } + + return urlEncoder.encode(key) + EQUALS + urlEncoder.encode(value) + }).join(AMPERSAND) : '' +} + +/** + * Correctly URL encodes query parameters in a URL and returns the final parsed URL. + * + * @param str + */ +module.exports = function (str) { + var parsed = url.parse(str) + var rawQs + var search + var path + var qs + + rawQs = parsed.query + + if (rawQs && rawQs.length) { + qs = stringify(parse(parsed.query)) + search = QUESTION_MARK + qs + path = parsed.pathname + search + + parsed.query = qs + parsed.search = search + parsed.path = path + + str = url.format(parsed) + } + + // Parse again, because Node does not guarantee consistency of properties + return url.parse(str) +} + +module.exports.parse = parse +module.exports.stringify = stringify diff --git a/package.json b/package.json index 86e4266bf..88783bccb 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,5 @@ { - "name": "request", + "name": "postman-request", "description": "Simplified HTTP request client.", "keywords": [ "http", @@ -7,14 +7,10 @@ "util", "utility" ], - "version": "2.88.1", - "author": "Mikeal Rogers ", + "version": "2.88.1-postman.16", "repository": { "type": "git", - "url": "https://github.com/request/request.git" - }, - "bugs": { - "url": "http://github.com/request/request/issues" + "url": "https://github.com/postmanlabs/postman-request.git" }, "license": "Apache-2.0", "engines": { @@ -33,7 +29,7 @@ "combined-stream": "~1.0.6", "extend": "~3.0.2", "forever-agent": "~0.6.1", - "form-data": "~2.3.2", + "@postman/form-data": "~3.1.0", "har-validator": "~5.1.3", "http-signature": "~1.2.0", "is-typedarray": "~1.0.0", @@ -42,10 +38,12 @@ "mime-types": "~2.1.19", "oauth-sign": "~0.9.0", "performance-now": "^2.1.0", + "postman-url-encoder": "1.0.1", "qs": "~6.5.2", "safe-buffer": "^5.1.2", + "stream-length": "^1.0.2", "tough-cookie": "~2.5.0", - "tunnel-agent": "^0.6.0", + "@postman/tunnel-agent": "^0.6.3", "uuid": "^3.3.2" }, "scripts": { @@ -79,7 +77,6 @@ }, "greenkeeper": { "ignore": [ - "hawk", "har-validator" ] } diff --git a/request.js b/request.js index 198b76093..bc9383ab2 100644 --- a/request.js +++ b/request.js @@ -1,5 +1,6 @@ 'use strict' +var tls = require('tls') var http = require('http') var https = require('https') var url = require('url') @@ -8,6 +9,7 @@ var stream = require('stream') var zlib = require('zlib') var aws2 = require('aws-sign2') var aws4 = require('aws4') +var uuid = require('uuid/v4') var httpSignature = require('http-signature') var mime = require('mime-types') var caseless = require('caseless') @@ -15,6 +17,7 @@ var ForeverAgent = require('forever-agent') var FormData = require('form-data') var extend = require('extend') var isstream = require('isstream') +var streamLength = require('stream-length') var isTypedArray = require('is-typedarray').strict var helpers = require('./lib/helpers') var cookies = require('./lib/cookies') @@ -29,7 +32,8 @@ var Redirect = require('./lib/redirect').Redirect var Tunnel = require('./lib/tunnel').Tunnel var now = require('performance-now') var Buffer = require('safe-buffer').Buffer - +var inflate = require('./lib/inflate') +var urlParse = require('./lib/url-parse') var safeStringify = helpers.safeStringify var isReadStream = helpers.isReadStream var toBase64 = helpers.toBase64 @@ -69,6 +73,38 @@ function filterOutReservedFunctions (reserved, options) { return object } +function transformFormData (formData) { + // Transform the object representation of form-data fields to array representation. + // This might not preserve the order of form fields defined in object representation. + // But, this transformation is required to support backward compatibility. + // + // Form-Data should be stored as an array to respect the fields order. + // RFC 7578#section-5.2 Ordered Fields and Duplicated Field Names + // https://tools.ietf.org/html/rfc7578#section-5.2 + + var transformedFormData = [] + var appendFormParam = function (key, param) { + transformedFormData.push({ + key: key, + value: param && param.hasOwnProperty('value') ? param.value : param, + options: param && param.hasOwnProperty('options') ? param.options : undefined + }) + } + for (var formKey in formData) { + if (formData.hasOwnProperty(formKey)) { + var formValue = formData[formKey] + if (Array.isArray(formValue)) { + for (var j = 0; j < formValue.length; j++) { + appendFormParam(formKey, formValue[j]) + } + } else { + appendFormParam(formKey, formValue) + } + } + } + return transformedFormData +} + // Return a simpler request object to allow serialization function requestToJSON () { var self = this @@ -106,6 +142,12 @@ function Request (options) { options = self._har.options(options) } + // transform `formData` for backward compatibility + // don't check for explicit object type to support legacy shenanigans + if (options.formData && !Array.isArray(options.formData)) { + options.formData = transformFormData(options.formData) + } + stream.Stream.call(self) var reserved = Object.keys(Request.prototype) var nonReserved = filterForNonReserved(reserved, options) @@ -115,6 +157,7 @@ function Request (options) { self.readable = true self.writable = true + self._debug = [] if (options.method) { self.explicitMethod = true } @@ -131,11 +174,13 @@ util.inherits(Request, stream.Stream) // Debugging Request.debug = process.env.NODE_DEBUG && /\brequest\b/.test(process.env.NODE_DEBUG) + function debug () { if (Request.debug) { console.error('REQUEST %s', util.format.apply(util, arguments)) } } + Request.prototype.debug = debug Request.prototype.init = function (options) { @@ -148,6 +193,31 @@ Request.prototype.init = function (options) { } self.headers = self.headers ? copy(self.headers) : {} + // for this request (or redirect) store its debug logs in `_reqResInfo` and + // store its reference in `_debug` which holds debug logs of every request + self._reqResInfo = {} + self._debug.push(self._reqResInfo) + + // additional postman feature starts + // bind default events sent via options + if (options.bindOn) { + Object.keys(options.bindOn).forEach(function (eventName) { + !Array.isArray(options.bindOn[eventName]) && (options.bindOn[eventName] = [options.bindOn[eventName]]) + options.bindOn[eventName].forEach(function (listener) { + self.on(eventName, listener) + }) + }) + } + if (options.once) { + Object.keys(options.once).forEach(function (eventName) { + !Array.isArray(options.bindOnce[eventName]) && (options.bindOnce[eventName] = [options.bindOnce[eventName]]) + options.bindOnce[eventName].forEach(function (listener) { + self.once(eventName, listener) + }) + }) + } + // additional postman feature ends + // Delete headers with value undefined since they break // ClientRequest.OutgoingMessage.setHeader in node 0.12 for (var headerName in self.headers) { @@ -177,12 +247,12 @@ Request.prototype.init = function (options) { // Protect against double callback if (!self._callback && self.callback) { self._callback = self.callback - self.callback = function () { + self.callback = function (error, response, body) { if (self._callbackCalled) { return // Print a warning maybe? } self._callbackCalled = true - self._callback.apply(self, arguments) + self._callback(error, response, body, self._debug) } self.on('error', self.callback.bind()) self.on('complete', self.callback.bind(self, null)) @@ -286,7 +356,7 @@ Request.prototype.init = function (options) { self.setHost = false if (!self.hasHeader('host')) { - var hostHeaderName = self.originalHostHeaderName || 'host' + var hostHeaderName = self.originalHostHeaderName || 'Host' self.setHeader(hostHeaderName, self.uri.host) // Drop :port suffix from Host header if known protocol. if (self.uri.port) { @@ -298,8 +368,6 @@ Request.prototype.init = function (options) { self.setHost = true } - self.jar(self._jar || options.jar) - if (!self.uri.port) { if (self.uri.protocol === 'http:') { self.uri.port = 80 } else if (self.uri.protocol === 'https:') { self.uri.port = 443 } } @@ -319,23 +387,13 @@ Request.prototype.init = function (options) { if (options.formData) { var formData = options.formData var requestForm = self.form() - var appendFormValue = function (key, value) { - if (value && value.hasOwnProperty('value') && value.hasOwnProperty('options')) { - requestForm.append(key, value.value, value.options) + for (var i = 0, ii = formData.length; i < ii; i++) { + var formParam = formData[i] + if (!formParam) { continue } + if (formParam.options) { + requestForm.append(formParam.key, formParam.value, formParam.options) } else { - requestForm.append(key, value) - } - } - for (var formKey in formData) { - if (formData.hasOwnProperty(formKey)) { - var formValue = formData[formKey] - if (formValue instanceof Array) { - for (var j = 0; j < formValue.length; j++) { - appendFormValue(formKey, formValue[j]) - } - } else { - appendFormValue(formKey, formValue) - } + requestForm.append(formParam.key, formParam.value) } } } @@ -384,7 +442,7 @@ Request.prototype.init = function (options) { } if (self.gzip && !self.hasHeader('accept-encoding')) { - self.setHeader('accept-encoding', 'gzip, deflate') + self.setHeader('Accept-Encoding', 'gzip, deflate') } if (self.uri.auth && !self.hasHeader('authorization')) { @@ -395,7 +453,7 @@ Request.prototype.init = function (options) { if (!self.tunnel && self.proxy && self.proxy.auth && !self.hasHeader('proxy-authorization')) { var proxyAuthPieces = self.proxy.auth.split(':').map(function (item) { return self._qs.unescape(item) }) var authHeader = 'Basic ' + toBase64(proxyAuthPieces.join(':')) - self.setHeader('proxy-authorization', authHeader) + self.setHeader('Proxy-Authorization', authHeader) } if (self.proxy && !self.tunnel) { @@ -409,13 +467,22 @@ Request.prototype.init = function (options) { self.multipart(options.multipart) } - if (options.time) { + // enable timings if verbose is true + if (options.time || options.verbose) { self.timing = true // NOTE: elapsedTime is deprecated in favor of .timings self.elapsedTime = self.elapsedTime || 0 } + if (options.verbose) { + self.verbose = true + } + + if (typeof options.maxResponseSize === 'number') { + self.maxResponseSize = options.maxResponseSize + } + function setContentLength () { if (isTypedArray(self.body)) { self.body = Buffer.from(self.body) @@ -432,12 +499,13 @@ Request.prototype.init = function (options) { } if (length) { - self.setHeader('content-length', length) + self.setHeader('Content-Length', length) } else { self.emit('error', new Error('Argument error, options.body.')) } } } + if (self.body && !isstream(self.body)) { setContentLength() } @@ -497,7 +565,7 @@ Request.prototype.init = function (options) { self.src = src if (isReadStream(src)) { if (!self.hasHeader('content-type')) { - self.setHeader('content-type', mime.lookup(src.path)) + self.setHeader('Content-Type', mime.lookup(src.path)) } } else { if (src.headers) { @@ -508,16 +576,16 @@ Request.prototype.init = function (options) { } } if (self._json && !self.hasHeader('content-type')) { - self.setHeader('content-type', 'application/json') + self.setHeader('Content-Type', 'application/json') } if (src.method && !self.explicitMethod) { self.method = src.method } } - // self.on('pipe', function () { - // console.error('You have already piped to this stream. Pipeing twice is likely to break the request.') - // }) + // self.on('pipe', function () { + // console.error('You have already piped to this stream. Pipeing twice is likely to break the request.') + // }) }) defer(function () { @@ -527,10 +595,14 @@ Request.prototype.init = function (options) { var end = function () { if (self._form) { - if (!self._auth.hasAuth) { - self._form.pipe(self) - } else if (self._auth.hasAuth && self._auth.sentAuth) { - self._form.pipe(self) + if (!self._auth.hasAuth || (self._auth.hasAuth && self._auth.sentAuth)) { + try { + self._form.pipe(self) + } catch (err) { + self.abort() + options.callback && options.callback(err) + return + } } } if (self._multipart && self._multipart.chunked) { @@ -538,7 +610,16 @@ Request.prototype.init = function (options) { } if (self.body) { if (isstream(self.body)) { - self.body.pipe(self) + if (self.hasHeader('content-length')) { + self.body.pipe(self) + } else { // certain servers require content-length to function. we try to pre-detect if possible + streamLength(self.body, {}, function (err, len) { + if (!(err || self._started || self.hasHeader('content-length') || len === null || len < 0)) { + self.setHeader('Content-Length', len) + } + self.body.pipe(self) + }) + } } else { setContentLength() if (Array.isArray(self.body)) { @@ -559,24 +640,25 @@ Request.prototype.init = function (options) { return } if (self.method !== 'GET' && typeof self.method !== 'undefined') { - self.setHeader('content-length', 0) + self.setHeader('Content-Length', 0) } self.end() } } - if (self._form && !self.hasHeader('content-length')) { - // Before ending the request, we had to compute the length of the whole form, asyncly - self.setHeader(self._form.getHeaders(), true) - self._form.getLength(function (err, length) { - if (!err && !isNaN(length)) { - self.setHeader('content-length', length) - } + self.jar(self._jar || options.jar, function () { + if (self._form && !self.hasHeader('content-length')) { + // Before ending the request, we had to compute the length of the whole form, asyncly + self._form.getLength(function (err, length) { + if (!err && !isNaN(length)) { + self.setHeader('Content-Length', length) + } + end() + }) + } else { end() - }) - } else { - end() - } + } + }) self.ntick = true }) @@ -594,6 +676,9 @@ Request.prototype.getNewAgent = function () { if (self.ca) { options.ca = self.ca } + if (self.extraCA) { + options.extraCA = self.extraCA + } if (self.ciphers) { options.ciphers = self.ciphers } @@ -642,6 +727,14 @@ Request.prototype.getNewAgent = function () { poolKey += options.ca } + // only add when NodeExtraCACerts is enabled + if (tls.__createSecureContext && options.extraCA) { + if (poolKey) { + poolKey += ':' + } + poolKey += options.extraCA + } + if (typeof options.rejectUnauthorized !== 'undefined') { if (poolKey) { poolKey += ':' @@ -663,6 +756,13 @@ Request.prototype.getNewAgent = function () { poolKey += options.pfx.toString('ascii') } + if (options.passphrase) { + if (poolKey) { + poolKey += ':' + } + poolKey += options.passphrase + } + if (options.ciphers) { if (poolKey) { poolKey += ':' @@ -724,22 +824,39 @@ Request.prototype.start = function () { return } + // postman: emit start event + self.emit('start') + self._started = true self.method = self.method || 'GET' self.href = self.uri.href if (self.src && self.src.stat && self.src.stat.size && !self.hasHeader('content-length')) { - self.setHeader('content-length', self.src.stat.size) + self.setHeader('Content-Length', self.src.stat.size) } if (self._aws) { self.aws(self._aws, true) } + self._reqResInfo.request = { + method: self.method, + href: self.uri.href, + proxy: (self.proxy && { href: self.proxy.href }) || undefined, + httpVersion: '1.1' + } + // We have a method named auth, which is completely different from the http.request // auth option. If we don't remove it, we're gonna have a bad time. var reqOptions = copy(self) delete reqOptions.auth + // Workaround for a bug in Node: https://github.com/nodejs/node/issues/8321 + if (!(self.disableUrlEncoding || self.proxy || self.uri.isUnix)) { + try { + extend(reqOptions, urlParse(self.uri.href)) + } catch (e) { } // nothing to do if urlParse fails, "extend" never throws an error. + } + debug('make request', self.uri.href) // node v6.8.0 now supports a `timeout` value in `http.request()`, but we @@ -779,6 +896,26 @@ Request.prototype.start = function () { }) self.req.on('socket', function (socket) { + if (self.verbose) { + // The reused socket holds all the session data which was injected in + // during the first connection. This is done because events like + // `lookup`, `connect` & `secureConnect` will not be triggered for a + // reused socket and debug information will be lost for that request. + var reusedSocket = Boolean(socket.__SESSION_ID && socket.__SESSION_DATA) + + if (!reusedSocket) { + socket.__SESSION_ID = uuid() + socket.__SESSION_DATA = {} + } + + // @note make sure you don't serialize this object to avoid memory leak + self._reqResInfo.session = { + id: socket.__SESSION_ID, + reused: reusedSocket, + data: socket.__SESSION_DATA + } + } + // `._connecting` was the old property which was made public in node v6.1.0 var isConnecting = socket._connecting || socket.connecting if (self.timing) { @@ -791,10 +928,85 @@ Request.prototype.start = function () { var onConnectTiming = function () { self.timings.connect = now() - self.startTimeNow + + if (self.verbose) { + socket.__SESSION_DATA.addresses = { + // local address + // @note there's no `socket.localFamily` but `.address` method + // returns same output as of remote. + local: (typeof socket.address === 'function') && socket.address(), + + // remote address + remote: { + address: socket.remoteAddress, + family: socket.remoteFamily, + port: socket.remotePort + } + } + } + } + + var onSecureConnectTiming = function () { + self.timings.secureConnect = now() - self.startTimeNow + + if (self.verbose) { + socket.__SESSION_DATA.tls = { + // true if the session was reused + reused: (typeof socket.isSessionReused === 'function') && socket.isSessionReused(), + + // true if the peer certificate was signed by one of the CAs specified + authorized: socket.authorized, + + // reason why the peer's certificate was not been verified + authorizationError: socket.authorizationError, + + // negotiated cipher name + cipher: (typeof socket.getCipher === 'function') && socket.getCipher(), + + // negotiated SSL/TLS protocol version + // @note Node >= v5.7.0 + protocol: (typeof socket.getProtocol === 'function') && socket.getProtocol(), + + // type, name, and size of parameter of an ephemeral key exchange + // @note Node >= v5.0.0 + ephemeralKeyInfo: (typeof socket.getEphemeralKeyInfo === 'function') && socket.getEphemeralKeyInfo() + } + + // peer certificate information + // @note if session is reused, all certificate information is + // stripped from the socket (returns {}). + // Refer: https://github.com/nodejs/node/issues/3940 + var peerCert = (typeof socket.getPeerCertificate === 'function') && (socket.getPeerCertificate() || {}) + + socket.__SESSION_DATA.tls.peerCertificate = { + subject: peerCert.subject && { + country: peerCert.subject.C, + stateOrProvince: peerCert.subject.ST, + locality: peerCert.subject.L, + organization: peerCert.subject.O, + organizationalUnit: peerCert.subject.OU, + commonName: peerCert.subject.CN, + alternativeNames: peerCert.subjectaltname + }, + issuer: peerCert.issuer && { + country: peerCert.issuer.C, + stateOrProvince: peerCert.issuer.ST, + locality: peerCert.issuer.L, + organization: peerCert.issuer.O, + organizationalUnit: peerCert.issuer.OU, + commonName: peerCert.issuer.CN + }, + validFrom: peerCert.valid_from && new Date(peerCert.valid_from), + validTo: peerCert.valid_to && new Date(peerCert.valid_to), + fingerprint: peerCert.fingerprint, + serialNumber: peerCert.serialNumber + } + } } socket.once('lookup', onLookupTiming) socket.once('connect', onConnectTiming) + socket.once('secureConnect', onSecureConnectTiming) // clean up timing event listeners if needed on error self.req.once('error', function () { @@ -868,7 +1080,7 @@ Request.prototype.onRequestError = function (error) { } if (self.req && self.req._reusedSocket && error.code === 'ECONNRESET' && self.agent.addRequestNoreuse) { - self.agent = { addRequest: self.agent.addRequestNoreuse.bind(self.agent) } + self.agent = {addRequest: self.agent.addRequestNoreuse.bind(self.agent)} self.start() self.req.end() return @@ -889,6 +1101,7 @@ Request.prototype.onRequestResponse = function (response) { if (self.timing) { self.timings.end = now() - self.startTimeNow response.timingStart = self.startTime + response.timingStartTimer = self.startTimeNow // fill in the blanks for any periods that didn't trigger, such as // no lookup or connect due to keep alive @@ -901,6 +1114,9 @@ Request.prototype.onRequestResponse = function (response) { if (!self.timings.connect) { self.timings.connect = self.timings.lookup } + if (!self.timings.secureConnect && self.httpModule === https) { + self.timings.secureConnect = self.timings.connect + } if (!self.timings.response) { self.timings.response = self.timings.connect } @@ -925,7 +1141,14 @@ Request.prototype.onRequestResponse = function (response) { download: self.timings.end - self.timings.response, total: self.timings.end } + + // if secureConnect is present, add secureHandshake and update firstByte + if (self.timings.secureConnect) { + response.timingPhases.secureHandshake = self.timings.secureConnect - self.timings.connect + response.timingPhases.firstByte = self.timings.response - self.timings.secureConnect + } } + debug('response end', self.uri.href, response.statusCode, response.headers) }) @@ -935,6 +1158,17 @@ Request.prototype.onRequestResponse = function (response) { return } + self._reqResInfo.response = { + statusCode: response.statusCode, + httpVersion: response.httpVersion + } + + if (self.timing) { + self._reqResInfo.timingStart = self.startTime + self._reqResInfo.timingStartTimer = self.startTimeNow + self._reqResInfo.timings = self.timings + } + self.response = response response.request = self response.toJSON = responseToJSON @@ -942,7 +1176,7 @@ Request.prototype.onRequestResponse = function (response) { // XXX This is different on 0.10, because SSL is strict by default if (self.httpModule === https && self.strictSSL && (!response.hasOwnProperty('socket') || - !response.socket.authorized)) { + !response.socket.authorized)) { debug('strict ssl error', self.uri.href) var sslErr = response.hasOwnProperty('socket') ? response.socket.authorizationError : self.uri.href + ' does not support SSL' self.emit('error', new Error('SSL Error: ' + sslErr)) @@ -962,33 +1196,14 @@ Request.prototype.onRequestResponse = function (response) { } self.clearTimeout() - var targetCookieJar = (self._jar && self._jar.setCookie) ? self._jar : globalCookieJar - var addCookie = function (cookie) { - // set the cookie if it's domain in the href's domain. - try { - targetCookieJar.setCookie(cookie, self.uri.href, {ignoreError: true}) - } catch (e) { - self.emit('error', e) + function responseHandler () { + if (self._redirect.onResponse(response)) { + return // Ignore the rest of the response } - } - - response.caseless = caseless(response.headers) - if (response.caseless.has('set-cookie') && (!self._disableCookies)) { - var headerName = response.caseless.has('set-cookie') - if (Array.isArray(response.headers[headerName])) { - response.headers[headerName].forEach(addCookie) - } else { - addCookie(response.headers[headerName]) - } - } - - if (self._redirect.onResponse(response)) { - return // Ignore the rest of the response - } else { // Be a good stream and emit end when the response is finished. // Hack to emit end on close because of a core bug that never fires end - response.on('close', function () { + response.once('close', function () { if (!self._ended) { self.response.emit('end') } @@ -1028,7 +1243,7 @@ Request.prototype.onRequestResponse = function (response) { responseContent = zlib.createGunzip(zlibOptions) response.pipe(responseContent) } else if (contentEncoding === 'deflate') { - responseContent = zlib.createInflate(zlibOptions) + responseContent = inflate.createInflate(zlibOptions) response.pipe(responseContent) } else { // Since previous versions didn't check for Content-Encoding header, @@ -1062,6 +1277,14 @@ Request.prototype.onRequestResponse = function (response) { self.pipeDest(dest) }) + var responseThresholdEnabled = false + var responseBytesLeft + + if (typeof self.maxResponseSize === 'number') { + responseThresholdEnabled = true + responseBytesLeft = self.maxResponseSize + } + responseContent.on('data', function (chunk) { if (self.timing && !self.responseStarted) { self.responseStartTime = (new Date()).getTime() @@ -1069,6 +1292,17 @@ Request.prototype.onRequestResponse = function (response) { // NOTE: responseStartTime is deprecated in favor of .timings response.responseStartTime = self.responseStartTime } + // if response threshold is set, update the response bytes left to hit + // threshold. If exceeds, abort the request. + if (responseThresholdEnabled) { + responseBytesLeft -= chunk.length + if (responseBytesLeft < 0) { + self.emit('error', new Error('Maximum response size reached')) + self.destroy() + self.abort() + return + } + } self._destdata = true self.emit('data', chunk) }) @@ -1092,12 +1326,63 @@ Request.prototype.onRequestResponse = function (response) { }) } } + + function forEachAsync (items, fn, cb) { + !cb && (cb = function () { /* (ಠ_ಠ) */ }) + + if (!(Array.isArray(items) && fn)) { return cb() } + + var index = 0 + var totalItems = items.length + function next (err) { + if (err || index >= totalItems) { + return cb(err) + } + + try { + fn.call(items, items[index++], next) + } catch (error) { + return cb(error) + } + } + + if (!totalItems) { return cb() } + + next() + } + + var targetCookieJar = (self._jar && self._jar.setCookie) ? self._jar : globalCookieJar + var addCookie = function (cookie, cb) { + // set the cookie if it's domain in the href's domain. + targetCookieJar.setCookie(cookie, self.uri.href, {ignoreError: true}, function () { + // swallow the error, don't fail the request because of cookie jar failure + cb() + }) + } + + response.caseless = caseless(response.headers) + + if (response.caseless.has('set-cookie') && (!self._disableCookies)) { + var headerName = response.caseless.has('set-cookie') + if (Array.isArray(response.headers[headerName])) { + forEachAsync(response.headers[headerName], addCookie, function (err) { + if (err) { return self.emit('error', err) } + + responseHandler() + }) + } else { + addCookie(response.headers[headerName], responseHandler) + } + } else { + responseHandler() + } + debug('finish init function', self.uri.href) } Request.prototype.readResponseBody = function (response) { var self = this - debug("reading response's body") + debug('reading response\'s body') var buffers = [] var bufferLength = 0 var strings = [] @@ -1238,22 +1523,32 @@ Request.prototype.qs = function (q, clobber) { } Request.prototype.form = function (form) { var self = this + var contentType = self.getHeader('content-type') if (form) { - if (!/^application\/x-www-form-urlencoded\b/.test(self.getHeader('content-type'))) { - self.setHeader('content-type', 'application/x-www-form-urlencoded') + if (!/^application\/x-www-form-urlencoded\b/.test(contentType)) { + self.setHeader('Content-Type', 'application/x-www-form-urlencoded') } self.body = (typeof form === 'string') ? self._qs.rfc3986(form.toString('utf8')) : self._qs.stringify(form).toString('utf8') return self } + // form-data + var contentTypeMatch = contentType && contentType.match && + contentType.match(/^multipart\/form-data;.*boundary=(?:"([^"]+)"|([^;]+))/) + var boundary = contentTypeMatch && (contentTypeMatch[1] || contentTypeMatch[2]) // create form-data object - self._form = new FormData() + // set custom boundary if present in content-type else auto-generate + self._form = new FormData({ _boundary: boundary }) self._form.on('error', function (err) { err.message = 'form-data: ' + err.message self.emit('error', err) self.abort() }) + if (!contentTypeMatch) { + // overrides invalid or missing content-type + self.setHeader('Content-Type', 'multipart/form-data; boundary=' + self._form.getBoundary()) + } return self._form } Request.prototype.multipart = function (multipart) { @@ -1271,7 +1566,7 @@ Request.prototype.json = function (val) { var self = this if (!self.hasHeader('accept')) { - self.setHeader('accept', 'application/json') + self.setHeader('Accept', 'application/json') } if (typeof self.jsonReplacer === 'function') { @@ -1287,13 +1582,13 @@ Request.prototype.json = function (val) { self.body = self._qs.rfc3986(self.body) } if (!self.hasHeader('content-type')) { - self.setHeader('content-type', 'application/json') + self.setHeader('Content-Type', 'application/json') } } } else { self.body = safeStringify(val, self._jsonReplacer) if (!self.hasHeader('content-type')) { - self.setHeader('content-type', 'application/json') + self.setHeader('Content-Type', 'application/json') } } @@ -1367,15 +1662,15 @@ Request.prototype.aws = function (opts, now) { secretAccessKey: opts.secret, sessionToken: opts.session }) - self.setHeader('authorization', signRes.headers.Authorization) - self.setHeader('x-amz-date', signRes.headers['X-Amz-Date']) + self.setHeader('Authorization', signRes.headers.Authorization) + self.setHeader('X-Amz-Date', signRes.headers['X-Amz-Date']) if (signRes.headers['X-Amz-Security-Token']) { - self.setHeader('x-amz-security-token', signRes.headers['X-Amz-Security-Token']) + self.setHeader('X-Amz-Security-Token', signRes.headers['X-Amz-Security-Token']) } } else { // default: use aws-sign2 var date = new Date() - self.setHeader('date', date.toUTCString()) + self.setHeader('Date', date.toUTCString()) var auth = { key: opts.key, secret: opts.secret, @@ -1396,7 +1691,7 @@ Request.prototype.aws = function (opts, now) { auth.resource = '/' } auth.resource = aws2.canonicalizeResource(auth.resource) - self.setHeader('authorization', aws2.authorization(auth)) + self.setHeader('Authorization', aws2.authorization(auth)) } return self @@ -1429,38 +1724,41 @@ Request.prototype.oauth = function (_oauth) { return self } -Request.prototype.jar = function (jar) { +Request.prototype.jar = function (jar, cb) { var self = this - var cookies - - if (self._redirect.redirectsFollowed === 0) { - self.originalCookieHeader = self.getHeader('cookie') - } + self._jar = jar if (!jar) { // disable cookies - cookies = false self._disableCookies = true - } else { - var targetCookieJar = jar.getCookieString ? jar : globalCookieJar - var urihref = self.uri.href - // fetch cookie in the Specified host - if (targetCookieJar) { - cookies = targetCookieJar.getCookieString(urihref) - } + return cb() } - // if need cookie and cookie is not empty - if (cookies && cookies.length) { - if (self.originalCookieHeader) { - // Don't overwrite existing Cookie header - self.setHeader('cookie', self.originalCookieHeader + '; ' + cookies) - } else { - self.setHeader('cookie', cookies) - } + if (self._redirect.redirectsFollowed === 0) { + self.originalCookieHeader = self.getHeader('cookie') } - self._jar = jar - return self + + var targetCookieJar = jar.getCookieString ? jar : globalCookieJar + var urihref = self.uri.href + // fetch cookie in the Specified host + targetCookieJar.getCookieString(urihref, function (err, cookies) { + if (err) { return cb() } + + // if need cookie and cookie is not empty + if (cookies && cookies.length) { + if (self.originalCookieHeader) { + if (Array.isArray(self.originalCookieHeader)) { + self.originalCookieHeader = self.originalCookieHeader.join('; ') + } + // Don't overwrite existing Cookie header + self.setHeader('Cookie', self.originalCookieHeader + '; ' + cookies) + } else { + self.setHeader('Cookie', cookies) + } + } + + cb() + }) } // Stream API diff --git a/tests/ssl/ca/client.pfx b/tests/ssl/ca/client.pfx new file mode 100644 index 000000000..d5022d683 Binary files /dev/null and b/tests/ssl/ca/client.pfx differ diff --git a/tests/ssl/ca/gen-client.sh b/tests/ssl/ca/gen-client.sh index 1d5cfd203..fa6fe4508 100755 --- a/tests/ssl/ca/gen-client.sh +++ b/tests/ssl/ca/gen-client.sh @@ -23,3 +23,10 @@ openssl x509 -req \ # Encrypt with password openssl rsa -aes128 -in client.key -out client-enc.key -passout 'pass:password' + +# Use the generated CRT and the KEY to create a PKCS12 certificate +openssl pkcs12 -export \ + -inkey client.key \ + -in client.crt \ + -out client.pfx \ + -passout 'pass:password' diff --git a/tests/test-api.js b/tests/test-api.js index 3aa12fdc3..0ee8c27dc 100644 --- a/tests/test-api.js +++ b/tests/test-api.js @@ -1,9 +1,14 @@ 'use strict' +var tls = require('tls') var http = require('http') -var request = require('../index') + var tape = require('tape') + +var request = require('../index') + var server +var origCreateSecureContext = tls.createSecureContext // fixture tape('setup', function (t) { server = http.createServer() @@ -28,6 +33,111 @@ tape('callback option', function (t) { }) }) +tape('enableNodeExtraCACerts', function (t) { + request.enableNodeExtraCACerts(function (err) { + t.error(err) + t.equal(typeof tls.createSecureContext, 'function') + t.equal(typeof tls.__createSecureContext, 'function') + t.equal(tls.__createSecureContext, origCreateSecureContext) // backup + request.disableNodeExtraCACerts() // RESET + t.end() + }) +}) + +tape('enableNodeExtraCACerts: without callback', function (t) { + request.enableNodeExtraCACerts() + + t.equal(typeof tls.createSecureContext, 'function') + t.equal(typeof tls.__createSecureContext, 'function') + t.equal(tls.__createSecureContext, origCreateSecureContext) // backup + request.disableNodeExtraCACerts() // RESET + t.end() +}) + +tape('enableNodeExtraCACerts: with missing addCACert', function (t) { + // override createSecureContext + tls.createSecureContext = function () { + return { + context: { + addCACert: undefined + } + } + } + + request.enableNodeExtraCACerts(function (err) { + t.ok(err) + t.equal(err.message, 'SecureContext.addCACert is not a function') + t.equal(typeof tls.__createSecureContext, 'undefined') + tls.createSecureContext = origCreateSecureContext // RESET + t.end() + }) +}) + +tape('enableNodeExtraCACerts: on createSecureContext error', function (t) { + // override createSecureContext + tls.createSecureContext = function () { + throw new Error('something went wrong') + } + + request.enableNodeExtraCACerts(function (err) { + t.ok(err) + t.equal(err.message, 'something went wrong') + t.equal(typeof tls.__createSecureContext, 'undefined') + tls.createSecureContext = origCreateSecureContext // RESET + t.end() + }) +}) + +tape('enableNodeExtraCACerts: called twice', function (t) { + request.enableNodeExtraCACerts(function (err) { + t.error(err) + t.equal(typeof tls.createSecureContext, 'function') + t.equal(typeof tls.__createSecureContext, 'function') + t.equal(tls.__createSecureContext, origCreateSecureContext) + + // called twice + request.enableNodeExtraCACerts(function (err) { + t.error(err) + t.equal(typeof tls.createSecureContext, 'function') + t.equal(typeof tls.__createSecureContext, 'function') + t.equal(tls.__createSecureContext, origCreateSecureContext) + request.disableNodeExtraCACerts() // RESET + t.end() + }) + }) +}) + +tape('disableNodeExtraCACerts', function (t) { + // enable first + request.enableNodeExtraCACerts(function (err) { + t.error(err) + + // disable + request.disableNodeExtraCACerts() + + t.equal(typeof tls.createSecureContext, 'function') + t.equal(typeof tls.__createSecureContext, 'undefined') + t.equal(tls.createSecureContext, origCreateSecureContext) // restored + t.end() + }) +}) + +tape('disableNodeExtraCACerts: called twice', function (t) { + // enable first + request.enableNodeExtraCACerts(function (err) { + t.error(err) + + // disable + request.disableNodeExtraCACerts() + request.disableNodeExtraCACerts() + + t.equal(typeof tls.createSecureContext, 'function') + t.equal(typeof tls.__createSecureContext, 'undefined') + t.equal(tls.createSecureContext, origCreateSecureContext) // restored + t.end() + }) +}) + tape('cleanup', function (t) { server.close(t.end) }) diff --git a/tests/test-cookies.js b/tests/test-cookies.js index 6bebcaf12..52bbb7c00 100644 --- a/tests/test-cookies.js +++ b/tests/test-cookies.js @@ -52,10 +52,10 @@ tape('after server sends a cookie', function (t) { }, function (error, response, body) { t.equal(error, null) - t.equal(jar1.getCookieString(validUrl), 'foo=bar') + t.equal(jar1.getCookieStringSync(validUrl), 'foo=bar') t.equal(body, 'okay') - var cookies = jar1.getCookies(validUrl) + var cookies = jar1.getCookiesSync(validUrl) t.equal(cookies.length, 1) t.equal(cookies[0].key, 'foo') t.equal(cookies[0].value, 'bar') @@ -72,10 +72,10 @@ tape('after server sends a malformed cookie', function (t) { }, function (error, response, body) { t.equal(error, null) - t.equal(jar.getCookieString(malformedUrl), 'foo') + t.equal(jar.getCookieStringSync(malformedUrl), 'foo') t.equal(body, 'okay') - var cookies = jar.getCookies(malformedUrl) + var cookies = jar.getCookiesSync(malformedUrl) t.equal(cookies.length, 1) t.equal(cookies[0].key, '') t.equal(cookies[0].value, 'foo') @@ -92,8 +92,8 @@ tape('after server sends a cookie for a different domain', function (t) { }, function (error, response, body) { t.equal(error, null) - t.equal(jar2.getCookieString(validUrl), '') - t.deepEqual(jar2.getCookies(validUrl), []) + t.equal(jar2.getCookieStringSync(validUrl), '') + t.deepEqual(jar2.getCookiesSync(validUrl), []) t.equal(body, 'okay') t.end() }) @@ -103,12 +103,12 @@ tape('make sure setCookie works', function (t) { var jar3 = request.jar() var err = null try { - jar3.setCookie(request.cookie('foo=bar'), validUrl) + jar3.setCookieSync(request.cookie('foo=bar'), validUrl) } catch (e) { err = e } t.equal(err, null) - var cookies = jar3.getCookies(validUrl) + var cookies = jar3.getCookiesSync(validUrl) t.equal(cookies.length, 1) t.equal(cookies[0].key, 'foo') t.equal(cookies[0].value, 'bar') @@ -119,7 +119,7 @@ tape('custom store', function (t) { var Store = function () {} var store = new Store() var jar = request.jar(store) - t.equals(store, jar._jar.store) + t.equals(store, jar.store) t.end() }) diff --git a/tests/test-defaults.js b/tests/test-defaults.js index f75f5d7bc..7618a4b34 100644 --- a/tests/test-defaults.js +++ b/tests/test-defaults.js @@ -230,7 +230,7 @@ tape('recursive defaults requester', function (t) { }) tape('test custom request handler function', function (t) { - t.plan(3) + t.plan(6) var requestWithCustomHandler = request.defaults({ headers: { foo: 'bar' }, @@ -241,11 +241,13 @@ tape('test custom request handler function', function (t) { return request(params.uri, params, params.callback) }) - t.throws(function () { - requestWithCustomHandler.head(s.url + '/', function (e, r, b) { - throw new Error('We should never get here') - }) - }, /HTTP HEAD requests MUST NOT include a request body/) + requestWithCustomHandler.head(s.url + '/', function (e, r, b) { + t.equal(r.request.method, 'HEAD') + t.equal(r.request.body, 'TESTING!') + // body not parsed by Node's http-parser + t.equal(r.body, '') + t.equal(b, '') + }) requestWithCustomHandler.get(s.url + '/', function (e, r, b) { b = JSON.parse(b) diff --git a/tests/test-disablePostmanUrlEncoder.js b/tests/test-disablePostmanUrlEncoder.js new file mode 100644 index 000000000..ef65a4153 --- /dev/null +++ b/tests/test-disablePostmanUrlEncoder.js @@ -0,0 +1,141 @@ +var tape = require('tape') +var server = require('./server') +var request = require('../index') +var destroyable = require('server-destroy') + +var plainServer = server.createServer() + +destroyable(plainServer) + +tape('setup', function (t) { + plainServer.listen(0, function () { + plainServer.on('/query', function (req, res) { + res.writeHead(200) + res.end(req.url) + }) + + plainServer.on('/redirect', function (req, res) { + res.writeHead(301, { + 'Location': 'http://localhost:' + plainServer.port + '/query?%E9%82%AE=%E5%B7%AE' + }) + res.end() + }) + + plainServer.on('/redirect2', function (req, res) { + res.writeHead(301, { + 'Location': 'http://localhost:' + plainServer.port + '/query?!foo!=!bar!' + }) + res.end() + }) + + t.end() + }) +}) + +tape('UTF-8 URL without disableUrlEncoding option', function (t) { + var requestUrl = 'http://localhost:' + plainServer.port + '/query?邮=差' + + request(requestUrl, function (err, res, body) { + t.equal(err, null) + t.equal(body, '/query?%E9%82%AE=%E5%B7%AE') + t.end() + }) +}) + +tape('URL containing character \'!\' without disableUrlEncoding option', function (t) { + var requestUrl = 'http://localhost:' + plainServer.port + '/query?!foo!=!bar!' + + request(requestUrl, function (err, res, body) { + t.equal(err, null) + t.equal(body, '/query?%21foo%21=%21bar%21') + t.end() + }) +}) + +tape('Encoded UTF-8 URL in redirect without disableUrlEncoding option', function (t) { + var requestUrl = 'http://localhost:' + plainServer.port + '/redirect' + + request(requestUrl, function (err, res, body) { + t.equal(err, null) + t.equal(body, '/query?%E9%82%AE=%E5%B7%AE') + t.end() + }) +}) + +tape('URL containing character \'!\' in redirect without disableUrlEncoding option', function (t) { + var requestUrl = 'http://localhost:' + plainServer.port + '/redirect2' + + request(requestUrl, function (err, res, body) { + t.equal(err, null) + t.equal(body, '/query?%21foo%21=%21bar%21') + t.end() + }) +}) + +tape('Encoded UTF-8 URL with disableUrlEncoding=true option', function (t) { + // given URL should be pre-encoded because encoder is disabled. So the request will fail otherwise + var requestUrl = 'http://localhost:' + plainServer.port + '/query?%E9%82%AE=%E5%B7%AE' + var options = { + disableUrlEncoding: true + } + + request(requestUrl, options, function (err, res, body) { + t.equal(err, null) + t.equal(body, '/query?%E9%82%AE=%E5%B7%AE') + t.end() + }) +}) + +tape('URL containing character \'!\' with disableUrlEncoding=true option', function (t) { + var requestUrl = 'http://localhost:' + plainServer.port + '/query?!foo!=!bar!' + var options = { disableUrlEncoding: true } + + request(requestUrl, options, function (err, res, body) { + t.equal(err, null) + t.equal(body, '/query?!foo!=!bar!') + t.end() + }) +}) + +tape('Encoded UTF-8 URL in redirect with disableUrlEncoding=true option', function (t) { + // given URL should be pre-encoded because encoder is disabled. So the request will fail otherwise + var requestUrl = 'http://localhost:' + plainServer.port + '/query?%E9%82%AE=%E5%B7%AE' + var options = { + disableUrlEncoding: true + } + + request(requestUrl, options, function (err, res, body) { + t.equal(err, null) + t.equal(body, '/query?%E9%82%AE=%E5%B7%AE') + t.end() + }) +}) + +tape('URL with character \'!\' in redirect with disableUrlEncoding=true option', function (t) { + var requestUrl = 'http://localhost:' + plainServer.port + '/redirect2' + var options = { disableUrlEncoding: true } + + request(requestUrl, options, function (err, res, body) { + t.equal(err, null) + t.equal(body, '/query?!foo!=!bar!') + t.end() + }) +}) + +tape('UTF-8 URL with disableUrlEncoding=true option', function (t) { + var requestUrl = 'http://localhost:' + plainServer.port + '/query?邮=差' + var options = { disableUrlEncoding: true } + + // this request should fail because encoding is off and URL contains UTF-8 characters + request(requestUrl, options, function (err, res, body) { + t.notEqual(err, null) + t.equal(body, undefined) + t.end() + }) +}) + +tape('cleanup', function (t) { + plainServer.destroy(function () { + t.end() + }) +}) diff --git a/tests/test-errors.js b/tests/test-errors.js index 7060e9fca..2efde32e3 100644 --- a/tests/test-errors.js +++ b/tests/test-errors.js @@ -87,22 +87,3 @@ tape('multipart without body 2', function (t) { }, /^Error: Body attribute missing in multipart\.$/) t.end() }) - -tape('head method with a body', function (t) { - t.throws(function () { - request(local, { - method: 'HEAD', - body: 'foo' - }) - }, /HTTP HEAD requests MUST NOT include a request body/) - t.end() -}) - -tape('head method with a body 2', function (t) { - t.throws(function () { - request.head(local, { - body: 'foo' - }) - }, /HTTP HEAD requests MUST NOT include a request body/) - t.end() -}) diff --git a/tests/test-follow-307.js b/tests/test-follow-307.js new file mode 100644 index 000000000..dbb394bf8 --- /dev/null +++ b/tests/test-follow-307.js @@ -0,0 +1,62 @@ +'use strict' + +var http = require('http') +var request = require('../index') +var tape = require('tape') + +// test data +var redirecter = 'test1.local.omg' // always resolves to 127.0.0.1 +var responder = 'test2.local.omg' // this too. + +var server = http.createServer(function (req, res) { + if (req.headers.host.indexOf(redirecter) === 0) { + res.setHeader('location', `http://${responder}:${port}/foo`) + res.statusCode = 307 + return res.end('try again') + } else if (req.headers.host.indexOf(responder) === 0) { + res.statusCode = 200 + return res.end('ok') + } + + res.statusCode = 404 + return res.end('not found') +}) + +var port + +tape('setup', function (t) { + server.listen(0, function () { + port = this.address().port + t.end() + }) +}) + +tape('307 redirect should work when host is set explicitly, but changes on redirect', function (t) { + var redirects = 0 + + request({ + url: `http://${redirecter}:${port}/foo`, + headers: { + Host: redirecter + }, + followAllRedirects: true, + followRedirect: true, + encoding: null, + lookup: function (hostname, options, callback) { + callback(null, '127.0.0.1', 4) // All hosts will resolve to 127.0.0.1 + } + }, function (err, res, body) { + t.equal(err, null) + t.equal(body.toString(), 'ok') + t.equal(redirects, 1) + t.end() + }).on('redirect', function () { + redirects++ + }) +}) + +tape('cleanup', function (t) { + server.close(function () { + t.end() + }) +}) diff --git a/tests/test-follow-auth-header.js b/tests/test-follow-auth-header.js new file mode 100644 index 000000000..25056c467 --- /dev/null +++ b/tests/test-follow-auth-header.js @@ -0,0 +1,100 @@ +'use strict' + +var tape = require('tape') +var destroyable = require('server-destroy') + +var server = require('./server') +var request = require('../index') + +function runTest (t, statusCode, followAuthorizationHeader) { + var s = server.createServer() + var redirects = 0 + var authHeader = 'Basic aGVsbG86d29ybGQ=' + + destroyable(s) + + s.on('/', function (req, res) { + if (req.headers.host === `${s.redirectHost}:${s.port}`) { + res.writeHead(statusCode || 302, { + location: `http://${s.respondHost}:${s.port}/` + }) + res.end() + } else if (req.headers.host === `${s.respondHost}:${s.port}`) { + res.writeHead(200) + res.end('ok') + } else { + res.writeHead(400) + res.end('unknown host') + } + }) + + s.listen(0, function () { + s.redirectHost = 'test1.local.omg' // resolves to 127.0.0.1 + s.respondHost = 'test2.local.omg' // resolves to 127.0.0.1 + + request({ + url: `http://${s.redirectHost}:${s.port}`, + headers: { + authorization: authHeader + }, + followAllRedirects: true, + followRedirect: true, + followAuthorizationHeader: followAuthorizationHeader, + lookup: function (hostname, options, callback) { + callback(null, '127.0.0.1', 4) // All hosts will resolve to 127.0.0.1 + } + }, function (err, res, body) { + t.equal(err, null) + t.equal(redirects, 1) + t.equal(body.toString(), 'ok') + t.equal(res.request.headers.authorization, followAuthorizationHeader ? authHeader : undefined) + s.destroy(function () { + t.end() + }) + }).on('redirect', function () { + redirects++ + t.equal(this.response.statusCode, statusCode) + t.equal(this.uri.href, `http://${s.respondHost}:${s.port}/`) + }) + }) +} + +tape('301 redirect', function (t) { + runTest(t, 301) +}) + +tape('301 redirect + followAuthorizationHeader', function (t) { + runTest(t, 301, true) +}) + +tape('302 redirect', function (t) { + runTest(t, 302) +}) + +tape('302 redirect + followAuthorizationHeader', function (t) { + runTest(t, 302, true) +}) + +tape('303 redirect', function (t) { + runTest(t, 303) +}) + +tape('303 redirect + followAuthorizationHeader', function (t) { + runTest(t, 303, true) +}) + +tape('307 redirect', function (t) { + runTest(t, 307) +}) + +tape('307 redirect + followAuthorizationHeader', function (t) { + runTest(t, 307, true) +}) + +tape('308 redirect', function (t) { + runTest(t, 308) +}) + +tape('308 redirect + followAuthorizationHeader', function (t) { + runTest(t, 308, true) +}) diff --git a/tests/test-form-data-array.js b/tests/test-form-data-array.js new file mode 100644 index 000000000..85faeacbe --- /dev/null +++ b/tests/test-form-data-array.js @@ -0,0 +1,147 @@ +'use strict' + +var http = require('http') +var path = require('path') +var mime = require('mime-types') +var request = require('../index') +var fs = require('fs') +var tape = require('tape') +var destroyable = require('server-destroy') + +function runTest (t, options) { + var remoteFile = path.join(__dirname, 'googledoodle.jpg') + var localFile = path.join(__dirname, 'unicycle.jpg') + var multipartFormData = [] + + var server = http.createServer(function (req, res) { + if (req.url === '/file') { + res.writeHead(200, {'content-type': 'image/jpg', 'content-length': 7187}) + res.end(fs.readFileSync(remoteFile), 'binary') + return + } + + if (options.auth) { + if (!req.headers.authorization) { + res.writeHead(401, {'www-authenticate': 'Basic realm="Private"'}) + res.end() + return + } else { + t.ok(req.headers.authorization === 'Basic ' + Buffer.from('user:pass').toString('base64')) + } + } + + t.ok(/multipart\/form-data; boundary=--------------------------\d+/ + .test(req.headers['content-type'])) + + // temp workaround + var data = '' + req.setEncoding('utf8') + + req.on('data', function (d) { + data += d + }) + + req.on('end', function () { + // check for the fields' traces + + // 1st field : my_field + t.ok(data.indexOf('form-data; name="my_field"') !== -1) + t.ok(data.indexOf(multipartFormData[0].value) !== -1) + + // 2nd field : my_buffer + t.ok(data.indexOf('form-data; name="my_buffer"') !== -1) + t.ok(data.indexOf(multipartFormData[1].value) !== -1) + + // 3rd field : my_file + t.ok(data.indexOf('form-data; name="my_file"') !== -1) + t.ok(data.indexOf('; filename="' + path.basename(multipartFormData[2].value.path) + '"') !== -1) + // check for unicycle.jpg traces + t.ok(data.indexOf('2005:06:21 01:44:12') !== -1) + t.ok(data.indexOf('Content-Type: ' + mime.lookup(multipartFormData[2].value.path)) !== -1) + + // 4th field : remote_file + t.ok(data.indexOf('form-data; name="remote_file"') !== -1) + t.ok(data.indexOf('; filename="' + path.basename(multipartFormData[3].value.path) + '"') !== -1) + + // 5th field : file with metadata + t.ok(data.indexOf('form-data; name="secret_file"') !== -1) + t.ok(data.indexOf('Content-Disposition: form-data; name="secret_file"; filename="topsecret.jpg"') !== -1) + t.ok(data.indexOf('Content-Type: image/custom') !== -1) + + // 6th field : batch of files + t.ok(data.indexOf('form-data; name="batch"') !== -1) + t.ok(data.match(/form-data; name="batch"/g).length === 2) + + // 7th field : 0 + t.ok(data.indexOf('form-data; name="0"') !== -1) + t.ok(data.indexOf(multipartFormData[7].value) !== -1) + + // check for http://localhost:nnnn/file traces + t.ok(data.indexOf('Photoshop ICC') !== -1) + t.ok(data.indexOf('Content-Type: ' + mime.lookup(remoteFile)) !== -1) + + // check for form-data fields order + var prevFieldIndex = data.indexOf(`form-data; name="${multipartFormData[0].key}"`) + for (var i = 1, ii = multipartFormData.length; i < ii; i++) { + var fieldIndex = data.indexOf(`form-data; name="${multipartFormData[i].key}"`, prevFieldIndex + 1) + t.ok(fieldIndex > prevFieldIndex) + } + + res.writeHead(200) + res.end(options.json ? JSON.stringify({status: 'done'}) : 'done') + }) + }) + + destroyable(server) + + server.listen(0, function () { + var url = 'http://localhost:' + this.address().port + + multipartFormData.push({key: 'my_field', value: 'my_value'}) + multipartFormData.push({key: 'my_buffer', value: Buffer.from([1, 2, 3])}) + multipartFormData.push({key: 'my_file', value: fs.createReadStream(localFile)}) + multipartFormData.push({key: 'remote_file', value: request(url + '/file')}) + multipartFormData.push({ + key: 'secret_file', + value: fs.createReadStream(localFile), + options: { + filename: 'topsecret.jpg', + contentType: 'image/custom' + } + }) + multipartFormData.push({key: 'batch', value: fs.createReadStream(localFile)}) + multipartFormData.push({key: 'batch', value: fs.createReadStream(localFile)}) + multipartFormData.push({key: '0', value: 'numeric_field_value'}) + + var reqOptions = { + url: url + '/upload', + formData: multipartFormData + } + if (options.json) { + reqOptions.json = true + } + if (options.auth) { + reqOptions.auth = {user: 'user', pass: 'pass', sendImmediately: false} + } + request.post(reqOptions, function (err, res, body) { + t.equal(err, null) + t.equal(res.statusCode, 200) + t.deepEqual(body, options.json ? {status: 'done'} : 'done') + server.destroy(function () { + t.end() + }) + }) + }) +} + +tape('multipart formData', function (t) { + runTest(t, {json: false}) +}) + +tape('multipart formData + JSON', function (t) { + runTest(t, {json: true}) +}) + +tape('multipart formData + basic auth', function (t) { + runTest(t, {json: false, auth: true}) +}) diff --git a/tests/test-form-data-boundary.js b/tests/test-form-data-boundary.js new file mode 100644 index 000000000..652e09b8e --- /dev/null +++ b/tests/test-form-data-boundary.js @@ -0,0 +1,233 @@ +'use strict' + +var http = require('http') +var request = require('../') +var tape = require('tape') +var destroyable = require('server-destroy') + +var server = http.createServer(function (req, res) { + var data = '' + + req.on('data', function (d) { + data += d + }) + + req.once('end', function () { + res.writeHead(200) + res.end(JSON.stringify({ + headers: req.headers, + body: data + })) + }) +}) + +destroyable(server) + +tape('setup', function (t) { + server.listen(0, function () { + server.url = 'http://localhost:' + this.address().port + t.end() + }) +}) + +tape('default boundary', function (t) { + request.post({ + url: server.url, + formData: { + formKey: 'formValue' + } + }, function (err, res, body) { + var req = JSON.parse(body) + var boundary + t.equal(err, null) + t.equal(res.statusCode, 200) + t.ok(/multipart\/form-data; boundary=--------------------------\d+/ + .test(req.headers['content-type'])) + + boundary = req.headers['content-type'].split('boundary=')[1] + t.ok(/--------------------------\d+/.test(boundary)) + t.ok(req.body.startsWith('--' + boundary)) + t.ok(req.body.indexOf('name="formKey"') !== -1) + t.ok(req.body.indexOf('formValue') !== -1) + t.ok(req.body.endsWith(boundary + '--\r\n')) + t.end() + }) +}) + +tape('custom boundary', function (t) { + var boundary = 'X-FORM-DATA-BOUNDARY' + request.post({ + url: server.url, + headers: { + 'content-type': 'multipart/form-data; boundary=' + boundary + }, + formData: { + formKey: 'formValue' + } + }, function (err, res, body) { + var req = JSON.parse(body) + t.equal(err, null) + t.equal(res.statusCode, 200) + t.equal(req.headers['content-type'], 'multipart/form-data; boundary=' + boundary) + t.ok(req.body.startsWith('--' + boundary)) + t.ok(req.body.indexOf('name="formKey"') !== -1) + t.ok(req.body.indexOf('formValue') !== -1) + t.ok(req.body.endsWith(boundary + '--\r\n')) + t.end() + }) +}) + +tape('custom boundary within quotes', function (t) { + var boundary = 'X-FORM-DATA-BOUNDARY' + request.post({ + url: server.url, + headers: { + 'content-type': 'multipart/form-data; boundary="' + boundary + '"' + }, + formData: { + formKey: 'formValue' + } + }, function (err, res, body) { + var req = JSON.parse(body) + t.equal(err, null) + t.equal(res.statusCode, 200) + t.equal(req.headers['content-type'], 'multipart/form-data; boundary="' + boundary + '"') + t.ok(req.body.startsWith('--' + boundary)) + t.ok(req.body.indexOf('name="formKey"') !== -1) + t.ok(req.body.indexOf('formValue') !== -1) + t.ok(req.body.endsWith(boundary + '--\r\n')) + t.end() + }) +}) + +tape('content-length without content-type', function (t) { + request.post({ + url: server.url, + headers: { + 'content-length': '171' + }, + formData: { + formKey: 'formValue' + } + }, function (err, res, body) { + var req = JSON.parse(body) + var boundary + t.equal(err, null) + t.equal(res.statusCode, 200) + t.notEqual(req.headers['content-type'], null) + boundary = req.headers['content-type'].split('boundary=')[1] + t.equal(req.headers['content-type'], 'multipart/form-data; boundary=' + boundary) + t.equal(req.headers['content-length'], '171') + t.ok(req.body.startsWith('--' + boundary)) + t.ok(req.body.endsWith(boundary + '--\r\n')) + t.ok(req.body.indexOf('name="formKey"') !== -1) + t.ok(req.body.indexOf('formValue') !== -1) + t.end() + }) +}) + +tape('custom boundary with content-length', function (t) { + var boundary = 'X-FORM-DATA-BOUNDARY' + request.post({ + url: server.url, + headers: { + 'content-type': 'multipart/form-data; boundary=' + boundary, + 'content-length': '111' + }, + formData: { + formKey: 'formValue' + } + }, function (err, res, body) { + var req = JSON.parse(body) + t.equal(err, null) + t.equal(res.statusCode, 200) + t.equal(req.headers['content-type'], 'multipart/form-data; boundary=' + boundary) + t.equal(req.headers['content-length'], '111') + t.ok(req.body.startsWith('--' + boundary)) + t.ok(req.body.indexOf('name="formKey"') !== -1) + t.ok(req.body.indexOf('formValue') !== -1) + t.ok(req.body.endsWith(boundary + '--\r\n')) + t.end() + }) +}) + +tape('custom boundary and charset', function (t) { + var boundary = 'X-FORM-DATA-BOUNDARY' + request.post({ + url: server.url, + headers: { + 'content-type': 'multipart/form-data; charset=UTF-8; boundary=' + boundary + }, + formData: { + formKey: 'formValue' + } + }, function (err, res, body) { + var req = JSON.parse(body) + t.equal(err, null) + t.equal(res.statusCode, 200) + t.equal(req.headers['content-type'], 'multipart/form-data; charset=UTF-8; boundary=' + boundary) + t.ok(req.body.startsWith('--' + boundary)) + t.ok(req.body.indexOf('name="formKey"') !== -1) + t.ok(req.body.indexOf('formValue') !== -1) + t.ok(req.body.endsWith(boundary + '--\r\n')) + t.end() + }) +}) + +tape('custom boundary with single quotations', function (t) { + var boundary = '"X-FORM-DATA-BOUNDARY' + request.post({ + url: server.url, + headers: { + 'content-type': 'multipart/form-data; boundary=' + boundary + }, + formData: { + formKey: 'formValue' + } + }, function (err, res, body) { + var req = JSON.parse(body) + t.equal(err, null) + t.equal(res.statusCode, 200) + t.equal(req.headers['content-type'], 'multipart/form-data; boundary=' + boundary) + t.ok(req.body.startsWith('--' + boundary)) + t.ok(req.body.indexOf('name="formKey"') !== -1) + t.ok(req.body.indexOf('formValue') !== -1) + t.ok(req.body.endsWith(boundary + '--\r\n')) + t.end() + }) +}) + +tape('custom boundary with multipart/mixed', function (t) { + var boundary = 'X-FORM-DATA-BOUNDARY' + request.post({ + url: server.url, + headers: { + 'content-type': 'multipart/mixed; boundary=' + boundary + }, + formData: { + formKey: 'formValue' + } + }, function (err, res, body) { + var req = JSON.parse(body) + var boundary + t.equal(err, null) + t.equal(res.statusCode, 200) + // it should override invalid content-type + t.ok(/multipart\/form-data; boundary=--------------------------\d+/ + .test(req.headers['content-type'])) + + boundary = req.headers['content-type'].split('boundary=')[1] + t.ok(/--------------------------\d+/.test(boundary)) + t.ok(req.body.startsWith('--' + boundary)) + t.ok(req.body.indexOf('name="formKey"') !== -1) + t.ok(req.body.indexOf('formValue') !== -1) + t.ok(req.body.endsWith(boundary + '--\r\n')) + t.end() + }) +}) + +tape('cleanup', function (t) { + server.destroy(function () { + t.end() + }) +}) diff --git a/tests/test-form-data-postman.js b/tests/test-form-data-postman.js new file mode 100644 index 000000000..0c40ae183 --- /dev/null +++ b/tests/test-form-data-postman.js @@ -0,0 +1,44 @@ +'use strict' + +var request = require('../index') +var server = require('./server') +var tape = require('tape') + +var s = server.createServer() + +var path = '/upload' + +s.on(path, function (req, res) { + res.writeHead(200, { + 'Content-Type': 'application/json' + }) + res.end(JSON.stringify(req.headers)) +}) + +tape('setup', function (t) { + s.listen(0, function () { + t.end() + }) +}) + +// since form-data has been updated to ~2.3.1, this test appears to have become redundant +// The try catch wrapper around the code for this flow has been retained for sanity +tape.skip('large formData should return an error', function (t) { + request({ + uri: s.url + path, + method: 'post', + formData: {foo: new Array(3e4).fill('bar')} + }, function (err) { + t.notEqual(err, null) + t.equal(typeof err, 'object') + t.equal(err.name, 'Error') + t.equal(err.message, 'write EPIPE') + t.end() + }) +}) + +tape('cleanup', function (t) { + s.close(function () { + t.end() + }) +}) diff --git a/tests/test-form-data-redirect.js b/tests/test-form-data-redirect.js new file mode 100644 index 000000000..7fbab7c3c --- /dev/null +++ b/tests/test-form-data-redirect.js @@ -0,0 +1,127 @@ +'use strict' + +var http = require('http') +var path = require('path') +var request = require('../') +var fs = require('fs') +var tape = require('tape') +var destroyable = require('server-destroy') + +function runTest (t, options) { + var localFile = path.join(__dirname, 'googledoodle.jpg') + var multipartFormData = {} + var redirects = 0 + + var server = http.createServer(function (req, res) { + if (req.url === '/redirect') { + res.writeHead(options.responseCode, {location: options.location}) + res.end() + return + } + + t.ok(/multipart\/form-data; boundary=--------------------------\d+/ + .test(req.headers['content-type'])) + + // temp workaround + var data = '' + + req.on('data', function (d) { + data += d + }) + + req.once('end', function () { + // check for the fields' traces + + if (options.batch) { + t.ok(data.indexOf('form-data; name="my_field_batch"') !== -1) + t.ok(data.indexOf(multipartFormData.my_field_batch[0]) !== -1) + t.ok(data.indexOf(multipartFormData.my_field_batch[1]) !== -1) + } else if (options.preserveOrder) { + // 1st field : my_field + t.ok(data.indexOf('form-data; name="my_field"') !== -1) + t.ok(data.indexOf(multipartFormData[0].value) !== -1) + + // 2nd field : file with metadata + t.ok(data.indexOf('Content-Disposition: form-data; name="0"; filename="topsecret.jpg"') !== -1) + t.ok(data.indexOf('Content-Type: image/custom') !== -1) + + // check for form-data fields order + t.ok(data.indexOf(`form-data; name="${multipartFormData[0].key}"`) < + data.indexOf(`form-data; name="${multipartFormData[1].key}"`)) + } else { + // 1st field : my_field + t.ok(data.indexOf('form-data; name="my_field"') !== -1) + t.ok(data.indexOf(multipartFormData.my_field) !== -1) + + // 2nd field : my_buffer + t.ok(data.indexOf('form-data; name="my_buffer"') !== -1) + t.ok(data.indexOf(multipartFormData.my_buffer) !== -1) + } + + // formdata boundary + t.ok(data.endsWith((/boundary=(.*)$/).exec(req.headers['content-type'])[1] + '--\r\n')) + + res.writeHead(200) + res.end('done') + }) + }) + + destroyable(server) + + // this will cause servers to listen on a random port + server.listen(0, function () { + var url = 'http://localhost:' + this.address().port + // both together have flaky behavior because of the following issue: + // https://github.com/request/request/issues/887#issuecomment-347050137 + if (options.batch) { + multipartFormData.my_field_batch = ['my_value_1', 'my_value_2'] + } else if (options.preserveOrder) { + multipartFormData = [{ + key: 'my_field', + value: 'my_value' + }, { + key: '0', + value: fs.createReadStream(localFile), + options: { + filename: 'topsecret.jpg', + contentType: 'image/custom' + } + }] + } else { + multipartFormData.my_field = 'my_value' + multipartFormData.my_buffer = Buffer.from([1, 2, 3]) + } + + request.post({ + url: url + options.url, + formData: multipartFormData, + followAllRedirects: true + }, function (err, res, body) { + t.equal(err, null) + t.equal(redirects, 1) + t.equal(res.statusCode, 200) + t.deepEqual(body, options.json ? {status: 'done'} : 'done') + server.destroy(function () { + t.end() + }) + }).on('redirect', function () { + redirects++ + }) + }) +} + +tape('multipart formData + 307 redirect', function (t) { + runTest(t, {url: '/redirect', responseCode: 307, location: '/upload'}) +}) + +tape('multipart formData + 307 redirect + batch', function (t) { + runTest(t, {url: '/redirect', responseCode: 307, location: '/upload', batch: true}) +}) + +tape('multipart formData + 308 redirect', function (t) { + runTest(t, {url: '/redirect', responseCode: 308, location: '/upload'}) +}) + +tape('multipart formData + 308 redirect + preserveOrder', function (t) { + runTest(t, {url: '/redirect', responseCode: 308, location: '/upload', preserveOrder: true}) +}) diff --git a/tests/test-form-data.js b/tests/test-form-data.js index 990562be5..b9f417d4c 100644 --- a/tests/test-form-data.js +++ b/tests/test-form-data.js @@ -6,6 +6,7 @@ var mime = require('mime-types') var request = require('../index') var fs = require('fs') var tape = require('tape') +var destroyable = require('server-destroy') function runTest (t, options) { var remoteFile = path.join(__dirname, 'googledoodle.jpg') @@ -80,6 +81,8 @@ function runTest (t, options) { }) }) + destroyable(server) + server.listen(0, function () { var url = 'http://localhost:' + this.address().port // @NOTE: multipartFormData properties must be set here so that my_file read stream does not leak in node v0.8 @@ -113,7 +116,7 @@ function runTest (t, options) { t.equal(err, null) t.equal(res.statusCode, 200) t.deepEqual(body, options.json ? {status: 'done'} : 'done') - server.close(function () { + server.destroy(function () { t.end() }) }) diff --git a/tests/test-headers.js b/tests/test-headers.js index 68b748691..5e0ce9d10 100644 --- a/tests/test-headers.js +++ b/tests/test-headers.js @@ -62,17 +62,18 @@ function addTests () { }) var jar = request.jar() - jar.setCookie('quux=baz', s.url) + jar.setCookieSync('c1=v1', s.url) + jar.setCookieSync('c2=v2', s.url) runTest( '#125: headers.cookie + cookie jar', 'header-and-jar', - {jar: jar, headers: {cookie: 'foo=bar'}}, + {jar: jar, headers: {cookie: ['c3=v3', 'c4=v4']}}, function (t, req, res) { - t.equal(req.headers.cookie, 'foo=bar; quux=baz') + t.equal(req.headers.cookie, 'c3=v3; c4=v4; c1=v1; c2=v2') }) var jar2 = request.jar() - jar2.setCookie('quux=baz; Domain=foo.bar.com', s.url, {ignoreError: true}) + jar2.setCookieSync('quux=baz; Domain=foo.bar.com', s.url, {ignoreError: true}) runTest( '#794: ignore cookie parsing and domain errors', 'ignore-errors', diff --git a/tests/test-maxResponseSize.js b/tests/test-maxResponseSize.js new file mode 100644 index 000000000..aa85910ef --- /dev/null +++ b/tests/test-maxResponseSize.js @@ -0,0 +1,132 @@ +var request = require('../index') +var http = require('http') +var zlib = require('zlib') +var tape = require('tape') +var url = require('url') + +var CHAR = 'X' + +// request path to this server should be of the form '/?gzip=[true/false]' +// response from the server will have size of from request path +var server = http.createServer(function (req, res) { + var parsedUrl = url.parse(req.url, {parseQueryString: true}) + var bytes = parseInt(parsedUrl.pathname.substring(1)) || 0 + var gzip = parsedUrl.query.gzip + var data = Buffer.from(CHAR.repeat(bytes)) + + res.setHeader('Content-Type', 'text/plain') + + if (gzip === 'true') { + zlib.gzip(data, function (err, compressedData) { + if (err) { + res.writeHead(500) + res.end() + return + } + + res.setHeader('Content-Encoding', 'gzip') + res.setHeader('Content-Length', compressedData.length) + res.writeHead(200) + res.write(compressedData) + res.end() + }) + } else { + res.setHeader('Content-Length', data.length) + res.writeHead(200) + res.write(data) + res.end() + } +}) + +tape('setup', function (t) { + server.listen(0, function () { + server.port = this.address().port + server.url = 'http://localhost:' + server.port + t.end() + }) +}) + +tape('response < maxResponseSize', function (t) { + var options = { + method: 'GET', + uri: server.url + '/50', + maxResponseSize: 100 + } + + request(options, function (err, res, body) { + t.equal(err, null) + t.ok(body, 'Should receive body') + t.ok(body.length < options.maxResponseSize) + t.end() + }) +}) + +tape('response = maxResponseSize', function (t) { + var options = { + method: 'GET', + uri: server.url + '/100', + maxResponseSize: 100 + } + + request(options, function (err, res, body) { + t.equal(err, null) + t.ok(body, 'Should receive body') + t.ok(body.length === options.maxResponseSize) + t.end() + }) +}) + +tape('response > maxResponseSize', function (t) { + var options = { + method: 'GET', + uri: server.url + '/200', + maxResponseSize: 100 + } + + request(options, function (err, res, body) { + t.notEqual(err, null) + t.equal(typeof err, 'object') + t.equal(err.name, 'Error') + t.equal(err.message, 'Maximum response size reached') + t.end() + }) +}) + +tape('extracted gzip response > maxResponseSize but content-length < maxResponseSize', function (t) { + var options = { + method: 'GET', + uri: server.url + '/500?gzip=true', // for 500 bytes gzip response, content-length will be around 30 + maxResponseSize: 490, + gzip: true + } + + request(options, function (err, res, body) { + t.notEqual(err, null) + t.equal(typeof err, 'object') + t.equal(err.name, 'Error') + t.equal(err.message, 'Maximum response size reached') + t.end() + }) +}) + +tape('extracted gzip response < maxResponseSize', function (t) { + var options = { + method: 'GET', + uri: server.url + '/100?gzip=true', + maxResponseSize: 200, + gzip: true + } + + request(options, function (err, res, body) { + t.equal(err, null) + t.ok(body, 'Should receive body') + t.ok(body.length < options.maxResponseSize) + t.end() + }) +}) + +tape('cleanup', function (t) { + server.close(function () { + t.end() + }) +}) diff --git a/tests/test-proxy-connect.js b/tests/test-proxy-connect.js index 06800d00e..08047caae 100644 --- a/tests/test-proxy-connect.js +++ b/tests/test-proxy-connect.js @@ -65,7 +65,7 @@ tape('proxy', function (t) { 'dont-send-to-proxy: ok', 'accept: yo', 'user-agent: just another foobar', - 'host: google.com' + 'Host: google.com' ].join('\r\n')) t.equal(true, re.test(data)) t.equal(called, true, 'the request must be made to the proxy server') diff --git a/tests/test-redirect.js b/tests/test-redirect.js index b7b5ca676..f161b3f74 100644 --- a/tests/test-redirect.js +++ b/tests/test-redirect.js @@ -86,7 +86,7 @@ tape('setup', function (t) { }) tape('permanent bounce', function (t) { - jar.setCookie('quux=baz', s.url) + jar.setCookieSync('quux=baz', s.url) hits = {} request({ uri: s.url + '/perm', @@ -103,7 +103,7 @@ tape('permanent bounce', function (t) { }) tape('preserve HEAD method when using followAllRedirects', function (t) { - jar.setCookie('quux=baz', s.url) + jar.setCookieSync('quux=baz', s.url) hits = {} request({ method: 'HEAD', @@ -268,6 +268,69 @@ tape('should follow delete redirects when followallredirects true', function (t) }) }) +// @note Previously all the methods get redirected with their request method +// preserved(not changed to GET) other than the following 4: +// PATCH, PUT, POST, DELETE (Probably accounted for only GET & HEAD). +// BUT, with followAllRedirects set, the request method is changed to GET for +// non GET & HEAD methods. +// +// Since: https://github.com/postmanlabs/postman-request/pull/31 +// non GET & HEAD methods get redirects with their method changed to GET unless +// followOriginalHttpMethod is set. +tape('should follow options redirects by default', function (t) { + hits = {} + request({ + method: 'OPTIONS', // options because custom method is not supported by Node's HTTP server + uri: s.url + '/temp', + jar: jar, + headers: { cookie: 'foo=bar' } + }, function (err, res, body) { + t.equal(err, null) + t.equal(res.statusCode, 200) + t.ok(hits.temp, 'Original request is to /temp') + t.ok(hits.temp_landing, 'Forward to temporary landing URL') + t.equal(body, 'GET temp_landing', 'Got temporary landing content') // Previously redirected with OPTIONS + t.end() + }) +}) + +tape('should follow options redirects when followallredirects true', function (t) { + hits = {} + request({ + method: 'OPTIONS', + uri: s.url + '/temp', + followAllRedirects: true, + jar: jar, + headers: { cookie: 'foo=bar' } + }, function (err, res, body) { + t.equal(err, null) + t.equal(res.statusCode, 200) + t.ok(hits.temp, 'Original request is to /temp') + t.ok(hits.temp_landing, 'Forward to temporary landing URL') + t.equal(body, 'GET temp_landing', 'Got temporary landing content') + t.end() + }) +}) + +tape('should follow options redirects when followallredirects true and followOriginalHttpMethod is enabled', function (t) { + hits = {} + request({ + method: 'OPTIONS', + uri: s.url + '/temp', + followAllRedirects: true, + followOriginalHttpMethod: true, + jar: jar, + headers: { cookie: 'foo=bar' } + }, function (err, res, body) { + t.equal(err, null) + t.equal(res.statusCode, 200) + t.ok(hits.temp, 'Original request is to /temp') + t.ok(hits.temp_landing, 'Forward to temporary landing URL') + t.equal(body, 'OPTIONS temp_landing', 'Got temporary landing content') + t.end() + }) +}) + tape('should follow 307 delete redirects when followallredirects true', function (t) { hits = {} request.del(s.url + '/fwd', { @@ -366,7 +429,7 @@ tape('should have referer header by default when following redirect', function ( t.end() }) .on('redirect', function () { - t.equal(this.headers.referer, s.url + '/temp') + t.equal(this.headers.Referer, s.url + '/temp') }) }) @@ -383,7 +446,7 @@ tape('should not have referer header when removeRefererHeader is true', function t.end() }) .on('redirect', function () { - t.equal(this.headers.referer, undefined) + t.equal(this.headers.Referer, undefined) }) }) diff --git a/tests/test-ssl.js b/tests/test-ssl.js new file mode 100644 index 000000000..dd0e3f4d8 --- /dev/null +++ b/tests/test-ssl.js @@ -0,0 +1,160 @@ +'use strict' + +// this also validates that for each configuration new Agent is created +// previously same Agent was re-used on passphrase change + +var server = require('./server') +var request = require('../index') +var fs = require('fs') +var path = require('path') +var tape = require('tape') + +var caPath = path.resolve(__dirname, 'ssl/ca/ca.crt') +var ca = fs.readFileSync(caPath) +var clientPfx = fs.readFileSync(path.resolve(__dirname, 'ssl/ca/client.pfx')) +var clientKey = fs.readFileSync(path.resolve(__dirname, 'ssl/ca/client.key')) +var clientCert = fs.readFileSync(path.resolve(__dirname, 'ssl/ca/client.crt')) +var clientKeyEnc = fs.readFileSync(path.resolve(__dirname, 'ssl/ca/client-enc.key')) +var clientPassword = 'password' + +var sslServer = server.createSSLServer({ + key: path.resolve(__dirname, 'ssl/ca/localhost.key'), + cert: path.resolve(__dirname, 'ssl/ca/localhost.crt'), + ca: caPath, + requestCert: true, + rejectUnauthorized: true +}) + +tape('setup', function (t) { + sslServer.on('/', function (req, res) { + if (req.client.authorized) { + res.writeHead(200, { 'Content-Type': 'text/plain' }) + res.end('authorized') + } else { + res.writeHead(401, { 'Content-Type': 'text/plain' }) + res.end('unauthorized') + } + }) + + sslServer.listen(0, function () { + t.end() + }) +}) + +tape('key + cert', function (t) { + request({ + url: sslServer.url, + ca: ca, + key: clientKey, + cert: clientCert + }, function (err, res, body) { + t.equal(err, null) + t.equal(body.toString(), 'authorized') + t.end() + }) +}) + +tape('key + cert + passphrase', function (t) { + request({ + url: sslServer.url, + ca: ca, + key: clientKeyEnc, + cert: clientCert, + passphrase: clientPassword + }, function (err, res, body) { + t.equal(err, null) + t.equal(body.toString(), 'authorized') + t.end() + }) +}) + +tape('key + cert + passphrase(invalid)', function (t) { + request({ + url: sslServer.url, + ca: ca, + key: clientKeyEnc, + cert: clientCert, + passphrase: 'invalidPassphrase' + }, function (err, res, body) { + t.ok(err) + t.end() + }) +}) + +tape('pfx + passphrase', function (t) { + request({ + url: sslServer.url, + ca: ca, + pfx: clientPfx, + passphrase: clientPassword + }, function (err, res, body) { + t.equal(err, null) + t.equal(body.toString(), 'authorized') + t.end() + }) +}) + +tape('pfx + passphrase(invalid)', function (t) { + request({ + url: sslServer.url, + ca: ca, + pfx: clientPfx, + passphrase: 'invalidPassphrase' + }, function (err, res, body) { + t.ok(err) + t.end() + }) +}) + +tape('extraCA(NodeExtraCACerts: enabled)', function (t) { + // enable extraCA support + request.enableNodeExtraCACerts() + + request({ + url: sslServer.url, + extraCA: ca, + key: clientKey, + cert: clientCert + }, function (err, res, body) { + t.equal(err, null) + t.equal(body.toString(), 'authorized') + request.disableNodeExtraCACerts() // RESET + t.end() + }) +}) + +tape('extraCA(NodeExtraCACerts: disabled)', function (t) { + request({ + url: sslServer.url, + extraCA: ca, + key: clientKey, + cert: clientCert + }, function (err, res, body) { + t.ok(err) + t.end() + }) +}) + +tape('ca + extraCA(NodeExtraCACerts: enabled)', function (t) { + // enable extraCA support + request.enableNodeExtraCACerts() + + request({ + url: sslServer.url, + ca: ca, + extraCA: '---INVALID CERT---', // make sure this won't affect options.ca + key: clientKey, + cert: clientCert + }, function (err, res, body) { + t.equal(err, null) + t.equal(body.toString(), 'authorized') + request.disableNodeExtraCACerts() // RESET + t.end() + }) +}) + +tape('cleanup', function (t) { + sslServer.close(function () { + t.end() + }) +}) diff --git a/tests/test-stream.js b/tests/test-stream.js index 1d7bf3de0..b9603561f 100644 --- a/tests/test-stream.js +++ b/tests/test-stream.js @@ -31,6 +31,22 @@ tape('request body stream', function (t) { }) }) +tape('request body stream content-length', function (t) { + var fpath = path.join(__dirname, 'unicycle.jpg') + var input = fs.createReadStream(fpath, {highWaterMark: 1000}) + var r = request({ + uri: server.url, + method: 'POST', + body: input, + encoding: null + }, function (err, res, body) { + t.error(err) + t.equal(body.length, fs.statSync(fpath).size) + t.equal(r.getHeader('content-length'), body.length) + t.end() + }) +}) + tape('after', function (t) { server.close(t.end) }) diff --git a/tests/test-timing.js b/tests/test-timing.js index f3e77f929..ff63825e9 100644 --- a/tests/test-timing.js +++ b/tests/test-timing.js @@ -1,13 +1,20 @@ 'use strict' -var server = require('./server') -var request = require('../index') var tape = require('tape') var http = require('http') +var https = require('https') +var destroyable = require('server-destroy') + +var server = require('./server') +var request = require('../index') var plainServer = server.createServer() +var httpsServer = server.createSSLServer() var redirectMockTime = 10 +destroyable(plainServer) +destroyable(httpsServer) + tape('setup', function (t) { plainServer.listen(0, function () { plainServer.on('/', function (req, res) { @@ -22,11 +29,25 @@ tape('setup', function (t) { }, redirectMockTime) }) - t.end() + httpsServer.listen(0, function () { + httpsServer.on('/', function (req, res) { + res.writeHead(200) + res.end('https') + }) + httpsServer.on('/redir', function (req, res) { + // fake redirect delay to ensure strong signal for rollup check + setTimeout(function () { + res.writeHead(301, { 'location': 'https://localhost:' + httpsServer.port + '/' }) + res.end() + }, redirectMockTime) + }) + + t.end() + }) }) }) -tape('non-redirected request is timed', function (t) { +tape('HTTP: non-redirected request is timed', function (t) { var options = {time: true} var start = new Date().getTime() @@ -36,6 +57,7 @@ tape('non-redirected request is timed', function (t) { t.equal(err, null) t.equal(typeof res.elapsedTime, 'number') t.equal(typeof res.responseStartTime, 'number') + t.equal(typeof res.timingStartTimer, 'number') t.equal(typeof res.timingStart, 'number') t.equal((res.timingStart >= start), true) t.equal(typeof res.timings, 'object') @@ -77,7 +99,7 @@ tape('non-redirected request is timed', function (t) { }) }) -tape('redirected request is timed with rollup', function (t) { +tape('HTTP: redirected request is timed with rollup', function (t) { var options = {time: true} var r = request('http://localhost:' + plainServer.port + '/redir', options, function (err, res, body) { t.equal(err, null) @@ -91,7 +113,7 @@ tape('redirected request is timed with rollup', function (t) { }) }) -tape('keepAlive is timed', function (t) { +tape('HTTP: keepAlive is timed', function (t) { var agent = new http.Agent({ keepAlive: true }) var options = { time: true, agent: agent } var start1 = new Date().getTime() @@ -140,8 +162,130 @@ tape('keepAlive is timed', function (t) { }) }) -tape('cleanup', function (t) { - plainServer.close(function () { +tape('HTTPS: non-redirected request is timed', function (t) { + var options = {time: true, strictSSL: false} + + var start = new Date().getTime() + var r = request('https://localhost:' + httpsServer.port + '/', options, function (err, res, body) { + var end = new Date().getTime() + + t.equal(err, null) + t.equal(typeof res.elapsedTime, 'number') + t.equal(typeof res.responseStartTime, 'number') + t.equal(typeof res.timingStartTimer, 'number') + t.equal(typeof res.timingStart, 'number') + t.equal((res.timingStart >= start), true) + t.equal(typeof res.timings, 'object') + t.equal((res.elapsedTime > 0), true) + t.equal((res.elapsedTime <= (end - start)), true) + t.equal((res.responseStartTime > r.startTime), true) + t.equal((res.timings.socket >= 0), true) + t.equal((res.timings.lookup >= res.timings.socket), true) + t.equal((res.timings.connect >= res.timings.lookup), true) + t.equal((res.timings.secureConnect >= res.timings.connect), true) + t.equal((res.timings.response >= res.timings.secureConnect), true) + t.equal((res.timings.end >= res.timings.response), true) + t.equal(typeof res.timingPhases, 'object') + t.equal((res.timingPhases.wait >= 0), true) + t.equal((res.timingPhases.dns >= 0), true) + t.equal((res.timingPhases.tcp >= 0), true) + t.equal((res.timingPhases.secureHandshake >= 0), true) + t.equal((res.timingPhases.firstByte > 0), true) + t.equal((res.timingPhases.download > 0), true) + t.equal((res.timingPhases.total > 0), true) + t.equal((res.timingPhases.total <= (end - start)), true) + + // validate there are no unexpected properties + var propNames = [] + for (var propName in res.timings) { + if (res.timings.hasOwnProperty(propName)) { + propNames.push(propName) + } + } + t.deepEqual(propNames, ['socket', 'lookup', 'connect', 'secureConnect', 'response', 'end']) + + propNames = [] + for (propName in res.timingPhases) { + if (res.timingPhases.hasOwnProperty(propName)) { + propNames.push(propName) + } + } + t.deepEqual(propNames, ['wait', 'dns', 'tcp', 'firstByte', 'download', 'total', 'secureHandshake']) + + t.end() + }) +}) + +tape('HTTPS: redirected request is timed with rollup', function (t) { + var options = {time: true, strictSSL: false} + var r = request('https://localhost:' + httpsServer.port + '/redir', options, function (err, res, body) { + t.equal(err, null) + t.equal(typeof res.elapsedTime, 'number') + t.equal(typeof res.responseStartTime, 'number') + t.equal((res.elapsedTime > 0), true) + t.equal((res.responseStartTime > 0), true) + t.equal((res.elapsedTime > redirectMockTime), true) + t.equal((res.responseStartTime > r.startTime), true) t.end() }) }) + +tape('HTTPS: keepAlive is timed', function (t) { + var agent = new https.Agent({ keepAlive: true }) + var options = { time: true, agent: agent, strictSSL: false } + var start1 = new Date().getTime() + + request('https://localhost:' + httpsServer.port + '/', options, function (err1, res1, body1) { + var end1 = new Date().getTime() + + // ensure the first request's timestamps look ok + t.equal((res1.timingStart >= start1), true) + t.equal((start1 <= end1), true) + + t.equal((res1.timings.socket >= 0), true) + t.equal((res1.timings.lookup >= res1.timings.socket), true) + t.equal((res1.timings.connect >= res1.timings.lookup), true) + t.equal((res1.timings.secureConnect >= res1.timings.connect), true) + t.equal((res1.timings.response >= res1.timings.secureConnect), true) + + // open a second request with the same agent so we re-use the same connection + var start2 = new Date().getTime() + + request('https://localhost:' + httpsServer.port + '/', options, function (err2, res2, body2) { + var end2 = new Date().getTime() + + // ensure the second request's timestamps look ok + t.equal((res2.timingStart >= start2), true) + t.equal((start2 <= end2), true) + + // ensure socket==lookup==connect==secureConnect for the second request + t.equal((res2.timings.socket >= 0), true) + t.equal((res2.timings.lookup === res2.timings.socket), true) + t.equal((res2.timings.connect === res2.timings.lookup), true) + t.equal((res2.timings.secureConnect === res2.timings.connect), true) + t.equal((res2.timings.response >= res2.timings.connect), true) + + // explicitly shut down the agent + if (typeof agent.destroy === 'function') { + agent.destroy() + } else { + // node < 0.12 + Object.keys(agent.sockets).forEach(function (name) { + agent.sockets[name].forEach(function (socket) { + socket.end() + }) + }) + } + + t.end() + }) + }) +}) + +tape('cleanup', function (t) { + plainServer.destroy(function () { + httpsServer.destroy(function () { + t.end() + }) + }) +}) diff --git a/tests/test-tunnel-response.js b/tests/test-tunnel-response.js new file mode 100644 index 000000000..b0108a971 --- /dev/null +++ b/tests/test-tunnel-response.js @@ -0,0 +1,379 @@ +'use strict' + +// @note The proxy server in this test file sends a response body for the +// `CONNECT` request. The RFC says that the `CONNECT` response does not have a +// response body, but some proxy servers might be sending it anyway. Here we +// ensure that we can handle this case. + +var server = require('./server') +var tape = require('tape') +var request = require('../index') +var https = require('https') +var net = require('net') +var fs = require('fs') +var path = require('path') +var util = require('util') +var url = require('url') +var destroyable = require('server-destroy') + +var events = [] +var caFile = path.resolve(__dirname, 'ssl/ca/ca.crt') +var ca = fs.readFileSync(caFile) +var clientCert = fs.readFileSync(path.resolve(__dirname, 'ssl/ca/client.crt')) +var clientKey = fs.readFileSync(path.resolve(__dirname, 'ssl/ca/client-enc.key')) +var invalidClientKey = fs.readFileSync(path.resolve(__dirname, 'ssl/ca/localhost.key')) +var clientPassword = 'password' +var sslOpts = { + key: path.resolve(__dirname, 'ssl/ca/localhost.key'), + cert: path.resolve(__dirname, 'ssl/ca/localhost.crt') +} + +var mutualSSLOpts = { + key: path.resolve(__dirname, 'ssl/ca/localhost.key'), + cert: path.resolve(__dirname, 'ssl/ca/localhost.crt'), + ca: caFile, + requestCert: true, + rejectUnauthorized: true +} + +var s = server.createServer() +var ss = server.createSSLServer(sslOpts) +var ss2 = server.createSSLServer(mutualSSLOpts) + +// XXX when tunneling https over https, connections get left open so the server +// doesn't want to close normally (and same issue with http server on v0.8.x) +destroyable(s) +destroyable(ss) +destroyable(ss2) + +function event () { + events.push(util.format.apply(null, arguments)) +} + +function setListeners (server, type) { + server.on('/', function (req, res) { + event('%s response', type) + res.end(type + ' ok') + }) + + server.on('request', function (req, res) { + if (/^https?:/.test(req.url)) { + // This is a proxy request + var dest = req.url.split(':')[0] + // Is it a redirect? + var match = req.url.match(/\/redirect\/(https?)$/) + if (match) { + dest += '->' + match[1] + } + event('%s proxy to %s', type, dest) + request(req.url, { followRedirect: false }).pipe(res) + } + }) + + server.on('/redirect/http', function (req, res) { + event('%s redirect to http', type) + res.writeHead(301, { + location: s.url + }) + res.end() + }) + + server.on('/redirect/https', function (req, res) { + event('%s redirect to https', type) + res.writeHead(301, { + location: ss.url + }) + res.end() + }) + + server.on('connect', function (req, client, head) { + var u = url.parse(req.url) + var server = net.connect(u.host, u.port, function () { + event('%s connect to %s', type, req.url) + client.write('HTTP/1.1 200 Connection established\r\n' + + 'Proxy-Agent: postman-proxy-agent\r\n' + + '\r\n' + + 'OK') + client.pipe(server) + server.write(head) + server.pipe(client) + }) + }) +} + +setListeners(s, 'http') +setListeners(ss, 'https') +setListeners(ss2, 'https') + +// monkey-patch since you can't set a custom certificate authority for the +// proxy in tunnel-agent (this is necessary for "* over https" tests) +var customCaCount = 0 +var httpsRequestOld = https.request +https.request = function (options) { + if (customCaCount) { + options.ca = ca + customCaCount-- + } + return httpsRequestOld.apply(this, arguments) +} + +function runTest (name, opts, expected) { + tape(name, function (t) { + opts.ca = ca + if (opts.proxy === ss.url) { + customCaCount = (opts.url === ss.url ? 2 : 1) + } + request(opts, function (err, res, body) { + event(err ? 'err ' + err.message : res.statusCode + ' ' + body) + t.deepEqual(events, expected) + events = [] + t.end() + }) + }) +} + +function addTests () { + // HTTP OVER HTTP + + runTest('http over http, tunnel=true, with proxy response', { + url: s.url, + proxy: s.url, + tunnel: true + }, [ + 'http connect to localhost:' + s.port, + 'http response', + '200 http ok' + ]) + + runTest('http over http, tunnel=default, with proxy response', { + url: s.url, + proxy: s.url + }, [ + 'http proxy to http', + 'http response', + '200 http ok' + ]) + + // HTTP OVER HTTPS + + runTest('http over https, tunnel=true, with proxy response', { + url: s.url, + proxy: ss.url, + tunnel: true + }, [ + 'https connect to localhost:' + s.port, + 'http response', + '200 http ok' + ]) + + runTest('http over https, tunnel=default, with proxy response', { + url: s.url, + proxy: ss.url + }, [ + 'https proxy to http', + 'http response', + '200 http ok' + ]) + + // HTTPS OVER HTTP + + runTest('https over http, tunnel=true, with proxy response', { + url: ss.url, + proxy: s.url, + tunnel: true + }, [ + 'http connect to localhost:' + ss.port, + 'https response', + '200 https ok' + ]) + + runTest('https over http, tunnel=default, with proxy response', { + url: ss.url, + proxy: s.url + }, [ + 'http connect to localhost:' + ss.port, + 'https response', + '200 https ok' + ]) + + // HTTPS OVER HTTPS + + runTest('https over https, tunnel=true, with proxy response', { + url: ss.url, + proxy: ss.url, + tunnel: true + }, [ + 'https connect to localhost:' + ss.port, + 'https response', + '200 https ok' + ]) + + runTest('https over https, tunnel=default, with proxy response', { + url: ss.url, + proxy: ss.url + }, [ + 'https connect to localhost:' + ss.port, + 'https response', + '200 https ok' + ]) + + // HTTP->HTTP OVER HTTP + + runTest('http->http over http, tunnel=true, with proxy response', { + url: s.url + '/redirect/http', + proxy: s.url, + tunnel: true + }, [ + 'http connect to localhost:' + s.port, + 'http redirect to http', + 'http connect to localhost:' + s.port, + 'http response', + '200 http ok' + ]) + + runTest('http->http over http, tunnel=default, with proxy response', { + url: s.url + '/redirect/http', + proxy: s.url + }, [ + 'http proxy to http->http', + 'http redirect to http', + 'http proxy to http', + 'http response', + '200 http ok' + ]) + + // HTTP->HTTPS OVER HTTP + + runTest('http->https over http, tunnel=true, with proxy response', { + url: s.url + '/redirect/https', + proxy: s.url, + tunnel: true + }, [ + 'http connect to localhost:' + s.port, + 'http redirect to https', + 'http connect to localhost:' + ss.port, + 'https response', + '200 https ok' + ]) + + runTest('http->https over http, tunnel=default, with proxy response', { + url: s.url + '/redirect/https', + proxy: s.url + }, [ + 'http proxy to http->https', + 'http redirect to https', + 'http connect to localhost:' + ss.port, + 'https response', + '200 https ok' + ]) + + // HTTPS->HTTP OVER HTTP + + runTest('https->http over http, tunnel=true, with proxy response', { + url: ss.url + '/redirect/http', + proxy: s.url, + tunnel: true + }, [ + 'http connect to localhost:' + ss.port, + 'https redirect to http', + 'http connect to localhost:' + s.port, + 'http response', + '200 http ok' + ]) + + runTest('https->http over http, tunnel=default, with proxy response', { + url: ss.url + '/redirect/http', + proxy: s.url + }, [ + 'http connect to localhost:' + ss.port, + 'https redirect to http', + 'http proxy to http', + 'http response', + '200 http ok' + ]) + + // HTTPS->HTTPS OVER HTTP + + runTest('https->https over http, tunnel=true, with proxy response', { + url: ss.url + '/redirect/https', + proxy: s.url, + tunnel: true + }, [ + 'http connect to localhost:' + ss.port, + 'https redirect to https', + 'http connect to localhost:' + ss.port, + 'https response', + '200 https ok' + ]) + + runTest('https->https over http, tunnel=default, with proxy response', { + url: ss.url + '/redirect/https', + proxy: s.url + }, [ + 'http connect to localhost:' + ss.port, + 'https redirect to https', + 'http connect to localhost:' + ss.port, + 'https response', + '200 https ok' + ]) + + // MUTUAL HTTPS OVER HTTP + + runTest('mutual https over http, tunnel=true, with proxy response', { + url: ss2.url, + proxy: s.url, + tunnel: true, + cert: clientCert, + key: clientKey, + passphrase: clientPassword + }, [ + 'http connect to localhost:' + ss2.port, + 'https response', + '200 https ok' + ]) + + runTest('mutual https over http, tunnel=default, with proxy response', { + url: ss2.url, + proxy: s.url, + cert: clientCert, + key: clientKey, + passphrase: clientPassword + }, [ + 'http connect to localhost:' + ss2.port, + 'https response', + '200 https ok' + ]) + + // Client key mismatch for HTTPS over HTTP + + runTest('mutual https over http with client key mismatch, tunnel=default, with proxy response', { + url: ss2.url, + proxy: s.url, + cert: clientCert, + key: invalidClientKey + }, [ + 'http connect to localhost:' + ss2.port, + // it should bubble up the key mismatch error + 'err error:0B080074:x509 certificate routines:X509_check_private_key:key values mismatch' + ]) +} + +tape('setup', function (t) { + s.listen(0, function () { + ss.listen(0, function () { + ss2.listen(0, 'localhost', function () { + addTests() + tape('cleanup', function (t) { + s.destroy(function () { + ss.destroy(function () { + ss2.destroy(function () { + t.end() + }) + }) + }) + }) + t.end() + }) + }) + }) +}) diff --git a/tests/test-tunnel.js b/tests/test-tunnel.js index fa2ebce33..7e97b7d21 100644 --- a/tests/test-tunnel.js +++ b/tests/test-tunnel.js @@ -16,6 +16,7 @@ var caFile = path.resolve(__dirname, 'ssl/ca/ca.crt') var ca = fs.readFileSync(caFile) var clientCert = fs.readFileSync(path.resolve(__dirname, 'ssl/ca/client.crt')) var clientKey = fs.readFileSync(path.resolve(__dirname, 'ssl/ca/client-enc.key')) +var invalidClientKey = fs.readFileSync(path.resolve(__dirname, 'ssl/ca/localhost.key')) var clientPassword = 'password' var sslOpts = { key: path.resolve(__dirname, 'ssl/ca/localhost.key'), @@ -443,6 +444,19 @@ function addTests () { 'https response', '200 https ok' ]) + + // Client key mismatch for HTTPS over HTTP + + runTest('mutual https over http with client key mismatch, tunnel=default', { + url: ss2.url, + proxy: s.url, + cert: clientCert, + key: invalidClientKey + }, [ + 'http connect to localhost:' + ss2.port, + // it should bubble up the key mismatch error + 'err error:0B080074:x509 certificate routines:X509_check_private_key:key values mismatch' + ]) } tape('setup', function (t) { diff --git a/tests/test-url-parse.js b/tests/test-url-parse.js new file mode 100644 index 000000000..c1edd5e59 --- /dev/null +++ b/tests/test-url-parse.js @@ -0,0 +1,350 @@ +'use strict' + +var url = require('../lib/url-parse') +var tape = require('tape') + +tape('parse - "a=b&c=d"', function (t) { + var str = 'a=b&c=d' + t.deepEqual(url.parse(str), [ + { key: 'a', value: 'b' }, + { key: 'c', value: 'd' } + ]) + t.end() +}) + +tape('parse - "a=обязательный&c=d"', function (t) { + var str = 'a=обязательный&c=d' + t.deepEqual(url.parse(str), [ + { key: 'a', value: 'обязательный' }, + { key: 'c', value: 'd' } + ]) + t.end() +}) + +tape('parse - "a=b&c"', function (t) { + var str = 'a=b&c' + t.deepEqual(url.parse(str), [ + { key: 'a', value: 'b' }, + { key: 'c', value: null } + ]) + t.end() +}) + +tape('parse - "a=b&c="', function (t) { + var str = 'a=b&c=' + t.deepEqual(url.parse(str), [ + { key: 'a', value: 'b' }, + { key: 'c', value: '' } + ]) + t.end() +}) + +tape('parse - "a=b&c=&d=e"', function (t) { + var str = 'a=b&c=&d=e' + t.deepEqual(url.parse(str), [ + { key: 'a', value: 'b' }, + { key: 'c', value: '' }, + { key: 'd', value: 'e' } + ]) + t.end() +}) + +tape('parse - "a=b&c&d=e"', function (t) { + var str = 'a=b&c&d=e' + t.deepEqual(url.parse(str), [ + { key: 'a', value: 'b' }, + { key: 'c', value: null }, + { key: 'd', value: 'e' } + ]) + t.end() +}) + +tape('parse - "a=b&a=c"', function (t) { + var str = 'a=b&a=c' + t.deepEqual(url.parse(str), [ + { key: 'a', value: 'b' }, + { key: 'a', value: 'c' } + ]) + t.end() +}) + +tape('parse - "a=b&a"', function (t) { + var str = 'a=b&a' + t.deepEqual(url.parse(str), [ + { key: 'a', value: 'b' }, + { key: 'a', value: null } + ]) + t.end() +}) + +tape('parse - "a=b&=cd"', function (t) { + var str = 'a=b&=cd' + t.deepEqual(url.parse(str), [ + { key: 'a', value: 'b' }, + { key: '', value: 'cd' } + ]) + t.end() +}) + +tape('parse - "a=b&=&"', function (t) { + var str = 'a=b&=&' + t.deepEqual(url.parse(str), [ + { key: 'a', value: 'b' }, + { key: '', value: '' }, + { key: '', value: null } + ]) + t.end() +}) + +tape('parse - "a=b&&"', function (t) { + var str = 'a=b&&' + t.deepEqual(url.parse(str), [ + { key: 'a', value: 'b' }, + { key: null, value: null }, + { key: '', value: null } + ]) + t.end() +}) + +tape('parse - "a=b&&c=d"', function (t) { + var str = 'a=b&&c=d' + t.deepEqual(url.parse(str), [ + { key: 'a', value: 'b' }, + { key: null, value: null }, + { key: 'c', value: 'd' } + ]) + t.end() +}) + +// Stringification +tape('stringify - "a=b&c=d"', function (t) { + var parsed = [ + { key: 'a', value: 'b' }, + { key: 'c', value: 'd' } + ] + t.equal(url.stringify(parsed), 'a=b&c=d') + t.end() +}) + +tape('stringify - "a=обязательный&c=d"', function (t) { + var parsed = [ + { key: 'a', value: 'обязательный' }, + { key: 'c', value: 'd' } + ] + t.equal(url.stringify(parsed), 'a=%D0%BE%D0%B1%D1%8F%D0%B7%D0%B0%D1%82%D0%B5%D0%BB%D1%8C%D0%BD%D1%8B%D0%B9&c=d') + t.end() +}) + +tape('stringify - "a=b&c"', function (t) { + var parsed = [ + { key: 'a', value: 'b' }, + { key: 'c', value: null } + ] + t.equal(url.stringify(parsed), 'a=b&c') + t.end() +}) + +tape('stringify - "a=b&c="', function (t) { + var parsed = [ + { key: 'a', value: 'b' }, + { key: 'c', value: '' } + ] + + t.equal(url.stringify(parsed), 'a=b&c=') + t.end() +}) + +tape('stringify - "a=b&c=&d=e"', function (t) { + var parsed = [ + { key: 'a', value: 'b' }, + { key: 'c', value: '' }, + { key: 'd', value: 'e' } + ] + + t.equal(url.stringify(parsed), 'a=b&c=&d=e') + t.end() +}) + +tape('stringify - "a=b&c&d=e"', function (t) { + var parsed = [ + { key: 'a', value: 'b' }, + { key: 'c', value: null }, + { key: 'd', value: 'e' } + ] + + t.equal(url.stringify(parsed), 'a=b&c&d=e') + t.end() +}) + +tape('stringify - "a=b&a=c"', function (t) { + var parsed = [ + { key: 'a', value: 'b' }, + { key: 'a', value: 'c' } + ] + + t.equal(url.stringify(parsed), 'a=b&a=c') + t.end() +}) + +tape('stringify - "a=b&a"', function (t) { + var parsed = [ + { key: 'a', value: 'b' }, + { key: 'a', value: null } + ] + t.equal(url.stringify(parsed), 'a=b&a') + t.end() +}) + +tape('stringify - "a=b&=cd"', function (t) { + var parsed = [ + { key: 'a', value: 'b' }, + { key: '', value: 'cd' } + ] + t.equal(url.stringify(parsed), 'a=b&=cd') + t.end() +}) + +tape('stringify - "a=b&=&"', function (t) { + var parsed = [ + { key: 'a', value: 'b' }, + { key: '', value: '' }, + { key: '', value: null } + ] + t.equal(url.stringify(parsed), 'a=b&=&') + t.end() +}) + +tape('stringify - "a=b&&"', function (t) { + var parsed = [ + { key: 'a', value: 'b' }, + { key: null, value: null }, + { key: '', value: null } + ] + t.equal(url.stringify(parsed), 'a=b&&') + t.end() +}) + +tape('stringify - "a=b&&c=d"', function (t) { + var parsed = [ + { key: 'a', value: 'b' }, + { key: null, value: null }, + { key: 'c', value: 'd' } + ] + t.equal(url.stringify(parsed), 'a=b&&c=d') + t.end() +}) + +tape('stringify - "email=foo+bar-xyz@gmail.com"', function (t) { + var parsed = [ + { key: 'email', value: 'foo+bar-xyz@gmail.com' } + ] + t.equal(url.stringify(parsed), 'email=foo+bar-xyz@gmail.com') + t.end() +}) + +tape('stringify pre encoded text( must avoid double encoding) - "email=foo%2Bbar%40domain.com"', function (t) { + var parsed = [ + { key: 'email', value: 'foo%2Bbar%40domain.com' } + ] + t.equal(url.stringify(parsed), 'email=foo%2Bbar%40domain.com') + t.end() +}) + +tape('stringify multibyte character - "multibyte=𝌆"', function (t) { + var parsed = [ + { key: 'multibyte', value: '𝌆' } + ] + t.equal(url.stringify(parsed), 'multibyte=%F0%9D%8C%86') + t.end() +}) + +tape('stringify pre-encoded multibyte character - "multibyte=%F0%9D%8C%86"', function (t) { + var parsed = [ + { key: 'multibyte', value: '%F0%9D%8C%86' } + ] + t.equal(url.stringify(parsed), 'multibyte=%F0%9D%8C%86') + t.end() +}) + +tape('stringify encoding percentage - "charwithPercent=%foo"', function (t) { + var parsed = [ + { key: 'multibyte', value: '%foo' } + ] + t.equal(url.stringify(parsed), 'multibyte=%25foo') + t.end() +}) + +tape('stringify - "a[0]=foo&a[1]=bar"', function (t) { + var parsed = [ + { key: 'a[0]', value: 'foo' }, + { key: 'a[1]', value: 'bar' } + ] + t.equal(url.stringify(parsed), 'a[0]=foo&a[1]=bar') + t.end() +}) + +tape('stringify encodes ( and )- "a=foo(a)"', function (t) { + var parsed = [ + { key: 'a', value: 'foo(a)' } + ] + t.equal(url.stringify(parsed), 'a=foo%28a%29') + t.end() +}) + +tape('stringify Russian - "a=Привет Почтальон"', function (t) { + var parsed = [ + { key: 'a', value: 'Привет Почтальон' } + ] + t.equal(url.stringify(parsed), 'a=%D0%9F%D1%80%D0%B8%D0%B2%D0%B5%D1%82%20%D0%9F%D0%BE%D1%87%D1%82%D0%B0%D0%BB%D1%8C%D0%BE%D0%BD') + t.end() +}) + +tape('stringify Chinese- "a=你好"', function (t) { + var parsed = [ + { key: 'a', value: '你好' } + ] + t.equal(url.stringify(parsed), 'a=%E4%BD%A0%E5%A5%BD') + t.end() +}) + +tape('stringify Japanese- "a=ハローポストマン"', function (t) { + var parsed = [ + { key: 'a', value: 'ハローポストマン' } + ] + t.equal(url.stringify(parsed), 'a=%E3%83%8F%E3%83%AD%E3%83%BC%E3%83%9D%E3%82%B9%E3%83%88%E3%83%9E%E3%83%B3') + t.end() +}) + +tape('stringify Partial Russian - "a=Hello Почтальон"', function (t) { + var parsed = [ + { key: 'a', value: 'Hello Почтальон' } + ] + t.equal(url.stringify(parsed), 'a=Hello%20%D0%9F%D0%BE%D1%87%D1%82%D0%B0%D0%BB%D1%8C%D0%BE%D0%BD') + t.end() +}) + +tape('stringify pre encoded russian text - a=Hello%20%D0%9F%D0%BE%D1%87%D1%82%D0%B0%D0%BB%D1%8C%D0%BE%D0%BD', function (t) { + var parsed = [ + { key: 'a', value: 'Hello%20%D0%9F%D0%BE%D1%87%D1%82%D0%B0%D0%BB%D1%8C%D0%BE%D0%BD' } + ] + t.equal(url.stringify(parsed), 'a=Hello%20%D0%9F%D0%BE%D1%87%D1%82%D0%B0%D0%BB%D1%8C%D0%BE%D0%BD') + t.end() +}) + +tape('url parse - "http://httpbin.org/get?z=b,c"', function (t) { + var parsed = url('http://httpbin.org/get?z=b,c') + + t.equal(parsed.search, '?z=b,c') + t.equal(parsed.query, 'z=b,c') + t.equal(parsed.path, '/get?z=b,c') + t.equal(parsed.href, 'http://httpbin.org/get?z=b,c') + + t.end() +}) + +tape('url parse with invalid encoded parameters - ""', function (t) { + t.doesNotThrow(function () { + url('http://httpbin.org/get?&c=%d') + t.end() + }) +}) diff --git a/tests/test-verbose.js b/tests/test-verbose.js new file mode 100644 index 000000000..b83f600df --- /dev/null +++ b/tests/test-verbose.js @@ -0,0 +1,169 @@ +'use strict' + +var tape = require('tape') +var destroyable = require('server-destroy') + +var server = require('./server') +var request = require('../index') + +var plainServer = server.createServer() +var httpsServer = server.createSSLServer() + +destroyable(plainServer) +destroyable(httpsServer) + +tape('setup', function (t) { + plainServer.listen(0, function () { + plainServer.on('/', function (req, res) { + res.writeHead(200) + res.end('plain') + }) + plainServer.on('/redir', function (req, res) { + res.writeHead(301, { 'location': 'https://localhost:' + httpsServer.port + '/' }) + res.end() + }) + + httpsServer.listen(0, function () { + httpsServer.on('/', function (req, res) { + res.writeHead(200) + res.end('https') + }) + httpsServer.on('/redir', function (req, res) { + res.writeHead(301, { 'location': 'http://localhost:' + plainServer.port + '/' }) + res.end() + }) + + t.end() + }) + }) +}) + +tape('verbose=false [default]', function (t) { + var options = {} + + request('http://localhost:' + plainServer.port + '/', options, function (err, res, body, debug) { + t.equal(err, null) + t.equal(body, 'plain') + t.equal(Array.isArray(debug), true) + t.equal(debug.length, 1) + + t.equal(res.socket.__SESSION_ID, undefined) + t.equal(res.socket.__SESSION_DATA, undefined) + t.deepEqual(Object.keys(debug[0]), ['request', 'response']) + + t.end() + }) +}) + +tape('HTTP: verbose=true', function (t) { + var options = { verbose: true, time: false } // verbose overrides timing setting + + request('http://localhost:' + plainServer.port + '/', options, function (err, res, body, debug) { + t.equal(err, null) + t.equal(body, 'plain') + t.equal(Array.isArray(debug), true) + t.equal(debug.length, 1) + + t.equal(typeof res.socket.__SESSION_ID, 'string') + t.equal(typeof res.socket.__SESSION_DATA, 'object') + t.deepEqual(Object.keys(debug[0]), ['request', 'session', 'response', 'timingStart', 'timingStartTimer', 'timings']) + t.deepEqual(Object.keys(debug[0].session), ['id', 'reused', 'data']) + t.deepEqual(Object.keys(debug[0].session.data), ['addresses']) + t.equal(debug[0].session.reused, false) + + t.end() + }) +}) + +tape('HTTP: redirect(HTTPS) + verbose=true', function (t) { + var options = { + verbose: true, + strictSSL: false + } + + request('http://localhost:' + plainServer.port + '/redir', options, function (err, res, body, debug) { + t.equal(err, null) + t.equal(body, 'https') + t.equal(Array.isArray(debug), true) + t.equal(debug.length, 2) + + t.equal(typeof res.socket.__SESSION_ID, 'string') + t.equal(typeof res.socket.__SESSION_DATA, 'object') + + t.deepEqual(Object.keys(debug[0]), ['request', 'session', 'response', 'timingStart', 'timingStartTimer', 'timings']) + t.deepEqual(Object.keys(debug[0].session), ['id', 'reused', 'data']) + t.deepEqual(Object.keys(debug[0].session.data), ['addresses']) + t.equal(debug[0].session.reused, false) + + t.deepEqual(Object.keys(debug[1]), ['request', 'session', 'response', 'timingStart', 'timingStartTimer', 'timings']) + t.deepEqual(Object.keys(debug[1].session), ['id', 'reused', 'data']) + t.deepEqual(Object.keys(debug[1].session.data), ['addresses', 'tls']) + t.deepEqual(Object.keys(debug[1].session.data.tls), ['reused', 'authorized', 'authorizationError', 'cipher', 'protocol', 'ephemeralKeyInfo', 'peerCertificate']) + t.equal(debug[1].session.reused, false) + + t.end() + }) +}) + +tape('HTTPS: verbose=true', function (t) { + var options = { + verbose: true, + strictSSL: false, + time: false // verbose overrides timing setting + } + + request('https://localhost:' + httpsServer.port + '/', options, function (err, res, body, debug) { + t.equal(err, null) + t.equal(body, 'https') + t.equal(Array.isArray(debug), true) + t.equal(debug.length, 1) + + t.equal(typeof res.socket.__SESSION_ID, 'string') + t.equal(typeof res.socket.__SESSION_DATA, 'object') + t.deepEqual(Object.keys(debug[0]), ['request', 'session', 'response', 'timingStart', 'timingStartTimer', 'timings']) + t.deepEqual(Object.keys(debug[0].session), ['id', 'reused', 'data']) + t.deepEqual(Object.keys(debug[0].session.data), ['addresses', 'tls']) + t.deepEqual(Object.keys(debug[0].session.data.tls), ['reused', 'authorized', 'authorizationError', 'cipher', 'protocol', 'ephemeralKeyInfo', 'peerCertificate']) + t.equal(debug[0].session.reused, false) + + t.end() + }) +}) + +tape('HTTPS: redirect(HTTP) + verbose=true', function (t) { + var options = { + verbose: true, + strictSSL: false + } + + request('https://localhost:' + httpsServer.port + '/redir', options, function (err, res, body, debug) { + t.equal(err, null) + t.equal(body, 'plain') + t.equal(Array.isArray(debug), true) + t.equal(debug.length, 2) + + t.equal(typeof res.socket.__SESSION_ID, 'string') + t.equal(typeof res.socket.__SESSION_DATA, 'object') + + t.deepEqual(Object.keys(debug[0]), ['request', 'session', 'response', 'timingStart', 'timingStartTimer', 'timings']) + t.deepEqual(Object.keys(debug[0].session), ['id', 'reused', 'data']) + t.deepEqual(Object.keys(debug[0].session.data), ['addresses', 'tls']) + t.deepEqual(Object.keys(debug[0].session.data.tls), ['reused', 'authorized', 'authorizationError', 'cipher', 'protocol', 'ephemeralKeyInfo', 'peerCertificate']) + t.equal(debug[0].session.reused, false) + + t.deepEqual(Object.keys(debug[1]), ['request', 'session', 'response', 'timingStart', 'timingStartTimer', 'timings']) + t.deepEqual(Object.keys(debug[1].session), ['id', 'reused', 'data']) + t.deepEqual(Object.keys(debug[1].session.data), ['addresses']) + t.equal(debug[1].session.reused, false) + + t.end() + }) +}) + +tape('cleanup', function (t) { + plainServer.destroy(function () { + httpsServer.destroy(function () { + t.end() + }) + }) +})