diff --git a/packages/server/__snapshots__/request_spec.coffee.js b/packages/server/__snapshots__/request_spec.js similarity index 100% rename from packages/server/__snapshots__/request_spec.coffee.js rename to packages/server/__snapshots__/request_spec.js diff --git a/packages/server/lib/request.coffee b/packages/server/lib/request.coffee deleted file mode 100644 index c900cf6baca1..000000000000 --- a/packages/server/lib/request.coffee +++ /dev/null @@ -1,668 +0,0 @@ -_ = require("lodash") -r = require("@cypress/request") -rp = require("@cypress/request-promise") -url = require("url") -tough = require("tough-cookie") -debug = require("debug")("cypress:server:request") -Promise = require("bluebird") -stream = require("stream") -duplexify = require("duplexify") -agent = require("@packages/network").agent -statusCode = require("./util/status_code") -streamBuffer = require("./util/stream_buffer").streamBuffer - -SERIALIZABLE_COOKIE_PROPS = ['name', 'value', 'domain', 'expiry', 'path', 'secure', 'hostOnly', 'httpOnly', 'sameSite'] -NETWORK_ERRORS = "ECONNREFUSED ECONNRESET EPIPE EHOSTUNREACH EAI_AGAIN ENOTFOUND".split(" ") -VERBOSE_REQUEST_OPTS = "followRedirect strictSSL".split(" ") -HTTP_CLIENT_REQUEST_EVENTS = "abort connect continue information socket timeout upgrade".split(" ") -TLS_VERSION_ERROR_RE = /TLSV1_ALERT_PROTOCOL_VERSION|UNSUPPORTED_PROTOCOL/ -SAMESITE_NONE_RE = /; +samesite=(?:'none'|"none"|none)/i - -process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0" - -convertSameSiteToughToExtension = (sameSite, setCookie) => - ## tough-cookie@4.0.0 uses 'none' as a default, so run this regex to detect if - ## SameSite=None was not explicitly set - ## @see https://github.com/salesforce/tough-cookie/issues/191 - isUnspecified = sameSite is "none" and !SAMESITE_NONE_RE.test(setCookie) - - if isUnspecified - ## not explicitly set, so fall back to the browser's default - return undefined - - if sameSite is 'none' - return 'no_restriction' - - return sameSite - -getOriginalHeaders = (req = {}) -> - ## the request instance holds an instance - ## of the original ClientRequest - ## as the 'req' property which holds the - ## original headers else fall back to - ## the normal req.headers - _.get(req, 'req.headers', req.headers) - -getDelayForRetry = (options = {}) -> - { err, opts, delaysRemaining, retryIntervals, onNext, onElse } = options - - delay = delaysRemaining.shift() - - if not _.isNumber(delay) - ## no more delays, bailing - debug("exhausted all attempts retrying request %o", merge(opts, { err })) - - return onElse() - - ## figure out which attempt we're on... - attempt = retryIntervals.length - delaysRemaining.length - - ## if this ECONNREFUSED and we are - ## retrying greater than 1 second - ## then divide the delay interval - ## by 10 so it doesn't wait as long to retry - ## TODO: do we really want to do this? - if delay >= 1000 and _.get(err, "code") is "ECONNREFUSED" - delay = delay / 10 - - debug("retrying request %o", merge(opts, { - delay, - attempt, - })) - - return onNext(delay, attempt) - -hasRetriableStatusCodeFailure = (res, retryOnStatusCodeFailure) -> - ## everything must be true in order to - ## retry a status code failure - _.every([ - retryOnStatusCodeFailure, - !statusCode.isOk(res.statusCode) - ]) - -isRetriableError = (err = {}, retryOnNetworkFailure) -> - _.every([ - retryOnNetworkFailure, - _.includes(NETWORK_ERRORS, err.code) - ]) - -maybeRetryOnNetworkFailure = (err, options = {}) -> - { - opts, - retryIntervals, - delaysRemaining, - retryOnNetworkFailure, - onNext, - onElse, - } = options - - debug("received an error making http request %o", merge(opts, { err })) - - isTlsVersionError = TLS_VERSION_ERROR_RE.test(err.message) - - if isTlsVersionError - ## because doing every connection via TLSv1 can lead to slowdowns, we set it only on failure - ## https://github.com/cypress-io/cypress/pull/6705 - debug('detected TLS version error, setting min version to TLSv1') - opts.minVersion = 'TLSv1' - - if not isTlsVersionError and not isRetriableError(err, retryOnNetworkFailure) - return onElse() - - ## else see if we have more delays left... - getDelayForRetry({ - err, - opts, - retryIntervals, - delaysRemaining, - onNext, - onElse, - }) - -maybeRetryOnStatusCodeFailure = (res, options = {}) -> - { - err, - opts, - requestId, - retryIntervals, - delaysRemaining, - retryOnStatusCodeFailure, - onNext, - onElse, - } = options - - debug("received status code & headers on request %o", { - requestId, - statusCode: res.statusCode, - headers: _.pick(res.headers, 'content-type', 'set-cookie', 'location') - }) - - ## is this a retryable status code failure? - if not hasRetriableStatusCodeFailure(res, retryOnStatusCodeFailure) - ## if not then we're done here - return onElse() - - ## else see if we have more delays left... - getDelayForRetry({ - err, - opts, - retryIntervals, - delaysRemaining, - onNext, - onElse, - }) - -merge = (args...) -> - _.chain({}) - .extend(args...) - .omit(VERBOSE_REQUEST_OPTS) - .value() - -pick = (resp = {}) -> - req = resp.request ? {} - - headers = getOriginalHeaders(req) - - { - "Request Body": req.body ? null - "Request Headers": headers - "Request URL": req.href - "Response Body": resp.body ? null - "Response Headers": resp.headers - "Response Status": resp.statusCode - } - -createRetryingRequestPromise = (opts) -> - { - requestId, - retryIntervals, - delaysRemaining, - retryOnNetworkFailure, - retryOnStatusCodeFailure - } = opts - - retry = (delay) -> - return Promise.delay(delay) - .then -> - createRetryingRequestPromise(opts) - - return rp(opts) - .catch (err) -> - - ## rp wraps network errors in a RequestError, so might need to unwrap it to check - maybeRetryOnNetworkFailure(err.error or err, { - opts, - retryIntervals, - delaysRemaining, - retryOnNetworkFailure, - onNext: retry - onElse: -> - throw err - }) - .then (res) -> - ## ok, no net error, but what about a bad status code? - maybeRetryOnStatusCodeFailure(res, { - opts, - requestId, - retryIntervals, - delaysRemaining, - retryOnStatusCodeFailure, - onNext: retry - onElse: _.constant(res) - }) - -pipeEvent = (source, destination, event) -> - source.on event, (args...) -> - destination.emit(event, args...) - -createRetryingRequestStream = (opts = {}) -> - { - requestId, - retryIntervals, - delaysRemaining, - retryOnNetworkFailure, - retryOnStatusCodeFailure - } = opts - - req = null - - delayStream = stream.PassThrough() - reqBodyBuffer = streamBuffer() - retryStream = duplexify(reqBodyBuffer, delayStream) - - cleanup = -> - if reqBodyBuffer - ## null req body out to free memory - reqBodyBuffer.unpipeAll() - reqBodyBuffer = null - - emitError = (err) -> - retryStream.emit("error", err) - - cleanup() - - tryStartStream = -> - ## if our request has been aborted - ## in the time that we were waiting to retry - ## then immediately bail - if retryStream.aborted - return - - reqStream = r(opts) - didReceiveResponse = false - - retry = (delay, attempt) -> - retryStream.emit("retry", { attempt, delay }) - - setTimeout(tryStartStream, delay) - - ## if we're retrying and we previous piped - ## into the reqStream, then reapply this now - if req - reqStream.emit('pipe', req) - reqBodyBuffer.createReadStream().pipe(reqStream) - - ## forward the abort call to the underlying request - retryStream.abort = -> - debug('aborting', { requestId }) - retryStream.aborted = true - - reqStream.abort() - - onPiped = (src) -> - ## store this IncomingMessage so we can reapply it - ## if we need to retry - req = src - - ## https://github.com/request/request/blob/b3a218dc7b5689ce25be171e047f0d4f0eef8919/request.js#L493 - ## the request lib expects this 'pipe' event in - ## order to copy the request headers onto the - ## outgoing message - so we manually pipe it here - src.pipe(reqStream) - - ## when this passthrough stream is being piped into - ## then make sure we properly "forward" and connect - ## forward it to the real reqStream which enables - ## request to read off the IncomingMessage readable stream - retryStream.once("pipe", onPiped) - - reqStream.on "error", (err) -> - if didReceiveResponse - ## if we've already begun processing the requests - ## response, then that means we failed during transit - ## and its no longer safe to retry. all we can do now - ## is propogate the error upwards - debug("received an error on request after response started %o", merge(opts, { err })) - - return emitError(err) - - ## otherwise, see if we can retry another request under the hood... - maybeRetryOnNetworkFailure(err, { - opts, - retryIntervals, - delaysRemaining, - retryOnNetworkFailure, - onNext: retry - onElse: -> - emitError(err) - }) - - reqStream.once "request", (req) -> - ## remove the pipe listener since once the request has - ## been made, we cannot pipe into the reqStream anymore - retryStream.removeListener("pipe", onPiped) - - reqStream.once "response", (incomingRes) -> - didReceiveResponse = true - - ## ok, no net error, but what about a bad status code? - maybeRetryOnStatusCodeFailure(incomingRes, { - opts, - requestId, - delaysRemaining, - retryIntervals, - retryOnStatusCodeFailure, - onNext: retry - onElse: -> - debug("successful response received", { requestId }) - - cleanup() - - ## forward the response event upwards which should happen - ## prior to the pipe event, same as what request does - ## https://github.com/request/request/blob/master/request.js#L1059 - retryStream.emit("response", incomingRes) - - reqStream.pipe(delayStream) - - ## `http.ClientRequest` events - _.map(HTTP_CLIENT_REQUEST_EVENTS, _.partial(pipeEvent, reqStream, retryStream)) - }) - - tryStartStream() - - return retryStream - -caseInsensitiveGet = (obj, property) -> - lowercaseProperty = property.toLowerCase() - - for key in Object.keys(obj) - if key.toLowerCase() == lowercaseProperty - return obj[key] - -## first, attempt to set on an existing property with differing case -## if that fails, set the lowercase `property` -caseInsensitiveSet = (obj, property, val) -> - lowercaseProperty = property.toLowerCase() - - for key in Object.keys(obj) - if key.toLowerCase() == lowercaseProperty - return obj[key] = val - - obj[lowercaseProperty] = val - -setDefaults = (opts) -> - _ - .chain(opts) - .defaults({ - requestId: _.uniqueId('request') - retryIntervals: [0, 1000, 2000, 2000] - retryOnNetworkFailure: true - retryOnStatusCodeFailure: false - }) - .thru (opts) -> - _.defaults(opts, { - delaysRemaining: _.clone(opts.retryIntervals) - }) - .value() - -module.exports = (options = {}) -> - defaults = { - timeout: options.timeout - agent: agent - ## send keep-alive with requests since Chrome won't send it in proxy mode - ## https://github.com/cypress-io/cypress/pull/3531#issuecomment-476269041 - headers: { - "Connection": "keep-alive" - } - proxy: null ## upstream proxying is handled by CombinedAgent - } - - r = r.defaults(defaults) - rp = rp.defaults(defaults) - - return { - r: require("@cypress/request") - - rp: require("@cypress/request-promise") - - getDelayForRetry - - setDefaults - - create: (strOrOpts, promise) -> - switch - when _.isString(strOrOpts) - opts = { - url: strOrOpts - } - else - opts = strOrOpts - - opts = setDefaults(opts) - - if promise - createRetryingRequestPromise(opts) - else - createRetryingRequestStream(opts) - - contentTypeIsJson: (response) -> - ## TODO: use https://github.com/jshttp/type-is for this - ## https://github.com/cypress-io/cypress/pull/5166 - response?.headers?["content-type"]?.split(';', 2)[0].endsWith("json") - - parseJsonBody: (body) -> - try - JSON.parse(body) - catch e - body - - normalizeResponse: (push, response) -> - req = response.request ? {} - - push(response) - - response = _.pick(response, "statusCode", "body", "headers") - - ## normalize status - response.status = response.statusCode - delete response.statusCode - - _.extend(response, { - ## normalize what is an ok status code - statusText: statusCode.getText(response.status) - isOkStatusCode: statusCode.isOk(response.status) - requestHeaders: getOriginalHeaders(req) - requestBody: req.body - }) - - ## if body is a string and content type is json - ## try to convert the body to JSON - if _.isString(response.body) and @contentTypeIsJson(response) - response.body = @parseJsonBody(response.body) - - return response - - setRequestCookieHeader: (req, reqUrl, automationFn, existingHeader) -> - automationFn('get:cookies', { url: reqUrl }) - .then (cookies) -> - debug('got cookies from browser %o', { reqUrl, cookies }) - header = cookies.map (cookie) -> - "#{cookie.name}=#{cookie.value}" - .join("; ") || undefined - - if header - if existingHeader - ## existingHeader = whatever Cookie header the user is already trying to set - debug('there is an existing cookie header, merging %o', { header, existingHeader }) - ## order does not not matter here - ## @see https://tools.ietf.org/html/rfc6265#section-4.2.2 - header = [existingHeader, header].join(';') - - caseInsensitiveSet(req.headers, 'Cookie', header) - - setCookiesOnBrowser: (res, resUrl, automationFn) -> - cookies = res.headers['set-cookie'] - if !cookies - return Promise.resolve() - - if !(cookies instanceof Array) - cookies = [cookies] - - parsedUrl = url.parse(resUrl) - defaultDomain = parsedUrl.hostname - - debug('setting cookies on browser %o', { url: parsedUrl.href, defaultDomain, cookies }) - - Promise.map cookies, (cyCookie) -> - cookie = tough.Cookie.parse(cyCookie, { loose: true }) - - debug('parsing cookie %o', { cyCookie, toughCookie: cookie }) - - if not cookie - ## ignore invalid cookies (same as browser behavior) - ## https://github.com/cypress-io/cypress/issues/6890 - debug('tough-cookie failed to parse, ignoring') - return - - cookie.name = cookie.key - - if not cookie.domain - ## take the domain from the URL - cookie.domain = defaultDomain - cookie.hostOnly = true - - if not tough.domainMatch(defaultDomain, cookie.domain) - debug('domain match failed:', { defaultDomain }) - return - - expiry = cookie.expiryTime() - if isFinite(expiry) - cookie.expiry = expiry / 1000 - - cookie.sameSite = convertSameSiteToughToExtension(cookie.sameSite, cyCookie) - - cookie = _.pick(cookie, SERIALIZABLE_COOKIE_PROPS) - - automationCmd = 'set:cookie' - - if expiry <= 0 - automationCmd = 'clear:cookie' - - automationFn(automationCmd, cookie) - .catch (err) -> - debug('automation threw an error during cookie change %o', { automationCmd, cyCookie, cookie, err }) - - sendStream: (headers, automationFn, options = {}) -> - _.defaults options, { - headers: {} - onBeforeReqInit: (fn) -> fn() - } - - if not caseInsensitiveGet(options.headers, "user-agent") and (ua = headers["user-agent"]) - options.headers["user-agent"] = ua - - _.extend options, { - strictSSL: false - } - - self = @ - - followRedirect = options.followRedirect - - currentUrl = options.url - - options.followRedirect = (incomingRes) -> - if followRedirect and not followRedirect(incomingRes) - return false - - newUrl = url.resolve(currentUrl, incomingRes.headers.location) - - ## and when we know we should follow the redirect - ## we need to override the init method and - ## first set the received cookies on the browser - ## and then grab the cookies for the new url - self.setCookiesOnBrowser(incomingRes, currentUrl, automationFn) - .then (cookies) => - self.setRequestCookieHeader(@, newUrl, automationFn) - .then => - currentUrl = newUrl - true - - @setRequestCookieHeader(options, options.url, automationFn, caseInsensitiveGet(options.headers, 'cookie')) - .then => - return => - debug("sending request as stream %o", merge(options)) - - @create(options) - - sendPromise: (headers, automationFn, options = {}) -> - _.defaults options, { - headers: {} - gzip: true - cookies: true - followRedirect: true - } - - if not caseInsensitiveGet(options.headers, "user-agent") and (ua = headers["user-agent"]) - options.headers["user-agent"] = ua - - ## normalize case sensitivity - ## to be lowercase - if a = options.headers.Accept - delete options.headers.Accept - options.headers.accept = a - - ## https://github.com/cypress-io/cypress/issues/338 - _.defaults(options.headers, { - accept: "*/*" - }) - - _.extend(options, { - strictSSL: false - simple: false - resolveWithFullResponse: true - }) - - ## https://github.com/cypress-io/cypress/issues/322 - ## either turn these both on or off - options.followAllRedirects = options.followRedirect - - if options.form is true - ## reset form to whatever body is - ## and nuke body - options.form = options.body - delete options.json - delete options.body - - self = @ - - send = => - ms = Date.now() - - redirects = [] - requestResponses = [] - - push = (response) -> - requestResponses.push(pick(response)) - - currentUrl = options.url - - if options.followRedirect - options.followRedirect = (incomingRes) -> - newUrl = url.resolve(currentUrl, incomingRes.headers.location) - - ## normalize the url - redirects.push([incomingRes.statusCode, newUrl].join(": ")) - - push(incomingRes) - - ## and when we know we should follow the redirect - ## we need to override the init method and - ## first set the new cookies on the browser - ## and then grab the cookies for the new url - self.setCookiesOnBrowser(incomingRes, currentUrl, automationFn) - .then => - self.setRequestCookieHeader(@, newUrl, automationFn) - .then => - currentUrl = newUrl - true - - @create(options, true) - .then(@normalizeResponse.bind(@, push)) - .then (resp) => - ## TODO: move duration somewhere...? - ## does node store this somewhere? - ## we could probably calculate this ourselves - ## by using the date headers - resp.duration = Date.now() - ms - resp.allRequestResponses = requestResponses - - if redirects.length - resp.redirects = redirects - - if options.followRedirect is false and (loc = resp.headers.location) - ## resolve the new location head against - ## the current url - resp.redirectedToUrl = url.resolve(options.url, loc) - - @setCookiesOnBrowser(resp, currentUrl, automationFn) - .return(resp) - - if c = options.cookies - self.setRequestCookieHeader(options, options.url, automationFn, caseInsensitiveGet(options.headers, 'cookie')) - .then(send) - else - send() - - } diff --git a/packages/server/lib/request.js b/packages/server/lib/request.js new file mode 100644 index 000000000000..53751ec5b06d --- /dev/null +++ b/packages/server/lib/request.js @@ -0,0 +1,779 @@ +const _ = require('lodash') +let r = require('@cypress/request') +let rp = require('@cypress/request-promise') +const url = require('url') +const tough = require('tough-cookie') +const debug = require('debug')('cypress:server:request') +const Promise = require('bluebird') +const stream = require('stream') +const duplexify = require('duplexify') +const { agent } = require('@packages/network') +const statusCode = require('./util/status_code') +const { streamBuffer } = require('./util/stream_buffer') + +const SERIALIZABLE_COOKIE_PROPS = ['name', 'value', 'domain', 'expiry', 'path', 'secure', 'hostOnly', 'httpOnly', 'sameSite'] +const NETWORK_ERRORS = 'ECONNREFUSED ECONNRESET EPIPE EHOSTUNREACH EAI_AGAIN ENOTFOUND'.split(' ') +const VERBOSE_REQUEST_OPTS = 'followRedirect strictSSL'.split(' ') +const HTTP_CLIENT_REQUEST_EVENTS = 'abort connect continue information socket timeout upgrade'.split(' ') +const TLS_VERSION_ERROR_RE = /TLSV1_ALERT_PROTOCOL_VERSION|UNSUPPORTED_PROTOCOL/ +const SAMESITE_NONE_RE = /; +samesite=(?:'none'|"none"|none)/i + +process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0' + +const convertSameSiteToughToExtension = (sameSite, setCookie) => { + // tough-cookie@4.0.0 uses 'none' as a default, so run this regex to detect if + // SameSite=None was not explicitly set + // @see https://github.com/salesforce/tough-cookie/issues/191 + const isUnspecified = (sameSite === 'none') && !SAMESITE_NONE_RE.test(setCookie) + + if (isUnspecified) { + // not explicitly set, so fall back to the browser's default + return undefined + } + + if (sameSite === 'none') { + return 'no_restriction' + } + + return sameSite +} + +const getOriginalHeaders = (req = {}) => { + // the request instance holds an instance + // of the original ClientRequest + // as the 'req' property which holds the + // original headers else fall back to + // the normal req.headers + return _.get(req, 'req.headers', req.headers) +} + +const getDelayForRetry = function (options = {}) { + const { err, opts, delaysRemaining, retryIntervals, onNext, onElse } = options + + let delay = delaysRemaining.shift() + + if (!_.isNumber(delay)) { + // no more delays, bailing + debug('exhausted all attempts retrying request %o', merge(opts, { err })) + + return onElse() + } + + // figure out which attempt we're on... + const attempt = retryIntervals.length - delaysRemaining.length + + // if this ECONNREFUSED and we are + // retrying greater than 1 second + // then divide the delay interval + // by 10 so it doesn't wait as long to retry + // TODO: do we really want to do this? + if ((delay >= 1000) && (_.get(err, 'code') === 'ECONNREFUSED')) { + delay = delay / 10 + } + + debug('retrying request %o', merge(opts, { + delay, + attempt, + })) + + return onNext(delay, attempt) +} + +const hasRetriableStatusCodeFailure = (res, retryOnStatusCodeFailure) => { + // everything must be true in order to + // retry a status code failure + return _.every([ + retryOnStatusCodeFailure, + !statusCode.isOk(res.statusCode), + ]) +} + +const isRetriableError = (err = {}, retryOnNetworkFailure) => { + return _.every([ + retryOnNetworkFailure, + _.includes(NETWORK_ERRORS, err.code), + ]) +} + +const maybeRetryOnNetworkFailure = function (err, options = {}) { + const { + opts, + retryIntervals, + delaysRemaining, + retryOnNetworkFailure, + onNext, + onElse, + } = options + + debug('received an error making http request %o', merge(opts, { err })) + + const isTlsVersionError = TLS_VERSION_ERROR_RE.test(err.message) + + if (isTlsVersionError) { + // because doing every connection via TLSv1 can lead to slowdowns, we set it only on failure + // https://github.com/cypress-io/cypress/pull/6705 + debug('detected TLS version error, setting min version to TLSv1') + opts.minVersion = 'TLSv1' + } + + if (!isTlsVersionError && !isRetriableError(err, retryOnNetworkFailure)) { + return onElse() + } + + // else see if we have more delays left... + return getDelayForRetry({ + err, + opts, + retryIntervals, + delaysRemaining, + onNext, + onElse, + }) +} + +const maybeRetryOnStatusCodeFailure = function (res, options = {}) { + const { + err, + opts, + requestId, + retryIntervals, + delaysRemaining, + retryOnStatusCodeFailure, + onNext, + onElse, + } = options + + debug('received status code & headers on request %o', { + requestId, + statusCode: res.statusCode, + headers: _.pick(res.headers, 'content-type', 'set-cookie', 'location'), + }) + + // is this a retryable status code failure? + if (!hasRetriableStatusCodeFailure(res, retryOnStatusCodeFailure)) { + // if not then we're done here + return onElse() + } + + // else see if we have more delays left... + return getDelayForRetry({ + err, + opts, + retryIntervals, + delaysRemaining, + onNext, + onElse, + }) +} + +const merge = (...args) => { + return _.chain({}) + .extend(...args) + .omit(VERBOSE_REQUEST_OPTS) + .value() +} + +const pick = function (resp = {}) { + const req = resp.request != null ? resp.request : {} + + const headers = getOriginalHeaders(req) + + return { + 'Request Body': req.body != null ? req.body : null, + 'Request Headers': headers, + 'Request URL': req.href, + 'Response Body': resp.body != null ? resp.body : null, + 'Response Headers': resp.headers, + 'Response Status': resp.statusCode, + } +} + +const createRetryingRequestPromise = function (opts) { + const { + requestId, + retryIntervals, + delaysRemaining, + retryOnNetworkFailure, + retryOnStatusCodeFailure, + } = opts + + const retry = (delay) => { + return Promise.delay(delay) + .then(() => { + return createRetryingRequestPromise(opts) + }) + } + + return rp(opts) + .catch((err) => { + // rp wraps network errors in a RequestError, so might need to unwrap it to check + return maybeRetryOnNetworkFailure(err.error || err, { + opts, + retryIntervals, + delaysRemaining, + retryOnNetworkFailure, + onNext: retry, + onElse () { + throw err + }, + }) + }).then((res) => { + // ok, no net error, but what about a bad status code? + return maybeRetryOnStatusCodeFailure(res, { + opts, + requestId, + retryIntervals, + delaysRemaining, + retryOnStatusCodeFailure, + onNext: retry, + onElse: _.constant(res), + }) + }) +} + +const pipeEvent = (source, destination, event) => { + return source.on(event, (...args) => { + destination.emit(event, ...args) + }) +} + +const createRetryingRequestStream = function (opts = {}) { + const { + requestId, + retryIntervals, + delaysRemaining, + retryOnNetworkFailure, + retryOnStatusCodeFailure, + } = opts + + let req = null + + const delayStream = stream.PassThrough() + let reqBodyBuffer = streamBuffer() + const retryStream = duplexify(reqBodyBuffer, delayStream) + + const cleanup = function () { + if (reqBodyBuffer) { + // null req body out to free memory + reqBodyBuffer.unpipeAll() + reqBodyBuffer = null + } + } + + const emitError = function (err) { + retryStream.emit('error', err) + + cleanup() + } + + const tryStartStream = function () { + // if our request has been aborted + // in the time that we were waiting to retry + // then immediately bail + if (retryStream.aborted) { + return + } + + const reqStream = r(opts) + let didReceiveResponse = false + + const retry = function (delay, attempt) { + retryStream.emit('retry', { attempt, delay }) + + return setTimeout(tryStartStream, delay) + } + + // if we're retrying and we previous piped + // into the reqStream, then reapply this now + if (req) { + reqStream.emit('pipe', req) + reqBodyBuffer.createReadStream().pipe(reqStream) + } + + // forward the abort call to the underlying request + retryStream.abort = function () { + debug('aborting', { requestId }) + retryStream.aborted = true + + reqStream.abort() + } + + const onPiped = function (src) { + // store this IncomingMessage so we can reapply it + // if we need to retry + req = src + + // https://github.com/request/request/blob/b3a218dc7b5689ce25be171e047f0d4f0eef8919/request.js#L493 + // the request lib expects this 'pipe' event in + // order to copy the request headers onto the + // outgoing message - so we manually pipe it here + src.pipe(reqStream) + } + + // when this passthrough stream is being piped into + // then make sure we properly "forward" and connect + // forward it to the real reqStream which enables + // request to read off the IncomingMessage readable stream + retryStream.once('pipe', onPiped) + + reqStream.on('error', (err) => { + if (didReceiveResponse) { + // if we've already begun processing the requests + // response, then that means we failed during transit + // and its no longer safe to retry. all we can do now + // is propogate the error upwards + debug('received an error on request after response started %o', merge(opts, { err })) + + return emitError(err) + } + + // otherwise, see if we can retry another request under the hood... + return maybeRetryOnNetworkFailure(err, { + opts, + retryIntervals, + delaysRemaining, + retryOnNetworkFailure, + onNext: retry, + onElse () { + return emitError(err) + }, + }) + }) + + reqStream.once('request', (req) => { + // remove the pipe listener since once the request has + // been made, we cannot pipe into the reqStream anymore + retryStream.removeListener('pipe', onPiped) + }) + + reqStream.once('response', (incomingRes) => { + didReceiveResponse = true + + // ok, no net error, but what about a bad status code? + return maybeRetryOnStatusCodeFailure(incomingRes, { + opts, + requestId, + delaysRemaining, + retryIntervals, + retryOnStatusCodeFailure, + onNext: retry, + onElse () { + debug('successful response received', { requestId }) + + cleanup() + + // forward the response event upwards which should happen + // prior to the pipe event, same as what request does + // https://github.com/request/request/blob/master/request.js#L1059 + retryStream.emit('response', incomingRes) + + reqStream.pipe(delayStream) + + // `http.ClientRequest` events + return _.map(HTTP_CLIENT_REQUEST_EVENTS, _.partial(pipeEvent, reqStream, retryStream)) + }, + }) + }) + } + + tryStartStream() + + return retryStream +} + +const caseInsensitiveGet = function (obj, property) { + const lowercaseProperty = property.toLowerCase() + + for (let key of Object.keys(obj)) { + if (key.toLowerCase() === lowercaseProperty) { + return obj[key] + } + } +} + +// first, attempt to set on an existing property with differing case +// if that fails, set the lowercase `property` +const caseInsensitiveSet = function (obj, property, val) { + const lowercaseProperty = property.toLowerCase() + + for (let key of Object.keys(obj)) { + if (key.toLowerCase() === lowercaseProperty) { + obj[key] = val + } + } + + obj[lowercaseProperty] = val +} + +const setDefaults = (opts) => { + return _ + .chain(opts) + .defaults({ + requestId: _.uniqueId('request'), + retryIntervals: [0, 1000, 2000, 2000], + retryOnNetworkFailure: true, + retryOnStatusCodeFailure: false, + }) + .thru((opts) => { + return _.defaults(opts, { + delaysRemaining: _.clone(opts.retryIntervals), + }) + }).value() +} + +module.exports = function (options = {}) { + const defaults = { + timeout: options.timeout, + agent, + // send keep-alive with requests since Chrome won't send it in proxy mode + // https://github.com/cypress-io/cypress/pull/3531#issuecomment-476269041 + headers: { + 'Connection': 'keep-alive', + }, + proxy: null, // upstream proxying is handled by CombinedAgent + } + + r = r.defaults(defaults) + rp = rp.defaults(defaults) + + return { + r: require('@cypress/request'), + + rp: require('@cypress/request-promise'), + + getDelayForRetry, + + setDefaults, + + create (strOrOpts, promise) { + let opts + + if (_.isString(strOrOpts)) { + opts = { + url: strOrOpts, + } + } else { + opts = strOrOpts + } + + opts = setDefaults(opts) + + if (promise) { + return createRetryingRequestPromise(opts) + } + + return createRetryingRequestStream(opts) + }, + + contentTypeIsJson (response) { + // TODO: use https://github.com/jshttp/type-is for this + // https://github.com/cypress-io/cypress/pull/5166 + if (response && response.headers && response.headers['content-type']) { + return response.headers['content-type'].split(';', 2)[0].endsWith('json') + } + }, + + parseJsonBody (body) { + try { + return JSON.parse(body) + } catch (e) { + return body + } + }, + + normalizeResponse (push, response) { + const req = response.request != null ? response.request : {} + + push(response) + + response = _.pick(response, 'statusCode', 'body', 'headers') + + // normalize status + response.status = response.statusCode + delete response.statusCode + + _.extend(response, { + // normalize what is an ok status code + statusText: statusCode.getText(response.status), + isOkStatusCode: statusCode.isOk(response.status), + requestHeaders: getOriginalHeaders(req), + requestBody: req.body, + }) + + // if body is a string and content type is json + // try to convert the body to JSON + if (_.isString(response.body) && this.contentTypeIsJson(response)) { + response.body = this.parseJsonBody(response.body) + } + + return response + }, + + setRequestCookieHeader (req, reqUrl, automationFn, existingHeader) { + return automationFn('get:cookies', { url: reqUrl }) + .then((cookies) => { + debug('got cookies from browser %o', { reqUrl, cookies }) + let header = cookies.map((cookie) => { + return `${cookie.name}=${cookie.value}` + }).join('; ') || undefined + + if (header) { + if (existingHeader) { + // existingHeader = whatever Cookie header the user is already trying to set + debug('there is an existing cookie header, merging %o', { header, existingHeader }) + // order does not not matter here + // @see https://tools.ietf.org/html/rfc6265#section-4.2.2 + header = [existingHeader, header].join(';') + } + + return caseInsensitiveSet(req.headers, 'Cookie', header) + } + }) + }, + + setCookiesOnBrowser (res, resUrl, automationFn) { + let cookies = res.headers['set-cookie'] + + if (!cookies) { + return Promise.resolve() + } + + if (!(cookies instanceof Array)) { + cookies = [cookies] + } + + const parsedUrl = url.parse(resUrl) + const defaultDomain = parsedUrl.hostname + + debug('setting cookies on browser %o', { url: parsedUrl.href, defaultDomain, cookies }) + + return Promise.map(cookies, (cyCookie) => { + let cookie = tough.Cookie.parse(cyCookie, { loose: true }) + + debug('parsing cookie %o', { cyCookie, toughCookie: cookie }) + + if (!cookie) { + // ignore invalid cookies (same as browser behavior) + // https://github.com/cypress-io/cypress/issues/6890 + debug('tough-cookie failed to parse, ignoring') + + return + } + + cookie.name = cookie.key + + if (!cookie.domain) { + // take the domain from the URL + cookie.domain = defaultDomain + cookie.hostOnly = true + } + + if (!tough.domainMatch(defaultDomain, cookie.domain)) { + debug('domain match failed:', { defaultDomain }) + + return + } + + const expiry = cookie.expiryTime() + + if (isFinite(expiry)) { + cookie.expiry = expiry / 1000 + } + + cookie.sameSite = convertSameSiteToughToExtension(cookie.sameSite, cyCookie) + + cookie = _.pick(cookie, SERIALIZABLE_COOKIE_PROPS) + + let automationCmd = 'set:cookie' + + if (expiry <= 0) { + automationCmd = 'clear:cookie' + } + + return automationFn(automationCmd, cookie) + .catch((err) => { + return debug('automation threw an error during cookie change %o', { automationCmd, cyCookie, cookie, err }) + }) + }) + }, + + sendStream (headers, automationFn, options = {}) { + let ua + + _.defaults(options, { + headers: {}, + onBeforeReqInit (fn) { + return fn() + }, + }) + + if (!caseInsensitiveGet(options.headers, 'user-agent') && (ua = headers['user-agent'])) { + options.headers['user-agent'] = ua + } + + _.extend(options, { + strictSSL: false, + }) + + const self = this + + const { + followRedirect, + } = options + + let currentUrl = options.url + + options.followRedirect = function (incomingRes) { + if (followRedirect && !followRedirect(incomingRes)) { + return false + } + + const newUrl = url.resolve(currentUrl, incomingRes.headers.location) + + // and when we know we should follow the redirect + // we need to override the init method and + // first set the received cookies on the browser + // and then grab the cookies for the new url + return self.setCookiesOnBrowser(incomingRes, currentUrl, automationFn) + .then(() => { + return self.setRequestCookieHeader(this, newUrl, automationFn) + }).then(() => { + currentUrl = newUrl + + return true + }) + } + + return this.setRequestCookieHeader(options, options.url, automationFn, caseInsensitiveGet(options.headers, 'cookie')) + .then(() => { + return () => { + debug('sending request as stream %o', merge(options)) + + return this.create(options) + } + }) + }, + + sendPromise (headers, automationFn, options = {}) { + let a; let c; let ua + + _.defaults(options, { + headers: {}, + gzip: true, + cookies: true, + followRedirect: true, + }) + + if (!caseInsensitiveGet(options.headers, 'user-agent') && (ua = headers['user-agent'])) { + options.headers['user-agent'] = ua + } + + // normalize case sensitivity + // to be lowercase + a = options.headers.Accept + + if (a) { + delete options.headers.Accept + options.headers.accept = a + } + + // https://github.com/cypress-io/cypress/issues/338 + _.defaults(options.headers, { + accept: '*/*', + }) + + _.extend(options, { + strictSSL: false, + simple: false, + resolveWithFullResponse: true, + }) + + // https://github.com/cypress-io/cypress/issues/322 + // either turn these both on or off + options.followAllRedirects = options.followRedirect + + if (options.form === true) { + // reset form to whatever body is + // and nuke body + options.form = options.body + delete options.json + delete options.body + } + + const self = this + + const send = () => { + const ms = Date.now() + + const redirects = [] + const requestResponses = [] + + const push = (response) => { + return requestResponses.push(pick(response)) + } + + let currentUrl = options.url + + if (options.followRedirect) { + options.followRedirect = function (incomingRes) { + const newUrl = url.resolve(currentUrl, incomingRes.headers.location) + + // normalize the url + redirects.push([incomingRes.statusCode, newUrl].join(': ')) + + push(incomingRes) + + // and when we know we should follow the redirect + // we need to override the init method and + // first set the new cookies on the browser + // and then grab the cookies for the new url + return self.setCookiesOnBrowser(incomingRes, currentUrl, automationFn) + .then(() => { + return self.setRequestCookieHeader(this, newUrl, automationFn) + }).then(() => { + currentUrl = newUrl + + return true + }) + } + } + + return this.create(options, true) + .then(this.normalizeResponse.bind(this, push)) + .then((resp) => { + // TODO: move duration somewhere...? + // does node store this somewhere? + // we could probably calculate this ourselves + // by using the date headers + let loc + + resp.duration = Date.now() - ms + resp.allRequestResponses = requestResponses + + if (redirects.length) { + resp.redirects = redirects + } + + if ((options.followRedirect === false) && (loc = resp.headers.location)) { + // resolve the new location head against + // the current url + resp.redirectedToUrl = url.resolve(options.url, loc) + } + + return this.setCookiesOnBrowser(resp, currentUrl, automationFn) + .return(resp) + }) + } + + c = options.cookies + + if (c) { + return self.setRequestCookieHeader(options, options.url, automationFn, caseInsensitiveGet(options.headers, 'cookie')) + .then(send) + } + + return send() + }, + + } +} diff --git a/packages/server/test/unit/request_spec.coffee b/packages/server/test/unit/request_spec.coffee deleted file mode 100644 index 756829e8490c..000000000000 --- a/packages/server/test/unit/request_spec.coffee +++ /dev/null @@ -1,887 +0,0 @@ -require("../spec_helper") - -_ = require("lodash") -http = require("http") -Bluebird = require("bluebird") -Request = require("#{root}lib/request") -snapshot = require("snap-shot-it") - -request = Request({timeout: 100}) - -testAttachingCookiesWith = (fn) -> - set = sinon.spy(request, 'setCookiesOnBrowser') - get = sinon.spy(request, 'setRequestCookieHeader') - - nock("http://localhost:1234") - .get("/") - .reply(302, "", { - 'set-cookie': 'one=1' - location: "/second" - }) - .get("/second") - .reply(302, "", { - 'set-cookie': 'two=2' - location: "/third" - }) - .get("/third") - .reply(200, "", { - 'set-cookie': 'three=3' - }) - - fn() - .then -> - snapshot({ - setCalls: set.getCalls().map (call) -> - { - currentUrl: call.args[1], - setCookie: call.args[0].headers['set-cookie'] - } - getCalls: get.getCalls().map (call) -> - { - newUrl: _.get(call, 'args.1') - } - }) - -describe "lib/request", -> - beforeEach -> - @fn = sinon.stub() - @fn.withArgs('set:cookie').resolves({}) - @fn.withArgs('get:cookies').resolves([]) - - it "is defined", -> - expect(request).to.be.an("object") - - context "#getDelayForRetry", -> - it "divides by 10 when delay >= 1000 and err.code = ECONNREFUSED", -> - retryIntervals = [1,2,3,4] - delaysRemaining = [0, 999, 1000, 2000] - - err = { - code: "ECONNREFUSED" - } - - onNext = sinon.stub() - - retryIntervals.forEach -> - request.getDelayForRetry({ - err, - onNext, - retryIntervals, - delaysRemaining, - }) - - expect(delaysRemaining).to.be.empty - expect(onNext.args).to.deep.eq([ - [0, 1] - [999, 2] - [100, 3] - [200, 4] - ]) - - it "does not divide by 10 when err.code != ECONNREFUSED", -> - retryIntervals = [1,2,3,4] - delaysRemaining = [2000, 2000, 2000, 2000] - - err = { - code: "ECONNRESET" - } - - onNext = sinon.stub() - - request.getDelayForRetry({ - err, - onNext, - retryIntervals, - delaysRemaining, - }) - - expect(delaysRemaining).to.have.length(3) - expect(onNext).to.be.calledWith(2000, 1) - - it "calls onElse when delaysRemaining is exhausted", -> - retryIntervals = [1,2,3,4] - delaysRemaining = [] - - onNext = sinon.stub() - onElse = sinon.stub() - - request.getDelayForRetry({ - onElse - onNext, - retryIntervals, - delaysRemaining, - }) - - expect(onElse).to.be.calledWithExactly() - expect(onNext).not.to.be.called - - context "#setDefaults", -> - it "delaysRemaining to retryIntervals clone", -> - retryIntervals = [1,2,3,4] - - opts = request.setDefaults({ retryIntervals }) - - expect(opts.retryIntervals).to.eq(retryIntervals) - expect(opts.delaysRemaining).not.to.eq(retryIntervals) - expect(opts.delaysRemaining).to.deep.eq(retryIntervals) - - it "retryIntervals to [0, 1000, 2000, 2000] by default", -> - opts = request.setDefaults({}) - - expect(opts.retryIntervals).to.deep.eq([0, 1000, 2000, 2000]) - - it "delaysRemaining can be overridden", -> - delaysRemaining = [1] - opts = request.setDefaults({ delaysRemaining }) - - expect(opts.delaysRemaining).to.eq(delaysRemaining) - - context "#normalizeResponse", -> - beforeEach -> - @push = sinon.stub() - - it "sets status to statusCode and deletes statusCode", -> - expect(request.normalizeResponse(@push, { - statusCode: 404 - request: { - headers: {foo: "bar"} - body: "body" - } - })).to.deep.eq({ - status: 404 - statusText: "Not Found" - isOkStatusCode: false - requestHeaders: {foo: "bar"} - requestBody: "body" - }) - - expect(@push).to.be.calledOnce - - it "picks out status body and headers", -> - expect(request.normalizeResponse(@push, { - foo: "bar" - req: {} - originalHeaders: {} - headers: {"Content-Length": 50} - body: "foo" - statusCode: 200 - request: { - headers: {foo: "bar"} - body: "body" - } - })).to.deep.eq({ - body: "foo" - headers: {"Content-Length": 50} - status: 200 - statusText: "OK" - isOkStatusCode: true - requestHeaders: {foo: "bar"} - requestBody: "body" - }) - - expect(@push).to.be.calledOnce - - context "#create", -> - beforeEach (done) -> - @hits = 0 - - @srv = http.createServer (req, res) => - @hits++ - - switch req.url - when "/never-ends" - res.writeHead(200) - res.write("foo\n") - when "/econnreset" - req.socket.destroy() - - @srv.listen(9988, done) - - afterEach -> - @srv.close() - - context "retries for streams", -> - it "does not retry on a timeout", -> - opts = request.setDefaults({ - url: "http://localhost:9988/never-ends" - timeout: 1000 - }) - - stream = request.create(opts) - - retries = 0 - - stream.on "retry", -> - retries++ - - p = Bluebird.fromCallback (cb) -> - stream.on "error", cb - - expect(p).to.be.rejected - .then (err) -> - expect(err.code).to.eq('ESOCKETTIMEDOUT') - expect(retries).to.eq(0) - - it "retries 4x on a connection reset", -> - opts = { - url: "http://localhost:9988/econnreset" - retryIntervals: [0, 1, 2, 3] - timeout: 1000 - } - - stream = request.create(opts) - - retries = 0 - - stream.on "retry", -> - retries++ - - p = Bluebird.fromCallback (cb) -> - stream.on "error", cb - - expect(p).to.be.rejected - .then (err) -> - expect(err.code).to.eq('ECONNRESET') - expect(retries).to.eq(4) - - it "retries 4x on a NXDOMAIN (ENOTFOUND)", -> - nock.enableNetConnect() - - opts = { - url: "http://will-never-exist.invalid.example.com" - retryIntervals: [0, 1, 2, 3] - timeout: 1000 - } - - stream = request.create(opts) - - retries = 0 - - stream.on "retry", -> - retries++ - - p = Bluebird.fromCallback (cb) -> - stream.on "error", cb - - expect(p).to.be.rejected - .then (err) -> - expect(err.code).to.eq('ENOTFOUND') - expect(retries).to.eq(4) - - context "retries for promises", -> - it "does not retry on a timeout", -> - opts = { - url: "http://localhost:9988/never-ends" - timeout: 100 - } - - request.create(opts, true) - .then -> - throw new Error('should not reach') - .catch (err) => - expect(err.error.code).to.eq('ESOCKETTIMEDOUT') - expect(@hits).to.eq(1) - - it "retries 4x on a connection reset", -> - opts = { - url: "http://localhost:9988/econnreset" - retryIntervals: [0, 1, 2, 3] - timeout: 250 - } - - request.create(opts, true) - .then -> - throw new Error('should not reach') - .catch (err) => - expect(err.error.code).to.eq('ECONNRESET') - expect(@hits).to.eq(5) - - context "#sendPromise", -> - it "sets strictSSL=false", -> - init = sinon.spy(request.rp.Request.prototype, "init") - - nock("http://www.github.com") - .get("/foo") - .reply 200, "hello", { - "Content-Type": "text/html" - } - - request.sendPromise({}, @fn, { - url: "http://www.github.com/foo" - cookies: false - }) - .then -> - expect(init).to.be.calledWithMatch({strictSSL: false}) - - it "sets simple=false", -> - nock("http://www.github.com") - .get("/foo") - .reply(500, "") - - ## should not bomb on 500 - ## because simple = false - request.sendPromise({}, @fn, { - url: "http://www.github.com/foo" - cookies: false - }) - - it "sets resolveWithFullResponse=true", -> - nock("http://www.github.com") - .get("/foo") - .reply(200, "hello", { - "Content-Type": "text/html" - }) - - request.sendPromise({}, @fn, { - url: "http://www.github.com/foo" - cookies: false - body: "foobarbaz" - }) - .then (resp) -> - expect(resp).to.have.keys("status", "body", "headers", "duration", "isOkStatusCode", "statusText", "allRequestResponses", "requestBody", "requestHeaders") - - expect(resp.status).to.eq(200) - expect(resp.statusText).to.eq("OK") - expect(resp.body).to.eq("hello") - expect(resp.headers).to.deep.eq({"content-type": "text/html"}) - expect(resp.isOkStatusCode).to.be.true - expect(resp.requestBody).to.eq("foobarbaz") - expect(resp.requestHeaders).to.deep.eq({ - "accept": "*/*" - "accept-encoding": "gzip, deflate" - "connection": "keep-alive" - "content-length": 9 - "host": "www.github.com" - }) - expect(resp.allRequestResponses).to.deep.eq([ - { - "Request Body": "foobarbaz" - "Request Headers": {"accept": "*/*", "accept-encoding": "gzip, deflate", "connection": "keep-alive", "content-length": 9, "host": "www.github.com"} - "Request URL": "http://www.github.com/foo" - "Response Body": "hello" - "Response Headers": {"content-type": "text/html"} - "Response Status": 200 - } - ]) - - it "includes redirects", -> - @fn.resolves() - - nock("http://www.github.com") - .get("/dashboard") - .reply(301, null, { - "location": "/auth" - }) - .get("/auth") - .reply(302, null, { - "location": "/login" - }) - .get("/login") - .reply(200, "log in", { - "Content-Type": "text/html" - }) - - request.sendPromise({}, @fn, { - url: "http://www.github.com/dashboard" - cookies: false - }) - .then (resp) -> - expect(resp).to.have.keys("status", "body", "headers", "duration", "isOkStatusCode", "statusText", "allRequestResponses", "redirects", "requestBody", "requestHeaders") - - expect(resp.status).to.eq(200) - expect(resp.statusText).to.eq("OK") - expect(resp.body).to.eq("log in") - expect(resp.headers).to.deep.eq({"content-type": "text/html"}) - expect(resp.isOkStatusCode).to.be.true - expect(resp.requestBody).to.be.undefined - expect(resp.redirects).to.deep.eq([ - "301: http://www.github.com/auth" - "302: http://www.github.com/login" - ]) - expect(resp.requestHeaders).to.deep.eq({ - "accept": "*/*" - "accept-encoding": "gzip, deflate" - "connection": "keep-alive" - "referer": "http://www.github.com/auth" - "host": "www.github.com" - }) - expect(resp.allRequestResponses).to.deep.eq([ - { - "Request Body": null - "Request Headers": {"accept": "*/*", "accept-encoding": "gzip, deflate", "connection": "keep-alive", "host": "www.github.com"} - "Request URL": "http://www.github.com/dashboard" - "Response Body": null - "Response Headers": {"content-type": "application/json", "location": "/auth"} - "Response Status": 301 - }, { - "Request Body": null - "Request Headers": {"accept": "*/*", "accept-encoding": "gzip, deflate", "connection": "keep-alive", "host": "www.github.com", "referer": "http://www.github.com/dashboard"} - "Request URL": "http://www.github.com/auth" - "Response Body": null - "Response Headers": {"content-type": "application/json", "location": "/login"} - "Response Status": 302 - }, { - "Request Body": null - "Request Headers": {"accept": "*/*", "accept-encoding": "gzip, deflate", "connection": "keep-alive", "host": "www.github.com", "referer": "http://www.github.com/auth"} - "Request URL": "http://www.github.com/login" - "Response Body": "log in" - "Response Headers": {"content-type": "text/html"} - "Response Status": 200 - } - ]) - - it "catches errors", -> - nock.enableNetConnect() - - req = Request({ timeout: 2000 }) - - req.sendPromise({}, @fn, { - url: "http://localhost:1111/foo" - cookies: false - }) - .then -> - throw new Error("should have failed but didnt") - .catch (err) -> - expect(err.message).to.eq("Error: connect ECONNREFUSED 127.0.0.1:1111") - - it "parses response body as json if content-type application/json response headers", -> - nock("http://localhost:8080") - .get("/status.json") - .reply(200, JSON.stringify({status: "ok"}), { - "Content-Type": "application/json" - }) - - request.sendPromise({}, @fn, { - url: "http://localhost:8080/status.json" - cookies: false - }) - .then (resp) -> - expect(resp.body).to.deep.eq({status: "ok"}) - - it "parses response body as json if content-type application/vnd.api+json response headers", -> - nock("http://localhost:8080") - .get("/status.json") - .reply(200, JSON.stringify({status: "ok"}), { - "Content-Type": "application/vnd.api+json" - }) - - request.sendPromise({}, @fn, { - url: "http://localhost:8080/status.json" - cookies: false - }) - .then (resp) -> - expect(resp.body).to.deep.eq({status: "ok"}) - - it "revives from parsing bad json", -> - nock("http://localhost:8080") - .get("/status.json") - .reply(200, "{bad: 'json'}", { - "Content-Type": "application/json" - }) - - request.sendPromise({}, @fn, { - url: "http://localhost:8080/status.json" - cookies: false - }) - .then (resp) -> - expect(resp.body).to.eq("{bad: 'json'}") - - it "sets duration on response", -> - nock("http://localhost:8080") - .get("/foo") - .delay(10) - .reply(200, "123", { - "Content-Type": "text/plain" - }) - - request.sendPromise({}, @fn, { - url: "http://localhost:8080/foo" - cookies: false - }) - .then (resp) -> - expect(resp.duration).to.be.a("Number") - expect(resp.duration).to.be.gt(0) - - it "sends up user-agent headers", -> - nock("http://localhost:8080") - .matchHeader("user-agent", "foobarbaz") - .get("/foo") - .reply(200, "derp") - - headers = {} - headers["user-agent"] = "foobarbaz" - - request.sendPromise(headers, @fn, { - url: "http://localhost:8080/foo" - cookies: false - }) - .then (resp) -> - expect(resp.body).to.eq("derp") - - it "sends connection: keep-alive by default", -> - nock("http://localhost:8080") - .matchHeader("connection", "keep-alive") - .get("/foo") - .reply(200, "it worked") - - request.sendPromise({}, @fn, { - url: "http://localhost:8080/foo" - cookies: false - }) - .then (resp) -> - expect(resp.body).to.eq("it worked") - - it "lower cases headers", -> - nock("http://localhost:8080") - .matchHeader("test", "true") - .get("/foo") - .reply(200, "derp") - - headers = {} - headers["user-agent"] = "foobarbaz" - - request.sendPromise(headers, @fn, { - url: "http://localhost:8080/foo" - cookies: false, - headers: { - 'TEST': true, - } - }) - .then (resp) -> - expect(resp.body).to.eq("derp") - - it "allows overriding user-agent in headers", -> - nock("http://localhost:8080") - .matchHeader("user-agent", "custom-agent") - .get("/foo") - .reply(200, "derp") - - headers = {'user-agent': 'test'} - - request.sendPromise(headers, @fn, { - url: "http://localhost:8080/foo" - cookies: false, - headers: { - 'User-Agent': "custom-agent", - }, - }) - .then (resp) -> - expect(resp.body).to.eq("derp") - - context "accept header", -> - it "sets to */* by default", -> - nock("http://localhost:8080") - .matchHeader("accept", "*/*") - .get("/headers") - .reply(200) - - request.sendPromise({}, @fn, { - url: "http://localhost:8080/headers" - cookies: false - }) - .then (resp) -> - expect(resp.status).to.eq(200) - - it "can override accept header", -> - nock("http://localhost:8080") - .matchHeader("accept", "text/html") - .get("/headers") - .reply(200) - - request.sendPromise({}, @fn, { - url: "http://localhost:8080/headers" - cookies: false - headers: { - accept: "text/html" - } - }) - .then (resp) -> - expect(resp.status).to.eq(200) - - it "can override Accept header", -> - nock("http://localhost:8080") - .matchHeader("accept", "text/plain") - .get("/headers") - .reply(200) - - request.sendPromise({}, @fn, { - url: "http://localhost:8080/headers" - cookies: false - headers: { - Accept: "text/plain" - } - }) - .then (resp) -> - expect(resp.status).to.eq(200) - - context "qs", -> - it "can accept qs", -> - nock("http://localhost:8080") - .get("/foo?bar=baz&q=1") - .reply(200) - - request.sendPromise({}, @fn, { - url: "http://localhost:8080/foo" - cookies: false - qs: { - bar: "baz" - q: 1 - } - }) - .then (resp) -> - expect(resp.status).to.eq(200) - - context "followRedirect", -> - beforeEach -> - @fn.resolves() - - it "by default follow redirects", -> - nock("http://localhost:8080") - .get("/dashboard") - .reply(302, "", { - location: "http://localhost:8080/login" - }) - .get("/login") - .reply(200, "login") - - request.sendPromise({}, @fn, { - url: "http://localhost:8080/dashboard" - cookies: false - followRedirect: true - }) - .then (resp) -> - expect(resp.status).to.eq(200) - expect(resp.body).to.eq("login") - expect(resp).not.to.have.property("redirectedToUrl") - - it "follows non-GET redirects by default", -> - nock("http://localhost:8080") - .post("/login") - .reply(302, "", { - location: "http://localhost:8080/dashboard" - }) - .get("/dashboard") - .reply(200, "dashboard") - - request.sendPromise({}, @fn, { - method: "POST" - url: "http://localhost:8080/login" - cookies: false - }) - .then (resp) -> - expect(resp.status).to.eq(200) - expect(resp.body).to.eq("dashboard") - expect(resp).not.to.have.property("redirectedToUrl") - - it "can turn off following redirects", -> - nock("http://localhost:8080") - .get("/dashboard") - .reply(302, "", { - location: "http://localhost:8080/login" - }) - .get("/login") - .reply(200, "login") - - request.sendPromise({}, @fn, { - url: "http://localhost:8080/dashboard" - cookies: false - followRedirect: false - }) - .then (resp) -> - expect(resp.status).to.eq(302) - expect(resp.body).to.eq("") - expect(resp.redirectedToUrl).to.eq("http://localhost:8080/login") - - it "resolves redirectedToUrl on relative redirects", -> - nock("http://localhost:8080") - .get("/dashboard") - .reply(302, "", { - location: "/login" ## absolute-relative pathname - }) - .get("/login") - .reply(200, "login") - - request.sendPromise({}, @fn, { - url: "http://localhost:8080/dashboard" - cookies: false - followRedirect: false - }) - .then (resp) -> - expect(resp.status).to.eq(302) - expect(resp.redirectedToUrl).to.eq("http://localhost:8080/login") - - it "resolves redirectedToUrl to another domain", -> - nock("http://localhost:8080") - .get("/dashboard") - .reply(301, "", { - location: "https://www.google.com/login" - }) - .get("/login") - .reply(200, "login") - - request.sendPromise({}, @fn, { - url: "http://localhost:8080/dashboard" - cookies: false - followRedirect: false - }) - .then (resp) -> - expect(resp.status).to.eq(301) - expect(resp.redirectedToUrl).to.eq("https://www.google.com/login") - - it "does not included redirectedToUrl when following redirects", -> - nock("http://localhost:8080") - .get("/dashboard") - .reply(302, "", { - location: "http://localhost:8080/login" - }) - .get("/login") - .reply(200, "login") - - request.sendPromise({}, @fn, { - url: "http://localhost:8080/dashboard" - cookies: false - }) - .then (resp) -> - expect(resp.status).to.eq(200) - expect(resp).not.to.have.property("redirectedToUrl") - - it "gets + attaches the cookies at each redirect", -> - testAttachingCookiesWith => - request.sendPromise({}, @fn, { - url: "http://localhost:1234/" - }) - - context "form=true", -> - beforeEach -> - nock("http://localhost:8080") - .matchHeader("Content-Type", "application/x-www-form-urlencoded") - .post("/login", "foo=bar&baz=quux") - .reply(200, "") - - it "takes converts body to x-www-form-urlencoded and sets header", -> - request.sendPromise({}, @fn, { - url: "http://localhost:8080/login" - method: "POST" - cookies: false - form: true - body: { - foo: "bar" - baz: "quux" - } - }) - .then (resp) -> - expect(resp.status).to.eq(200) - expect(resp.body).to.eq("") - - it "does not send body", -> - init = sinon.spy(request.rp.Request.prototype, "init") - - body = { - foo: "bar" - baz: "quux" - } - - request.sendPromise({}, @fn, { - url: "http://localhost:8080/login" - method: "POST" - cookies: false - form: true - json: true - body: body - }) - .then (resp) -> - expect(resp.status).to.eq(200) - expect(resp.body).to.eq("") - expect(init).not.to.be.calledWithMatch({body: body}) - - it "does not set json=true", -> - init = sinon.spy(request.rp.Request.prototype, "init") - - request.sendPromise({}, @fn, { - url: "http://localhost:8080/login" - method: "POST" - cookies: false - form: true - json: true - body: { - foo: "bar" - baz: "quux" - } - }) - .then (resp) -> - expect(resp.status).to.eq(200) - expect(resp.body).to.eq("") - expect(init).not.to.be.calledWithMatch({json: true}) - - context "bad headers", -> - beforeEach (done) -> - @srv = http.createServer (req, res) -> - res.writeHead(200) - res.end() - - @srv.listen(9988, done) - - afterEach -> - @srv.close() - - it "recovers from bad headers", -> - request.sendPromise({}, @fn, { - url: "http://localhost:9988/foo" - cookies: false - headers: { - "x-text": "אבגד" - } - }) - .then -> - throw new Error("should have failed") - .catch (err) -> - expect(err.message).to.eq "TypeError [ERR_INVALID_CHAR]: Invalid character in header content [\"x-text\"]" - - it "handles weird content in the body just fine", -> - request.sendPromise({}, @fn, { - url: "http://localhost:9988/foo" - cookies: false - json: true - body: { - "x-text": "אבגד" - } - }) - - context "#sendStream", -> - it "allows overriding user-agent in headers", -> - nock("http://localhost:8080") - .matchHeader("user-agent", "custom-agent") - .get("/foo") - .reply(200, "derp") - - sinon.spy(request, "create") - @fn.resolves({}) - - headers = {'user-agent': 'test'} - - options = { - url: "http://localhost:8080/foo" - cookies: false, - headers: { - 'user-agent': "custom-agent", - }, - } - - request.sendStream(headers, @fn, options) - .then (beginFn) -> - beginFn() - expect(request.create).to.be.calledOnce - expect(request.create).to.be.calledWith(options) - - it "gets + attaches the cookies at each redirect", -> - testAttachingCookiesWith => - request.sendStream({}, @fn, { - url: "http://localhost:1234/" - followRedirect: _.stubTrue - }) - .then (fn) => - req = fn() - - new Promise (resolve, reject) => - req.on('response', resolve) - req.on('error', reject) diff --git a/packages/server/test/unit/request_spec.js b/packages/server/test/unit/request_spec.js new file mode 100644 index 000000000000..7110a2b5b045 --- /dev/null +++ b/packages/server/test/unit/request_spec.js @@ -0,0 +1,1034 @@ +require('../spec_helper') + +const _ = require('lodash') +const http = require('http') +const Bluebird = require('bluebird') +const Request = require(`${root}lib/request`) +const snapshot = require('snap-shot-it') + +const request = Request({ timeout: 100 }) + +const testAttachingCookiesWith = function (fn) { + const set = sinon.spy(request, 'setCookiesOnBrowser') + const get = sinon.spy(request, 'setRequestCookieHeader') + + nock('http://localhost:1234') + .get('/') + .reply(302, '', { + 'set-cookie': 'one=1', + location: '/second', + }) + .get('/second') + .reply(302, '', { + 'set-cookie': 'two=2', + location: '/third', + }) + .get('/third') + .reply(200, '', { + 'set-cookie': 'three=3', + }) + + return fn() + .then(() => { + return snapshot({ + setCalls: set.getCalls().map((call) => { + return { + currentUrl: call.args[1], + setCookie: call.args[0].headers['set-cookie'], + } + }), + getCalls: get.getCalls().map((call) => { + return { + newUrl: _.get(call, 'args.1'), + } + }), + }) + }) +} + +describe('lib/request', () => { + beforeEach(function () { + this.fn = sinon.stub() + this.fn.withArgs('set:cookie').resolves({}) + this.fn.withArgs('get:cookies').resolves([]) + }) + + it('is defined', () => { + expect(request).to.be.an('object') + }) + + context('#getDelayForRetry', () => { + it('divides by 10 when delay >= 1000 and err.code = ECONNREFUSED', () => { + const retryIntervals = [1, 2, 3, 4] + const delaysRemaining = [0, 999, 1000, 2000] + + const err = { + code: 'ECONNREFUSED', + } + + const onNext = sinon.stub() + + retryIntervals.forEach(() => { + return request.getDelayForRetry({ + err, + onNext, + retryIntervals, + delaysRemaining, + }) + }) + + expect(delaysRemaining).to.be.empty + + expect(onNext.args).to.deep.eq([ + [0, 1], + [999, 2], + [100, 3], + [200, 4], + ]) + }) + + it('does not divide by 10 when err.code != ECONNREFUSED', () => { + const retryIntervals = [1, 2, 3, 4] + const delaysRemaining = [2000, 2000, 2000, 2000] + + const err = { + code: 'ECONNRESET', + } + + const onNext = sinon.stub() + + request.getDelayForRetry({ + err, + onNext, + retryIntervals, + delaysRemaining, + }) + + expect(delaysRemaining).to.have.length(3) + + expect(onNext).to.be.calledWith(2000, 1) + }) + + it('calls onElse when delaysRemaining is exhausted', () => { + const retryIntervals = [1, 2, 3, 4] + const delaysRemaining = [] + + const onNext = sinon.stub() + const onElse = sinon.stub() + + request.getDelayForRetry({ + onElse, + onNext, + retryIntervals, + delaysRemaining, + }) + + expect(onElse).to.be.calledWithExactly() + + expect(onNext).not.to.be.called + }) + }) + + context('#setDefaults', () => { + it('delaysRemaining to retryIntervals clone', () => { + const retryIntervals = [1, 2, 3, 4] + + const opts = request.setDefaults({ retryIntervals }) + + expect(opts.retryIntervals).to.eq(retryIntervals) + expect(opts.delaysRemaining).not.to.eq(retryIntervals) + + expect(opts.delaysRemaining).to.deep.eq(retryIntervals) + }) + + it('retryIntervals to [0, 1000, 2000, 2000] by default', () => { + const opts = request.setDefaults({}) + + expect(opts.retryIntervals).to.deep.eq([0, 1000, 2000, 2000]) + }) + + it('delaysRemaining can be overridden', () => { + const delaysRemaining = [1] + const opts = request.setDefaults({ delaysRemaining }) + + expect(opts.delaysRemaining).to.eq(delaysRemaining) + }) + }) + + context('#normalizeResponse', () => { + beforeEach(function () { + this.push = sinon.stub() + }) + + it('sets status to statusCode and deletes statusCode', function () { + expect(request.normalizeResponse(this.push, { + statusCode: 404, + request: { + headers: { foo: 'bar' }, + body: 'body', + }, + })).to.deep.eq({ + status: 404, + statusText: 'Not Found', + isOkStatusCode: false, + requestHeaders: { foo: 'bar' }, + requestBody: 'body', + }) + + expect(this.push).to.be.calledOnce + }) + + it('picks out status body and headers', function () { + expect(request.normalizeResponse(this.push, { + foo: 'bar', + req: {}, + originalHeaders: {}, + headers: { 'Content-Length': 50 }, + body: 'foo', + statusCode: 200, + request: { + headers: { foo: 'bar' }, + body: 'body', + }, + })).to.deep.eq({ + body: 'foo', + headers: { 'Content-Length': 50 }, + status: 200, + statusText: 'OK', + isOkStatusCode: true, + requestHeaders: { foo: 'bar' }, + requestBody: 'body', + }) + + expect(this.push).to.be.calledOnce + }) + }) + + context('#create', () => { + beforeEach(function (done) { + this.hits = 0 + + this.srv = http.createServer((req, res) => { + this.hits++ + + switch (req.url) { + case '/never-ends': + res.writeHead(200) + + return res.write('foo\n') + case '/econnreset': + return req.socket.destroy() + default: + break + } + }) + + this.srv.listen(9988, done) + }) + + afterEach(function () { + return this.srv.close() + }) + + context('retries for streams', () => { + it('does not retry on a timeout', () => { + const opts = request.setDefaults({ + url: 'http://localhost:9988/never-ends', + timeout: 1000, + }) + + const stream = request.create(opts) + + let retries = 0 + + stream.on('retry', () => { + retries++ + }) + + const p = Bluebird.fromCallback((cb) => { + stream.on('error', cb) + }) + + return expect(p).to.be.rejected + .then((err) => { + expect(err.code).to.eq('ESOCKETTIMEDOUT') + + expect(retries).to.eq(0) + }) + }) + + it('retries 4x on a connection reset', () => { + const opts = { + url: 'http://localhost:9988/econnreset', + retryIntervals: [0, 1, 2, 3], + timeout: 1000, + } + + const stream = request.create(opts) + + let retries = 0 + + stream.on('retry', () => { + retries++ + }) + + const p = Bluebird.fromCallback((cb) => { + stream.on('error', cb) + }) + + return expect(p).to.be.rejected + .then((err) => { + expect(err.code).to.eq('ECONNRESET') + + expect(retries).to.eq(4) + }) + }) + + it('retries 4x on a NXDOMAIN (ENOTFOUND)', () => { + nock.enableNetConnect() + + const opts = { + url: 'http://will-never-exist.invalid.example.com', + retryIntervals: [0, 1, 2, 3], + timeout: 1000, + } + + const stream = request.create(opts) + + let retries = 0 + + stream.on('retry', () => { + retries++ + }) + + const p = Bluebird.fromCallback((cb) => { + stream.on('error', cb) + }) + + return expect(p).to.be.rejected + .then((err) => { + expect(err.code).to.eq('ENOTFOUND') + + expect(retries).to.eq(4) + }) + }) + }) + + context('retries for promises', () => { + it('does not retry on a timeout', function () { + const opts = { + url: 'http://localhost:9988/never-ends', + timeout: 100, + } + + return request.create(opts, true) + .then(() => { + throw new Error('should not reach') + }).catch((err) => { + expect(err.error.code).to.eq('ESOCKETTIMEDOUT') + + expect(this.hits).to.eq(1) + }) + }) + + it('retries 4x on a connection reset', function () { + const opts = { + url: 'http://localhost:9988/econnreset', + retryIntervals: [0, 1, 2, 3], + timeout: 250, + } + + return request.create(opts, true) + .then(() => { + throw new Error('should not reach') + }).catch((err) => { + expect(err.error.code).to.eq('ECONNRESET') + + expect(this.hits).to.eq(5) + }) + }) + }) + }) + + context('#sendPromise', () => { + it('sets strictSSL=false', function () { + const init = sinon.spy(request.rp.Request.prototype, 'init') + + nock('http://www.github.com') + .get('/foo') + .reply(200, 'hello', { + 'Content-Type': 'text/html', + }) + + return request.sendPromise({}, this.fn, { + url: 'http://www.github.com/foo', + cookies: false, + }) + .then(() => { + expect(init).to.be.calledWithMatch({ strictSSL: false }) + }) + }) + + it('sets simple=false', function () { + nock('http://www.github.com') + .get('/foo') + .reply(500, '') + + // should not bomb on 500 + // because simple = false + return request.sendPromise({}, this.fn, { + url: 'http://www.github.com/foo', + cookies: false, + }) + }) + + it('sets resolveWithFullResponse=true', function () { + nock('http://www.github.com') + .get('/foo') + .reply(200, 'hello', { + 'Content-Type': 'text/html', + }) + + return request.sendPromise({}, this.fn, { + url: 'http://www.github.com/foo', + cookies: false, + body: 'foobarbaz', + }) + .then((resp) => { + expect(resp).to.have.keys('status', 'body', 'headers', 'duration', 'isOkStatusCode', 'statusText', 'allRequestResponses', 'requestBody', 'requestHeaders') + + expect(resp.status).to.eq(200) + expect(resp.statusText).to.eq('OK') + expect(resp.body).to.eq('hello') + expect(resp.headers).to.deep.eq({ 'content-type': 'text/html' }) + expect(resp.isOkStatusCode).to.be.true + expect(resp.requestBody).to.eq('foobarbaz') + expect(resp.requestHeaders).to.deep.eq({ + 'accept': '*/*', + 'accept-encoding': 'gzip, deflate', + 'connection': 'keep-alive', + 'content-length': 9, + 'host': 'www.github.com', + }) + + expect(resp.allRequestResponses).to.deep.eq([ + { + 'Request Body': 'foobarbaz', + 'Request Headers': { 'accept': '*/*', 'accept-encoding': 'gzip, deflate', 'connection': 'keep-alive', 'content-length': 9, 'host': 'www.github.com' }, + 'Request URL': 'http://www.github.com/foo', + 'Response Body': 'hello', + 'Response Headers': { 'content-type': 'text/html' }, + 'Response Status': 200, + }, + ]) + }) + }) + + it('includes redirects', function () { + this.fn.resolves() + + nock('http://www.github.com') + .get('/dashboard') + .reply(301, null, { + 'location': '/auth', + }) + .get('/auth') + .reply(302, null, { + 'location': '/login', + }) + .get('/login') + .reply(200, 'log in', { + 'Content-Type': 'text/html', + }) + + return request.sendPromise({}, this.fn, { + url: 'http://www.github.com/dashboard', + cookies: false, + }) + .then((resp) => { + expect(resp).to.have.keys('status', 'body', 'headers', 'duration', 'isOkStatusCode', 'statusText', 'allRequestResponses', 'redirects', 'requestBody', 'requestHeaders') + + expect(resp.status).to.eq(200) + expect(resp.statusText).to.eq('OK') + expect(resp.body).to.eq('log in') + expect(resp.headers).to.deep.eq({ 'content-type': 'text/html' }) + expect(resp.isOkStatusCode).to.be.true + expect(resp.requestBody).to.be.undefined + expect(resp.redirects).to.deep.eq([ + '301: http://www.github.com/auth', + '302: http://www.github.com/login', + ]) + + expect(resp.requestHeaders).to.deep.eq({ + 'accept': '*/*', + 'accept-encoding': 'gzip, deflate', + 'connection': 'keep-alive', + 'referer': 'http://www.github.com/auth', + 'host': 'www.github.com', + }) + + expect(resp.allRequestResponses).to.deep.eq([ + { + 'Request Body': null, + 'Request Headers': { 'accept': '*/*', 'accept-encoding': 'gzip, deflate', 'connection': 'keep-alive', 'host': 'www.github.com' }, + 'Request URL': 'http://www.github.com/dashboard', + 'Response Body': null, + 'Response Headers': { 'content-type': 'application/json', 'location': '/auth' }, + 'Response Status': 301, + }, { + 'Request Body': null, + 'Request Headers': { 'accept': '*/*', 'accept-encoding': 'gzip, deflate', 'connection': 'keep-alive', 'host': 'www.github.com', 'referer': 'http://www.github.com/dashboard' }, + 'Request URL': 'http://www.github.com/auth', + 'Response Body': null, + 'Response Headers': { 'content-type': 'application/json', 'location': '/login' }, + 'Response Status': 302, + }, { + 'Request Body': null, + 'Request Headers': { 'accept': '*/*', 'accept-encoding': 'gzip, deflate', 'connection': 'keep-alive', 'host': 'www.github.com', 'referer': 'http://www.github.com/auth' }, + 'Request URL': 'http://www.github.com/login', + 'Response Body': 'log in', + 'Response Headers': { 'content-type': 'text/html' }, + 'Response Status': 200, + }, + ]) + }) + }) + + it('catches errors', function () { + nock.enableNetConnect() + + const req = Request({ timeout: 2000 }) + + return req.sendPromise({}, this.fn, { + url: 'http://localhost:1111/foo', + cookies: false, + }) + .then(() => { + throw new Error('should have failed but didnt') + }).catch((err) => { + expect(err.message).to.eq('Error: connect ECONNREFUSED 127.0.0.1:1111') + }) + }) + + it('parses response body as json if content-type application/json response headers', function () { + nock('http://localhost:8080') + .get('/status.json') + .reply(200, JSON.stringify({ status: 'ok' }), { + 'Content-Type': 'application/json', + }) + + return request.sendPromise({}, this.fn, { + url: 'http://localhost:8080/status.json', + cookies: false, + }) + .then((resp) => { + expect(resp.body).to.deep.eq({ status: 'ok' }) + }) + }) + + it('parses response body as json if content-type application/vnd.api+json response headers', function () { + nock('http://localhost:8080') + .get('/status.json') + .reply(200, JSON.stringify({ status: 'ok' }), { + 'Content-Type': 'application/vnd.api+json', + }) + + return request.sendPromise({}, this.fn, { + url: 'http://localhost:8080/status.json', + cookies: false, + }) + .then((resp) => { + expect(resp.body).to.deep.eq({ status: 'ok' }) + }) + }) + + it('revives from parsing bad json', function () { + nock('http://localhost:8080') + .get('/status.json') + .reply(200, '{bad: \'json\'}', { + 'Content-Type': 'application/json', + }) + + return request.sendPromise({}, this.fn, { + url: 'http://localhost:8080/status.json', + cookies: false, + }) + .then((resp) => { + expect(resp.body).to.eq('{bad: \'json\'}') + }) + }) + + it('sets duration on response', function () { + nock('http://localhost:8080') + .get('/foo') + .delay(10) + .reply(200, '123', { + 'Content-Type': 'text/plain', + }) + + return request.sendPromise({}, this.fn, { + url: 'http://localhost:8080/foo', + cookies: false, + }) + .then((resp) => { + expect(resp.duration).to.be.a('Number') + + expect(resp.duration).to.be.gt(0) + }) + }) + + it('sends up user-agent headers', function () { + nock('http://localhost:8080') + .matchHeader('user-agent', 'foobarbaz') + .get('/foo') + .reply(200, 'derp') + + const headers = {} + + headers['user-agent'] = 'foobarbaz' + + return request.sendPromise(headers, this.fn, { + url: 'http://localhost:8080/foo', + cookies: false, + }) + .then((resp) => { + expect(resp.body).to.eq('derp') + }) + }) + + it('sends connection: keep-alive by default', function () { + nock('http://localhost:8080') + .matchHeader('connection', 'keep-alive') + .get('/foo') + .reply(200, 'it worked') + + return request.sendPromise({}, this.fn, { + url: 'http://localhost:8080/foo', + cookies: false, + }) + .then((resp) => { + expect(resp.body).to.eq('it worked') + }) + }) + + it('lower cases headers', function () { + nock('http://localhost:8080') + .matchHeader('test', 'true') + .get('/foo') + .reply(200, 'derp') + + const headers = {} + + headers['user-agent'] = 'foobarbaz' + + return request.sendPromise(headers, this.fn, { + url: 'http://localhost:8080/foo', + cookies: false, + headers: { + 'TEST': true, + }, + }) + .then((resp) => { + expect(resp.body).to.eq('derp') + }) + }) + + it('allows overriding user-agent in headers', function () { + nock('http://localhost:8080') + .matchHeader('user-agent', 'custom-agent') + .get('/foo') + .reply(200, 'derp') + + const headers = { 'user-agent': 'test' } + + return request.sendPromise(headers, this.fn, { + url: 'http://localhost:8080/foo', + cookies: false, + headers: { + 'User-Agent': 'custom-agent', + }, + }) + .then((resp) => { + expect(resp.body).to.eq('derp') + }) + }) + + context('accept header', () => { + it('sets to */* by default', function () { + nock('http://localhost:8080') + .matchHeader('accept', '*/*') + .get('/headers') + .reply(200) + + return request.sendPromise({}, this.fn, { + url: 'http://localhost:8080/headers', + cookies: false, + }) + .then((resp) => { + expect(resp.status).to.eq(200) + }) + }) + + it('can override accept header', function () { + nock('http://localhost:8080') + .matchHeader('accept', 'text/html') + .get('/headers') + .reply(200) + + return request.sendPromise({}, this.fn, { + url: 'http://localhost:8080/headers', + cookies: false, + headers: { + accept: 'text/html', + }, + }) + .then((resp) => { + expect(resp.status).to.eq(200) + }) + }) + + it('can override Accept header', function () { + nock('http://localhost:8080') + .matchHeader('accept', 'text/plain') + .get('/headers') + .reply(200) + + return request.sendPromise({}, this.fn, { + url: 'http://localhost:8080/headers', + cookies: false, + headers: { + Accept: 'text/plain', + }, + }) + .then((resp) => { + expect(resp.status).to.eq(200) + }) + }) + }) + + context('qs', () => { + it('can accept qs', function () { + nock('http://localhost:8080') + .get('/foo?bar=baz&q=1') + .reply(200) + + return request.sendPromise({}, this.fn, { + url: 'http://localhost:8080/foo', + cookies: false, + qs: { + bar: 'baz', + q: 1, + }, + }) + .then((resp) => { + expect(resp.status).to.eq(200) + }) + }) + }) + + context('followRedirect', () => { + beforeEach(function () { + this.fn.resolves() + }) + + it('by default follow redirects', function () { + nock('http://localhost:8080') + .get('/dashboard') + .reply(302, '', { + location: 'http://localhost:8080/login', + }) + .get('/login') + .reply(200, 'login') + + return request.sendPromise({}, this.fn, { + url: 'http://localhost:8080/dashboard', + cookies: false, + followRedirect: true, + }) + .then((resp) => { + expect(resp.status).to.eq(200) + expect(resp.body).to.eq('login') + + expect(resp).not.to.have.property('redirectedToUrl') + }) + }) + + it('follows non-GET redirects by default', function () { + nock('http://localhost:8080') + .post('/login') + .reply(302, '', { + location: 'http://localhost:8080/dashboard', + }) + .get('/dashboard') + .reply(200, 'dashboard') + + return request.sendPromise({}, this.fn, { + method: 'POST', + url: 'http://localhost:8080/login', + cookies: false, + }) + .then((resp) => { + expect(resp.status).to.eq(200) + expect(resp.body).to.eq('dashboard') + + expect(resp).not.to.have.property('redirectedToUrl') + }) + }) + + it('can turn off following redirects', function () { + nock('http://localhost:8080') + .get('/dashboard') + .reply(302, '', { + location: 'http://localhost:8080/login', + }) + .get('/login') + .reply(200, 'login') + + return request.sendPromise({}, this.fn, { + url: 'http://localhost:8080/dashboard', + cookies: false, + followRedirect: false, + }) + .then((resp) => { + expect(resp.status).to.eq(302) + expect(resp.body).to.eq('') + + expect(resp.redirectedToUrl).to.eq('http://localhost:8080/login') + }) + }) + + it('resolves redirectedToUrl on relative redirects', function () { + nock('http://localhost:8080') + .get('/dashboard') + .reply(302, '', { + location: '/login', // absolute-relative pathname + }) + .get('/login') + .reply(200, 'login') + + return request.sendPromise({}, this.fn, { + url: 'http://localhost:8080/dashboard', + cookies: false, + followRedirect: false, + }) + .then((resp) => { + expect(resp.status).to.eq(302) + + expect(resp.redirectedToUrl).to.eq('http://localhost:8080/login') + }) + }) + + it('resolves redirectedToUrl to another domain', function () { + nock('http://localhost:8080') + .get('/dashboard') + .reply(301, '', { + location: 'https://www.google.com/login', + }) + .get('/login') + .reply(200, 'login') + + return request.sendPromise({}, this.fn, { + url: 'http://localhost:8080/dashboard', + cookies: false, + followRedirect: false, + }) + .then((resp) => { + expect(resp.status).to.eq(301) + + expect(resp.redirectedToUrl).to.eq('https://www.google.com/login') + }) + }) + + it('does not included redirectedToUrl when following redirects', function () { + nock('http://localhost:8080') + .get('/dashboard') + .reply(302, '', { + location: 'http://localhost:8080/login', + }) + .get('/login') + .reply(200, 'login') + + return request.sendPromise({}, this.fn, { + url: 'http://localhost:8080/dashboard', + cookies: false, + }) + .then((resp) => { + expect(resp.status).to.eq(200) + + expect(resp).not.to.have.property('redirectedToUrl') + }) + }) + + it('gets + attaches the cookies at each redirect', function () { + return testAttachingCookiesWith(() => { + return request.sendPromise({}, this.fn, { + url: 'http://localhost:1234/', + }) + }) + }) + }) + + context('form=true', () => { + beforeEach(() => { + nock('http://localhost:8080') + .matchHeader('Content-Type', 'application/x-www-form-urlencoded') + .post('/login', 'foo=bar&baz=quux') + .reply(200, '') + }) + + it('takes converts body to x-www-form-urlencoded and sets header', function () { + return request.sendPromise({}, this.fn, { + url: 'http://localhost:8080/login', + method: 'POST', + cookies: false, + form: true, + body: { + foo: 'bar', + baz: 'quux', + }, + }) + .then((resp) => { + expect(resp.status).to.eq(200) + + expect(resp.body).to.eq('') + }) + }) + + it('does not send body', function () { + const init = sinon.spy(request.rp.Request.prototype, 'init') + + const body = { + foo: 'bar', + baz: 'quux', + } + + return request.sendPromise({}, this.fn, { + url: 'http://localhost:8080/login', + method: 'POST', + cookies: false, + form: true, + json: true, + body, + }) + .then((resp) => { + expect(resp.status).to.eq(200) + expect(resp.body).to.eq('') + + expect(init).not.to.be.calledWithMatch({ body }) + }) + }) + + it('does not set json=true', function () { + const init = sinon.spy(request.rp.Request.prototype, 'init') + + return request.sendPromise({}, this.fn, { + url: 'http://localhost:8080/login', + method: 'POST', + cookies: false, + form: true, + json: true, + body: { + foo: 'bar', + baz: 'quux', + }, + }) + .then((resp) => { + expect(resp.status).to.eq(200) + expect(resp.body).to.eq('') + + expect(init).not.to.be.calledWithMatch({ json: true }) + }) + }) + }) + + context('bad headers', () => { + beforeEach(function (done) { + this.srv = http.createServer((req, res) => { + res.writeHead(200) + + res.end() + }) + + this.srv.listen(9988, done) + }) + + afterEach(function () { + return this.srv.close() + }) + + it('recovers from bad headers', function () { + return request.sendPromise({}, this.fn, { + url: 'http://localhost:9988/foo', + cookies: false, + headers: { + 'x-text': 'אבגד', + }, + }) + .then(() => { + throw new Error('should have failed') + }).catch((err) => { + expect(err.message).to.eq('TypeError [ERR_INVALID_CHAR]: Invalid character in header content ["x-text"]') + }) + }) + + it('handles weird content in the body just fine', function () { + return request.sendPromise({}, this.fn, { + url: 'http://localhost:9988/foo', + cookies: false, + json: true, + body: { + 'x-text': 'אבגד', + }, + }) + }) + }) + }) + + context('#sendStream', () => { + it('allows overriding user-agent in headers', function () { + nock('http://localhost:8080') + .matchHeader('user-agent', 'custom-agent') + .get('/foo') + .reply(200, 'derp') + + sinon.spy(request, 'create') + this.fn.resolves({}) + + const headers = { 'user-agent': 'test' } + + const options = { + url: 'http://localhost:8080/foo', + cookies: false, + headers: { + 'user-agent': 'custom-agent', + }, + } + + return request.sendStream(headers, this.fn, options) + .then((beginFn) => { + beginFn() + expect(request.create).to.be.calledOnce + + expect(request.create).to.be.calledWith(options) + }) + }) + + it('gets + attaches the cookies at each redirect', function () { + return testAttachingCookiesWith(() => { + return request.sendStream({}, this.fn, { + url: 'http://localhost:1234/', + followRedirect: _.stubTrue, + }) + .then((fn) => { + const req = fn() + + return new Promise((resolve, reject) => { + req.on('response', resolve) + + req.on('error', reject) + }) + }) + }) + }) + }) +})