main.js

// Copyright 2010-2011 Mikeal Rogers
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

var http = require('http')
  , https = false
  , tls = false
  , url = require('url')
  , util = require('util')
  , stream = require('stream')
  , qs = require('querystring')
  , mimetypes = require('./mimetypes')
  , oauth = require('./oauth')
  , uuid = require('./uuid')
  , ForeverAgent = require('./forever')
  , Cookie = require('./vendor/cookie')
  , CookieJar = require('./vendor/cookie/jar')
  , cookieJar = new CookieJar
  ;
  
if (process.logging) {
  var log = process.logging('request')
}

try {
  https = require('https')
} catch (e) {}

try {
  tls = require('tls')
} catch (e) {}

function toBase64 (str) {
  return (new Buffer(str || "", "ascii")).toString("base64")
}

// Hacky fix for pre-0.4.4 https
if (https && !https.Agent) {
  https.Agent = function (options) {
    http.Agent.call(this, options)
  }
  util.inherits(https.Agent, http.Agent)
  https.Agent.prototype._getConnection = function(host, port, cb) {
    var s = tls.connect(port, host, this.options, function() {
      // do other checks here?
      if (cb) cb()
    })
    return s
  }
}

function isReadStream (rs) {
  if (rs.readable && rs.path && rs.mode) {
    return true
  }
}

function copy (obj) {
  var o = {}
  for (var i in obj) o[i] = obj[i]
  return o
}

var isUrl = /^https?:/

var globalPool = {}

function Request (options) {
  stream.Stream.call(this)
  this.readable = true
  this.writable = true

  if (typeof options === 'string') {
    options = {uri:options}
  }

  for (var i in options) {
    this[i] = options[i]
  }
  if (!this.pool) this.pool = globalPool
  this.dests = []
  this.__isRequestRequest = true
}
util.inherits(Request, stream.Stream)
Request.prototype.getAgent = function (host, port) {
  if (!this.pool[host+':'+port]) {
    this.pool[host+':'+port] = new this.httpModule.Agent({host:host, port:port})
  }
  return this.pool[host+':'+port]
}
Request.prototype.request = function () {
  var self = this

  // Protect against double callback
  if (!self._callback && self.callback) {
    self._callback = self.callback
    self.callback = function () {
      if (self._callbackCalled) return // Print a warning maybe?
      self._callback.apply(self, arguments)
      self._callbackCalled = true
    }
  }

  if (self.url) {
    // People use this property instead all the time so why not just support it.
    self.uri = self.url
    delete self.url
  }

  if (!self.uri) {
    throw new Error("options.uri is a required argument")
  } else {
    if (typeof self.uri == "string") self.uri = url.parse(self.uri)
  }
  if (self.proxy) {
    if (typeof self.proxy == 'string') self.proxy = url.parse(self.proxy)
  }

  self._redirectsFollowed = self._redirectsFollowed || 0
  self.maxRedirects = (self.maxRedirects !== undefined) ? self.maxRedirects : 10
  self.followRedirect = (self.followRedirect !== undefined) ? self.followRedirect : true
  if (self.followRedirect)
    self.redirects = self.redirects || []

  self.headers = self.headers ? copy(self.headers) : {}

  var setHost = false
  if (!self.headers.host) {
    self.headers.host = self.uri.hostname
    if (self.uri.port) {
      if ( !(self.uri.port === 80 && self.uri.protocol === 'http:') &&
           !(self.uri.port === 443 && self.uri.protocol === 'https:') )
      self.headers.host += (':'+self.uri.port)
    }
    setHost = true
  }

  if (self.jar === false) {
    // disable cookies
    var cookies = false;
    self._disableCookies = true;
  } else if (self.jar) {
    // fetch cookie from the user defined cookie jar
    var cookies = self.jar.get({ url: self.uri.href })
  } else {
    // fetch cookie from the global cookie jar
    var cookies = cookieJar.get({ url: self.uri.href })
  }
  if (cookies) {
    var cookieString = cookies.map(function (c) {
      return c.name + "=" + c.value;
    }).join("; ");
    
    self.headers.Cookie = cookieString;
  }

  if (!self.uri.pathname) {self.uri.pathname = '/'}
  if (!self.uri.port) {
    if (self.uri.protocol == 'http:') {self.uri.port = 80}
    else if (self.uri.protocol == 'https:') {self.uri.port = 443}
  }

  if (self.proxy) {
    self.port = self.proxy.port
    self.host = self.proxy.hostname
  } else {
    self.port = self.uri.port
    self.host = self.uri.hostname
  }

  if (self.onResponse === true) {
    self.onResponse = self.callback
    delete self.callback
  }

  var clientErrorHandler = function (error) {
    if (setHost) delete self.headers.host
    if (self.req._reusedSocket && error.code === 'ECONNRESET') {
      self.agent = {addRequest: ForeverAgent.prototype.addRequestNoreuse.bind(self.agent)}
      self.start()
      self.req.end()
      return
    }
    if (self.timeout && self.timeoutTimer) {
        clearTimeout(self.timeoutTimer);
        self.timeoutTimer = null;
    }
    self.emit('error', error)
  }
  if (self.onResponse) self.on('error', function (e) {self.onResponse(e)})
  if (self.callback) self.on('error', function (e) {self.callback(e)})

  if (self.form) {
    self.headers['content-type'] = 'application/x-www-form-urlencoded; charset=utf-8'
    self.body = qs.stringify(self.form).toString('utf8')
  }

  if (self.oauth) {
    var form
    if (self.headers['content-type'] && 
        self.headers['content-type'].slice(0, 'application/x-www-form-urlencoded'.length) ===
          'application/x-www-form-urlencoded' 
       ) {
      form = qs.parse(self.body)
    } 
    if (self.uri.query) {
      form = qs.parse(self.uri.query)
    } 
    if (!form) form = {}
    var oa = {}
    for (var i in form) oa[i] = form[i]
    for (var i in self.oauth) oa['oauth_'+i] = self.oauth[i]
    if (!oa.oauth_version) oa.oauth_version = '1.0'
    if (!oa.oauth_timestamp) oa.oauth_timestamp = Math.floor( (new Date()).getTime() / 1000 ).toString()
    if (!oa.oauth_nonce) oa.oauth_nonce = uuid().replace(/-/g, '')
    
    oa.oauth_signature_method = 'HMAC-SHA1'
    
    var consumer_secret = oa.oauth_consumer_secret
    delete oa.oauth_consumer_secret
    var token_secret = oa.oauth_token_secret
    delete oa.oauth_token_secret
    
    var baseurl = self.uri.protocol + '//' + self.uri.host + self.uri.pathname
    var signature = oauth.hmacsign(self.method, baseurl, oa, consumer_secret, token_secret)
    
    // oa.oauth_signature = signature
    for (var i in form) {
      if ( i.slice(0, 'oauth_') in self.oauth) {
        // skip 
      } else {
        delete oa['oauth_'+i]
      }
    }
    self.headers.authorization = 
      'OAuth '+Object.keys(oa).sort().map(function (i) {return i+'="'+oauth.rfc3986(oa[i])+'"'}).join(',')
    self.headers.authorization += ',oauth_signature="'+oauth.rfc3986(signature)+'"'  
  }

  if (self.uri.auth && !self.headers.authorization) {
    self.headers.authorization = "Basic " + toBase64(self.uri.auth.split(':').map(function(item){ return qs.unescape(item)}).join(':'))
  }
  if (self.proxy && self.proxy.auth && !self.headers['proxy-authorization']) {
    self.headers['proxy-authorization'] = "Basic " + toBase64(self.proxy.auth.split(':').map(function(item){ return qs.unescape(item)}).join(':'))
  }

  if (self.uri.path) {
    self.path = self.uri.path
  } else {
    self.path = self.uri.pathname + (self.uri.search || "")
  }

  if (self.path.length === 0) self.path = '/'

  if (self.proxy) self.path = (self.uri.protocol + '//' + self.uri.host + self.path)

  if (self.json) {
    self.headers['content-type'] = 'application/json'
    if (typeof self.json === 'boolean') {
      if (typeof self.body === 'object') self.body = JSON.stringify(self.body)
    } else {
      self.body = JSON.stringify(self.json)
    }

  } else if (self.multipart) {
    self.body = []

    if (!self.headers['content-type']) {
      self.headers['content-type'] = 'multipart/related;boundary="frontier"';
    } else {
      self.headers['content-type'] = self.headers['content-type'].split(';')[0] + ';boundary="frontier"';
    }

    if (!self.multipart.forEach) throw new Error('Argument error, options.multipart.')

    self.multipart.forEach(function (part) {
      var body = part.body
      if(!body) throw Error('Body attribute missing in multipart.')
      delete part.body
      var preamble = '--frontier\r\n'
      Object.keys(part).forEach(function(key){
        preamble += key + ': ' + part[key] + '\r\n'
      })
      preamble += '\r\n'
      self.body.push(new Buffer(preamble))
      self.body.push(new Buffer(body))
      self.body.push(new Buffer('\r\n'))
    })
    self.body.push(new Buffer('--frontier--'))
  }

  if (self.body) {
    var length = 0
    if (!Buffer.isBuffer(self.body)) {
      if (Array.isArray(self.body)) {
        for (var i = 0; i < self.body.length; i++) {
          length += self.body[i].length
        }
      } else {
        self.body = new Buffer(self.body)
        length = self.body.length
      }
    } else {
      length = self.body.length
    }
    if (length) {
      self.headers['content-length'] = length
    } else {
      throw new Error('Argument error, options.body.')
    }
  }

  var protocol = self.proxy ? self.proxy.protocol : self.uri.protocol
    , defaultModules = {'http:':http, 'https:':https}
    , httpModules = self.httpModules || {}
    ;
  self.httpModule = httpModules[protocol] || defaultModules[protocol]

  if (!self.httpModule) throw new Error("Invalid protocol")

  if (self.pool === false) {
    self.agent = false
  } else {
    if (self.maxSockets) {
      // Don't use our pooling if node has the refactored client
      self.agent = self.agent || self.httpModule.globalAgent || self.getAgent(self.host, self.port)
      self.agent.maxSockets = self.maxSockets
    }
    if (self.pool.maxSockets) {
      // Don't use our pooling if node has the refactored client
      self.agent = self.agent || self.httpModule.globalAgent || self.getAgent(self.host, self.port)
      self.agent.maxSockets = self.pool.maxSockets
    }
  }

  self.start = function () {
    self._started = true
    self.method = self.method || 'GET'
    self.href = self.uri.href
    if (log) log('%method %href', self)
    self.req = self.httpModule.request(self, function (response) {
      self.response = response
      response.request = self

      if (self.httpModule === https &&
          self.strictSSL &&
          !response.client.authorized) {
        var sslErr = response.client.authorizationError
        self.emit('error', new Error('SSL Error: '+ sslErr))
        return
      }

      if (setHost) delete self.headers.host
      if (self.timeout && self.timeoutTimer) {
          clearTimeout(self.timeoutTimer);
          self.timeoutTimer = null;
      }  
      
      if (response.headers['set-cookie'] && (!self._disableCookies)) {
        response.headers['set-cookie'].forEach(function(cookie) {
          if (self.jar) self.jar.add(new Cookie(cookie))
          else cookieJar.add(new Cookie(cookie))
        })
      }

      if (response.statusCode >= 300 &&
          response.statusCode < 400  &&
          self.followRedirect     &&
          self.method !== 'PUT' &&
          self.method !== 'POST' &&
          response.headers.location) {
        if (self._redirectsFollowed >= self.maxRedirects) {
          self.emit('error', new Error("Exceeded maxRedirects. Probably stuck in a redirect loop."))
          return
        }
        self._redirectsFollowed += 1

        if (!isUrl.test(response.headers.location)) {
          response.headers.location = url.resolve(self.uri.href, response.headers.location)
        }
        self.uri = response.headers.location
        self.redirects.push(
          { statusCode : response.statusCode
          , redirectUri: response.headers.location 
          }
        )
        delete self.req
        delete self.agent
        delete self._started
        if (self.headers) {
          delete self.headers.host
        }
        if (log) log('Redirect to %uri', self)
        request(self, self.callback)
        return // Ignore the rest of the response
      } else {
        self._redirectsFollowed = self._redirectsFollowed || 0
        // Be a good stream and emit end when the response is finished.
        // Hack to emit end on close because of a core bug that never fires end
        response.on('close', function () {
          if (!self._ended) self.response.emit('end')
        })

        if (self.encoding) {
          if (self.dests.length !== 0) {
            console.error("Ingoring encoding parameter as this stream is being piped to another stream which makes the encoding option invalid.")
          } else {
            response.setEncoding(self.encoding)
          }
        }

        self.pipeDest = function (dest) {
          if (dest.headers) {
            dest.headers['content-type'] = response.headers['content-type']
            if (response.headers['content-length']) {
              dest.headers['content-length'] = response.headers['content-length']
            }
          }
          if (dest.setHeader) {
            for (var i in response.headers) {
              dest.setHeader(i, response.headers[i])
            }
            dest.statusCode = response.statusCode
          }
          if (self.pipefilter) self.pipefilter(response, dest)
        }

        self.dests.forEach(function (dest) {
          self.pipeDest(dest)
        })

        response.on("data", function (chunk) {
          self._destdata = true
          self.emit("data", chunk)
        })
        response.on("end", function (chunk) {
          self._ended = true
          self.emit("end", chunk)
        })
        response.on("close", function () {self.emit("close")})

        self.emit('response', response)

        if (self.onResponse) {
          self.onResponse(null, response)
        }
        if (self.callback) {
          var buffer = []
          var bodyLen = 0
          self.on("data", function (chunk) {
            buffer.push(chunk)
            bodyLen += chunk.length
          })
          self.on("end", function () {
            if (buffer.length && Buffer.isBuffer(buffer[0])) {
              var body = new Buffer(bodyLen)
              var i = 0
              buffer.forEach(function (chunk) {
                chunk.copy(body, i, 0, chunk.length)
                i += chunk.length
              })
              if (self.encoding === null) {
                response.body = body
              } else {
                response.body = body.toString()
              }
            } else if (buffer.length) {
              response.body = buffer.join('')
            }

            if (self.json) {
              try {
                response.body = JSON.parse(response.body)
              } catch (e) {}
            }

            self.callback(null, response, response.body)
          })
        }
      }
    })

    if (self.timeout && !self.timeoutTimer) {
      self.timeoutTimer = setTimeout(function() {
        self.req.abort()
        var e = new Error("ETIMEDOUT")
        e.code = "ETIMEDOUT"
        self.emit("error", e)
      }, self.timeout)
      
      // Set additional timeout on socket - in case if remote
      // server freeze after sending headers
      self.req.setTimeout(self.timeout, function(){
          if (self.req) {
              self.req.abort()
              var e = new Error("ESOCKETTIMEDOUT")
              e.code = "ESOCKETTIMEDOUT"
              self.emit("error", e)
          }
      });
    }
    
    self.req.on('error', clientErrorHandler)
  }

  self.once('pipe', function (src) {
    if (self.ntick) throw new Error("You cannot pipe to this stream after the first nextTick() after creation of the request stream.")
    self.src = src
    if (isReadStream(src)) {
      if (!self.headers['content-type'] && !self.headers['Content-Type'])
        self.headers['content-type'] = mimetypes.lookup(src.path.slice(src.path.lastIndexOf('.')+1))
    } else {
      if (src.headers) {
        for (var i in src.headers) {
          if (!self.headers[i]) {
            self.headers[i] = src.headers[i]
          }
        }
      }
      if (src.method && !self.method) {
        self.method = src.method
      }
    }

    self.on('pipe', function () {
      console.error("You have already piped to this stream. Pipeing twice is likely to break the request.")
    })
  })

  process.nextTick(function () {
    if (self.body) {
      if (Array.isArray(self.body)) {
        self.body.forEach(function(part) {
          self.write(part)
        })
      } else {
        self.write(self.body)
      }
      self.end()
    } else if (self.requestBodyStream) {
      console.warn("options.requestBodyStream is deprecated, please pass the request object to stream.pipe.")
      self.requestBodyStream.pipe(self)
    } else if (!self.src) {
      self.headers['content-length'] = 0
      self.end()
    }
    self.ntick = true
  })
}
Request.prototype.pipe = function (dest) {
  if (this.response) {
    if (this._destdata) {
      throw new Error("You cannot pipe after data has been emitted from the response.")
    } else if (this._ended) {
      throw new Error("You cannot pipe after the response has been ended.")
    } else {
      stream.Stream.prototype.pipe.call(this, dest)
      this.pipeDest(dest)
      return dest
    }
  } else {
    this.dests.push(dest)
    stream.Stream.prototype.pipe.call(this, dest)
    return dest
  }
}
Request.prototype.write = function () {
  if (!this._started) this.start()
  if (!this.req) throw new Error("This request has been piped before http.request() was called.")
  this.req.write.apply(this.req, arguments)
}
Request.prototype.end = function () {
  if (!this._started) this.start()
  if (!this.req) throw new Error("This request has been piped before http.request() was called.")
  this.req.end.apply(this.req, arguments)
}
Request.prototype.pause = function () {
  if (!this.response) throw new Error("This request has been piped before http.request() was called.")
  this.response.pause.apply(this.response, arguments)
}
Request.prototype.resume = function () {
  if (!this.response) throw new Error("This request has been piped before http.request() was called.")
  this.response.resume.apply(this.response, arguments)
}

function request (options, callback) {
  if (typeof options === 'string') options = {uri:options}
  if (callback) options.callback = callback
  var r = new Request(options)
  r.request()
  return r
}

module.exports = request

request.defaults = function (options) {
  var def = function (method) {
    var d = function (opts, callback) {
      if (typeof opts === 'string') opts = {uri:opts}
      for (var i in options) {
        if (opts[i] === undefined) opts[i] = options[i]
      }
      return method(opts, callback)
    }
    return d
  }
  var de = def(request)
  de.get = def(request.get)
  de.post = def(request.post)
  de.put = def(request.put)
  de.head = def(request.head)
  de.del = def(request.del)
  de.cookie = def(request.cookie)
  de.jar = def(request.jar)
  return de
}

request.forever = function (agentOptions, optionsArg) {
  var options = {}
  if (agentOptions) {
    for (option in optionsArg) {
      options[option] = optionsArg[option]
    }
  }
  options.agent = new ForeverAgent(agentOptions)
  return request.defaults(options)
}

request.get = request
request.post = function (options, callback) {
  if (typeof options === 'string') options = {uri:options}
  options.method = 'POST'
  return request(options, callback)
}
request.put = function (options, callback) {
  if (typeof options === 'string') options = {uri:options}
  options.method = 'PUT'
  return request(options, callback)
}
request.head = function (options, callback) {
  if (typeof options === 'string') options = {uri:options}
  options.method = 'HEAD'
  if (options.body || options.requestBodyStream || options.json || options.multipart) {
    throw new Error("HTTP HEAD requests MUST NOT include a request body.")
  }
  return request(options, callback)
}
request.del = function (options, callback) {
  if (typeof options === 'string') options = {uri:options}
  options.method = 'DELETE'
  return request(options, callback)
}
request.jar = function () {
  return new CookieJar
}
request.cookie = function (str) {
  if (typeof str !== 'string') throw new Error("The cookie function only accepts STRING as param")
  return new Cookie(str)
}