request.rb

require 'tempfile'
require 'cgi'
require 'netrc'
require 'set'

begin
  # Use mime/types/columnar if available, for reduced memory usage
  require 'mime/types/columnar'
rescue LoadError
  require 'mime/types'
end

module RestClient
  # This class is used internally by RestClient to send the request, but you can also
  # call it directly if you'd like to use a method not supported by the
  # main API.
  #
  # @example Using {.execute} class method:
  #   RestClient::Request.execute(method: :head, url: 'http://example.com')
  #
  # @example Initializing {#initialize} and then calling {#execute}:
  #   req = RestClient::Request.new(method: :get, url: 'http://example.com', timeout: 5)
  #   req.execute
  #
  # The `:method` and `:url` parameters are required. All others are optional.
  #
  # **See {#initialize} for the full list of available options.**
  #
  # **Deprecation note:**
  # Certain options are accepted as keys in the headers hash, but doing so is
  # deprecated. This misfeature in the RestClient API dates to the original
  # design, where certain options are destructively pulled out of the
  # `:headers` hash that normally contains HTTP request headers. This is
  # because in the top-level helper shortcuts like {RestClient.get}, the only
  # hash argument permitted is the headers hash, so there is no place to put
  # options. For example, while it is currently allowed to pass options like
  # `:params` or `:cookies` as keys inside the `:headers` hash, this is
  # strongly discouraged.
  #
  class Request

    attr_reader :method, :uri, :url, :headers, :payload, :proxy,
                :user, :password, :read_timeout, :max_redirects,
                :open_timeout, :raw_response, :processed_headers, :args,
                :ssl_opts

    # An array of previous redirection responses
    attr_accessor :redirection_history

    # Shorthand for initializing a Request and executing it.
    #
    # `RestClient::Request.execute` is the recommended way to pass complex
    # options. It is shorthand for `RestClient::Request.new(args).execute`.
    #
    # @example
    #   RestClient::Request.execute(method: :get, url: 'http://example.com', timeout: 5)
    #
    # @example
    #   RestClient::Request.execute(method: :get, url: 'http://httpbin.org/redirect/2', max_redirects: 1)
    #
    # @see RestClient::Request#initialize
    #
    # @return [RestClient::Response, RestClient::RawResponse]
    #
    def self.execute(args, & block)
      new(args).execute(& block)
    end

    SSLOptionList = %w{client_cert client_key ca_file ca_path cert_store
                       version ciphers verify_callback verify_callback_warnings}

    def inspect
      "<RestClient::Request @method=#{@method.inspect}, @url=#{@url.inspect}>"
    end

    # Create a new Request object. This will not make a connection to the
    # server to send the request until {#execute} is called.
    #
    # The `:url` and `:method` are required parameters.
    #
    # @param [Hash] args
    #
    # @option args [String] :url **Required.** The HTTP URL to request.
    # @option args [String, Symbol] :method **Required.** The HTTP request method
    #   or verb, such as `"GET"`, `"HEAD"`, or `"POST"`.
    #
    # @option args [Hash] :headers The HTTP request headers. Keys may be
    #   Symbol or String. Symbol keys will be converted to String header names
    #   by {#stringify_headers}. For backwards compatibility, this Hash
    #   recognizes certain keys that will be pulled out as options, but relying
    #   on this behavior is deprecated and strongly discouraged.
    # @option args :cookies [HTTP::CookieJar, Hash{String, Symbol => String},
    #   Array<HTTP::Cookie>] The cookies to be sent with the request. This can
    #   be passed as a Hash, an array of HTTP::Cookie objects, or as a full
    #   HTTP::CookieJar. Regardless, they will be processed into a cookie jar
    #   before the request is sent. {#cookie_jar}
    # @option args [String] :user The username used for HTTP Basic
    #   Authentication. A username/password in the `:url` takes precedence over
    #   this option, which takes precedence over a netrc file.
    # @option args [String] :password The password used for HTTP Basic
    #   Authentication. A username/password in the `:url` takes precedence over
    #   this option, which takes precedence over a netrc file.
    # @option args [Proc] :block_response When this is passed, the normal HTTP
    #   response / exception processing will be bypassed. The provided block
    #   will be called with the raw {Net::HTTPResponse} object returned by
    #   {Net::HTTP}, allowing for fully custom response handling.
    # @option args [Boolean] :raw_response Return a low-level {RawResponse}
    #   instead of a {Response}. This is good for streaming large response
    #   downloads, because the response will be downloaded directly to a
    #   {Tempfile} rather than loaded into memory. The normal error handling
    #   and exceptions still run as usual.
    # @option args [Logger, #<<] Set the log for this request only, overriding
    #   RestClient.log (if any). Accepts any object that implements a `<<`
    #   method, such as a Logger, file handle, or other IO.
    # @option args [Integer] :stream_log_percent (10) Only relevant with
    #   `:raw_response => true`. Customize the interval at which download
    #   progress is logged.
    # @option args [Integer] :max_redirects (10) Set the maximum number of
    #   redirections to follow. Set this to 0 to disable following redirects or
    #   to handle redirects manually.
    # @option args [String, nil] :proxy An HTTP proxy URI to use for this
    #   request. Any value here (including nil) will override
    #   {RestClient.proxy}.
    # @option args [Boolean, Integer] :verify_ssl (OpenSSL::SSL::VERIFY_PEER)
    #   Enable ssl verification, possible values are constants from
    #   OpenSSL::SSL::VERIFY_*, defaults to verifying SSL. There is little
    #   point to using HTTPS at all without verifying certificates.
    # @option args [Numeric, nil] :read_timeout Number of seconds to wait for server
    #   to respond with data after establishing the connection. This sets a
    #   timeout on an individual network read, but does not limit the overall
    #   duration of the request so long as the server continues sending data at
    #   a trickle. Pass nil to disable the timeout. See
    #   `Net::HTTP#read_timeout`.
    # @option args [Numeric, nil] :open_timeout Number of seconds to wait for
    #   the connection to be established. Pass nil to disable the timeout. See
    #   `Net::HTTP#open_timeout`.
    # @option args [Numeric, nil] :timeout Set both `:read_timeout` and
    #   `:open_timeout`
    # @option args :ssl_client_cert
    # @option args :ssl_client_key
    # @option args :ssl_ca_file
    # @option args :ssl_ca_path
    # @option args :ssl_cert_store
    # @option args :ssl_verify_callback
    # @option args :ssl_verify_callback_warnings
    # @option args :ssl_version Set the SSL version for the underlying
    #   Net::HTTP connection.
    # @option args :ssl_ciphers Set SSL ciphers for the connection. See
    #     {OpenSSL::SSL::SSLContext#ciphers=}
    # @option args [Proc] :before_execution_proc A Proc to call before
    #   executing the request. This proc, like procs from
    #   {RestClient.before_execution_procs}, will be called with the HTTP
    #   request and request params.
    #
    def initialize(args)
      @method = normalize_method(args[:method])
      @headers = (args[:headers] || {}).dup
      if args[:url]
        @url = process_url_params(normalize_url(args[:url]), headers)
      else
        raise ArgumentError, "must pass :url"
      end

      @user = @password = nil
      parse_url_with_auth!(url)

      # process cookie arguments found in headers or args
      @cookie_jar = process_cookie_args!(@uri, @headers, args)

      @payload = Payload.generate(args[:payload])

      @user = args[:user] if args.include?(:user)
      @password = args[:password] if args.include?(:password)

      if args.include?(:timeout)
        @read_timeout = args[:timeout]
        @open_timeout = args[:timeout]
      end
      if args.include?(:read_timeout)
        @read_timeout = args[:read_timeout]
      end
      if args.include?(:open_timeout)
        @open_timeout = args[:open_timeout]
      end
      @block_response = args[:block_response]
      @raw_response = args[:raw_response] || false

      @stream_log_percent = args[:stream_log_percent] || 10
      if @stream_log_percent <= 0 || @stream_log_percent > 100
        raise ArgumentError.new(
          "Invalid :stream_log_percent #{@stream_log_percent.inspect}")
      end

      @proxy = args.fetch(:proxy) if args.include?(:proxy)

      @ssl_opts = {}

      if args.include?(:verify_ssl)
        v_ssl = args.fetch(:verify_ssl)
        if v_ssl
          if v_ssl == true
            # interpret :verify_ssl => true as VERIFY_PEER
            @ssl_opts[:verify_ssl] = OpenSSL::SSL::VERIFY_PEER
          else
            # otherwise pass through any truthy values
            @ssl_opts[:verify_ssl] = v_ssl
          end
        else
          # interpret all falsy :verify_ssl values as VERIFY_NONE
          @ssl_opts[:verify_ssl] = OpenSSL::SSL::VERIFY_NONE
        end
      else
        # if :verify_ssl was not passed, default to VERIFY_PEER
        @ssl_opts[:verify_ssl] = OpenSSL::SSL::VERIFY_PEER
      end

      SSLOptionList.each do |key|
        source_key = ('ssl_' + key).to_sym
        if args.has_key?(source_key)
          @ssl_opts[key.to_sym] = args.fetch(source_key)
        end
      end

      # Set some other default SSL options, but only if we have an HTTPS URI.
      if use_ssl?

        # If there's no CA file, CA path, or cert store provided, use default
        if !ssl_ca_file && !ssl_ca_path && !@ssl_opts.include?(:cert_store)
          @ssl_opts[:cert_store] = self.class.default_ssl_cert_store
        end
      end

      @log = args[:log]
      @max_redirects = args[:max_redirects] || 10
      @processed_headers = make_headers headers
      @processed_headers_lowercase = Hash[@processed_headers.map {|k, v| [k.downcase, v]}]
      @args = args

      @before_execution_proc = args[:before_execution_proc]
    end

    def execute & block
      # With 2.0.0+, net/http accepts URI objects in requests and handles wrapping
      # IPv6 addresses in [] for use in the Host request header.
      transmit uri, net_http_request_class(method).new(uri, processed_headers), payload, & block
    ensure
      payload.close if payload
    end

    # SSL-related options
    def verify_ssl
      @ssl_opts.fetch(:verify_ssl)
    end
    SSLOptionList.each do |key|
      define_method('ssl_' + key) do
        @ssl_opts[key.to_sym]
      end
    end

    # Return true if the request URI will use HTTPS.
    #
    # @return [Boolean]
    #
    def use_ssl?
      uri.is_a?(URI::HTTPS)
    end

    # Extract the query parameters and append them to the url
    #
    # Look through the headers hash for a `:params` option (case-insensitive,
    # may be string or symbol). If present and the value is a Hash or
    # RestClient::ParamsArray, *delete* the key/value pair from the headers
    # hash and encode the value into a query string. Append this query string
    # to the URL and return the resulting URL.
    #
    # @param [String] url
    # @param [Hash] headers An options/headers hash to process. Mutation
    #   warning: the `params` key may be removed if present!
    #
    # @return [String] resulting url with query string
    #
    # @api private
    #
    def process_url_params(url, headers)
      url_params = nil

      # find and extract/remove "params" key if the value is a Hash/ParamsArray
      headers.delete_if do |key, value|
        if key.to_s.downcase == 'params' &&
            (value.is_a?(Hash) || value.is_a?(RestClient::ParamsArray))
          if url_params
            raise ArgumentError.new("Multiple 'params' options passed")
          end
          url_params = value
          true
        else
          false
        end
      end

      # build resulting URL with query string
      if url_params && !url_params.empty?
        query_string = RestClient::Utils.encode_query_string(url_params)

        if url.include?('?')
          url + '&' + query_string
        else
          url + '?' + query_string
        end
      else
        url
      end
    end

    # Render a hash of key => value pairs for cookies in the Request#cookie_jar
    # that are valid for the Request#uri. This will not necessarily include all
    # cookies if there are duplicate keys. It's safer to use the cookie_jar
    # directly if that's a concern.
    #
    # @see Request#cookie_jar
    #
    # @return [Hash]
    #
    def cookies
      hash = {}

      @cookie_jar.cookies(uri).each do |c|
        hash[c.name] = c.value
      end

      hash
    end

    # @return [HTTP::CookieJar]
    def cookie_jar
      @cookie_jar
    end

    # Render a Cookie HTTP request header from the contents of the @cookie_jar,
    # or nil if the jar is empty.
    #
    # @see Request#cookie_jar
    #
    # @return [String, nil]
    #
    def make_cookie_header
      return nil if cookie_jar.nil?

      arr = cookie_jar.cookies(url)
      return nil if arr.empty?

      return HTTP::Cookie.cookie_value(arr)
    end

    # Process cookies passed as hash or as HTTP::CookieJar. For backwards
    # compatibility, these may be passed as a `:cookies` option masquerading
    # inside the headers hash. To avoid confusion, if `:cookies` is passed in
    # both headers and {#initialize}, raise an error.
    #
    # `:cookies` may be a:
    #
    # - `Hash{String/Symbol => String}`
    # - `Array<HTTP::Cookie>`
    # - `HTTP::CookieJar`
    #
    # **Passing as a hash:**
    #
    #   Keys may be symbols or strings. Values must be strings.
    #   Infer the domain name from the request URI and allow subdomains (as
    #   though '.example.com' had been set in a `Set-Cookie` header). Assume a
    #   path of '/'.
    #
    #     RestClient::Request.new(url: 'http://example.com', method: :get,
    #       :cookies => {:foo => 'Value', 'bar' => '123'}
    #     )
    #
    # results in cookies as though set from the server by:
    #
    #     Set-Cookie: foo=Value; Domain=.example.com; Path=/
    #     Set-Cookie: bar=123; Domain=.example.com; Path=/
    #
    # which yields a client cookie header of:
    #
    #     Cookie: foo=Value; bar=123
    #
    # **Passing as HTTP::CookieJar, which will be passed through directly:**
    #
    #     jar = HTTP::CookieJar.new
    #     jar.add(HTTP::Cookie.new('foo', 'Value', domain: 'example.com',
    #                              path: '/', for_domain: false))
    #
    #     RestClient::Request.new('...', :cookies => jar)
    #
    # @param [URI::HTTP] uri The URI for the request. This will be used to
    #   infer the domain name for cookies passed as strings in a hash. To avoid
    #   this implicit behavior, pass a full cookie jar or use `HTTP::Cookie`
    #   hash values.
    # @param [Hash] headers The headers hash from which to pull the `:cookies`
    #   option. **MUTATION NOTE:** This key will be deleted from the hash if
    #   present. Passing cookies in this way is deprecated.
    # @param [Hash] args The options passed to {Request#initialize}. This hash
    #   will be used as another potential source for the :cookies key.
    #   These args will not be mutated.
    #
    # @return [HTTP::CookieJar] A cookie jar containing the parsed cookies.
    #
    def process_cookie_args!(uri, headers, args)

      # Avoid ambiguity in whether options from headers or options from
      # Request#initialize should take precedence by raising ArgumentError when
      # both are present. Prior versions of rest-client claimed to give
      # precedence to init options, but actually gave precedence to headers.
      # Avoid that mess by erroring out instead.
      if headers[:cookies] && args[:cookies]
        raise ArgumentError.new(
          "Cannot pass :cookies in Request.new() and in headers hash")
      end

      cookies_data = headers.delete(:cookies) || args[:cookies]

      # return copy of cookie jar as is
      if cookies_data.is_a?(HTTP::CookieJar)
        return cookies_data.dup
      end

      # convert cookies hash into a CookieJar
      jar = HTTP::CookieJar.new

      (cookies_data || []).each do |key, val|

        # Support for Array<HTTP::Cookie> mode:
        # If key is a cookie object, add it to the jar directly and assert that
        # there is no separate val.
        if key.is_a?(HTTP::Cookie)
          if val
            raise ArgumentError.new("extra cookie val: #{val.inspect}")
          end

          jar.add(key)
          next
        end

        if key.is_a?(Symbol)
          key = key.to_s
        end

        # assume implicit domain from the request URI, and set for_domain to
        # permit subdomains
        jar.add(HTTP::Cookie.new(key, val, domain: uri.hostname.downcase,
                                 path: '/', for_domain: true))
      end

      jar
    end

    # Generate headers for use by a request. Header keys will be stringified
    # using {#stringify_headers} to normalize them as capitalized strings.
    #
    # The final headers consist of:
    #
    #   - default headers from {#default_headers}
    #   - `user_headers` provided here
    #   - headers from the payload object (e.g. Content-Type, Content-Lenth)
    #   - cookie headers from {#make_cookie_header}
    #
    # **BUG:** stringify_headers does not alter the capitalization of headers
    # that are passed as strings, it only normalizes those passed as symbols.
    # This behavior will probably remain for a while for compatibility, but it
    # means that the warnings that attempt to detect accidental header
    # overrides may not always work.
    # https://github.com/rest-client/rest-client/issues/599
    #
    # @param [Hash] user_headers User-provided headers to include
    #
    # @return [Hash<String, String>] A hash of HTTP headers => values
    #
    def make_headers(user_headers)
      headers = stringify_headers(default_headers).merge(stringify_headers(user_headers))

      # override headers from the payload (e.g. Content-Type, Content-Length)
      if @payload
        payload_headers = @payload.headers

        # Warn the user if we override any headers that were previously
        # present. This usually indicates that rest-client was passed
        # conflicting information, e.g. if it was asked to render a payload as
        # x-www-form-urlencoded but a Content-Type application/json was
        # also supplied by the user.
        payload_headers.each_pair do |key, val|
          if headers.include?(key) && headers[key] != val
            warn("warning: Overriding #{key.inspect} header " +
                 "#{headers.fetch(key).inspect} with #{val.inspect} " +
                 "due to payload")
          end
        end

        headers.merge!(payload_headers)
      end

      # merge in cookies
      cookies = make_cookie_header
      if cookies && !cookies.empty?
        if headers['Cookie']
          warn('warning: overriding "Cookie" header with :cookies option')
        end
        headers['Cookie'] = cookies
      end

      headers
    end

    # The proxy URI for this request. If `:proxy` was provided on this request,
    # use it over {RestClient.proxy}.
    #
    # Return false if a proxy was explicitly set and is falsy.
    #
    # @return [URI, false, nil]
    #
    def proxy_uri
      if defined?(@proxy)
        if @proxy
          URI.parse(@proxy)
        else
          false
        end
      elsif RestClient.proxy_set?
        if RestClient.proxy
          URI.parse(RestClient.proxy)
        else
          false
        end
      else
        nil
      end
    end

    # Create a new Net::HTTP object representing a connection to a server
    # without actually opening the connection. This method will set up an HTTP
    # proxy according to {#proxy_uri}.
    #
    # @param hostname [String]
    # @param port [Integer]
    #
    # @return [Net::HTTP]
    #
    # @api private
    #
    def net_http_object(hostname, port)
      p_uri = proxy_uri

      if p_uri.nil?
        # no proxy set
        Net::HTTP.new(hostname, port)
      elsif !p_uri
        # proxy explicitly set to none
        Net::HTTP.new(hostname, port, nil, nil, nil, nil)
      else
        Net::HTTP.new(hostname, port,
                      p_uri.hostname, p_uri.port, p_uri.user, p_uri.password)

      end
    end

    # Find the Net::HTTPRequest subclass for a given HTTP method/verb.
    #
    # @param method [Symbol, String]
    #
    # @return [Class] A subclass of Net::HTTPRequest.
    #
    # @api private
    #
    def net_http_request_class(method)
      Net::HTTP.const_get(method.capitalize, false)
    end

    # Actually execute the request with Net::HTTP.
    #
    # @param http [Net::HTTP]
    # @param req [Net::HTTPRequest]
    # @param body [String, IO, nil]
    #
    # @see https://ruby-doc.org/stdlib-2.4.0/libdoc/net/http/rdoc/Net/HTTP.html#method-i-request
    #   Net::HTTP#request
    #
    # @api private
    def net_http_do_request(http, req, body=nil, &block)
      if body && body.respond_to?(:read)
        req.body_stream = body
        return http.request(req, nil, &block)
      else
        return http.request(req, body, &block)
      end
    end

    # Normalize a URL by adding a protocol if none is present.
    #
    # If the string has no HTTP-like scheme (i.e. scheme followed by '//'), a
    # scheme of 'http' will be added. This mimics the behavior of browsers and
    # user agents like cURL.
    #
    # @param [String] url A URL string.
    #
    # @return [String]
    #
    # @api private
    #
    def normalize_url(url)
      url = 'http://' + url unless url.match(%r{\A[a-z][a-z0-9+.-]*://}i)
      url
    end

    # Return a certificate store that can be used to validate certificates with
    # the system certificate authorities. This will probably not do anything on
    # OS X, which monkey patches OpenSSL in terrible ways to insert its own
    # validation. On most *nix platforms, this will add the system certifcates
    # using OpenSSL::X509::Store#set_default_paths. On Windows, this will use
    # RestClient::Windows::RootCerts to look up the CAs trusted by the system.
    #
    # @return [OpenSSL::X509::Store]
    #
    def self.default_ssl_cert_store
      cert_store = OpenSSL::X509::Store.new
      cert_store.set_default_paths

      # set_default_paths() doesn't do anything on Windows, so look up
      # certificates using the win32 API.
      if RestClient::Platform.windows?
        RestClient::Windows::RootCerts.instance.to_a.uniq.each do |cert|
          begin
            cert_store.add_cert(cert)
          rescue OpenSSL::X509::StoreError => err
            # ignore duplicate certs
            raise unless err.message == 'cert already in hash table'
          end
        end
      end

      cert_store
    end

    def redacted_uri
      if uri.password
        sanitized_uri = uri.dup
        sanitized_uri.password = 'REDACTED'
        sanitized_uri
      else
        uri
      end
    end

    def redacted_url
      redacted_uri.to_s
    end

    # The log used for this request.
    #
    # Defaults to the global logger {RestClient.log} if there was no `:log`
    # option set on this request.
    #
    def log
      @log || RestClient.log
    end

    # Write log information about the request. Called just prior to sending the
    # request to the server.
    #
    # @return [void]
    #
    def log_request
      return unless log

      out = []

      out << "RestClient.#{method} #{redacted_url.inspect}"
      out << payload.short_inspect if payload
      out << processed_headers.to_a.sort.map { |(k, v)| [k.inspect, v.inspect].join("=>") }.join(", ")
      log << out.join(', ') + "\n"
    end

    # Return a hash of headers whose keys are capitalized strings
    #
    # BUG: stringify_headers does not fix the capitalization of headers that
    # are already Strings. Leaving this behavior as is for now for
    # backwards compatibility.
    # https://github.com/rest-client/rest-client/issues/599
    #
    def stringify_headers headers
      headers.inject({}) do |result, (key, value)|
        if key.is_a? Symbol
          key = key.to_s.split(/_/).map(&:capitalize).join('-')
        end
        if 'CONTENT-TYPE' == key.upcase
          result[key] = maybe_convert_extension(value.to_s)
        elsif 'ACCEPT' == key.upcase
          # Accept can be composed of several comma-separated values
          if value.is_a? Array
            target_values = value
          else
            target_values = value.to_s.split ','
          end
          result[key] = target_values.map { |ext|
            maybe_convert_extension(ext.to_s.strip)
          }.join(', ')
        else
          result[key] = value.to_s
        end
        result
      end
    end

    # Default headers set by RestClient. In addition to these headers, servers
    # will receive headers set by Net::HTTP, such as Accept-Encoding and Host.
    #
    # @return [Hash<Symbol, String>]
    def default_headers
      {
        :accept => '*/*',
        :user_agent => RestClient::Platform.default_user_agent,
      }
    end

    private

    # Parse the `@url` string into a URI object and save it as
    # `@uri`. Also save any basic auth user or password as @user and @password.
    # If no auth info was passed, check for credentials in a Netrc file.
    #
    # @param [String] url A URL string.
    #
    # @return [URI]
    #
    # @raise URI::InvalidURIError on invalid URIs
    #
    def parse_url_with_auth!(url)
      uri = URI.parse(url)

      if uri.hostname.nil?
        raise URI::InvalidURIError.new("bad URI(no host provided): #{url}")
      end

      @user = CGI.unescape(uri.user) if uri.user
      @password = CGI.unescape(uri.password) if uri.password
      if !@user && !@password
        @user, @password = Netrc.read[uri.hostname]
      end

      @uri = uri
    end

    def print_verify_callback_warnings
      warned = false
      if RestClient::Platform.mac_mri?
        warn('warning: ssl_verify_callback return code is ignored on OS X')
        warned = true
      end
      if RestClient::Platform.jruby?
        warn('warning: SSL verify_callback may not work correctly in jruby')
        warn('see https://github.com/jruby/jruby/issues/597')
        warned = true
      end
      warned
    end

    # Parse a method and return a normalized string version.
    #
    # Raise ArgumentError if the method is falsy, but otherwise do no
    # validation.
    #
    # @param method [String, Symbol]
    #
    # @return [String]
    #
    # @see net_http_request_class
    #
    def normalize_method(method)
      raise ArgumentError.new('must pass :method') unless method
      method.to_s.downcase
    end

    def transmit uri, req, payload, & block

      # We set this to true in the net/http block so that we can distinguish
      # read_timeout from open_timeout. Now that we only support Ruby 2.0+,
      # this is only needed for Timeout exceptions thrown outside of Net::HTTP.
      established_connection = false

      setup_credentials req

      net = net_http_object(uri.hostname, uri.port)
      net.use_ssl = uri.is_a?(URI::HTTPS)
      net.ssl_version = ssl_version if ssl_version
      net.ciphers = ssl_ciphers if ssl_ciphers

      net.verify_mode = verify_ssl

      net.cert = ssl_client_cert if ssl_client_cert
      net.key = ssl_client_key if ssl_client_key
      net.ca_file = ssl_ca_file if ssl_ca_file
      net.ca_path = ssl_ca_path if ssl_ca_path
      net.cert_store = ssl_cert_store if ssl_cert_store

      # We no longer rely on net.verify_callback for the main SSL verification
      # because it's not well supported on all platforms (see comments below).
      # But do allow users to set one if they want.
      if ssl_verify_callback
        net.verify_callback = ssl_verify_callback

        # Hilariously, jruby only calls the callback when cert_store is set to
        # something, so make sure to set one.
        # https://github.com/jruby/jruby/issues/597
        if RestClient::Platform.jruby?
          net.cert_store ||= OpenSSL::X509::Store.new
        end

        if ssl_verify_callback_warnings != false
          if print_verify_callback_warnings
            warn('pass :ssl_verify_callback_warnings => false to silence this')
          end
        end
      end

      if OpenSSL::SSL::VERIFY_PEER == OpenSSL::SSL::VERIFY_NONE
        warn('WARNING: OpenSSL::SSL::VERIFY_PEER == OpenSSL::SSL::VERIFY_NONE')
        warn('This dangerous monkey patch leaves you open to MITM attacks!')
        warn('Try passing :verify_ssl => false instead.')
      end

      if defined? @read_timeout
        if @read_timeout == -1
          warn 'Deprecated: to disable timeouts, please use nil instead of -1'
          @read_timeout = nil
        end
        net.read_timeout = @read_timeout
      end
      if defined? @open_timeout
        if @open_timeout == -1
          warn 'Deprecated: to disable timeouts, please use nil instead of -1'
          @open_timeout = nil
        end
        net.open_timeout = @open_timeout
      end

      RestClient.before_execution_procs.each do |before_proc|
        before_proc.call(req, args)
      end

      if @before_execution_proc
        @before_execution_proc.call(req, args)
      end

      log_request

      start_time = Time.now
      tempfile = nil

      net.start do |http|
        established_connection = true

        if @block_response
          net_http_do_request(http, req, payload, &@block_response)
        else
          res = net_http_do_request(http, req, payload) { |http_response|
            if @raw_response
              # fetch body into tempfile
              tempfile = fetch_body_to_tempfile(http_response)
            else
              # fetch body
              http_response.read_body
            end
            http_response
          }
          process_result(res, start_time, tempfile, &block)
        end
      end
    rescue EOFError
      raise RestClient::ServerBrokeConnection
    rescue Net::OpenTimeout => err
      raise RestClient::Exceptions::OpenTimeout.new(nil, err)
    rescue Net::ReadTimeout => err
      raise RestClient::Exceptions::ReadTimeout.new(nil, err)
    rescue Timeout::Error, Errno::ETIMEDOUT => err
      # handling for non-Net::HTTP timeouts
      if established_connection
        raise RestClient::Exceptions::ReadTimeout.new(nil, err)
      else
        raise RestClient::Exceptions::OpenTimeout.new(nil, err)
      end

    rescue OpenSSL::SSL::SSLError => error
      # TODO: deprecate and remove RestClient::SSLCertificateNotVerified and just
      # pass through OpenSSL::SSL::SSLError directly.
      #
      # Exceptions in verify_callback are ignored [1], and jruby doesn't support
      # it at all [2]. RestClient has to catch OpenSSL::SSL::SSLError and either
      # re-throw it as is, or throw SSLCertificateNotVerified based on the
      # contents of the message field of the original exception.
      #
      # The client has to handle OpenSSL::SSL::SSLError exceptions anyway, so
      # we shouldn't make them handle both OpenSSL and RestClient exceptions.
      #
      # [1] https://github.com/ruby/ruby/blob/89e70fe8e7/ext/openssl/ossl.c#L238
      # [2] https://github.com/jruby/jruby/issues/597

      if error.message.include?("certificate verify failed")
        raise SSLCertificateNotVerified.new(error.message)
      else
        raise error
      end
    end

    def setup_credentials(req)
      if user && !@processed_headers_lowercase.include?('authorization')
        req.basic_auth(user, password)
      end
    end

    def fetch_body_to_tempfile(http_response)
      # Taken from Chef, which as in turn...
      # Stolen from http://www.ruby-forum.com/topic/166423
      # Kudos to _why!
      tf = Tempfile.new('rest-client.')
      tf.binmode

      size = 0
      total = http_response['Content-Length'].to_i
      stream_log_bucket = nil

      http_response.read_body do |chunk|
        tf.write chunk
        size += chunk.size
        if log
          if total == 0
            log << "streaming %s %s (%d of unknown) [0 Content-Length]\n" % [@method.upcase, @url, size]
          else
            percent = (size * 100) / total
            current_log_bucket, _ = percent.divmod(@stream_log_percent)
            if current_log_bucket != stream_log_bucket
              stream_log_bucket = current_log_bucket
              log << "streaming %s %s %d%% done (%d of %d)\n" % [@method.upcase, @url, (size * 100) / total, size, total]
            end
          end
        end
      end
      tf.close
      tf
    end

    # @param res The Net::HTTP response object
    # @param start_time [Time] Time of request start
    def process_result(res, start_time, tempfile=nil, &block)
      if @raw_response
        unless tempfile
          raise ArgumentError.new('tempfile is required')
        end
        response = RawResponse.new(tempfile, res, self, start_time)
      else
        response = Response.create(res.body, res, self, start_time)
      end

      response.log_response

      if block_given?
        block.call(response, self, res, & block)
      else
        response.return!(&block)
      end

    end

    def parser
      URI.const_defined?(:Parser) ? URI::Parser.new : URI
    end

    # Given a MIME type or file extension, return either a MIME type or, if
    # none is found, the input unchanged.
    #
    #     >> maybe_convert_extension('json')
    #     => 'application/json'
    #
    #     >> maybe_convert_extension('unknown')
    #     => 'unknown'
    #
    #     >> maybe_convert_extension('application/xml')
    #     => 'application/xml'
    #
    # @param ext [String]
    #
    # @return [String]
    #
    def maybe_convert_extension(ext)
      unless ext =~ /\A[a-zA-Z0-9_@-]+\z/
        # Don't look up strings unless they look like they could be a file
        # extension known to mime-types.
        #
        # There currently isn't any API public way to look up extensions
        # directly out of MIME::Types, but the type_for() method only strips
        # off after a period anyway.
        return ext
      end

      types = MIME::Types.type_for(ext)
      if types.empty?
        ext
      else
        types.first.content_type
      end
    end
  end
end