Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Fetching contributors…

Cannot retrieve contributors at this time

1019 lines (884 sloc) 25.42 kB
# HTTPClient - HTTP client library.
# Copyright (C) 2000-2009 NAKAMURA, Hiroshi <nahi@ruby-lang.org>.
#
# This program is copyrighted free software by NAKAMURA, Hiroshi. You can
# redistribute it and/or modify it under the same terms of Ruby's license;
# either the dual license version in 2003, or any later version.
#
# httpclient/session.rb is based on http-access.rb in http-access/0.0.4. Some
# part of it is copyrighted by Maebashi-san who made and published
# http-access/0.0.4. http-access/0.0.4 did not include license notice but when
# I asked Maebashi-san he agreed that I can redistribute it under the same terms
# of Ruby. Many thanks to Maebashi-san.
require 'socket'
require 'thread'
require 'stringio'
require 'zlib'
require 'httpclient/timeout'
require 'httpclient/ssl_config'
require 'httpclient/http'
class HTTPClient
# Represents a Site: protocol scheme, host String and port Number.
class Site
# Protocol scheme.
attr_accessor :scheme
# Host String.
attr_accessor :host
# Port number.
attr_accessor :port
# Creates a new Site based on the given URI.
def initialize(uri = nil)
if uri
@scheme = uri.scheme
@host = uri.host
@port = uri.port.to_i
else
@scheme = 'tcp'
@host = '0.0.0.0'
@port = 0
end
end
# Returns address String.
def addr
"#{@scheme}://#{@host}:#{@port.to_s}"
end
# Returns true is scheme, host and port are '=='
def ==(rhs)
(@scheme == rhs.scheme) and (@host == rhs.host) and (@port == rhs.port)
end
# Same as ==.
def eql?(rhs)
self == rhs
end
def hash # :nodoc:
[@scheme, @host, @port].hash
end
def to_s # :nodoc:
addr
end
# Returns true if scheme, host and port of the given URI matches with this.
def match(uri)
(@scheme == uri.scheme) and (@host == uri.host) and (@port == uri.port.to_i)
end
def inspect # :nodoc:
sprintf("#<%s:0x%x %s>", self.class.name, __id__, addr)
end
EMPTY = Site.new.freeze
end
# Manages sessions for a HTTPClient instance.
class SessionManager
# Name of this client. Used for 'User-Agent' header in HTTP request.
attr_accessor :agent_name
# Owner of this client. Used for 'From' header in HTTP request.
attr_accessor :from
# Requested protocol version
attr_accessor :protocol_version
# Chunk size for chunked request
attr_accessor :chunk_size
# Device for dumping log for debugging
attr_accessor :debug_dev
# Boolean value for Socket#sync
attr_accessor :socket_sync
attr_accessor :connect_timeout
# Maximum retry count. 0 for infinite.
attr_accessor :connect_retry
attr_accessor :send_timeout
attr_accessor :receive_timeout
attr_accessor :keep_alive_timeout
attr_accessor :read_block_size
attr_accessor :protocol_retry_count
# Local address to bind local side of the socket to
attr_accessor :socket_local
attr_accessor :ssl_config
attr_reader :test_loopback_http_response
attr_accessor :transparent_gzip_decompression
def initialize(client)
@client = client
@proxy = client.proxy
@agent_name = nil
@from = nil
@protocol_version = nil
@debug_dev = client.debug_dev
@socket_sync = true
@chunk_size = ::HTTP::Message::Body::DEFAULT_CHUNK_SIZE
@connect_timeout = 60
@connect_retry = 1
@send_timeout = 120
@receive_timeout = 60 # For each read_block_size bytes
@keep_alive_timeout = 15 # '15' is from Apache 2 default
@read_block_size = 1024 * 16 # follows net/http change in 1.8.7
@protocol_retry_count = 5
@ssl_config = nil
@test_loopback_http_response = []
@transparent_gzip_decompression = false
@socket_local = Site.new
@sess_pool = {}
@sess_pool_mutex = Mutex.new
@sess_pool_last_checked = Time.now
end
def proxy=(proxy)
if proxy.nil?
@proxy = nil
else
@proxy = Site.new(proxy)
end
end
def query(req, via_proxy)
req.http_body.chunk_size = @chunk_size
sess = open(req.header.request_uri, via_proxy)
begin
sess.query(req)
rescue
sess.close
raise
end
sess
end
def reset(uri)
site = Site.new(uri)
close(site)
end
def reset_all
close_all
end
# assert: sess.last_used must not be nil
def keep(sess)
add_cached_session(sess)
end
def invalidate(site)
@sess_pool_mutex.synchronize do
if pool = @sess_pool[site]
pool.each do |sess|
sess.invalidate
end
end
end
end
private
def open(uri, via_proxy = false)
site = Site.new(uri)
sess = nil
if cached = get_cached_session(site)
sess = cached
else
sess = Session.new(@client, site, @agent_name, @from)
sess.proxy = via_proxy ? @proxy : nil
sess.socket_sync = @socket_sync
sess.requested_version = @protocol_version if @protocol_version
sess.connect_timeout = @connect_timeout
sess.connect_retry = @connect_retry
sess.send_timeout = @send_timeout
sess.receive_timeout = @receive_timeout
sess.read_block_size = @read_block_size
sess.protocol_retry_count = @protocol_retry_count
sess.ssl_config = @ssl_config
sess.debug_dev = @debug_dev
sess.socket_local = @socket_local
sess.test_loopback_http_response = @test_loopback_http_response
sess.transparent_gzip_decompression = @transparent_gzip_decompression
end
sess
end
def close_all
@sess_pool_mutex.synchronize do
@sess_pool.each do |site, pool|
pool.each do |sess|
sess.close
end
end
end
@sess_pool.clear
end
# This method might not work as you expected...
def close(dest)
if cached = get_cached_session(Site.new(dest))
cached.close
true
else
false
end
end
def get_cached_session(site)
@sess_pool_mutex.synchronize do
now = Time.now
if now > @sess_pool_last_checked + @keep_alive_timeout
scrub_cached_session(now)
@sess_pool_last_checked = now
end
if pool = @sess_pool[site]
pool.each_with_index do |sess, idx|
if valid_session?(sess, now)
return pool.slice!(idx)
end
end
end
end
nil
end
def scrub_cached_session(now)
@sess_pool.each do |site, pool|
pool.replace(pool.select { |sess|
if valid_session?(sess, now)
true
else
sess.close # close & remove from the pool
false
end
})
end
end
def valid_session?(sess, now)
!sess.invalidated? and (now <= sess.last_used + @keep_alive_timeout)
end
def add_cached_session(sess)
@sess_pool_mutex.synchronize do
(@sess_pool[sess.dest] ||= []).unshift(sess)
end
end
end
# Wraps up OpenSSL::SSL::SSLSocket and offers debugging features.
class SSLSocketWrap
def initialize(socket, context, debug_dev = nil)
unless SSLEnabled
raise ConfigurationError.new('Ruby/OpenSSL module is required')
end
@context = context
@socket = socket
@ssl_socket = create_openssl_socket(@socket)
@debug_dev = debug_dev
end
def ssl_connect(hostname = nil)
if hostname && @ssl_socket.respond_to?(:hostname=)
@ssl_socket.hostname = hostname
end
@ssl_socket.connect
end
def post_connection_check(host)
verify_mode = @context.verify_mode || OpenSSL::SSL::VERIFY_NONE
if verify_mode == OpenSSL::SSL::VERIFY_NONE
return
elsif @ssl_socket.peer_cert.nil? and
check_mask(verify_mode, OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT)
raise OpenSSL::SSL::SSLError.new('no peer cert')
end
hostname = host.host
if @ssl_socket.respond_to?(:post_connection_check) and RUBY_VERSION > "1.8.4"
@ssl_socket.post_connection_check(hostname)
else
@context.post_connection_check(@ssl_socket.peer_cert, hostname)
end
end
def ssl_version
@ssl_socket.ssl_version if @ssl_socket.respond_to?(:ssl_version)
end
def ssl_cipher
@ssl_socket.cipher
end
def ssl_state
@ssl_socket.state
end
def peer_cert
@ssl_socket.peer_cert
end
def close
@ssl_socket.close
@socket.close
end
def closed?
@socket.closed?
end
def eof?
@ssl_socket.eof?
end
def gets(*args)
str = @ssl_socket.gets(*args)
debug(str)
str
end
def read(*args)
str = @ssl_socket.read(*args)
debug(str)
str
end
def readpartial(*args)
str = @ssl_socket.readpartial(*args)
debug(str)
str
end
def <<(str)
rv = @ssl_socket.write(str)
debug(str)
rv
end
def flush
@ssl_socket.flush
end
def sync
@ssl_socket.sync
end
def sync=(sync)
@ssl_socket.sync = sync
end
private
def check_mask(value, mask)
value & mask == mask
end
def create_openssl_socket(socket)
ssl_socket = nil
if OpenSSL::SSL.const_defined?("SSLContext")
ctx = OpenSSL::SSL::SSLContext.new
@context.set_context(ctx)
ssl_socket = OpenSSL::SSL::SSLSocket.new(socket, ctx)
else
ssl_socket = OpenSSL::SSL::SSLSocket.new(socket)
@context.set_context(ssl_socket)
end
ssl_socket
end
def debug(str)
@debug_dev << str if @debug_dev && str
end
end
# Wraps up a Socket for method interception.
module SocketWrap
def initialize(socket, *args)
super(*args)
@socket = socket
end
def close
@socket.close
end
def closed?
@socket.closed?
end
def eof?
@socket.eof?
end
def gets(*args)
@socket.gets(*args)
end
def read(*args)
@socket.read(*args)
end
def readpartial(*args)
# StringIO doesn't support :readpartial
if @socket.respond_to?(:readpartial)
@socket.readpartial(*args)
else
@socket.read(*args)
end
end
def <<(str)
@socket << str
end
def flush
@socket.flush
end
def sync
@socket.sync
end
def sync=(sync)
@socket.sync = sync
end
end
# Module for intercepting Socket methods and dumps in/out to given debugging
# device. debug_dev must respond to <<.
module DebugSocket
extend SocketWrap
def debug_dev=(debug_dev)
@debug_dev = debug_dev
end
def close
super
debug("! CONNECTION CLOSED\n")
end
def gets(*args)
str = super
debug(str)
str
end
def read(*args)
str = super
debug(str)
str
end
def readpartial(*args)
str = super
debug(str)
str
end
def <<(str)
super
debug(str)
end
private
def debug(str)
if str && @debug_dev
if str.index("\0")
require 'hexdump'
str.force_encoding('BINARY') if str.respond_to?(:force_encoding)
@debug_dev << HexDump.encode(str).join("\n")
else
@debug_dev << str
end
end
end
end
# Dummy Socket for emulating loopback test.
class LoopBackSocket
include SocketWrap
def initialize(host, port, response)
super(response.is_a?(StringIO) ? response : StringIO.new(response))
@host = host
@port = port
end
def <<(str)
# ignored
end
end
# Manages a HTTP session with a Site.
class Session
include HTTPClient::Timeout
include Util
# Destination site
attr_reader :dest
# Proxy site
attr_accessor :proxy
# Boolean value for Socket#sync
attr_accessor :socket_sync
# Requested protocol version
attr_accessor :requested_version
# Device for dumping log for debugging
attr_accessor :debug_dev
attr_accessor :connect_timeout
attr_accessor :connect_retry
attr_accessor :send_timeout
attr_accessor :receive_timeout
attr_accessor :read_block_size
attr_accessor :protocol_retry_count
attr_accessor :socket_local
attr_accessor :ssl_config
attr_reader :ssl_peer_cert
attr_accessor :test_loopback_http_response
attr_accessor :transparent_gzip_decompression
attr_reader :last_used
def initialize(client, dest, agent_name, from)
@client = client
@dest = dest
@invalidated = false
@proxy = nil
@socket_sync = true
@requested_version = nil
@debug_dev = nil
@connect_timeout = nil
@connect_retry = 1
@send_timeout = nil
@receive_timeout = nil
@read_block_size = nil
@protocol_retry_count = 5
@ssl_config = nil
@ssl_peer_cert = nil
@test_loopback_http_response = nil
@socket_local = Site::EMPTY
@agent_name = agent_name
@from = from
@state = :INIT
@requests = []
@status = nil
@reason = nil
@headers = []
@socket = nil
@readbuf = nil
@transparent_gzip_decompression = false
@last_used = nil
end
# Send a request to the server
def query(req)
connect if @state == :INIT
# Use absolute URI (not absolute path) iif via proxy AND not HTTPS.
req.header.request_absolute_uri = !@proxy.nil? and !https?(@dest)
begin
timeout(@send_timeout, SendTimeoutError) do
set_header(req)
req.dump(@socket)
# flush the IO stream as IO::sync mode is false
@socket.flush unless @socket_sync
end
rescue Errno::ECONNABORTED, Errno::ECONNRESET, Errno::EPIPE, IOError
# JRuby can raise IOError instead of ECONNRESET for now
close
raise KeepAliveDisconnected.new(self)
rescue HTTPClient::TimeoutError
close
raise
rescue
close
if SSLEnabled and $!.is_a?(OpenSSL::SSL::SSLError)
raise KeepAliveDisconnected.new(self)
else
raise
end
end
@state = :META if @state == :WAIT
@next_connection = nil
@requests.push(req)
@last_used = Time.now
end
def close
if !@socket.nil? and !@socket.closed?
# @socket.flush may block when it the socket is already closed by
# foreign host and the client runs under MT-condition.
@socket.close
end
@state = :INIT
end
def closed?
@state == :INIT
end
def invalidate
@invalidated = true
end
def invalidated?
@invalidated
end
def get_header
begin
if @state != :META
raise RuntimeError.new("get_status must be called at the beginning of a session")
end
read_header
rescue
close
raise
end
[@version, @status, @reason, @headers]
end
def eof?
if !@content_length.nil?
@content_length == 0
else
@socket.closed? or @socket.eof?
end
end
def get_body(&block)
begin
read_header if @state == :META
return nil if @state != :DATA
if @gzipped and @transparent_gzip_decompression
# zlib itself has a functionality to decompress gzip stream.
# - zlib 1.2.5 Manual
# http://www.zlib.net/manual.html#Advanced
# > windowBits can also be greater than 15 for optional gzip decoding. Add 32 to
# > windowBits to enable zlib and gzip decoding with automatic header detection,
# > or add 16 to decode only the gzip format
inflate_stream = Zlib::Inflate.new(Zlib::MAX_WBITS + 32)
original_block = block
block = Proc.new { |buf|
original_block.call(inflate_stream.inflate(buf))
}
end
if @chunked
read_body_chunked(&block)
elsif @content_length
read_body_length(&block)
else
read_body_rest(&block)
end
rescue
close
raise
end
if eof?
if @next_connection
@state = :WAIT
else
close
end
end
nil
end
private
def set_header(req)
if @requested_version
if /^(?:HTTP\/|)(\d+.\d+)$/ =~ @requested_version
req.http_version = $1
end
end
if @agent_name
req.header.set('User-Agent', "#{@agent_name} #{LIB_NAME}")
end
if @from
req.header.set('From', @from)
end
if req.header.get('Accept').empty?
req.header.set('Accept', '*/*')
end
if @transparent_gzip_decompression
req.header.set('Accept-Encoding', 'gzip,deflate')
end
if req.header.get('Date').empty?
req.header.set_date_header
end
end
# Connect to the server
def connect
site = @proxy || @dest
retry_number = 0
begin
timeout(@connect_timeout, ConnectTimeoutError) do
@socket = create_socket(site)
if https?(@dest)
if @socket.is_a?(LoopBackSocket)
connect_ssl_proxy(@socket, urify(@dest.to_s)) if @proxy
else
@socket = create_ssl_socket(@socket)
connect_ssl_proxy(@socket, urify(@dest.to_s)) if @proxy
begin
@socket.ssl_connect(@dest.host)
ensure
if $DEBUG
warn("Protocol version: #{@socket.ssl_version}")
warn("Cipher: #{@socket.ssl_cipher.inspect}")
warn("State: #{@socket.ssl_state}")
end
end
@socket.post_connection_check(@dest)
@ssl_peer_cert = @socket.peer_cert
end
end
# Use Ruby internal buffering instead of passing data immediately
# to the underlying layer
# => we need to to call explicitly flush on the socket
@socket.sync = @socket_sync
end
rescue RetryableResponse
retry_number += 1
if retry_number < @protocol_retry_count
retry
end
raise BadResponseError.new("connect to the server failed with status #{@status} #{@reason}")
rescue TimeoutError
if @connect_retry == 0
retry
else
retry_number += 1
retry if retry_number < @connect_retry
end
close
raise
end
@state = :WAIT
end
def create_socket(site)
socket = nil
begin
@debug_dev << "! CONNECT TO #{site.host}:#{site.port}\n" if @debug_dev
if str = @test_loopback_http_response.shift
socket = LoopBackSocket.new(site.host, site.port, str)
elsif @socket_local == Site::EMPTY
socket = TCPSocket.new(site.host, site.port)
else
socket = TCPSocket.new(site.host, site.port, @socket_local.host, @socket_local.port)
end
if @debug_dev
@debug_dev << "! CONNECTION ESTABLISHED\n"
socket.extend(DebugSocket)
socket.debug_dev = @debug_dev
end
rescue SystemCallError => e
e.message << " (#{site})"
raise
rescue SocketError => e
e.message << " (#{site})"
raise
end
socket
end
# wrap socket with OpenSSL.
def create_ssl_socket(raw_socket)
SSLSocketWrap.new(raw_socket, @ssl_config, @debug_dev)
end
def connect_ssl_proxy(socket, uri)
req = HTTP::Message.new_connect_request(uri)
@client.request_filter.each do |filter|
filter.filter_request(req)
end
set_header(req)
req.dump(@socket)
@socket.flush unless @socket_sync
res = HTTP::Message.new_response('')
parse_header
res.http_version, res.status, res.reason = @version, @status, @reason
@headers.each do |key, value|
res.header.set(key.to_s, value)
end
commands = @client.request_filter.collect { |filter|
filter.filter_response(req, res)
}
if commands.find { |command| command == :retry }
raise RetryableResponse.new
end
unless @status == 200
raise BadResponseError.new("connect to ssl proxy failed with status #{@status} #{@reason}", res)
end
end
# Read status block.
def read_header
@content_length = nil
@chunked = false
@gzipped = false
@chunk_length = 0
parse_header
# Header of the request has been parsed.
@state = :DATA
req = @requests.shift
if req.header.request_method == 'HEAD' or no_message_body?(@status)
@content_length = 0
if @next_connection
@state = :WAIT
else
close
end
end
@next_connection = false if !@content_length and !@chunked
end
StatusParseRegexp = %r(\AHTTP/(\d+\.\d+)\s+(\d\d\d)\s*([^\r\n]+)?\r?\n\z)
def parse_header
timeout(@receive_timeout, ReceiveTimeoutError) do
initial_line = nil
begin
begin
initial_line = @socket.gets("\n")
if initial_line.nil?
close
raise KeepAliveDisconnected.new(self)
end
rescue Errno::ECONNABORTED, Errno::ECONNRESET, Errno::EPIPE, IOError
# JRuby can raise IOError instead of ECONNRESET for now
close
raise KeepAliveDisconnected.new(self)
end
if StatusParseRegexp !~ initial_line
@version = '0.9'
@status = nil
@reason = nil
@next_connection = false
@content_length = nil
@readbuf = initial_line
break
end
@version, @status, @reason = $1, $2.to_i, $3
@next_connection = HTTP::Message.keep_alive_enabled?(@version)
@headers = []
while true
line = @socket.gets("\n")
unless line
raise BadResponseError.new('unexpected EOF')
end
line.chomp!
break if line.empty?
if line[0] == ?\ or line[0] == ?\t
last = @headers.last[1]
last << ' ' unless last.empty?
last << line.strip
else
key, value = line.strip.split(/\s*:\s*/, 2)
parse_keepalive_header(key, value)
@headers << [key, value]
end
end
end while (@version == '1.1' && @status == 100)
end
end
def no_message_body?(status)
!status.nil? && # HTTP/0.9
((status >= 100 && status < 200) || status == 204 || status == 304)
end
def parse_keepalive_header(key, value)
key = key.downcase
if key == 'content-length'
@content_length = value.to_i
elsif key == 'content-encoding' and ( value.downcase == 'gzip' or
value.downcase == 'x-gzip' or value.downcase == 'deflate' )
@gzipped = true
elsif key == 'transfer-encoding' and value.downcase == 'chunked'
@chunked = true
@chunk_length = 0
@content_length = nil
elsif key == 'connection' or key == 'proxy-connection'
if value.downcase == 'keep-alive'
@next_connection = true
else
@next_connection = false
end
end
end
def read_body_length(&block)
return nil if @content_length == 0
while true
buf = ''
maxbytes = @read_block_size
maxbytes = @content_length if maxbytes > @content_length
timeout(@receive_timeout, ReceiveTimeoutError) do
begin
@socket.readpartial(maxbytes, buf)
rescue EOFError
close
buf = nil
end
end
if buf && buf.bytesize > 0
@content_length -= buf.bytesize
yield buf
else
@content_length = 0
end
return if @content_length == 0
end
end
RS = "\r\n"
def read_body_chunked(&block)
buf = ''
while true
len = @socket.gets(RS)
if len.nil? # EOF
close
return
end
@chunk_length = len.hex
if @chunk_length == 0
@content_length = 0
@socket.gets(RS)
return
end
timeout(@receive_timeout, ReceiveTimeoutError) do
@socket.read(@chunk_length + 2, buf)
end
unless buf.empty?
yield buf.slice(0, @chunk_length)
end
end
end
def read_body_rest
if @readbuf and @readbuf.bytesize > 0
yield @readbuf
@readbuf = nil
end
while true
buf = ''
timeout(@receive_timeout, ReceiveTimeoutError) do
begin
@socket.readpartial(@read_block_size, buf)
rescue EOFError
buf = nil
end
end
if buf && buf.bytesize > 0
yield buf
else
return
end
end
end
end
end
Jump to Line
Something went wrong with that request. Please try again.