diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a17630b --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +/TAGS +/pkg +/doc +*.swp diff --git a/History.txt b/History.txt new file mode 100644 index 0000000..0edf5a1 --- /dev/null +++ b/History.txt @@ -0,0 +1,4 @@ +=== 1.0.0 / 2010-09-16 + +* Major Enhancements + * Based on drbrain/net-http-persistent but uses a connection pool instead of connection/thread diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..01dbba9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2010 Eric Hodel, Aaron Patterson + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +'Software'), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.rdoc b/README.rdoc new file mode 100644 index 0000000..d7de80f --- /dev/null +++ b/README.rdoc @@ -0,0 +1,58 @@ += persistent_http + +* http://github.com/bpardee/persistent_http + +== DESCRIPTION: + +Persistent connections using Net::HTTP with a connection pool. + +This is based on Eric Holder's Net::HTTP::Persistent libary but uses +a connection pool of Net::HTTP objects instead of a connection per +thread. C/T is fine if you're only using your http threads to make +connections but if you use them in child threads then I suspect you +will have a thread memory leak. Also, you will generally get less +connection resets if the most recently used connection is always +returned. + +== FEATURES/PROBLEMS: + +* Supports SSL +* Thread-safe +* Pure ruby +* Timeout-less speed boost for 1.8 (by Aaron Patterson) + +== INSTALL: + + gem install persistent_http + +== EXAMPLE USAGE: + + require 'persistent_http' + + class MyHTTPClient + @@persistent_http = PersistentHTTP.new( + :name => 'MyHTTPClient', + :logger => Rails.logger, + :pool_size => 10, + :warn_timeout => 0.25, + :force_retry => true, + :url => 'https://www.example.com/echo/foo' # equivalent to :use_ssl => true, :host => 'www.example.com', :default_path => '/echo/foo' + ) + + def send_get_message + response = @@persistent_http.request + ... Handle response as you would a normal Net::HTTPResponse ... + end + + def send_post_message + request = Net::HTTP::Post.new('/perform_service) + ... Modify request as needed ... + response = @@persistent_http.request(request) + ... Handle response as you would a normal Net::HTTPResponse ... + end + end + + +== Copyright + +Copyright (c) 2010 Eric Hodel, Aaron Patterson, Brad Pardee. See LICENSE for details. diff --git a/Rakefile b/Rakefile new file mode 100644 index 0000000..acb8f30 --- /dev/null +++ b/Rakefile @@ -0,0 +1,16 @@ +require 'rubygems' +require 'rake' + +begin + require 'jeweler' + Jeweler::Tasks.new do |gemspec| + gemspec.name = "persistent_http" + gemspec.summary = "Persistent HTTP connections using a connection pool" + gemspec.description = "Persistent HTTP connections using a connection pool" + gemspec.email = "bradpardee@gmail.com" + gemspec.homepage = "http://github.com/bpardee/persistent_http" + gemspec.authors = ["Brad Pardee"] + end +rescue LoadError + puts "Jeweler not available. Install it with: gem install jeweler" +end diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..3eefcb9 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +1.0.0 diff --git a/lib/persistent_http.rb b/lib/persistent_http.rb new file mode 100644 index 0000000..922fc11 --- /dev/null +++ b/lib/persistent_http.rb @@ -0,0 +1,444 @@ +require 'net/http' +require 'persistent_http/faster' +require 'uri' +require 'gene_pool' + +## +# Persistent connections for Net::HTTP +# +# PersistentHTTP maintains a connection pool of Net::HTTP persistent connections. +# When connections fail due to resets or bad responses, the connection is renewed +# and the request is retried per RFC 2616 (POST requests will only get retried if +# the :force_retry option is set to true). +# +# Example: +# +# @@persistent_http = PersistentHTTP.new( +# :name => 'MyHTTPClient', +# :logger => Rails.logger, +# :pool_size => 10, +# :warn_timeout => 0.25, +# :force_retry => true, +# :url => 'https://www.example.com/echo/foo' # equivalent to :use_ssl => true, :host => 'www.example.com', :default_path => '/echo/foo' +# ) +# +# def send_get_message +# response = @@persistent_http.request +# ... Handle response as you would a normal Net::HTTPResponse ... +# end +# +# def send_post_message +# request = Net::HTTP::Post.new('/perform_service) +# ... Modify request as needed ... +# response = @@persistent_http.request(request) +# ... Handle response as you would a normal Net::HTTPResponse ... +# end + +class PersistentHTTP + + ## + # The version of PersistentHTTP use are using + VERSION = '1.0.0' + + ## + # Error class for errors raised by PersistentHTTP. Various + # SystemCallErrors are re-raised with a human-readable message under this + # class. + class Error < StandardError; end + + ## + # An SSL certificate authority. Setting this will set verify_mode to + # VERIFY_PEER. + attr_accessor :ca_file + + ## + # This client's OpenSSL::X509::Certificate + attr_accessor :certificate + + ## + # Sends debug_output to this IO via Net::HTTP#set_debug_output. + # + # Never use this method in production code, it causes a serious security + # hole. + attr_accessor :debug_output + + ## + # Default path for the request + attr_accessor :default_path + + ## + # Retry even for non-idempotent (POST) requests. + attr_accessor :force_retry + + ## + # Headers that are added to every request + attr_accessor :headers + + ## + # Host for the Net:HTTP connection + attr_reader :host + + ## + # HTTP version to enable version specific features. + attr_reader :http_version + + ## + # The value sent in the Keep-Alive header. Defaults to 30. Not needed for + # HTTP/1.1 servers. + # + # This may not work correctly for HTTP/1.0 servers + # + # This method may be removed in a future version as RFC 2616 does not + # require this header. + attr_accessor :keep_alive + + ## + # Logger for message logging. + attr_accessor :logger + + ## + # A name for this connection. Allows you to keep your connections apart + # from everybody else's. + attr_reader :name + + ## + # Seconds to wait until a connection is opened. See Net::HTTP#open_timeout + attr_accessor :open_timeout + + ## + # The maximum size of the connection pool + attr_reader :pool_size + + ## + # Port for the Net:HTTP connection + attr_reader :port + + ## + # This client's SSL private key + attr_accessor :private_key + + ## + # The URL through which requests will be proxied + attr_reader :proxy_uri + + ## + # Seconds to wait until reading one block. See Net::HTTP#read_timeout + attr_accessor :read_timeout + + ## + # Use ssl if set + attr_reader :use_ssl + + ## + # SSL verification callback. Used when ca_file is set. + attr_accessor :verify_callback + + ## + # HTTPS verify mode. Defaults to OpenSSL::SSL::VERIFY_NONE which ignores + # certificate problems. + # + # You can use +verify_mode+ to override any default values. + attr_accessor :verify_mode + + ## + # The threshold in seconds for checking out a connection at which a warning + # will be logged via the logger + attr_accessor :warn_timeout + + ## + # Creates a new PersistentHTTP. + # + # Set +name+ to keep your connections apart from everybody else's. Not + # required currently, but highly recommended. Your library name should be + # good enough. This parameter will be required in a future version. + # + # +proxy+ may be set to a URI::HTTP or :ENV to pick up proxy options from + # the environment. See proxy_from_env for details. + # + # In order to use a URI for the proxy you'll need to do some extra work + # beyond URI.parse: + # + # proxy = URI.parse 'http://proxy.example' + # proxy.user = 'AzureDiamond' + # proxy.password = 'hunter2' + + def initialize(options={}) + @name = options[:name] || 'PersistentHTTP' + @ca_file = options[:ca_file] + @certificate = options[:certificate] + @debug_output = options[:debug_output] + @default_path = options[:default_path] + @force_retry = options[:force_retry] + @headers = options[:header] || {} + @host = options[:host] + @keep_alive = options[:keep_alive] || 30 + @logger = options[:logger] + @open_timeout = options[:open_timeout] + @pool_size = options[:pool_size] || 1 + @port = options[:port] + @private_key = options[:private_key] + @read_timeout = options[:read_timeout] + @use_ssl = options[:use_ssl] + @verify_callback = options[:verify_callback] + @verify_mode = options[:verify_mode] + @warn_timeout = options[:warn_timeout] || 0.5 + + url = options[:url] + if url + url = URI.parse(url) if url.kind_of? String + @default_path ||= url.request_uri + @host ||= url.host + @port ||= url.port + @use_ssl ||= url.scheme == 'https' + end + + @port ||= (@use_ssl ? 443 : 80) + + # Hash containing the request counts based on the connection + @count_hash = Hash.new(0) + + raise 'host not set' unless @host + net_http_args = [@host, @port] + connection_id = net_http_args.join ':' + + proxy = options[:proxy] + + @proxy_uri = case proxy + when :ENV then proxy_from_env + when URI::HTTP then proxy + when nil then # ignore + else raise ArgumentError, 'proxy must be :ENV or a URI::HTTP' + end + + if @proxy_uri then + @proxy_args = [ + @proxy_uri.host, + @proxy_uri.port, + @proxy_uri.user, + @proxy_uri.password, + ] + + @proxy_connection_id = [nil, *@proxy_args].join ':' + + connection_id << @proxy_connection_id + net_http_args.concat @proxy_args + end + + @pool = GenePool.new(:name => name + '-' + connection_id, + :pool_size => @pool_size, + :warn_timeout => @warn_timeout, + :logger => @logger) do + begin + connection = Net::HTTP.new(*net_http_args) + connection.set_debug_output @debug_output if @debug_output + connection.open_timeout = @open_timeout if @open_timeout + connection.read_timeout = @read_timeout if @read_timeout + + ssl connection if @use_ssl + + connection.start + connection + rescue Errno::ECONNREFUSED + raise Error, "connection refused: #{connection.address}:#{connection.port}" + rescue Errno::EHOSTDOWN + raise Error, "host down: #{connection.address}:#{connection.port}" + end + end + end + + ## + # Makes a request per +req+. If +req+ is nil a Net::HTTP::Get is performed + # against +default_path+. + # + # If a block is passed #request behaves like Net::HTTP#request (the body of + # the response will not have been read). + # + # +req+ must be a Net::HTTPRequest subclass (see Net::HTTP for a list). + # + # If there is an error and the request is idempontent according to RFC 2616 + # it will be retried automatically. + + def request req = nil, &block + retried = false + bad_response = false + + req = Net::HTTP::Get.new @default_path unless req + + headers.each do |pair| + req.add_field(*pair) + end + + req.add_field 'Connection', 'keep-alive' + req.add_field 'Keep-Alive', @keep_alive + + @pool.with_connection do |connection| + begin + response = connection.request req, &block + @http_version ||= response.http_version + @count_hash[connection.object_id] += 1 + return response + + rescue Timeout::Error => e + due_to = "(due to #{e.message} - #{e.class})" + message = error_message connection + @logger.info "#{name}: Removing connection #{due_to} #{message}" if @logger + remove connection + raise + + rescue Net::HTTPBadResponse => e + message = error_message connection + if bad_response or not (idempotent? req or @force_retry) + @logger.info "#{name}: Removing connection because of too many bad responses #{message}" if @logger + remove connection + raise Error, "too many bad responses #{message}" + else + bad_response = true + @logger.info "#{name}: Renewing connection because of bad response #{message}" if @logger + connection = renew connection + retry + end + + rescue IOError, EOFError, Errno::ECONNABORTED, Errno::ECONNRESET, Errno::EPIPE => e + due_to = "(due to #{e.message} - #{e.class})" + message = error_message connection + if retried or not (idempotent? req or @force_retry) + @logger.info "#{name}: Removing connection #{due_to} #{message}" if @logger + remove connection + raise Error, "too many connection resets #{due_to} #{message}" + else + retried = true + @logger.info "#{name}: Renewing connection #{due_to} #{message}" if @logger + connection = renew connection + retry + end + end + end + end + + ## + # Shuts down all connections. + + def shutdown + raise 'Shutdown not implemented' + # TBD - need to think about this one + @count_hash = nil + end + + ####### + private + ####### + + ## + # Returns an error message containing the number of requests performed on + # this connection + + def error_message connection + requests = @count_hash[connection.object_id] || 0 + "after #{requests} requests on #{connection.object_id}" + end + + ## + # URI::escape wrapper + + def escape str + URI.escape str if str + end + + ## + # Finishes the Net::HTTP +connection+ + + def finish connection + @count_hash.delete(connection.object_id) + connection.finish + rescue IOError + end + + ## + # Is +req+ idempotent according to RFC 2616? + + def idempotent? req + case req + when Net::HTTP::Delete, Net::HTTP::Get, Net::HTTP::Head, + Net::HTTP::Options, Net::HTTP::Put, Net::HTTP::Trace then + true + end + end + + ## + # Adds "http://" to the String +uri+ if it is missing. + + def normalize_uri uri + (uri =~ /^https?:/) ? uri : "http://#{uri}" + end + + ## + # Creates a URI for an HTTP proxy server from ENV variables. + # + # If +HTTP_PROXY+ is set a proxy will be returned. + # + # If +HTTP_PROXY_USER+ or +HTTP_PROXY_PASS+ are set the URI is given the + # indicated user and password unless HTTP_PROXY contains either of these in + # the URI. + # + # For Windows users lowercase ENV variables are preferred over uppercase ENV + # variables. + + def proxy_from_env + env_proxy = ENV['http_proxy'] || ENV['HTTP_PROXY'] + + return nil if env_proxy.nil? or env_proxy.empty? + + uri = URI.parse(normalize_uri(env_proxy)) + + unless uri.user or uri.password then + uri.user = escape ENV['http_proxy_user'] || ENV['HTTP_PROXY_USER'] + uri.password = escape ENV['http_proxy_pass'] || ENV['HTTP_PROXY_PASS'] + end + + uri + end + + ## + # Finishes then removes the Net::HTTP +connection+ + + def remove connection + finish connection + @pool.remove(connection) + end + + ## + # Finishes then renews the Net::HTTP +connection+. It may be unnecessary + # to completely recreate the connection but connections that get timed out + # in JRuby leave the ssl context in a frozen object state. + + def renew connection + finish connection + connection = @pool.renew(connection) + end + + ## + # Enables SSL on +connection+ + + def ssl connection + require 'net/https' + connection.use_ssl = true + + # suppress warning but allow override + connection.verify_mode = OpenSSL::SSL::VERIFY_NONE unless @verify_mode + + if @ca_file then + connection.ca_file = @ca_file + connection.verify_mode = OpenSSL::SSL::VERIFY_PEER + connection.verify_callback = @verify_callback if @verify_callback + end + + if @certificate and @private_key then + connection.cert = @certificate + connection.key = @private_key + end + + connection.verify_mode = @verify_mode if @verify_mode + end + +end + diff --git a/lib/persistent_http/faster.rb b/lib/persistent_http/faster.rb new file mode 100644 index 0000000..46a4daa --- /dev/null +++ b/lib/persistent_http/faster.rb @@ -0,0 +1,27 @@ +require 'net/protocol' + +## +# Aaron Patterson's monkeypatch (accepted into 1.9.1) to fix Net::HTTP's speed +# problems. +# +# http://gist.github.com/251244 + +class Net::BufferedIO #:nodoc: + alias :old_rbuf_fill :rbuf_fill + + def rbuf_fill + if @io.respond_to? :read_nonblock then + begin + @rbuf << @io.read_nonblock(65536) + rescue Errno::EWOULDBLOCK => e + retry if IO.select [@io], nil, nil, @read_timeout + raise Timeout::Error, e.message + end + else # SSL sockets do not have read_nonblock + timeout @read_timeout do + @rbuf << @io.sysread(65536) + end + end + end +end if RUBY_VERSION < '1.9' + diff --git a/persistent_http.gemspec b/persistent_http.gemspec new file mode 100644 index 0000000..2a0f061 --- /dev/null +++ b/persistent_http.gemspec @@ -0,0 +1,49 @@ +# Generated by jeweler +# DO NOT EDIT THIS FILE DIRECTLY +# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command +# -*- encoding: utf-8 -*- + +Gem::Specification.new do |s| + s.name = %q{gene_pool} + s.version = "1.0.1" + + s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version= + s.authors = ["Brad Pardee"] + s.date = %q{2010-09-12} + s.description = %q{Generic pooling library for creating a connection pool} + s.email = %q{bradpardee@gmail.com} + s.extra_rdoc_files = [ + "LICENSE", + "README.rdoc" + ] + s.files = [ + ".gitignore", + "History.txt", + "LICENSE", + "README.rdoc", + "Rakefile", + "VERSION", + "gene_pool.gemspec", + "lib/gene_pool.rb", + "test/gene_pool_test.rb" + ] + s.homepage = %q{http://github.com/bpardee/gene_pool} + s.rdoc_options = ["--charset=UTF-8"] + s.require_paths = ["lib"] + s.rubygems_version = %q{1.3.6} + s.summary = %q{Generic pooling library for creating a connection pool} + s.test_files = [ + "test/gene_pool_test.rb" + ] + + if s.respond_to? :specification_version then + current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION + s.specification_version = 3 + + if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then + else + end + else + end +end + diff --git a/test/persistent_http_test.rb b/test/persistent_http_test.rb new file mode 100644 index 0000000..78484e4 --- /dev/null +++ b/test/persistent_http_test.rb @@ -0,0 +1,647 @@ +require 'rubygems' +require 'test/unit' +require 'shoulda' +require 'persistent_http' +require 'openssl' +require 'stringio' +require 'logger' + +CMD_SUCCESS = 'success' +CMD_SLEEP = 'sleep' +CMD_BAD_RESPONSE = 'bad_response' +CMD_EOF_ERROR = 'eof_error' +CMD_CONNRESET = 'connreset' +CMD_ECHO = 'echo' + +PASS = 'pass' +FAIL = 'fail' + +DUMMY_OPEN_TIMEOUT_FOR_HOSTDOWN = 9000 +DUMMY_OPEN_TIMEOUT_FOR_CONNREFUSED = 9001 + +$debug = false +$count = -1 + +class Net::HTTP + def connect + raise Errno::EHOSTDOWN if open_timeout == DUMMY_OPEN_TIMEOUT_FOR_HOSTDOWN + raise Errno::ECONNREFUSED if open_timeout == DUMMY_OPEN_TIMEOUT_FOR_CONNREFUSED + end + + def successful_response + r = Net::HTTPResponse.allocate + def r.http_version() '1.1'; end + def r.read_body() :read_body; end + yield r if block_given? + r + end + + def request(req, &block) + $count += 1 + puts "path=#{req.path} count=#{$count}" if $debug + args = req.path[1..-1].split('/') + cmd = args.shift + i = $count % args.size if args.size > 0 + puts "i=#{i}" if $debug + if cmd == CMD_ECHO + res = successful_response(&block) + eval "def res.body() \"#{req.body}\" end" + return res + elsif cmd == CMD_SUCCESS || args[i] == PASS + return successful_response(&block) + end + case cmd + when CMD_SLEEP + sleep args[i].to_i + return successful_response(&block) + when CMD_BAD_RESPONSE + raise Net::HTTPBadResponse.new('Dummy bad response') + when CMD_EOF_ERROR + raise EOFError.new('Dummy EOF error') + when CMD_CONNRESET + raise Errno::ECONNRESET + else + return successful_response(&block) + end + end +end + +class PersistentHTTP + attr_reader :pool + + # Make private methods public + send(:public, *(self.private_instance_methods - Object.private_instance_methods)) +end + +class PersistentHTTPTest < Test::Unit::TestCase + + def clear_proxy_env + ENV.delete 'http_proxy' + ENV.delete 'HTTP_PROXY' + ENV.delete 'http_proxy_user' + ENV.delete 'HTTP_PROXY_USER' + ENV.delete 'http_proxy_pass' + ENV.delete 'HTTP_PROXY_PASS' + end + + def uri_for(*args) + '/' + args.join('/') + end + + def get_request(*args) + puts "uri=#{uri_for(args)}" if $debug + $count = -1 + return Net::HTTP::Get.new(uri_for(args)) + end + + def post_request(*args) + puts "uri=#{uri_for(args)}" if $debug + $count = -1 + return Net::HTTP::Post.new(uri_for(args)) + end + + def http_and_io(options={}) + io = StringIO.new + logger = Logger.new(io) + logger.level = Logger::INFO + default_options = {:name => 'TestNetHTTPPersistent', :logger => logger, :pool_size => 1} + http = PersistentHTTP.new(default_options.merge(options)) + [http, io] + end + + context 'simple setup' do + setup do + @io = StringIO.new + logger = Logger.new(@io) + logger.level = Logger::INFO + @http = PersistentHTTP.new(:host => 'example.com', :name => 'TestNetHTTPPersistent', :logger => logger) + @http.headers['user-agent'] = 'test ua' + end + + should 'have options set' do + assert_equal @http.proxy_uri, nil + assert_equal 'TestNetHTTPPersistent', @http.name + end + + should 'handle escape' do + assert_equal nil, @http.escape(nil) + assert_equal '%20', @http.escape(' ') + end + + should 'handle error' do + req = get_request CMD_EOF_ERROR, PASS, PASS, PASS, PASS, FAIL, PASS, PASS + 6.times do + @http.request(req) + end + assert_match "after 4 requests on", @io.string + end + + should 'handle finish' do + c = Object.new + def c.finish; @finished = true end + def c.finished?; @finished end + def c.start; @started = true end + def c.started?; @started end + + @http.finish c + + assert !c.started? + assert c.finished? + end + + should 'handle finish io error' do + c = Object.new + def c.finish; @finished = true; raise IOError end + def c.finished?; @finished end + def c.start; @started = true end + def c.started?; @started end + + @http.finish c + + assert !c.started? + assert c.finished? + end + + should 'fill in http version' do + assert_nil @http.http_version + @http.request(get_request(CMD_SUCCESS)) + assert_equal '1.1', @http.http_version + end + + should 'handle idempotent' do + assert @http.idempotent? Net::HTTP::Delete.new '/' + assert @http.idempotent? Net::HTTP::Get.new '/' + assert @http.idempotent? Net::HTTP::Head.new '/' + assert @http.idempotent? Net::HTTP::Options.new '/' + assert @http.idempotent? Net::HTTP::Put.new '/' + assert @http.idempotent? Net::HTTP::Trace.new '/' + + assert !@http.idempotent?(Net::HTTP::Post.new '/') + end + + should 'handle normalize_uri' do + assert_equal 'http://example', @http.normalize_uri('example') + assert_equal 'http://example', @http.normalize_uri('http://example') + assert_equal 'https://example', @http.normalize_uri('https://example') + end + + should 'handle simple request' do + req = get_request(CMD_SUCCESS) + res = @http.request(req) + + assert_kind_of Net::HTTPResponse, res + + assert_kind_of Net::HTTP::Get, req + assert_equal uri_for(CMD_SUCCESS), req.path + assert_equal 'keep-alive', req['connection'] + assert_equal '30', req['keep-alive'] + assert_match %r%test ua%, req['user-agent'] + end + + should 'handle request with block' do + body = nil + + req = get_request(CMD_SUCCESS) + res = @http.request(req) do |r| + body = r.read_body + end + + assert_kind_of Net::HTTPResponse, res + assert !body.nil? + + assert_kind_of Net::HTTP::Get, req + assert_equal uri_for(CMD_SUCCESS), req.path + assert_equal 'keep-alive', req['connection'] + assert_equal '30', req['keep-alive'] + assert_match %r%test ua%, req['user-agent'] + end + + should 'handle bad response' do + req = get_request(CMD_BAD_RESPONSE, FAIL, FAIL) + e = assert_raises PersistentHTTP::Error do + @http.request req + end + assert_match %r%too many bad responses%, e.message + assert_match %r%Renewing connection because of bad response%, @io.string + assert_match %r%Removing connection because of too many bad responses%, @io.string + + res = @http.request(get_request(CMD_SUCCESS)) + assert_kind_of Net::HTTPResponse, res + end + + should 'handle connection reset' do + req = get_request(CMD_CONNRESET, FAIL, FAIL) + e = assert_raises PersistentHTTP::Error do + @http.request req + end + + assert_match %r%too many connection resets%, e.message + assert_match %r%Renewing connection %, @io.string + assert_match %r%Removing connection %, @io.string + + res = @http.request(get_request(CMD_SUCCESS)) + assert_kind_of Net::HTTPResponse, res + end + + should 'retry on bad response' do + res = @http.request(get_request(CMD_BAD_RESPONSE, FAIL, PASS)) + assert_match %r%Renewing connection because of bad response%, @io.string + assert_kind_of Net::HTTPResponse, res + end + + should 'retry on connection reset' do + res = @http.request(get_request(CMD_CONNRESET, FAIL, PASS)) + assert_match %r%Renewing connection %, @io.string + assert_kind_of Net::HTTPResponse, res + end + + should 'not retry on bad response from post' do + post = post_request(CMD_BAD_RESPONSE, FAIL, PASS) + e = assert_raises PersistentHTTP::Error do + @http.request(post) + end + assert_match %r%too many bad responses%, e.message + assert_match %r%Removing connection because of too many bad responses%, @io.string + + res = @http.request(get_request(CMD_SUCCESS)) + assert_kind_of Net::HTTPResponse, res + end + + should 'not retry on connection reset from post' do + post = post_request(CMD_CONNRESET, FAIL, PASS) + e = assert_raises PersistentHTTP::Error do + @http.request(post) + end + assert_match %r%too many connection resets%, e.message + assert_match %r%Removing connection %, @io.string + + res = @http.request(get_request(CMD_SUCCESS)) + assert_kind_of Net::HTTPResponse, res + end + + should 'retry on bad response from post when force_retry set' do + @http.force_retry = true + post = post_request(CMD_BAD_RESPONSE, FAIL, PASS) + res = @http.request post + assert_match %r%Renewing connection because of bad response%, @io.string + assert_kind_of Net::HTTPResponse, res + end + + should 'retry on connection reset from post when force_retry set' do + @http.force_retry = true + post = post_request(CMD_CONNRESET, FAIL, PASS) + res = @http.request post + assert_match %r%Renewing connection %, @io.string + assert_kind_of Net::HTTPResponse, res + end + + should 'allow post' do + post = Net::HTTP::Post.new(uri_for CMD_ECHO) + post.body = 'hello PersistentHTTP' + res = @http.request(post) + assert_kind_of Net::HTTPResponse, res + assert_equal post.body, res.body + end + + should 'allow ssl' do + @http.verify_callback = :callback + c = Net::HTTP.new('localhost', 80) + + @http.ssl c + + assert c.use_ssl? + assert_equal OpenSSL::SSL::VERIFY_NONE, c.verify_mode + assert_nil c.verify_callback + end + + should 'allow ssl ca_file' do + @http.ca_file = 'ca_file' + @http.verify_callback = :callback + c = Net::HTTP.new('localhost', 80) + + @http.ssl c + + assert c.use_ssl? + assert_equal OpenSSL::SSL::VERIFY_PEER, c.verify_mode + assert_equal :callback, c.verify_callback + end + + should 'allow ssl certificate' do + @http.certificate = :cert + @http.private_key = :key + c = Net::HTTP.new('localhost', 80) + + @http.ssl c + + assert c.use_ssl? + assert_equal :cert, c.cert + assert_equal :key, c.key + end + + should 'allow ssl verify_mode' do + @http.verify_mode = OpenSSL::SSL::VERIFY_NONE + c = Net::HTTP.new('localhost', 80) + + @http.ssl c + + assert c.use_ssl? + assert_equal OpenSSL::SSL::VERIFY_NONE, c.verify_mode + end + end + + context 'initialize proxy by env' do + setup do + clear_proxy_env + ENV['HTTP_PROXY'] = 'proxy.example' + @http = PersistentHTTP.new(:host => 'foobar', :proxy => :ENV) + end + + should 'match HTTP_PROXY' do + assert_equal URI.parse('http://proxy.example'), @http.proxy_uri + assert_equal 'foobar', @http.host + end + end + + context 'initialize proxy by uri' do + setup do + @proxy_uri = URI.parse 'http://proxy.example' + @proxy_uri.user = 'johndoe' + @proxy_uri.password = 'muffins' + @http = PersistentHTTP.new(:url => 'https://zulu.com/foobar', :proxy => @proxy_uri) + end + + should 'match proxy_uri and have proxy connection' do + assert_equal @proxy_uri, @http.proxy_uri + assert_equal true, @http.use_ssl + assert_equal 'zulu.com', @http.host + assert_equal '/foobar', @http.default_path + + @http.pool.with_connection do |c| + assert c.started? + assert c.proxy? + end + end + end + + context 'initialize proxy by env' do + setup do + clear_proxy_env + ENV['HTTP_PROXY'] = 'proxy.example' + ENV['HTTP_PROXY_USER'] = 'johndoe' + ENV['HTTP_PROXY_PASS'] = 'muffins' + @http = PersistentHTTP.new(:url => 'https://zulu.com/foobar', :proxy => :ENV) + end + + should 'create proxy_uri from env' do + expected = URI.parse 'http://proxy.example' + expected.user = 'johndoe' + expected.password = 'muffins' + + assert_equal expected, @http.proxy_uri + end + end + + context 'initialize proxy by env lower' do + setup do + clear_proxy_env + ENV['http_proxy'] = 'proxy.example' + ENV['http_proxy_user'] = 'johndoe' + ENV['http_proxy_pass'] = 'muffins' + @http = PersistentHTTP.new(:url => 'https://zulu.com/foobar', :proxy => :ENV) + end + + should 'create proxy_uri from env' do + expected = URI.parse 'http://proxy.example' + expected.user = 'johndoe' + expected.password = 'muffins' + + assert_equal expected, @http.proxy_uri + end + end + + context 'with timeouts set' do + setup do + @http = PersistentHTTP.new(:url => 'http://example.com') + @http.open_timeout = 123 + @http.read_timeout = 321 + end + + should 'have timeouts set' do + @http.pool.with_connection do |c| + assert c.started? + assert !c.proxy? + + assert_equal 123, c.open_timeout + assert_equal 321, c.read_timeout + + assert_equal 'example.com', c.address + assert_equal 80, c.port + assert !@http.use_ssl + end + end + + should 'reuse same connection' do + c1, c2 = nil, nil + @http.pool.with_connection do |c| + c1 = c + assert c.started? + end + @http.pool.with_connection do |c| + c2 = c + assert c.started? + end + assert_same c1,c2 + end + end + + context 'with debug_output' do + setup do + @io = StringIO.new + @http = PersistentHTTP.new(:url => 'http://example.com', :debug_output => @io) + end + + should 'have debug_output set' do + @http.pool.with_connection do |c| + assert c.started? + assert_equal @io, c.instance_variable_get(:@debug_output) + assert_equal 'example.com', c.address + assert_equal 80, c.port + end + end + end + + context 'with host down' do + setup do + @http = PersistentHTTP.new(:url => 'http://example.com', :open_timeout => DUMMY_OPEN_TIMEOUT_FOR_HOSTDOWN) + end + + should 'assert error' do + e = assert_raises PersistentHTTP::Error do + @http.request(get_request(CMD_SUCCESS)) + end + assert_match %r%host down%, e.message + end + end + + context 'with connection refused' do + setup do + @http = PersistentHTTP.new(:url => 'http://example.com', :open_timeout => DUMMY_OPEN_TIMEOUT_FOR_CONNREFUSED) + end + + should 'assert error' do + e = assert_raises PersistentHTTP::Error do + @http.request(get_request(CMD_SUCCESS)) + end + assert_match %r%connection refused%, e.message + end + end + + context 'with pool size of 3' do + setup do + @http = PersistentHTTP.new(:url => 'http://example.com', :pool_size => 3) + end + + should 'only allow 3 connections checked out at a time' do + @http.request(get_request(CMD_SUCCESS)) + pool = @http.pool + 2.times do + conns = [] + pool.with_connection do |c1| + pool.with_connection do |c2| + conns << c2 + pool.with_connection do |c3| + conns << c3 + begin + Timeout.timeout(2) do + pool.with_connection { |c4| } + assert false, 'should NOT have been able to get 4th connection' + end + rescue Timeout::Error => e + # successfully failed to get a connection + end + @http.remove(c1) + Timeout.timeout(1) do + begin + pool.with_connection do |c4| + conns << c4 + end + rescue Timeout::Error => e + assert false, 'should have been able to get 4th connection' + end + end + end + end + end + pool.with_connection do |c1| + pool.with_connection do |c2| + pool.with_connection do |c3| + assert_equal conns, [c1,c2,c3] + end + end + end + # Do it a 2nd time with finish returning an IOError + c1 = conns[0] + def c1.finish + super + raise IOError + end + end + end + + should 'handle renew' do + @http.request(get_request(CMD_SUCCESS)) + pool = @http.pool + 2.times do + conns = [] + pool.with_connection do |c1| + pool.with_connection do |c2| + conns << c2 + pool.with_connection do |c3| + conns << c3 + new_c1 = @http.renew(c1) + assert c1 != new_c1 + conns.unshift(new_c1) + end + end + end + pool.with_connection do |c1| + pool.with_connection do |c2| + pool.with_connection do |c3| + assert_equal conns, [c1,c2,c3] + end + end + end + # Do it a 2nd time with finish returning an IOError + c1 = conns[0] + def c1.finish + super + raise IOError + end + end + end + + should 'handle renew with exception' do + pool = @http.pool + [[DUMMY_OPEN_TIMEOUT_FOR_HOSTDOWN, %r%host down%], [DUMMY_OPEN_TIMEOUT_FOR_CONNREFUSED, %r%connection refused%]].each do |pair| + dummy_open_timeout = pair.first + error_message = pair.last + pool.with_connection do |c| + old_c = c + @http.open_timeout = dummy_open_timeout + e = assert_raises PersistentHTTP::Error do + new_c = @http.renew c + end + assert_match error_message, e.message + + # Make sure our pool is still in good shape + @http.open_timeout = 5 # Any valid timeout will do + pool.with_connection do |c1| + assert old_c != c1 + pool.with_connection do |c2| + assert old_c != c2 + end + end + end + end + end + end +# +# # def test_shutdown +# # c = connection +# # cs = conns +# # rs = reqs +# # +# # orig = @http +# # @http = PersistentHTTP.new 'name' +# # c2 = connection +# # +# # orig.shutdown +# # +# # assert c.finished? +# # refute c2.finished? +# # +# # refute_same cs, conns +# # refute_same rs, reqs +# # end +# # +# # def test_shutdown_not_started +# # c = Object.new +# # def c.finish() raise IOError end +# # +# # conns["#{@uri.host}:#{@uri.port}"] = c +# # +# # @http.shutdown +# # +# # assert_nil Thread.current[@http.connection_key] +# # assert_nil Thread.current[@http.request_key] +# # end +# # +# # def test_shutdown_no_connections +# # @http.shutdown +# # +# # assert_nil Thread.current[@http.connection_key] +# # assert_nil Thread.current[@http.request_key] +# # end +# +end +