Permalink
Browse files

Thread-safe persistent connections for Net::HTTP.

  • Loading branch information...
0 parents commit ddc3cdeabfeb2c374a7546e083ffdf2b93a36de0 @drbrain committed May 5, 2010
Showing with 566 additions and 0 deletions.
  1. +7 −0 .autotest
  2. +4 −0 .gitignore
  3. +6 −0 History.txt
  4. +8 −0 Manifest.txt
  5. +44 −0 README.txt
  6. +11 −0 Rakefile
  7. +27 −0 lib/net/http/faster.rb
  8. +215 −0 lib/net/http/persistent.rb
  9. +244 −0 test/test_net_http_persistent.rb
@@ -0,0 +1,7 @@
+# -*- ruby -*-
+
+require 'autotest/restart'
+
+Autotest.add_hook :initialize do |at|
+ at.testlib = 'minitest/unit'
+end
@@ -0,0 +1,4 @@
+/TAGS
+/pkg
+/doc
+*.swp
@@ -0,0 +1,6 @@
+=== 1.0.0 / 2010-05-04
+
+* 1 major enhancement
+
+ * Birthday!
+
@@ -0,0 +1,8 @@
+.autotest
+History.txt
+Manifest.txt
+README.txt
+Rakefile
+lib/net/http/faster.rb
+lib/net/http/persistent.rb
+test/test_net_http_persistent.rb
@@ -0,0 +1,44 @@
+= net_http_persistent
+
+* http://seattlerb.rubyforge.org/net-http-persistent
+
+== DESCRIPTION:
+
+Persistent connections using Net::HTTP plus a speed fix for 1.8. It's
+thread-safe too!
+
+== FEATURES/PROBLEMS:
+
+* Supports SSL
+* Thread-safe
+* Pure ruby
+* Timeout-less speed boost for 1.8 (by Aaron Patterson)
+
+== INSTALL:
+
+ gem install net-http-persistent
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2010 Eric Hodel, Aaron Patterson
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,11 @@
+# -*- ruby -*-
+
+require 'rubygems'
+require 'hoe'
+
+Hoe.spec 'net-http-persistent' do |p|
+ self.rubyforge_name = 'seattlerb'
+ developer 'Eric Hodel', 'drbrain@segment7.net'
+end
+
+# vim: syntax=Ruby
@@ -0,0 +1,27 @@
+require 'net/protocol'
+
+##
+# Aaron Patterson's monkeypatch (accepted into 1.9.1) to fix Net::HTTP's speed
+# problems.
+#
+# http://gist.github.com/251244
+
+class Net::BufferedIO #:nodoc:
+ alias :old_rbuf_fill :rbuf_fill
+
+ def rbuf_fill
+ if @io.respond_to? :read_nonblock then
+ begin
+ @rbuf << @io.read_nonblock(65536)
+ rescue Errno::EWOULDBLOCK => e
+ retry if IO.select [@io], nil, nil, @read_timeout
+ raise Timeout::Error, e.message
+ end
+ else # SSL sockets do not have read_nonblock
+ timeout @read_timeout do
+ @rbuf << @io.sysread(65536)
+ end
+ end
+ end
+end
+
@@ -0,0 +1,215 @@
+require 'net/http'
+require 'net/http/faster'
+require 'uri'
+
+##
+# Persistent connections for Net::HTTP
+#
+# Net::HTTP::Persistent maintains persistent connections across all the
+# servers you wish to talk to. For each host:port you communicate with a
+# single persistent connection is created.
+#
+# Multiple Net::HTTP::Persistent objects will share the same set of
+# connections.
+#
+# Example:
+#
+# uri = URI.parse 'http://example.com/awesome/web/service'
+# http = Net::HTTP::Persistent
+# stuff = http.request uri # performs a GET
+#
+# # perform a POST
+# post_uri = uri + 'create'
+# post = Net::HTTP::Post.new uri.path
+# post.set_form_data 'some' => 'cool data'
+# http.request post_uri, post # URI is always required
+
+class Net::HTTP::Persistent
+
+ ##
+ # The version of Net::HTTP::Persistent use are using
+
+ VERSION = '1.0'
+
+ ##
+ # Error class for errors raised by Net::HTTP::Persistent. Various
+ # SystemCallErrors are re-raised with a human-readable message under this
+ # class.
+
+ class Error < StandardError; end
+
+ ##
+ # This client's OpenSSL::X509::Certificate
+
+ attr_accessor :certificate
+
+ ##
+ # An SSL certificate authority. Setting this will set verify_mode to
+ # VERIFY_PEER.
+
+ attr_accessor :ca_file
+
+ ##
+ # Headers that are added to every request
+
+ attr_reader :headers
+
+ ##
+ # The value sent in the Keep-Alive header. Defaults to 30 seconds
+
+ attr_accessor :keep_alive
+
+ ##
+ # This client's SSL private key
+
+ attr_accessor :private_key
+
+ ##
+ # SSL verification callback. Used when ca_file is set.
+
+ attr_accessor :verify_callback
+
+ ##
+ # HTTPS verify mode. Set to OpenSSL::SSL::VERIFY_NONE to ignore certificate
+ # problems.
+ #
+ # You can use +verify_mode+ to override any default values.
+
+ attr_accessor :verify_mode
+
+ def initialize # :nodoc:
+ @keep_alive = 30
+ @headers = {}
+
+ @certificate = nil
+ @ca_file = nil
+ @private_key = nil
+ @verify_callback = nil
+ @verify_mode = nil
+ end
+
+ ##
+ # Creates a new connection for +uri+
+
+ def connection_for uri
+ Thread.current[:net_http_persistent_connections] ||= {}
+ connections = Thread.current[:net_http_persistent_connections]
+
+ connection_id = [uri.host, uri.port].join ':'
+
+ connections[connection_id] ||= Net::HTTP.new uri.host, uri.port
+ connection = connections[connection_id]
+
+ ssl connection if uri.scheme == 'https' and not connection.started?
+
+ connection.start unless connection.started?
+
+ connection
+ rescue Errno::ECONNREFUSED
+ raise Error, "connection refused: #{connection.address}:#{connection.port}"
+ end
+
+ ##
+ # Returns an error message containing the number of requests performed on
+ # this connection
+
+ def error_message connection
+ requests =
+ Thread.current[:net_http_persistent_requests][connection.object_id]
+
+ "after #{requests} requests on #{connection.object_id}"
+ end
+
+ ##
+ # Finishes then restarts the Net::HTTP +connection+
+
+ def reset connection
+ Thread.current[:net_http_persistent_requests].delete connection.object_id
+
+ begin
+ connection.finish
+ rescue IOError
+ end
+
+ connection.start
+ rescue Errno::ECONNREFUSED
+ raise Error, "connection refused: #{connection.address}:#{connection.port}"
+ rescue Errno::EHOSTDOWN
+ raise Error, "host down: #{connection.address}:#{connection.port}"
+ end
+
+ ##
+ # Makes a request on +uri+. If +req+ is nil a Net::HTTP::Get is performed
+ # against +uri+.
+ #
+ # +req+ must be a Net::HTTPRequest subclass (see Net::HTTP for a list).
+
+ def request uri, req = nil
+ Thread.current[:net_http_persistent_requests] ||= Hash.new 0
+ retried = false
+ bad_response = false
+
+ req = Net::HTTP::Get.new uri.request_uri unless req
+
+ headers.each do |pair|
+ req.add_field(*pair)
+ end
+
+ req.add_field 'Connection', 'keep-alive'
+ req.add_field 'Keep-Alive', @keep_alive
+
+ connection = connection_for uri
+ connection_id = connection.object_id
+
+ begin
+ count = Thread.current[:net_http_persistent_requests][connection_id] += 1
+ response = connection.request req
+
+ rescue Net::HTTPBadResponse => e
+ message = error_message connection
+
+ reset connection
+
+ raise Error, "too many bad responses #{message}" if bad_response
+
+ bad_response = true
+ retry
+ rescue IOError, EOFError, Timeout::Error,
+ Errno::ECONNABORTED, Errno::ECONNRESET, Errno::EPIPE => e
+ due_to = "(due to #{e.message} - #{e.class})"
+ message = error_message connection
+
+ reset connection
+
+ raise Error, "too many connection resets #{due_to} #{message}" if retried
+
+ retried = true
+ retry
+ end
+
+ response
+ end
+
+ ##
+ # Enables SSL on +connection+
+
+ def ssl connection
+ require 'net/https'
+ connection.use_ssl = true
+
+ if @ca_file then
+ connection.ca_file = @ca_file
+ connection.verify_mode = OpenSSL::SSL::VERIFY_PEER
+ connection.verify_callback = @verify_callback if @verify_callback
+ end
+
+ if @certificate and @private_key then
+ connection.cert = @certificate
+ connection.key = @private_key
+ end
+
+ connection.verify_mode = @verify_mode if @verify_mode
+ end
+
+end
+
Oops, something went wrong.

0 comments on commit ddc3cde

Please sign in to comment.