benchmarks/profile.rb examples/twitter.rb lib/typhoeus/response.rb @@ -11,4 +11,7 @@ task :install do rm_rf "*.gem" puts `gem build typhoeus.gemspec` puts `sudo gem install typhoeus-#{Typhoeus::VERSION}.gem` -end \ No newline at end of file +end + +desc "Run all the tests" +task :default => :spec Rakefile @@ -10,24 +10,26 @@ calls = 20 include Typhoeus end +Typhoeus.init_easy_objects + benchmark do |t| - t.report("httpmachine") do + t.report("net::http") do responses = [] - calls.times do - responses << @klass.get("http://127.0.0.1:3000") + calls.times do |i| + responses << open("http://127.0.0.1:3000/#{i}").read end - responses.each {|r| raise unless r.response_body == "whatever"} + responses.each {|r| raise unless r == "whatever"} end - - t.report("net::http") do + + t.report("typhoeus") do responses = [] - calls.times do - responses << open("http://127.0.0.1:3000").read + calls.times do |i| + responses << @klass.get("http://127.0.0.1:3000/#{i}") end - responses.each {|r| raise unless r == "whatever"} + responses.each {|r| raise unless r.body == "whatever"} end end benchmarks/vs_nethttp.rb @@ -76,23 +76,6 @@ static VALUE easy_perform(VALUE self) { return Qnil; } -static VALUE easy_reset(VALUE self) { - CurlEasy *curl_easy; - Data_Get_Struct(self, CurlEasy, curl_easy); - - curl_easy_reset(curl_easy->curl); - - if (curl_easy->request_chunk != NULL) { - free(curl_easy->request_chunk); - } - - if (curl_easy->headers != NULL) { - curl_slist_free_all(curl_easy->headers); - } - - return Qnil; -} - static size_t write_data_handler(char *stream, size_t size, size_t nmemb, VALUE val) { rb_funcall(val, idAppend, 1, rb_str_new(stream, size * nmemb)); return size * nmemb; @@ -114,6 +97,37 @@ static size_t read_callback(void *ptr, size_t size, size_t nmemb, void *data) { return realsize; } +static void set_response_handlers(VALUE easy, CURL *curl) { + rb_iv_set(easy, "@response_body", rb_str_new2("")); + rb_iv_set(easy, "@response_header", rb_str_new2("")); + + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, (curl_write_callback)&write_data_handler); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, rb_iv_get(easy, "@response_body")); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, (curl_write_callback)&write_data_handler); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, rb_iv_get(easy, "@response_header")); + + return NULL; +} + +static VALUE easy_reset(VALUE self) { + CurlEasy *curl_easy; + Data_Get_Struct(self, CurlEasy, curl_easy); + + curl_easy_reset(curl_easy->curl); + + if (curl_easy->request_chunk != NULL) { + free(curl_easy->request_chunk); + } + + if (curl_easy->headers != NULL) { + curl_slist_free_all(curl_easy->headers); + } + + set_response_handlers(self, curl_easy->curl); + + return Qnil; +} + static VALUE easy_add_header(VALUE self, VALUE header) { CurlEasy *curl_easy; Data_Get_Struct(self, CurlEasy, curl_easy); @@ -154,6 +168,10 @@ static VALUE easy_escape(VALUE self, VALUE data, VALUE length) { return rb_str_new2(curl_easy_escape(curl_easy->curl, StringValuePtr(data), NUM2INT(length))); } +static VALUE version(VALUE self) { + return rb_str_new2(curl_version()); +} + static VALUE new(int argc, VALUE *argv, VALUE klass) { CURL *curl = curl_easy_init(); CurlEasy *curl_easy = ALLOC(CurlEasy); @@ -161,14 +179,8 @@ static VALUE new(int argc, VALUE *argv, VALUE klass) { curl_easy->headers = NULL; curl_easy->request_chunk = NULL; VALUE easy = Data_Wrap_Struct(cTyphoeusEasy, 0, dealloc, curl_easy); - - rb_iv_set(easy, "@response_body", rb_str_new2("")); - rb_iv_set(easy, "@response_header", rb_str_new2("")); - - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, (curl_write_callback)&write_data_handler); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, rb_iv_get(easy, "@response_body")); - curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, (curl_write_callback)&write_data_handler); - curl_easy_setopt(curl, CURLOPT_HEADERDATA, rb_iv_get(easy, "@response_header")); + + set_response_handlers(easy, curl); rb_obj_call_init(easy, argc, argv); @@ -190,4 +202,5 @@ void init_typhoeus_easy() { rb_define_private_method(klass, "easy_set_headers", easy_set_headers, 0); rb_define_private_method(klass, "easy_add_header", easy_add_header, 1); rb_define_private_method(klass, "easy_escape", easy_escape, 2); + rb_define_private_method(klass, "version", version, 0); } \ No newline at end of file ext/typhoeus/typhoeus_easy.c @@ -9,10 +9,34 @@ require 'typhoeus/filter' require 'typhoeus/remote_method' require 'typhoeus/remote' require 'typhoeus/remote_proxy_object' +require 'typhoeus/response' module Typhoeus VERSION = "0.0.6" + def self.easy_object_pool + @easy_objects ||= [] + end + + def self.init_easy_object_pool + 20.times do + easy_object_pool << Typhoeus::Easy.new + end + end + + def self.release_easy_object(easy) + easy.reset + easy_object_pool << easy + end + + def self.get_easy_object + if easy_object_pool.empty? + Typhoeus::Easy.new + else + easy_object_pool.pop + end + end + def self.add_easy_request(easy_object) Thread.current[:curl_multi] ||= Typhoeus::Multi.new Thread.current[:curl_multi].add(easy_object) lib/typhoeus.rb @@ -171,6 +171,9 @@ module Typhoeus end def reset + @response_code = 0 + @response_header = "" + @response_body = "" easy_reset() end @@ -185,5 +188,9 @@ module Typhoeus def get_info_double(option) easy_getinfo_double(option) end + + def curl_version + version + end end end \ No newline at end of file lib/typhoeus/easy.rb @@ -23,13 +23,14 @@ module Typhoeus end def remote_proxy_object(url, method, options) - easy = Typhoeus::Easy.new + easy = Typhoeus.get_easy_object easy.url = url easy.method = method easy.headers["User-Agent"] = (options[:user_agent] || Typhoeus::USER_AGENT) easy.params = options[:params] if options[:params] easy.request_body = options[:body] if options[:body] + easy.timeout = options[:timeout] if options[:timeout] easy lib/typhoeus/remote.rb @@ -21,13 +21,16 @@ module Typhoeus unless @proxied_object Typhoeus.perform_easy_requests + response = Response.new(@easy.response_code, @easy.response_header, @easy.response_body, @easy.total_time_taken) if @easy.response_code >= 200 && @easy.response_code < 300 - @proxied_object = @success.nil? ? @easy : @success.call(@easy) + Typhoeus.release_easy_object(@easy) + @proxied_object = @success.nil? ? response : @success.call(response) + if @cache && @cache_key @cache.set(@cache_key, @proxied_object, @timeout) end else - @proxied_object = @failure.nil? ? @easy : @failure.call(@easy) + @proxied_object = @failure.nil? ? response : @failure.call(response) end @clear_memoized_store_proc.call end lib/typhoeus/remote_proxy_object.rb @@ -4,7 +4,7 @@ require 'sinatra' get '/**' do puts request.inspect puts "**#{request.body.read}**" - sleep 1 + sleep 3 "hello world" end spec/servers/app.rb @@ -9,7 +9,7 @@ class DelayFixtureServer < EventMachine::Connection def process_http_request EventMachine.stop if ENV["PATH_INFO"] == "/die" - puts "got a request" + puts "got a request #{ENV['PATH_INFO']}" resp = EventMachine::DelegatedHttpResponse.new( self ) # Block which fulfills the request @@ -56,7 +56,7 @@ end port = (ARGV[0] || 3000).to_i -DelayFixtureServer.response_delay = 0.1 +DelayFixtureServer.response_delay = 0.5 DelayFixtureServer.response_number = 0 #DelayFixtureServer.response_fixture = File.read(File.dirname(__FILE__) + "/../fixtures/result_set.xml") spec/servers/delay_fixture_server.rb @@ -5,10 +5,6 @@ describe Typhoeus::RemoteProxyObject do @easy = Typhoeus::Easy.new @easy.method = :get @easy.url = "http://localhost:3001" - @klass = Class.new do - def self.clear_memoized_proxy_objects - end - end end before(:all) do @@ -20,51 +16,52 @@ describe Typhoeus::RemoteProxyObject do end it "should take a caller and call the clear_memoized_proxy_objects" do - @klass.should_receive(:clear_memoized_proxy_objects) - easy = Typhoeus::RemoteProxyObject.new(@klass, @easy) - easy.response_code.should == 200 + clear_proxy = lambda {} + clear_proxy.should_receive(:call) + response = Typhoeus::RemoteProxyObject.new(clear_proxy, @easy) + response.code.should == 200 end it "should take an easy object and return the body when requested" do - easy = Typhoeus::RemoteProxyObject.new(@klass, @easy) + response = Typhoeus::RemoteProxyObject.new(lambda {}, @easy) @easy.response_code.should == 0 - easy.response_code.should == 200 + response.code.should == 200 end it "should perform requests only on the first access" do - easy = Typhoeus::RemoteProxyObject.new(@klass, @easy) - easy.response_code.should == 200 + response = Typhoeus::RemoteProxyObject.new(lambda {}, @easy) + response.code.should == 200 Typhoeus.should_receive(:perform_easy_requests).exactly(0).times - easy.response_code.should == 200 + response.code.should == 200 end it "should call the on_success method with an easy object and proxy to the result of on_success" do klass = Class.new do - def initialize(e) - @easy = e + def initialize(r) + @response = r end def blah - @easy.response_code + @response.code end end - k = Typhoeus::RemoteProxyObject.new(@klass, @easy, :on_success => lambda {|e| klass.new(e)}) + k = Typhoeus::RemoteProxyObject.new(lambda {}, @easy, :on_success => lambda {|e| klass.new(e)}) k.blah.should == 200 end it "should call the on_failure method with an easy object and proxy to the result of on_failure" do klass = Class.new do - def initialize(e) - @easy = e + def initialize(r) + @response = r end def blah - @easy.response_code + @response.code end end @easy.url = "http://localhost:3002" #bad port - k = Typhoeus::RemoteProxyObject.new(@klass, @easy, :on_failure => lambda {|e| klass.new(e)}) + k = Typhoeus::RemoteProxyObject.new(lambda {}, @easy, :on_failure => lambda {|e| klass.new(e)}) k.blah.should == 0 end end \ No newline at end of file spec/typhoeus/remote_proxy_object_spec.rb @@ -18,45 +18,45 @@ describe Typhoeus do describe "get" do it "should add a get method" do easy = @klass.get("http://localhost:3001/posts.xml") - easy.response_code.should == 200 - easy.response_body.should include("REQUEST_METHOD=GET") - easy.response_body.should include("REQUEST_URI=/posts.xml") + easy.code.should == 200 + easy.body.should include("REQUEST_METHOD=GET") + easy.body.should include("REQUEST_URI=/posts.xml") end it "should take passed in params and add them to the query string" do easy = @klass.get("http://localhost:3001", {:params => {:foo => :bar}}) - easy.response_body.should include("QUERY_STRING=foo=bar") + easy.body.should include("QUERY_STRING=foo=bar") end end # get describe "post" do it "should add a post method" do easy = @klass.post("http://localhost:3001/posts.xml", {:params => {:post => {:author => "paul", :title => "a title", :body => "a body"}}}) - easy.response_code.should == 200 - easy.response_body.should include("post%5Bbody%5D=a+body") - easy.response_body.should include("post%5Bauthor%5D=paul") - easy.response_body.should include("post%5Btitle%5D=a+title") - easy.response_body.should include("REQUEST_METHOD=POST") + easy.code.should == 200 + easy.body.should include("post%5Bbody%5D=a+body") + easy.body.should include("post%5Bauthor%5D=paul") + easy.body.should include("post%5Btitle%5D=a+title") + easy.body.should include("REQUEST_METHOD=POST") end it "should add a body" do easy = @klass.post("http://localhost:3001/posts.xml", {:body => "this is a request body"}) - easy.response_code.should == 200 - easy.response_body.should include("this is a request body") - easy.response_body.should include("REQUEST_METHOD=POST") + easy.code.should == 200 + easy.body.should include("this is a request body") + easy.body.should include("REQUEST_METHOD=POST") end end # post it "should add a put method" do easy = @klass.put("http://localhost:3001/posts/3.xml") - easy.response_code.should == 200 - easy.response_body.should include("REQUEST_METHOD=PUT") + easy.code.should == 200 + easy.body.should include("REQUEST_METHOD=PUT") end it "should add a delete method" do easy = @klass.delete("http://localhost:3001/posts/3.xml") - easy.response_code.should == 200 - easy.response_body.should include("REQUEST_METHOD=DELETE") + easy.code.should == 200 + easy.body.should include("REQUEST_METHOD=DELETE") end describe "#define_remote_method" do @@ -211,7 +211,7 @@ describe Typhoeus do describe "on_success" do it "should take :on_success as an argument" do @klass.instance_eval do - define_remote_method :do_stuff, :base_uri => "http://localhost:3001", :on_success => lambda {|e| e.response_code.should == 200; :foo} + define_remote_method :do_stuff, :base_uri => "http://localhost:3001", :on_success => lambda {|e| e.code.should == 200; :foo} end @klass.do_stuff.should == :foo @@ -219,7 +219,7 @@ describe Typhoeus do it "should use default_on_success if no on_success provided" do @klass.instance_eval do - remote_defaults :on_success => lambda {|e| e.response_code.should == 200; :foo} + remote_defaults :on_success => lambda {|e| e.code.should == 200; :foo} define_remote_method :do_stuff, :base_uri => "http://localhost:3001" end @@ -229,7 +229,7 @@ describe Typhoeus do it "should override default_on_success if on_success is provided" do @klass.instance_eval do remote_defaults :on_success => lambda {|e| :foo} - define_remote_method :do_stuff, :base_uri => "http://localhost:3001", :on_success => lambda {|e| e.response_code.should == 200; :bar} + define_remote_method :do_stuff, :base_uri => "http://localhost:3001", :on_success => lambda {|e| e.code.should == 200; :bar} end @klass.do_stuff.should == :bar @@ -239,7 +239,7 @@ describe Typhoeus do describe "on_failure" do it "should take :on_failure as an argument" do @klass.instance_eval do - define_remote_method :do_stuff, :base_uri => "http://localhost:9999", :on_failure => lambda {|e| e.response_code.should == 0; :foo} + define_remote_method :do_stuff, :base_uri => "http://localhost:9999", :on_failure => lambda {|e| e.code.should == 0; :foo} end @klass.do_stuff.should == :foo @@ -247,7 +247,7 @@ describe Typhoeus do it "should use default_on_failure if no on_success provided" do @klass.instance_eval do - remote_defaults :on_failure => lambda {|e| e.response_code.should == 0; :bar} + remote_defaults :on_failure => lambda {|e| e.code.should == 0; :bar} define_remote_method :do_stuff, :base_uri => "http://localhost:9999" end @@ -257,7 +257,7 @@ describe Typhoeus do it "should override default_on_failure if no method is provided" do @klass.instance_eval do remote_defaults :on_failure => lambda {|e| :foo} - define_remote_method :do_stuff, :base_uri => "http://localhost:9999", :on_failure => lambda {|e| e.response_code.should == 0; :bar} + define_remote_method :do_stuff, :base_uri => "http://localhost:9999", :on_failure => lambda {|e| e.code.should == 0; :bar} end @klass.do_stuff.should == :bar @@ -270,7 +270,7 @@ describe Typhoeus do define_remote_method :do_stuff, :base_uri => "http://localhost:3001", :params => {:foo => :bar} end - @klass.do_stuff.response_body.should include("QUERY_STRING=foo=bar") + @klass.do_stuff.body.should include("QUERY_STRING=foo=bar") end it "should add :params from remote method definition with params passed in when called" do @@ -278,7 +278,7 @@ describe Typhoeus do define_remote_method :do_stuff, :base_uri => "http://localhost:3001", :params => {:foo => :bar} end - @klass.do_stuff(:params => {:asdf => :jkl}).response_body.should include("QUERY_STRING=asdf=jkl&foo=bar") + @klass.do_stuff(:params => {:asdf => :jkl}).body.should include("QUERY_STRING=foo=bar&asdf=jkl") end end spec/typhoeus/remote_spec.rb spec/typhoeus/http_machine_spec.rb 60a23f380e949a099b3a03a54505d598c0579668 Paul Dix paul@pauldix.net http://github.com/pauldix/typhoeus/commit/1a60706ae912fba95958608fd684d046dc77dd51 1a60706ae912fba95958608fd684d046dc77dd51 2009-05-07T09:36:10-07:00 2009-05-07T09:36:10-07:00 sweet zombie jesus, this is a huge commit. Got too caught up hacking new ideas while at the conf. Main change is the addition of a response object and the easy object pool 37eed6c0ab284943ec8b7a7e04238bde788fde8e Paul Dix paul@pauldix.net