Skip to content
Browse files

Extract easy pool.

  • Loading branch information...
1 parent be0253c commit ec56521595e778d7016027076b034793df5ff406 @i0rek i0rek committed Jun 26, 2012
View
34 lib/typhoeus.rb
@@ -9,41 +9,9 @@
module Typhoeus
extend self
+ extend Hydras::EasyPool
USER_AGENT = "Typhoeus - https://github.com/typhoeus/typhoeus"
- def easy_object_pool
- @easy_objects ||= []
- end
-
- def init_easy_object_pool
- 20.times do
- easy_object_pool << Ethon::Easy.new
- end
- end
-
- def release_easy_object(easy)
- easy.reset
- easy_object_pool << easy
- end
-
- def get_easy_object
- easy_object_pool.pop || Ethon::Easy.new
- end
-
- def add_easy_request(easy_object)
- Thread.current[:curl_multi] ||= Ethon::Multi.new
- Thread.current[:curl_multi].add(easy_object)
- end
-
- def perform_easy_requests
- multi = Thread.current[:curl_multi]
- start_time = Time.now
- multi.easy_handles.each do |easy|
- easy.start_time = start_time
- end
- multi.perform
- end
-
def configure
yield Config
end
View
5 lib/typhoeus/hydra.rb
@@ -1,3 +1,4 @@
+require 'typhoeus/hydras/easy_pool'
require 'typhoeus/hydras/easy_factory'
require 'typhoeus/hydras/memoizable'
require 'typhoeus/hydras/queueable'
@@ -8,11 +9,12 @@ module Typhoeus
# Hydra manages making parallel HTTP requests
#
class Hydra
+ include Hydras::EasyPool
include Hydras::Queueable
include Hydras::Runnable
include Hydras::Memoizable
- attr_reader :queued_requests, :max_concurrency, :easy_pool, :multi
+ attr_reader :queued_requests, :max_concurrency, :multi
class << self
def hydra
@@ -27,7 +29,6 @@ def hydra=(val)
def initialize(options = {})
@options = options
@queued_requests = []
- @easy_pool = []
@max_concurrency = @options[:max_concurrency] || 200
@multi = Ethon::Multi.new
end
View
5 lib/typhoeus/hydras/easy_factory.rb
@@ -9,7 +9,7 @@ def initialize(request, hydra)
end
def easy
- @easy ||= (hydra.easy_pool.pop || Ethon::Easy.new)
+ @easy ||= hydra.get_easy
@ifesdjeen
ifesdjeen added a note Jun 30, 2012

I would say that this code is potentially non-tread-safe on some rubies. I am not sure about internal MRI context switches, honestly. I'm not sure wether context switch may actually occur in-between the check for nil and an actual assignment.

As EasyFactory could potentially be called from the multiple threads, and the code should actually expand to a more elaborate construct. Therefore, several instances of Easy may get created here.

It's just a thought though.

@i0rek
Typhoeus member
i0rek added a note Jul 1, 2012

Thats a good point. I've already thought about typhoeus easy_pool implementation and would love to use eg https://github.com/basho/innertube. But it doesn't seem to fit, because of the asynchron nature of the easy handling in hydra.

Any other ideas?

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
end
def get
@@ -22,8 +22,7 @@ def get
def set_callback
easy.on_complete do |easy|
request.response = Response.new(easy.to_hash)
- easy.reset
- hydra.easy_pool.push easy
+ hydra.release_easy(easy)
hydra.queue(hydra.queued_requests.shift) unless hydra.queued_requests.empty?
request.complete
end
View
18 lib/typhoeus/hydras/easy_pool.rb
@@ -0,0 +1,18 @@
+module Typhoeus
+ module Hydras
+ module EasyPool
+ def easy_pool
+ @easy_pool ||= []
+ end
+
+ def release_easy(easy)
+ easy.reset
+ easy_pool << easy
+ end
+
+ def get_easy
+ easy_pool.pop || Ethon::Easy.new
+ end
+ end
+ end
+end
View
4 lib/typhoeus/requests/operations.rb
@@ -13,12 +13,12 @@ def run(url, params = {})
end
def run
- easy = Typhoeus.get_easy_object
+ easy = Typhoeus.get_easy
easy.http_request(url, options[:method] || :get, options)
easy.prepare
easy.perform
@response = Response.new(easy.to_hash)
- Typhoeus.release_easy_object(easy)
+ Typhoeus.release_easy(easy)
complete
@response
end
View
1 perf/vs_nethttp.rb
@@ -4,7 +4,6 @@
require 'benchmark'
URL = "http://localhost:300"
-Typhoeus.init_easy_object_pool
Typhoeus::Hydra.hydra = Typhoeus::Hydra.new(max_concurrency: 3)
def url_for(i)
View
41 spec/typhoeus/hydras/easy_pool_spec.rb
@@ -0,0 +1,41 @@
+require 'spec_helper'
+
+describe Typhoeus::Hydras::EasyPool do
+ let(:easy) { Ethon::Easy.new }
+
+ describe "#easy_pool" do
+ it "returns array" do
+ Typhoeus.easy_pool.should be_a(Array)
+ end
+ end
+
+ describe "#release_easy" do
+ it "resets easy" do
+ easy.expects(:reset)
+ Typhoeus.release_easy(easy)
+ end
+
+ it "puts easy back into pool" do
+ Typhoeus.release_easy(easy)
+ Typhoeus.easy_pool.should include(easy)
+ end
+ end
+
+ describe "#get_easy" do
+ context "when easy in pool" do
+ before { Typhoeus.easy_pool << easy }
+ after { Typhoeus.easy_pool.clear }
+
+ it "takes" do
+ Typhoeus.get_easy.should eq(easy)
+ end
+ end
+
+ context "when no easy in pool" do
+ it "creates" do
+ Ethon::Easy.expects(:new)
+ Typhoeus.get_easy
+ end
+ end
+ end
+end
View
4 spec/typhoeus/requests/operations_spec.rb
@@ -6,7 +6,7 @@
describe "#run" do
let(:easy) { Ethon::Easy.new }
- before { Typhoeus.expects(:get_easy_object).returns(easy) }
+ before { Typhoeus.expects(:get_easy).returns(easy) }
it "grabs an easy" do
request.run
@@ -28,7 +28,7 @@
end
it "releases easy" do
- Typhoeus.expects(:release_easy_object)
+ Typhoeus.expects(:release_easy)
request.run
end

0 comments on commit ec56521

Please sign in to comment.
Something went wrong with that request. Please try again.