Skip to content

Commit

Permalink
Merge pull request #15 from spagalloco/downstream_forks
Browse files Browse the repository at this point in the history 
Downstream forks
  • Loading branch information
@voloko
voloko committed Jun 22, 2011
2 parents 0305636 + 161f82c commit e13023c
Show file tree
Hide file tree
Showing 5 changed files with 191 additions and 83 deletions.
18 changes: 4 additions & 14 deletions fixtures/twitter/basic_http.txt → fixtures/twitter/tweets.txt
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,4 @@
HTTP/1.1 200 OK
Content-Type: application/json
Transfer-Encoding: chunked
Server: Jetty(6.1.17)

{"text":"Just wanted the world to know what our theologically deep pastor @bprentice has for a desktop background. http://yfrog.com/1rl4ij","favorited":false,"in_reply_to_user_id":null,"in_reply_to_screen_name":null,"source":"<a href=\"http://www.atebits.com/\" rel=\"nofollow\">Tweetie</a>","truncated":false,"created_at":"Thu Oct 08 19:34:09 +0000 2009","geo":null,"user":{"geo_enabled":false,"profile_text_color":"5b5252","location":"Stillwater, Oklahoma","statuses_count":122,"followers_count":70,"profile_link_color":"220099","description":"Taking an unchanging Savior to a changing world! Eagle Heights in Stillwater, Oklahoma.","following":null,"friends_count":136,"profile_sidebar_fill_color":"f37a20","url":"http://www.eagleheights.com","profile_image_url":"http://a3.twimg.com/profile_images/249941843/online_logo_normal.jpg","verified":false,"notifications":null,"favourites_count":0,"profile_sidebar_border_color":"c5f109","protected":false,"screen_name":"eagleheights","profile_background_tile":false,"profile_background_image_url":"http://a1.twimg.com/profile_background_images/5935314/EHBC_LOGO.jpg","created_at":"Tue Mar 17 14:52:04 +0000 2009","name":"Eagle Heights","time_zone":"Central Time (US & Canada)","id":24892440,"utc_offset":-21600,"profile_background_color":"3d3d3d"},"id":4714982187,"in_reply_to_status_id":null}

{"text":"I finally took a good pic of our resident BobCat @ the JakeCruise Ranch: http://yfrog.com/769vij","favorited":false,"in_reply_to_user_id":null,"in_reply_to_screen_name":null,"source":"<a href=\"http://www.atebits.com/\" rel=\"nofollow\">Tweetie</a>","truncated":false,"created_at":"Thu Oct 08 19:34:12 +0000 2009","geo":null,"user":{"geo_enabled":false,"profile_text_color":"141318","location":"West Hollywood, CA","statuses_count":265,"followers_count":80,"profile_link_color":"DA4425","description":"Gay Programmer in SoCal","following":null,"friends_count":37,"profile_sidebar_fill_color":"5DD2F4","url":"http://www.kelvo.com/","profile_image_url":"http://a1.twimg.com/profile_images/197950488/kelvis_green_current_normal.jpeg","verified":false,"notifications":null,"favourites_count":1,"profile_sidebar_border_color":"1F6926","protected":false,"screen_name":"KelvisWeHo","profile_background_tile":false,"profile_background_image_url":"http://a1.twimg.com/profile_background_images/9898116/thaneeya3.jpg","created_at":"Fri Apr 17 18:44:57 +0000 2009","name":"Kelvis Del Rio","time_zone":"Pacific Time (US & Canada)","id":32517583,"utc_offset":-28800,"profile_background_color":"cccccc"},"id":4714983168,"in_reply_to_status_id":null}

{"text":"Thursdays are long. But at least we're allowed a cheat sheet for stats. http://yfrog.com/9gjc7j","favorited":false,"in_reply_to_user_id":null,"in_reply_to_screen_name":null,"source":"<a href=\"http://twitterrific.com\" rel=\"nofollow\">Twitterrific</a>","truncated":false,"created_at":"Thu Oct 08 19:34:21 +0000 2009","geo":null,"user":{"geo_enabled":false,"profile_text_color":"663B12","location":"Calgary ","statuses_count":68,"followers_count":10,"profile_link_color":"1F98C7","description":"pure. love. art. music. ","following":null,"friends_count":22,"profile_sidebar_fill_color":"DAECF4","url":null,"profile_image_url":"http://a3.twimg.com/profile_images/391882135/Photo_550_normal.jpg","verified":false,"notifications":null,"favourites_count":5,"profile_sidebar_border_color":"C6E2EE","protected":false,"screen_name":"KarinRudmik","profile_background_tile":false,"profile_background_image_url":"http://a3.twimg.com/profile_background_images/33520483/shimmering__by_AnBystrowska.jpg","created_at":"Wed Jul 29 02:37:13 +0000 2009","name":"Karin Rudmik","time_zone":"Mountain Time (US & Canada)","id":61091098,"utc_offset":-25200,"profile_background_color":"C6E2EE"},"id":4714986148,"in_reply_to_status_id":null}



{"text":"acabando de almorzar, les comparto mi ultima creacion: http://yfrog.com/5mi94j pollo a la maracuya, con verduras sofritas ^^","favorited":false,"in_reply_to_user_id":null,"in_reply_to_screen_name":null,"source":"<a href=\"http://www.seesmic.com/\" rel=\"nofollow\">Seesmic</a>","truncated":false,"created_at":"Thu Oct 08 19:34:28 +0000 2009","geo":null,"user":{"geo_enabled":false,"profile_text_color":"3C3940","location":"Cartagena de Indias","statuses_count":1016,"followers_count":190,"profile_link_color":"0099B9","description":"Cartagenero extremadamente consentido, flojo y amante del anime, el manga, la cocina y los mmorpgs. pdt: y de los postres por supuesto!!!","following":null,"friends_count":253,"profile_sidebar_fill_color":"95E8EC","url":"http://www.flickr.com/photos/lobitokun/","profile_image_url":"http://a1.twimg.com/profile_images/451679242/shippo_normal.jpg","verified":false,"notifications":null,"favourites_count":9,"profile_sidebar_border_color":"5ED4DC","protected":false,"screen_name":"lobitokun","profile_background_tile":false,"profile_background_image_url":"http://a3.twimg.com/profile_background_images/24664583/hideki1.jpg","created_at":"Fri Jul 17 15:32:14 +0000 2009","name":"emiro gomez beltran","time_zone":"Bogota","id":57672295,"utc_offset":-18000,"profile_background_color":"0099B9"},"id":4714988998,"in_reply_to_status_id":null}
{"text":"Just wanted the world to know what our theologically deep pastor @bprentice has for a desktop background. http://yfrog.com/1rl4ij","favorited":false,"in_reply_to_user_id":null,"in_reply_to_screen_name":null,"source":"<a href=\"http://www.atebits.com/\" rel=\"nofollow\">Tweetie</a>","truncated":false,"created_at":"Thu Oct 08 19:34:09 +0000 2009","geo":null,"user":{"geo_enabled":false,"profile_text_color":"5b5252","location":"Stillwater, Oklahoma","statuses_count":122,"followers_count":70,"profile_link_color":"220099","description":"Taking an unchanging Savior to a changing world! Eagle Heights in Stillwater, Oklahoma.","following":null,"friends_count":136,"profile_sidebar_fill_color":"f37a20","url":"http://www.eagleheights.com","profile_image_url":"http://a3.twimg.com/profile_images/249941843/online_logo_normal.jpg","verified":false,"notifications":null,"favourites_count":0,"profile_sidebar_border_color":"c5f109","protected":false,"screen_name":"eagleheights","profile_background_tile":false,"profile_background_image_url":"http://a1.twimg.com/profile_background_images/5935314/EHBC_LOGO.jpg","created_at":"Tue Mar 17 14:52:04 +0000 2009","name":"Eagle Heights","time_zone":"Central Time (US & Canada)","id":24892440,"utc_offset":-21600,"profile_background_color":"3d3d3d"},"id":4714982187,"in_reply_to_status_id":null}
{"text":"I finally took a good pic of our resident BobCat @ the JakeCruise Ranch: http://yfrog.com/769vij","favorited":false,"in_reply_to_user_id":null,"in_reply_to_screen_name":null,"source":"<a href=\"http://www.atebits.com/\" rel=\"nofollow\">Tweetie</a>","truncated":false,"created_at":"Thu Oct 08 19:34:12 +0000 2009","geo":null,"user":{"geo_enabled":false,"profile_text_color":"141318","location":"West Hollywood, CA","statuses_count":265,"followers_count":80,"profile_link_color":"DA4425","description":"Gay Programmer in SoCal","following":null,"friends_count":37,"profile_sidebar_fill_color":"5DD2F4","url":"http://www.kelvo.com/","profile_image_url":"http://a1.twimg.com/profile_images/197950488/kelvis_green_current_normal.jpeg","verified":false,"notifications":null,"favourites_count":1,"profile_sidebar_border_color":"1F6926","protected":false,"screen_name":"KelvisWeHo","profile_background_tile":false,"profile_background_image_url":"http://a1.twimg.com/profile_background_images/9898116/thaneeya3.jpg","created_at":"Fri Apr 17 18:44:57 +0000 2009","name":"Kelvis Del Rio","time_zone":"Pacific Time (US & Canada)","id":32517583,"utc_offset":-28800,"profile_background_color":"cccccc"},"id":4714983168,"in_reply_to_status_id":null}
{"text":"Thursdays are long. But at least we're allowed a cheat sheet for stats. http://yfrog.com/9gjc7j","favorited":false,"in_reply_to_user_id":null,"in_reply_to_screen_name":null,"source":"<a href=\"http://twitterrific.com\" rel=\"nofollow\">Twitterrific</a>","truncated":false,"created_at":"Thu Oct 08 19:34:21 +0000 2009","geo":null,"user":{"geo_enabled":false,"profile_text_color":"663B12","location":"Calgary ","statuses_count":68,"followers_count":10,"profile_link_color":"1F98C7","description":"pure. love. art. music. ","following":null,"friends_count":22,"profile_sidebar_fill_color":"DAECF4","url":null,"profile_image_url":"http://a3.twimg.com/profile_images/391882135/Photo_550_normal.jpg","verified":false,"notifications":null,"favourites_count":5,"profile_sidebar_border_color":"C6E2EE","protected":false,"screen_name":"KarinRudmik","profile_background_tile":false,"profile_background_image_url":"http://a3.twimg.com/profile_background_images/33520483/shimmering__by_AnBystrowska.jpg","created_at":"Wed Jul 29 02:37:13 +0000 2009","name":"Karin Rudmik","time_zone":"Mountain Time (US & Canada)","id":61091098,"utc_offset":-25200,"profile_background_color":"C6E2EE"},"id":4714986148,"in_reply_to_status_id":null}
{"text":"acabando de almorzar, les comparto mi ultima creacion: http://yfrog.com/5mi94j pollo a la maracuya, con verduras sofritas ^^","favorited":false,"in_reply_to_user_id":null,"in_reply_to_screen_name":null,"source":"<a href=\"http://www.seesmic.com/\" rel=\"nofollow\">Seesmic</a>","truncated":false,"created_at":"Thu Oct 08 19:34:28 +0000 2009","geo":null,"user":{"geo_enabled":false,"profile_text_color":"3C3940","location":"Cartagena de Indias","statuses_count":1016,"followers_count":190,"profile_link_color":"0099B9","description":"Cartagenero extremadamente consentido, flojo y amante del anime, el manga, la cocina y los mmorpgs. pdt: y de los postres por supuesto!!!","following":null,"friends_count":253,"profile_sidebar_fill_color":"95E8EC","url":"http://www.flickr.com/photos/lobitokun/","profile_image_url":"http://a1.twimg.com/profile_images/451679242/shippo_normal.jpg","verified":false,"notifications":null,"favourites_count":9,"profile_sidebar_border_color":"5ED4DC","protected":false,"screen_name":"lobitokun","profile_background_tile":false,"profile_background_image_url":"http://a3.twimg.com/profile_background_images/24664583/hideki1.jpg","created_at":"Fri Jul 17 15:32:14 +0000 2009","name":"emiro gomez beltran","time_zone":"Bogota","id":57672295,"utc_offset":-18000,"profile_background_color":"0099B9"},"id":4714988998,"in_reply_to_status_id":null}
141 changes: 78 additions & 63 deletions lib/twitter/json_stream.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
require 'em/buftok'
require 'uri'
require 'simple_oauth'
require 'http/parser'

module Twitter
class JSONStream < EventMachine::Connection
Expand All @@ -20,21 +21,22 @@ class JSONStream < EventMachine::Connection
RETRIES_MAX = 10

DEFAULT_OPTIONS = {
:method => 'GET',
:path => '/',
:content_type => "application/x-www-form-urlencoded",
:content => '',
:path => '/1/statuses/filter.json',
:host => 'stream.twitter.com',
:port => 80,
:ssl => false,
:user_agent => 'TwitterStream',
:timeout => 0,
:proxy => ENV['HTTP_PROXY'],
:auth => nil,
:oauth => {},
:filters => [],
:params => {},
:method => 'GET',
:path => '/',
:content_type => "application/x-www-form-urlencoded",
:content => '',
:path => '/1/statuses/filter.json',
:host => 'stream.twitter.com',
:port => 80,
:ssl => false,
:user_agent => 'TwitterStream',
:timeout => 0,
:proxy => ENV['HTTP_PROXY'],
:auth => nil,
:oauth => {},
:filters => [],
:params => {},
:auto_reconnect => true
}

attr_accessor :code
Expand Down Expand Up @@ -89,6 +91,10 @@ def on_max_reconnects &block
@max_reconnects_callback = block
end

def on_close &block
@close_callback = block
end

def stop
@gracefully_closed = true
close_connection
Expand All @@ -101,20 +107,18 @@ def immediate_reconnect
end

def unbind
receive_line(@buffer.flush) unless @buffer.empty?
schedule_reconnect unless @gracefully_closed
if @state == :stream && !@buffer.empty?
parse_stream_line(@buffer.flush)
end
schedule_reconnect if @options[:auto_reconnect] && !@gracefully_closed
@close_callback.call if @close_callback

end

def receive_data data
begin
@buffer.extract(data).each do |line|
receive_line(line)
end
rescue Exception => e
receive_error("#{e.class}: " + [e.message, e.backtrace].flatten.join("\n\t"))
close_connection
return
end
# Receives raw data from the HTTP connection and pushes it into the
# HTTP parser which then drives subsequent callbacks.
def receive_data(data)
@parser << data
end

def connection_completed
Expand Down Expand Up @@ -175,9 +179,40 @@ def reconnect_timeout
def reset_state
set_comm_inactivity_timeout @options[:timeout] if @options[:timeout] > 0
@code = 0
@headers = []
@headers = {}
@state = :init
@buffer = BufferedTokenizer.new("\r", MAX_LINE_LENGTH)
@stream = ''

@parser = Http::Parser.new
@parser.on_headers_complete = method(:handle_headers_complete)
@parser.on_body = method(:receive_stream_data)
end

# Called when the status line and all headers have been read from the
# stream.
def handle_headers_complete(headers)
@code = @parser.status_code.to_i
if @code != 200
receive_error("invalid status code: #{@code}.")
end
self.headers = headers
@state = :stream
end

# Called every time a chunk of data is read from the connection once it has
# been opened and after the headers have been processed.
def receive_stream_data(data)
begin
@buffer.extract(data).each do |line|
parse_stream_line(line)
end
@stream = ''
rescue Exception => e
receive_error("#{e.class}: " + [e.message, e.backtrace].flatten.join("\n\t"))
close_connection
return
end
end

def send_request
Expand Down Expand Up @@ -217,57 +252,37 @@ def send_request
data << "Content-type: #{@options[:content_type]}"
data << "Content-length: #{content.length}"
end

if @options[:headers]
@options[:headers].each do |name,value|
data << "#{name}: #{value}"
end
end

data << "\r\n"

send_data data.join("\r\n") << content
end

def receive_line ln
case @state
when :init
parse_response_line ln
when :headers
parse_header_line ln
when :stream
parse_stream_line ln
end
end

def receive_error e
@error_callback.call(e) if @error_callback
end

def parse_stream_line ln
ln.strip!
unless ln.empty?
if ln[0,1] == '{'
@each_item_callback.call(ln) if @each_item_callback
if ln[0,1] == '{' || ln[ln.length-1,1] == '}'
@stream << ln
if @stream[0,1] == '{' && @stream[@stream.length-1,1] == '}'
@each_item_callback.call(@stream) if @each_item_callback
@stream = ''
end
end
end
end

def parse_header_line ln
ln.strip!
if ln.empty?
reset_timeouts if @code == 200
@state = :stream
else
headers << ln
end
end

def parse_response_line ln
if ln =~ /\AHTTP\/1\.[01] ([\d]{3})/
@code = $1.to_i
@state = :headers
receive_error("invalid status code: #{@code}. #{ln}") unless @code == 200
else
receive_error('invalid response')
close_connection
end
end

def reset_timeouts
set_comm_inactivity_timeout @options[:timeout] if @options[:timeout] > 0
@nf_last_reconnect = @af_last_reconnect = nil
@reconnect_retries = 0
end
Expand Down Expand Up @@ -309,7 +324,7 @@ def uri_base
def params
flat = {}
@options[:params].merge( :track => @options[:filters] ).each do |param, val|
next if val.empty?
next if val.to_s.empty? || (val.respond_to?(:empty?) && val.empty?)
val = val.join(",") if val.respond_to?(:join)
flat[escape(param)] = escape(val)
end
Expand Down
25 changes: 25 additions & 0 deletions spec/spec_helper.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,4 +26,29 @@ def connect_stream(opts={}, &blk)
blk.call if blk
EM.add_timer(stop_in){ EM.stop }
}
end

def http_response(status_code, status_text, headers, body)
res = "HTTP/1.1 #{status_code} #{status_text}\r\n"
headers = {
"Content-Type"=>"application/json",
"Transfer-Encoding"=>"chunked"
}.merge(headers)
headers.each do |key,value|
res << "#{key}: #{value}\r\n"
end
res << "\r\n"
if headers["Transfer-Encoding"] == "chunked" && body.kind_of?(Array)
body.each do |data|
res << http_chunk(data)
end
else
res << body
end
res
end

def http_chunk(data)
# See http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.6.1
"#{data.length.to_s(16)}\r\n#{data}\r\n"
end
Loading

0 comments on commit e13023c

Please sign in to comment.