Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
branch: master
Fetching contributors…

Cannot retrieve contributors at this time

61 lines (50 sloc) 1.54 kb
#!/usr/bin/env ruby
#
# fetcher.rb -- grab twoots from the twitter API
#
# Part of the twootr! "project", hosted on github:
# http://github.com/bleything/twootr
#
# Copyright (c) 2010, Ben Bleything
# Released under the MIT License
#
require 'uri'
require 'rubygems'
require 'yajl/http_stream'
require 'resque'
require 'resque_jobs/intake'
TIMING_GRANULARITY = 50
CREDENTIALS = YAML.load_file( 'credentials.yml' )[ 'twitter' ]
puts "Connecting to Twitter..."
auth_string = "#{CREDENTIALS['username']}:#{CREDENTIALS['password']}"
uri = URI.parse( "http://#{auth_string}@stream.twitter.com/1/statuses/sample.json" )
# timing stuff
twoots = []
t_per_s = []
timings = []
puts "Fetching..."
start = Time.now
Yajl::HttpStream.get(uri, :symbolize_keys => true) do |hash|
twoots << hash
if twoots.size == TIMING_GRANULARITY
begin
twoots.each {|t| Resque.enqueue Intake, t[:id], t }
rescue Errno::ECONNREFUSED
$stderr.puts "Could not connect to redis... check to make sure that it's running:"
$stderr.puts " $ redis-server redis/redis.conf"
exit 1
end
stop = Time.now
# timing report
timings << ( stop - start )
t_per_s << ( TIMING_GRANULARITY / timings.last )
puts "* last #{TIMING_GRANULARITY} in %.2f seconds (avg: %.2f; %.2f tweets/second; %.2f avg t/s)" % [
timings.last,
timings.inject(0) {|sum, t| sum + t } / timings.size,
t_per_s.last,
t_per_s.inject(0) {|sum, t| sum + t } / t_per_s.size
]
start = Time.now
twoots = []
end
end
Jump to Line
Something went wrong with that request. Please try again.