Collect user timeline status and store them in mongodb

Gemfile

@@ -0,0 +1,9 @@
+source "http://rubygems.org"
+
+gem "tweetstream"
+gem "mongo"
+gem "bson_ext"
+
+group :development do
+  gem "awesome_print"
+end

Gemfile.lock

@@ -0,0 +1,31 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    awesome_print (1.0.2)
+    bson (1.6.4)
+    bson_ext (1.6.4)
+      bson (~> 1.6.4)
+    daemons (1.1.8)
+    eventmachine (0.12.10)
+    http_parser.rb (0.5.3)
+    mongo (1.6.2)
+      bson (~> 1.6.2)
+    multi_json (1.3.6)
+    simple_oauth (0.1.5)
+    tweetstream (1.1.4)
+      daemons (~> 1.1)
+      multi_json (>= 1.0)
+      twitter-stream (= 0.1.14)
+    twitter-stream (0.1.14)
+      eventmachine (>= 0.12.8)
+      http_parser.rb (~> 0.5.1)
+      simple_oauth (~> 0.1.4)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  awesome_print
+  bson_ext
+  mongo
+  tweetstream

collect.rb

@@ -0,0 +1,35 @@
+# Collects user tweets and saves them to a mongodb
+
+require "bundler"
+require File.dirname(__FILE__) + "/tweetminer"
+
+Bundler.require
+
+# We use the TweetStream gem to access Twitter's Streaming API.
+# https://github.com/intridea/tweetstream
+
+TweetStream.configure do |config|
+  settings = YAML.load_file File.dirname(__FILE__) + '/twitter.yml'
+
+  config.consumer_key       = settings['consumer_key']
+  config.consumer_secret    = settings['consumer_secret']
+  config.oauth_token        = settings['oauth_token']
+  config.oauth_token_secret = settings['oauth_token_secret']
+end
+
+settings = YAML.load_file File.dirname(__FILE__) + '/mongo.yml'
+miner = TweetMiner.new(settings)
+
+stream = TweetStream::Client.new
+
+stream.on_error do |msg|
+  puts msg
+end
+
+stream.on_timeline_status do |status|
+  miner.insert_status status
+  print "."
+end
+
+# Do not forget this to trigger the collection of tweets
+stream.userstream

mongo.yml

@@ -0,0 +1,3 @@
+host:     localhost
+port:     27017
+database: twitter

tweetminer.rb

@@ -0,0 +1,33 @@
+require "mongo"
+
+class TweetMiner
+  attr_writer :db_connector
+  attr_reader :options
+
+  def initialize(options)
+    @options = options
+  end
+
+  def db
+    @db ||= connect_to_db
+  end
+
+  def insert_status(status)
+    statuses.insert status
+  end
+
+  def statuses
+    @statuses ||= db["statuses"]
+  end
+
+  private
+
+  def connect_to_db
+    db_connector.call(options["host"], options["port"]).db(options["database"])
+  end
+
+  def db_connector
+    @db_connector ||= Mongo::Connection.public_method :new
+  end
+
+end

twitter.yml

@@ -0,0 +1,9 @@
+# Vist http://dev.twitter.com to register your application
+# and get oauth settings.
+
+consumer_key:    'KEY'
+consumer_secret: 'SECRET'
+
+oauth_token:       'TOKEN'
+oauth_token_secret: 'TOKEN SECRET'
+