Permalink
Browse files

add twitter favorites importer

  • Loading branch information...
1 parent fba585c commit ef36d4656c2bfd78250b8b124ae5724ec086eab1 @technoweenie committed Mar 18, 2011
Showing with 83 additions and 33 deletions.
  1. +1 −1 Gemfile
  2. +13 −1 Rakefile
  3. +18 −31 lib/allofthestars/client/importer.rb
  4. +49 −0 lib/allofthestars/client/importers/twitter.rb
  5. +2 −0 lib/allofthestars/server.rb
View
@@ -2,6 +2,7 @@ source "http://rubygems.org"
gem 'excon'
gem 'yajl-ruby'
+gem "redis"
group :client do
gem 'faraday'
@@ -12,7 +13,6 @@ group :server do
gem "adapter"
gem "adapter-riak"
gem "adapter-redis"
- gem "redis"
gem "riak-client"
gem "sinatra"
gem "stratocaster", :path => "vendor/stratocaster"
View
@@ -21,7 +21,17 @@ end
namespace :import do
desc "Import a single Tweet. Needs CLUSTER_ID= and TWEET={status-id}"
task :tweet => :init do
- import_tweet ENV['TWEET'].to_i
+ ENV['TWEET'].to_s.split(',').each do |tweet|
+ tweet.strip!
+ import_tweet tweet.to_i
+ end
+ end
+
+ task :twitter_favorites => :init do
+ ENV['USER'].to_s.split(',').each do |user|
+ user.strip!
+ import_twitter_favorites user
+ end
end
task :instagram => :init do
@@ -37,13 +47,15 @@ namespace :import do
require 'bundler'
Bundler.setup :default, :client, :importers
$:.unshift File.expand_path("../lib", __FILE__)
+ require 'redis'
require 'allofthestars/client/importer'
if url = ENV['STARS_URL']
AllOfTheStars::Client.default_url = url
end
extend AllOfTheStars::Importer
+ self.redis = Redis.new
self.cluster_id = ENV['CLUSTER_ID']
debug! if ENV['DEBUG']
end
@@ -3,38 +3,12 @@
require 'pp'
module AllOfTheStars
- module Importer
- def import_tweet(id)
- require 'twitter'
- import_tweet_status Twitter.status(id)
- end
-
- def import_tweet_status(status)
- created = Time.parse(status.created_at)
- data = {
- :type => "Twitter",
- :content => status.text,
- :source_url =>
- 'http://twitter.com/%s/status/%s' % [
- status.user.screen_name,
- status.id],
- :custom => {
- :id => status.id_str,
- :user => status.user.screen_name,
- :retweet_count => status.retweet_count.to_i
- },
- :created_at => created.to_i
- }
-
- if in_reply_name = status.in_reply_to_screen_name
- data[:custom][:in_reply_to] =
- 'http://twitter.com/%s/status/%s' % [
- in_reply_name,
- status.in_reply_to_status_id_str]
- end
+ module Importers
+ autoload :Twitter, 'allofthestars/client/importers/twitter'
+ end
- add_star(data, status)
- end
+ module Importer
+ include Importers::Twitter
def add_star(data, *debuggables)
if debug?
@@ -72,5 +46,18 @@ def cluster_id
def cluster_id=(val)
@cluster_id = val
end
+
+ def config(key, value = nil)
+ return nil if !@redis
+ if value
+ @redis.set key, value.to_s
+ else
+ @redis.get key
+ end
+ end
+
+ def redis=(r)
+ @redis = r
+ end
end
end
@@ -0,0 +1,49 @@
+module AllOfTheStars::Importers::Twitter
+ def import_twitter_favorites(user)
+ require 'twitter'
+
+ options = {}
+ config_key = "twitter:favorites:#{user}"
+ if since_id = config(config_key)
+ options[:since_id] = since_id
+ end
+
+ tweet = Twitter.favorites(user, options).each do |tweet|
+ import_tweet_status tweet
+ end.first
+
+ config(config_key, tweet.id) if tweet
+ end
+
+ def import_tweet(id)
+ require 'twitter'
+ import_tweet_status Twitter.status(id)
+ end
+
+ def import_tweet_status(status)
+ created = Time.parse(status.created_at)
+ data = {
+ :type => "Twitter",
+ :content => status.text,
+ :source_url =>
+ 'http://twitter.com/%s/status/%s' % [
+ status.user.screen_name,
+ status.id],
+ :custom => {
+ :id => status.id_str,
+ :user => status.user.screen_name,
+ :retweet_count => status.retweet_count.to_i
+ },
+ :created_at => created.to_i
+ }
+
+ if in_reply_name = status.in_reply_to_screen_name
+ data[:custom][:in_reply_to] =
+ 'http://twitter.com/%s/status/%s' % [
+ in_reply_name,
+ status.in_reply_to_status_id_str]
+ end
+
+ add_star(data, status)
+ end
+end
@@ -36,6 +36,7 @@ def self.stratocaster
module Timelines
class Type < Stratocaster::Timeline
adapter Stratocaster::Adapters::Redis.new(Redis.new, :prefix => "strat")
+
key_format "type:%s" do |msg|
msg.type
end
@@ -45,6 +46,7 @@ class HashTag < Stratocaster::Timeline
extend Twitter::Extractor
adapter Type.adapters.first
+
key_format "hashtag:%s" do |msg, keys|
keys.push *extract_hashtags(msg.content)
end

0 comments on commit ef36d46

Please sign in to comment.