Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

added .info methods to crawler, twiterator and puller, added .debug m…

…ethods to saver, removed the puts statements we were using for this purpose
  • Loading branch information...
commit 2fc8429145287b5c5621f8135bd108c9d1ed5914 1 parent b946978
Sam Gilbert authored
View
7 lib/wetk_twitter/crawler.rb
@@ -16,7 +16,7 @@ def crawl_type_str_to_proc(str)
# crawl type = RT_TO_USER_CRAWL, RT_FROM_USER_CRAWL, 'mention_to_user' 'mention_from_user', 'reply_to_user', 'reply_from_user',
def crawl(search_query = nil)
- unless @users.empty?
+ unless @users.empty?
while @depth > 0
@users.dup.each do |user|
unless user.crawled?
@@ -60,20 +60,24 @@ def append(user, user_info = nil)
end
SEARCH_CRAWL = lambda do |user, search_query, count|
+ $LOG.info "SEARCH CRAWL"
$TWITERATOR.twiterate({:collect_users => true}, {:search_query => search_query}, &SEARCH_ITER)
# @users.keys
search_query
end
FOLLOWER_IDS_CRAWL = lambda do |user, search_query, count|
+ $LOG.info "FOLLOWER IDS CRAWL"
$PULLER.pull({:user_id => user, :collect_users => true}, &FOLLOWER_IDS_PULL)
end
FRIEND_IDS_CRAWL = lambda do |user, search_query, count|
+ $LOG.info "FRIEND IDS CRAWL"
$PULLER.pull({:user_id => user, :collect_users => true}, &FRIEND_IDS_PULL)
end
FOLLOWERS_CRAWL = lambda do |user, search_query, count|
+ $LOG.info "FOLLOWERS CRAWL"
if not user.db_user_info
user.db_user_info = $PULLER.pull({:user_id => user.search}, &USER_PULL)
end
@@ -81,6 +85,7 @@ def append(user, user_info = nil)
end
FRIENDS_CRAWL = lambda do |user, search_query, count|
+ $LOG.info "FRIENDS CRAWL"
if not user.db_user_info
user.db_user_info = $PULLER.pull({:user_id => user.search}, &USER_PULL)
end
View
16 lib/wetk_twitter/puller.rb
@@ -6,7 +6,6 @@ def initialize(base = nil)
end
def pull(rules, &pull_type)
-
@rules = rules.dup
begin
pull_type.call(@rules, @base)
@@ -32,8 +31,7 @@ def pull(rules, &pull_type)
SEARCH_PULL = lambda do |rules, base|
- puts "SEARCH PULL"
-
+ $LOG.info "SEARCH PULL"
@search_query = rules.delete(:search_query)
@results = Twitter::Search.new(@search_query, rules).per_page(100)
rules[:max_id] ? @results.max(rules.delete(:max_id)) : nil
@@ -46,14 +44,14 @@ def pull(rules, &pull_type)
end
USER_PULL = lambda do |rules, base|
- puts "USER PULL = " + rules[:user_id].to_s
+ $LOG.info "USER PULL FOR " + rules[:user_id].to_s
@user_id = rules.delete(:user_id)
@results = base.user(@user_id, rules)
$SAVER.save(@results, &TWITTER_ACCOUNT_SAVE)
end
FOLLOWERS_PULL = lambda do |rules, base|
- puts "FOLLOWERS PULL"
+ $LOG.info "FOLLOWERS PULL"
@user = rules.delete(:user)
if @user.by_id
rules[:user_id] = @user.by_id
@@ -72,7 +70,7 @@ def pull(rules, &pull_type)
end
FOLLOWER_IDS_PULL = lambda do |rules, base|
- puts "FOLLOWER_IDS PULL"
+ $LOG.info "FOLLOWER IDS PULL"
@user = rules.delete(:user)
unless @user.db_user_info
@user.db_user_info = $PULLER.pull({:user_id => @user.search}, &USER_PULL)
@@ -95,7 +93,7 @@ def pull(rules, &pull_type)
end
FRIENDS_PULL = lambda do |rules, base|
- puts "FRIENDS PULL"
+ $LOG.info "FRIENDS PULL"
@user = rules.delete(:user)
if @user.by_id
rules[:user_id] = @user.by_id
@@ -114,7 +112,7 @@ def pull(rules, &pull_type)
end
FRIEND_IDS_PULL = lambda do |rules, base|
- puts "FRIEND_IDS PULL"
+ $LOG.info "FRIEND IDS PULL"
@user = rules.delete(:user)
unless @user.db_user_info
@user.db_user_info = $PULLER.pull({:user_id => @user.search}, &USER_PULL)
@@ -137,7 +135,7 @@ def pull(rules, &pull_type)
end
USER_TWEETS_PULL = lambda do |rules, base|
- puts "USER TWEETS PULL"
+ $LOG.info "USER TWEETS PULL"
rules[:count] = 200
@results = base.user_timeline(rules)
@results.each do |result|
View
11 lib/wetk_twitter/saver.rb
@@ -19,6 +19,7 @@ def save(to_save, &save_type)
end
TWEET_SAVE = lambda do |tweet_to_save, rules|
+ $LOG.debug "TWEET SAVE"
tweet = Tweet.new(:text => tweet_to_save.text,
:time_of_tweet => tweet_to_save.created_at,
:to_user_id => tweet_to_save.to_user_id,
@@ -46,9 +47,7 @@ def save(to_save, &save_type)
end
USER_TWEET_SAVE = lambda do |tweet_to_save, rules|
- # debugger
- # nil
- puts "user_tweet_save"
+ $LOG.debug "USER TWEET SAVE"
tweet = Tweet.new(:text => tweet_to_save.text,
:time_of_tweet => tweet_to_save.created_at,
:to_user_id => tweet_to_save.in_reply_to_user_id,
@@ -75,7 +74,7 @@ def save(to_save, &save_type)
end
TWITTER_ACCOUNT_SAVE = lambda do |twitter_account_to_save, rules|
- puts "twitter account save"
+ $LOG.debug "TWITTER ACCOUNT SAVE"
if twitter_account_to_save.class == SearchUser
twitter_account_attribute_hash = {:twitter_id => twitter_account_to_save.by_id,
:screen_name => twitter_account_to_save.by_screen_name}
@@ -138,6 +137,7 @@ def save(to_save, &save_type)
end
CALL_SAVE = lambda do |call_to_save, rules|
+ $LOG.debug "CALL SAVE"
call = Call.new(:query => call_to_save.query,
:completed_in => call_to_save.completed_in,
:since_id => call_to_save.since_id,
@@ -156,9 +156,9 @@ def save(to_save, &save_type)
end
RELATIONSHIP_SAVE = lambda do |users_to_save, rules|
+ $LOG.debug "RELATIONSHIP SAVE"
#Assuming hash of two SearchUser objects
#saves a lot of code
-# puts "relationship save"
# users_to_save.each_key do |k|
# if k.kind_of?(Hash)
@@ -202,6 +202,7 @@ def save(to_save, &save_type)
REACTION_SAVE = lambda do |reaction_to_save, rules|
+ $LOG.debug "REACTION SAVE"
if reaction_to_save.kind_of?(Hash)
reaction_to_save.each_key do |k|
if k.kind_of?(Hash)
View
7 lib/wetk_twitter/twiterator.rb
@@ -17,7 +17,7 @@ def twiterate(my_rules = {},puller_rules = {}, &type)
end
SEARCH_ITER = lambda do |my_rules, puller_rules|
-
+ $LOG.info "SEARCH ITERATION"
my_rules[:cursor] ? puller_rules[:max_id] = my_rules[:cursor] : nil
@results = $PULLER.pull(puller_rules, &SEARCH_PULL)
my_rules[:collect_users] == true ? @results.results.each do |tweet| $CRAWLER.append(tweet.from_user) end : nil
@@ -25,14 +25,14 @@ def twiterate(my_rules = {},puller_rules = {}, &type)
end
USER_TWEETS_ITER = lambda do |my_rules, puller_rules|
-
+ $LOG.info "USER TWEETS ITERATION"
my_rules[:cursor] ? puller_rules[:max_id] = my_rules[:cursor] : nil
@result = $PULLER.pull(puller_rules, &USER_TWEETS_PULL)
{:result => @result.last.id, :count => @results.length}
end
FOLLOWERS_ITER = lambda do |my_rules, puller_rules|
-
+ $LOG.info "FOLLOWERS ITERATION"
my_rules[:cursor] ? puller_rules[:cursor] = my_rules[:cursor] : puller_rules[:cursor] = -1
if my_rules[:count] == 99999999999999
$SAVER.rules[:complete_follower_set] = true
@@ -51,6 +51,7 @@ def twiterate(my_rules = {},puller_rules = {}, &type)
end
FRIENDS_ITER = lambda do |my_rules, puller_rules|
+ $LOG.info "FRIENDS ITERATION"
my_rules[:cursor] ? puller_rules[:cursor] = my_rules[:cursor] : puller_rules[:cursor] = -1
if my_rules[:count] == 99999999999999
$SAVER.rules[:complete_friend_set] = true
Please sign in to comment.
Something went wrong with that request. Please try again.