Permalink
Browse files

add pickup lost users to user helper haphazardly so i don't lose the …

…code
  • Loading branch information...
2 parents cc2f565 + cf0e049 commit 4f9f9b989536a70ad79bd7be22b09bdb80228599 Devin Gaffney committed Dec 22, 2010
@@ -1,5 +1,9 @@
def time_based_summary(collection_id, save_path)
+<<<<<<< HEAD:back-end/cluster-code/analyzer/tools/time_based_summary.rb
+ granularity = "year"
+=======
granularity = "date"
+>>>>>>> 9afbda7ec65c2bcceeabc95c8b121146353bb298:back-end/cluster-code/analyzer/tools/time_based_summary.rb
time_queries = resolve_time_query(granularity)
time_queries.each_pair do |time_granularity,time_query|
collection = Collection.find({:id => collection_id})
@@ -95,4 +99,4 @@ def resolve_time(granularity, time_slice)
year = time.year
end
return time, hour, date, month, year
-end
+end
@@ -2,9 +2,9 @@ def word_frequency(collection_id, save_path)
collection = Collection.find({:id => collection_id})
query = "select text from tweets "+Analysis.conditional(collection)
frequency_listing = get_frequency_listing(query)
- generate_graph_points([{"title" => "hashtags", "style" => "word_frequency", "collection" => collection},
- {"title" => "mentions", "style" => "word_frequency", "collection" => collection},
- {"title" => "significant_words", "style" => "word_frequency", "collection" => collection},
+ generate_graph_points([ #{"title" => "hashtags", "style" => "word_frequency", "collection" => collection},
+ # {"title" => "mentions", "style" => "word_frequency", "collection" => collection},
+ # {"title" => "significant_words", "style" => "word_frequency", "collection" => collection},
{"title" => "urls", "style" => "word_frequency", "collection" => collection}]) do |fs, graph, tmp_folder|
generate_word_frequency(fs, tmp_folder, frequency_listing, collection, graph)
end
@@ -80,4 +80,4 @@ def hashes_to_graph_points(hash, collection, graph)
temp_graph_points << graph_point
end
return temp_graph_points
-end
+end
@@ -42,12 +42,13 @@ def self.claim_new_users
def self.create_temp_file
`mkdir ../tmp_files/#{$w.instance_id}`
- source_data = `curl #{SITE_URL}/files/#{$w.rest_instance.metadata.source_data}`
+# source_data = `curl #{SITE_URL}/files/#{$w.rest_instance.metadata.source_data}`
+ source_data = File.open("../tmp_files/users.txt").read
f = File.open("../tmp_files/#{$w.instance_id}/source_data.txt", "w")
f.write(source_data)
f.close
f = File.open("../tmp_files/#{$w.instance_id}/source_data.txt", "r")
- data = f.read.split(",").collect{|d| d.strip}
+ data = f.read.split(",").flatten.collect{|x| x.split("\n")}.flatten.collect{|d| d.strip}
f.close
return data, "../tmp_files/#{$w.instance_id}/source_data.txt"
end
@@ -86,4 +87,4 @@ def self.check_finished
full_collection.save
end
end
-end
+end
@@ -1,3 +1,3 @@
class Whitelisting < SiteData
- attr_accessor :hostname, :ip, :whitelisted
-end
+ attr_accessor :hostname, :ip, :whitelisted, :id
+end
@@ -21,13 +21,13 @@ def self.hash_tweet(entry)
def self.scrape_tweet_attributes(entry)
allowed_fields = ["lat", "twitter_id", "metadata_id", "in_reply_to_user_id", "lon", "language", "scrape_id", "favorited", "text", "user_id", "truncated", "source", "screen_name", "created_at", "in_reply_to_screen_name", "location", "id", "in_reply_to_status_id"]
- twitter_id = entry["id"]
+ twitter_id = entry["id"].to_i
tweet = {}
entry.delete_if{|k, v| k == "title" || k == "profile_image_url" || k == "from_user_id"}
entry.flatify.each_pair do |key, value|
case key
when "id"
- tweet["twitter_id"] = value
+ tweet["twitter_id"] = value.to_i
when "user-lang"
tweet["language"] = value
when "coordinates"
@@ -117,4 +117,4 @@ def self.prep_branch_terms(branch_terms)
def self.collect_screen_names(tweets)
return tweets.collect {|tweet| tweet.values_at("screen_name")}.flatten.uniq.first
end
-end
+end
@@ -11,7 +11,7 @@ def self.hash_users(dataset)
end
def self.hash_user(raw_user)
- disallowed_keys = ["profile_use_background_image", "follow_request_sent", "show_all_inline_media"]
+ disallowed_keys = ["profile_use_background_image", "follow_request_sent", "show_all_inline_media", "id_str"]
user = {}
user["screen_name"] = raw_user["screen_name"]
raw_user.delete_if {|k, v| k == "following" || k == "status"}
@@ -31,26 +31,4 @@ def self.hash_user(raw_user)
# puts "Hashed user: #{raw_user["screen_name"]}"
return user
end
-
-def pickup_lost_users(metadata_id, metadata_type)
- metadata = metadata_type.classify.constantize.find(:id => metadata_id)
- user_names = Tweet.find_all(:metadata_id => metadata_id, :metadata_type => metadata_type).collect{|x| x.screen_name }.compact.uniq
- users = []
- user_names.each do |user_name|
- url = "http://api.twitter.com/1/statuses/user_timeline.json?screen_name=#{user_name}"
- data = U.return_data(url)
- if !data.nil?
- if !data.empty?
- data = JSON.parse(data)
- user = UserHelper.hash_user(data.first["user"])
- user["metadata_id"] = metadata_id
- user["metadata_type"] = metadata_type
- user["scrape_id"] = metadata.scrape.id
- users << user
- end
- end
- end
- Database.save_all(:users => users)
-end
-
end
@@ -61,7 +61,7 @@ def self.type_attributes(row, query_result)
type_map = query_result.fetch_fields.collect{|field| map_hash[field.name] = field.type}
row.each_pair do |k,v|
if map_hash[k] == 1
- final_row[k] = v.class == nil ? nil : v.to_bool
+ final_row[k] = v.class == NilClass ? nil : v.to_bool
elsif map_hash[k] == 3 || map_hash[k] == 8
final_row[k] = v.nil? ? nil : v.to_i
elsif map_hash[k] == 246 || map_hash[k] == 5
@@ -113,7 +113,7 @@ def self.prep_attribute(attribute)
elsif attribute.class == NilClass
attribute = "NULL"
end
- attribute = "'#{attribute}'" if attribute != "NULL"
+ attribute = "'#{attribute}'" if attribute != "NULL" && attribute.class == String || attribute.class == FalseClass || attribute.class == TrueClass
if attribute.class == String && ["''", "'\n'", "'\n '"].include?(attribute.gsub(" ", ""))
attribute = "NULL"
end
@@ -1,11 +1,11 @@
class RollBackGraphIdentifiers < ActiveRecord::Migration
def self.up
- remove_column :graphs, :day
+# remove_column :graphs, :day
remove_column :graphs, :minute
end
def self.down
- add_column :graphs, :day, :integer
+ # add_column :graphs, :day, :integer
add_column :graphs, :minute, :integer
end
-end
+end
@@ -1,9 +1,11 @@
class ReinstateGraphTimeSeparations < ActiveRecord::Migration
def self.up
+ remove_column :graphs, :date
add_column :graphs, :date, :integer
end
def self.down
remove_column :graphs, :date
+ add_column :graphs, :date, :string
end
end
@@ -1,11 +1,13 @@
class ChangeOutUniqueGraphIndexing < ActiveRecord::Migration
def self.up
- remove_index("graphs", "title_style_collection")
+# remove_index("graphs", "title_style_collection")
+ add_column :graphs, :month, :integer
add_index(:graphs, [:title, :style, :collection_id, :time_slice, :year, :month, :date, :hour], :unique => true, :name => "unique_graph")
end
def self.down
add_index(:graphs, [:title, :style, :collection_id], :unique => true, :name => "title_style_collection")
+ remove_column :graphs, :year
remove_index("graphs", "unique_graph")
end
end
Oops, something went wrong.

0 comments on commit 4f9f9b9

Please sign in to comment.