app/models/mozes_user.rb db/migrate/20081024164149_create_mozes_users.rb db/migrate/20081024170120_add_reports_mozes_fields.rb test/fixtures/mozes_users.yml test/unit/mozes_user_test.rb @@ -1 +1,3 @@ config/database.yml +log/* +.DS_Store .gitignore @@ -12,6 +12,9 @@ class Report < ActiveRecord::Base after_save :assign_filters named_scope :with_location, :conditions => 'location_id IS NOT NULL' + + SOURCE_TWITTER = 1 + SOURCE_MOZES = 2 def name "#votereport #{self[:id]}" + (self.twitter_user ? " - #{twitter_user.name}" : "") app/models/report.rb @@ -6,7 +6,7 @@ development: database: votereport_development username: root password: - socket: /private/var/mysql/mysql.sock + socket: /tmp/mysql.sock test: adapter: mysql config/database.yml @@ -9,7 +9,7 @@ # # It's strongly recommended to check this file into your version control system. -ActiveRecord::Schema.define(:version => 20081023221200) do +ActiveRecord::Schema.define(:version => 20081024170120) do create_table "filters", :options=>'ENGINE=MyISAM', :force => true do |t| t.column "name", :string, :limit => 80 @@ -57,6 +57,14 @@ ActiveRecord::Schema.define(:version => 20081023221200) do add_index "locations", ["point"], :name => "index_locations_on_point", :spatial=> true + create_table "mozes_users", :force => true do |t| + t.column "mozes_id", :integer, :null => false + t.column "created_at", :datetime + t.column "updated_at", :datetime + end + + add_index "mozes_users", ["mozes_id"], :name => "index_mozes_users_on_mozes_id" + create_table "report_filters", :options=>'ENGINE=MyISAM', :force => true do |t| t.column "report_id", :integer t.column "filter_id", :integer @@ -86,9 +94,13 @@ ActiveRecord::Schema.define(:version => 20081023221200) do t.column "wait_time", :integer t.column "created_at", :datetime t.column "updated_at", :datetime + t.column "mozes_user_id", :integer + t.column "mozes_feed_id", :integer end add_index "reports", ["tid"], :name => "index_reports_on_tid", :unique => true + add_index "reports", ["mozes_user_id"], :name => "index_reports_on_mozes_user_id" + add_index "reports", ["mozes_feed_id"], :name => "index_reports_on_mozes_feed_id" create_table "tags", :force => true do |t| t.column "pattern", :string, :limit => 30 db/schema.rb @@ -7,7 +7,7 @@ <description>Nationwide VoteReport</description> <title>Messages for votereport on Mozes.com</title> <item> - <title><![CDATA[Message to votereport]]></title> + <title><![CDATA[#votereport they won't let grandma vote! L:Lansing, MI]]></title> <description><![CDATA[ <a href="http://www.mozes.com/302574403185/"><img src="http://www.mozes.com/_common/images/default_avatars/phone_default_0.jpg" width="40" height="40" title=""/></a> long lines at l:02130 @@ -17,7 +17,7 @@ <mozes:mozesUserId>302574403185</mozes:mozesUserId> </item> <item> - <title><![CDATA[Message from yotowoti]]></title> + <title><![CDATA[#votereport long lines here L:11211 ]]></title> <description><![CDATA[ <a href="http://www.mozes.com/3216502430/yotowoti"><img src="http://www.mozes.com/_common/images/default_avatars/phone_default_0.jpg" width="40" height="40" title="yotowoti"/></a> test @@ -26,5 +26,15 @@ <link><![CDATA[http://www.mozes.com/3216502430/yotowoti]]></link> <mozes:mozesUserId>3216502430</mozes:mozesUserId> </item> + <item> + <title><![CDATA[#votereport long lines where? Jacksonville, FL ]]></title> + <description><![CDATA[ + <a href="http://www.mozes.com/3216502430/yotowoti"><img src="http://www.mozes.com/_common/images/default_avatars/phone_default_0.jpg" width="40" height="40" title="yotowoti"/></a> + test + ]]></description> + <pubDate>Thu, 23 Oct 2008 14:11:20 -0700</pubDate> + <link><![CDATA[http://www.mozes.com/3216502430/yotowoti]]></link> + <mozes:mozesUserId>3216502430</mozes:mozesUserId> + </item> </channel> </rss> lib/daemons/mozes_sample.xml @@ -1,12 +1,9 @@ ENV["RAILS_ENV"] ||= 'production' -# TODO: this needs completed! Only a skeleton right now! -# See mozes_sample.xml for contents of feed - -FEED = "http://www.mozes.com/_/rss?keyword_id=1031894" +FEED = 'http://www.mozes.com/_/rss?keyword_id=1031894' require File.dirname(__FILE__) + "/../../config/environment" -require 'json' +require 'hpricot' require 'open-uri' $running = true @@ -14,19 +11,42 @@ Signal.trap("TERM") do $running = false end -while($running) do - # Populate these fields ONLY - # report.text = description body (exclude <a><img>*</a>) - # report.callerid = mozes:mozesUserId - # report.input_source_id = 2 +# for chucking errors about bad data +class PollMozesException < StandardError +end +while($running) do + begin # DO NOT populate these fields for this dataset # report.uniqueid = nil (asterisk ONLY) # report.tid = nil (twitter ONLY) # report.twitter_user_id = nil (twitter ONLY) - - messages = open(FEED).read + doc = Hpricot.XML(open(FEED)) + + (doc/:item).each do |item| + begin + user = MozesUser.find_or_create_by_mozes_id( (item/'mozes:mozesUserId').inner_html.strip ) + + # Populate these fields ONLY + # report.text = description body (exclude <a><img>*</a>) + # report.callerid = mozes:mozesUserId + # report.input_source_id = 2 + user.reports.create! { + :text => (item/:description).inner_text, + :mozes_user_id => user.id, + :mozes_feed_id => nil, + :input_source_id => REPORT::SOURCE_MOZES + } + + rescue ActiveRecord::InvalidRecord => e + puts "Error while creating report from Mozes feed: #{e.class}: #{e.message}" + end + end + rescue Exception => e + puts "Uncaught exception during loop: \n#{e.class}: #{e.message}\n\t#{e.backtrace.join("\n")} " + end + #messages = open(FEED).read sleep 10 end lib/daemons/poll_mozes.rb @@ -22,7 +22,7 @@ while($running) do next unless user = TwitterUser.add(user_info) screen_name, text = entry['title'].match(EXTRACTOR).captures - report_info = { :text => text, :tid => entry['status_id'], :input_source_id => 1 } + report_info = { :text => text, :tid => entry['status_id'], :input_source_id => REPORT::SOURCE_TWITTER } user.reports.create(report_info) end sleep 10 lib/daemons/poll_tweets.rb 5d8483833ed0e2ec8e54a022b54b3aa25d9a06a1 Billy Gray wgray@zetetic.net http://github.com/davetroy/votereport/commit/9930911adeedac244a2eed8c6725557f2ec0bd02 9930911adeedac244a2eed8c6725557f2ec0bd02 2008-10-24T10:33:55-07:00 2008-10-24T10:33:55-07:00 - okay, shaping up here - beginning to get these mozes items imported f573d19e2498d694b53ef6061b16846f98a5f61b Billy Gray wgray@zetetic.net