README @@ -21,7 +21,6 @@ namespace :db do create_table 'events' do |t| t.string :type t.string :user - t.string :repo t.datetime :published end end @@ -34,13 +33,21 @@ end desc "Parse away" task :parse do urls = [] - (265..3120).each { |x| urls << "http://github.com/timeline.atom?page=#{x}" } + (275..3150).each { |x| urls << "http://github.com/timeline.atom?page=#{x}" } - start_date = DateTime.parse("2008-02-01") - end_date = DateTime.parse("2008-02-07 23:59:59") + start_date = DateTime.parse("2009-02-01") + end_date = DateTime.parse("2009-02-07 23:59:59") + + Feedzirra::Feed.fetch_and_parse(urls, + :on_success => lambda {|u, a| puts "Got #{u}"} ).each do |k, v| + + if v.is_a?(Fixnum) + puts "\nBroken page: #{k}" + next + else + puts "\nWorking on #{k}, #{v.entries.size} entries..." + end - Feedzirra::Feed.fetch_and_parse(urls).each do |k, v| - puts "\nWorking on #{k}, #{v.entries.size} entries..." v.entries.each do |entry| next unless start_date < entry.published && entry.published < end_date @@ -50,7 +57,6 @@ task :parse do title = entry.title.split e.type = title[1] - e.repo = title.last e.save p e end Rakefile 367f1893c80c5f3097a0f2310dd6b70339f99750 Nick Quaranto nick@quaran.to http://github.com/qrush/rebase/commit/24a5c7d7894d96a3993d2ee218a7f3f7fbacd704 24a5c7d7894d96a3993d2ee218a7f3f7fbacd704 2009-02-08T16:51:42-08:00 2009-02-08T16:51:42-08:00 It helps to use 2009 instead of 2008 when checking for dates 192146a2daba46decf692fa688bdee72cb30a9b5 Nick Quaranto nick@quaran.to