@@ -106,9 +106,16 @@ url = base puts "CONTACTING INTERNETS... #{url}" agent = WWW::Mechanize.new +agent.read_timeout = 30 # agent.user_agent = "KANYE RANT DETECTOR <http://fffff.at>" agent.user_agent_alias = "Mac Safari" -page = agent.get(url) +retries = 3 +begin + page = agent.get(url) +rescue Exception # Timeout::Error does not derive from StandardException, I h8 it O_o + STDERR.puts "ERROR FETCHING: #{$!} RETRIES REMAINING: #{retries}" + retry if (retries -= 1) > 0 +end # FOR PROPER ARCHIVAL reverse_pagination = true @@ -117,7 +124,12 @@ reverse_pagination = true first = 0 # GETS OVERRIDDEN loop { # IN REVERSE MODE... - posts = (page/'.rapper').to_a.reverse + posts = (page/'.rapper').to_a.reverse rescue nil + if posts.blank? + STDERR.puts "ERROR: NO POSTS ON PAGE; ABORTING..." + exit 1 # Our whole purpose is to parse this page; bail + end + puts "PROCESSING #{posts.length} POSTS ..." posts.each { |post| RANT-DETECTOR.rb 10c8b84e3903cf546146be7326714ad17826d244 Jamie Wilkinson jamie@tramchase.com http://github.com/jamiew/KANYE-RANT-DETECTOR/commit/886b399ed5d355761f1bef0be6369ebc7ea232b7 886b399ed5d355761f1bef0be6369ebc7ea232b7 2009-09-15T14:35:17-07:00 2009-09-15T14:35:17-07:00 better error handling... Kanye's website is down today. Fetch will retry 3 times before giving up. e142a219c36c5adbc5a5093dcb83cb706e66968f Jamie Wilkinson jamie@tramchase.com