Allow a spider to fail if it generates a 404

tombh · Jun 2, 2016 · 43f6430 · 43f6430
1 parent 03fdbc1
commit 43f6430
Showing 1 changed file with 5 additions and 1 deletion.
diff --git a/Rakefile b/Rakefile
@@ -45,7 +45,11 @@ task :crawl do
       # Capitalize first letter of spider, eg; dharmaseed to Dharmaseed
       # Then turn that string into a true language constant representing a class name
       # Then call the .run() method on the object
-      spider.capitalize.constantize.new(recrawl: recrawl).run
+      begin
+        spider.capitalize.constantize.new(recrawl: recrawl).run
+      rescue OpenURI::HTTPError
+        puts "Skipping #{spider}"
+      end
     end
 
   else