Release commit for v.0.9.0

dynamic_scraper/spiders/django_spider.py

@@ -374,7 +374,7 @@ def parse(self, response):
                         only_main_page_idfs = False
 
                 is_double = False
-                if only_main_page_idfs:
+                if only_main_page_idfs and self.conf['DO_ACTION']:
                     item, is_double = self._check_for_double_item(item)
 
                 # Don't go on reading detail pages when...

tests/scraper/scraper_test.py

@@ -3,6 +3,7 @@
 from django.test import TestCase
 
 from scrapy import log, signals
+from scrapy.exceptions import DropItem
 
 from scrapy.utils.project import get_project_settings
 settings = get_project_settings()
@@ -38,16 +39,22 @@ def __init__(self, *args, **kwargs):
 class DjangoWriterPipeline(object):
 
     def process_item(self, item, spider):
-        item['event_website'] = spider.ref_object
+        if spider.conf['DO_ACTION']:
+            try:
+                item['event_website'] = spider.ref_object
+
+                checker_rt = SchedulerRuntime()
+                checker_rt.save()
+                item['checker_runtime'] = checker_rt
+
+                if not 'description' in item or item['description'] == None:
+                    item['description'] = u''
+
+                item.save()
+            except IntegrityError, e:
+                spider.log(str(e), log.ERROR)
+                raise DropItem("Missing attribute.")
 
-        checker_rt = SchedulerRuntime()
-        checker_rt.save()
-        item['checker_runtime'] = checker_rt
-
-        if not 'description' in item or item['description'] == None:
-            item['description'] = u''
-
-        item.save()
         return item