Merge b74479a into 6e74516

kingfisher_scrapy/pipelines.py

@@ -44,6 +44,27 @@ def process_item(self, item, spider):
         return item
 
 
+class Sample:
+    """
+    Drop items and close the spider when more than 1 item is scraped
+    """
+    def __init__(self):
+        self.item_count = 0
+
+    def process_item(self, item, spider):
+        if not spider.sample:
+            return item
+        if self.item_count >= 1:
+            spider.crawler.engine.close_spider(spider, 'closespider_itemcount')
+            raise DropItem
+        self.item_count += 1
+        return item
+
+    def open_spider(self, spider):
+        if spider.sample:
+            spider.crawler.engine.downloader.total_concurrency = 1
+
+
 class Pluck:
     def __init__(self):
         self.processed = set()

kingfisher_scrapy/settings.py

@@ -81,6 +81,7 @@
 # Configure item pipelines
 # See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
 ITEM_PIPELINES = {
+   'kingfisher_scrapy.pipelines.Sample': 200,
    'kingfisher_scrapy.pipelines.Validate': 300,
    'kingfisher_scrapy.pipelines.Pluck': 301,
 }