Add crawl_time and keep_collection_open spider parameters

kingfisher_scrapy/base_spider.py

@@ -40,6 +40,12 @@ class BaseSpider(scrapy.Spider):
     .. code:: bash
 
         scrapy crawl spider_name -a note='Started by NAME.'
+
+    Use the same collection_id in Kingfisher Process for a source_id:
+
+     .. code:: bash
+
+        scrapy crawl spider_name -a custom_collection_data_version=2020-01-01
     """
 
     MAX_SAMPLE = 10
@@ -49,7 +55,8 @@ class BaseSpider(scrapy.Spider):
     ocds_version = '1.1'
     date_format = 'date'
 
-    def __init__(self, sample=None, note=None, from_date=None, until_date=None, latest=None, *args,
+    def __init__(self, sample=None, note=None, from_date=None, until_date=None, latest=None,
+                 custom_collection_data_version=None, *args,
                  **kwargs):
         super().__init__(*args, **kwargs)
 
@@ -60,13 +67,15 @@ def __init__(self, sample=None, note=None, from_date=None, until_date=None, late
         self.until_date = until_date
         self.date_format = self.VALID_DATE_FORMATS[self.date_format]
         self.latest = latest == 'true'
+        self.custom_collection_data_version = custom_collection_data_version
 
         spider_arguments = {
             'sample': sample,
             'note': note,
             'from_date': from_date,
             'until_date': until_date,
             'latest': latest,
+            'custom_collection_data_version': custom_collection_data_version,
         }
         spider_arguments.update(kwargs)
         self.logger.info('Spider arguments: {!r}'.format(spider_arguments))
@@ -75,6 +84,14 @@ def __init__(self, sample=None, note=None, from_date=None, until_date=None, late
     def from_crawler(cls, crawler, *args, **kwargs):
         spider = super(BaseSpider, cls).from_crawler(crawler, *args, **kwargs)
 
+        if spider.custom_collection_data_version:
+            try:
+                spider.custom_collection_data_version = datetime.strptime(spider.custom_collection_data_version,
+                                                                          '%Y-%m-%d')
+            except ValueError as e:
+                raise SpiderArgumentError('spider argument custom_collection_data_version: '
+                                          'invalid date value: {}'.format(e))
+
         # Checks Spider date ranges arguments
         if spider.from_date or spider.until_date:
             if not spider.from_date:

kingfisher_scrapy/extensions.py

@@ -134,7 +134,7 @@ def spider_closed(self, spider, reason):
         Sends an API request to end the collection's store step.
         """
         # https://docs.scrapy.org/en/latest/topics/signals.html#spider-closed
-        if reason != 'finished' or spider.latest:
+        if reason != 'finished' or spider.latest or spider.custom_collection_data_version:
             return
 
         response = self.client.end_collection_store({
@@ -154,9 +154,13 @@ def item_scraped(self, item, spider):
 
         if not item.get('post_to_api', True) or isinstance(item, LatestReleaseDateItem):
             return
+        date_format = '%Y-%m-%d %H:%M:%S'
+        collection_data_version = spider.get_start_time(date_format)
+        if spider.custom_collection_data_version:
+            collection_data_version = spider.custom_collection_data_version.strftime(date_format)
         data = {
             'collection_source': spider.name,
-            'collection_data_version': spider.get_start_time('%Y-%m-%d %H:%M:%S'),
+            'collection_data_version': collection_data_version,
             'collection_sample': spider.sample,
             'file_name': item['file_name'],
             'url': item['url'],

tests/extensions/test_kingfisher_process_api.py

@@ -45,9 +45,11 @@ def test_from_crawler_missing_arguments(api_url, api_key):
 @pytest.mark.parametrize('directory', [False, True])
 @pytest.mark.parametrize('ok', [True, False])
 @pytest.mark.parametrize('post_to_api', [True, False])
-def test_item_scraped_file(sample, is_sample, path, note, encoding, encoding2, directory, ok, post_to_api, tmpdir,
-                           caplog):
-    spider = spider_with_files_store(tmpdir, sample=sample, note=note)
+@pytest.mark.parametrize('custom_collection_data_version', [None, '2020-01-01'])
+def test_item_scraped_file(sample, is_sample, path, note, encoding, encoding2, directory, ok, post_to_api,
+                           custom_collection_data_version, tmpdir, caplog):
+    spider = spider_with_files_store(tmpdir, sample=sample, note=note,
+                                     custom_collection_data_version=custom_collection_data_version)
 
     if directory:
         spider.crawler.settings['KINGFISHER_API_LOCAL_DIRECTORY'] = str(tmpdir.join('xxx'))
@@ -96,6 +98,8 @@ def test_item_scraped_file(sample, is_sample, path, note, encoding, encoding2, d
             expected['collection_note'] = note
         if directory:
             expected['local_file_name'] = tmpdir.join('xxx', path)
+        if custom_collection_data_version:
+            expected['collection_data_version'] = '2020-01-01 00:00:00'
         if not post_to_api:
             assert mocked.call_count == 0
         else:

tests/test_base_spider.py

@@ -90,3 +90,13 @@ def test_date_arguments():
     with pytest.raises(SpiderArgumentError) as e:
         assert spider_with_crawler(until_date='test', default_from_date=test_date)
     assert str(e.value) == 'spider argument until_date: invalid date value: {}'.format(error_message)
+
+
+def test_custom_collection_data_version():
+    error_message = "time data '2020' does not match format '%Y-%m-%d'"
+
+    assert spider_with_crawler(custom_collection_data_version='2020-01-01')
+    with pytest.raises(SpiderArgumentError) as e:
+        assert spider_with_crawler(custom_collection_data_version='2020')
+    assert str(e.value) == 'spider argument custom_collection_data_version: invalid date value: {}'.format(
+        error_message)