Merge c578da5 into b527492

kingfisher_scrapy/base_spider.py

@@ -374,7 +374,7 @@ def parse(self, response):
                 data = compressed_file
 
             yield File({
-                'file_name': basename,
+                'file_name': f'{archive_name}-{basename}',
                 'data': data,
                 'data_type': self.data_type,
                 'url': response.request.url,

tests/middlewares/test_kingfisher_transform_middleware.py

@@ -90,8 +90,8 @@ def test_data_types(data_type, data, root_path):
     assert item == expected
 
 
-@pytest.mark.parametrize('sample,len_releases', [(None, 100), (5, 5)])
-def test_parse_release_package(sample, len_releases):
+@pytest.mark.parametrize('sample,len_releases,file_name', [(None, 100, 'test'), (5, 5, 'test')])
+def test_parse_release_package(sample, len_releases, file_name):
     spider = spider_with_crawler(spider_class=CompressedFileSpider, sample=sample)
     spider.data_type = 'release_package'
     spider.resize_package = True
@@ -106,7 +106,7 @@ def test_parse_release_package(sample, len_releases):
     with ZipFile(io, 'w', compression=ZIP_DEFLATED) as zipfile:
         zipfile.writestr('test.json', json.dumps(package))
 
-    response = response_fixture(body=io.getvalue(), meta={'file_name': 'test.zip'})
+    response = response_fixture(body=io.getvalue(), meta={'file_name': f'{file_name}.zip'})
     generator = spider.parse(response)
     item = next(generator)
 
@@ -116,7 +116,7 @@ def test_parse_release_package(sample, len_releases):
     for i, item in enumerate(transformed_items, 1):
         assert type(item) is FileItem
         assert len(item) == 6
-        assert item['file_name'] == 'test.json'
+        assert item['file_name'] == f'{file_name}-test.json'
         assert item['url'] == 'http://example.com'
         assert item['number'] == i
         assert len(json.loads(item['data'])['releases']) == len_releases
@@ -181,7 +181,7 @@ def test_line_delimited_json_middleware_compressed(sample):
     for i, item in enumerate(transformed_items, 1):
         assert type(item) is FileItem
         assert item == {
-            'file_name': 'test.json',
+            'file_name': 'test-test.json',
             'url': 'http://example.com',
             'number': i,
             'data': '{"key": %s}\n' % i,

tests/test_compressed_file_spider.py

@@ -24,7 +24,7 @@ def test_parse():
     item = next(generator)
 
     assert type(item) is File
-    assert item['file_name'] == 'test.json'
+    assert item['file_name'] == 'test-test.json'
     assert item['url'] == 'http://example.com'
     assert item['data_type'] == 'release_package'
     assert item['encoding'] == 'utf-8'
@@ -35,8 +35,8 @@ def test_parse():
         next(generator)
 
 
-@pytest.mark.parametrize('sample,len_items', [(None, 20), (5, 5)])
-def test_parse_line_delimited(sample, len_items):
+@pytest.mark.parametrize('sample,len_items,file_name', [(None, 20, 'test'), (5, 5, 'test')])
+def test_parse_line_delimited(sample, len_items, file_name):
     spider = spider_with_crawler(spider_class=CompressedFileSpider, sample=sample)
     spider.data_type = 'release_package'
     spider.line_delimited = True
@@ -49,13 +49,13 @@ def test_parse_line_delimited(sample, len_items):
     with ZipFile(io, 'w', compression=ZIP_DEFLATED) as zipfile:
         zipfile.writestr('test.json', ''.join(content))
 
-    response = response_fixture(body=io.getvalue(), meta={'file_name': 'test.zip'})
+    response = response_fixture(body=io.getvalue(), meta={'file_name': f'{file_name}.zip'})
     generator = spider.parse(response)
     item = next(generator)
 
     assert type(item) is File
     assert len(item) == 5
-    assert item['file_name'] == 'test.json'
+    assert item['file_name'] == f'{file_name}-test.json'
     assert item['url'] == 'http://example.com'
     assert item['data_type'] == 'release_package'
     assert item['encoding'] == 'utf-8'
@@ -66,8 +66,8 @@ def test_parse_line_delimited(sample, len_items):
         next(generator)
 
 
-@pytest.mark.parametrize('sample,len_items,len_releases', [(None, 2, 100), (5, 1, 5)])
-def test_parse_release_package(sample, len_items, len_releases):
+@pytest.mark.parametrize('sample,len_items,len_releases, file_name', [(None, 2, 100, 'test'), (5, 1, 5, 'test')])
+def test_parse_release_package(sample, len_items, len_releases, file_name):
     spider = spider_with_crawler(spider_class=CompressedFileSpider, sample=sample)
     spider.data_type = 'release_package'
     spider.resize_package = True
@@ -80,13 +80,13 @@ def test_parse_release_package(sample, len_items, len_releases):
     with ZipFile(io, 'w', compression=ZIP_DEFLATED) as zipfile:
         zipfile.writestr('test.json', json.dumps(package))
 
-    response = response_fixture(body=io.getvalue(), meta={'file_name': 'test.zip'})
+    response = response_fixture(body=io.getvalue(), meta={'file_name': f'{file_name}.zip'})
     generator = spider.parse(response)
     item = next(generator)
 
     assert type(item) is File
     assert len(item) == 5
-    assert item['file_name'] == 'test.json'
+    assert item['file_name'] == f'{file_name}-test.json'
     assert item['url'] == 'http://example.com'
     assert item['data_type'] == 'release_package'
     assert item['encoding'] == 'utf-8'
@@ -126,7 +126,7 @@ def test_parse_rar_file():
 
     assert type(item) is File
     assert len(item) == 5
-    assert item['file_name'] == 'test.json'
+    assert item['file_name'] == 'test-test.json'
     assert item['url'] == 'http://example.com'
     assert item['data_type'] == 'release_package'
     assert item['encoding'] == 'utf-8'