Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

CompressedFileSpider: name files including archive name #681

Merged
merged 1 commit into from
Mar 19, 2021
Merged

CompressedFileSpider: name files including archive name #681

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion kingfisher_scrapy/base_spider.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -374,7 +374,7 @@ def parse(self, response):
data = compressed_file

yield File({
'file_name': basename,
'file_name': f'{archive_name}-{basename}',
'data': data,
'data_type': self.data_type,
'url': response.request.url,
Expand Down
10 changes: 5 additions & 5 deletions tests/middlewares/test_kingfisher_transform_middleware.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,8 +90,8 @@ def test_data_types(data_type, data, root_path):
assert item == expected


@pytest.mark.parametrize('sample,len_releases', [(None, 100), (5, 5)])
def test_parse_release_package(sample, len_releases):
@pytest.mark.parametrize('sample,len_releases,file_name', [(None, 100, 'test'), (5, 5, 'test')])
def test_parse_release_package(sample, len_releases, file_name):
spider = spider_with_crawler(spider_class=CompressedFileSpider, sample=sample)
spider.data_type = 'release_package'
spider.resize_package = True
Expand All @@ -106,7 +106,7 @@ def test_parse_release_package(sample, len_releases):
with ZipFile(io, 'w', compression=ZIP_DEFLATED) as zipfile:
zipfile.writestr('test.json', json.dumps(package))

response = response_fixture(body=io.getvalue(), meta={'file_name': 'test.zip'})
response = response_fixture(body=io.getvalue(), meta={'file_name': f'{file_name}.zip'})
generator = spider.parse(response)
item = next(generator)

Expand All @@ -116,7 +116,7 @@ def test_parse_release_package(sample, len_releases):
for i, item in enumerate(transformed_items, 1):
assert type(item) is FileItem
assert len(item) == 6
assert item['file_name'] == 'test.json'
assert item['file_name'] == f'{file_name}-test.json'
assert item['url'] == 'http://example.com'
assert item['number'] == i
assert len(json.loads(item['data'])['releases']) == len_releases
Expand Down Expand Up @@ -181,7 +181,7 @@ def test_line_delimited_json_middleware_compressed(sample):
for i, item in enumerate(transformed_items, 1):
assert type(item) is FileItem
assert item == {
'file_name': 'test.json',
'file_name': 'test-test.json',
'url': 'http://example.com',
'number': i,
'data': '{"key": %s}\n' % i,
Expand Down
20 changes: 10 additions & 10 deletions tests/test_compressed_file_spider.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ def test_parse():
item = next(generator)

assert type(item) is File
assert item['file_name'] == 'test.json'
assert item['file_name'] == 'test-test.json'
assert item['url'] == 'http://example.com'
assert item['data_type'] == 'release_package'
assert item['encoding'] == 'utf-8'
Expand All @@ -35,8 +35,8 @@ def test_parse():
next(generator)


@pytest.mark.parametrize('sample,len_items', [(None, 20), (5, 5)])
def test_parse_line_delimited(sample, len_items):
@pytest.mark.parametrize('sample,len_items,file_name', [(None, 20, 'test'), (5, 5, 'test')])
def test_parse_line_delimited(sample, len_items, file_name):
spider = spider_with_crawler(spider_class=CompressedFileSpider, sample=sample)
spider.data_type = 'release_package'
spider.line_delimited = True
Expand All @@ -49,13 +49,13 @@ def test_parse_line_delimited(sample, len_items):
with ZipFile(io, 'w', compression=ZIP_DEFLATED) as zipfile:
zipfile.writestr('test.json', ''.join(content))

response = response_fixture(body=io.getvalue(), meta={'file_name': 'test.zip'})
response = response_fixture(body=io.getvalue(), meta={'file_name': f'{file_name}.zip'})
generator = spider.parse(response)
item = next(generator)

assert type(item) is File
assert len(item) == 5
assert item['file_name'] == 'test.json'
assert item['file_name'] == f'{file_name}-test.json'
assert item['url'] == 'http://example.com'
assert item['data_type'] == 'release_package'
assert item['encoding'] == 'utf-8'
Expand All @@ -66,8 +66,8 @@ def test_parse_line_delimited(sample, len_items):
next(generator)


@pytest.mark.parametrize('sample,len_items,len_releases', [(None, 2, 100), (5, 1, 5)])
def test_parse_release_package(sample, len_items, len_releases):
@pytest.mark.parametrize('sample,len_items,len_releases, file_name', [(None, 2, 100, 'test'), (5, 1, 5, 'test')])
def test_parse_release_package(sample, len_items, len_releases, file_name):
spider = spider_with_crawler(spider_class=CompressedFileSpider, sample=sample)
spider.data_type = 'release_package'
spider.resize_package = True
Expand All @@ -80,13 +80,13 @@ def test_parse_release_package(sample, len_items, len_releases):
with ZipFile(io, 'w', compression=ZIP_DEFLATED) as zipfile:
zipfile.writestr('test.json', json.dumps(package))

response = response_fixture(body=io.getvalue(), meta={'file_name': 'test.zip'})
response = response_fixture(body=io.getvalue(), meta={'file_name': f'{file_name}.zip'})
generator = spider.parse(response)
item = next(generator)

assert type(item) is File
assert len(item) == 5
assert item['file_name'] == 'test.json'
assert item['file_name'] == f'{file_name}-test.json'
assert item['url'] == 'http://example.com'
assert item['data_type'] == 'release_package'
assert item['encoding'] == 'utf-8'
Expand Down Expand Up @@ -126,7 +126,7 @@ def test_parse_rar_file():

assert type(item) is File
assert len(item) == 5
assert item['file_name'] == 'test.json'
assert item['file_name'] == 'test-test.json'
assert item['url'] == 'http://example.com'
assert item['data_type'] == 'release_package'
assert item['encoding'] == 'utf-8'
Expand Down