Skip to content

Commit

Permalink
chore: Reduce repetition in RootPathMiddleware. Put package metadata …
Browse files Browse the repository at this point in the history
…first in AddPackageMiddleware.
  • Loading branch information
jpmckinney committed Oct 5, 2023
1 parent c393bea commit 9c66ea5
Showing 1 changed file with 7 additions and 5 deletions.
12 changes: 7 additions & 5 deletions kingfisher_scrapy/spidermiddlewares.py
Original file line number Diff line number Diff line change
Expand Up @@ -130,14 +130,16 @@ def process_spider_output(self, response, result, spider):
if sample_filled(spider, number):
return

# Omit the None values returned by `grouper(*, fillvalue=None)`.
items = filter(None, items)

if is_package:
# Assume that the `extensions` are the same for all packages.
package = items[0]
for other in filter(None, items[1:]):
package = next(items)
for other in items:
package[key].extend(other[key])
else:
# Omit the None values returned by `grouper(*, fillvalue=None)`.
package = {'version': spider.ocds_version, key: list(filter(None, items))}
package = {'version': spider.ocds_version, key: list(items)}

yield spider.build_file_item(number, package, item)
else:
Expand Down Expand Up @@ -178,7 +180,7 @@ def process_spider_output(self, response, result, spider):
else:
key = 'records'

item['data'] = {key: [data], 'version': spider.ocds_version}
item['data'] = {'version': spider.ocds_version, key: [data]}
item['data_type'] += '_package'

yield item
Expand Down

0 comments on commit 9c66ea5

Please sign in to comment.