Skip to content

Commit

Permalink
chore(spidermiddlewares): Avoid shadowing data variable
Browse files Browse the repository at this point in the history
  • Loading branch information
jpmckinney committed Oct 4, 2023
1 parent 925c38b commit 03b1891
Showing 1 changed file with 5 additions and 4 deletions.
9 changes: 5 additions & 4 deletions kingfisher_scrapy/spidermiddlewares.py
Original file line number Diff line number Diff line change
Expand Up @@ -197,17 +197,18 @@ def process_spider_output(self, response, result, spider):
else:
key = 'records'

package = self._get_package_metadata(spider, data['package'], key, item['data_type'])
template = self._get_package_metadata(spider, data['package'], key, item['data_type'])
iterable = util.transcode(spider, ijson.items, data['data'], f'{key}.item')

for number, items in enumerate(util.grouper(iterable, group_size(spider)), 1):
if sample_filled(spider, number):
return

data = copy.deepcopy(package)
package = copy.deepcopy(template)
# Omit the None values returned by `grouper(*, fillvalue=None)`.
data[key] = list(filter(None, items))
package[key] = list(filter(None, items))

yield spider.build_file_item(number, data, item)
yield spider.build_file_item(number, package, item)

def _get_package_metadata(self, spider, data, skip_key, data_type):
"""
Expand Down

0 comments on commit 03b1891

Please sign in to comment.