Skip to content

Commit

Permalink
Update changes from review
Browse files Browse the repository at this point in the history
  • Loading branch information
aguilerapy committed Sep 21, 2020
1 parent 5b4723d commit 8a8898b
Showing 1 changed file with 6 additions and 5 deletions.
11 changes: 6 additions & 5 deletions kingfisher_scrapy/spiders/honduras_oncae.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,18 +45,19 @@ def start_requests(self):

@handle_http_error
def parse_list(self, response):
systems_flags = {system: False for system in self.systems}
urls = response.xpath('//a[contains(., "[json]")]/@href').getall()
for url in urls:
path, file = split(urlparse(url).path)
current_system = path.replace('/datosabiertos/', "")
if self.system and current_system != self.system:
continue
if self.sample:
if systems_flags[current_system]:
continue
if next((system for system in systems_flags if not system), False):
# if we already downloaded a package for all the available systems
if not self.systems:
return
systems_flags[current_system] = True
# if we already processed a file for the current system
if current_system not in self.systems:
continue
self.systems.remove(current_system)
# URL looks like http://200.13.162.79/datosabiertos/HC1/HC1_datos_2020_json.zip
yield self.build_request(url, formatter=components(-1))

0 comments on commit 8a8898b

Please sign in to comment.