Fix portugal scraper with json lines

kingfisher_scrapy/base_spider.py

@@ -107,20 +107,40 @@ def _get_crawl_path(self):
 # https://docs.python.org/3.8/library/functions.html#super
 # https://rhettinger.wordpress.com/2011/05/26/super-considered-super/
 class BaseSpider(KingfisherSpiderMixin, scrapy.Spider):
-    def parse_zipfile(self, response, data_type):
+
+    def parse_json_lines(self, f, data_type, url, encoding='utf-8'):
+        for number, line in enumerate(f):
+            number += 1
+            yield {
+                'success': True,
+                'number': number,
+                'file_name': 'data.json',
+                'data': line,
+                'data_type': data_type,
+                'url': url,
+                'encoding': encoding,
+            }
+            if self.sample and number > 10:
+                break
+
+    def parse_zipfile(self, response, data_type, file_format=None, encoding='utf-8'):
         """
         Handling response with JSON data in ZIP files
         """
         if response.status == 200:
+            if file_format == 'json_lines':
+                self.save_response_to_disk(response, 'file.zip')
             zip_file = ZipFile(BytesIO(response.body))
             for finfo in zip_file.infolist():
-                data = zip_file.open(finfo.filename).read()
-                if finfo.filename.endswith('.json'):
-                    filename = finfo.filename
+                filename = finfo.filename
+                if not filename.endswith('.json'):
+                    filename += '.json'
+                data = zip_file.open(finfo.filename)
+                if file_format == 'json_lines':
+                    yield from self.parse_json_lines(data, data_type, response.request.url, encoding=encoding)
                 else:
-                    filename = finfo.filename + '.json'
-                yield self.save_data_to_disk(data, filename, data_type=data_type, url=response.request.url)
-
+                    yield self.save_data_to_disk(data.read(), filename, data_type, response.request.url,
+                                                 encoding=encoding)
         else:
             yield {
                 'success': False,

kingfisher_scrapy/spiders/digiwhist_base.py

@@ -23,24 +23,9 @@ def parse(self, response):
                 }))
 
             # Load a line at the time, pass it to API
-            number = 0
             with tarfile.open(save_file_name, "r:gz") as tar:
                 with tar.extractfile(tar.getnames()[0]) as readfp:
-                    line = readfp.readline()
-                    while line:
-                        number += 1
-                        yield {
-                            'success': True,
-                            'number': number,
-                            'file_name': 'data.json',
-                            'data': line,
-                            'data_type': 'release_package',
-                            'url': self.start_urls[0],
-                        }
-                        line = readfp.readline()
-                        if self.sample and number > 10:
-                            break
-
+                    yield from self.parse_json_lines(readfp, 'release_package', url=self.start_urls[0])
         else:
             yield {
                 'success': False,

kingfisher_scrapy/spiders/portugal.py

@@ -41,4 +41,5 @@ def parse_list(self, response):
             }
 
     def parse(self, response):
-        yield from self.parse_zipfile(response, data_type='record_package_json_lines')
+        yield from self.parse_zipfile(response, data_type='record_package_json_lines',
+                                      file_format='json_lines', encoding='ISO-8859-1')