fix(uganda_releases): Add support for XLSX for 2023-2024. Rate limit …

…to avoid 403 error.

kingfisher_scrapy/pipelines.py

@@ -138,6 +138,14 @@ def process_item(self, item, spider):
             return item
 
         input_name = item.file_name
+
+        # uganda_releases yields JSON until 2023-2024, using the same URL pattern.
+        if input_name.endswith('.json'):
+            if item.data.startswith(b'PK\x03\x04'):
+                input_name = f'{os.path.splitext(input_name)[0]}.xlsx'
+            else:
+                return item
+
         if input_name.endswith('.csv'):
             item.file_name = f'{item.file_name[:-4]}.json'
             input_format = 'csv'

kingfisher_scrapy/spiders/uganda_releases.py

@@ -30,9 +30,13 @@ class UgandaReleases(PeriodicSpider):
         # returns HTTP 500 error FileNotFoundException. Therefore, we retry all codes in RETRY_HTTP_CODES except 500.
         'RETRY_HTTP_CODES': [status for status in RETRY_HTTP_CODES if status != 500],
     }
+    # Returns HTTP 403 if too many requests. (0.5 is too short.)
+    download_delay = 1
+
     # BaseSpider
     date_format = 'year'
     default_from_date = '2019'
+    unflatten = True
 
     # SimpleSpider
     data_type = 'release_package'
@@ -49,8 +53,7 @@ def parse(self, response):
         else:
             yield from super().parse(response)
             code = int(get_parameter_value(response.request.url, 'code')) + 1
-            yield self.build_request(replace_parameters(response.request.url, code=code),
-                                     formatter=self.formatter)
+            yield self.build_request(replace_parameters(response.request.url, code=code), formatter=self.formatter)
 
     def build_urls(self, date):
         yield self.pattern.format(date, date + 1)