Merge pull request #598 from open-contracting/595-afghanistan

Add new afghanistan endpoints

kingfisher_scrapy/spiders/afghanistan_packages_base.py

@@ -0,0 +1,31 @@
+from datetime import datetime
+
+import scrapy
+
+from kingfisher_scrapy.base_spider import SimpleSpider
+from kingfisher_scrapy.util import components, handle_http_error
+
+
+class AfghanistanPackagesBase(SimpleSpider):
+    default_from_date = '2018-12-15'
+    download_delay = 1.5  # for 'too many requests' errors
+
+    def start_requests(self):
+        yield scrapy.Request(self.base_url, meta={'file_name': 'list.json'}, callback=self.parse_list)
+
+    @handle_http_error
+    def parse_list(self, response):
+        urls = response.json()
+        for url in urls:
+            if self.from_date and self.until_date:
+                date = datetime.strptime(url[-10:], self.date_format)
+                if not (self.from_date <= date <= self.until_date):
+                    continue
+            yield self.build_request(url, formatter=components(-2), callback=self.parse_release_list)
+
+    @handle_http_error
+    def parse_release_list(self, response):
+        urls = response.xpath('//a/text()').getall()
+        for url in urls:
+            if 'https://ocds.ageops.net/api/' in url:
+                yield self.build_request(url.strip(), formatter=components(-2))

kingfisher_scrapy/spiders/afghanistan_record_packages.py

@@ -0,0 +1,18 @@
+from kingfisher_scrapy.spiders.afghanistan_packages_base import AfghanistanPackagesBase
+
+
+class AfghanistanRecordPackages(AfghanistanPackagesBase):
+    """
+    Domain
+      Afghanistan Government Electronic & Open Procurement System (AGEOPS)
+    Spider arguments
+      from_date
+        Download only data from this date onward (YYYY-MM-DD format). Defaults to '2018-12-15'.
+      until_date
+        Download only data until this date (YYYY-MM-DD format). Defaults to today.
+    API documentation
+      https://ocds.ageops.net/
+    """
+    name = 'afghanistan_record_packages'
+    base_url = 'https://ocds.ageops.net/api/ocds/record-package/dates'
+    data_type = 'record_package'

kingfisher_scrapy/spiders/afghanistan_release_packages.py

@@ -0,0 +1,18 @@
+from kingfisher_scrapy.spiders.afghanistan_packages_base import AfghanistanPackagesBase
+
+
+class AfghanistanReleasePackages(AfghanistanPackagesBase):
+    """
+    Domain
+      Afghanistan Government Electronic & Open Procurement System (AGEOPS)
+    Spider arguments
+      from_date
+        Download only data from this date onward (YYYY-MM-DD format). Defaults to '2018-12-15'.
+      until_date
+        Download only data until this date (YYYY-MM-DD format). Defaults to today.
+    API documentation
+      https://ocds.ageops.net/
+    """
+    name = 'afghanistan_release_packages'
+    base_url = 'https://ocds.ageops.net/api/ocds/release-package/dates'
+    data_type = 'release_package'