Merge 2562571 into ec0953b

kingfisher_scrapy/spiders/scotland.py → kingfisher_scrapy/spiders/scotland_base.py

@@ -1,22 +1,16 @@
-from datetime import date, timedelta
+from datetime import date
 
 from kingfisher_scrapy.base_spider import SimpleSpider
-from kingfisher_scrapy.util import parameters
+from kingfisher_scrapy.util import date_range_by_month, parameters
 
 
-class Scotland(SimpleSpider):
-    """
-    API documentation
-      https://api.publiccontractsscotland.gov.uk/v1
-    Spider arguments
-      sample
-        Downloads packages for releases dated one year ago, for each notice type available.
-    """
-    name = 'scotland'
-    data_type = 'release_package'
+class ScotlandBase(SimpleSpider):
+    @classmethod
+    def from_crawler(cls, crawler, from_date=None, *args, **kwargs):
+        spider = super().from_crawler(crawler, date_format='year-month', from_date=from_date, *args, **kwargs)
+        return spider
 
-    def start_requests(self):
-        pattern = 'https://api.publiccontractsscotland.gov.uk/v1/Notices?dateFrom={}&outputType=1&noticeType={}'
+    def parse_requests(self, pattern):
 
         notice_types = [
             1,  # OJEU - F1 - Prior Information Notice
@@ -43,16 +37,17 @@ def start_requests(self):
         ]
 
         now = date.today()
+        if self.from_date:
+            start = date(self.from_date.year, self.from_date.month, 1)
+        else:
+            start = date(now.year - 1, now.month, 1)
+        if self.sample:
+            start = now
 
-        # It's meant to go back a year, but in testing it seemed to be year minus one day!
-        marker = now - timedelta(days=364)
-        while marker <= now:
-            datestring = '{:04d}-{:02d}-{:02d}'.format(marker.year, marker.month, marker.day)
+        for d in date_range_by_month(start, now):
+            date_string = '{:02d}-{:04d}'.format(d.month, d.year)
             for notice_type in notice_types:
                 yield self.build_request(
-                    pattern.format(datestring, notice_type),
+                    pattern.format(date_string, notice_type),
                     formatter=parameters('noticeType', 'dateFrom')
                 )
-            marker = marker + timedelta(days=14)
-            if self.sample:
-                break

kingfisher_scrapy/spiders/scotland_proactis.py

@@ -0,0 +1,19 @@
+from kingfisher_scrapy.spiders.scotland_base import ScotlandBase
+
+
+class ScotlandProactis(ScotlandBase):
+    """
+    API documentation
+      https://sandbox4.proactislabs.com/v1
+    Spider arguments
+      sample
+        Downloads packages for releases dated one year ago, for each notice type available.
+      from_date
+        Download only data from this month onward (YYYY-MM format). Defaults to one year back.
+    """
+    name = 'scotland_proactis'
+    data_type = 'release_package'
+
+    def start_requests(self):
+        pattern = 'https://sandbox4.proactislabs.com/v1/Notices?dateFrom={}&outputType=0&noticeType={}'
+        return self.parse_requests(pattern)

kingfisher_scrapy/spiders/scotland_public_contracts.py

@@ -0,0 +1,19 @@
+from kingfisher_scrapy.spiders.scotland_base import ScotlandBase
+
+
+class ScotlandPublicContracts(ScotlandBase):
+    """
+    API documentation
+      https://api.publiccontractsscotland.gov.uk/v1
+    Spider arguments
+      sample
+        Downloads packages for releases dated one year ago, for each notice type available.
+      from_date
+        Download only data from this month onward (YYYY-MM format). Defaults to one year back.
+    """
+    name = 'scotland_public_contracts'
+    data_type = 'release_package'
+
+    def start_requests(self):
+        pattern = 'https://api.publiccontractsscotland.gov.uk/v1/Notices?dateFrom={}&outputType=1&noticeType={}'
+        return self.parse_requests(pattern)