From ae9daeff6f81d2c2b7fa4bf9541e7c3840e43f25 Mon Sep 17 00:00:00 2001 From: cecicasco Date: Wed, 29 Sep 2021 21:53:29 -0400 Subject: [PATCH 1/7] change response manage to get codes --- kingfisher_scrapy/spiders/indonesia_opentender.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kingfisher_scrapy/spiders/indonesia_opentender.py b/kingfisher_scrapy/spiders/indonesia_opentender.py index 17095215..04b42a7c 100644 --- a/kingfisher_scrapy/spiders/indonesia_opentender.py +++ b/kingfisher_scrapy/spiders/indonesia_opentender.py @@ -1,3 +1,5 @@ +import json + from kingfisher_scrapy.base_spider import CompressedFileSpider, PeriodicSpider from kingfisher_scrapy.util import components, handle_http_error, join, parameters @@ -31,7 +33,7 @@ class IndonesiaOpentender(CompressedFileSpider, PeriodicSpider): @handle_http_error def parse_list(self, response): - data = response.json() + data = json.loads(response.xpath('//*[@id="content"]/div[2]/div[4]/pre/text()').get()) year = response.request.url.split('=')[1] requested_codes = [] for item in data['data']: From 10722f7e0b4744044dfd5cb72ceb5059e1a960bf Mon Sep 17 00:00:00 2001 From: cecicasco Date: Wed, 29 Sep 2021 22:24:36 -0400 Subject: [PATCH 2/7] change url and add &format=json --- kingfisher_scrapy/spiders/indonesia_opentender.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kingfisher_scrapy/spiders/indonesia_opentender.py b/kingfisher_scrapy/spiders/indonesia_opentender.py index 04b42a7c..69e134e7 100644 --- a/kingfisher_scrapy/spiders/indonesia_opentender.py +++ b/kingfisher_scrapy/spiders/indonesia_opentender.py @@ -28,12 +28,12 @@ class IndonesiaOpentender(CompressedFileSpider, PeriodicSpider): base_url = 'https://opentender.net/api/' # PeriodicSpider - pattern = base_url + 'master/lpse?year={}' + pattern = base_url + 'master/lpse?year={}&format=json' start_requests_callback = 'parse_list' @handle_http_error def parse_list(self, response): - data = json.loads(response.xpath('//*[@id="content"]/div[2]/div[4]/pre/text()').get()) + data = response.json() year = response.request.url.split('=')[1] requested_codes = [] for item in data['data']: From b1e00a496c5cec31d871d77ba690fb41496c4540 Mon Sep 17 00:00:00 2001 From: cecicasco Date: Wed, 29 Sep 2021 22:30:25 -0400 Subject: [PATCH 3/7] remove unused import --- kingfisher_scrapy/spiders/indonesia_opentender.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/kingfisher_scrapy/spiders/indonesia_opentender.py b/kingfisher_scrapy/spiders/indonesia_opentender.py index 69e134e7..1afcc28e 100644 --- a/kingfisher_scrapy/spiders/indonesia_opentender.py +++ b/kingfisher_scrapy/spiders/indonesia_opentender.py @@ -1,5 +1,3 @@ -import json - from kingfisher_scrapy.base_spider import CompressedFileSpider, PeriodicSpider from kingfisher_scrapy.util import components, handle_http_error, join, parameters From 24d1cb61a3d7e19de4f45ac942f62ddf4d8d0fc5 Mon Sep 17 00:00:00 2001 From: cecicasco Date: Wed, 29 Sep 2021 22:44:06 -0400 Subject: [PATCH 4/7] add formatter atribute --- kingfisher_scrapy/spiders/indonesia_opentender.py | 1 + 1 file changed, 1 insertion(+) diff --git a/kingfisher_scrapy/spiders/indonesia_opentender.py b/kingfisher_scrapy/spiders/indonesia_opentender.py index 1afcc28e..288f3d31 100644 --- a/kingfisher_scrapy/spiders/indonesia_opentender.py +++ b/kingfisher_scrapy/spiders/indonesia_opentender.py @@ -27,6 +27,7 @@ class IndonesiaOpentender(CompressedFileSpider, PeriodicSpider): base_url = 'https://opentender.net/api/' # PeriodicSpider pattern = base_url + 'master/lpse?year={}&format=json' + formatter = join(components(-1)) start_requests_callback = 'parse_list' @handle_http_error From 2b738b2f2aa482b0ec605b9734414766da65095c Mon Sep 17 00:00:00 2001 From: cecicasco Date: Wed, 29 Sep 2021 22:55:54 -0400 Subject: [PATCH 5/7] add formatter atribute --- kingfisher_scrapy/spiders/indonesia_opentender.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kingfisher_scrapy/spiders/indonesia_opentender.py b/kingfisher_scrapy/spiders/indonesia_opentender.py index 54a01950..cb28b048 100644 --- a/kingfisher_scrapy/spiders/indonesia_opentender.py +++ b/kingfisher_scrapy/spiders/indonesia_opentender.py @@ -28,7 +28,7 @@ class IndonesiaOpentender(CompressedFileSpider, PeriodicSpider): # PeriodicSpider pattern = base_url + 'master/lpse?year={}&format=json' - formatter = join(components(-1)) + formatter = staticmethod(components(-1)) start_requests_callback = 'parse_list' @handle_http_error From c741a65103cf34e344f71a950de58c1a262c48e5 Mon Sep 17 00:00:00 2001 From: Yohanna Lisnichuk Date: Wed, 29 Sep 2021 22:58:08 -0400 Subject: [PATCH 6/7] indonesia_opentender: get year from url and add formatter Signed-off-by: Yohanna Lisnichuk --- kingfisher_scrapy/spiders/indonesia_opentender.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kingfisher_scrapy/spiders/indonesia_opentender.py b/kingfisher_scrapy/spiders/indonesia_opentender.py index 7b46df2b..07919b19 100644 --- a/kingfisher_scrapy/spiders/indonesia_opentender.py +++ b/kingfisher_scrapy/spiders/indonesia_opentender.py @@ -1,5 +1,5 @@ from kingfisher_scrapy.base_spider import CompressedFileSpider, PeriodicSpider -from kingfisher_scrapy.util import components, handle_http_error, join, parameters +from kingfisher_scrapy.util import components, get_parameter_value, handle_http_error, join, parameters class IndonesiaOpentender(CompressedFileSpider, PeriodicSpider): @@ -28,12 +28,13 @@ class IndonesiaOpentender(CompressedFileSpider, PeriodicSpider): # PeriodicSpider pattern = base_url + 'master/lpse?year={}&format=json' + formatter = staticmethod(components(-1)) start_requests_callback = 'parse_list' @handle_http_error def parse_list(self, response): data = response.json() - year = response.request.url.split('=')[1] + year = get_parameter_value(response.request.url, 'year') requested_codes = [] for item in data['data']: code = item['code'] From 3671efa02cb1ccc8c2523e45d6587993c6bbf275 Mon Sep 17 00:00:00 2001 From: Yohanna Lisnichuk Date: Wed, 29 Sep 2021 23:00:27 -0400 Subject: [PATCH 7/7] indonesia_opentender: remove additional space Signed-off-by: Yohanna Lisnichuk --- kingfisher_scrapy/spiders/indonesia_opentender.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kingfisher_scrapy/spiders/indonesia_opentender.py b/kingfisher_scrapy/spiders/indonesia_opentender.py index 07919b19..84fc890a 100644 --- a/kingfisher_scrapy/spiders/indonesia_opentender.py +++ b/kingfisher_scrapy/spiders/indonesia_opentender.py @@ -34,7 +34,7 @@ class IndonesiaOpentender(CompressedFileSpider, PeriodicSpider): @handle_http_error def parse_list(self, response): data = response.json() - year = get_parameter_value(response.request.url, 'year') + year = get_parameter_value(response.request.url, 'year') requested_codes = [] for item in data['data']: code = item['code']