Adiciona item e validação para pagamentos (#26)

scraper/scraper/items.py

@@ -55,3 +55,21 @@ class CityHallBidItem(BaseItem):
     date = scrapy.Field()
     file_urls = scrapy.Field()
     file_content = scrapy.Field()
+
+
+class CityHallPaymentsItem(BaseItem):
+    published_at = scrapy.Field()
+    phase = scrapy.Field()
+    company_or_person = scrapy.Field()
+    value = scrapy.Field()
+    number = scrapy.Field()
+    document = scrapy.Field()
+    date = scrapy.Field()
+    process_number = scrapy.Field()
+    summary = scrapy.Field()
+    group = scrapy.Field()
+    action = scrapy.Field()
+    function = scrapy.Field()
+    subfunction = scrapy.Field()
+    type_of_process = scrapy.Field()
+    resource = scrapy.Field()

scraper/scraper/settings.py

@@ -3,6 +3,7 @@
     CityCouncilAgendaItem,
     CityHallBidItem,
     CityHallContractItem,
+    CityHallPaymentsItem,
     GazetteEventItem,
     LegacyGazetteItem,
 )
@@ -38,4 +39,5 @@
     CityCouncilAgendaItem: "scraper.validators.CityCouncilAgendaItem",
     CityHallContractItem: "scraper.validators.CityHallContractItem",
     CityHallBidItem: "scraper.validators.CityHallBidItem",
+    CityHallPaymentsItem: "scraper.validators.CityHallPaymentsItem",
 }

scraper/scraper/spiders/cityhall.py

@@ -1,7 +1,7 @@
 from datetime import datetime, date, timedelta
 import re
 
-from scraper.items import CityHallBidItem, CityHallContractItem
+from scraper.items import CityHallBidItem, CityHallContractItem, CityHallPaymentsItem
 import scrapy
 from .utils import identify_contract_id
 
@@ -263,13 +263,14 @@ def parse_page(self, response):
 
         for headline, raw_details in zip(headlines, details):
             headline = [text.strip() for text in headline.css("td ::text").extract()]
-            data = {
-                "published_at": headline[0],
-                "phase": headline[1],
-                "company_or_person": headline[2],
-                "value": headline[3],
-                "crawled_at": response.url,
-            }
+            item = CityHallPaymentsItem(
+                published_at=headline[0],
+                phase=headline[1],
+                company_or_person=headline[2],
+                value=headline[3],
+                crawled_at=datetime.now(),
+                crawled_from=response.url,
+            )
             details = [
                 detail.strip() for detail in raw_details.css("td ::text").extract()
             ]
@@ -290,6 +291,6 @@ def parse_page(self, response):
             while details_copy:
                 key = details_copy.pop(0)
                 value = details_copy.pop(0)
-                data[mapping[key]] = value
+                item[mapping[key]] = value
 
-            yield data
+            yield item

scraper/scraper/validators.py

@@ -65,3 +65,21 @@ class CityHallBidItem(BaseModel):
     date = DateTimeType(formats=("%d/%m/%Y %Hh%M"))
     file_urls = ListType(StringType)
     file_content = StringType()
+
+
+class CityHallPaymentsItem(BaseModel):
+    published_at = DateType(formats=("%d/%m/%Y", "%d/%m/%y"))
+    phase = StringType()
+    company_or_person = StringType(required=True)
+    value = StringType(required=True)
+    number = StringType()
+    document = StringType(required=True)
+    date = DateType(formats=("%d/%m/%Y", "%d/%m/%y"))
+    process_number = StringType()
+    summary = StringType()
+    group = StringType()
+    action = StringType()
+    function = StringType()
+    subfunction = StringType()
+    type_of_process = StringType()
+    resource = StringType()