diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 00000000..ba2778dc --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,6 @@ +[settings] +multi_line_output=3 +include_trailing_comma=True +force_grid_wrap=0 +use_parentheses=True +line_length=88 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6843cee3..0c90000c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,6 +11,10 @@ repos: rev: 3.7.6 hooks: - id: flake8 +- repo: https://github.com/pre-commit/mirrors-isort + rev: v4.3.21 + hooks: + - id: isort - repo: https://github.com/ambv/black rev: stable hooks: diff --git a/scraper/scraper/pipelines.py b/scraper/scraper/pipelines.py index 018e7da3..42d27ace 100644 --- a/scraper/scraper/pipelines.py +++ b/scraper/scraper/pipelines.py @@ -1,11 +1,10 @@ -from scraper.settings import FILES_STORE, KEEP_FILES +import hashlib import os -import hashlib +from scraper.settings import FILES_STORE, KEEP_FILES +from scrapy.pipelines.files import FilesPipeline from scrapy.utils.python import to_bytes from six.moves.urllib.parse import urlparse - -from scrapy.pipelines.files import FilesPipeline from tika import parser diff --git a/scraper/scraper/settings.py b/scraper/scraper/settings.py index bc0f03ca..a091a15b 100644 --- a/scraper/scraper/settings.py +++ b/scraper/scraper/settings.py @@ -1,4 +1,5 @@ import os + from .items import ( CityCouncilAgendaItem, CityHallBidItem, diff --git a/scraper/scraper/spiders/citycouncil.py b/scraper/scraper/spiders/citycouncil.py index 912e5417..608cf730 100644 --- a/scraper/scraper/spiders/citycouncil.py +++ b/scraper/scraper/spiders/citycouncil.py @@ -1,7 +1,7 @@ from datetime import datetime -from scraper.items import CityCouncilAgendaItem import scrapy +from scraper.items import CityCouncilAgendaItem class AgendaSpider(scrapy.Spider): diff --git a/scraper/scraper/spiders/cityhall.py b/scraper/scraper/spiders/cityhall.py index 0aa4d833..a6895604 100644 --- a/scraper/scraper/spiders/cityhall.py +++ b/scraper/scraper/spiders/cityhall.py @@ -1,8 +1,9 @@ -from datetime import datetime, date, timedelta import re +from datetime import date, datetime, timedelta -from scraper.items import CityHallBidItem, CityHallContractItem, CityHallPaymentsItem import scrapy +from scraper.items import CityHallBidItem, CityHallContractItem, CityHallPaymentsItem + from .utils import identify_contract_id diff --git a/scraper/scraper/spiders/gazette.py b/scraper/scraper/spiders/gazette.py index 0987037e..a230f8d1 100644 --- a/scraper/scraper/spiders/gazette.py +++ b/scraper/scraper/spiders/gazette.py @@ -1,7 +1,9 @@ from datetime import datetime + import scrapy -from scrapy import Request from scraper.items import GazetteEventItem, LegacyGazetteItem +from scrapy import Request + from .utils import replace_query_param diff --git a/scraper/scraper/validators.py b/scraper/scraper/validators.py index 63aae991..1e9e108d 100644 --- a/scraper/scraper/validators.py +++ b/scraper/scraper/validators.py @@ -1,7 +1,7 @@ from schematics.models import Model from schematics.types import ( - DateType, DateTimeType, + DateType, DictType, IntType, ListType,