From 8cccd852659ab30ca2f59b00f05faad5b28639b9 Mon Sep 17 00:00:00 2001 From: Michael Wood Date: Tue, 25 May 2021 19:20:39 +0100 Subject: [PATCH] datastore: Update to new python black and flakes linting --- datastore/additional_data/generator.py | 4 ++-- datastore/additional_data/models.py | 4 ++-- .../additional_data/sources/find_that_charity.py | 4 ++-- datastore/additional_data/sources/local_files.py | 6 +++--- datastore/additional_data/sources/tsg_org_types.py | 2 +- datastore/api/grantnav/api.py | 2 +- datastore/api/urls.py | 5 +++-- datastore/data_quality/__init__.py | 2 -- datastore/data_quality/quality_data.py | 2 -- .../db/management/commands/create_data_package.py | 2 +- .../db/management/commands/load_datagetter_data.py | 6 +++--- datastore/db/management/spinner.py | 2 +- datastore/db/migrations/0011_auto_20210525_1336.py | 10 +++++----- datastore/db/migrations/0012_sourcefile_grants.py | 6 +++--- datastore/db/models.py | 12 +++++++----- datastore/tests/test_browser.py | 2 +- datastore/tests/test_commands.py | 6 ++++-- datastore/tests/test_quality_data.py | 1 - datastore/tests/test_views_respond.py | 2 +- 19 files changed, 40 insertions(+), 40 deletions(-) diff --git a/datastore/additional_data/generator.py b/datastore/additional_data/generator.py index bf1f4d79..ab63ac19 100644 --- a/datastore/additional_data/generator.py +++ b/datastore/additional_data/generator.py @@ -6,7 +6,7 @@ class AdditionalDataGenerator(object): - """ Adds additional data to grant data """ + """Adds additional data to grant data""" def __init__(self): self.local_files_source = LocalFilesSource() @@ -17,7 +17,7 @@ def __init__(self): # Initialise Other Sources heres def create(self, grant): - """ Takes a grant's data and returns a dict of additional data """ + """Takes a grant's data and returns a dict of additional data""" additional_data = {} diff --git a/datastore/additional_data/models.py b/datastore/additional_data/models.py index fe686045..0c61bddc 100644 --- a/datastore/additional_data/models.py +++ b/datastore/additional_data/models.py @@ -124,10 +124,10 @@ class GeoLookup(models.Model): class TSGOrgType(models.Model): - """ ThreeSixtyGiving Org Type mappings """ + """ThreeSixtyGiving Org Type mappings""" def validate_regex(value): - """ Check that the input regex is valid """ + """Check that the input regex is valid""" try: re.compile(value) except re.error as e: diff --git a/datastore/additional_data/sources/find_that_charity.py b/datastore/additional_data/sources/find_that_charity.py index 683ec934..438cd6c2 100644 --- a/datastore/additional_data/sources/find_that_charity.py +++ b/datastore/additional_data/sources/find_that_charity.py @@ -53,7 +53,7 @@ def update_additional_data(self, grant, additional_data): self._cache[org_id] = None def process_csv(self, file_data, org_type): - """ Returns total added. file_data array from csv """ + """Returns total added. file_data array from csv""" added = 0 bulk_list = [] @@ -89,7 +89,7 @@ def process_csv(self, file_data, org_type): return added def import_from_path(self, path, org_type=None): - """ Path can be http or file path, org_type if omitted we guess from the filename """ + """Path can be http or file path, org_type if omitted we guess from the filename""" added = 0 # Have a guess at the org type from the path diff --git a/datastore/additional_data/sources/local_files.py b/datastore/additional_data/sources/local_files.py index 2c8e977e..1832617f 100644 --- a/datastore/additional_data/sources/local_files.py +++ b/datastore/additional_data/sources/local_files.py @@ -51,7 +51,7 @@ def update_additional_data(self, grant, additional_data): self.update_additional_with_region(grant, additional_data) def _setup_charity_mappings(self): - """ Setup info for charity names """ + """Setup info for charity names""" with open(os.path.join(self.data_files_dir, "charity_names.json")) as fd: @@ -59,7 +59,7 @@ def _setup_charity_mappings(self): self.id_name_org_mappings["recipientOrganization"].update(charity_names) def _setup_org_name_mappings(self): - """ Setup overrides for org name """ + """Setup overrides for org name""" with open( os.path.join(self.data_files_dir, "primary_funding_org_name.json") @@ -68,7 +68,7 @@ def _setup_org_name_mappings(self): self.id_name_org_mappings["fundingOrganization"].update(funding_org_name) def _setup_area_mappings(self): - """ Setup the area/district mappings """ + """Setup the area/district mappings""" with open( os.path.join(self.data_files_dir, "codelist.csv") diff --git a/datastore/additional_data/sources/tsg_org_types.py b/datastore/additional_data/sources/tsg_org_types.py index 57a955e2..ee7f6388 100644 --- a/datastore/additional_data/sources/tsg_org_types.py +++ b/datastore/additional_data/sources/tsg_org_types.py @@ -4,7 +4,7 @@ class TSGOrgTypesSource(object): - """ This adds a custom ThreeSixtyGiving organisation type of the funding organisation to the additional data""" + """This adds a custom ThreeSixtyGiving organisation type of the funding organisation to the additional data""" ADDITIONAL_DATA_KEY = "TSGFundingOrgType" diff --git a/datastore/api/grantnav/api.py b/datastore/api/grantnav/api.py index 6812775d..3d2909bd 100644 --- a/datastore/api/grantnav/api.py +++ b/datastore/api/grantnav/api.py @@ -6,7 +6,7 @@ class GrantNavPollForNewData(View): - """ API endpoint for GrantNav to poll to know that new data is available """ + """API endpoint for GrantNav to poll to know that new data is available""" def get(self, *args, **kwargs): statuses = db.Status.objects.all() diff --git a/datastore/api/urls.py b/datastore/api/urls.py index 684cddca..15a0397d 100644 --- a/datastore/api/urls.py +++ b/datastore/api/urls.py @@ -10,9 +10,10 @@ urlpatterns = [ path("", TemplateView.as_view(template_name="api.html"), name="index"), - path("dashboard/publishers", + path( + "dashboard/publishers", api.dashboard.api.Publishers.as_view(), - name="publishers" + name="publishers", ), path( "grantnav/updates", diff --git a/datastore/data_quality/__init__.py b/datastore/data_quality/__init__.py index 5517d913..d009a366 100644 --- a/datastore/data_quality/__init__.py +++ b/datastore/data_quality/__init__.py @@ -4,8 +4,6 @@ ) from lib360dataquality.cove.schema import Schema360 -import json - schema = Schema360() diff --git a/datastore/data_quality/quality_data.py b/datastore/data_quality/quality_data.py index 4bb91271..74b7f425 100644 --- a/datastore/data_quality/quality_data.py +++ b/datastore/data_quality/quality_data.py @@ -3,8 +3,6 @@ common_checks_360, ) from lib360dataquality.cove.schema import Schema360 - -import json from tempfile import TemporaryDirectory schema = Schema360() diff --git a/datastore/db/management/commands/create_data_package.py b/datastore/db/management/commands/create_data_package.py index c434468a..bc51dd5e 100644 --- a/datastore/db/management/commands/create_data_package.py +++ b/datastore/db/management/commands/create_data_package.py @@ -50,7 +50,7 @@ def handle(self, *args, **options): data_all_file = "%s/data_all.json" % options["dir"] def flatten_grant(in_grant): - """ Add the additional_data inside grant object """ + """Add the additional_data inside grant object""" out_grant = {} out_grant.update(in_grant["data"]) try: diff --git a/datastore/db/management/commands/load_datagetter_data.py b/datastore/db/management/commands/load_datagetter_data.py index 12aaa04f..2b1c46c8 100644 --- a/datastore/db/management/commands/load_datagetter_data.py +++ b/datastore/db/management/commands/load_datagetter_data.py @@ -23,7 +23,7 @@ def add_arguments(self, parser): ) def check_dir_looks_right(self): - """ Quickly check if the supplied dir looks correct """ + """Quickly check if the supplied dir looks correct""" ls = os.listdir(self.options["data_dir"][0]) if "data_all.json" not in ls or "json_all" not in ls: @@ -33,13 +33,13 @@ def check_dir_looks_right(self): ) def load_dataset_data(self): - """ Loads the dataset data which describes the grant data """ + """Loads the dataset data which describes the grant data""" path = os.path.join(self.options["data_dir"][0], "data_all.json") with open(path, encoding="utf-8") as f: return json.loads(f.read()) def load_grant_data(self, path): - """ return the grant json for the given path """ + """return the grant json for the given path""" # As we want to use the path given by option to the command # reconstruct the file path with this value diff --git a/datastore/db/management/spinner.py b/datastore/db/management/spinner.py index 241f578e..0467944c 100644 --- a/datastore/db/management/spinner.py +++ b/datastore/db/management/spinner.py @@ -6,7 +6,7 @@ # Spinner - Michael Wood - Toaster - lsupdates GPLv2 class Spinner(threading.Thread): - """ A simple progress spinner to indicate download/parsing is happening""" + """A simple progress spinner to indicate download/parsing is happening""" def __init__(self, *args, **kwargs): super(Spinner, self).__init__(*args, **kwargs) diff --git a/datastore/db/migrations/0011_auto_20210525_1336.py b/datastore/db/migrations/0011_auto_20210525_1336.py index b9ec4990..e547a1c8 100644 --- a/datastore/db/migrations/0011_auto_20210525_1336.py +++ b/datastore/db/migrations/0011_auto_20210525_1336.py @@ -7,18 +7,18 @@ class Migration(migrations.Migration): dependencies = [ - ('db', '0010_auto_20200916_1109'), + ("db", "0010_auto_20200916_1109"), ] operations = [ migrations.AddField( - model_name='getterrun', - name='archived', + model_name="getterrun", + name="archived", field=models.BooleanField(default=False), ), migrations.AddField( - model_name='sourcefile', - name='quality', + model_name="sourcefile", + name="quality", field=django.contrib.postgres.fields.jsonb.JSONField(null=True), ), ] diff --git a/datastore/db/migrations/0012_sourcefile_grants.py b/datastore/db/migrations/0012_sourcefile_grants.py index 44357d16..149bdf3d 100644 --- a/datastore/db/migrations/0012_sourcefile_grants.py +++ b/datastore/db/migrations/0012_sourcefile_grants.py @@ -6,13 +6,13 @@ class Migration(migrations.Migration): dependencies = [ - ('db', '0011_auto_20210525_1336'), + ("db", "0011_auto_20210525_1336"), ] operations = [ migrations.AddField( - model_name='sourcefile', - name='grants', + model_name="sourcefile", + name="grants", field=models.IntegerField(default=0), ), ] diff --git a/datastore/db/models.py b/datastore/db/models.py index 37cf51be..66759226 100644 --- a/datastore/db/models.py +++ b/datastore/db/models.py @@ -5,7 +5,7 @@ class Latest(models.Model): - """ Latest best data we have """ + """Latest best data we have""" NEXT = "NEXT" CURRENT = "CURRENT" @@ -123,7 +123,7 @@ def delete_all_data_from_run(self): self.publisher_set.all().delete() def archive_run(self): - """ Archive the run and delete grant data """ + """Archive the run and delete grant data""" self.grant_set.all().delete() self.archived = True self.save() @@ -148,7 +148,7 @@ class SourceFile(models.Model): # We have this as an array but for now we can assume it will only have # one item for the purposes of our api. def get_distribution(self): - return self.data['distribution'][0] + return self.data["distribution"][0] def save(self, *args, **kwargs): try: @@ -182,7 +182,9 @@ class Publisher(models.Model): prefix = models.CharField(max_length=300) def get_sourcefiles(self): - return SourceFile.objects.filter(getter_run=self.getter_run, data__publisher__prefix=self.prefix) + return SourceFile.objects.filter( + getter_run=self.getter_run, data__publisher__prefix=self.prefix + ) # Update the convenience fields def save(self, *args, **kwargs): @@ -217,7 +219,7 @@ class Grant(models.Model): @staticmethod def estimated_total(): - """ Big table count() is expensive so estimate instead """ + """Big table count() is expensive so estimate instead""" try: with connection.cursor() as c: # https://www.citusdata.com/blog/2016/10/12/count-performance/ diff --git a/datastore/tests/test_browser.py b/datastore/tests/test_browser.py index 320eeb8b..ecf0f8af 100644 --- a/datastore/tests/test_browser.py +++ b/datastore/tests/test_browser.py @@ -5,7 +5,7 @@ class BrowserTests(StaticLiveServerTestCase): - """ Browser test using latest Chrome/Chromium stable""" + """Browser test using latest Chrome/Chromium stable""" def setUp(self, *args, **kwargs): capabilities = DesiredCapabilities.CHROME diff --git a/datastore/tests/test_commands.py b/datastore/tests/test_commands.py index 602e77f5..72c797a9 100644 --- a/datastore/tests/test_commands.py +++ b/datastore/tests/test_commands.py @@ -10,7 +10,7 @@ class CustomMgmtCommandsTest(TestCase): - """ Test custom management commands """ + """Test custom management commands""" fixtures = ["test_data.json"] @@ -74,7 +74,9 @@ def test_archive_getter_run_no_options(self): try: call_command("archive_getter_run", stderr=err_out) except CommandError as e: - self.assertTrue("No datagetter data specified" in str(e), "Unexpected exception") + self.assertTrue( + "No datagetter data specified" in str(e), "Unexpected exception" + ) self.assertEqual(len(err_out.getvalue()), 0, "Errors output by command") diff --git a/datastore/tests/test_quality_data.py b/datastore/tests/test_quality_data.py index a999d8ce..46a583ad 100644 --- a/datastore/tests/test_quality_data.py +++ b/datastore/tests/test_quality_data.py @@ -2,7 +2,6 @@ from data_quality import quality_data import db.models as db -import json class TestDataQualityData(TestCase): diff --git a/datastore/tests/test_views_respond.py b/datastore/tests/test_views_respond.py index 313bb0fa..672b6c6c 100644 --- a/datastore/tests/test_views_respond.py +++ b/datastore/tests/test_views_respond.py @@ -39,7 +39,7 @@ def _test_url(self, path, namespace=False): ) def test_url_responds(self): - """ Basic test to make sure all urls/views return """ + """Basic test to make sure all urls/views return""" for path in root_urls: self._test_url(path)