datastore: Update to new python black and flakes linting

datastore/additional_data/generator.py

@@ -6,7 +6,7 @@
 
 
 class AdditionalDataGenerator(object):
-    """ Adds additional data to grant data """
+    """Adds additional data to grant data"""
 
     def __init__(self):
         self.local_files_source = LocalFilesSource()
@@ -17,7 +17,7 @@ def __init__(self):
         # Initialise Other Sources heres
 
     def create(self, grant):
-        """ Takes a grant's data and returns a dict of additional data """
+        """Takes a grant's data and returns a dict of additional data"""
 
         additional_data = {}
 

datastore/additional_data/models.py

@@ -124,10 +124,10 @@ class GeoLookup(models.Model):
 
 
 class TSGOrgType(models.Model):
-    """ ThreeSixtyGiving Org Type mappings """
+    """ThreeSixtyGiving Org Type mappings"""
 
     def validate_regex(value):
-        """ Check that the input regex is valid """
+        """Check that the input regex is valid"""
         try:
             re.compile(value)
         except re.error as e:

datastore/additional_data/sources/find_that_charity.py

@@ -53,7 +53,7 @@ def update_additional_data(self, grant, additional_data):
             self._cache[org_id] = None
 
     def process_csv(self, file_data, org_type):
-        """ Returns total added. file_data array from csv """
+        """Returns total added. file_data array from csv"""
         added = 0
         bulk_list = []
 
@@ -89,7 +89,7 @@ def process_csv(self, file_data, org_type):
         return added
 
     def import_from_path(self, path, org_type=None):
-        """ Path can be http or file path, org_type if omitted we guess from the filename """
+        """Path can be http or file path, org_type if omitted we guess from the filename"""
         added = 0
 
         # Have a guess at the org type from the path

datastore/additional_data/sources/local_files.py

@@ -51,15 +51,15 @@ def update_additional_data(self, grant, additional_data):
         self.update_additional_with_region(grant, additional_data)
 
     def _setup_charity_mappings(self):
-        """ Setup info for charity names """
+        """Setup info for charity names"""
 
         with open(os.path.join(self.data_files_dir, "charity_names.json")) as fd:
 
             charity_names = json.load(fd)
         self.id_name_org_mappings["recipientOrganization"].update(charity_names)
 
     def _setup_org_name_mappings(self):
-        """ Setup overrides for org name """
+        """Setup overrides for org name"""
 
         with open(
             os.path.join(self.data_files_dir, "primary_funding_org_name.json")
@@ -68,7 +68,7 @@ def _setup_org_name_mappings(self):
         self.id_name_org_mappings["fundingOrganization"].update(funding_org_name)
 
     def _setup_area_mappings(self):
-        """ Setup the area/district mappings """
+        """Setup the area/district mappings"""
 
         with open(
             os.path.join(self.data_files_dir, "codelist.csv")

datastore/additional_data/sources/tsg_org_types.py

@@ -4,7 +4,7 @@
 
 
 class TSGOrgTypesSource(object):
-    """ This adds a custom ThreeSixtyGiving organisation type of the funding organisation to the additional data"""
+    """This adds a custom ThreeSixtyGiving organisation type of the funding organisation to the additional data"""
 
     ADDITIONAL_DATA_KEY = "TSGFundingOrgType"
 

datastore/api/grantnav/api.py

@@ -6,7 +6,7 @@
 
 
 class GrantNavPollForNewData(View):
-    """ API endpoint for GrantNav to poll to know that new data is available """
+    """API endpoint for GrantNav to poll to know that new data is available"""
 
     def get(self, *args, **kwargs):
         statuses = db.Status.objects.all()

datastore/api/urls.py

@@ -10,9 +10,10 @@
 
 urlpatterns = [
     path("", TemplateView.as_view(template_name="api.html"), name="index"),
-    path("dashboard/publishers",
+    path(
+        "dashboard/publishers",
         api.dashboard.api.Publishers.as_view(),
-        name="publishers"
+        name="publishers",
     ),
     path(
         "grantnav/updates",

datastore/data_quality/__init__.py

@@ -4,8 +4,6 @@
 )
 from lib360dataquality.cove.schema import Schema360
 
-import json
-
 schema = Schema360()
 
 

datastore/data_quality/quality_data.py

@@ -3,8 +3,6 @@
     common_checks_360,
 )
 from lib360dataquality.cove.schema import Schema360
-
-import json
 from tempfile import TemporaryDirectory
 
 schema = Schema360()

datastore/db/management/commands/create_data_package.py

@@ -50,7 +50,7 @@ def handle(self, *args, **options):
         data_all_file = "%s/data_all.json" % options["dir"]
 
         def flatten_grant(in_grant):
-            """ Add the additional_data inside grant object """
+            """Add the additional_data inside grant object"""
             out_grant = {}
             out_grant.update(in_grant["data"])
             try:

datastore/db/management/commands/load_datagetter_data.py

@@ -23,7 +23,7 @@ def add_arguments(self, parser):
         )
 
     def check_dir_looks_right(self):
-        """ Quickly check if the supplied dir looks correct """
+        """Quickly check if the supplied dir looks correct"""
         ls = os.listdir(self.options["data_dir"][0])
 
         if "data_all.json" not in ls or "json_all" not in ls:
@@ -33,13 +33,13 @@ def check_dir_looks_right(self):
             )
 
     def load_dataset_data(self):
-        """ Loads the dataset data which describes the grant data """
+        """Loads the dataset data which describes the grant data"""
         path = os.path.join(self.options["data_dir"][0], "data_all.json")
         with open(path, encoding="utf-8") as f:
             return json.loads(f.read())
 
     def load_grant_data(self, path):
-        """ return the grant json for the given path """
+        """return the grant json for the given path"""
 
         # As we want to use the path given by option to the command
         # reconstruct the file path with this value

datastore/db/management/spinner.py

@@ -6,7 +6,7 @@
 
 # Spinner - Michael Wood - Toaster - lsupdates GPLv2
 class Spinner(threading.Thread):
-    """ A simple progress spinner to indicate download/parsing is happening"""
+    """A simple progress spinner to indicate download/parsing is happening"""
 
     def __init__(self, *args, **kwargs):
         super(Spinner, self).__init__(*args, **kwargs)

datastore/db/migrations/0011_auto_20210525_1336.py

@@ -7,18 +7,18 @@
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('db', '0010_auto_20200916_1109'),
+        ("db", "0010_auto_20200916_1109"),
     ]
 
     operations = [
         migrations.AddField(
-            model_name='getterrun',
-            name='archived',
+            model_name="getterrun",
+            name="archived",
             field=models.BooleanField(default=False),
         ),
         migrations.AddField(
-            model_name='sourcefile',
-            name='quality',
+            model_name="sourcefile",
+            name="quality",
             field=django.contrib.postgres.fields.jsonb.JSONField(null=True),
         ),
     ]

datastore/db/migrations/0012_sourcefile_grants.py

@@ -6,13 +6,13 @@
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('db', '0011_auto_20210525_1336'),
+        ("db", "0011_auto_20210525_1336"),
     ]
 
     operations = [
         migrations.AddField(
-            model_name='sourcefile',
-            name='grants',
+            model_name="sourcefile",
+            name="grants",
             field=models.IntegerField(default=0),
         ),
     ]

datastore/db/models.py

@@ -5,7 +5,7 @@
 
 
 class Latest(models.Model):
-    """ Latest best data we have """
+    """Latest best data we have"""
 
     NEXT = "NEXT"
     CURRENT = "CURRENT"
@@ -123,7 +123,7 @@ def delete_all_data_from_run(self):
         self.publisher_set.all().delete()
 
     def archive_run(self):
-        """ Archive the run and delete grant data """
+        """Archive the run and delete grant data"""
         self.grant_set.all().delete()
         self.archived = True
         self.save()
@@ -148,7 +148,7 @@ class SourceFile(models.Model):
     # We have this as an array but for now we can assume it will only have
     # one item for the purposes of our api.
     def get_distribution(self):
-        return self.data['distribution'][0]
+        return self.data["distribution"][0]
 
     def save(self, *args, **kwargs):
         try:
@@ -182,7 +182,9 @@ class Publisher(models.Model):
     prefix = models.CharField(max_length=300)
 
     def get_sourcefiles(self):
-        return SourceFile.objects.filter(getter_run=self.getter_run, data__publisher__prefix=self.prefix)
+        return SourceFile.objects.filter(
+            getter_run=self.getter_run, data__publisher__prefix=self.prefix
+        )
 
     #  Update the convenience fields
     def save(self, *args, **kwargs):
@@ -217,7 +219,7 @@ class Grant(models.Model):
 
     @staticmethod
     def estimated_total():
-        """ Big table count() is expensive so estimate instead """
+        """Big table count() is expensive so estimate instead"""
         try:
             with connection.cursor() as c:
                 # https://www.citusdata.com/blog/2016/10/12/count-performance/

datastore/tests/test_browser.py

@@ -5,7 +5,7 @@
 
 
 class BrowserTests(StaticLiveServerTestCase):
-    """ Browser test using latest Chrome/Chromium stable"""
+    """Browser test using latest Chrome/Chromium stable"""
 
     def setUp(self, *args, **kwargs):
         capabilities = DesiredCapabilities.CHROME

datastore/tests/test_commands.py

@@ -10,7 +10,7 @@
 
 
 class CustomMgmtCommandsTest(TestCase):
-    """ Test custom management commands """
+    """Test custom management commands"""
 
     fixtures = ["test_data.json"]
 
@@ -74,7 +74,9 @@ def test_archive_getter_run_no_options(self):
         try:
             call_command("archive_getter_run", stderr=err_out)
         except CommandError as e:
-            self.assertTrue("No datagetter data specified" in str(e), "Unexpected exception")
+            self.assertTrue(
+                "No datagetter data specified" in str(e), "Unexpected exception"
+            )
 
         self.assertEqual(len(err_out.getvalue()), 0, "Errors output by command")
 

datastore/tests/test_quality_data.py

@@ -2,7 +2,6 @@
 
 from data_quality import quality_data
 import db.models as db
-import json
 
 
 class TestDataQualityData(TestCase):

datastore/tests/test_views_respond.py

@@ -39,7 +39,7 @@ def _test_url(self, path, namespace=False):
         )
 
     def test_url_responds(self):
-        """ Basic test to make sure all urls/views return """
+        """Basic test to make sure all urls/views return"""
         for path in root_urls:
             self._test_url(path)