Merge 2416cd7 into 9fc84dc

.coveragerc

@@ -0,0 +1,2 @@
+[run]
+source = datastore

.travis.yml

@@ -40,8 +40,10 @@ before_script:
 
 script:
   - "./datastore/manage.py migrate"
-  - "./datastore/manage.py test tests"
+  - "coverage run --source='./datastore' ./datastore/manage.py test tests"
   - "datagetter.py --help" # Verify it compiles and is installed
   - "flake8"
-  - "isort --check-only --recursive ./"
+  - "isort --check-only ./"
   - "black --check ./"
+
+after_success: coveralls

README.md

@@ -2,7 +2,8 @@
 
 [![Build Status](https://travis-ci.com/ThreeSixtyGiving/datastore.svg?branch=master)](https://travis-ci.com/ThreeSixtyGiving/datastore)
 
-
+[![Coverage Status](https://coveralls.io/repos/github/ThreeSixtyGiving/Dashboard/badge.svg?branch=master)](https://coveralls.io/github/ThreeSixtyGiving/Dashboard?branch=master)
+
 ## Postgres setup
 
 Example:

datastore/additional_data/admin.py

@@ -1,9 +1,8 @@
+import additional_data.models as db
 from django.contrib import admin
 from django.contrib.postgres.fields import JSONField
 from prettyjson import PrettyJSONWidget
 
-import additional_data.models as db
-
 
 class JSONFieldAdmin(admin.ModelAdmin):
     list_per_page = 20

datastore/additional_data/management/commands/delete_org_data.py

@@ -1,14 +1,15 @@
-from django.core.management.base import BaseCommand
-
 from additional_data.models import OrgInfoCache
+from django.core.management.base import BaseCommand
 
 
 class Command(BaseCommand):
     help = "Delete find that charity data"
 
     def add_arguments(self, parser):
         parser.add_argument(
-            "--no-prompt", action="store_true", help="Don't prompt before delete",
+            "--no-prompt",
+            action="store_true",
+            help="Don't prompt before delete",
         )
 
     def handle(self, *args, **options):

datastore/additional_data/management/commands/load_code_names.py

@@ -1,6 +1,5 @@
-from django.core.management.base import BaseCommand
-
 from additional_data.sources.code_names import CodeNamesSource
+from django.core.management.base import BaseCommand
 
 
 class Command(BaseCommand):

datastore/additional_data/management/commands/load_geolookups.py

@@ -1,6 +1,5 @@
-from django.core.management.base import BaseCommand
-
 from additional_data.sources.geo_lookup import GeoLookupSource
+from django.core.management.base import BaseCommand
 
 
 class Command(BaseCommand):

datastore/additional_data/management/commands/load_nspl.py

@@ -1,6 +1,5 @@
-from django.core.management.base import BaseCommand
-
 from additional_data.sources.nspl import NSPLSource
+from django.core.management.base import BaseCommand
 
 
 class Command(BaseCommand):

datastore/additional_data/management/commands/load_org_data.py

@@ -1,8 +1,7 @@
-from django.core.management.base import BaseCommand
-from django.db import transaction
-
 from additional_data.models import OrgInfoCache
 from additional_data.sources.find_that_charity import FindThatCharitySource
+from django.core.management.base import BaseCommand
+from django.db import transaction
 
 
 class Command(BaseCommand):

datastore/additional_data/management/commands/rewrite_additional_data.py

@@ -1,7 +1,6 @@
-from django.core.management.base import BaseCommand
-
 from additional_data.generator import AdditionalDataGenerator
 from db.models import Grant, Latest
+from django.core.management.base import BaseCommand
 
 
 class Command(BaseCommand):

datastore/additional_data/migrations/0006_tsgorgtype.py

@@ -1,8 +1,7 @@
 # Generated by Django 2.2.13 on 2020-09-16 14:20
 
-from django.db import migrations, models
-
 import additional_data.models
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):

datastore/additional_data/sources/code_names.py

@@ -4,7 +4,6 @@
 from datetime import datetime
 
 import requests
-
 from additional_data.models import CodeName
 
 # based on 'import_chd' function in

datastore/additional_data/sources/data_files/fetch_charity_data.py

@@ -6,7 +6,6 @@
 from io import StringIO
 
 import requests
-
 import titlecase
 
 current_dir = os.path.dirname(os.path.realpath(__file__))

datastore/additional_data/sources/find_that_charity.py

@@ -2,13 +2,12 @@
 import json
 
 import requests
-
 from additional_data.models import OrgInfoCache
 
 
 class FindThatCharitySource(object):
-    """ This is responsible for inputting and outputting data from the
-    FindThatCharity (FTC) organisation info data sources """
+    """This is responsible for inputting and outputting data from the
+    FindThatCharity (FTC) organisation info data sources"""
 
     ADDITIONAL_DATA_KEY = "recipientOrgInfos"
 

datastore/additional_data/sources/geo_lookup.py

@@ -2,7 +2,6 @@
 import io
 
 import requests
-
 from additional_data.models import GeoLookup
 
 
@@ -17,14 +16,20 @@ class GeoLookupSource(object):
             "url_latlong": "https://raw.githubusercontent.com/drkane/geo-lookups/master/lsoa_latlong.csv",
             "field_areacode": "lsoa11cd",
             "field_areaname": "lsoa11nm",
-            "field_transforms": {"lad20cd": "ladcd", "lad20nm": "ladnm",},
+            "field_transforms": {
+                "lad20cd": "ladcd",
+                "lad20nm": "ladnm",
+            },
         },
         "msoa": {
             "url_lookup": "https://raw.githubusercontent.com/drkane/geo-lookups/master/msoa_la.csv",
             "url_latlong": "https://raw.githubusercontent.com/drkane/geo-lookups/master/msoa_latlong.csv",
             "field_areacode": "msoa11cd",
             "field_areaname": "msoa11hclnm",
-            "field_transforms": {"lad20cd": "ladcd", "lad20nm": "ladnm",},
+            "field_transforms": {
+                "lad20cd": "ladcd",
+                "lad20nm": "ladnm",
+            },
         },
         "ward": {
             "url_lookup": "https://raw.githubusercontent.com/drkane/geo-lookups/master/ward_all_codes.csv",
@@ -36,7 +41,10 @@ class GeoLookupSource(object):
             "url_lookup": "https://raw.githubusercontent.com/drkane/geo-lookups/master/la_all_codes.csv",
             "field_areacode": "ladcd",
             "field_areaname": "ladnm",
-            "field_transforms": {"lad20cd": "ladcd", "lad20nm": "ladnm",},
+            "field_transforms": {
+                "lad20cd": "ladcd",
+                "lad20nm": "ladnm",
+            },
         },
     }
 

datastore/additional_data/sources/nspl.py

@@ -5,7 +5,6 @@
 from datetime import datetime
 
 import requests
-
 from additional_data.models import NSPL, CodeName
 
 # based on https://github.com/drkane/find-that-postcode/blob/master/findthatpostcode/commands/postcodes.py

datastore/api/control/api.py

@@ -2,12 +2,11 @@
 import signal
 import subprocess
 
+import db.models as db
 from django.conf import settings
 from django.http.response import JsonResponse
 from django.views.generic import View
 
-import db.models as db
-
 
 class StatusView(View):
     def get(self, *args, **kwargs):

datastore/api/experimental/api.py

@@ -1,10 +1,9 @@
+import db.models as db
 import django_filters.rest_framework
+from api.experimental import serializers
 from rest_framework import filters, generics
 from rest_framework.pagination import LimitOffsetPagination
 
-import db.models as db
-from api.experimental import serializers
-
 
 class CurrentLatestGrantsPaginator(LimitOffsetPagination):
     default_limit = 60

datastore/api/experimental/serializers.py

@@ -1,6 +1,5 @@
-from rest_framework import serializers
-
 import db.models as db
+from rest_framework import serializers
 
 
 class GrantSerializer(serializers.ModelSerializer):

datastore/api/grantnav/api.py

@@ -1,9 +1,8 @@
+import db.models as db
 from django.conf import settings
 from django.http.response import JsonResponse
 from django.views import View
 
-import db.models as db
-
 
 class GrantNavPollForNewData(View):
     """ API endpoint for GrantNav to poll to know that new data is available """

datastore/api/urls.py

@@ -1,9 +1,8 @@
-from django.urls import path
-from django.views.generic import TemplateView
-
 import api.control.api
 import api.experimental.api
 import api.grantnav.api
+from django.urls import path
+from django.views.generic import TemplateView
 
 app_name = "api"
 

datastore/db/admin.py

@@ -1,9 +1,8 @@
+import db.models as db
 from django.contrib import admin
 from django.contrib.postgres.fields import JSONField
 from prettyjson import PrettyJSONWidget
 
-import db.models as db
-
 
 class JSONFieldAdmin(admin.ModelAdmin):
     list_per_page = 20

datastore/db/management/commands/create_datagetter_data.py

@@ -1,11 +1,10 @@
 import json
 import os
 
-from django.core.management.base import BaseCommand
-
 from additional_data.sources.local_files import LocalFilesSource
 from db.management.spinner import Spinner
 from db.models import Latest
+from django.core.management.base import BaseCommand
 
 
 class Command(BaseCommand):
@@ -31,13 +30,13 @@ def add_arguments(self, parser):
         )
 
     def handle(self, *args, **options):
-        """ Create grantnav package:
+        """Create grantnav package:
 
-          - data_all.json (all sources)
-          - json_all/
-                |- grants.json (lists of grants)
-                |- grants.json
-                ...
+        - data_all.json (all sources)
+        - json_all/
+              |- grants.json (lists of grants)
+              |- grants.json
+              ...
         """
         spinner = Spinner()
         spinner.start()

datastore/db/management/commands/delete_datagetter_data.py

@@ -1,6 +1,5 @@
-from django.core.management.base import BaseCommand, CommandError
-
 from db.models import GetterRun
+from django.core.management.base import BaseCommand, CommandError
 
 
 class Command(BaseCommand):
@@ -16,11 +15,15 @@ def add_arguments(self, parser):
         )
 
         parser.add_argument(
-            "--no-prompt", action="store_true", help="Don't prompt for deletes",
+            "--no-prompt",
+            action="store_true",
+            help="Don't prompt for deletes",
         )
 
         parser.add_argument(
-            "--oldest", action="store_true", help="Delete the oldest datagetter data",
+            "--oldest",
+            action="store_true",
+            help="Delete the oldest datagetter data",
         )
 
     def handle(self, *args, **options):

datastore/db/management/commands/list_datagetter_runs.py

@@ -1,6 +1,5 @@
-from django.core.management.base import BaseCommand
-
 from db.models import GetterRun
+from django.core.management.base import BaseCommand
 
 
 class Command(BaseCommand):

datastore/db/management/commands/load_datagetter_data.py

@@ -1,12 +1,11 @@
 import json
 import os
 
-from django.core.management.base import BaseCommand, CommandError
-from django.db import transaction
-
 import db.models as db
 from additional_data.generator import AdditionalDataGenerator
 from db.management.spinner import Spinner
+from django.core.management.base import BaseCommand, CommandError
+from django.db import transaction
 
 
 class Command(BaseCommand):
@@ -97,7 +96,8 @@ def load_data(self):
                 grants_added = grants_added + len(grant_data["grants"])
             except (FileNotFoundError, KeyError, TypeError, json.JSONDecodeError) as e:
                 print(
-                    "Skipping loading due to: '%s'" % e, file=self.stdout,
+                    "Skipping loading due to: '%s'" % e,
+                    file=self.stdout,
                 )
                 pass
 

datastore/db/management/commands/set_status.py

@@ -1,6 +1,5 @@
-from django.core.management.base import BaseCommand, CommandError
-
 from db.models import Status, Statuses
+from django.core.management.base import BaseCommand, CommandError
 
 
 class Command(BaseCommand):

datastore/db/migrations/0001_initial.py

@@ -50,7 +50,9 @@ class Migration(migrations.Migration):
                 ),
                 ("prefix", models.CharField(default=0, max_length=300)),
             ],
-            options={"unique_together": {("getter_run", "prefix")},},
+            options={
+                "unique_together": {("getter_run", "prefix")},
+            },
         ),
         migrations.CreateModel(
             name="SourceFile",

datastore/db/migrations/0005_auto_20190816_1015.py

@@ -11,6 +11,8 @@ class Migration(migrations.Migration):
 
     operations = [
         migrations.AlterField(
-            model_name="status", name="when", field=models.DateTimeField(auto_now=True),
+            model_name="status",
+            name="when",
+            field=models.DateTimeField(auto_now=True),
         ),
     ]

datastore/db/migrations/0008_remove_latest_total_grants.py

@@ -10,5 +10,8 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.RemoveField(model_name="latest", name="total_grants",),
+        migrations.RemoveField(
+            model_name="latest",
+            name="total_grants",
+        ),
     ]