Merge 0b35c99 into ffbcb09

.github/workflows/lint.yml

@@ -20,5 +20,5 @@ jobs:
     - run: pip install -r requirements_dev.txt
 
     - run: flake8
-    # To avoid datagetter errors, we must specifically exclude src/datagetter
-    - run: black --check --exclude "/(\.eggs|\.git|_cache|src\/datagetter)/" ./
+    # To local srcs, we must specifically exclude src
+    - run: black --check --exclude "/(\.eggs|\.git|_cache|src)/" ./

.github/workflows/test.yml

@@ -41,7 +41,7 @@ jobs:
     - run: psql -c "CREATE USER test WITH ENCRYPTED PASSWORD 'test'; GRANT ALL PRIVILEGES ON DATABASE \"360givingdatastore\" TO test; ALTER USER test CREATEDB;" -W -h localhost postgres postgres
 
     - run: ./datastore/manage.py migrate
-    - run: coverage run --source='./datastore' ./datastore/manage.py test tests
+    - run: coverage run --source='./datastore' ./datastore/manage.py test -v 3 tests
       env:
         DATA_RUN_PID_FILE: datarunpidfile.pid
     - name: Verify it compiles and is installed

datastore/additional_data/generator.py

@@ -6,7 +6,7 @@
 
 
 class AdditionalDataGenerator(object):
-    """ Adds additional data to grant data """
+    """Adds additional data to grant data"""
 
     def __init__(self):
         self.local_files_source = LocalFilesSource()
@@ -17,7 +17,7 @@ def __init__(self):
         # Initialise Other Sources heres
 
     def create(self, grant):
-        """ Takes a grant's data and returns a dict of additional data """
+        """Takes a grant's data and returns a dict of additional data"""
 
         additional_data = {}
 

datastore/additional_data/models.py

@@ -124,10 +124,10 @@ class GeoLookup(models.Model):
 
 
 class TSGOrgType(models.Model):
-    """ ThreeSixtyGiving Org Type mappings """
+    """ThreeSixtyGiving Org Type mappings"""
 
     def validate_regex(value):
-        """ Check that the input regex is valid """
+        """Check that the input regex is valid"""
         try:
             re.compile(value)
         except re.error as e:

datastore/additional_data/sources/find_that_charity.py

@@ -53,7 +53,7 @@ def update_additional_data(self, grant, additional_data):
             self._cache[org_id] = None
 
     def process_csv(self, file_data, org_type):
-        """ Returns total added. file_data array from csv """
+        """Returns total added. file_data array from csv"""
         added = 0
         bulk_list = []
 
@@ -89,7 +89,7 @@ def process_csv(self, file_data, org_type):
         return added
 
     def import_from_path(self, path, org_type=None):
-        """ Path can be http or file path, org_type if omitted we guess from the filename """
+        """Path can be http or file path, org_type if omitted we guess from the filename"""
         added = 0
 
         # Have a guess at the org type from the path

datastore/additional_data/sources/local_files.py

@@ -51,15 +51,15 @@ def update_additional_data(self, grant, additional_data):
         self.update_additional_with_region(grant, additional_data)
 
     def _setup_charity_mappings(self):
-        """ Setup info for charity names """
+        """Setup info for charity names"""
 
         with open(os.path.join(self.data_files_dir, "charity_names.json")) as fd:
 
             charity_names = json.load(fd)
         self.id_name_org_mappings["recipientOrganization"].update(charity_names)
 
     def _setup_org_name_mappings(self):
-        """ Setup overrides for org name """
+        """Setup overrides for org name"""
 
         with open(
             os.path.join(self.data_files_dir, "primary_funding_org_name.json")
@@ -68,7 +68,7 @@ def _setup_org_name_mappings(self):
         self.id_name_org_mappings["fundingOrganization"].update(funding_org_name)
 
     def _setup_area_mappings(self):
-        """ Setup the area/district mappings """
+        """Setup the area/district mappings"""
 
         with open(
             os.path.join(self.data_files_dir, "codelist.csv")

datastore/additional_data/sources/tsg_org_types.py

@@ -4,7 +4,7 @@
 
 
 class TSGOrgTypesSource(object):
-    """ This adds a custom ThreeSixtyGiving organisation type of the funding organisation to the additional data"""
+    """This adds a custom ThreeSixtyGiving organisation type of the funding organisation to the additional data"""
 
     ADDITIONAL_DATA_KEY = "TSGFundingOrgType"
 

datastore/api/dashboard/api.py

@@ -0,0 +1,84 @@
+from django.http.response import JsonResponse
+from django.views import View
+from django.core.cache import cache
+
+import django_filters.rest_framework
+from rest_framework import filters, generics
+
+import db.models as db
+from api.dashboard import serializers
+from api.dashboard.permissions import ReadOnly
+from data_quality import quality_data
+
+
+class PublisherFilters(django_filters.rest_framework.FilterSet):
+    quality__hasGrantProgrammeTitle = django_filters.CharFilter(
+        method="hasGrantProgrammeTitle_filter", label="hasGrantProgrammeTitle"
+    )
+
+    def hasGrantProgrammeTitle_filter(self, queryset, name, value):
+        return queryset.filter(quality__hasGrantProgrammeTitle__iexact=value)
+
+    class Meta:
+        model = db.Publisher
+        fields = ["quality__hasGrantProgrammeTitle"]
+
+
+class Publishers(generics.ListAPIView):
+    serializer_class = serializers.PublishersSerializer
+    permission_classes = [ReadOnly]
+
+    filter_backends = (
+        filters.SearchFilter,
+        django_filters.rest_framework.DjangoFilterBackend,
+        filters.OrderingFilter,
+    )
+
+    filterset_class = PublisherFilters
+    search_fields = ("^data__name", "^prefix")
+    ordering_fields = [
+        "data__name",
+    ]
+
+    def get_queryset(self):
+        return db.Publisher.objects.filter(getter_run=db.GetterRun.objects.last())
+
+
+class Publisher(generics.RetrieveAPIView):
+    lookup_field = "prefix"
+    lookup_url_kwarg = "publisher_prefix"
+    serializer_class = serializers.PublisherSerializer
+
+    def get_queryset(self):
+        return db.Publisher.objects.filter(getter_run=db.GetterRun.objects.last())
+
+
+class Sources(generics.ListAPIView):
+    serializer_class = serializers.SourcesSerializer
+    #  pagination_class = CurrentLatestGrantsPaginator
+
+    def get_queryset(self):
+        return db.SourceFile.objects.filter(getter_run=db.GetterRun.objects.last())
+
+
+class Overview(View):
+    def get(self, *args, **kwargs):
+        # If we have a cache of this uri then return that.
+        # All caches are cleared if the dataload happens
+        full_request_uri = self.request.build_absolute_uri()
+
+        # Don't cache if we have ?nocache in the query
+        if not self.request.GET.get("nocache"):
+            ret = cache.get(full_request_uri)
+            if ret:
+                return JsonResponse(ret, safe=False)
+
+        mode = "overview_%s" % self.request.GET.get("mode")
+
+        latest = db.Latest.objects.get(series=db.Latest.CURRENT)
+        source_file_set = latest.sourcefile_set.all()
+
+        ret = quality_data.aggregated_stats(source_file_set, mode)
+        cache.set(full_request_uri, ret)
+
+        return JsonResponse(ret, safe=False)

datastore/api/dashboard/permissions.py

@@ -0,0 +1,14 @@
+from rest_framework import permissions
+
+
+class ReadOnly(permissions.BasePermission):
+    """
+    Always read-only even if authenticated
+    """
+
+    def has_object_permission(self, request, view, obj):
+        # we'll always allow GET, HEAD or OPTIONS requests.
+        if request.method in permissions.SAFE_METHODS:
+            return True
+
+        return False

datastore/api/dashboard/serializers.py

@@ -0,0 +1,63 @@
+from rest_framework import serializers
+
+import db.models as db
+
+
+class SourcesSerializer(serializers.ModelSerializer):
+    id = serializers.JSONField(source="data.identifier")
+    datagetter_data = serializers.JSONField()
+    modified = serializers.JSONField(source="data.modified")
+    grants = serializers.IntegerField()
+    distribution = serializers.JSONField(source="get_distribution")
+    quality = serializers.JSONField()
+    aggregate = serializers.JSONField()
+
+    class Meta:
+        model = db.SourceFile
+        fields = (
+            "id",
+            "datagetter_data",
+            "grants",
+            "distribution",
+            "modified",
+            "quality",
+            "aggregate",
+        )
+
+
+class PublishersSerializer(serializers.ModelSerializer):
+    name = serializers.JSONField(source="data.name")
+    logo = serializers.JSONField(source="data.logo")
+    prefix = serializers.JSONField(source="data.prefix")
+    website = serializers.JSONField(source="data.website")
+    aggregate = serializers.JSONField()
+    quality = serializers.JSONField()
+
+    class Meta:
+        model = db.Publisher
+        fields = (
+            "name",
+            "logo",
+            "prefix",
+            "website",
+            "aggregate",
+            "quality",
+        )
+
+
+class PublisherSerializer(PublishersSerializer):
+
+    # Same as Publishers but with added `files`
+    files = SourcesSerializer(source="get_sourcefiles", many=True)
+
+    class Meta:
+        model = db.Publisher
+        fields = (
+            "name",
+            "logo",
+            "prefix",
+            "website",
+            "aggregate",
+            "quality",
+            "files",
+        )

datastore/api/grantnav/api.py

@@ -6,7 +6,7 @@
 
 
 class GrantNavPollForNewData(View):
-    """ API endpoint for GrantNav to poll to know that new data is available """
+    """API endpoint for GrantNav to poll to know that new data is available"""
 
     def get(self, *args, **kwargs):
         statuses = db.Status.objects.all()

datastore/api/templates/api.html

@@ -2,11 +2,13 @@
 {% block title %}API{% endblock %}
 {% block content %}
 <div class="container">
-    <h2>API</h2>
+    <h2>API Index</h2>
     <ul>
         <li><a href="{% url "api:current-latest-grants" %}">{% url "api:current-latest-grants" %}</a></li>
         <li><a href="{% url "api:status" %}">{% url "api:status" %}</a></li>
         <li><a href="{% url "api:grantnav-updates" %}">{% url "api:grantnav-updates" %}</a></li>
+        <li><a href="{% url "api:publishers" %}">{% url "api:publishers" %}</a></li>
+        <li><a href="{% url "api:overview" %}">{% url "api:overview" %}</a></li>
     </ul>
 </div>
 {% endblock %}

datastore/api/urls.py

@@ -4,11 +4,27 @@
 import api.control.api
 import api.experimental.api
 import api.grantnav.api
+import api.dashboard.api
 
 app_name = "api"
 
 urlpatterns = [
     path("", TemplateView.as_view(template_name="api.html"), name="index"),
+    path(
+        "dashboard/publishers",
+        api.dashboard.api.Publishers.as_view(),
+        name="publishers",
+    ),
+    path(
+        "dashboard/overview",
+        api.dashboard.api.Overview.as_view(),
+        name="overview",
+    ),
+    path(
+        "dashboard/publisher/<str:publisher_prefix>",
+        api.dashboard.api.Publisher.as_view(),
+        name="publisher",
+    ),
     path(
         "grantnav/updates",
         api.grantnav.api.GrantNavPollForNewData.as_view(),

datastore/data_quality/__init__.py

@@ -0,0 +1,17 @@
+from lib360dataquality.cove.threesixtygiving import (
+    USEFULNESS_TEST_CLASS,
+    common_checks_360,
+)
+from lib360dataquality.cove.schema import Schema360
+
+schema = Schema360()
+
+
+def create(grants):
+    """grants: grants json"""
+
+    result = {}
+
+    common_checks_360(result, "/", grants, schema, test_classes=[USEFULNESS_TEST_CLASS])
+
+    return result

datastore/data_quality/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class AdditionalDataConfig(AppConfig):
+    name = "data_quality"