refactor: Use django-rest-framework for processor views.

- Make endpoints more RESTful.
- Return HTTP 202 Accepted for creating and deleting datasets.
- Use DELETE method to delete datasets.
- Use status code instead of {"status": "ok"}.
- Return empty dict instead of None or int.

backend/core/settings.py

@@ -50,6 +50,7 @@
     "django.contrib.sessions",
     "django.contrib.messages",
     "django.contrib.staticfiles",
+    "rest_framework",
     "tastypie",
     "corsheaders",
     "dqt",
@@ -221,6 +222,12 @@
 elif not production:
     CORS_ALLOW_ALL_ORIGINS = True
 
+REST_FRAMEWORK = {
+    "DEFAULT_PARSER_CLASSES": [
+        "rest_framework.parsers.JSONParser",
+    ]
+}
+
 
 # Project configuration
 

backend/processor/urls.py

@@ -1,21 +1,11 @@
-from django.urls import path
+from django.urls import include, path
+from processor import views
+from rest_framework.routers import SimpleRouter
 
-from .views import (
-    create_dataset_filter,
-    dataset_availability,
-    dataset_id,
-    dataset_metadata,
-    dataset_progress,
-    dataset_start,
-    dataset_wipe,
-)
+router = SimpleRouter()
+router.register(r"datasets", views.DatasetViewSet, basename="dataset")
 
 urlpatterns = [
-    path("api/create_dataset_filter", create_dataset_filter, name="create_dataset_filter"),
-    path("api/dataset_status/<dataset_id>", dataset_progress, name="dataset_status"),
-    path("api/dataset_id", dataset_id, name="dataset_id"),
-    path("api/dataset_availability/<dataset_id>", dataset_availability, name="dataset_availability"),
-    path("api/dataset_metadata/<dataset_id>", dataset_metadata, name="dataset_metadata"),
-    path("api/dataset_start", dataset_start, name="dataset_start"),
-    path("api/dataset_wipe", dataset_wipe, name="dataset_wipe"),
+    path("api/create_dataset_filter", views.create_dataset_filter, name="create_dataset_filter"),
+    path("", include(router.urls)),
 ]

backend/processor/views.py

@@ -1,10 +1,14 @@
 import simplejson as json
 from django.db import connections
 from django.http import JsonResponse
+from django.shortcuts import get_object_or_404
 from django.views.decorators.csrf import csrf_exempt
 from django.views.decorators.http import require_POST
 from dqt.models import Dataset, FieldLevelCheck, ProgressMonitorDataset
 from psycopg2.sql import SQL, Identifier
+from rest_framework import status, viewsets
+from rest_framework.decorators import action
+from rest_framework.response import Response
 
 from .rabbitmq import publish
 
@@ -17,107 +21,94 @@ def create_dataset_filter(request):
     return JsonResponse({"status": "ok"})
 
 
-@csrf_exempt
-@require_POST
-def dataset_start(request):
-    body = json.loads(request.body.decode("utf-8"))
-
-    message = {
-        "name": body.get("name"),
-        "collection_id": body.get("collection_id"),
-    }
-    publish(json.dumps(message), "ocds_kingfisher_extractor_init")
-
-    return JsonResponse(
-        {"status": "ok", "data": {"message": "Started dataset '%(name)s' for collection %(collection_id)s" % message}}
-    )
-
-
-@csrf_exempt
-@require_POST
-def dataset_wipe(request):
-    body = json.loads(request.body.decode("utf-8"))
-
-    message = {
-        "dataset_id": body.get("dataset_id"),
-    }
-    publish(json.dumps(message), "wiper_init")
-
-    return JsonResponse({"status": "ok", "data": {"message": "Wiping dataset %(dataset_id)s" % message}})
-
-
-def dataset_progress(request, dataset_id):
-    try:
-        monitor = ProgressMonitorDataset.objects.values("state", "phase").get(dataset__id=dataset_id)
-        return JsonResponse({"status": "ok", "data": monitor})
-    except ProgressMonitorDataset.DoesNotExist:
-        return JsonResponse({"status": "ok", "data": None})
-
-
-def dataset_id(request):
-    dataset = Dataset.objects.get(name=request.GET.get("name"))
-
-    return JsonResponse({"status": "ok", "data": dataset.id if dataset else None})
-
-
-def dataset_availability(request, dataset_id):
-    map = {
-        "parties": ["parties.id"],
-        "plannings": ["planning.budget"],
-        "tenders": ["tender.id"],
-        "tenderers": ["tenderers.id"],
-        "tenders_items": ["tender.items.id"],
-        "awards": ["awards.id"],
-        "awards_items": ["awards.items.id"],
-        "awards_suppliers": ["awards.suppliers.id"],
-        "contracts": ["contracts.id"],
-        "contracts_items": ["contracts.items.id"],
-        "contracts_transactions": ["contracts.implementation.transactions.id"],
-        "documents": [
-            "planning.documents.id",
-            "tender.documents.id",
-            "awards.documents.id",
-            "contracts.documents.id",
-            "contracts.implementation.documents.id",
-        ],
-        "milestones": [
-            "planning.milestones.id",
-            "tender.milestones.id",
-            "contracts.milestones.id",
-            "contracts.implementation.milestones.id",
-        ],
-        "amendments": ["tender.amendments.id", "awards.amendments.id", "contract.amendments.id"],
-    }
-
-    with connections["data"].cursor() as cursor:
-        statement = """
-            SELECT c.key AS check, SUM(jsonb_array_length(c.value)) AS count
-            FROM {table} flc, jsonb_each(flc.result->'checks') c
-            WHERE dataset_id = %(dataset_id)s
-                AND c.key IN %(checks)s
-            GROUP BY c.key
-            ORDER BY c.key
-            """
-
-        cursor.execute(
-            SQL(statement).format(table=Identifier(FieldLevelCheck._meta.db_table)),
-            {"checks": tuple(j for i in map.values() for j in i), "dataset_id": dataset_id},
-        )
-
-        results = cursor.fetchall()
-
-        counts = {}
-        for key, items in map.items():
-            counts[key] = 0
-            for i in items:
-                for r in results:
-                    if r[0] == i:
-                        counts[key] += int(r[1])
-
-    return JsonResponse({"status": "ok", "data": counts})
-
-
-def dataset_metadata(request, dataset_id):
-    meta = Dataset.objects.values_list("meta__collection_metadata", flat=True).get(id=dataset_id)
-
-    return JsonResponse({"status": "ok", "data": meta})
+class DatasetViewSet(viewsets.GenericViewSet):
+    queryset = Dataset.objects.all()
+    # ViewSet's don't allow typed paths like <int:pk>.
+    # https://github.com/encode/django-rest-framework/pull/6789
+    # https://github.com/encode/django-rest-framework/issues/6148#issuecomment-725297421
+    lookup_value_regex = "[0-9]+"
+
+    def create(self, request):
+        message = {"name": request.data.get("name"), "collection_id": request.data.get("collection_id")}
+        publish(json.dumps(message), "ocds_kingfisher_extractor_init")
+        return Response(status=status.HTTP_202_ACCEPTED)
+
+    def destroy(self, request, pk=None):
+        message = {"dataset_id": int(pk)}
+        publish(json.dumps(message), "wiper_init")
+        return Response(status=status.HTTP_202_ACCEPTED)
+
+    @action(detail=False)
+    def find_by_name(self, request):
+        dataset = get_object_or_404(self.get_queryset(), name=request.GET.get("name"))
+        return Response({"id": dataset.id})
+
+    @action(detail=True)
+    def status(self, request, pk=None):
+        try:
+            monitor = ProgressMonitorDataset.objects.values("state", "phase").get(dataset__pk=pk)
+        except ProgressMonitorDataset.DoesNotExist:
+            monitor = {}
+        return Response(monitor)
+
+    @action(detail=True)
+    def metadata(self, request, pk=None):
+        meta = self.get_queryset().values_list("meta__collection_metadata", flat=True).get(pk=pk)
+        return Response(meta or {})
+
+    @action(detail=True)
+    def coverage(self, request, pk=None):
+        map = {
+            "parties": ["parties.id"],
+            "plannings": ["planning.budget"],
+            "tenders": ["tender.id"],
+            "tenderers": ["tenderers.id"],
+            "tenders_items": ["tender.items.id"],
+            "awards": ["awards.id"],
+            "awards_items": ["awards.items.id"],
+            "awards_suppliers": ["awards.suppliers.id"],
+            "contracts": ["contracts.id"],
+            "contracts_items": ["contracts.items.id"],
+            "contracts_transactions": ["contracts.implementation.transactions.id"],
+            "documents": [
+                "planning.documents.id",
+                "tender.documents.id",
+                "awards.documents.id",
+                "contracts.documents.id",
+                "contracts.implementation.documents.id",
+            ],
+            "milestones": [
+                "planning.milestones.id",
+                "tender.milestones.id",
+                "contracts.milestones.id",
+                "contracts.implementation.milestones.id",
+            ],
+            "amendments": ["tender.amendments.id", "awards.amendments.id", "contract.amendments.id"],
+        }
+
+        with connections["data"].cursor() as cursor:
+            statement = """
+                SELECT c.key AS check, SUM(jsonb_array_length(c.value)) AS count
+                FROM {table} flc, jsonb_each(flc.result->'checks') c
+                WHERE dataset_id = %(dataset_id)s
+                    AND c.key IN %(checks)s
+                GROUP BY c.key
+                ORDER BY c.key
+                """
+
+            cursor.execute(
+                SQL(statement).format(table=Identifier(FieldLevelCheck._meta.db_table)),
+                {"checks": tuple(j for i in map.values() for j in i), "dataset_id": pk},
+            )
+
+            results = cursor.fetchall()
+
+            counts = {}
+            for key, items in map.items():
+                counts[key] = 0
+                for i in items:
+                    for r in results:
+                        if r[0] == i:
+                            counts[key] += int(r[1])
+
+        return Response(counts)

backend/requirements.in

@@ -1,6 +1,7 @@
 dj-database-url
 django
 django-cors-headers
+djangorestframework
 # 0.14.3 is not compatible with Django 3.2.
 -e git+https://github.com/django-tastypie/django-tastypie.git@ac18543860a1deb88ed83c6d2eecad02a525419d#egg=django-tastypie
 google-api-python-client

backend/requirements.txt

@@ -24,8 +24,11 @@ django==3.2.8
     # via
     #   -r requirements.in
     #   django-cors-headers
+    #   djangorestframework
 django-cors-headers==3.10.0
     # via -r requirements.in
+djangorestframework==3.12.4
+    # via -r requirements.in
 google-api-core==1.21.0
     # via google-api-python-client
 google-api-python-client==1.9.3

backend/requirements_dev.txt

@@ -51,8 +51,11 @@ django==3.2.8
     # via
     #   -r requirements.txt
     #   django-cors-headers
+    #   djangorestframework
 django-cors-headers==3.10.0
     # via -r requirements.txt
+djangorestframework==3.12.4
+    # via -r requirements.txt
 filelock==3.0.12
     # via virtualenv
 flake8==3.9.1
@@ -128,7 +131,7 @@ pillow==8.3.2
     # via
     #   -r requirements.txt
     #   matplotlib
-pip-tools==6.1.0
+pip-tools==6.4.0
     # via -r requirements_dev.in
 pluggy==0.13.1
     # via pytest
@@ -246,6 +249,8 @@ urllib3==1.26.5
     #   transifex-client
 virtualenv==20.4.6
     # via pre-commit
+wheel==0.37.0
+    # via pip-tools
 
 # The following packages are considered to be unsafe in a requirements file:
 # pip