feat: implement google sheet import for stock verification receipts

config/api_router.py

@@ -2,6 +2,7 @@
 from rest_framework.routers import DefaultRouter, SimpleRouter
 
 from fahari.common.views import FacilityViewSet, SystemViewSet, UserFacilityViewSet
+from fahari.misc.views import StockVerificationReceiptsAdapterView
 from fahari.ops.views import (
     ActivityLogViewSet,
     CommodityViewSet,
@@ -47,6 +48,7 @@
 router.register("facility_devices", FacilityDeviceViewSet)
 router.register("facility_device_requests", FacilityDeviceRequestViewSet)
 router.register("security_incidents", SecurityIncidenceViewSet)
+router.register("stock_receipts_adapters", StockVerificationReceiptsAdapterView)
 
 app_name = "api"
 urlpatterns = router.urls

config/asgi.py

@@ -11,8 +11,13 @@
 import sys
 from pathlib import Path
 
+from channels.auth import AuthMiddlewareStack
+from channels.routing import ProtocolTypeRouter, URLRouter
+from django.conf.urls import re_path  # type: ignore
 from django.core.asgi import get_asgi_application
 
+from fahari.misc.consumers import StockVerificationReceiptsAdapterConsumer
+
 # This allows easy placement of apps within the interior
 # fahari directory.
 ROOT_DIR = Path(__file__).resolve(strict=True).parent.parent
@@ -21,20 +26,22 @@
 # If DJANGO_SETTINGS_MODULE is unset, default to the local settings
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local")
 
-# This application object is used by any ASGI server configured to use this file.
-django_application = get_asgi_application()
-# Apply ASGI middleware here.
-# from helloworld.asgi import HelloWorldApplication
-# application = HelloWorldApplication(application)
-
-# Import websocket application here, so apps from django_application are loaded first
-from config.websocket import websocket_application  # noqa isort:skip
-
-
-async def application(scope, receive, send):
-    if scope["type"] == "http":
-        await django_application(scope, receive, send)
-    elif scope["type"] == "websocket":
-        await websocket_application(scope, receive, send)
-    else:
-        raise NotImplementedError(f"Unknown scope type {scope['type']}")
+django_asgi_app = get_asgi_application()
+
+application = ProtocolTypeRouter(
+    {
+        # Django's ASGI application to handle traditional HTTP requests
+        "http": django_asgi_app,
+        # WebSocket router
+        "websocket": AuthMiddlewareStack(
+            URLRouter(
+                [
+                    re_path(
+                        r"^ws/misc/stock_receipts_verification_ingest/$",
+                        StockVerificationReceiptsAdapterConsumer.as_asgi(),
+                    ),
+                ]
+            )
+        ),
+    }
+)

config/settings/base.py

@@ -52,6 +52,7 @@
 # URLS
 # ------------------------------------------------------------------------------
 ROOT_URLCONF = "config.urls"
+
 WSGI_APPLICATION = "config.wsgi.application"
 
 # APPS
@@ -69,6 +70,7 @@
     "django.contrib.postgres",
 ]
 THIRD_PARTY_APPS = [
+    # "channels",
     "crispy_forms",
     "allauth",
     "allauth.account",
@@ -88,6 +90,7 @@
     "fahari.common.apps.CommonConfig",
     "fahari.ops.apps.OpsConfig",
     "fahari.mle.apps.MLEConfig",
+    "fahari.misc.apps.MiscConfig",
 ]
 INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS
 

config/settings/test.py

@@ -55,7 +55,7 @@ def gen_func():
 # STORAGES
 # ------------------------------------------------------------------------------
 # https://django-storages.readthedocs.io/en/latest/#installation
-INSTALLED_APPS += ["storages"]  # noqa F405
+INSTALLED_APPS += ["storages", "channels"]  # noqa F405
 GS_BUCKET_NAME = env("DJANGO_GCP_STORAGE_BUCKET_NAME", default="fahari-ya-jamii-test")
 GS_DEFAULT_ACL = "project-private"
 # STATIC

data/kajiodo_svr_sheet_to_db_mappings_metadata.json

@@ -0,0 +1,49 @@
+{
+	"batch_number": {
+		"column_index": 9
+	},
+	"comments": {
+		"column_index": 11,
+        "default_value": "-",
+        "optional": true
+	},
+    "commodity": {
+    	"column_index": 4,
+    	"value_mappings": {
+    		"Tenofovir/Lamivudine/Dolutegravir TLD": "Tenofovir/Lamivudine/Dolutegravir (TDF/3TC/DTG) FDC (300/300/50mg) FDC Tablets",
+    		"Dolutegravir 50mg": "Dolutegravir(DTG) 50mg tabs",
+    		"Zidovudine/Lamivudine AZT/3tC": "Zidovudine/Lamivudine (AZT/3TC) FDC (300/150mg) Tablets"
+    	},
+        "lookup": "name"
+    },
+    "delivery_date": {
+    	"column_index": 3,
+    	"datetime_format": "%m/%d/%Y"
+    },
+    "delivery_note_number": {
+    	"column_index": 6
+    },
+    "delivery_note_quantity": {
+    	"column_index": 7
+    },
+    "expiry_date": {
+    	"column_index": 10,
+    	"datetime_format": "%m/%d/%Y"
+    },
+    "facility": {
+    	"column_index": 2,
+    	"lookup": "mfl_code"
+    },
+    "pack_size": {
+    	"column_index": 5,
+    	"value_mappings": {
+    		"30": "Packs of 30s",
+    		"60": "Packs of 60s",
+    		"90": "Packs of 90s"
+    	},
+        "lookup": "name"
+    },
+    "quantity_received": {
+    	"column_index": 8
+    }
+}

data/nairobi_svr_sheet_to_db_mappings_metadata.json

@@ -0,0 +1,49 @@
+{
+	"batch_number": {
+		"column_index": 10
+	},
+	"comments": {
+		"column_index": 12,
+        "default_value": "-",
+        "optional": true
+	},
+    "commodity": {
+    	"column_index": 6,
+    	"value_mappings": {
+    		"Tenofovir/Lamivudine/Dolutegravir TLD": "Tenofovir/Lamivudine/Dolutegravir (TDF/3TC/DTG) FDC (300/300/50mg) FDC Tablets",
+    		"Dolutegravir 50mg": "Dolutegravir(DTG) 50mg tabs",
+    		"Zidovudine/Lamivudine AZT/3tC": "Zidovudine/Lamivudine (AZT/3TC) FDC (300/150mg) Tablets"
+    	},
+        "lookup": "name"
+    },
+    "delivery_date": {
+    	"column_index": 5,
+    	"datetime_format": "%m/%d/%Y"
+    },
+    "delivery_note_number": {
+    	"column_index": 4
+    },
+    "delivery_note_quantity": {
+    	"column_index": 8
+    },
+    "expiry_date": {
+    	"column_index": 11,
+    	"datetime_format": "%m/%d/%Y"
+    },
+    "facility": {
+    	"column_index": 3,
+    	"lookup": "mfl_code"
+    },
+    "pack_size": {
+    	"column_index": 7,
+    	"value_mappings": {
+    		"30": "Packs of 30s",
+    		"60": "Packs of 60s",
+    		"90": "Packs of 90s"
+    	},
+        "lookup": "name"
+    },
+    "quantity_received": {
+    	"column_index": 9
+    }
+}

fahari/common/admin.py

@@ -1,10 +1,12 @@
+from typing import Tuple
+
 from django.contrib import admin
 
 from .models import Facility, FacilityAttachment, Organisation, System
 
 
 class BaseAdmin(admin.ModelAdmin):
-    readonly_fields = (
+    readonly_fields: Tuple[str, ...] = (
         "created",
         "created_by",
         "updated",

fahari/common/permissions.py

@@ -0,0 +1,25 @@
+from rest_framework.permissions import BasePermission
+from rest_framework.request import Request
+from rest_framework.views import APIView
+
+
+class CanExportData(BasePermission):
+    """Allows access only to users who have the export data permission."""
+
+    def has_permission(self, request: Request, view: APIView) -> bool:
+        return (
+            request.user
+            and request.user.is_authenticated
+            and request.user.has_perm("can_export_data")
+        )
+
+
+class CanImportData(BasePermission):
+    """Allow access only to users who have the import data permission."""
+
+    def has_permission(self, request: Request, view: APIView) -> bool:
+        return (
+            request.user
+            and request.user.is_authenticated
+            and request.user.has_perm("can_import_data")
+        )

fahari/common/views/mixins/drf_mixins.py

@@ -5,13 +5,15 @@
 from openpyxl import Workbook
 from rest_framework import status
 from rest_framework.decorators import action
+from rest_framework.permissions import DjangoModelPermissions
 from rest_framework.relations import ManyRelatedField, PrimaryKeyRelatedField
 from rest_framework.renderers import StaticHTMLRenderer
 from rest_framework.request import Request
 from rest_framework.response import Response
 from rest_framework.serializers import Serializer
 from rest_framework.viewsets import GenericViewSet
 
+from fahari.common.permissions import CanExportData
 from fahari.common.renderers import ExcelIORenderer
 from fahari.utils.excel_utils import (
     AuditSerializerExcelIO,
@@ -32,8 +34,13 @@ class ExcelIOMixin(Generic[EIO, EIO_T], GenericViewSet):
     excel_io_template_class: Optional[Type[EIO_T]] = None
     filename: Optional[str] = None
 
-    @action(detail=False, methods=["GET"], renderer_classes=(ExcelIORenderer,))
-    def dump_data(self, request: Request, pk=None):
+    @action(
+        detail=False,
+        methods=["GET"],
+        permission_classes=(DjangoModelPermissions, CanExportData),
+        renderer_classes=(ExcelIORenderer,),
+    )
+    def dump_data(self, request: Request, pk=None) -> Response:
         """Export data in excel format."""
 
         workbook = self.perform_dump_data(request)

fahari/misc/admin.py

@@ -0,0 +1,16 @@
+from django.contrib import admin
+
+from fahari.common.admin import BaseAdmin
+
+from .models import SheetToDBMappingsMetadata, StockVerificationReceiptsAdapter
+
+
+@admin.register(SheetToDBMappingsMetadata)
+class StockToDBMappingsMetadata(BaseAdmin):
+    list_display = ("name", "version")
+
+
+@admin.register(StockVerificationReceiptsAdapter)
+class StockVerificationReceiptsAdapterAdmin(BaseAdmin):
+    list_display = ("county", "position")
+    readonly_fields = BaseAdmin.readonly_fields + ("target_model",)

fahari/misc/apps.py

@@ -0,0 +1,7 @@
+from django.apps import AppConfig
+from django.utils.translation import gettext_lazy as _
+
+
+class MiscConfig(AppConfig):
+    name = "fahari.misc"
+    verbose_name = _("Miscellaneous")

fahari/misc/consumers.py

@@ -0,0 +1,85 @@
+from typing import Any, Callable, Dict, Generic, Optional, TypeVar
+
+from channels.generic.websocket import JsonWebsocketConsumer
+
+from .exceptions import ProcessGoogleSheetRowError
+from .models import AbstractGoogleSheetToDjangoModelAdapter, StockVerificationReceiptsAdapter
+
+A = TypeVar("A", bound=AbstractGoogleSheetToDjangoModelAdapter, covariant=True)
+ProgressCallback = Callable[[int, int, float], None]
+
+
+class AbstractGoogleSheetToDjangoModelAdapterConsumer(Generic[A], JsonWebsocketConsumer):
+    """Read, process and persist data from a Google Sheets spreadsheet."""
+
+    def connect(self) -> None:
+        self.user = self.scope["user"]  # noqa
+        # Ensure that the user has permissions to import data
+        if not self.user.has_perm("can_import_data"):
+            self.close(code=1008)
+        self.accept()
+
+    def receive_json(self, content: Any, **kwargs) -> None:
+        adapter = self.get_adapter(content)
+        try:
+            ingested_rows = adapter.ingest_from_last_position(
+                progress_callback=self.get_progress_callback(), **self.get_adapter_context()
+            )
+        except ProcessGoogleSheetRowError as exp:
+            error_details = {"row_index": exp.row_index}
+            error_messages = []
+            while exp is not None:
+                error_messages.append(str(exp))
+                exp = exp.__cause__  # type: ignore
+            error_details["error_messages"] = error_messages
+            self.send_data("error", error_details)
+            return
+
+        self.send_data("success", {"ingested_rows": ingested_rows})  # noqa
+
+    def send_data(self, data_type: str, data_content: Any) -> None:
+        """Send the given data of to the party on the other end of this connection."""
+
+        self.send_json({"type": data_type, "data": data_content})
+
+    def get_adapter(self, content: Any) -> A:
+        """Return an adapter instance given input."""
+
+        raise NotImplementedError("`get_adapter` must be implemented.")
+
+    def get_adapter_context(self) -> Dict[str, Any]:
+        """Return extra context to be injected in an adapter's `ingest_from_last_position` method.
+
+        Subclasses can override this method to add or change the returned
+        context dict.
+        """
+
+        return {
+            "extra_kwargs": {
+                "created_by": self.user.pk,
+                "organisation": self.user.organisation,  # noqa
+                "updated_by": self.user.pk,
+            }
+        }
+
+    def get_progress_callback(self) -> Optional[ProgressCallback]:
+        """Optionally return a function that acts a progress callback."""
+
+        def simple_progress_callback(current: int, total: int, percentage: float) -> None:
+            """Send progress data through a web-socket to the client on the other end."""
+
+            self.send_data(
+                "progress", {"current": current, "total": total, "percentage": percentage}
+            )
+
+        return simple_progress_callback
+
+
+class StockVerificationReceiptsAdapterConsumer(
+    AbstractGoogleSheetToDjangoModelAdapterConsumer[StockVerificationReceiptsAdapter]
+):
+    """StockVerificationReceiptsAdapter consumer."""
+
+    def get_adapter(self, content: Any) -> A:
+        adapter: str = content["adapter"]
+        return StockVerificationReceiptsAdapter.objects.active().get(pk=adapter)

fahari/misc/exceptions.py

@@ -0,0 +1,10 @@
+from typing import Sequence
+
+
+class ProcessGoogleSheetRowError(RuntimeError):
+    """Raised when processing a google sheet row fails."""
+
+    def __init__(self, row: Sequence[str], row_index: int, *args, **kwargs):
+        self.row: Sequence[str] = tuple(row)
+        self.row_index: int = row_index
+        super().__init__(*args, **kwargs)  # type: ignore