From 5d7200ef37f763f07924e3e52d6b631e7b984f76 Mon Sep 17 00:00:00 2001 From: Wille Marcel Date: Fri, 29 Aug 2025 23:40:11 -0300 Subject: [PATCH] Add Raster and Tabular data models and views --- requirements.txt | 1 + vbos/config/common.py | 6 +- vbos/datasets/admin.py | 23 ++++- vbos/datasets/filters.py | 37 ++++++++ .../0002_tabulardataset_tabularitem.py | 59 +++++++++++++ .../datasets/migrations/0003_rasterdataset.py | 34 ++++++++ vbos/datasets/models.py | 36 ++++++++ vbos/datasets/serializers.py | 58 ++++++++++++- vbos/datasets/test/test_raster_views.py | 32 +++++++ vbos/datasets/test/test_tabular_views.py | 84 +++++++++++++++++++ .../{test_views.py => test_vector_views.py} | 14 +++- vbos/datasets/urls.py | 20 +++++ vbos/datasets/views.py | 68 ++++++++++++++- 13 files changed, 464 insertions(+), 8 deletions(-) create mode 100644 vbos/datasets/filters.py create mode 100644 vbos/datasets/migrations/0002_tabulardataset_tabularitem.py create mode 100644 vbos/datasets/migrations/0003_rasterdataset.py create mode 100644 vbos/datasets/test/test_raster_views.py create mode 100644 vbos/datasets/test/test_tabular_views.py rename vbos/datasets/test/{test_views.py => test_vector_views.py} (87%) diff --git a/requirements.txt b/requirements.txt index fa62e17..7af577b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,6 +18,7 @@ djangorestframework-gis==1.2.0 django-filter==24.3 drf_spectacular==0.28.0 django-cors-headers==4.7.0 +drf-excel==2.5.3 # Developer Tools ipdb==0.13.13 diff --git a/vbos/config/common.py b/vbos/config/common.py index 3e1bb49..4a45be2 100755 --- a/vbos/config/common.py +++ b/vbos/config/common.py @@ -182,12 +182,16 @@ class Common(Configuration): # Django Rest Framework REST_FRAMEWORK = { "DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.PageNumberPagination", - "PAGE_SIZE": int(os.getenv("DJANGO_PAGINATION_LIMIT", 10)), + "PAGE_SIZE": int(os.getenv("DJANGO_PAGINATION_LIMIT", 20)), "DATETIME_FORMAT": "%Y-%m-%dT%H:%M:%S%z", "DEFAULT_RENDERER_CLASSES": ( "rest_framework.renderers.JSONRenderer", "rest_framework.renderers.BrowsableAPIRenderer", + "drf_excel.renderers.XLSXRenderer", ), + "DEFAULT_FILTER_BACKENDS": [ + "django_filters.rest_framework.DjangoFilterBackend" + ], "DEFAULT_PERMISSION_CLASSES": [ "rest_framework.permissions.IsAuthenticated", ], diff --git a/vbos/datasets/admin.py b/vbos/datasets/admin.py index ef2f4bb..9e898f5 100644 --- a/vbos/datasets/admin.py +++ b/vbos/datasets/admin.py @@ -1,5 +1,16 @@ from django.contrib.gis import admin -from .models import VectorDataset, VectorItem +from .models import ( + RasterDataset, + TabularDataset, + TabularItem, + VectorDataset, + VectorItem, +) + + +@admin.register(RasterDataset) +class RasterDatasetAdmin(admin.ModelAdmin): + list_display = ["id", "name", "created", "updated", "file_path"] @admin.register(VectorDataset) @@ -10,3 +21,13 @@ class VectorDatasetAdmin(admin.ModelAdmin): @admin.register(VectorItem) class VectorItemAdmin(admin.GISModelAdmin): list_display = ["id", "metadata"] + + +@admin.register(TabularDataset) +class TabularDatasetAdmin(admin.ModelAdmin): + list_display = ["id", "name", "created", "updated"] + + +@admin.register(TabularItem) +class TabularItemAdmin(admin.GISModelAdmin): + list_display = ["id", "data"] diff --git a/vbos/datasets/filters.py b/vbos/datasets/filters.py new file mode 100644 index 0000000..419e0ac --- /dev/null +++ b/vbos/datasets/filters.py @@ -0,0 +1,37 @@ +from django_filters import ( + FilterSet, + BooleanFilter, + CharFilter, + OrderingFilter, + DateFromToRangeFilter, + ModelChoiceFilter, +) + +from .models import RasterDataset, VectorDataset, TabularDataset + + +class DatasetFilter(FilterSet): + name = CharFilter(field_name="name", lookup_expr="icontains") + created = DateFromToRangeFilter() + updated = DateFromToRangeFilter() + order_by = OrderingFilter( + fields=("name", "id", "updated", "created"), + ) + + +class RasterDatasetFilter(DatasetFilter): + class Meta: + model = RasterDataset + fields = ["name", "created", "updated"] + + +class VectorDatasetFilter(DatasetFilter): + class Meta: + model = VectorDataset + fields = ["name", "created", "updated"] + + +class TabularDatasetFilter(DatasetFilter): + class Meta: + model = TabularDataset + fields = ["name", "created", "updated"] diff --git a/vbos/datasets/migrations/0002_tabulardataset_tabularitem.py b/vbos/datasets/migrations/0002_tabulardataset_tabularitem.py new file mode 100644 index 0000000..a0a51e8 --- /dev/null +++ b/vbos/datasets/migrations/0002_tabulardataset_tabularitem.py @@ -0,0 +1,59 @@ +# Generated by Django 5.2.5 on 2025-08-29 20:49 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("datasets", "0001_initial"), + ] + + operations = [ + migrations.CreateModel( + name="TabularDataset", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(max_length=155)), + ("created", models.DateTimeField(auto_now_add=True)), + ("updated", models.DateTimeField(auto_now=True)), + ], + options={ + "ordering": ["id"], + }, + ), + migrations.CreateModel( + name="TabularItem", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("data", models.JSONField(default=dict)), + ( + "dataset", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="datasets.tabulardataset", + ), + ), + ], + options={ + "ordering": ["id"], + }, + ), + ] diff --git a/vbos/datasets/migrations/0003_rasterdataset.py b/vbos/datasets/migrations/0003_rasterdataset.py new file mode 100644 index 0000000..9858d3c --- /dev/null +++ b/vbos/datasets/migrations/0003_rasterdataset.py @@ -0,0 +1,34 @@ +# Generated by Django 5.2.5 on 2025-08-29 21:22 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("datasets", "0002_tabulardataset_tabularitem"), + ] + + operations = [ + migrations.CreateModel( + name="RasterDataset", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(max_length=155)), + ("created", models.DateTimeField(auto_now_add=True)), + ("updated", models.DateTimeField(auto_now=True)), + ("file_path", models.CharField(max_length=2000)), + ], + options={ + "ordering": ["id"], + }, + ), + ] diff --git a/vbos/datasets/models.py b/vbos/datasets/models.py index 81aecad..d512c0f 100644 --- a/vbos/datasets/models.py +++ b/vbos/datasets/models.py @@ -1,6 +1,19 @@ from django.contrib.gis.db import models +class RasterDataset(models.Model): + name = models.CharField(max_length=155) + created = models.DateTimeField(auto_now_add=True) + updated = models.DateTimeField(auto_now=True) + file_path = models.CharField(max_length=2000, null=False, blank=False) + + def __str__(self): + return self.name + + class Meta: + ordering = ["id"] + + class VectorDataset(models.Model): name = models.CharField(max_length=155) created = models.DateTimeField(auto_now_add=True) @@ -23,3 +36,26 @@ def __str__(self): class Meta: ordering = ["id"] + + +class TabularDataset(models.Model): + name = models.CharField(max_length=155) + created = models.DateTimeField(auto_now_add=True) + updated = models.DateTimeField(auto_now=True) + + def __str__(self): + return self.name + + class Meta: + ordering = ["id"] + + +class TabularItem(models.Model): + dataset = models.ForeignKey(TabularDataset, on_delete=models.CASCADE) + data = models.JSONField(default=dict) + + def __str__(self): + return f"{self.id}" + + class Meta: + ordering = ["id"] diff --git a/vbos/datasets/serializers.py b/vbos/datasets/serializers.py index 2409c7d..36d9802 100644 --- a/vbos/datasets/serializers.py +++ b/vbos/datasets/serializers.py @@ -1,7 +1,19 @@ from rest_framework import serializers from rest_framework_gis.serializers import GeoFeatureModelSerializer -from .models import VectorDataset, VectorItem +from .models import ( + RasterDataset, + TabularDataset, + TabularItem, + VectorDataset, + VectorItem, +) + + +class RasterDatasetSerializer(serializers.ModelSerializer): + class Meta: + model = RasterDataset + fields = "__all__" class VectorDatasetSerializer(serializers.ModelSerializer): @@ -30,3 +42,47 @@ def unformat_geojson(self, feature): attrs[self.Meta.bbox_geo_field] = Polygon.from_bbox(feature["bbox"]) return attrs + + +class TabularDatasetSerializer(serializers.ModelSerializer): + class Meta: + model = TabularDataset + fields = "__all__" + + +class TabularItemSerializer(serializers.ModelSerializer): + class Meta: + model = TabularItem + fields = ["id", "data"] + + def to_representation(self, instance): + representation = super().to_representation(instance) + + # Extract the data field and merge it with the top level fields + data_content = representation.pop("data", {}) + + return {**representation, **data_content} + + +class TabularItemExcelSerializer(serializers.ModelSerializer): + # Dynamically add fields based on all possible keys in the data + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # Get all possible keys from the queryset + if self.context.get("view"): + queryset = self.context["view"].get_queryset() + all_keys = set() + for item in queryset: + if item.data and isinstance(item.data, dict): + all_keys.update(item.data.keys()) + + # Create a field for each key + for key in all_keys: + self.fields[key] = serializers.CharField( + source=f"data.{key}", required=False, allow_blank=True, default="" + ) + + class Meta: + model = TabularItem + fields = ["id"] diff --git a/vbos/datasets/test/test_raster_views.py b/vbos/datasets/test/test_raster_views.py new file mode 100644 index 0000000..a7d0ad8 --- /dev/null +++ b/vbos/datasets/test/test_raster_views.py @@ -0,0 +1,32 @@ +from rest_framework import status +from rest_framework.test import APITestCase +from django.urls import reverse + +from ..models import RasterDataset + + +class TestRasterDatasetListDetailViews(APITestCase): + def setUp(self): + self.dataset_1 = RasterDataset.objects.create( + name="Rainfall", file_path="cogs/rainfall.tiff" + ) + self.dataset_2 = RasterDataset.objects.create( + name="Coastline changes", file_path="cogs/coastlines.tiff" + ) + self.url = reverse("datasets:raster-list") + + def test_raster_datasets_list(self): + req = self.client.get(self.url) + assert req.status_code == status.HTTP_200_OK + assert req.data.get("count") == 2 + assert req.data.get("results")[0]["name"] == "Rainfall" + assert req.data.get("results")[1]["name"] == "Coastline changes" + + def test_raster_datasets_detail(self): + url = reverse("datasets:raster-detail", args=[self.dataset_1.id]) + req = self.client.get(url) + assert req.status_code == status.HTTP_200_OK + assert req.data.get("name") == "Rainfall" + assert req.data.get("file_path") == "cogs/rainfall.tiff" + assert req.data.get("created") + assert req.data.get("updated") diff --git a/vbos/datasets/test/test_tabular_views.py b/vbos/datasets/test/test_tabular_views.py new file mode 100644 index 0000000..609de65 --- /dev/null +++ b/vbos/datasets/test/test_tabular_views.py @@ -0,0 +1,84 @@ +from rest_framework import status +from rest_framework.test import APITestCase +from django.urls import reverse + +from ..models import TabularDataset, TabularItem +from ...users.test.factories import UserFactory + + +class TestTabularDatasetListDetailViews(APITestCase): + def setUp(self): + self.user = UserFactory() + self.dataset_1 = TabularDataset.objects.create(name="Population") + self.dataset_2 = TabularDataset.objects.create(name="Prices") + self.url = reverse("datasets:tabular-list") + + def test_tabular_datasets_list(self): + req = self.client.get(self.url) + assert req.status_code == status.HTTP_200_OK + assert req.data.get("count") == 2 + assert req.data.get("results")[0]["name"] == "Population" + assert req.data.get("results")[1]["name"] == "Prices" + + def test_tabular_datasets_detail(self): + url = reverse("datasets:tabular-detail", args=[self.dataset_1.id]) + req = self.client.get(url) + assert req.status_code == status.HTTP_200_OK + assert req.data.get("name") == "Population" + assert req.data.get("created") + assert req.data.get("updated") + + +class TestTabularDatasetDataView(APITestCase): + def setUp(self): + self.user = UserFactory() + self.dataset_1 = TabularDataset.objects.create(name="Population") + self.dataset_2 = TabularDataset.objects.create(name="Employment") + self.item = TabularItem.objects.create( + dataset=self.dataset_1, + data={"province": "A", "population": 1902, "year": 2025}, + ) + TabularItem.objects.create( + dataset=self.dataset_1, + data={"province": "B", "population": 10902, "year": 2025}, + ) + TabularItem.objects.create( + dataset=self.dataset_1, + data={"province": "C", "population": 875, "year": 2025}, + ) + TabularItem.objects.create( + dataset=self.dataset_2, + data={"employed_population": 0.75, "year": 2025, "month": 1}, + ) + TabularItem.objects.create( + dataset=self.dataset_2, + data={"employed_population": 0.85, "year": 2024, "month": 7}, + ) + TabularItem.objects.create( + dataset=self.dataset_2, + data={"employed_population": 0.82, "year": 2024, "month": 1}, + ) + TabularItem.objects.create( + dataset=self.dataset_2, + data={"employed_population": 0.80, "year": 2023, "month": 7}, + ) + self.url = reverse("datasets:tabular-data", args=[self.dataset_1.id]) + + def test_tabular_datasets_data(self): + req = self.client.get(self.url) + assert req.status_code == status.HTTP_200_OK + assert req.data.get("count") == 3 + assert len(req.data.get("results")) == 3 + assert req.data.get("results")[0]["province"] == "A" + assert req.data.get("results")[0]["population"] == 1902 + assert req.data.get("results")[0]["year"] == 2025 + + # fetch second dataset's data + url = reverse("datasets:tabular-data", args=[self.dataset_2.id]) + req = self.client.get(url) + assert req.status_code == status.HTTP_200_OK + assert req.data.get("count") == 4 + assert len(req.data.get("results")) == 4 + assert req.data.get("results")[0]["employed_population"] == 0.75 + assert req.data.get("results")[0]["month"] == 1 + assert req.data.get("results")[0]["year"] == 2025 diff --git a/vbos/datasets/test/test_views.py b/vbos/datasets/test/test_vector_views.py similarity index 87% rename from vbos/datasets/test/test_views.py rename to vbos/datasets/test/test_vector_views.py index 0e8bd68..dcf6b71 100644 --- a/vbos/datasets/test/test_views.py +++ b/vbos/datasets/test/test_vector_views.py @@ -4,12 +4,10 @@ from django.contrib.gis.geos import Polygon, LineString, Point from ..models import VectorDataset, VectorItem -from ...users.test.factories import UserFactory class TestVectorDatasetListDetailViews(APITestCase): def setUp(self): - self.user = UserFactory() self.dataset_1 = VectorDataset.objects.create(name="Boundaries") self.dataset_2 = VectorDataset.objects.create(name="Roads") self.url = reverse("datasets:vector-list") @@ -32,7 +30,6 @@ def test_vector_datasets_detail(self): class TestVectorDatasetDataView(APITestCase): def setUp(self): - self.user = UserFactory() self.dataset_1 = VectorDataset.objects.create(name="Boundaries") self.dataset_2 = VectorDataset.objects.create(name="Roads") VectorItem.objects.create( @@ -76,3 +73,14 @@ def test_vector_datasets_data(self): [[0.0, 0.0], [0.0, 3.0], [3.0, 3.0], [3.0, 0.0], [0.0, 0.0]] ], } + + def test_filters(self): + req = self.client.get(self.url, {"in_bbox": "80,10,81,11"}) + assert req.status_code == status.HTTP_200_OK + assert req.data.get("count") == 1 + assert len(req.data.get("features")) == 1 + assert req.data.get("features")[0]["geometry"] == { + "type": "Point", + "coordinates": [80.5, 10.232], + } + assert req.data.get("features")[0]["properties"]["name"] == "Point 1" diff --git a/vbos/datasets/urls.py b/vbos/datasets/urls.py index 4d0d9c5..aac8722 100644 --- a/vbos/datasets/urls.py +++ b/vbos/datasets/urls.py @@ -5,6 +5,14 @@ app_name = "datasets" urlpatterns = [ + # raster + path("raster/", views.RasterDatasetListView.as_view(), name="raster-list"), + path( + "raster//", + views.RasterDatasetDetailView.as_view(), + name="raster-detail", + ), + # vector path("vector/", views.VectorDatasetListView.as_view(), name="vector-list"), path( "vector//", @@ -16,4 +24,16 @@ views.VectorDatasetDataView.as_view(), name="vector-data", ), + # tabular + path("tabular/", views.TabularDatasetListView.as_view(), name="tabular-list"), + path( + "tabular//", + views.TabularDatasetDetailView.as_view(), + name="tabular-detail", + ), + path( + "tabular//data/", + views.TabularDatasetDataView.as_view(), + name="tabular-data", + ), ] diff --git a/vbos/datasets/views.py b/vbos/datasets/views.py index c54d740..b99bd5e 100644 --- a/vbos/datasets/views.py +++ b/vbos/datasets/views.py @@ -2,10 +2,44 @@ from rest_framework.generics import ListAPIView, RetrieveAPIView from rest_framework.permissions import IsAuthenticatedOrReadOnly from rest_framework_gis.pagination import GeoJsonPagination +from rest_framework_gis.filters import InBBoxFilter -from .models import VectorDataset, VectorItem +from vbos.datasets.filters import ( + RasterDatasetFilter, + TabularDatasetFilter, + VectorDatasetFilter, +) + +from .models import ( + RasterDataset, + TabularDataset, + TabularItem, + VectorDataset, + VectorItem, +) from .pagination import StandardResultsSetPagination -from .serializers import VectorDatasetSerializer, VectorItemSerializer +from .serializers import ( + RasterDatasetSerializer, + TabularDatasetSerializer, + TabularItemExcelSerializer, + TabularItemSerializer, + VectorDatasetSerializer, + VectorItemSerializer, +) + + +class RasterDatasetListView(ListAPIView): + queryset = RasterDataset.objects.all() + serializer_class = RasterDatasetSerializer + permission_classes = [IsAuthenticatedOrReadOnly] + pagination_class = StandardResultsSetPagination + filterset_class = RasterDatasetFilter + + +class RasterDatasetDetailView(RetrieveAPIView): + queryset = RasterDataset.objects.all() + serializer_class = RasterDatasetSerializer + permission_classes = [IsAuthenticatedOrReadOnly] class VectorDatasetListView(ListAPIView): @@ -13,6 +47,7 @@ class VectorDatasetListView(ListAPIView): serializer_class = VectorDatasetSerializer permission_classes = [IsAuthenticatedOrReadOnly] pagination_class = StandardResultsSetPagination + filterset_class = VectorDatasetFilter class VectorDatasetDetailView(RetrieveAPIView): @@ -25,6 +60,35 @@ class VectorDatasetDataView(ListAPIView): serializer_class = VectorItemSerializer permission_classes = [IsAuthenticatedOrReadOnly] pagination_class = GeoJsonPagination + bbox_filter_field = "geometry" + filter_backends = (InBBoxFilter,) def get_queryset(self): return VectorItem.objects.filter(dataset=self.kwargs.get("pk")) + + +class TabularDatasetListView(ListAPIView): + queryset = TabularDataset.objects.all() + serializer_class = TabularDatasetSerializer + permission_classes = [IsAuthenticatedOrReadOnly] + pagination_class = StandardResultsSetPagination + filterset_class = TabularDatasetFilter + + +class TabularDatasetDetailView(RetrieveAPIView): + queryset = TabularDataset.objects.all() + serializer_class = TabularDatasetSerializer + permission_classes = [IsAuthenticatedOrReadOnly] + + +class TabularDatasetDataView(ListAPIView): + permission_classes = [IsAuthenticatedOrReadOnly] + + def get_queryset(self): + return TabularItem.objects.filter(dataset=self.kwargs.get("pk")) + + def get_serializer_class(self): + # Use different serializer for Excel format + if self.request.query_params.get("format") == "xlsx": + return TabularItemExcelSerializer + return TabularItemSerializer