diff --git a/vbos/datasets/admin.py b/vbos/datasets/admin.py index 177b01f..a816b8f 100644 --- a/vbos/datasets/admin.py +++ b/vbos/datasets/admin.py @@ -8,8 +8,12 @@ from django.shortcuts import render, redirect, reverse from django.urls import path +from vbos.datasets.utils import CSVRow, GeoJSONProperties + from .models import ( + AreaCouncil, Cluster, + Province, RasterDataset, RasterFile, TabularDataset, @@ -44,8 +48,8 @@ class VectorDatasetAdmin(admin.ModelAdmin): @admin.register(VectorItem) class VectorItemAdmin(admin.GISModelAdmin): - list_display = ["id", "dataset", "metadata"] - list_filter = ["dataset"] + list_display = ["id", "dataset", "name", "attribute", "province", "area_council"] + list_filter = ["dataset", "province", "area_council"] def get_urls(self): urls = super().get_urls() @@ -82,10 +86,20 @@ def import_file(self, request): error_count = 0 for item in geojson_content["features"]: + metadata = GeoJSONProperties(item["properties"]) try: VectorItem.objects.create( dataset=form.cleaned_data["dataset"], - metadata=item["properties"], + metadata=metadata.properties, + name=metadata.name, + ref=metadata.ref, + attribute=metadata.attribute, + province=Province.objects.filter( + name__iexact=metadata.province + ).first(), + area_council=AreaCouncil.objects.filter( + name__iexact=metadata.area_council + ).first(), geometry=GEOSGeometry(json.dumps(item["geometry"])), ) created_count += 1 @@ -126,8 +140,8 @@ class TabularDatasetAdmin(admin.ModelAdmin): @admin.register(TabularItem) class TabularItemAdmin(admin.GISModelAdmin): - list_display = ["id", "dataset", "data"] - list_filter = ["dataset"] + list_display = ["id", "dataset", "province", "area_council", "attribute", "value"] + list_filter = ["dataset", "province", "area_council"] def get_urls(self): urls = super().get_urls() @@ -166,8 +180,19 @@ def import_file(self, request): for row in reader: # start=2 to account for header row try: + csv_row = CSVRow(row) TabularItem.objects.create( - dataset=form.cleaned_data["dataset"], data=row + dataset=form.cleaned_data["dataset"], + metadata=csv_row.metadata, + attribute=csv_row.attribute, + value=csv_row.value, + date=csv_row.date, + province=Province.objects.filter( + name__iexact=csv_row.province + ).first(), + area_council=AreaCouncil.objects.filter( + name__iexact=csv_row.area_council + ).first(), ) created_count += 1 except Exception as e: diff --git a/vbos/datasets/filters.py b/vbos/datasets/filters.py index 9dc83d1..f7c3cd2 100644 --- a/vbos/datasets/filters.py +++ b/vbos/datasets/filters.py @@ -7,6 +7,8 @@ ) from .models import ( + AreaCouncil, + Province, RasterDataset, TabularItem, VectorDataset, @@ -50,9 +52,20 @@ class Meta: fields = ["name", "source", "cluster", "created", "updated"] -class TabularItemFilter(FilterSet): - filter = CharFilter( - field_name="data", +class DataItemsBaseFilter(FilterSet): + attribute = CharFilter(lookup_expr="icontains") + province = ModelChoiceFilter( + field_name="province__name", + to_field_name="name__iexact", + queryset=Province.objects.all(), + ) + area_council = ModelChoiceFilter( + field_name="area_council__name", + to_field_name="name__iexact", + queryset=AreaCouncil.objects.all(), + ) + metadata = CharFilter( + field_name="metadata", method="filter_metadata", help_text="""Filter by the content of the data JSONField.""", ) @@ -91,18 +104,27 @@ def filter_metadata(self, queryset, name, value): return queryset + +class TabularItemFilter(DataItemsBaseFilter): + date = DateFromToRangeFilter() + class Meta: model = TabularItem - fields = ["filter", "id"] + fields = ["metadata", "attribute", "province", "area_council", "id", "date"] -class VectorItemFilter(TabularItemFilter): - filter = CharFilter( - field_name="metadata", - method="filter_metadata", - help_text="""Filter by the content of the metadata JSONField.""", - ) +class VectorItemFilter(DataItemsBaseFilter): + name = CharFilter(lookup_expr="icontains") + ref = CharFilter(lookup_expr="icontains") class Meta: model = VectorItem - fields = ["filter", "id"] + fields = [ + "metadata", + "attribute", + "province", + "area_council", + "id", + "name", + "ref", + ] diff --git a/vbos/datasets/fixtures/test.csv b/vbos/datasets/fixtures/test.csv new file mode 100644 index 0000000..7abc801 --- /dev/null +++ b/vbos/datasets/fixtures/test.csv @@ -0,0 +1,4 @@ +Year,Month,Attribute,Province,Area Council,Value,Other +2024,January,ecce,TAFEA,Futuna,1154,yes +2022,may,secondary,TAFEA,Futuna,1154,no +2025,,primary,,,1154 diff --git a/vbos/datasets/fixtures/test.geojson b/vbos/datasets/fixtures/test.geojson index 59aef6b..9c24bf9 100644 --- a/vbos/datasets/fixtures/test.geojson +++ b/vbos/datasets/fixtures/test.geojson @@ -4,7 +4,12 @@ { "type": "Feature", "properties": { - "name": "Area 1" + "name": "Area 1", + "ref": "12NC", + "Province": "Torba", + "Area Council": "East Gaua", + "Attribute": "Schools", + "extra": "value" }, "geometry": { "coordinates": [ @@ -36,7 +41,13 @@ }, { "type": "Feature", - "properties": {"name": "Line 1", "col": "val"}, + "properties": { + "name": "Line 1", + "ref": "13NC", + "Province": "TAFEA", + "Area Council": "Futuna", + "Attribute": "Roads" + }, "geometry": { "coordinates": [ [ @@ -53,7 +64,14 @@ }, { "type": "Feature", - "properties": {"name": "Point 1", "source": "OpenStreetMap"}, + "properties": { + "name": "Point 2", + "ref": "14NC", + "Province": "TAFEA", + "Area Council": "Futuna", + "Attribute": "Business", + "source": "OpenStreetMap" + }, "geometry": { "coordinates": [ 167.72470, diff --git a/vbos/datasets/migrations/0011_rename_data_tabularitem_metadata_and_more.py b/vbos/datasets/migrations/0011_rename_data_tabularitem_metadata_and_more.py new file mode 100644 index 0000000..2008149 --- /dev/null +++ b/vbos/datasets/migrations/0011_rename_data_tabularitem_metadata_and_more.py @@ -0,0 +1,85 @@ +# Generated by Django 5.2.5 on 2025-10-03 12:44 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("datasets", "0010_auto_20251001_1132"), + ] + + operations = [ + migrations.RenameField( + model_name="tabularitem", + old_name="data", + new_name="metadata", + ), + migrations.AddField( + model_name="tabularitem", + name="area_council", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.PROTECT, + to="datasets.areacouncil", + ), + ), + migrations.AddField( + model_name="tabularitem", + name="attribute", + field=models.CharField(blank=True, max_length=155, null=True), + ), + migrations.AddField( + model_name="tabularitem", + name="date", + field=models.DateField(null=True), + ), + migrations.AddField( + model_name="tabularitem", + name="province", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.PROTECT, + to="datasets.province", + ), + ), + migrations.AddField( + model_name="tabularitem", + name="value", + field=models.FloatField(default=0), + ), + migrations.AddField( + model_name="vectoritem", + name="area_council", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.PROTECT, + to="datasets.areacouncil", + ), + ), + migrations.AddField( + model_name="vectoritem", + name="attribute", + field=models.CharField(blank=True, max_length=155, null=True), + ), + migrations.AddField( + model_name="vectoritem", + name="name", + field=models.CharField(blank=True, max_length=155, null=True), + ), + migrations.AddField( + model_name="vectoritem", + name="province", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.PROTECT, + to="datasets.province", + ), + ), + migrations.AddField( + model_name="vectoritem", + name="ref", + field=models.CharField(blank=True, max_length=50, null=True), + ), + ] diff --git a/vbos/datasets/models.py b/vbos/datasets/models.py index 3d4d2c0..c143bea 100644 --- a/vbos/datasets/models.py +++ b/vbos/datasets/models.py @@ -119,11 +119,19 @@ class Meta: class VectorItem(models.Model): dataset = models.ForeignKey(VectorDataset, on_delete=models.CASCADE) + name = models.CharField(max_length=155, blank=True, null=True) + ref = models.CharField(max_length=50, blank=True, null=True) + attribute = models.CharField(max_length=155, blank=True, null=True) + province = models.ForeignKey(Province, null=True, on_delete=models.PROTECT) + area_council = models.ForeignKey(AreaCouncil, null=True, on_delete=models.PROTECT) geometry = models.GeometryField() metadata = models.JSONField(default=dict, blank=True, null=True) def __str__(self): - return f"{self.id}" + if self.name: + return f"{self.id} ({self.name})" + else: + return f"{self.id}" class Meta: ordering = ["id"] @@ -149,7 +157,12 @@ class Meta: class TabularItem(models.Model): dataset = models.ForeignKey(TabularDataset, on_delete=models.CASCADE) - data = models.JSONField(default=dict) + date = models.DateField(null=True) + attribute = models.CharField(max_length=155, blank=True, null=True) + value = models.FloatField(default=0) + province = models.ForeignKey(Province, null=True, on_delete=models.PROTECT) + area_council = models.ForeignKey(AreaCouncil, null=True, on_delete=models.PROTECT) + metadata = models.JSONField(default=dict) def __str__(self): return f"{self.id}" diff --git a/vbos/datasets/serializers.py b/vbos/datasets/serializers.py index 7d408c0..45fc247 100644 --- a/vbos/datasets/serializers.py +++ b/vbos/datasets/serializers.py @@ -44,25 +44,21 @@ class Meta: class VectorItemSerializer(GeoFeatureModelSerializer): + province = serializers.ReadOnlyField(source="province.name") + area_council = serializers.ReadOnlyField(source="area_council.name") + class Meta: model = VectorItem geo_field = "geometry" - fields = ["id", "metadata"] - - def get_properties(self, instance, fields): - # This is a PostgreSQL HStore field, which django maps to a dict - return instance.metadata - - def unformat_geojson(self, feature): - attrs = { - self.Meta.geo_field: feature["geometry"], - "metadata": feature["properties"], - } - - if self.Meta.bbox_geo_field and "bbox" in feature: - attrs[self.Meta.bbox_geo_field] = Polygon.from_bbox(feature["bbox"]) - - return attrs + fields = [ + "id", + "name", + "ref", + "attribute", + "province", + "area_council", + "metadata", + ] class TabularDatasetSerializer(serializers.ModelSerializer): @@ -74,15 +70,26 @@ class Meta: class TabularItemSerializer(serializers.ModelSerializer): + province = serializers.ReadOnlyField(source="province.name") + area_council = serializers.ReadOnlyField(source="area_council.name") + class Meta: model = TabularItem - fields = ["id", "data"] + fields = [ + "id", + "attribute", + "date", + "value", + "province", + "area_council", + "metadata", + ] def to_representation(self, instance): representation = super().to_representation(instance) # Extract the data field and merge it with the top level fields - data_content = representation.pop("data", {}) + data_content = representation.pop("metadata", {}) return {**representation, **data_content} @@ -103,7 +110,10 @@ def __init__(self, *args, **kwargs): # Create a field for each key for key in all_keys: self.fields[key] = serializers.CharField( - source=f"data.{key}", required=False, allow_blank=True, default="" + source=f"metadata.{key}", + required=False, + allow_blank=True, + default="", ) class Meta: diff --git a/vbos/datasets/test/test_admin.py b/vbos/datasets/test/test_admin.py index 737b065..0a3e274 100644 --- a/vbos/datasets/test/test_admin.py +++ b/vbos/datasets/test/test_admin.py @@ -1,3 +1,4 @@ +from datetime import date import io from django.contrib.auth import get_user_model @@ -50,24 +51,36 @@ def test_post_invalid_file_type(self): self.assertContains(response, "Please upload a CSV file") def test_post_valid_csv_creates_items(self): - csv_content = "col1,col2\nval1,val2\nval3,val4\n" - file_data = io.BytesIO(csv_content.encode("utf-8")) - file_data.name = "test.csv" - response = self.client.post( - self.upload_url, - {"file": file_data, "dataset": self.dataset.id}, - follow=True, - ) - self.assertContains(response, "Successfully created 2 new records") - self.assertEqual(TabularItem.objects.count(), 2) - ti_1 = TabularItem.objects.first() + csv_path = "./vbos/datasets/fixtures/test.csv" + with open(csv_path, "rb") as file_data: + response = self.client.post( + self.upload_url, + {"file": file_data, "dataset": self.dataset.id}, + follow=True, + ) + self.assertContains(response, "Successfully created 3 new records") + self.assertEqual(TabularItem.objects.count(), 3) + ti_1, ti_2, ti_3 = TabularItem.objects.all() self.assertEqual(ti_1.dataset.id, self.dataset.id) - self.assertEqual(ti_1.data["col1"], "val1") - self.assertEqual(ti_1.data["col2"], "val2") - ti_2 = TabularItem.objects.last() + self.assertEqual(ti_1.date, date(2024, 1, 1)) + self.assertEqual(ti_1.attribute, "ecce") + self.assertEqual(ti_1.province.name, "TAFEA") + self.assertEqual(ti_1.area_council.name, "Futuna") + self.assertEqual(ti_1.value, 1154) + self.assertEqual(ti_1.metadata["Other"], "yes") self.assertEqual(ti_2.dataset.id, self.dataset.id) - self.assertEqual(ti_2.data["col1"], "val3") - self.assertEqual(ti_2.data["col2"], "val4") + self.assertEqual(ti_2.date, date(2022, 5, 1)) + self.assertEqual(ti_2.attribute, "secondary") + self.assertEqual(ti_2.province.name, "TAFEA") + self.assertEqual(ti_2.area_council.name, "Futuna") + self.assertEqual(ti_2.value, 1154) + self.assertEqual(ti_2.metadata["Other"], "no") + self.assertEqual(ti_3.dataset.id, self.dataset.id) + self.assertEqual(ti_3.date, date(2025, 1, 1)) + self.assertEqual(ti_3.attribute, "primary") + self.assertEqual(ti_3.province, None) + self.assertEqual(ti_3.area_council, None) + self.assertEqual(ti_3.value, 1154) class VectorItemAdminImportFileTests(TestCase): @@ -118,8 +131,21 @@ def test_post_valid_geojson_creates_items(self): self.assertEqual(VectorItem.objects.count(), 3) vi_1, vi_2, vi_3 = VectorItem.objects.all() self.assertEqual(vi_1.dataset.id, self.dataset.id) - self.assertEqual(vi_1.metadata["name"], "Area 1") + self.assertEqual(vi_1.name, "Area 1") + self.assertEqual(vi_1.ref, "12NC") + self.assertEqual(vi_1.attribute, "Schools") + self.assertEqual(vi_1.province.name, "TORBA") + self.assertEqual(vi_1.area_council.name, "East Gaua") self.assertEqual(vi_2.dataset.id, self.dataset.id) - self.assertEqual(vi_2.metadata["name"], "Line 1") + self.assertEqual(vi_2.name, "Line 1") + self.assertEqual(vi_2.ref, "13NC") + self.assertEqual(vi_2.attribute, "Roads") + self.assertEqual(vi_2.province.name, "TAFEA") + self.assertEqual(vi_2.area_council.name, "Futuna") self.assertEqual(vi_3.dataset.id, self.dataset.id) - self.assertEqual(vi_3.metadata["name"], "Point 1") + self.assertEqual(vi_3.name, "Point 2") + self.assertEqual(vi_3.ref, "14NC") + self.assertEqual(vi_3.attribute, "Business") + self.assertEqual(vi_3.province.name, "TAFEA") + self.assertEqual(vi_3.area_council.name, "Futuna") + self.assertEqual(vi_3.metadata["source"], "OpenStreetMap") diff --git a/vbos/datasets/test/test_tabular_views.py b/vbos/datasets/test/test_tabular_views.py index 813672e..dc7c282 100644 --- a/vbos/datasets/test/test_tabular_views.py +++ b/vbos/datasets/test/test_tabular_views.py @@ -1,8 +1,9 @@ +from datetime import date from rest_framework import status from rest_framework.test import APITestCase from django.urls import reverse -from ..models import Cluster, TabularDataset, TabularItem +from ..models import AreaCouncil, Cluster, Province, TabularDataset, TabularItem from ...users.test.factories import UserFactory @@ -68,51 +69,58 @@ def setUp(self): ) self.item = TabularItem.objects.create( dataset=self.dataset_1, - data={"province": "A", "population": 1902, "year": 2025}, + date=date(2025, 1, 1), + province=Province.objects.get(name="TORBA"), + attribute="Population", + value=13874, ) TabularItem.objects.create( dataset=self.dataset_1, - data={"province": "B", "population": 10902, "year": 2025}, + date=date(2025, 1, 1), + province=Province.objects.get(name="TAFEA"), + attribute="Population", + value=1230, ) TabularItem.objects.create( dataset=self.dataset_1, - data={"province": "C", "population": 875, "year": 2025}, + date=date(2025, 1, 1), + province=Province.objects.get(name="PENAMA"), + area_council=AreaCouncil.objects.get(name="North Maewo"), + attribute="Population", + value=5682, ) TabularItem.objects.create( dataset=self.dataset_2, - data={ - "employed_population": 0.75, - "year": 2025, - "month": 1, - "region": "North", - }, + date=date(2025, 1, 1), + attribute="Employed Population", + value=0.93, + province=Province.objects.get(name="TORBA"), + area_council=AreaCouncil.objects.get(name="East Gaua"), + metadata={"additional_value": "test"}, ) TabularItem.objects.create( dataset=self.dataset_2, - data={ - "employed_population": 0.85, - "year": 2024, - "month": 7, - "region": "North", - }, + date=date(2025, 2, 1), + attribute="Employed Population", + value=0.9, + province=Province.objects.get(name="TORBA"), + area_council=AreaCouncil.objects.get(name="East Gaua"), ) TabularItem.objects.create( dataset=self.dataset_2, - data={ - "employed_population": 0.82, - "year": 2024, - "month": 1, - "region": "South", - }, + date=date(2025, 3, 1), + attribute="Employed Population", + value=0.91, + province=Province.objects.get(name="TORBA"), + area_council=AreaCouncil.objects.get(name="East Gaua"), ) TabularItem.objects.create( dataset=self.dataset_2, - data={ - "employed_population": 0.80, - "year": 2023, - "month": 7, - "region": "East", - }, + date=date(2025, 4, 1), + attribute="Employed Population", + value=0.95, + province=Province.objects.get(name="TORBA"), + area_council=AreaCouncil.objects.get(name="East Gaua"), ) self.url = reverse("datasets:tabular-data", args=[self.dataset_1.id]) @@ -121,9 +129,15 @@ def test_tabular_datasets_data(self): assert req.status_code == status.HTTP_200_OK assert req.data.get("count") == 3 assert len(req.data.get("results")) == 3 - assert req.data.get("results")[0]["province"] == "A" - assert req.data.get("results")[0]["population"] == 1902 - assert req.data.get("results")[0]["year"] == 2025 + assert req.data.get("results")[0]["province"] == "TORBA" + assert req.data.get("results")[0]["value"] == 13874 + assert req.data.get("results")[0]["date"] == "2025-01-01" + assert req.data.get("results")[0]["attribute"] == "Population" + assert req.data.get("results")[2]["province"] == "PENAMA" + assert req.data.get("results")[2]["area_council"] == "North Maewo" + assert req.data.get("results")[2]["value"] == 5682 + assert req.data.get("results")[2]["date"] == "2025-01-01" + assert req.data.get("results")[2]["attribute"] == "Population" # fetch second dataset's data url = reverse("datasets:tabular-data", args=[self.dataset_2.id]) @@ -131,24 +145,42 @@ def test_tabular_datasets_data(self): assert req.status_code == status.HTTP_200_OK assert req.data.get("count") == 4 assert len(req.data.get("results")) == 4 - assert req.data.get("results")[0]["employed_population"] == 0.75 - assert req.data.get("results")[0]["month"] == 1 - assert req.data.get("results")[0]["year"] == 2025 + assert req.data.get("results")[0]["province"] == "TORBA" + assert req.data.get("results")[0]["area_council"] == "East Gaua" + assert req.data.get("results")[0]["value"] == 0.93 + assert req.data.get("results")[0]["date"] == "2025-01-01" + assert req.data.get("results")[0]["attribute"] == "Employed Population" + assert req.data.get("results")[0]["additional_value"] == "test" def test_filter_data(self): url = reverse("datasets:tabular-data", args=[self.dataset_2.id]) - req = self.client.get(url, {"filter": "year=2024"}) + req = self.client.get(url, {"date_after": "2025-01-01"}) + assert req.status_code == status.HTTP_200_OK + assert req.data.get("count") == 4 + + req = self.client.get(url, {"date_after": "2025-03-01"}) assert req.status_code == status.HTTP_200_OK assert req.data.get("count") == 2 - req = self.client.get(url, {"filter": "year=2024,month=1"}) + req = self.client.get(url, {"date_before": "2024-12-01"}) assert req.status_code == status.HTTP_200_OK - assert req.data.get("count") == 1 + assert req.data.get("count") == 0 - req = self.client.get(url, {"filter": "year__gte=2024,region__icontains=south"}) + req = self.client.get(url, {"province": "torba"}) assert req.status_code == status.HTTP_200_OK - assert req.data.get("count") == 1 + assert req.data.get("count") == 4 + + req = self.client.get(url, {"province": "south"}) + assert req.status_code == status.HTTP_400_BAD_REQUEST - req = self.client.get(url, {"filter": "region__icontains=north"}) + req = self.client.get(url, {"area_council": "North Maewo"}) assert req.status_code == status.HTTP_200_OK - assert req.data.get("count") == 2 + assert req.data.get("count") == 0 + + req = self.client.get(url, {"area_council": "East Gaua"}) + assert req.status_code == status.HTTP_200_OK + assert req.data.get("count") == 4 + + req = self.client.get(url, {"attribute": "population"}) + assert req.status_code == status.HTTP_200_OK + assert req.data.get("count") == 4 diff --git a/vbos/datasets/test/test_utils.py b/vbos/datasets/test/test_utils.py new file mode 100644 index 0000000..01796dc --- /dev/null +++ b/vbos/datasets/test/test_utils.py @@ -0,0 +1,67 @@ +from datetime import date +from django.test import TestCase + +from vbos.datasets.utils import CSVRow, GeoJSONProperties + + +class TestGeoJSONProperties(TestCase): + def setUp(self): + self.p1 = GeoJSONProperties( + { + "name": "Area 1", + "ref": "12NC", + "Province": "Torba", + "Area Council": "East Gaua", + "Attribute": "High School", + "extra": "value", + } + ) + + def test_class_attributes(self): + assert self.p1.properties == {"extra": "value"} + assert self.p1.attribute == "High School" + assert self.p1.name == "Area 1" + assert self.p1.ref == "12NC" + assert self.p1.province == "Torba" + assert self.p1.area_council == "East Gaua" + + +class TestCsvRow(TestCase): + def setUp(self): + self.r1 = CSVRow( + { + "value": "154", + "year": "2023", + "month": "March", + "Province": "Torba", + "Area Council": "East Gaua", + "Attribute": "High School", + "something": "yes", + } + ) + self.r2 = CSVRow( + { + "value": "154.2", + "year": "2022", + "Province": "TAFEA", + "Area Council": "Futuna", + "Attribute": "High School", + "something": "no", + "k": "v", + } + ) + + def test_class_attributes(self): + assert self.r1.metadata == {"something": "yes"} + assert self.r1.attribute == "High School" + assert self.r1.date == date(2023, 3, 1) + assert self.r1.province == "Torba" + assert self.r1.area_council == "East Gaua" + assert self.r1.value == "154" + + assert self.r2.metadata == {"something": "no", "k": "v"} + assert self.r2.attribute == "High School" + assert self.r2.date == date(2022, 1, 1) + assert self.r2.province == "TAFEA" + assert self.r2.area_council == "Futuna" + assert self.r2.value == "154.2" diff --git a/vbos/datasets/test/test_vector_views.py b/vbos/datasets/test/test_vector_views.py index fd0db41..d1d7975 100644 --- a/vbos/datasets/test/test_vector_views.py +++ b/vbos/datasets/test/test_vector_views.py @@ -1,9 +1,10 @@ +from pickle import OBJ from rest_framework import status from rest_framework.test import APITestCase from django.urls import reverse from django.contrib.gis.geos import Polygon, LineString, Point -from ..models import VectorDataset, VectorItem, Cluster +from ..models import AreaCouncil, Province, VectorDataset, VectorItem, Cluster class TestVectorDatasetListDetailViews(APITestCase): @@ -60,17 +61,32 @@ def setUp(self): VectorItem.objects.create( dataset=self.dataset_1, geometry=Point(80.5, 10.232), - metadata={"type": "administrative", "name": "Point 1", "area": 5000}, + name="Point 1", + ref="12NC", + attribute="administrative", + province=Province.objects.get(name="TORBA"), + area_council=AreaCouncil.objects.get(name="East Gaua"), + metadata={"area": 5000}, ) VectorItem.objects.create( dataset=self.dataset_1, geometry=LineString([(0, 0), (0, 3), (3, 3), (3, 0), (6, 6), (0, 0)]), - metadata={"type": "administrative", "name": "Line 123", "area": 5321}, + name="Line 1", + ref="13NC", + attribute="administrative", + province=Province.objects.get(name="TAFEA"), + area_council=AreaCouncil.objects.get(name="Futuna"), + metadata={"area": 5321}, ) VectorItem.objects.create( dataset=self.dataset_2, geometry=Polygon([(0, 0), (0, 3), (3, 3), (3, 0), (0, 0)]), - metadata={"type": "administrative", "name": "Area 1", "area": 3432}, + name="Area 1", + ref="14NC", + attribute="business", + province=Province.objects.get(name="TAFEA"), + area_council=AreaCouncil.objects.get(name="Futuna"), + metadata={"key": "value"}, ) self.url = reverse("datasets:vector-data", args=[self.dataset_1.id]) @@ -84,7 +100,13 @@ def test_vector_datasets_data(self): "coordinates": [80.5, 10.232], } assert req.data.get("features")[0]["properties"]["name"] == "Point 1" - assert req.data.get("features")[0]["properties"]["type"] == "administrative" + assert ( + req.data.get("features")[0]["properties"]["attribute"] == "administrative" + ) + assert req.data.get("features")[0]["properties"]["province"] == "TORBA" + assert req.data.get("features")[0]["properties"]["area_council"] == "East Gaua" + assert req.data.get("features")[0]["properties"]["ref"] == "12NC" + assert req.data.get("features")[0]["properties"]["metadata"]["area"] == 5000 # fetch second dataset's data url = reverse("datasets:vector-data", args=[self.dataset_2.id]) @@ -110,23 +132,37 @@ def test_filters(self): } assert req.data.get("features")[0]["properties"]["name"] == "Point 1" - # filter by metadata - req = self.client.get(self.url, {"filter": "name__icontains=Point"}) + # filter by name + req = self.client.get(self.url, {"name": "Point"}) + assert req.status_code == status.HTTP_200_OK + assert req.data.get("count") == 1 + + # filter by attribute + req = self.client.get(self.url, {"attribute": "ADMINISTRATIVE"}) + assert req.status_code == status.HTTP_200_OK + assert req.data.get("count") == 2 + + # filter by province + req = self.client.get(self.url, {"province": "tafea"}) assert req.status_code == status.HTTP_200_OK assert req.data.get("count") == 1 - req = self.client.get(self.url, {"filter": "area__lt=5000"}) + # filter by area council + req = self.client.get(self.url, {"area_council": "East Gaua"}) + assert req.status_code == status.HTTP_200_OK + assert req.data.get("count") == 1 + + # filter by metadata + req = self.client.get(self.url, {"metadata": "area__lt=5000"}) assert req.status_code == status.HTTP_200_OK assert req.data.get("count") == 0 - req = self.client.get( - self.url, {"filter": "area__gte=5000, type=administrative"} - ) + req = self.client.get(self.url, {"metadata": "area__gte=5000"}) assert req.status_code == status.HTTP_200_OK assert req.data.get("count") == 2 req = self.client.get( - self.url, {"filter": "area__gte=5000", "in_bbox": "80,10,81,11"} + self.url, {"metadata": "area__gte=5000", "in_bbox": "80,10,81,11"} ) assert req.status_code == status.HTTP_200_OK assert req.data.get("count") == 1 diff --git a/vbos/datasets/utils.py b/vbos/datasets/utils.py index e69de29..781c442 100644 --- a/vbos/datasets/utils.py +++ b/vbos/datasets/utils.py @@ -0,0 +1,60 @@ +from datetime import date +import calendar +from typing import Dict, List + + +class GeoJSONProperties: + def __init__(self, properties: Dict): + self.properties = properties + self.area_council = self.get_property( + ["Area Council", "area_council", "area council", "Area council"] + ) + self.province = self.get_property(["Province", "province"]) + self.name = self.get_property(["Name", "name"]) + self.ref = self.get_property(["ref", "Ref", "REF"]) + self.attribute = self.get_property(["Attribute", "attribute"]) + + def get_property(self, keys: List[str]) -> str: + value = "" + for key in keys: + try: + value = self.properties.pop(key) + break + except KeyError: + pass + return value + + +class CSVRow: + def __init__(self, row: Dict): + self.metadata = row + self.area_council = self.get_property( + ["Area Council", "area_council", "area council", "Area council"] + ) + self.province = self.get_property(["Province", "province"]) + self.value = self.get_property(["Value", "value", "VALUE"]) + self.attribute = self.get_property(["Attribute", "attribute"]) + self.year = self.get_property(["Year", "year", "YEAR"]) + self.month = self.get_property(["Month", "month", "MONTH"]) + try: + self.date = date(int(self.year), self.month_name_to_number(), 1) + except ValueError: + self.date = None + + def get_property(self, keys: List[str]) -> str: + value = "" + for key in keys: + try: + value = self.metadata.pop(key) + break + except KeyError: + pass + return value + + def month_name_to_number(self) -> int: + month_dict = { + month.lower(): index + for index, month in enumerate(calendar.month_name) + if month + } + return month_dict.get(self.month.lower()) or 1