From a47d1a2d46ca8c225dfe7fc3ef813b5abb7d9cf2 Mon Sep 17 00:00:00 2001 From: Isaiah King'ori Date: Fri, 11 Dec 2020 14:09:49 +0300 Subject: [PATCH 01/29] Ft locations endpoint (#75) * Adds Location Serializer * Allows users to list and create locations * Adds a basename for sensors_location_router * Adds Pagination --- sensorsafrica/api/v2/router.py | 8 ++++++-- sensorsafrica/api/v2/serializers.py | 20 ++++++++++++++++++++ sensorsafrica/api/v2/views.py | 26 ++++++++++++++++++++++++-- 3 files changed, 50 insertions(+), 4 deletions(-) diff --git a/sensorsafrica/api/v2/router.py b/sensorsafrica/api/v2/router.py index 69f17e7..6cc5a72 100644 --- a/sensorsafrica/api/v2/router.py +++ b/sensorsafrica/api/v2/router.py @@ -1,7 +1,7 @@ from rest_framework import routers from django.conf.urls import url, include -from .views import SensorDataStatView, CityView, NodesView +from .views import SensorDataStatView, CityView, NodesView, SensorsLocationView data_router = routers.DefaultRouter() @@ -15,8 +15,12 @@ nodes_router.register(r"", NodesView, basename="map") +sensors_location_router = routers.DefaultRouter() +sensors_location_router.register(r"", SensorsLocationView, basename="location") + api_urls = [ url(r"data/(?P[air]+)/", include(data_router.urls)), url(r"cities/", include(city_router.urls)), - url(r"nodes/", include(nodes_router.urls)) + url(r"nodes/", include(nodes_router.urls)), + url(r"locations/", include(sensors_location_router.urls)), ] diff --git a/sensorsafrica/api/v2/serializers.py b/sensorsafrica/api/v2/serializers.py index b86e9a4..875b094 100644 --- a/sensorsafrica/api/v2/serializers.py +++ b/sensorsafrica/api/v2/serializers.py @@ -1,4 +1,5 @@ from rest_framework import serializers +from feinstaub.sensors.serializers import NestedSensorLocationSerializer class SensorDataStatSerializer(serializers.Serializer): @@ -21,3 +22,22 @@ class CitySerializer(serializers.Serializer): def get_label(self, obj): return "{}, {}".format(obj.name, obj.country) + + +class SensorLocationSerializer(NestedSensorLocationSerializer): + class Meta(NestedSensorLocationSerializer.Meta): + fields = NestedSensorLocationSerializer.Meta.fields + \ + ( + 'longitude', + 'latitude', + 'altitude', + 'street_name', + 'street_number', + 'city', + 'country', + 'postalcode', + 'traffic_in_area', + 'oven_in_area', + 'industry_in_area', + 'owner', + ) diff --git a/sensorsafrica/api/v2/views.py b/sensorsafrica/api/v2/views.py index 332f494..d356227 100644 --- a/sensorsafrica/api/v2/views.py +++ b/sensorsafrica/api/v2/views.py @@ -11,8 +11,8 @@ from django.db.models.functions import Cast, TruncHour, TruncDay, TruncMonth from rest_framework import mixins, pagination, viewsets -from ..models import SensorDataStat, LastActiveNodes, City, Node -from .serializers import SensorDataStatSerializer, CitySerializer +from ..models import SensorDataStat, LastActiveNodes, City, Node, SensorLocation +from .serializers import SensorDataStatSerializer, CitySerializer, SensorLocationSerializer from feinstaub.sensors.views import StandardResultsSetPagination @@ -21,6 +21,8 @@ from django.utils.text import slugify from rest_framework.response import Response +from rest_framework.authentication import SessionAuthentication, TokenAuthentication +from rest_framework.permissions import IsAuthenticated from django.utils.decorators import method_decorator from django.views.decorators.cache import cache_page @@ -273,3 +275,23 @@ def list(self, request): } ) return Response(nodes) + + +class SensorsLocationView(viewsets.ViewSet): + authentication_classes = [SessionAuthentication, TokenAuthentication] + permission_classes = [IsAuthenticated] + pagination_class = StandardResultsSetPagination + + def list(self, request): + queryset = SensorLocation.objects.all() + serializer = SensorLocationSerializer(queryset, many=True) + + return Response(serializer.data) + + def create(self, request): + serializer = SensorLocationSerializer(data=request.data) + if serializer.is_valid(): + serializer.save() + return Response(serializer.data, status=204) + + return Response(serializer.errors, status=400) From 32bb168fa4b2a9aa28e36f79ed5943978cffe531 Mon Sep 17 00:00:00 2001 From: Isaiah King'ori Date: Fri, 11 Dec 2020 17:13:11 +0300 Subject: [PATCH 02/29] Adds Node creation API endpoint (#76) * Adds Node creation API endpoint * Remove un used serializer import --- sensorsafrica/api/v2/serializers.py | 18 ++++++++++++++++++ sensorsafrica/api/v2/views.py | 21 ++++++++++++++++++--- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/sensorsafrica/api/v2/serializers.py b/sensorsafrica/api/v2/serializers.py index 875b094..d5ab74e 100644 --- a/sensorsafrica/api/v2/serializers.py +++ b/sensorsafrica/api/v2/serializers.py @@ -1,5 +1,6 @@ from rest_framework import serializers from feinstaub.sensors.serializers import NestedSensorLocationSerializer +from feinstaub.sensors.models import Node class SensorDataStatSerializer(serializers.Serializer): @@ -41,3 +42,20 @@ class Meta(NestedSensorLocationSerializer.Meta): 'industry_in_area', 'owner', ) +class NodeSerializer(serializers.ModelSerializer): + class Meta: + model = Node + fields = ( + 'uid', + 'owner', + 'location', + 'name', + 'description', + 'height', + 'sensor_position', + 'email', + 'last_notify', + 'indoor', + 'inactive', + 'exact_location', + ) diff --git a/sensorsafrica/api/v2/views.py b/sensorsafrica/api/v2/views.py index d356227..344fa5c 100644 --- a/sensorsafrica/api/v2/views.py +++ b/sensorsafrica/api/v2/views.py @@ -12,7 +12,7 @@ from rest_framework import mixins, pagination, viewsets from ..models import SensorDataStat, LastActiveNodes, City, Node, SensorLocation -from .serializers import SensorDataStatSerializer, CitySerializer, SensorLocationSerializer +from .serializers import SensorDataStatSerializer, CitySerializer, SensorLocationSerializer, NodeSerializer from feinstaub.sensors.views import StandardResultsSetPagination @@ -22,7 +22,7 @@ from rest_framework.response import Response from rest_framework.authentication import SessionAuthentication, TokenAuthentication -from rest_framework.permissions import IsAuthenticated +from rest_framework.permissions import IsAuthenticated, AllowAny from django.utils.decorators import method_decorator from django.views.decorators.cache import cache_page @@ -199,6 +199,14 @@ class CityView(mixins.ListModelMixin, viewsets.GenericViewSet): class NodesView(viewsets.ViewSet): + authentication_classes = [SessionAuthentication, TokenAuthentication] + def get_permissions(self): + if self.action == 'create': + permission_classes = [IsAuthenticated] + else: + permission_classes = [AllowAny] + return [permission() for permission in permission_classes] + def list(self, request): nodes = [] # Loop through the last active nodes @@ -276,6 +284,14 @@ def list(self, request): ) return Response(nodes) + def create(self, request): + serializer = NodeSerializer(data=request.data) + if serializer.is_valid(): + serializer.save() + return Response(serializer.data, status=204) + + return Response(serializer.errors, status=400) + class SensorsLocationView(viewsets.ViewSet): authentication_classes = [SessionAuthentication, TokenAuthentication] @@ -293,5 +309,4 @@ def create(self, request): if serializer.is_valid(): serializer.save() return Response(serializer.data, status=204) - return Response(serializer.errors, status=400) From 79e61cac9d9611a5fe31d5a02ae22d741e37b440 Mon Sep 17 00:00:00 2001 From: Khadija Mahanga Date: Fri, 11 Dec 2020 17:28:08 +0300 Subject: [PATCH 03/29] Add Sensors + Sensor_Type Endpoint (#77) * sensors endpoint * fix serializers * authentication on post * sensors type endpoint * Update sensorsafrica/api/v2/views.py Co-authored-by: _ Kilemensi * Update sensorsafrica/api/v2/views.py Co-authored-by: _ Kilemensi * spelling Co-authored-by: _ Kilemensi --- sensorsafrica/api/v2/router.py | 10 +++++- sensorsafrica/api/v2/serializers.py | 8 +++-- sensorsafrica/api/v2/views.py | 51 +++++++++++++++++++++++++++-- 3 files changed, 63 insertions(+), 6 deletions(-) diff --git a/sensorsafrica/api/v2/router.py b/sensorsafrica/api/v2/router.py index 6cc5a72..7accd8d 100644 --- a/sensorsafrica/api/v2/router.py +++ b/sensorsafrica/api/v2/router.py @@ -1,7 +1,7 @@ from rest_framework import routers from django.conf.urls import url, include -from .views import SensorDataStatView, CityView, NodesView, SensorsLocationView +from .views import SensorDataStatView, CityView, NodesView, SensorsLocationView, SensorTypeView, SensorsView data_router = routers.DefaultRouter() @@ -15,12 +15,20 @@ nodes_router.register(r"", NodesView, basename="map") +sensors_router = routers.DefaultRouter() +sensors_router.register(r"", SensorsView, basename="sensors") + sensors_location_router = routers.DefaultRouter() sensors_location_router.register(r"", SensorsLocationView, basename="location") +sensor_type_router = routers.DefaultRouter() +sensor_type_router.register(r"", SensorTypeView, basename="sensor_type") + api_urls = [ url(r"data/(?P[air]+)/", include(data_router.urls)), url(r"cities/", include(city_router.urls)), url(r"nodes/", include(nodes_router.urls)), url(r"locations/", include(sensors_location_router.urls)), + url(r"sensors/", include(sensors_router.urls)), + url(r"sensor-type/", include(sensor_type_router.urls)), ] diff --git a/sensorsafrica/api/v2/serializers.py b/sensorsafrica/api/v2/serializers.py index d5ab74e..7ebb29a 100644 --- a/sensorsafrica/api/v2/serializers.py +++ b/sensorsafrica/api/v2/serializers.py @@ -1,6 +1,6 @@ from rest_framework import serializers -from feinstaub.sensors.serializers import NestedSensorLocationSerializer -from feinstaub.sensors.models import Node +from feinstaub.sensors.serializers import NestedSensorLocationSerializer, NestedSensorTypeSerializer +from feinstaub.sensors.models import Node, Sensor class SensorDataStatSerializer(serializers.Serializer): @@ -24,6 +24,10 @@ class CitySerializer(serializers.Serializer): def get_label(self, obj): return "{}, {}".format(obj.name, obj.country) +class SensorSerializer(serializers.ModelSerializer): + class Meta: + model = Sensor + fields = ('id', 'node', 'description', 'pin', 'sensor_type', 'public') class SensorLocationSerializer(NestedSensorLocationSerializer): class Meta(NestedSensorLocationSerializer.Meta): diff --git a/sensorsafrica/api/v2/views.py b/sensorsafrica/api/v2/views.py index 344fa5c..2cec95c 100644 --- a/sensorsafrica/api/v2/views.py +++ b/sensorsafrica/api/v2/views.py @@ -11,12 +11,12 @@ from django.db.models.functions import Cast, TruncHour, TruncDay, TruncMonth from rest_framework import mixins, pagination, viewsets -from ..models import SensorDataStat, LastActiveNodes, City, Node, SensorLocation -from .serializers import SensorDataStatSerializer, CitySerializer, SensorLocationSerializer, NodeSerializer +from ..models import SensorDataStat, LastActiveNodes, City, Node, Sensor, SensorLocation +from .serializers import SensorDataStatSerializer, CitySerializer, NestedSensorTypeSerializer, NodeSerializer, SensorSerializer, SensorLocationSerializer from feinstaub.sensors.views import StandardResultsSetPagination -from feinstaub.sensors.models import SensorLocation, SensorData, SensorDataValue +from feinstaub.sensors.models import SensorLocation, SensorData, SensorDataValue, SensorType from django.utils.text import slugify @@ -310,3 +310,48 @@ def create(self, request): serializer.save() return Response(serializer.data, status=204) return Response(serializer.errors, status=400) + +class SensorsView(viewsets.ViewSet): + authentication_classes = [SessionAuthentication, TokenAuthentication] + permission_classes = [IsAuthenticated] + pagination_class = StandardResultsSetPagination + + def get_permissions(self): + if self.action == 'create': + permission_classes = [IsAuthenticated] + else: + permission_classes = [AllowAny] + return [permission() for permission in permission_classes] + + def list(self, request): + queryset = Sensor.objects.all() + serializer = SensorSerializer(queryset, many=True) + + return Response(serializer.data) + + def create(self, request): + serializer = SensorSerializer(data=request.data) + if serializer.is_valid(): + serializer.save() + return Response(serializer.data, status=204) + + return Response(serializer.errors, status=400) + +class SensorTypeView(viewsets.ViewSet): + authentication_classes = [SessionAuthentication, TokenAuthentication] + permission_classes = [IsAuthenticated] + pagination_class = StandardResultsSetPagination + + def list(self, request): + queryset = SensorType.objects.all() + serializer = NestedSensorTypeSerializer(queryset, many=True) + + return Response(serializer.data) + + def create(self, request): + serializer = NestedSensorTypeSerializer(data=request.data) + if serializer.is_valid(): + serializer.save() + return Response(serializer.data, status=204) + + return Response(serializer.errors, status=400) From 09ce4ac53d40626100554cd86034982ee4ebe4eb Mon Sep 17 00:00:00 2001 From: _ Kilemensi Date: Sat, 12 Dec 2020 17:11:48 +0300 Subject: [PATCH 04/29] [Hotfix] Creation api (#79) * Standardize naming around views, models and routes * Add tests --- Makefile | 5 ++ sensorsafrica/api/v2/router.py | 22 ++--- sensorsafrica/api/v2/serializers.py | 66 ++++++++------- sensorsafrica/api/v2/views.py | 82 +++++++++++-------- sensorsafrica/tests/conftest.py | 1 - sensorsafrica/tests/test_node_view.py | 37 +++++++++ .../tests/test_sensor_location_view.py | 36 ++++++++ sensorsafrica/tests/test_sensor_type_view.py | 37 +++++++++ sensorsafrica/tests/test_sensor_view.py | 39 ++++++++- 9 files changed, 245 insertions(+), 80 deletions(-) create mode 100644 sensorsafrica/tests/test_node_view.py create mode 100644 sensorsafrica/tests/test_sensor_location_view.py create mode 100644 sensorsafrica/tests/test_sensor_type_view.py diff --git a/Makefile b/Makefile index 69071ce..f4f380d 100644 --- a/Makefile +++ b/Makefile @@ -25,9 +25,14 @@ migrate: test: $(COMPOSE) exec api pytest --pylama +testexpr: + $(COMPOSE) exec api pytest --pylama -k '$(expr)' + createsuperuser: $(COMPOSE) exec api python manage.py createsuperuser +down: + $(COMPOSE) down clean: @find . -name "*.pyc" -exec rm -rf {} \; diff --git a/sensorsafrica/api/v2/router.py b/sensorsafrica/api/v2/router.py index 7accd8d..85a709c 100644 --- a/sensorsafrica/api/v2/router.py +++ b/sensorsafrica/api/v2/router.py @@ -1,15 +1,15 @@ from rest_framework import routers from django.conf.urls import url, include -from .views import SensorDataStatView, CityView, NodesView, SensorsLocationView, SensorTypeView, SensorsView +from .views import CitiesView, NodesView, SensorDataStatsView, SensorLocationsView, SensorTypesView, SensorsView data_router = routers.DefaultRouter() -data_router.register(r"", SensorDataStatView) +data_router.register(r"", SensorDataStatsView) -city_router = routers.DefaultRouter() +cities_router = routers.DefaultRouter() -city_router.register(r"", CityView) +cities_router.register(r"", CitiesView, basename="cities") nodes_router = routers.DefaultRouter() @@ -18,17 +18,17 @@ sensors_router = routers.DefaultRouter() sensors_router.register(r"", SensorsView, basename="sensors") -sensors_location_router = routers.DefaultRouter() -sensors_location_router.register(r"", SensorsLocationView, basename="location") +sensor_locations_router = routers.DefaultRouter() +sensor_locations_router.register(r"", SensorLocationsView, basename="locations") -sensor_type_router = routers.DefaultRouter() -sensor_type_router.register(r"", SensorTypeView, basename="sensor_type") +sensor_types_router = routers.DefaultRouter() +sensor_types_router.register(r"", SensorTypesView, basename="sensor_types") api_urls = [ url(r"data/(?P[air]+)/", include(data_router.urls)), - url(r"cities/", include(city_router.urls)), + url(r"cities/", include(cities_router.urls)), url(r"nodes/", include(nodes_router.urls)), - url(r"locations/", include(sensors_location_router.urls)), + url(r"locations/", include(sensor_locations_router.urls)), url(r"sensors/", include(sensors_router.urls)), - url(r"sensor-type/", include(sensor_type_router.urls)), + url(r"sensor-types/", include(sensor_types_router.urls)), ] diff --git a/sensorsafrica/api/v2/serializers.py b/sensorsafrica/api/v2/serializers.py index 7ebb29a..d875603 100644 --- a/sensorsafrica/api/v2/serializers.py +++ b/sensorsafrica/api/v2/serializers.py @@ -1,5 +1,8 @@ from rest_framework import serializers -from feinstaub.sensors.serializers import NestedSensorLocationSerializer, NestedSensorTypeSerializer +from feinstaub.sensors.serializers import ( + NestedSensorLocationSerializer, + NestedSensorTypeSerializer, +) from feinstaub.sensors.models import Node, Sensor @@ -24,42 +27,45 @@ class CitySerializer(serializers.Serializer): def get_label(self, obj): return "{}, {}".format(obj.name, obj.country) + class SensorSerializer(serializers.ModelSerializer): class Meta: model = Sensor - fields = ('id', 'node', 'description', 'pin', 'sensor_type', 'public') + fields = ("id", "node", "description", "pin", "sensor_type", "public") + class SensorLocationSerializer(NestedSensorLocationSerializer): class Meta(NestedSensorLocationSerializer.Meta): - fields = NestedSensorLocationSerializer.Meta.fields + \ - ( - 'longitude', - 'latitude', - 'altitude', - 'street_name', - 'street_number', - 'city', - 'country', - 'postalcode', - 'traffic_in_area', - 'oven_in_area', - 'industry_in_area', - 'owner', - ) + fields = NestedSensorLocationSerializer.Meta.fields + ( + "longitude", + "latitude", + "altitude", + "street_name", + "street_number", + "city", + "country", + "postalcode", + "traffic_in_area", + "oven_in_area", + "industry_in_area", + "owner", + ) + + class NodeSerializer(serializers.ModelSerializer): class Meta: model = Node fields = ( - 'uid', - 'owner', - 'location', - 'name', - 'description', - 'height', - 'sensor_position', - 'email', - 'last_notify', - 'indoor', - 'inactive', - 'exact_location', - ) + "uid", + "owner", + "location", + "name", + "description", + "height", + "sensor_position", + "email", + "last_notify", + "indoor", + "inactive", + "exact_location", + ) diff --git a/sensorsafrica/api/v2/views.py b/sensorsafrica/api/v2/views.py index 2cec95c..abe712b 100644 --- a/sensorsafrica/api/v2/views.py +++ b/sensorsafrica/api/v2/views.py @@ -11,12 +11,26 @@ from django.db.models.functions import Cast, TruncHour, TruncDay, TruncMonth from rest_framework import mixins, pagination, viewsets -from ..models import SensorDataStat, LastActiveNodes, City, Node, Sensor, SensorLocation -from .serializers import SensorDataStatSerializer, CitySerializer, NestedSensorTypeSerializer, NodeSerializer, SensorSerializer, SensorLocationSerializer +from ..models import SensorDataStat, LastActiveNodes, City +from .serializers import ( + SensorDataStatSerializer, + CitySerializer, + NestedSensorTypeSerializer, + NodeSerializer, + SensorSerializer, + SensorLocationSerializer, +) from feinstaub.sensors.views import StandardResultsSetPagination -from feinstaub.sensors.models import SensorLocation, SensorData, SensorDataValue, SensorType +from feinstaub.sensors.models import ( + Node, + Sensor, + SensorData, + SensorDataValue, + SensorLocation, + SensorType, +) from django.utils.text import slugify @@ -79,7 +93,9 @@ def get_paginated_response(self, data_stats): results[city_slug][value_type] = [] if from_date or interval else {} values = results[city_slug][value_type] - include_result = getattr(values, "append" if from_date or interval else "update") + include_result = getattr( + values, "append" if from_date or interval else "update" + ) include_result( { "average": data_stat["calculated_average"], @@ -100,7 +116,7 @@ def get_paginated_response(self, data_stats): ) -class SensorDataStatView(mixins.ListModelMixin, viewsets.GenericViewSet): +class SensorDataStatsView(mixins.ListModelMixin, viewsets.GenericViewSet): queryset = SensorDataStat.objects.none() serializer_class = SensorDataStatSerializer pagination_class = CustomPagination @@ -135,7 +151,7 @@ def get_queryset(self): if not from_date and not to_date: to_date = timezone.now().replace(minute=0, second=0, microsecond=0) from_date = to_date - datetime.timedelta(hours=24) - interval = 'day' if not interval else interval + interval = "day" if not interval else interval elif not to_date: from_date = beginning_of_day(from_date) # Get data from_date until the end @@ -151,9 +167,9 @@ def get_queryset(self): timestamp__lte=to_date, ) - if interval == 'month': + if interval == "month": truncate = TruncMonth("timestamp") - elif interval == 'day': + elif interval == "day": truncate = TruncDay("timestamp") else: truncate = TruncHour("timestamp") @@ -162,11 +178,7 @@ def get_queryset(self): queryset = queryset.filter(city_slug__in=city_slugs.split(",")) return ( - queryset - .values( - "value_type", - "city_slug" - ) + queryset.values("value_type", "city_slug") .annotate( truncated_timestamp=truncate, start_datetime=Min("timestamp"), @@ -186,13 +198,13 @@ def get_queryset(self): "end_datetime", "calculated_average", "calculated_minimum", - "calculated_maximum" + "calculated_maximum", ) .order_by("city_slug", "-truncated_timestamp") ) -class CityView(mixins.ListModelMixin, viewsets.GenericViewSet): +class CitiesView(mixins.ListModelMixin, viewsets.GenericViewSet): queryset = City.objects.all() serializer_class = CitySerializer pagination_class = StandardResultsSetPagination @@ -200,11 +212,13 @@ class CityView(mixins.ListModelMixin, viewsets.GenericViewSet): class NodesView(viewsets.ViewSet): authentication_classes = [SessionAuthentication, TokenAuthentication] + def get_permissions(self): - if self.action == 'create': + if self.action == "create": permission_classes = [IsAuthenticated] else: permission_classes = [AllowAny] + return [permission() for permission in permission_classes] def list(self, request): @@ -265,10 +279,7 @@ def list(self, request): { "node_moved": moved_to is not None, "moved_to": moved_to, - "node": { - "uid": last_active.node.uid, - "id": last_active.node.id - }, + "node": {"uid": last_active.node.uid, "id": last_active.node.id}, "location": { "name": last_active.location.location, "longitude": last_active.location.longitude, @@ -288,12 +299,12 @@ def create(self, request): serializer = NodeSerializer(data=request.data) if serializer.is_valid(): serializer.save() - return Response(serializer.data, status=204) + return Response(serializer.data, status=201) return Response(serializer.errors, status=400) -class SensorsLocationView(viewsets.ViewSet): +class SensorLocationsView(viewsets.ViewSet): authentication_classes = [SessionAuthentication, TokenAuthentication] permission_classes = [IsAuthenticated] pagination_class = StandardResultsSetPagination @@ -301,43 +312,45 @@ class SensorsLocationView(viewsets.ViewSet): def list(self, request): queryset = SensorLocation.objects.all() serializer = SensorLocationSerializer(queryset, many=True) - return Response(serializer.data) - + def create(self, request): serializer = SensorLocationSerializer(data=request.data) if serializer.is_valid(): serializer.save() - return Response(serializer.data, status=204) + return Response(serializer.data, status=201) + return Response(serializer.errors, status=400) + class SensorsView(viewsets.ViewSet): authentication_classes = [SessionAuthentication, TokenAuthentication] permission_classes = [IsAuthenticated] pagination_class = StandardResultsSetPagination def get_permissions(self): - if self.action == 'create': + if self.action == "create": permission_classes = [IsAuthenticated] else: permission_classes = [AllowAny] + return [permission() for permission in permission_classes] def list(self, request): queryset = Sensor.objects.all() serializer = SensorSerializer(queryset, many=True) - return Response(serializer.data) - + def create(self, request): serializer = SensorSerializer(data=request.data) if serializer.is_valid(): serializer.save() - return Response(serializer.data, status=204) - + return Response(serializer.data, status=201) + return Response(serializer.errors, status=400) -class SensorTypeView(viewsets.ViewSet): + +class SensorTypesView(viewsets.ViewSet): authentication_classes = [SessionAuthentication, TokenAuthentication] permission_classes = [IsAuthenticated] pagination_class = StandardResultsSetPagination @@ -345,13 +358,12 @@ class SensorTypeView(viewsets.ViewSet): def list(self, request): queryset = SensorType.objects.all() serializer = NestedSensorTypeSerializer(queryset, many=True) - return Response(serializer.data) - + def create(self, request): serializer = NestedSensorTypeSerializer(data=request.data) if serializer.is_valid(): serializer.save() - return Response(serializer.data, status=204) - + return Response(serializer.data, status=201) + return Response(serializer.errors, status=400) diff --git a/sensorsafrica/tests/conftest.py b/sensorsafrica/tests/conftest.py index cbef41c..8a96df4 100644 --- a/sensorsafrica/tests/conftest.py +++ b/sensorsafrica/tests/conftest.py @@ -43,7 +43,6 @@ def sensor_type(): uid="a", name="b", manufacturer="c") return st - @pytest.fixture def node(logged_in_user, location): n, x = Node.objects.get_or_create( diff --git a/sensorsafrica/tests/test_node_view.py b/sensorsafrica/tests/test_node_view.py new file mode 100644 index 0000000..1b648b9 --- /dev/null +++ b/sensorsafrica/tests/test_node_view.py @@ -0,0 +1,37 @@ +import pytest +import datetime +from django.utils import timezone + +from rest_framework.test import APIRequestFactory + +from feinstaub.sensors.models import Node +from sensorsafrica.api.v2.views import NodesView + + +@pytest.mark.django_db +class TestNodesView: + @pytest.fixture + def data_fixture(self): + return { + "uid": "testnode1", + } + + def test_create_node(self, data_fixture, logged_in_user, location): + data_fixture["location"] = location.id + data_fixture["owner"] = logged_in_user.id + factory = APIRequestFactory() + url = "/v2/nodes/" + request = factory.post(url, data_fixture, format="json") + + # authenticate request + request.user = logged_in_user + + view = NodesView + view_function = view.as_view({"post": "create"}) + response = view_function(request) + + assert response.status_code == 201 + node = Node.objects.get(uid=response.data["uid"]) + assert node.uid == data_fixture["uid"] + assert node.location == location + assert node.owner == logged_in_user diff --git a/sensorsafrica/tests/test_sensor_location_view.py b/sensorsafrica/tests/test_sensor_location_view.py new file mode 100644 index 0000000..00032e2 --- /dev/null +++ b/sensorsafrica/tests/test_sensor_location_view.py @@ -0,0 +1,36 @@ +import pytest +import datetime +from django.utils import timezone + +from rest_framework.test import APIRequestFactory + +from feinstaub.sensors.models import SensorLocation +from sensorsafrica.api.v2.views import SensorLocationsView + + +@pytest.mark.django_db +class TestSensorLocationsView: + @pytest.fixture + def data_fixture(self): + return { + "location": "Code for Africa Offices", + } + + def test_create_sensor_location(self, data_fixture, logged_in_user): + data_fixture["owner"] = logged_in_user.id + factory = APIRequestFactory() + url = "/v2/locations/" + request = factory.post(url, data_fixture, format="json") + + # authenticate request + request.user = logged_in_user + + view = SensorLocationsView + view_function = view.as_view({"post": "create"}) + response = view_function(request) + + assert response.status_code == 201 + sensor_type = SensorLocation.objects.get(id=response.data["id"]) + + assert sensor_type.location == data_fixture["location"] + assert sensor_type.owner == logged_in_user diff --git a/sensorsafrica/tests/test_sensor_type_view.py b/sensorsafrica/tests/test_sensor_type_view.py new file mode 100644 index 0000000..a156b15 --- /dev/null +++ b/sensorsafrica/tests/test_sensor_type_view.py @@ -0,0 +1,37 @@ +import pytest +import datetime +from django.utils import timezone + +from rest_framework.test import APIRequestFactory + +from feinstaub.sensors.models import SensorType +from sensorsafrica.api.v2.views import SensorTypesView + + +@pytest.mark.django_db +class TestSensorTypesView: + @pytest.fixture + def data_fixture(self): + return { + "uid": "nm1", + "name": "N1", + "manufacturer": "M1", + } + + def test_create_sensor_type(self, data_fixture, logged_in_user): + factory = APIRequestFactory() + url = "/v2/sensor-types/" + request = factory.post(url, data_fixture, format="json") + + # authenticate request + request.user = logged_in_user + + view = SensorTypesView + view_function = view.as_view({"post": "create"}) + response = view_function(request) + + assert response.status_code == 201 + sensor_type = SensorType.objects.get(id=response.data["id"]) + + assert sensor_type.name == data_fixture["name"] + assert sensor_type.manufacturer == data_fixture["manufacturer"] diff --git a/sensorsafrica/tests/test_sensor_view.py b/sensorsafrica/tests/test_sensor_view.py index 8a43905..c159c19 100644 --- a/sensorsafrica/tests/test_sensor_view.py +++ b/sensorsafrica/tests/test_sensor_view.py @@ -2,9 +2,42 @@ import datetime from django.utils import timezone +from rest_framework.test import APIRequestFactory + +from feinstaub.sensors.models import Sensor +from sensorsafrica.api.v2.views import SensorsView + @pytest.mark.django_db -class TestGettingSensorPast5MinutesData: +class TestSensorsView: + @pytest.fixture + def data_fixture(self): + return { + "pin": "1", + "sensor_type": 1, + "node": 1, + "public": False, + } + + def test_create_sensor(self, data_fixture, logged_in_user, node, sensor_type): + data_fixture["node"] = node.id + data_fixture["sensor_type"] = sensor_type.id + factory = APIRequestFactory() + url = "/v2/sensors/" + request = factory.post(url, data_fixture, format="json") + + # authenticate request + request.user = logged_in_user + + view = SensorsView + view_function = view.as_view({"post": "create"}) + response = view_function(request) + + assert response.status_code == 201 + sensor = Sensor.objects.get(id=response.data["id"]) + assert sensor.pin == data_fixture["pin"] + assert sensor.public == data_fixture["public"] + def test_getting_past_5_minutes_data_for_sensor_with_id(self, client, sensors): response = client.get("/v1/sensors/%s/" % sensors[0].id, format="json") assert response.status_code == 200 @@ -15,5 +48,5 @@ def test_getting_past_5_minutes_data_for_sensor_with_id(self, client, sensors): assert len(results) == 7 assert "sensordatavalues" in results[0] assert "timestamp" in results[0] - assert "value_type" in results[0]['sensordatavalues'][0] - assert "value" in results[0]['sensordatavalues'][0] + assert "value_type" in results[0]["sensordatavalues"][0] + assert "value" in results[0]["sensordatavalues"][0] From 679344449f777a7a5232d829d4bd7031c4653371 Mon Sep 17 00:00:00 2001 From: _ Kilemensi Date: Mon, 14 Dec 2020 15:34:47 +0300 Subject: [PATCH 05/29] [Feature] Public / Private Data (#81) * Add SensorDataView using CursorPagination to show all available data * Make /v2/data return all data, keept /data/ for air/water, etc. * Switch computation to only work with public data (sensorsAFRICA data for now) * Upgrade backward compatible deps --- requirements.txt | 6 +- sensorsafrica/api/v2/router.py | 21 +- sensorsafrica/api/v2/views.py | 284 ++++++++++-------- .../commands/calculate_data_statistics.py | 52 ++-- 4 files changed, 208 insertions(+), 155 deletions(-) diff --git a/requirements.txt b/requirements.txt index b7653d4..fdca0c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,11 @@ -django==1.11.27 #LTS +django==1.11.29 #LTS coreapi==2.3.3 dj-database-url==0.5.0 timeago==1.0.10 -flower==0.9.2 +flower==0.9.5 tornado<6 -sentry-sdk==0.7.3 +sentry-sdk==0.19.5 celery==4.2.1 gevent==1.2.2 greenlet==0.4.12 diff --git a/sensorsafrica/api/v2/router.py b/sensorsafrica/api/v2/router.py index 85a709c..d0b9677 100644 --- a/sensorsafrica/api/v2/router.py +++ b/sensorsafrica/api/v2/router.py @@ -1,18 +1,26 @@ from rest_framework import routers from django.conf.urls import url, include -from .views import CitiesView, NodesView, SensorDataStatsView, SensorLocationsView, SensorTypesView, SensorsView +from .views import ( + CitiesView, + NodesView, + SensorDataStatsView, + SensorDataView, + SensorLocationsView, + SensorTypesView, + SensorsView, +) + +stat_data_router = routers.DefaultRouter() +stat_data_router.register(r"", SensorDataStatsView) data_router = routers.DefaultRouter() - -data_router.register(r"", SensorDataStatsView) +data_router.register(r"", SensorDataView) cities_router = routers.DefaultRouter() - cities_router.register(r"", CitiesView, basename="cities") nodes_router = routers.DefaultRouter() - nodes_router.register(r"", NodesView, basename="map") sensors_router = routers.DefaultRouter() @@ -25,7 +33,8 @@ sensor_types_router.register(r"", SensorTypesView, basename="sensor_types") api_urls = [ - url(r"data/(?P[air]+)/", include(data_router.urls)), + url(r"data/(?P[air]+)/", include(stat_data_router.urls)), + url(r"data/", include(data_router.urls)), url(r"cities/", include(cities_router.urls)), url(r"nodes/", include(nodes_router.urls)), url(r"locations/", include(sensor_locations_router.urls)), diff --git a/sensorsafrica/api/v2/views.py b/sensorsafrica/api/v2/views.py index abe712b..60891be 100644 --- a/sensorsafrica/api/v2/views.py +++ b/sensorsafrica/api/v2/views.py @@ -1,27 +1,25 @@ import datetime +import django_filters import pytz import json -from rest_framework.exceptions import ValidationError +from dateutil.relativedelta import relativedelta from django.conf import settings from django.utils import timezone -from dateutil.relativedelta import relativedelta from django.db.models import ExpressionWrapper, F, FloatField, Max, Min, Sum, Avg, Q from django.db.models.functions import Cast, TruncHour, TruncDay, TruncMonth -from rest_framework import mixins, pagination, viewsets +from django.utils.decorators import method_decorator +from django.utils.text import slugify +from django.views.decorators.cache import cache_page -from ..models import SensorDataStat, LastActiveNodes, City -from .serializers import ( - SensorDataStatSerializer, - CitySerializer, - NestedSensorTypeSerializer, - NodeSerializer, - SensorSerializer, - SensorLocationSerializer, -) +from rest_framework import mixins, pagination, viewsets +from rest_framework.authentication import SessionAuthentication, TokenAuthentication +from rest_framework.exceptions import ValidationError +from rest_framework.response import Response +from rest_framework.permissions import IsAuthenticated, AllowAny -from feinstaub.sensors.views import StandardResultsSetPagination +from feinstaub.sensors.views import SensorFilter, StandardResultsSetPagination from feinstaub.sensors.models import ( Node, @@ -32,14 +30,18 @@ SensorType, ) -from django.utils.text import slugify +from feinstaub.sensors.serializers import VerboseSensorDataSerializer -from rest_framework.response import Response -from rest_framework.authentication import SessionAuthentication, TokenAuthentication -from rest_framework.permissions import IsAuthenticated, AllowAny +from ..models import City, LastActiveNodes, SensorDataStat +from .serializers import ( + SensorDataStatSerializer, + CitySerializer, + NestedSensorTypeSerializer, + NodeSerializer, + SensorSerializer, + SensorLocationSerializer, +) -from django.utils.decorators import method_decorator -from django.views.decorators.cache import cache_page value_types = {"air": ["P1", "P2", "humidity", "temperature"]} @@ -116,94 +118,6 @@ def get_paginated_response(self, data_stats): ) -class SensorDataStatsView(mixins.ListModelMixin, viewsets.GenericViewSet): - queryset = SensorDataStat.objects.none() - serializer_class = SensorDataStatSerializer - pagination_class = CustomPagination - - @method_decorator(cache_page(3600)) - def dispatch(self, request, *args, **kwargs): - return super().dispatch(request, *args, **kwargs) - - def get_queryset(self): - sensor_type = self.kwargs["sensor_type"] - - city_slugs = self.request.query_params.get("city", None) - from_date = self.request.query_params.get("from", None) - to_date = self.request.query_params.get("to", None) - interval = self.request.query_params.get("interval", None) - - if to_date and not from_date: - raise ValidationError({"from": "Must be provide along with to query"}) - if from_date: - validate_date(from_date, {"from": "Must be a date in the format Y-m-d."}) - if to_date: - validate_date(to_date, {"to": "Must be a date in the format Y-m-d."}) - - value_type_to_filter = self.request.query_params.get("value_type", None) - - filter_value_types = value_types[sensor_type] - if value_type_to_filter: - filter_value_types = set(value_type_to_filter.upper().split(",")) & set( - [x.upper() for x in value_types[sensor_type]] - ) - - if not from_date and not to_date: - to_date = timezone.now().replace(minute=0, second=0, microsecond=0) - from_date = to_date - datetime.timedelta(hours=24) - interval = "day" if not interval else interval - elif not to_date: - from_date = beginning_of_day(from_date) - # Get data from_date until the end - # of day yesterday which is the beginning of today - to_date = beginning_of_today() - else: - from_date = beginning_of_day(from_date) - to_date = end_of_day(to_date) - - queryset = SensorDataStat.objects.filter( - value_type__in=filter_value_types, - timestamp__gte=from_date, - timestamp__lte=to_date, - ) - - if interval == "month": - truncate = TruncMonth("timestamp") - elif interval == "day": - truncate = TruncDay("timestamp") - else: - truncate = TruncHour("timestamp") - - if city_slugs: - queryset = queryset.filter(city_slug__in=city_slugs.split(",")) - - return ( - queryset.values("value_type", "city_slug") - .annotate( - truncated_timestamp=truncate, - start_datetime=Min("timestamp"), - end_datetime=Max("timestamp"), - calculated_average=ExpressionWrapper( - Sum(F("average") * F("sample_size")) / Sum("sample_size"), - output_field=FloatField(), - ), - calculated_minimum=Min("minimum"), - calculated_maximum=Max("maximum"), - ) - .values( - "value_type", - "city_slug", - "truncated_timestamp", - "start_datetime", - "end_datetime", - "calculated_average", - "calculated_minimum", - "calculated_maximum", - ) - .order_by("city_slug", "-truncated_timestamp") - ) - - class CitiesView(mixins.ListModelMixin, viewsets.GenericViewSet): queryset = City.objects.all() serializer_class = CitySerializer @@ -304,6 +218,130 @@ def create(self, request): return Response(serializer.errors, status=400) +class SensorDataPagination(pagination.CursorPagination): + cursor_query_param = "next_page" + ordering = "-timestamp" + page_size = 100 + + +class SensorDataView( + mixins.ListModelMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet +): + """This endpoint is to download sensor data from the api.""" + + authentication_classes = [SessionAuthentication, TokenAuthentication] + queryset = SensorData.objects.all() + pagination_class = SensorDataPagination + permission_classes = [IsAuthenticated] + filter_backends = (django_filters.rest_framework.DjangoFilterBackend,) + filter_class = SensorFilter + serializer_class = VerboseSensorDataSerializer + + def get_queryset(self): + if self.request.user.is_authenticated(): + if self.request.user.groups.filter(name="show_me_everything").exists(): + return SensorData.objects.all() + + # Return data from sensors owned or + # owned by someone in the same group as requesting user or + # public sensors + return SensorData.objects.filter( + Q(sensor__node__owner=self.request.user) + | Q(sensor__node__owner__groups__name__in=[g.name for g in self.request.user.groups.all()]) + | Q(sensor__public=True) + ) + + return SensorData.objects.filter(sensor__public=True) + + +class SensorDataStatsView(mixins.ListModelMixin, viewsets.GenericViewSet): + queryset = SensorDataStat.objects.none() + serializer_class = SensorDataStatSerializer + pagination_class = CustomPagination + + @method_decorator(cache_page(3600)) + def dispatch(self, request, *args, **kwargs): + return super().dispatch(request, *args, **kwargs) + + def get_queryset(self): + sensor_type = self.kwargs["sensor_type"] + + city_slugs = self.request.query_params.get("city", None) + from_date = self.request.query_params.get("from", None) + to_date = self.request.query_params.get("to", None) + interval = self.request.query_params.get("interval", None) + + if to_date and not from_date: + raise ValidationError({"from": "Must be provide along with to query"}) + if from_date: + validate_date(from_date, {"from": "Must be a date in the format Y-m-d."}) + if to_date: + validate_date(to_date, {"to": "Must be a date in the format Y-m-d."}) + + value_type_to_filter = self.request.query_params.get("value_type", None) + + filter_value_types = value_types[sensor_type] + if value_type_to_filter: + filter_value_types = set(value_type_to_filter.upper().split(",")) & set( + [x.upper() for x in value_types[sensor_type]] + ) + + if not from_date and not to_date: + to_date = timezone.now().replace(minute=0, second=0, microsecond=0) + from_date = to_date - datetime.timedelta(hours=24) + interval = "day" if not interval else interval + elif not to_date: + from_date = beginning_of_day(from_date) + # Get data from_date until the end + # of day yesterday which is the beginning of today + to_date = beginning_of_today() + else: + from_date = beginning_of_day(from_date) + to_date = end_of_day(to_date) + + queryset = SensorDataStat.objects.filter( + value_type__in=filter_value_types, + timestamp__gte=from_date, + timestamp__lte=to_date, + ) + + if interval == "month": + truncate = TruncMonth("timestamp") + elif interval == "day": + truncate = TruncDay("timestamp") + else: + truncate = TruncHour("timestamp") + + if city_slugs: + queryset = queryset.filter(city_slug__in=city_slugs.split(",")) + + return ( + queryset.values("value_type", "city_slug") + .annotate( + truncated_timestamp=truncate, + start_datetime=Min("timestamp"), + end_datetime=Max("timestamp"), + calculated_average=ExpressionWrapper( + Sum(F("average") * F("sample_size")) / Sum("sample_size"), + output_field=FloatField(), + ), + calculated_minimum=Min("minimum"), + calculated_maximum=Max("maximum"), + ) + .values( + "value_type", + "city_slug", + "truncated_timestamp", + "start_datetime", + "end_datetime", + "calculated_average", + "calculated_minimum", + "calculated_maximum", + ) + .order_by("city_slug", "-truncated_timestamp") + ) + + class SensorLocationsView(viewsets.ViewSet): authentication_classes = [SessionAuthentication, TokenAuthentication] permission_classes = [IsAuthenticated] @@ -323,26 +361,18 @@ def create(self, request): return Response(serializer.errors, status=400) -class SensorsView(viewsets.ViewSet): +class SensorTypesView(viewsets.ViewSet): authentication_classes = [SessionAuthentication, TokenAuthentication] permission_classes = [IsAuthenticated] pagination_class = StandardResultsSetPagination - def get_permissions(self): - if self.action == "create": - permission_classes = [IsAuthenticated] - else: - permission_classes = [AllowAny] - - return [permission() for permission in permission_classes] - def list(self, request): - queryset = Sensor.objects.all() - serializer = SensorSerializer(queryset, many=True) + queryset = SensorType.objects.all() + serializer = NestedSensorTypeSerializer(queryset, many=True) return Response(serializer.data) def create(self, request): - serializer = SensorSerializer(data=request.data) + serializer = NestedSensorTypeSerializer(data=request.data) if serializer.is_valid(): serializer.save() return Response(serializer.data, status=201) @@ -350,18 +380,26 @@ def create(self, request): return Response(serializer.errors, status=400) -class SensorTypesView(viewsets.ViewSet): +class SensorsView(viewsets.ViewSet): authentication_classes = [SessionAuthentication, TokenAuthentication] permission_classes = [IsAuthenticated] pagination_class = StandardResultsSetPagination + def get_permissions(self): + if self.action == "create": + permission_classes = [IsAuthenticated] + else: + permission_classes = [AllowAny] + + return [permission() for permission in permission_classes] + def list(self, request): - queryset = SensorType.objects.all() - serializer = NestedSensorTypeSerializer(queryset, many=True) + queryset = Sensor.objects.all() + serializer = SensorSerializer(queryset, many=True) return Response(serializer.data) def create(self, request): - serializer = NestedSensorTypeSerializer(data=request.data) + serializer = SensorSerializer(data=request.data) if serializer.is_valid(): serializer.save() return Response(serializer.data, status=201) diff --git a/sensorsafrica/management/commands/calculate_data_statistics.py b/sensorsafrica/management/commands/calculate_data_statistics.py index 6a9d9db..d96a38f 100644 --- a/sensorsafrica/management/commands/calculate_data_statistics.py +++ b/sensorsafrica/management/commands/calculate_data_statistics.py @@ -1,11 +1,12 @@ from django.core.management import BaseCommand +from django.core.paginator import Paginator from django.db.models import Avg, Count, FloatField, Max, Min, Q from django.db.models.functions import Cast, TruncHour from django.utils.text import slugify + from feinstaub.sensors.models import Node, Sensor, SensorDataValue, SensorLocation -from ...api.models import SensorDataStat -from django.core.paginator import Paginator +from sensorsafrica.api.models import SensorDataStat def map_stat(stat, city): @@ -55,6 +56,8 @@ def handle(self, *args, **options): if last_date_time: queryset = SensorDataValue.objects.filter( + # Pick data from public sensors only + Q(sensordata__sensor__public=True), Q(sensordata__location__city__iexact=city), # Get dates greater than last stat calculation Q(created__gt=last_date_time), @@ -65,6 +68,8 @@ def handle(self, *args, **options): ) else: queryset = SensorDataValue.objects.filter( + # Pick data from public sensors only + Q(sensordata__sensor__public=True), Q(sensordata__location__city__iexact=city), # Ignore timestamp values ~Q(value_type="timestamp"), @@ -74,27 +79,28 @@ def handle(self, *args, **options): for stats in chunked_iterator( queryset.annotate(timestamp=TruncHour("created")) - .values( - "timestamp", - "value_type", - "sensordata__sensor", - "sensordata__location", - "sensordata__sensor__node", - ) - .order_by() - .annotate( - last_datetime=Max("created"), - average=Avg(Cast("value", FloatField())), - minimum=Min(Cast("value", FloatField())), - maximum=Max(Cast("value", FloatField())), - sample_size=Count("created", FloatField()), - ) - .filter( - ~Q(average=float("NaN")), - ~Q(minimum=float("NaN")), - ~Q(maximum=float("NaN")), - ) - .order_by("timestamp")): + .values( + "timestamp", + "value_type", + "sensordata__sensor", + "sensordata__location", + "sensordata__sensor__node", + ) + .order_by() + .annotate( + last_datetime=Max("created"), + average=Avg(Cast("value", FloatField())), + minimum=Min(Cast("value", FloatField())), + maximum=Max(Cast("value", FloatField())), + sample_size=Count("created", FloatField()), + ) + .filter( + ~Q(average=float("NaN")), + ~Q(minimum=float("NaN")), + ~Q(maximum=float("NaN")), + ) + .order_by("timestamp") + ): SensorDataStat.objects.bulk_create( list(map(lambda stat: map_stat(stat, city), stats)) ) From 997d0c356f5472c682a93e205619b2e20ce143f0 Mon Sep 17 00:00:00 2001 From: _ Kilemensi Date: Tue, 15 Dec 2020 12:31:56 +0300 Subject: [PATCH 06/29] Add default AUTHENTICATION_CLASSES (#82) Feinstaub framework doesn't set per view authentication classes and hence we need default ones --- sensorsafrica/settings.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/sensorsafrica/settings.py b/sensorsafrica/settings.py index defbb24..ecde80b 100644 --- a/sensorsafrica/settings.py +++ b/sensorsafrica/settings.py @@ -34,9 +34,7 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = os.getenv("SENSORSAFRICA_DEBUG", "True") == "True" -ALLOWED_HOSTS = os.getenv( - "SENSORSAFRICA_ALLOWED_HOSTS", "*" -).split(",") +ALLOWED_HOSTS = os.getenv("SENSORSAFRICA_ALLOWED_HOSTS", "*").split(",") CORS_ORIGIN_ALLOW_ALL = True @@ -60,7 +58,7 @@ "feinstaub.sensors", # API "sensorsafrica", - 'corsheaders', + "corsheaders", ] MIDDLEWARE = [ @@ -77,6 +75,14 @@ ROOT_URLCONF = "sensorsafrica.urls" +REST_FRAMEWORK = { + "DEFAULT_AUTHENTICATION_CLASSES": ( + "rest_framework.authentication.TokenAuthentication", + "rest_framework.authentication.SessionAuthentication", + ), + "DEFAULT_FILTER_BACKENDS": ("django_filters.rest_framework.DjangoFilterBackend",), +} + TEMPLATES = [ { "BACKEND": "django.template.backends.django.DjangoTemplates", @@ -150,29 +156,30 @@ # Celery Broker CELERY_BROKER_URL = os.environ.get( - "SENSORSAFRICA_RABBITMQ_URL", "amqp://sensorsafrica:sensorsafrica@localhost//") + "SENSORSAFRICA_RABBITMQ_URL", "amqp://sensorsafrica:sensorsafrica@localhost//" +) CELERY_IGNORE_RESULT = True CELERY_BEAT_SCHEDULE = { "statistics-task": { "task": "sensorsafrica.tasks.calculate_data_statistics", - "schedule": crontab(hour="*", minute=0) + "schedule": crontab(hour="*", minute=0), }, "archive-task": { "task": "sensorsafrica.tasks.archive_data", - "schedule": crontab(hour="*", minute=0) + "schedule": crontab(hour="*", minute=0), }, "cache-lastactive-nodes-task": { "task": "sensorsafrica.tasks.cache_lastactive_nodes", - "schedule": crontab(minute="*/5") + "schedule": crontab(minute="*/5"), }, "cache-static-json-data": { "task": "sensorsafrica.tasks.cache_static_json_data", - "schedule": crontab(minute="*/5") + "schedule": crontab(minute="*/5"), }, "cache-static-json-data-1h-24h": { "task": "sensorsafrica.tasks.cache_static_json_data_1h_24h", - "schedule": crontab(hour="*", minute=0) + "schedule": crontab(hour="*", minute=0), }, } @@ -186,5 +193,5 @@ # Put fenstaub migrations into sensorsafrica MIGRATION_MODULES = { - 'sensors': 'sensorsafrica.openstuttgart.feinstaub.sensors.migrations' + "sensors": "sensorsafrica.openstuttgart.feinstaub.sensors.migrations" } From 7a8b68c41e36e9e05ab75956179ff4f6ec683fb9 Mon Sep 17 00:00:00 2001 From: _ Kilemensi Date: Tue, 15 Dec 2020 18:01:35 +0300 Subject: [PATCH 07/29] [Fix] serializer (#83) * Introduce SensorTypeSerializer that will handle uid * Use SensorTypeSerializer --- sensorsafrica/api/v2/serializers.py | 9 ++++++++- sensorsafrica/api/v2/views.py | 6 +++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/sensorsafrica/api/v2/serializers.py b/sensorsafrica/api/v2/serializers.py index d875603..80e9c16 100644 --- a/sensorsafrica/api/v2/serializers.py +++ b/sensorsafrica/api/v2/serializers.py @@ -3,7 +3,7 @@ NestedSensorLocationSerializer, NestedSensorTypeSerializer, ) -from feinstaub.sensors.models import Node, Sensor +from feinstaub.sensors.models import Node, Sensor, SensorType class SensorDataStatSerializer(serializers.Serializer): @@ -52,10 +52,17 @@ class Meta(NestedSensorLocationSerializer.Meta): ) +class SensorTypeSerializer(serializers.ModelSerializer): + class Meta: + model = SensorType + fields = ("id", "uid", "name", "manufacturer") + + class NodeSerializer(serializers.ModelSerializer): class Meta: model = Node fields = ( + "id", "uid", "owner", "location", diff --git a/sensorsafrica/api/v2/views.py b/sensorsafrica/api/v2/views.py index 60891be..cd2f403 100644 --- a/sensorsafrica/api/v2/views.py +++ b/sensorsafrica/api/v2/views.py @@ -36,7 +36,7 @@ from .serializers import ( SensorDataStatSerializer, CitySerializer, - NestedSensorTypeSerializer, + SensorTypeSerializer, NodeSerializer, SensorSerializer, SensorLocationSerializer, @@ -368,11 +368,11 @@ class SensorTypesView(viewsets.ViewSet): def list(self, request): queryset = SensorType.objects.all() - serializer = NestedSensorTypeSerializer(queryset, many=True) + serializer = SensorTypeSerializer(queryset, many=True) return Response(serializer.data) def create(self, request): - serializer = NestedSensorTypeSerializer(data=request.data) + serializer = SensorTypeSerializer(data=request.data) if serializer.is_valid(): serializer.save() return Response(serializer.data, status=201) From eabf492356cccd9d91809e39f53305910050895c Mon Sep 17 00:00:00 2001 From: _ Kilemensi Date: Wed, 13 Jan 2021 14:43:08 +0300 Subject: [PATCH 08/29] Ensure city and data exists before trying to upload (#85) --- .../management/commands/upload_to_ckan.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/sensorsafrica/management/commands/upload_to_ckan.py b/sensorsafrica/management/commands/upload_to_ckan.py index 0a45a2f..de3ccec 100644 --- a/sensorsafrica/management/commands/upload_to_ckan.py +++ b/sensorsafrica/management/commands/upload_to_ckan.py @@ -28,7 +28,15 @@ def handle(self, *args, **options): .distinct("city") ) for city in city_queryset.iterator(): - if not city: + # Ensure we have a city + if not city or city.isspace(): + continue + + # Ensure city has actual data we can upload + timestamp = SensorData.objects.filter(location__city=city).aggregate( + Max("timestamp"), Min("timestamp") + ) + if not timestamp or not timestamp['timestamp__min'] or not timestamp['timestamp__max']: continue try: @@ -53,14 +61,11 @@ def handle(self, *args, **options): if not start_date or date > start_date: start_date = date - timestamp = SensorData.objects.filter(location__city=city).aggregate( - Max("timestamp"), Min("timestamp") - ) - - if not start_date and "timestamp__min" in timestamp and timestamp["timestamp__min"] is not None: + if not start_date: start_date = timestamp["timestamp__min"].replace( day=1, hour=0, minute=0, second=0, microsecond=0 ) + end_date = timestamp["timestamp__max"].replace( day=1, hour=0, minute=0, second=0, microsecond=0 ) From 7a778ec6c058ba3283e2eecfc1e417893cdfdc7d Mon Sep 17 00:00:00 2001 From: _ Kilemensi Date: Thu, 21 Jan 2021 08:57:21 +0300 Subject: [PATCH 09/29] Upload data from public sensors only (#87) --- sensorsafrica/management/commands/upload_to_ckan.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sensorsafrica/management/commands/upload_to_ckan.py b/sensorsafrica/management/commands/upload_to_ckan.py index de3ccec..3b0a95c 100644 --- a/sensorsafrica/management/commands/upload_to_ckan.py +++ b/sensorsafrica/management/commands/upload_to_ckan.py @@ -77,6 +77,7 @@ def handle(self, *args, **options): while date <= end_date: qs = ( SensorData.objects.filter( + sensor__public=True, location__city=city, timestamp__month=date.month, timestamp__year=date.year, From 0339e4c072534e5aedbae755c81e3f245b4fcfe4 Mon Sep 17 00:00:00 2001 From: _ Kilemensi Date: Thu, 21 Jan 2021 13:17:13 +0300 Subject: [PATCH 10/29] [Feature] Make v1/now return public data only (#86) * Recreate feinstaub NowView but with public sensors filter * Switch to custom NowView --- sensorsafrica/api/v1/router.py | 3 +- sensorsafrica/api/v1/views.py | 56 +++++++++++++++++++++------------- 2 files changed, 36 insertions(+), 23 deletions(-) diff --git a/sensorsafrica/api/v1/router.py b/sensorsafrica/api/v1/router.py index d69f7cc..5140f78 100644 --- a/sensorsafrica/api/v1/router.py +++ b/sensorsafrica/api/v1/router.py @@ -2,14 +2,13 @@ from feinstaub.main.views import UsersView from feinstaub.sensors.views import ( NodeView, - NowView, PostSensorDataView, SensorView, StatisticsView, SensorDataView, ) -from .views import SensorDataView as SensorsAfricaSensorDataView, FilterView +from .views import FilterView, NowView, SensorDataView as SensorsAfricaSensorDataView from rest_framework import routers diff --git a/sensorsafrica/api/v1/views.py b/sensorsafrica/api/v1/views.py index 8e00237..60acccb 100644 --- a/sensorsafrica/api/v1/views.py +++ b/sensorsafrica/api/v1/views.py @@ -2,49 +2,63 @@ import pytz import json -from rest_framework.exceptions import ValidationError from django.conf import settings -from django.utils import timezone -from dateutil.relativedelta import relativedelta from django.db.models import ExpressionWrapper, F, FloatField, Max, Min, Sum, Avg, Q from django.db.models.functions import Cast, TruncDate +from dateutil.relativedelta import relativedelta +from django.utils import timezone from rest_framework import mixins, pagination, viewsets +from rest_framework.exceptions import ValidationError -from .serializers import SensorDataSerializer from feinstaub.sensors.models import SensorData +from feinstaub.sensors.serializers import NowSerializer +from .serializers import SensorDataSerializer -class SensorDataView(mixins.ListModelMixin, viewsets.GenericViewSet): +class FilterView(mixins.ListModelMixin, viewsets.GenericViewSet): serializer_class = SensorDataSerializer def get_queryset(self): + sensor_type = self.request.GET.get("type", r"\w+") + country = self.request.GET.get("country", r"\w+") + city = self.request.GET.get("city", r"\w+") return ( - SensorData.objects - .filter( + SensorData.objects.filter( timestamp__gte=timezone.now() - datetime.timedelta(minutes=5), - sensor=self.kwargs["sensor_id"] + sensor__sensor_type__uid__iregex=sensor_type, + location__country__iregex=country, + location__city__iregex=city, ) - .only('sensor', 'timestamp') - .prefetch_related('sensordatavalues') + .only("sensor", "timestamp") + .prefetch_related("sensordatavalues") ) -class FilterView(mixins.ListModelMixin, viewsets.GenericViewSet): +class NowView(mixins.ListModelMixin, viewsets.GenericViewSet): + """Show all public sensors active in the last 5 minutes with newest value""" + + permission_classes = [] + serializer_class = NowSerializer + queryset = SensorData.objects.none() + + def get_queryset(self): + now = timezone.now() + startdate = now - datetime.timedelta(minutes=5) + return SensorData.objects.filter( + sensor__public=True, modified__range=[startdate, now] + ) + + +class SensorDataView(mixins.ListModelMixin, viewsets.GenericViewSet): serializer_class = SensorDataSerializer def get_queryset(self): - sensor_type = self.request.GET.get('type', r'\w+') - country = self.request.GET.get('country', r'\w+') - city = self.request.GET.get('city', r'\w+') return ( - SensorData.objects - .filter( + SensorData.objects.filter( timestamp__gte=timezone.now() - datetime.timedelta(minutes=5), - sensor__sensor_type__uid__iregex=sensor_type, - location__country__iregex=country, - location__city__iregex=city + sensor=self.kwargs["sensor_id"], ) - .only('sensor', 'timestamp') - .prefetch_related('sensordatavalues') + .only("sensor", "timestamp") + .prefetch_related("sensordatavalues") ) From 848c261882dd8d47cb9d9e95c43ce880bcc000a9 Mon Sep 17 00:00:00 2001 From: _ Kilemensi Date: Fri, 22 Jan 2021 15:55:23 +0300 Subject: [PATCH 11/29] Add owner as part of node (#89) --- sensorsafrica/api/v2/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sensorsafrica/api/v2/views.py b/sensorsafrica/api/v2/views.py index cd2f403..ad5bacf 100644 --- a/sensorsafrica/api/v2/views.py +++ b/sensorsafrica/api/v2/views.py @@ -193,7 +193,7 @@ def list(self, request): { "node_moved": moved_to is not None, "moved_to": moved_to, - "node": {"uid": last_active.node.uid, "id": last_active.node.id}, + "node": {"uid": last_active.node.uid, "id": last_active.node.id, "owner": last_active.node.owner}, "location": { "name": last_active.location.location, "longitude": last_active.location.longitude, From 3aa3e128ccbc36b0031a3ba9b335672616a6c498 Mon Sep 17 00:00:00 2001 From: _ Kilemensi Date: Fri, 22 Jan 2021 16:47:09 +0300 Subject: [PATCH 12/29] [Fix] v1/node should check groups (#88) * Add NodeView that checks group ownership + pagination * Update router to use new NodeView --- sensorsafrica/api/v1/router.py | 13 ++++++++---- sensorsafrica/api/v1/views.py | 36 ++++++++++++++++++++++++++++++++-- 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/sensorsafrica/api/v1/router.py b/sensorsafrica/api/v1/router.py index 5140f78..88901a3 100644 --- a/sensorsafrica/api/v1/router.py +++ b/sensorsafrica/api/v1/router.py @@ -1,14 +1,18 @@ # The base version is entirely based on feinstaub from feinstaub.main.views import UsersView from feinstaub.sensors.views import ( - NodeView, PostSensorDataView, SensorView, StatisticsView, SensorDataView, ) -from .views import FilterView, NowView, SensorDataView as SensorsAfricaSensorDataView +from .views import ( + FilterView, + NodeView, + NowView, + SensorDataView as SensorsAfricaSensorDataView, +) from rest_framework import routers @@ -20,8 +24,9 @@ router.register(r"statistics", StatisticsView, basename="statistics") router.register(r"now", NowView) router.register(r"user", UsersView) -router.register(r"sensors/(?P\d+)", - SensorsAfricaSensorDataView, basename="sensors") +router.register( + r"sensors/(?P\d+)", SensorsAfricaSensorDataView, basename="sensors" +) router.register(r"filter", FilterView, basename="filter") api_urls = router.urls diff --git a/sensorsafrica/api/v1/views.py b/sensorsafrica/api/v1/views.py index 60acccb..53d9921 100644 --- a/sensorsafrica/api/v1/views.py +++ b/sensorsafrica/api/v1/views.py @@ -9,13 +9,17 @@ from dateutil.relativedelta import relativedelta from django.utils import timezone from rest_framework import mixins, pagination, viewsets +from rest_framework.authentication import SessionAuthentication, TokenAuthentication from rest_framework.exceptions import ValidationError +from rest_framework.permissions import IsAuthenticatedOrReadOnly -from feinstaub.sensors.models import SensorData -from feinstaub.sensors.serializers import NowSerializer +from feinstaub.sensors.models import Node, SensorData +from feinstaub.sensors.serializers import NodeSerializer, NowSerializer +from feinstaub.sensors.views import StandardResultsSetPagination from .serializers import SensorDataSerializer + class FilterView(mixins.ListModelMixin, viewsets.GenericViewSet): serializer_class = SensorDataSerializer @@ -35,6 +39,34 @@ def get_queryset(self): ) +class NodeView( + mixins.ListModelMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet +): + """Show all nodes belonging to authenticated user""" + + authentication_classes = [SessionAuthentication, TokenAuthentication] + pagination_class = StandardResultsSetPagination + permission_classes = [IsAuthenticatedOrReadOnly] + queryset = SensorData.objects.none() + serializer_class = NodeSerializer + + def get_queryset(self): + if self.request.user.is_authenticated(): + if self.request.user.groups.filter(name="show_me_everything").exists(): + return Node.objects.all() + + return Node.objects.filter( + Q(owner=self.request.user) + | Q( + owner__groups__name__in=[ + g.name for g in self.request.user.groups.all() + ] + ) + ) + + return Node.objects.none() + + class NowView(mixins.ListModelMixin, viewsets.GenericViewSet): """Show all public sensors active in the last 5 minutes with newest value""" From ce0bdb490a1055d3f481d268b3c7590a65e450c3 Mon Sep 17 00:00:00 2001 From: Isaiah King'ori Date: Fri, 22 Jan 2021 17:40:12 +0300 Subject: [PATCH 13/29] Use user id instead of user object (#90) --- sensorsafrica/api/v2/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sensorsafrica/api/v2/views.py b/sensorsafrica/api/v2/views.py index ad5bacf..52bab1a 100644 --- a/sensorsafrica/api/v2/views.py +++ b/sensorsafrica/api/v2/views.py @@ -193,7 +193,7 @@ def list(self, request): { "node_moved": moved_to is not None, "moved_to": moved_to, - "node": {"uid": last_active.node.uid, "id": last_active.node.id, "owner": last_active.node.owner}, + "node": {"uid": last_active.node.uid, "id": last_active.node.id, "owner": last_active.node.owner.id}, "location": { "name": last_active.location.location, "longitude": last_active.location.longitude, From 21ad544711ced1740c66f2f756645e40426aeeb1 Mon Sep 17 00:00:00 2001 From: Isaiah King'ori Date: Wed, 27 Jan 2021 14:40:51 +0300 Subject: [PATCH 14/29] Adds endpoint for providing metadata (#91) * Adds endpoint for providing metadata * Use dynamic database name * Returns sensors locations and last time database was updated. * Adds sensor networks to returned metadata * Adds authentication to the meta endpoint * Remove a default NETWORKS_OWNER * Use latest created SensorDataValue to know when the database was last updated. * Ensure we only return SensorLocations with a country --- sensorsafrica/api/v2/router.py | 2 ++ sensorsafrica/api/v2/views.py | 46 ++++++++++++++++++++++++++++++++++ sensorsafrica/settings.py | 2 ++ 3 files changed, 50 insertions(+) diff --git a/sensorsafrica/api/v2/router.py b/sensorsafrica/api/v2/router.py index d0b9677..46f4bc4 100644 --- a/sensorsafrica/api/v2/router.py +++ b/sensorsafrica/api/v2/router.py @@ -9,6 +9,7 @@ SensorLocationsView, SensorTypesView, SensorsView, + meta_data, ) stat_data_router = routers.DefaultRouter() @@ -40,4 +41,5 @@ url(r"locations/", include(sensor_locations_router.urls)), url(r"sensors/", include(sensors_router.urls)), url(r"sensor-types/", include(sensor_types_router.urls)), + url(r"meta/", meta_data), ] diff --git a/sensorsafrica/api/v2/views.py b/sensorsafrica/api/v2/views.py index 52bab1a..b430a2c 100644 --- a/sensorsafrica/api/v2/views.py +++ b/sensorsafrica/api/v2/views.py @@ -5,8 +5,10 @@ from dateutil.relativedelta import relativedelta +from django.contrib.auth.models import User from django.conf import settings from django.utils import timezone +from django.db import connection from django.db.models import ExpressionWrapper, F, FloatField, Max, Min, Sum, Avg, Q from django.db.models.functions import Cast, TruncHour, TruncDay, TruncMonth from django.utils.decorators import method_decorator @@ -18,6 +20,7 @@ from rest_framework.exceptions import ValidationError from rest_framework.response import Response from rest_framework.permissions import IsAuthenticated, AllowAny +from rest_framework.decorators import api_view, authentication_classes from feinstaub.sensors.views import SensorFilter, StandardResultsSetPagination @@ -405,3 +408,46 @@ def create(self, request): return Response(serializer.data, status=201) return Response(serializer.errors, status=400) + + +@api_view(['GET']) +@authentication_classes([SessionAuthentication, TokenAuthentication]) +def meta_data(request): + nodes_count = Node.objects.count() + sensors_count = Sensor.objects.count() + sensor_data_count = SensorData.objects.count() + + database_size = get_database_size() + database_last_updated = get_database_last_updated() + sensors_locations = get_sensors_locations() + + return Response({ + "sensor_networks": get_sensors_networks(), + "nodes_count": nodes_count, + "sensors_count": sensors_count, + "sensor_data_count": sensor_data_count, + "sensors_locations": sensors_locations, + "database_size": database_size[0], + "database_last_updated": database_last_updated, + }) + +def get_sensors_networks(): + user = User.objects.filter(username=settings.NETWORKS_OWNER).first() + if user: + networks = list(user.groups.values_list('name', flat=True)) + networks.append("sensors.AFRICA") + return {"networks": networks, "count": len(networks)} + +def get_sensors_locations(): + sensor_locations = SensorLocation.objects.filter(country__isnull=False).values_list('country', flat=True) + return set(sensor_locations) + +def get_database_size(): + with connection.cursor() as c: + c.execute(f"SELECT pg_size_pretty(pg_database_size('{connection.settings_dict['NAME']}'))") + return c.fetchall() + +def get_database_last_updated(): + sensor_data_value = SensorDataValue.objects.latest('created') + if sensor_data_value: + return sensor_data_value.modified diff --git a/sensorsafrica/settings.py b/sensorsafrica/settings.py index ecde80b..d18aabc 100644 --- a/sensorsafrica/settings.py +++ b/sensorsafrica/settings.py @@ -195,3 +195,5 @@ MIGRATION_MODULES = { "sensors": "sensorsafrica.openstuttgart.feinstaub.sensors.migrations" } + +NETWORKS_OWNER = os.getenv("NETWORKS_OWNER") From caf175b54d42f8194300d3506e1ff12677ad43e7 Mon Sep 17 00:00:00 2001 From: Isaiah King'ori Date: Wed, 27 Jan 2021 16:04:05 +0300 Subject: [PATCH 15/29] Ft increase gurnicorn timeout (#92) * Increase Gunicorn timeout to 3 minutes * Return sorted sensors locations --- contrib/start.sh | 3 ++- sensorsafrica/api/v2/views.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/contrib/start.sh b/contrib/start.sh index eb8b924..c8dfd08 100755 --- a/contrib/start.sh +++ b/contrib/start.sh @@ -16,7 +16,8 @@ celery -A sensorsafrica flower --basic_auth=$SENSORSAFRICA_FLOWER_ADMIN_USERNAME echo Starting Gunicorn. exec gunicorn \ --bind 0.0.0.0:8000 \ - --workers 3 \ + --timeout 180 \ + --workers 5 \ --worker-class gevent \ --log-level=info \ --log-file=/src/logs/gunicorn.log \ diff --git a/sensorsafrica/api/v2/views.py b/sensorsafrica/api/v2/views.py index b430a2c..030cd9f 100644 --- a/sensorsafrica/api/v2/views.py +++ b/sensorsafrica/api/v2/views.py @@ -440,7 +440,7 @@ def get_sensors_networks(): def get_sensors_locations(): sensor_locations = SensorLocation.objects.filter(country__isnull=False).values_list('country', flat=True) - return set(sensor_locations) + return sorted(set(sensor_locations)) def get_database_size(): with connection.cursor() as c: From 13a065fd939d87593bebd739e4d7517d3def93a7 Mon Sep 17 00:00:00 2001 From: Isaiah King'ori Date: Thu, 28 Jan 2021 03:46:53 +0300 Subject: [PATCH 16/29] Includes country in the sensor data serializer (#93) * Includes country in the sensor data serializer * Only return location id and country --- sensorsafrica/api/v2/serializers.py | 11 ++++++++++- sensorsafrica/api/v2/views.py | 4 ++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/sensorsafrica/api/v2/serializers.py b/sensorsafrica/api/v2/serializers.py index 80e9c16..f909cb6 100644 --- a/sensorsafrica/api/v2/serializers.py +++ b/sensorsafrica/api/v2/serializers.py @@ -3,7 +3,8 @@ NestedSensorLocationSerializer, NestedSensorTypeSerializer, ) -from feinstaub.sensors.models import Node, Sensor, SensorType +from feinstaub.sensors.models import Node, Sensor, SensorType, SensorLocation +from feinstaub.sensors.serializers import (VerboseSensorDataSerializer, ) class SensorDataStatSerializer(serializers.Serializer): @@ -76,3 +77,11 @@ class Meta: "inactive", "exact_location", ) + +class SensorDataSensorLocationSerializer(serializers.ModelSerializer): + class Meta: + model = SensorLocation + fields = ('id', "country", ) + +class SensorDataSerializer(VerboseSensorDataSerializer): + location = SensorDataSensorLocationSerializer() diff --git a/sensorsafrica/api/v2/views.py b/sensorsafrica/api/v2/views.py index 030cd9f..f4e7272 100644 --- a/sensorsafrica/api/v2/views.py +++ b/sensorsafrica/api/v2/views.py @@ -33,7 +33,6 @@ SensorType, ) -from feinstaub.sensors.serializers import VerboseSensorDataSerializer from ..models import City, LastActiveNodes, SensorDataStat from .serializers import ( @@ -43,6 +42,7 @@ NodeSerializer, SensorSerializer, SensorLocationSerializer, + SensorDataSerializer, ) @@ -238,7 +238,7 @@ class SensorDataView( permission_classes = [IsAuthenticated] filter_backends = (django_filters.rest_framework.DjangoFilterBackend,) filter_class = SensorFilter - serializer_class = VerboseSensorDataSerializer + serializer_class = SensorDataSerializer def get_queryset(self): if self.request.user.is_authenticated(): From 92d83113163e1412edbd247dd37099c785deae75 Mon Sep 17 00:00:00 2001 From: Isaiah King'ori Date: Mon, 1 Feb 2021 14:28:43 +0300 Subject: [PATCH 17/29] Ft add location filter (#95) * Adds a nodes filter using location * Adds nodes filter for v1 of the API * Remove filterling on v2 since it's only used for the F.E map * Use default django filter backend * Makes country location matching case insensitive. --- sensorsafrica/api/v1/filters.py | 9 +++++++++ sensorsafrica/api/v1/views.py | 4 +++- 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 sensorsafrica/api/v1/filters.py diff --git a/sensorsafrica/api/v1/filters.py b/sensorsafrica/api/v1/filters.py new file mode 100644 index 0000000..d525dd2 --- /dev/null +++ b/sensorsafrica/api/v1/filters.py @@ -0,0 +1,9 @@ +import django_filters + +from feinstaub.sensors.models import Node + +class NodeFilter(django_filters.FilterSet): + class Meta: + model = Node + fields = {"location__country": ["iexact"]} + diff --git a/sensorsafrica/api/v1/views.py b/sensorsafrica/api/v1/views.py index 53d9921..a51aecc 100644 --- a/sensorsafrica/api/v1/views.py +++ b/sensorsafrica/api/v1/views.py @@ -1,6 +1,7 @@ import datetime import pytz import json +import django_filters from django.conf import settings @@ -18,7 +19,7 @@ from feinstaub.sensors.views import StandardResultsSetPagination from .serializers import SensorDataSerializer - +from .filters import NodeFilter class FilterView(mixins.ListModelMixin, viewsets.GenericViewSet): serializer_class = SensorDataSerializer @@ -49,6 +50,7 @@ class NodeView( permission_classes = [IsAuthenticatedOrReadOnly] queryset = SensorData.objects.none() serializer_class = NodeSerializer + filter_class = NodeFilter def get_queryset(self): if self.request.user.is_authenticated(): From 67acd6553413f81d52ebf3b16d560e8b24f8c756 Mon Sep 17 00:00:00 2001 From: Khadija Mahanga Date: Mon, 1 Feb 2021 14:38:44 +0300 Subject: [PATCH 18/29] [Ft] Nodes last notify (#94) * update node last_notify * update nodeserializer * import from feinstaub * class naming * code refactor * remove repeating line * comment line * Update sensorsafrica/api/v1/serializers.py Co-authored-by: _ Kilemensi * rename verbose to lastnotify * update only when current notify is less than sensordata timestamp Co-authored-by: _ Kilemensi --- sensorsafrica/api/v1/router.py | 2 +- sensorsafrica/api/v1/serializers.py | 29 +++++++++++++++++++++++++++++ sensorsafrica/api/v1/views.py | 15 +++++++++++++-- 3 files changed, 43 insertions(+), 3 deletions(-) diff --git a/sensorsafrica/api/v1/router.py b/sensorsafrica/api/v1/router.py index 88901a3..b90b3b0 100644 --- a/sensorsafrica/api/v1/router.py +++ b/sensorsafrica/api/v1/router.py @@ -1,7 +1,6 @@ # The base version is entirely based on feinstaub from feinstaub.main.views import UsersView from feinstaub.sensors.views import ( - PostSensorDataView, SensorView, StatisticsView, SensorDataView, @@ -11,6 +10,7 @@ FilterView, NodeView, NowView, + PostSensorDataView, SensorDataView as SensorsAfricaSensorDataView, ) diff --git a/sensorsafrica/api/v1/serializers.py b/sensorsafrica/api/v1/serializers.py index 252d45d..1aba22d 100644 --- a/sensorsafrica/api/v1/serializers.py +++ b/sensorsafrica/api/v1/serializers.py @@ -1,10 +1,23 @@ from rest_framework import serializers from feinstaub.sensors.models import ( + Node, SensorData, SensorDataValue, SensorLocation ) +from feinstaub.sensors.serializers import ( + NestedSensorLocationSerializer, + NestedSensorSerializer, + SensorDataSerializer as PostSensorDataSerializer +) + +class NodeSerializer(serializers.ModelSerializer): + sensors = NestedSensorSerializer(many=True) + location = NestedSensorLocationSerializer() + class Meta: + model = Node + fields = ('id', 'sensors', 'uid', 'owner', 'location', 'last_notify') class SensorLocationSerializer(serializers.ModelSerializer): class Meta: @@ -25,3 +38,19 @@ class SensorDataSerializer(serializers.ModelSerializer): class Meta: model = SensorData fields = ['location', 'timestamp', 'sensordatavalues'] + +class LastNotifySensorDataSerializer(PostSensorDataSerializer): + + def create(self, validated_data): + sd = super().create(validated_data) + # use node from authenticator + successful_authenticator = self.context['request'].successful_authenticator + node, pin = successful_authenticator.authenticate(self.context['request']) + + #sometimes we post historical data (eg: from other network) + #this means we have to update last_notify only if current timestamp is greater than what's there + if node.last_notify is None or node.last_notify < sd.timestamp: + node.last_notify = sd.timestamp + node.save() + + return sd diff --git a/sensorsafrica/api/v1/views.py b/sensorsafrica/api/v1/views.py index a51aecc..c984da3 100644 --- a/sensorsafrica/api/v1/views.py +++ b/sensorsafrica/api/v1/views.py @@ -15,11 +15,12 @@ from rest_framework.permissions import IsAuthenticatedOrReadOnly from feinstaub.sensors.models import Node, SensorData -from feinstaub.sensors.serializers import NodeSerializer, NowSerializer +from feinstaub.sensors.serializers import NowSerializer from feinstaub.sensors.views import StandardResultsSetPagination +from feinstaub.sensors.authentication import NodeUidAuthentication -from .serializers import SensorDataSerializer from .filters import NodeFilter +from .serializers import LastNotifySensorDataSerializer, NodeSerializer, SensorDataSerializer class FilterView(mixins.ListModelMixin, viewsets.GenericViewSet): serializer_class = SensorDataSerializer @@ -83,6 +84,15 @@ def get_queryset(self): sensor__public=True, modified__range=[startdate, now] ) +class PostSensorDataView(mixins.CreateModelMixin, + viewsets.GenericViewSet): + """ This endpoint is to POST data from the sensor to the api. + """ + authentication_classes = (NodeUidAuthentication,) + permission_classes = tuple() + serializer_class = LastNotifySensorDataSerializer + queryset = SensorData.objects.all() + class SensorDataView(mixins.ListModelMixin, viewsets.GenericViewSet): serializer_class = SensorDataSerializer @@ -96,3 +106,4 @@ def get_queryset(self): .only("sensor", "timestamp") .prefetch_related("sensordatavalues") ) + From 38ed9e74d73b7535b95de0f2e259290fd870103c Mon Sep 17 00:00:00 2001 From: Isaiah King'ori Date: Mon, 1 Feb 2021 17:30:53 +0300 Subject: [PATCH 19/29] Ft add location filter for /v2/data (#97) * Adds location filter for /data endpoint * Temporaliry Remove iexact --- sensorsafrica/api/v1/filters.py | 2 +- sensorsafrica/api/v2/filters.py | 6 ++++++ sensorsafrica/api/v2/views.py | 3 ++- 3 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 sensorsafrica/api/v2/filters.py diff --git a/sensorsafrica/api/v1/filters.py b/sensorsafrica/api/v1/filters.py index d525dd2..3cee53b 100644 --- a/sensorsafrica/api/v1/filters.py +++ b/sensorsafrica/api/v1/filters.py @@ -5,5 +5,5 @@ class NodeFilter(django_filters.FilterSet): class Meta: model = Node - fields = {"location__country": ["iexact"]} + fields = {"location__country": ["exact"]} diff --git a/sensorsafrica/api/v2/filters.py b/sensorsafrica/api/v2/filters.py new file mode 100644 index 0000000..92d6f61 --- /dev/null +++ b/sensorsafrica/api/v2/filters.py @@ -0,0 +1,6 @@ +from feinstaub.sensors.views import SensorFilter + +class CustomSensorFilter(SensorFilter): + class Meta(SensorFilter.Meta): + # Pick the fields already defined and add the location__country field + fields = {**SensorFilter.Meta.fields, **{'location__country': ['exact']}} diff --git a/sensorsafrica/api/v2/views.py b/sensorsafrica/api/v2/views.py index f4e7272..b334231 100644 --- a/sensorsafrica/api/v2/views.py +++ b/sensorsafrica/api/v2/views.py @@ -45,6 +45,7 @@ SensorDataSerializer, ) +from .filters import CustomSensorFilter value_types = {"air": ["P1", "P2", "humidity", "temperature"]} @@ -237,7 +238,7 @@ class SensorDataView( pagination_class = SensorDataPagination permission_classes = [IsAuthenticated] filter_backends = (django_filters.rest_framework.DjangoFilterBackend,) - filter_class = SensorFilter + filter_class = CustomSensorFilter serializer_class = SensorDataSerializer def get_queryset(self): From 2bc8b2d501c905d8a74a52309655a71c574741f1 Mon Sep 17 00:00:00 2001 From: Khadija Mahanga Date: Mon, 1 Feb 2021 18:08:58 +0300 Subject: [PATCH 20/29] [FT] Time filter (#96) * add last_notify filter on nodes, sensorfilter * view class override * filter ovverides * add exact to look up exp * Includes missing import Co-authored-by: esir --- sensorsafrica/api/v1/filters.py | 25 +++++++++++++++++++++++-- sensorsafrica/api/v1/router.py | 5 +++-- sensorsafrica/api/v1/views.py | 9 ++++++--- 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/sensorsafrica/api/v1/filters.py b/sensorsafrica/api/v1/filters.py index 3cee53b..3989a2e 100644 --- a/sensorsafrica/api/v1/filters.py +++ b/sensorsafrica/api/v1/filters.py @@ -1,9 +1,30 @@ import django_filters +from django.db import models -from feinstaub.sensors.models import Node +from feinstaub.sensors.models import Node, SensorData class NodeFilter(django_filters.FilterSet): class Meta: model = Node - fields = {"location__country": ["exact"]} + fields = { + "location__country": ["exact"], + "last_notify": ["exact", "gte", "lte"]} + filter_overrides = { + models.DateTimeField: { + 'filter_class': django_filters.IsoDateTimeFilter, + }, + } + +class SensorFilter(django_filters.FilterSet): + class Meta: + model = SensorData + fields = { + "sensor": ["exact"], + "timestamp": ["exact", "gte", "lte"] + } + filter_overrides = { + models.DateTimeField: { + 'filter_class': django_filters.IsoDateTimeFilter, + }, + } diff --git a/sensorsafrica/api/v1/router.py b/sensorsafrica/api/v1/router.py index b90b3b0..c923328 100644 --- a/sensorsafrica/api/v1/router.py +++ b/sensorsafrica/api/v1/router.py @@ -11,7 +11,8 @@ NodeView, NowView, PostSensorDataView, - SensorDataView as SensorsAfricaSensorDataView, + SensorsAfricaSensorDataView, + VerboseSensorDataView, ) from rest_framework import routers @@ -20,7 +21,7 @@ router.register(r"push-sensor-data", PostSensorDataView) router.register(r"node", NodeView) router.register(r"sensor", SensorView) -router.register(r"data", SensorDataView) +router.register(r"data", VerboseSensorDataView) router.register(r"statistics", StatisticsView, basename="statistics") router.register(r"now", NowView) router.register(r"user", UsersView) diff --git a/sensorsafrica/api/v1/views.py b/sensorsafrica/api/v1/views.py index c984da3..822fbcd 100644 --- a/sensorsafrica/api/v1/views.py +++ b/sensorsafrica/api/v1/views.py @@ -16,10 +16,10 @@ from feinstaub.sensors.models import Node, SensorData from feinstaub.sensors.serializers import NowSerializer -from feinstaub.sensors.views import StandardResultsSetPagination +from feinstaub.sensors.views import SensorDataView, StandardResultsSetPagination from feinstaub.sensors.authentication import NodeUidAuthentication -from .filters import NodeFilter +from .filters import NodeFilter, SensorFilter from .serializers import LastNotifySensorDataSerializer, NodeSerializer, SensorDataSerializer class FilterView(mixins.ListModelMixin, viewsets.GenericViewSet): @@ -94,7 +94,10 @@ class PostSensorDataView(mixins.CreateModelMixin, queryset = SensorData.objects.all() -class SensorDataView(mixins.ListModelMixin, viewsets.GenericViewSet): +class VerboseSensorDataView(SensorDataView): + filter_class = SensorFilter + +class SensorsAfricaSensorDataView(mixins.ListModelMixin, viewsets.GenericViewSet): serializer_class = SensorDataSerializer def get_queryset(self): From 42a19cccf2bb2135e5883d13cb3b37d80d3b7ef4 Mon Sep 17 00:00:00 2001 From: khadija Date: Wed, 3 Feb 2021 11:38:40 +0300 Subject: [PATCH 21/29] add lat, long, city on to nodes location --- sensorsafrica/api/v1/serializers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sensorsafrica/api/v1/serializers.py b/sensorsafrica/api/v1/serializers.py index 1aba22d..2dd4348 100644 --- a/sensorsafrica/api/v1/serializers.py +++ b/sensorsafrica/api/v1/serializers.py @@ -10,10 +10,13 @@ NestedSensorSerializer, SensorDataSerializer as PostSensorDataSerializer ) +class NodesLocationSerializer(NestedSensorLocationSerializer): + class Meta(NestedSensorLocationSerializer.Meta): + fields = NestedSensorLocationSerializer.Meta.fields + ("latitude", "longitude", "city") class NodeSerializer(serializers.ModelSerializer): sensors = NestedSensorSerializer(many=True) - location = NestedSensorLocationSerializer() + location = NodesLocationSerializer() class Meta: model = Node From 33b6a5da84d17b4abae0b57d75a22cff016bbe07 Mon Sep 17 00:00:00 2001 From: khadija Date: Wed, 3 Feb 2021 13:27:54 +0300 Subject: [PATCH 22/29] add stats to node --- sensorsafrica/api/v1/views.py | 68 +++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/sensorsafrica/api/v1/views.py b/sensorsafrica/api/v1/views.py index 822fbcd..8a6d02f 100644 --- a/sensorsafrica/api/v1/views.py +++ b/sensorsafrica/api/v1/views.py @@ -70,6 +70,74 @@ def get_queryset(self): return Node.objects.none() + def add_stat_var_to_nodes(data): + nodes = [] + for node in data: + # last data received for this node + stats = [] + moved_to = None + + # Get data stats from 5mins before last_data_received_at + if node.last_notify: + last_5_mins = last_notify - datetime.timedelta(minutes=5) + stats = ( + SensorDataValue.objects.filter( + Q(sensordata__sensor__node=node.id), + Q(sensordata__location=node.location.id), + Q(sensordata__timestamp__gte=last_5_mins), + Q(sensordata__timestamp__lte=node.last_notify), + # Ignore timestamp values + ~Q(value_type="timestamp"), + # Match only valid float text + Q(value__regex=r"^\-?\d+(\.?\d+)?$"), + ) + .order_by() + .values("value_type") + .annotate( + sensor_id=F("sensordata__sensor__id"), + start_datetime=Min("sensordata__timestamp"), + end_datetime=Max("sensordata__timestamp"), + average=Avg(Cast("value", FloatField())), + minimum=Min(Cast("value", FloatField())), + maximum=Max(Cast("value", FloatField())), + ) + ) + + nodes.append( + { + "node_moved": moved_to is not None, + "moved_to": moved_to, + "node": {"uid": last_active.node.uid, "id": last_active.node.id, "owner": last_active.node.owner.id}, + "location": { + "name": last_active.location.location, + "longitude": last_active.location.longitude, + "latitude": last_active.location.latitude, + "city": { + "name": last_active.location.city, + "slug": slugify(last_active.location.city), + }, + }, + "last_data_received_at": last_data_received_at, + "stats": stats, + } + ) + return nodes + + + def list(self, request): + queryset = self.filter_queryset(self.get_queryset()) + + page = self.paginate_queryset(queryset) + if page is not None: + serializer = self.get_serializer(page, many=True) + data = add_stat_var_to_nodes(serializer.data) + return self.get_paginated_response(data) + + serializer = self.get_serializer(queryset, many=True) + data = add_stat_var_to_nodes(serializer.data) + return Response(data) + + class NowView(mixins.ListModelMixin, viewsets.GenericViewSet): """Show all public sensors active in the last 5 minutes with newest value""" From a0fbe0d9a8262a949ff5053590a68b168fb8d100 Mon Sep 17 00:00:00 2001 From: khadija Date: Wed, 3 Feb 2021 13:53:18 +0300 Subject: [PATCH 23/29] data --- sensorsafrica/api/v1/views.py | 63 +++-------------------------------- 1 file changed, 5 insertions(+), 58 deletions(-) diff --git a/sensorsafrica/api/v1/views.py b/sensorsafrica/api/v1/views.py index 8a6d02f..e1289f1 100644 --- a/sensorsafrica/api/v1/views.py +++ b/sensorsafrica/api/v1/views.py @@ -8,6 +8,8 @@ from django.db.models import ExpressionWrapper, F, FloatField, Max, Min, Sum, Avg, Q from django.db.models.functions import Cast, TruncDate from dateutil.relativedelta import relativedelta +from django.utils.text import slugify + from django.utils import timezone from rest_framework import mixins, pagination, viewsets from rest_framework.authentication import SessionAuthentication, TokenAuthentication @@ -70,72 +72,17 @@ def get_queryset(self): return Node.objects.none() - def add_stat_var_to_nodes(data): - nodes = [] - for node in data: - # last data received for this node - stats = [] - moved_to = None - - # Get data stats from 5mins before last_data_received_at - if node.last_notify: - last_5_mins = last_notify - datetime.timedelta(minutes=5) - stats = ( - SensorDataValue.objects.filter( - Q(sensordata__sensor__node=node.id), - Q(sensordata__location=node.location.id), - Q(sensordata__timestamp__gte=last_5_mins), - Q(sensordata__timestamp__lte=node.last_notify), - # Ignore timestamp values - ~Q(value_type="timestamp"), - # Match only valid float text - Q(value__regex=r"^\-?\d+(\.?\d+)?$"), - ) - .order_by() - .values("value_type") - .annotate( - sensor_id=F("sensordata__sensor__id"), - start_datetime=Min("sensordata__timestamp"), - end_datetime=Max("sensordata__timestamp"), - average=Avg(Cast("value", FloatField())), - minimum=Min(Cast("value", FloatField())), - maximum=Max(Cast("value", FloatField())), - ) - ) - - nodes.append( - { - "node_moved": moved_to is not None, - "moved_to": moved_to, - "node": {"uid": last_active.node.uid, "id": last_active.node.id, "owner": last_active.node.owner.id}, - "location": { - "name": last_active.location.location, - "longitude": last_active.location.longitude, - "latitude": last_active.location.latitude, - "city": { - "name": last_active.location.city, - "slug": slugify(last_active.location.city), - }, - }, - "last_data_received_at": last_data_received_at, - "stats": stats, - } - ) - return nodes - - def list(self, request): queryset = self.filter_queryset(self.get_queryset()) page = self.paginate_queryset(queryset) + if page is not None: serializer = self.get_serializer(page, many=True) - data = add_stat_var_to_nodes(serializer.data) - return self.get_paginated_response(data) + return self.get_paginated_response(serializer.data) serializer = self.get_serializer(queryset, many=True) - data = add_stat_var_to_nodes(serializer.data) - return Response(data) + return Response(serializer.data) class NowView(mixins.ListModelMixin, viewsets.GenericViewSet): From 707e322d3c46aac3b5b73e1043def832a0f20762 Mon Sep 17 00:00:00 2001 From: khadija Date: Wed, 3 Feb 2021 13:58:38 +0300 Subject: [PATCH 24/29] remove list function --- sensorsafrica/api/v1/views.py | 13 ------------- sensorsafrica/i.json | 25 +++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 13 deletions(-) create mode 100644 sensorsafrica/i.json diff --git a/sensorsafrica/api/v1/views.py b/sensorsafrica/api/v1/views.py index e1289f1..fed1926 100644 --- a/sensorsafrica/api/v1/views.py +++ b/sensorsafrica/api/v1/views.py @@ -72,19 +72,6 @@ def get_queryset(self): return Node.objects.none() - def list(self, request): - queryset = self.filter_queryset(self.get_queryset()) - - page = self.paginate_queryset(queryset) - - if page is not None: - serializer = self.get_serializer(page, many=True) - return self.get_paginated_response(serializer.data) - - serializer = self.get_serializer(queryset, many=True) - return Response(serializer.data) - - class NowView(mixins.ListModelMixin, viewsets.GenericViewSet): """Show all public sensors active in the last 5 minutes with newest value""" diff --git a/sensorsafrica/i.json b/sensorsafrica/i.json new file mode 100644 index 0000000..09b10b2 --- /dev/null +++ b/sensorsafrica/i.json @@ -0,0 +1,25 @@ +{"count": 1, "next": None, "previous": None, "results": [ + {"id": 2, "sensors": [ + {"id": 2, "description": "", "pin": "-", "sensor_type": {"id": 1, "name": "p3m3", "manufacturer": "sfjh" + }, "sensordatas": [ + {"id": 37, "sampling_rate": None, "timestamp": "2021-02-07T00: 04: 00Z", "sensordatavalues": [ + {"id": 73, "value": "10", "value_type": "P1" + }, + {"id": 74, "value": "99", "value_type": "P2" + } + ] + }, + {"id": 36, "sampling_rate": None, "timestamp": "2021-02-06T00: 04: 00Z", "sensordatavalues": [ + {"id": 71, "value": "10", "value_type": "P1" + }, + {"id": 72, "value": "99", "value_type": "P2" + } + ] + } + ] + } + ], "uid": "ad22", "owner": 1, "location": {"id": 1, "location": "location", "indoor": False, "description": "", "latitude": None, "longitude": None, "city": "" + }, "last_notify": "2021-02-07T00: 04: 00Z" + } + ] +} \ No newline at end of file From d1b49c68d04196af4571aad26e796d687293da41 Mon Sep 17 00:00:00 2001 From: Khadija Mahanga Date: Wed, 3 Feb 2021 13:59:49 +0300 Subject: [PATCH 25/29] Delete i.json --- sensorsafrica/i.json | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100644 sensorsafrica/i.json diff --git a/sensorsafrica/i.json b/sensorsafrica/i.json deleted file mode 100644 index 09b10b2..0000000 --- a/sensorsafrica/i.json +++ /dev/null @@ -1,25 +0,0 @@ -{"count": 1, "next": None, "previous": None, "results": [ - {"id": 2, "sensors": [ - {"id": 2, "description": "", "pin": "-", "sensor_type": {"id": 1, "name": "p3m3", "manufacturer": "sfjh" - }, "sensordatas": [ - {"id": 37, "sampling_rate": None, "timestamp": "2021-02-07T00: 04: 00Z", "sensordatavalues": [ - {"id": 73, "value": "10", "value_type": "P1" - }, - {"id": 74, "value": "99", "value_type": "P2" - } - ] - }, - {"id": 36, "sampling_rate": None, "timestamp": "2021-02-06T00: 04: 00Z", "sensordatavalues": [ - {"id": 71, "value": "10", "value_type": "P1" - }, - {"id": 72, "value": "99", "value_type": "P2" - } - ] - } - ] - } - ], "uid": "ad22", "owner": 1, "location": {"id": 1, "location": "location", "indoor": False, "description": "", "latitude": None, "longitude": None, "city": "" - }, "last_notify": "2021-02-07T00: 04: 00Z" - } - ] -} \ No newline at end of file From f2fefbddffef0322347da49853742c5bc97f6d79 Mon Sep 17 00:00:00 2001 From: Khadija Mahanga Date: Wed, 3 Feb 2021 14:00:28 +0300 Subject: [PATCH 26/29] Update views.py --- sensorsafrica/api/v1/views.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/sensorsafrica/api/v1/views.py b/sensorsafrica/api/v1/views.py index fed1926..822fbcd 100644 --- a/sensorsafrica/api/v1/views.py +++ b/sensorsafrica/api/v1/views.py @@ -8,8 +8,6 @@ from django.db.models import ExpressionWrapper, F, FloatField, Max, Min, Sum, Avg, Q from django.db.models.functions import Cast, TruncDate from dateutil.relativedelta import relativedelta -from django.utils.text import slugify - from django.utils import timezone from rest_framework import mixins, pagination, viewsets from rest_framework.authentication import SessionAuthentication, TokenAuthentication From c194dea3b8aa6aaffba8f4e1f9761d2c5dd30f6b Mon Sep 17 00:00:00 2001 From: Khadija Mahanga Date: Wed, 3 Feb 2021 14:38:11 +0300 Subject: [PATCH 27/29] Update sensorsafrica/api/v1/serializers.py Co-authored-by: _ Kilemensi --- sensorsafrica/api/v1/serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sensorsafrica/api/v1/serializers.py b/sensorsafrica/api/v1/serializers.py index 2dd4348..e7b3a5c 100644 --- a/sensorsafrica/api/v1/serializers.py +++ b/sensorsafrica/api/v1/serializers.py @@ -10,7 +10,7 @@ NestedSensorSerializer, SensorDataSerializer as PostSensorDataSerializer ) -class NodesLocationSerializer(NestedSensorLocationSerializer): +class NodeLocationSerializer(NestedSensorLocationSerializer): class Meta(NestedSensorLocationSerializer.Meta): fields = NestedSensorLocationSerializer.Meta.fields + ("latitude", "longitude", "city") From 73b8be7f774bebb4e14a8cbb241f5d5b0b79385b Mon Sep 17 00:00:00 2001 From: khadija Date: Wed, 3 Feb 2021 14:42:11 +0300 Subject: [PATCH 28/29] serializer name --- sensorsafrica/api/v1/serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sensorsafrica/api/v1/serializers.py b/sensorsafrica/api/v1/serializers.py index e7b3a5c..bf1b951 100644 --- a/sensorsafrica/api/v1/serializers.py +++ b/sensorsafrica/api/v1/serializers.py @@ -16,7 +16,7 @@ class Meta(NestedSensorLocationSerializer.Meta): class NodeSerializer(serializers.ModelSerializer): sensors = NestedSensorSerializer(many=True) - location = NodesLocationSerializer() + location = NodeLocationSerializer() class Meta: model = Node From 1d03730934b9988dd3cd5ba9bfa31e98a38428f6 Mon Sep 17 00:00:00 2001 From: khadija Date: Wed, 3 Feb 2021 14:44:37 +0300 Subject: [PATCH 29/29] revert merge error --- sensorsafrica/api/v2/filters.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/sensorsafrica/api/v2/filters.py b/sensorsafrica/api/v2/filters.py index b6b6e91..92d6f61 100644 --- a/sensorsafrica/api/v2/filters.py +++ b/sensorsafrica/api/v2/filters.py @@ -1,15 +1,6 @@ -from django.db import models -import django_filters from feinstaub.sensors.views import SensorFilter class CustomSensorFilter(SensorFilter): class Meta(SensorFilter.Meta): - fields = {"sensor": ["exact"], - "location__country": ['exact'], - "timestamp": ("gte", "lte"), - } - filter_overrides = { - models.DateTimeField: { - 'filter_class': django_filters.IsoDateTimeFilter, - }, - } + # Pick the fields already defined and add the location__country field + fields = {**SensorFilter.Meta.fields, **{'location__country': ['exact']}}