diff --git a/requirements.txt b/requirements.txt index b7653d4..fdca0c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,11 @@ -django==1.11.27 #LTS +django==1.11.29 #LTS coreapi==2.3.3 dj-database-url==0.5.0 timeago==1.0.10 -flower==0.9.2 +flower==0.9.5 tornado<6 -sentry-sdk==0.7.3 +sentry-sdk==0.19.5 celery==4.2.1 gevent==1.2.2 greenlet==0.4.12 diff --git a/sensorsafrica/api/v2/router.py b/sensorsafrica/api/v2/router.py index 85a709c..d0b9677 100644 --- a/sensorsafrica/api/v2/router.py +++ b/sensorsafrica/api/v2/router.py @@ -1,18 +1,26 @@ from rest_framework import routers from django.conf.urls import url, include -from .views import CitiesView, NodesView, SensorDataStatsView, SensorLocationsView, SensorTypesView, SensorsView +from .views import ( + CitiesView, + NodesView, + SensorDataStatsView, + SensorDataView, + SensorLocationsView, + SensorTypesView, + SensorsView, +) + +stat_data_router = routers.DefaultRouter() +stat_data_router.register(r"", SensorDataStatsView) data_router = routers.DefaultRouter() - -data_router.register(r"", SensorDataStatsView) +data_router.register(r"", SensorDataView) cities_router = routers.DefaultRouter() - cities_router.register(r"", CitiesView, basename="cities") nodes_router = routers.DefaultRouter() - nodes_router.register(r"", NodesView, basename="map") sensors_router = routers.DefaultRouter() @@ -25,7 +33,8 @@ sensor_types_router.register(r"", SensorTypesView, basename="sensor_types") api_urls = [ - url(r"data/(?P[air]+)/", include(data_router.urls)), + url(r"data/(?P[air]+)/", include(stat_data_router.urls)), + url(r"data/", include(data_router.urls)), url(r"cities/", include(cities_router.urls)), url(r"nodes/", include(nodes_router.urls)), url(r"locations/", include(sensor_locations_router.urls)), diff --git a/sensorsafrica/api/v2/views.py b/sensorsafrica/api/v2/views.py index abe712b..60891be 100644 --- a/sensorsafrica/api/v2/views.py +++ b/sensorsafrica/api/v2/views.py @@ -1,27 +1,25 @@ import datetime +import django_filters import pytz import json -from rest_framework.exceptions import ValidationError +from dateutil.relativedelta import relativedelta from django.conf import settings from django.utils import timezone -from dateutil.relativedelta import relativedelta from django.db.models import ExpressionWrapper, F, FloatField, Max, Min, Sum, Avg, Q from django.db.models.functions import Cast, TruncHour, TruncDay, TruncMonth -from rest_framework import mixins, pagination, viewsets +from django.utils.decorators import method_decorator +from django.utils.text import slugify +from django.views.decorators.cache import cache_page -from ..models import SensorDataStat, LastActiveNodes, City -from .serializers import ( - SensorDataStatSerializer, - CitySerializer, - NestedSensorTypeSerializer, - NodeSerializer, - SensorSerializer, - SensorLocationSerializer, -) +from rest_framework import mixins, pagination, viewsets +from rest_framework.authentication import SessionAuthentication, TokenAuthentication +from rest_framework.exceptions import ValidationError +from rest_framework.response import Response +from rest_framework.permissions import IsAuthenticated, AllowAny -from feinstaub.sensors.views import StandardResultsSetPagination +from feinstaub.sensors.views import SensorFilter, StandardResultsSetPagination from feinstaub.sensors.models import ( Node, @@ -32,14 +30,18 @@ SensorType, ) -from django.utils.text import slugify +from feinstaub.sensors.serializers import VerboseSensorDataSerializer -from rest_framework.response import Response -from rest_framework.authentication import SessionAuthentication, TokenAuthentication -from rest_framework.permissions import IsAuthenticated, AllowAny +from ..models import City, LastActiveNodes, SensorDataStat +from .serializers import ( + SensorDataStatSerializer, + CitySerializer, + NestedSensorTypeSerializer, + NodeSerializer, + SensorSerializer, + SensorLocationSerializer, +) -from django.utils.decorators import method_decorator -from django.views.decorators.cache import cache_page value_types = {"air": ["P1", "P2", "humidity", "temperature"]} @@ -116,94 +118,6 @@ def get_paginated_response(self, data_stats): ) -class SensorDataStatsView(mixins.ListModelMixin, viewsets.GenericViewSet): - queryset = SensorDataStat.objects.none() - serializer_class = SensorDataStatSerializer - pagination_class = CustomPagination - - @method_decorator(cache_page(3600)) - def dispatch(self, request, *args, **kwargs): - return super().dispatch(request, *args, **kwargs) - - def get_queryset(self): - sensor_type = self.kwargs["sensor_type"] - - city_slugs = self.request.query_params.get("city", None) - from_date = self.request.query_params.get("from", None) - to_date = self.request.query_params.get("to", None) - interval = self.request.query_params.get("interval", None) - - if to_date and not from_date: - raise ValidationError({"from": "Must be provide along with to query"}) - if from_date: - validate_date(from_date, {"from": "Must be a date in the format Y-m-d."}) - if to_date: - validate_date(to_date, {"to": "Must be a date in the format Y-m-d."}) - - value_type_to_filter = self.request.query_params.get("value_type", None) - - filter_value_types = value_types[sensor_type] - if value_type_to_filter: - filter_value_types = set(value_type_to_filter.upper().split(",")) & set( - [x.upper() for x in value_types[sensor_type]] - ) - - if not from_date and not to_date: - to_date = timezone.now().replace(minute=0, second=0, microsecond=0) - from_date = to_date - datetime.timedelta(hours=24) - interval = "day" if not interval else interval - elif not to_date: - from_date = beginning_of_day(from_date) - # Get data from_date until the end - # of day yesterday which is the beginning of today - to_date = beginning_of_today() - else: - from_date = beginning_of_day(from_date) - to_date = end_of_day(to_date) - - queryset = SensorDataStat.objects.filter( - value_type__in=filter_value_types, - timestamp__gte=from_date, - timestamp__lte=to_date, - ) - - if interval == "month": - truncate = TruncMonth("timestamp") - elif interval == "day": - truncate = TruncDay("timestamp") - else: - truncate = TruncHour("timestamp") - - if city_slugs: - queryset = queryset.filter(city_slug__in=city_slugs.split(",")) - - return ( - queryset.values("value_type", "city_slug") - .annotate( - truncated_timestamp=truncate, - start_datetime=Min("timestamp"), - end_datetime=Max("timestamp"), - calculated_average=ExpressionWrapper( - Sum(F("average") * F("sample_size")) / Sum("sample_size"), - output_field=FloatField(), - ), - calculated_minimum=Min("minimum"), - calculated_maximum=Max("maximum"), - ) - .values( - "value_type", - "city_slug", - "truncated_timestamp", - "start_datetime", - "end_datetime", - "calculated_average", - "calculated_minimum", - "calculated_maximum", - ) - .order_by("city_slug", "-truncated_timestamp") - ) - - class CitiesView(mixins.ListModelMixin, viewsets.GenericViewSet): queryset = City.objects.all() serializer_class = CitySerializer @@ -304,6 +218,130 @@ def create(self, request): return Response(serializer.errors, status=400) +class SensorDataPagination(pagination.CursorPagination): + cursor_query_param = "next_page" + ordering = "-timestamp" + page_size = 100 + + +class SensorDataView( + mixins.ListModelMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet +): + """This endpoint is to download sensor data from the api.""" + + authentication_classes = [SessionAuthentication, TokenAuthentication] + queryset = SensorData.objects.all() + pagination_class = SensorDataPagination + permission_classes = [IsAuthenticated] + filter_backends = (django_filters.rest_framework.DjangoFilterBackend,) + filter_class = SensorFilter + serializer_class = VerboseSensorDataSerializer + + def get_queryset(self): + if self.request.user.is_authenticated(): + if self.request.user.groups.filter(name="show_me_everything").exists(): + return SensorData.objects.all() + + # Return data from sensors owned or + # owned by someone in the same group as requesting user or + # public sensors + return SensorData.objects.filter( + Q(sensor__node__owner=self.request.user) + | Q(sensor__node__owner__groups__name__in=[g.name for g in self.request.user.groups.all()]) + | Q(sensor__public=True) + ) + + return SensorData.objects.filter(sensor__public=True) + + +class SensorDataStatsView(mixins.ListModelMixin, viewsets.GenericViewSet): + queryset = SensorDataStat.objects.none() + serializer_class = SensorDataStatSerializer + pagination_class = CustomPagination + + @method_decorator(cache_page(3600)) + def dispatch(self, request, *args, **kwargs): + return super().dispatch(request, *args, **kwargs) + + def get_queryset(self): + sensor_type = self.kwargs["sensor_type"] + + city_slugs = self.request.query_params.get("city", None) + from_date = self.request.query_params.get("from", None) + to_date = self.request.query_params.get("to", None) + interval = self.request.query_params.get("interval", None) + + if to_date and not from_date: + raise ValidationError({"from": "Must be provide along with to query"}) + if from_date: + validate_date(from_date, {"from": "Must be a date in the format Y-m-d."}) + if to_date: + validate_date(to_date, {"to": "Must be a date in the format Y-m-d."}) + + value_type_to_filter = self.request.query_params.get("value_type", None) + + filter_value_types = value_types[sensor_type] + if value_type_to_filter: + filter_value_types = set(value_type_to_filter.upper().split(",")) & set( + [x.upper() for x in value_types[sensor_type]] + ) + + if not from_date and not to_date: + to_date = timezone.now().replace(minute=0, second=0, microsecond=0) + from_date = to_date - datetime.timedelta(hours=24) + interval = "day" if not interval else interval + elif not to_date: + from_date = beginning_of_day(from_date) + # Get data from_date until the end + # of day yesterday which is the beginning of today + to_date = beginning_of_today() + else: + from_date = beginning_of_day(from_date) + to_date = end_of_day(to_date) + + queryset = SensorDataStat.objects.filter( + value_type__in=filter_value_types, + timestamp__gte=from_date, + timestamp__lte=to_date, + ) + + if interval == "month": + truncate = TruncMonth("timestamp") + elif interval == "day": + truncate = TruncDay("timestamp") + else: + truncate = TruncHour("timestamp") + + if city_slugs: + queryset = queryset.filter(city_slug__in=city_slugs.split(",")) + + return ( + queryset.values("value_type", "city_slug") + .annotate( + truncated_timestamp=truncate, + start_datetime=Min("timestamp"), + end_datetime=Max("timestamp"), + calculated_average=ExpressionWrapper( + Sum(F("average") * F("sample_size")) / Sum("sample_size"), + output_field=FloatField(), + ), + calculated_minimum=Min("minimum"), + calculated_maximum=Max("maximum"), + ) + .values( + "value_type", + "city_slug", + "truncated_timestamp", + "start_datetime", + "end_datetime", + "calculated_average", + "calculated_minimum", + "calculated_maximum", + ) + .order_by("city_slug", "-truncated_timestamp") + ) + + class SensorLocationsView(viewsets.ViewSet): authentication_classes = [SessionAuthentication, TokenAuthentication] permission_classes = [IsAuthenticated] @@ -323,26 +361,18 @@ def create(self, request): return Response(serializer.errors, status=400) -class SensorsView(viewsets.ViewSet): +class SensorTypesView(viewsets.ViewSet): authentication_classes = [SessionAuthentication, TokenAuthentication] permission_classes = [IsAuthenticated] pagination_class = StandardResultsSetPagination - def get_permissions(self): - if self.action == "create": - permission_classes = [IsAuthenticated] - else: - permission_classes = [AllowAny] - - return [permission() for permission in permission_classes] - def list(self, request): - queryset = Sensor.objects.all() - serializer = SensorSerializer(queryset, many=True) + queryset = SensorType.objects.all() + serializer = NestedSensorTypeSerializer(queryset, many=True) return Response(serializer.data) def create(self, request): - serializer = SensorSerializer(data=request.data) + serializer = NestedSensorTypeSerializer(data=request.data) if serializer.is_valid(): serializer.save() return Response(serializer.data, status=201) @@ -350,18 +380,26 @@ def create(self, request): return Response(serializer.errors, status=400) -class SensorTypesView(viewsets.ViewSet): +class SensorsView(viewsets.ViewSet): authentication_classes = [SessionAuthentication, TokenAuthentication] permission_classes = [IsAuthenticated] pagination_class = StandardResultsSetPagination + def get_permissions(self): + if self.action == "create": + permission_classes = [IsAuthenticated] + else: + permission_classes = [AllowAny] + + return [permission() for permission in permission_classes] + def list(self, request): - queryset = SensorType.objects.all() - serializer = NestedSensorTypeSerializer(queryset, many=True) + queryset = Sensor.objects.all() + serializer = SensorSerializer(queryset, many=True) return Response(serializer.data) def create(self, request): - serializer = NestedSensorTypeSerializer(data=request.data) + serializer = SensorSerializer(data=request.data) if serializer.is_valid(): serializer.save() return Response(serializer.data, status=201) diff --git a/sensorsafrica/management/commands/calculate_data_statistics.py b/sensorsafrica/management/commands/calculate_data_statistics.py index 6a9d9db..d96a38f 100644 --- a/sensorsafrica/management/commands/calculate_data_statistics.py +++ b/sensorsafrica/management/commands/calculate_data_statistics.py @@ -1,11 +1,12 @@ from django.core.management import BaseCommand +from django.core.paginator import Paginator from django.db.models import Avg, Count, FloatField, Max, Min, Q from django.db.models.functions import Cast, TruncHour from django.utils.text import slugify + from feinstaub.sensors.models import Node, Sensor, SensorDataValue, SensorLocation -from ...api.models import SensorDataStat -from django.core.paginator import Paginator +from sensorsafrica.api.models import SensorDataStat def map_stat(stat, city): @@ -55,6 +56,8 @@ def handle(self, *args, **options): if last_date_time: queryset = SensorDataValue.objects.filter( + # Pick data from public sensors only + Q(sensordata__sensor__public=True), Q(sensordata__location__city__iexact=city), # Get dates greater than last stat calculation Q(created__gt=last_date_time), @@ -65,6 +68,8 @@ def handle(self, *args, **options): ) else: queryset = SensorDataValue.objects.filter( + # Pick data from public sensors only + Q(sensordata__sensor__public=True), Q(sensordata__location__city__iexact=city), # Ignore timestamp values ~Q(value_type="timestamp"), @@ -74,27 +79,28 @@ def handle(self, *args, **options): for stats in chunked_iterator( queryset.annotate(timestamp=TruncHour("created")) - .values( - "timestamp", - "value_type", - "sensordata__sensor", - "sensordata__location", - "sensordata__sensor__node", - ) - .order_by() - .annotate( - last_datetime=Max("created"), - average=Avg(Cast("value", FloatField())), - minimum=Min(Cast("value", FloatField())), - maximum=Max(Cast("value", FloatField())), - sample_size=Count("created", FloatField()), - ) - .filter( - ~Q(average=float("NaN")), - ~Q(minimum=float("NaN")), - ~Q(maximum=float("NaN")), - ) - .order_by("timestamp")): + .values( + "timestamp", + "value_type", + "sensordata__sensor", + "sensordata__location", + "sensordata__sensor__node", + ) + .order_by() + .annotate( + last_datetime=Max("created"), + average=Avg(Cast("value", FloatField())), + minimum=Min(Cast("value", FloatField())), + maximum=Max(Cast("value", FloatField())), + sample_size=Count("created", FloatField()), + ) + .filter( + ~Q(average=float("NaN")), + ~Q(minimum=float("NaN")), + ~Q(maximum=float("NaN")), + ) + .order_by("timestamp") + ): SensorDataStat.objects.bulk_create( list(map(lambda stat: map_stat(stat, city), stats)) )