Merge da66d98 into 7189378

openwisp_monitoring/check/classes/base.py

@@ -35,15 +35,15 @@ def _get_or_create_metric(self, configuration=None):
         Gets or creates metric
         """
         check = self.check_instance
-        if check.object_id and check.content_type:
+        if check.object_id and check.content_type_id:
             obj_id = check.object_id
-            ct = check.content_type
+            ct = ContentType.objects.get_for_id(check.content_type_id)
         else:
             obj_id = str(check.id)
             ct = ContentType.objects.get_for_model(Check)
         options = dict(
             object_id=obj_id,
-            content_type=ct,
+            content_type_id=ct.id,
             configuration=configuration or self.__class__.__name__.lower(),
         )
         metric, created = Metric._get_or_create(**options)

openwisp_monitoring/check/tasks.py

@@ -84,7 +84,7 @@ def auto_create_ping(
     if has_check:
         return
     content_type_model = content_type_model or ContentType
-    ct = content_type_model.objects.get(app_label=app_label, model=model)
+    ct = content_type_model.objects.get_by_natural_key(app_label=app_label, model=model)
     check = Check(
         name='Ping', check_type=ping_path, content_type=ct, object_id=object_id
     )
@@ -108,7 +108,7 @@ def auto_create_config_check(
     if has_check:
         return
     content_type_model = content_type_model or ContentType
-    ct = content_type_model.objects.get(app_label=app_label, model=model)
+    ct = content_type_model.objects.get_by_natural_key(app_label=app_label, model=model)
     check = Check(
         name='Configuration Applied',
         check_type=config_check_path,
@@ -135,7 +135,7 @@ def auto_create_iperf3_check(
     if has_check:
         return
     content_type_model = content_type_model or ContentType
-    ct = content_type_model.objects.get(app_label=app_label, model=model)
+    ct = content_type_model.objects.get_by_natural_key(app_label=app_label, model=model)
     check = Check(
         name='Iperf3',
         check_type=iperf3_check_path,

openwisp_monitoring/check/tests/test_models.py

@@ -169,7 +169,9 @@ def test_config_modified_device_problem(self):
         self.assertEqual(m.key, 'config_applied')
         dm = d.monitoring
         dm.refresh_from_db()
+        m.refresh_from_db(fields=['is_healthy', 'is_healthy_tolerant'])
         self.assertEqual(m.is_healthy, False)
+        self.assertEqual(m.is_healthy_tolerant, False)
         self.assertEqual(dm.status, 'problem')
         self.assertEqual(Notification.objects.count(), 1)
 

openwisp_monitoring/db/backends/influxdb/client.py

@@ -117,6 +117,10 @@ def dbs(self):
             dbs['__all__'] = dbs['default']
         return dbs
 
+    @cached_property
+    def use_udp(self):
+        return TIMESERIES_DB.get('OPTIONS', {}).get('udp_writes', False)
+
     @retry
     def create_or_alter_retention_policy(self, name, duration):
         """creates or alters existing retention policy if necessary"""

openwisp_monitoring/db/backends/influxdb/tests.py

@@ -331,7 +331,7 @@ def test_timeseries_write_params(self, mock_write):
                 retention_policy=None,
                 tags={},
                 # this should be the original time at the moment of first failure
-                timestamp='2020-01-14T00:00:00Z',
+                timestamp=datetime(2020, 1, 14, tzinfo=tz('UTC')).isoformat(),
                 current=False,
             )
 

openwisp_monitoring/device/api/views.py

@@ -2,6 +2,7 @@
 import uuid
 from datetime import datetime
 
+from cache_memoize import cache_memoize
 from django.contrib.contenttypes.models import ContentType
 from django.core.exceptions import ValidationError
 from django.db.models import Count, Q
@@ -45,6 +46,17 @@
 Location = load_model('geo', 'Location')
 
 
+def get_device_args_rewrite(view, pk):
+    """
+    Use only the PK parameter for calculating the cache key
+    """
+    try:
+        pk = uuid.UUID(pk)
+    except ValueError:
+        return pk
+    return pk.hex
+
+
 class DeviceMetricView(MonitoringApiViewMixin, GenericAPIView):
     """
     Retrieve device information, monitoring status (health status),
@@ -57,15 +69,23 @@ class DeviceMetricView(MonitoringApiViewMixin, GenericAPIView):
     """
 
     model = DeviceData
-    queryset = (
-        DeviceData.objects.select_related('devicelocation')
-        .select_related('monitoring')
-        .all()
-    )
+    queryset = DeviceData.objects.only(
+        'id',
+        'key',
+    ).all()
     serializer_class = serializers.Serializer
     permission_classes = [DevicePermission]
     schema = schema
 
+    @classmethod
+    def invalidate_get_device_cache(cls, instance, **kwargs):
+        """
+        Called from signal receiver which performs cache invalidation
+        """
+        view = cls()
+        view.get_object.invalidate(view, str(instance.pk))
+        logger.debug(f'invalidated view cache for device ID {instance.pk}')
+
     def get_permissions(self):
         if self.request.method in SAFE_METHODS and not self.request.query_params.get(
             'key'
@@ -84,7 +104,7 @@ def get(self, request, pk):
             pk = str(uuid.UUID(pk))
         except ValueError:
             return Response({'detail': 'not found'}, status=404)
-        self.instance = self.get_object()
+        self.instance = self.get_object(pk)
         response = super().get(request, pk)
         if not request.query_params.get('csv'):
             charts_data = dict(response.data)
@@ -97,16 +117,20 @@ def get(self, request, pk):
     def _get_charts(self, request, *args, **kwargs):
         ct = ContentType.objects.get_for_model(Device)
         return Chart.objects.filter(
-            metric__object_id=args[0], metric__content_type=ct
+            metric__object_id=args[0], metric__content_type_id=ct.id
         ).select_related('metric')
 
     def _get_additional_data(self, request, *args, **kwargs):
         if request.query_params.get('status', False):
             return {'data': self.instance.data}
         return {}
 
+    @cache_memoize(24 * 60 * 60, args_rewrite=get_device_args_rewrite)
+    def get_object(self, pk):
+        return super().get_object()
+
     def post(self, request, pk):
-        self.instance = self.get_object()
+        self.instance = self.get_object(pk)
         self.instance.data = request.data
         # validate incoming data
         try:

openwisp_monitoring/device/apps.py

@@ -43,6 +43,7 @@ def ready(self):
         self.connect_is_working_changed()
         self.connect_device_signals()
         self.connect_config_status_changed()
+        self.connect_wifi_client_signals()
         self.connect_offline_device_close_wifisession()
         self.device_recovery_detection()
         self.set_update_config_model()
@@ -51,19 +52,48 @@ def ready(self):
         self.add_connection_ignore_notification_reasons()
 
     def connect_device_signals(self):
+        from .api.views import DeviceMetricView
+
         Device = load_model('config', 'Device')
+        DeviceData = load_model('device_monitoring', 'DeviceData')
+        DeviceLocation = load_model('geo', 'DeviceLocation')
 
         post_save.connect(
             self.device_post_save_receiver,
             sender=Device,
             dispatch_uid='device_post_save_receiver',
         )
+        post_save.connect(
+            DeviceData.invalidate_cache,
+            sender=Device,
+            dispatch_uid='post_save_device_invalidate_devicedata_cache',
+        )
+        post_save.connect(
+            DeviceData.invalidate_cache,
+            sender=DeviceLocation,
+            dispatch_uid='post_save_devicelocation_invalidate_devicedata_cache',
+        )
 
         post_delete.connect(
             self.device_post_delete_receiver,
             sender=Device,
             dispatch_uid='device_post_delete_receiver',
         )
+        post_delete.connect(
+            DeviceMetricView.invalidate_get_device_cache,
+            sender=Device,
+            dispatch_uid=('device_post_delete_invalidate_view_cache'),
+        )
+        post_delete.connect(
+            DeviceData.invalidate_cache,
+            sender=Device,
+            dispatch_uid='post_delete_device_invalidate_devicedata_cache',
+        )
+        post_delete.connect(
+            DeviceData.invalidate_cache,
+            sender=DeviceLocation,
+            dispatch_uid='post_delete_devicelocation_invalidate_devicedata_cache',
+        )
 
     @classmethod
     def device_post_save_receiver(cls, instance, created, **kwargs):
@@ -176,6 +206,22 @@ def connect_config_status_changed(cls):
                 dispatch_uid='monitoring.config_status_changed_receiver',
             )
 
+    @classmethod
+    def connect_wifi_client_signals(cls):
+        if not app_settings.WIFI_SESSIONS_ENABLED:
+            return
+        WifiClient = load_model('device_monitoring', 'WifiClient')
+        post_save.connect(
+            WifiClient.invalidate_cache,
+            sender=WifiClient,
+            dispatch_uid='post_save_invalidate_wificlient_cache',
+        )
+        post_delete.connect(
+            WifiClient.invalidate_cache,
+            sender=WifiClient,
+            dispatch_uid='post_delete_invalidate_wificlient_cache',
+        )
+
     @classmethod
     def connect_offline_device_close_wifisession(cls):
         if not app_settings.WIFI_SESSIONS_ENABLED:

openwisp_monitoring/device/base/models.py

@@ -54,6 +54,31 @@ def __init__(self, *args, **kwargs):
         self.writer = DeviceDataWriter(self)
         super().__init__(*args, **kwargs)
 
+    @classmethod
+    @cache_memoize(24 * 60 * 60)
+    def get_devicedata(cls, pk):
+        obj = (
+            cls.objects.select_related('devicelocation')
+            .only(
+                'id',
+                'organization_id',
+                'devicelocation__location_id',
+                'devicelocation__floorplan_id',
+            )
+            .get(id=pk)
+        )
+        return obj
+
+    @classmethod
+    def invalidate_cache(cls, instance, *args, **kwargs):
+        if isinstance(instance, load_model('geo', 'DeviceLocation')):
+            pk = instance.content_object_id
+        else:
+            if kwargs.get('created'):
+                return
+            pk = instance.pk
+        cls.get_devicedata.invalidate(cls, str(pk))
+
     def can_be_updated(self):
         """
         Do not attempt at pushing the conf if the device is not reachable
@@ -244,7 +269,7 @@ def save_data(self, time=None):
         self._transform_data()
         time = time or now()
         options = dict(tags={'pk': self.pk}, timestamp=time, retention_policy=SHORT_RP)
-        timeseries_write.delay(name=self.__key, values={'data': self.json()}, **options)
+        timeseries_write(name=self.__key, values={'data': self.json()}, **options)
         cache_key = get_device_cache_key(device=self, context='current-data')
         # cache current data to allow getting it without querying the timeseries DB
         cache.set(
@@ -258,13 +283,54 @@ def save_data(self, time=None):
             timeout=86400,  # 24 hours
         )
         if app_settings.WIFI_SESSIONS_ENABLED:
-            tasks.save_wifi_clients_and_sessions.delay(
-                device_data=self.data, device_pk=self.pk
-            )
+            self.save_wifi_clients_and_sessions()
 
     def json(self, *args, **kwargs):
         return json.dumps(self.data, *args, **kwargs)
 
+    def save_wifi_clients_and_sessions(self):
+        _WIFICLIENT_FIELDS = ['vendor', 'ht', 'vht', 'he', 'wmm', 'wds', 'wps']
+        WifiClient = load_model('device_monitoring', 'WifiClient')
+        WifiSession = load_model('device_monitoring', 'WifiSession')
+
+        active_sessions = []
+        interfaces = self.data.get('interfaces', [])
+        for interface in interfaces:
+            if interface.get('type') != 'wireless':
+                continue
+            interface_name = interface.get('name')
+            wireless = interface.get('wireless', {})
+            if not wireless or wireless['mode'] != 'access_point':
+                continue
+            ssid = wireless.get('ssid')
+            clients = wireless.get('clients', [])
+            for client in clients:
+                # Save WifiClient
+                client_obj = WifiClient.get_wifi_client(client.get('mac'))
+                update_fields = []
+                for field in _WIFICLIENT_FIELDS:
+                    if getattr(client_obj, field) != client.get(field):
+                        setattr(client_obj, field, client.get(field))
+                        update_fields.append(field)
+                if update_fields:
+                    client_obj.full_clean()
+                    client_obj.save(update_fields=update_fields)
+
+                # Save WifiSession
+                session_obj, _ = WifiSession.objects.get_or_create(
+                    device_id=self.id,
+                    interface_name=interface_name,
+                    ssid=ssid,
+                    wifi_client=client_obj,
+                    stop_time=None,
+                )
+                active_sessions.append(session_obj.pk)
+
+        # Close open WifiSession
+        WifiSession.objects.filter(device_id=self.id, stop_time=None,).exclude(
+            pk__in=active_sessions
+        ).update(stop_time=now())
+
 
 class AbstractDeviceMonitoring(TimeStampedEditableModel):
     device = models.OneToOneField(
@@ -385,6 +451,18 @@ class Meta:
         abstract = True
         verbose_name = _('WiFi Client')
 
+    @classmethod
+    @cache_memoize(24 * 60 * 60)
+    def get_wifi_client(cls, mac_address):
+        wifi_client, _ = cls.objects.get_or_create(mac_address=mac_address)
+        return wifi_client
+
+    @classmethod
+    def invalidate_cache(cls, instance, *args, **kwargs):
+        if kwargs.get('created'):
+            return
+        cls.get_wifi_client.invalidate(cls, instance.mac_address)
+
 
 class AbstractWifiSession(TimeStampedEditableModel):
     created = None