BITMAKER-3543 Stats module merging (#210)

* BITMAKER-3465 Stats module: Apply pagination to jobs per day in estela (#204)

* Added spiders endpoint to global stats and modified the stats serializers schema

* Fixed issues with api generation, changed some schema responses and updated viewsets names

* Added jobs action method to spiders viewset

* Added the integration in frontend with the backend, disabled the expandable feature on the stats table

* Disabling expandable row on stats table section

* Fix issues mentioned on my reviewer's comments

Author: webtaken

estela-api/api/serializers/stats.py

@@ -1,5 +1,7 @@
+from datetime import timedelta
 from rest_framework import serializers
-
+from api.serializers.spider import SpiderSerializer
+from api.serializers.job import SpiderJobSerializer
 from core.models import SpiderJob
 
 
@@ -55,30 +57,68 @@ class StatsSerializer(serializers.Serializer):
     jobs = JobsStatsSerializer(required=False)
     pages = PagesStatsSerializer()
     items_count = serializers.IntegerField(default=0)
-    runtime = serializers.FloatField(default=0.0)
+    runtime = serializers.DurationField(default=timedelta(hours=0, minutes=0))
     status_codes = StatusCodesStatsSerializer()
-    success_rate = serializers.FloatField(default=0.0)
+    success_rate = serializers.FloatField(default=0.0, required=False)
     logs = LogsStatsSerializer()
-    coverage = CoverageStatsSerializer()
+    coverage = CoverageStatsSerializer(required=False)
 
 
-class JobsMetadataSerializer(serializers.ModelSerializer):
-    class Meta:
-        model = SpiderJob
-        fields = ("jid", "spider", "job_status")
-
+class SpiderJobStatsSerializer(SpiderJobSerializer):
+    stats = StatsSerializer()
 
-class GetJobsStatsSerializer(serializers.Serializer):
-    jid = serializers.IntegerField(default=0)
-    spider = serializers.IntegerField(default=0)
-    stats = StatsSerializer(required=False)
 
+class SpiderJobStatsSerializer(SpiderJobSerializer):
+    stats = StatsSerializer()
 
-class GlobalStatsSerializer(serializers.Serializer):
+    class Meta:
+        model = SpiderJob
+        fields = (
+            "jid",
+            "spider",
+            "created",
+            "name",
+            "lifespan",
+            "total_response_bytes",
+            "item_count",
+            "request_count",
+            "args",
+            "env_vars",
+            "tags",
+            "job_status",
+            "cronjob",
+            "data_expiry_days",
+            "data_status",
+            "stats",
+        )
+
+
+class ProjectStatsSerializer(serializers.Serializer):
     date = serializers.DateField()
     stats = StatsSerializer()
-    jobs_metadata = JobsMetadataSerializer(many=True)
 
 
-class SpidersJobsStatsSerializer(GlobalStatsSerializer):
+class SpidersStatsSerializer(ProjectStatsSerializer):
     pass
+
+
+class SpidersPaginationSerializer(serializers.Serializer):
+    count = serializers.IntegerField()
+    next = serializers.HyperlinkedIdentityField(
+        view_name="project-stats", allow_null=True
+    )
+    previous = serializers.HyperlinkedIdentityField(
+        view_name="project-stats", allow_null=True
+    )
+    results = SpiderSerializer(many=True)
+
+
+class JobsPaginationSerializer(serializers.Serializer):
+    count = serializers.IntegerField()
+    next = serializers.HyperlinkedIdentityField(
+        view_name="project-stats", allow_null=True
+    )
+    previous = serializers.HyperlinkedIdentityField(
+        view_name="project-stats", allow_null=True
+    )
+    results = SpiderJobStatsSerializer(many=True)

estela-api/api/urls.py

@@ -50,8 +50,8 @@
 )
 router.register(
     prefix=r"stats/(?P<pid>[0-9a-z-]+)",
-    viewset=stats_views.GlobalStatsViewSet,
-    basename="global-stats",
+    viewset=stats_views.ProjectStatsViewSet,
+    basename="project-stats",
 )
 router.register(
     prefix=r"stats/(?P<pid>[0-9a-z-]+)/spider/(?P<sid>\d+)",

estela-api/api/views/notification.py

@@ -18,7 +18,9 @@ class NotificationViewSet(BaseViewSet, viewsets.ModelViewSet):
     def get_queryset(self):
         if self.request is None:
             return Notification.objects.none()
-        return Notification.objects.filter(user=self.request.user).order_by("-activity__created")
+        return Notification.objects.filter(user=self.request.user).order_by(
+            "-activity__created"
+        )
 
     @swagger_auto_schema(
         request_body=NotificationUpdateSerializer,