Feature/528 optimize charts

backend/api/v1/v1_data/functions.py

@@ -1,8 +1,32 @@
+import re
+import json
+from pathlib import Path
+from datetime import datetime
 from django.db import transaction, connection
+from rtmis.settings import CACHE_FOLDER
 
 
 @transaction.atomic
 def refresh_materialized_data():
     with connection.cursor() as cursor:
         cursor.execute("""
             REFRESH MATERIALIZED VIEW view_data_options;""")
+
+
+def get_cache(name):
+    name = re.sub(r'[\W_]+', '_', name)
+    today = datetime.now().strftime("%Y%m%d")
+    cache_name = f"{CACHE_FOLDER}{today}-{name}.json"
+    if Path(cache_name).exists():
+        with open(cache_name, 'r') as cache_file:
+            return json.load(cache_file)
+    return None
+
+
+def create_cache(name, resp):
+    name = re.sub(r'[\W_]+', '_', name)
+    today = datetime.now().strftime("%Y%m%d")
+    cache_name = f"{CACHE_FOLDER}{today}-{name}.json"
+    json_cache = json.dumps(resp, separators=(',', ":"))
+    with open(cache_name, "w") as outfile:
+        outfile.write(json_cache)

backend/api/v1/v1_data/management/commands/fake_data_seeder.py

@@ -10,6 +10,7 @@
 from api.v1.v1_forms.models import Forms
 from api.v1.v1_profile.models import Administration, Levels
 from api.v1.v1_users.models import SystemUser
+from api.v1.v1_data.functions import refresh_materialized_data
 
 fake = Faker()
 
@@ -130,3 +131,4 @@ def handle(self, *args, **options):
                 print(f"\nSeeding - {form.name}")
             seed_data(form, fake_geo, level_names, options.get("repeat"),
                       options.get("test"))
+        refresh_materialized_data()

backend/api/v1/v1_data/management/commands/refresh_database.py

@@ -0,0 +1,7 @@
+from django.core.management import BaseCommand
+from api.v1.v1_data.functions import refresh_materialized_data
+
+
+class Command(BaseCommand):
+    def handle(self, *args, **options):
+        refresh_materialized_data()

backend/api/v1/v1_data/tests/tests_visualization.py

@@ -1,3 +1,4 @@
+from datetime import datetime
 from django.core.management import call_command
 from django.test import TestCase
 from django.test.utils import override_settings
@@ -65,7 +66,7 @@ def test_chart_data(self):
         token = user_response.json().get('token')
 
         call_command("form_seeder", "--test")
-        call_command("fake_data_seeder", "-r", 2, '-t', True)
+        call_command("fake_data_seeder", "-r", 100, '-t', True)
         header = {'HTTP_AUTHORIZATION': f'Bearer {token}'}
 
         form = Forms.objects.first()
@@ -182,33 +183,92 @@ def test_chart_data(self):
                 self.assertEqual(list(d.get('child')[0]), ['name', 'value'])
 
         # CHART OVERVIEW CRITERIA API
-        payload = [{
-            "name": "Test",
-            "option": [{
-                "question": 102,
-                "options": ["Male"],
-            }, {
-                "question": 106,
-                "options": ["Parent"],
-            }],
-        }]
+        # INCORRECT PARAMETER
+        payload = {
+            "cache": "Testing-chart",
+            "data": [{
+                "name": "Test",
+                "option": [{
+                    "question": 102,
+                    "options": ["Male"],
+                }, {
+                    "question": 106,
+                    "options": ["Parent"],
+                }]
+            }]
+        }
         data = self.client.post(
             "/api/v1/chart/overview/criteria/{0}".
             format(form.id),
             payload,
             content_type='application/json',
             **header)
         self.assertEqual(data.status_code, 400)
-        payload = [{
-            "name": "Test",
-            "options": [{
-                "question": 102,
-                "option": ["Male"],
-            }, {
-                "question": 106,
-                "option": ["Parent"],
-            }],
-        }]
+
+        # INCORRECT PARAMETER
+        payload = {
+            "data": [{
+                "name": "Test",
+                "options": [{
+                    "question": 102,
+                    "option": ["Male"],
+                }, {
+                    "question": 106,
+                    "option": ["Parent"],
+                }]
+            }]
+        }
+        data = self.client.post(
+            "/api/v1/chart/overview/criteria/{0}".
+            format(form.id),
+            payload,
+            content_type='application/json',
+            **header)
+        self.assertEqual(data.status_code, 400)
+
+        # CORRECT PARAMETER, RUN FOR THE FIRST TIME
+        run_without_cache = datetime.now().timestamp()
+        payload = {
+            "cache": "Testing-chart",
+            "data": [{
+                "name": "Test",
+                "options": [{
+                    "question": 102,
+                    "option": ["Male"],
+                }, {
+                    "question": 106,
+                    "option": ["Parent"],
+                }]
+            }]
+        }
+        data = self.client.post(
+            "/api/v1/chart/overview/criteria/{0}".
+            format(form.id),
+            payload,
+            content_type='application/json',
+            **header)
+        self.assertEqual(data.status_code, 200)
+        self.assertEqual(data.json().get('type'), 'BARSTACK')
+        self.assertEqual(list(data.json().get('data')[0]), ['group', 'child'])
+        self.assertEqual(list(data.json().get('data')[0]['child'][0]),
+                         ['name', 'value'])
+        run_without_cache = datetime.now().timestamp() - run_without_cache
+
+        # RUN FOR THE SECOND TIME
+        run_with_cache = datetime.now().timestamp()
+        payload = {
+            "cache": "Testing-chart",
+            "data": [{
+                "name": "Test",
+                "options": [{
+                    "question": 102,
+                    "option": ["Male"],
+                }, {
+                    "question": 106,
+                    "option": ["Parent"],
+                }]
+            }]
+        }
         data = self.client.post(
             "/api/v1/chart/overview/criteria/{0}".
             format(form.id),
@@ -220,3 +280,5 @@ def test_chart_data(self):
         self.assertEqual(list(data.json().get('data')[0]), ['group', 'child'])
         self.assertEqual(list(data.json().get('data')[0]['child'][0]),
                          ['name', 'value'])
+        run_with_cache = datetime.now().timestamp() - run_with_cache
+        self.assertEqual(True, run_without_cache > run_with_cache)

backend/api/v1/v1_data/views.py

@@ -1,6 +1,7 @@
 # Create your views here.
-import datetime
+import json
 from math import ceil
+from datetime import datetime, timedelta
 from wsgiref.util import FileWrapper
 from django.utils import timezone
 
@@ -39,7 +40,8 @@
     ChartDataSerializer, ListChartCriteriaRequestSerializer, \
     ListMapOverviewDataPointSerializer, \
     ListMapOverviewDataPointRequestSerializer
-from api.v1.v1_data.functions import refresh_materialized_data
+from api.v1.v1_data.functions import refresh_materialized_data, \
+    get_cache, create_cache
 from api.v1.v1_forms.constants import QuestionTypes, FormTypes
 from api.v1.v1_forms.models import Forms, Questions
 from api.v1.v1_profile.models import Administration, Levels
@@ -105,8 +107,7 @@ def get(self, request, form_id, version):
 
         page_size = REST_FRAMEWORK.get('PAGE_SIZE')
 
-        the_past = datetime.datetime.now() - datetime.timedelta(
-            days=10 * 365)
+        the_past = datetime.now() - timedelta(days=10 * 365)
         queryset = form.form_form_data.filter(**filter_data).annotate(
             last_updated=Coalesce('updated', Value(the_past))).order_by(
             '-last_updated', '-created')
@@ -350,6 +351,13 @@ def get_map_data_point(request, version, form_id):
     summary='To get overview Map data points')
 @api_view(['GET'])
 def get_map_overview_data_point(request, version, form_id):
+    cache_name = request.GET.get("shape")
+    cache_name = f"ovw_maps-{cache_name}"
+    cache = get_cache(cache_name)
+    if cache:
+        return HttpResponse(
+                json.dumps(cache),
+                content_type="application/json;")
     instance = get_object_or_404(Forms, pk=form_id)
     serializer = ListMapOverviewDataPointRequestSerializer(
         data=request.GET, context={'form': instance})
@@ -377,6 +385,7 @@ def get_map_overview_data_point(request, version, form_id):
             'loc': adm.name,
             'shape': sum(fl.get('shape') for fl in list(filtered))
         })
+    create_cache(cache_name, counties)
     return Response(counties, status=status.HTTP_200_OK)
 
 
@@ -677,7 +686,9 @@ def get_chart_criteria(request, version, form_id):
 
 
 @extend_schema(
-    request=ListChartCriteriaRequestSerializer(many=True),
+    request=inline_serializer("OverviewCriteriaChart", fields={
+            "cache": serializers.CharField(),
+            "data": ListChartCriteriaRequestSerializer(many=True)}),
     responses={200: ChartDataSerializer},
     parameters=[OpenApiParameter(
         name='administration',
@@ -689,23 +700,30 @@ def get_chart_criteria(request, version, form_id):
     summary='To get overview with criteria chart at National level')
 @api_view(['POST'])
 def get_chart_overview_criteria(request, version, form_id):
-    administration_id = 1
-    if request.GET.get('administration'):
-        administration_id = request.GET.get('administration')
     instance = get_object_or_404(Forms, pk=form_id)
-    administration = get_object_or_404(Administration, pk=administration_id)
     serializer = ListChartCriteriaRequestSerializer(
-        data=request.data, context={'form': instance}, many=True)
+        data=request.data.get("data"), context={'form': instance}, many=True)
     if not serializer.is_valid():
         return Response(
             {'message': validate_serializers_message(serializer.errors)},
-            status=status.HTTP_400_BAD_REQUEST
-        )
+            status=status.HTTP_400_BAD_REQUEST)
+    administration_id = 1
+    if request.GET.get('administration'):
+        administration_id = request.GET.get('administration')
+    cache_name = request.data.get("cache")
+    if not cache_name:
+        return Response(
+            {'message': 'cache params not found'},
+            status=status.HTTP_400_BAD_REQUEST)
+    cache_name = f"ovw_chart_criteria-{cache_name}-{administration_id}"
+    cache = get_cache(cache_name)
+    if cache:
+        return HttpResponse(
+                json.dumps(cache),
+                content_type="application/json;")
+    administration = get_object_or_404(Administration, pk=administration_id)
     params = serializer.validated_data
     max_level = Levels.objects.order_by('-level').first()
-    # show only national level
-    # childs = Administration.objects.filter(level_id=1).all()
-    #
     childs = Administration.objects.filter(parent=administration).all()
     if administration.level.level == max_level.level:
         childs = [administration]
@@ -716,10 +734,6 @@ def get_chart_overview_criteria(request, version, form_id):
             'group': child.name,
             'child': []
         }
-        # show only national level
-        # data_ids = list(ViewDataOptions.objects.filter(
-        #     form_id=form_id).values_list('data_id', flat=True))
-        #
         filter_path = child.path
         if child.level.level < max_level.level:
             filter_path = "{0}{1}.".format(child.path, child.id)
@@ -756,8 +770,9 @@ def get_chart_overview_criteria(request, version, form_id):
                 'value': len(filter_criteria)
             })
         data.append(values)
-    return Response({'type': 'BARSTACK', 'data': data},
-                    status=status.HTTP_200_OK)
+    data = {"type": "BARSTACK", "data": data}
+    create_cache(cache_name, data)
+    return Response(data, status=status.HTTP_200_OK)
 
 
 @extend_schema(responses={

backend/manage.py

@@ -16,8 +16,9 @@ def main():
             "available on your PYTHONPATH environment variable? Did you "
             "forget to activate a virtual environment?"
         ) from exc
-    """Create fake storage"""
+    """Create fake storage and cache folder"""
     Path("./tmp/fake_storage").mkdir(parents=True, exist_ok=True)
+    Path("./tmp/cache").mkdir(parents=True, exist_ok=True)
     execute_from_command_line(sys.argv)
 
 

backend/rtmis/settings.py

@@ -171,6 +171,9 @@
 
 STATIC_URL = 'static-files/'
 
+# For Caching API call
+CACHE_FOLDER = './tmp/cache/'
+
 # Default primary key field type
 # https://docs.djangoproject.com/en/4.0/ref/settings/#default-auto-field
 

frontend/src/App.js

@@ -227,12 +227,14 @@ const App = () => {
     }
   }, [authUser, isLoggedIn, notify]);
 
+  const isHome = location.pathname === "/";
+
   return (
     <Layout>
       <Layout.Header />
       <Layout.Banner />
       <Layout.Body>
-        {loading ? (
+        {loading && !isHome ? (
           <PageLoader message="Initializing. Please wait.." />
         ) : (
           <RouteList />

frontend/src/components/visualisation/HomeAdministrationChart.jsx

@@ -9,7 +9,13 @@ import { Chart } from "../../components";
 import PropTypes from "prop-types";
 import { Color } from "../../components/chart/options/common";
 
-const HomeAdministrationChart = ({ config, formId, runNow, nextCall }) => {
+const HomeAdministrationChart = ({
+  config,
+  formId,
+  runNow,
+  nextCall,
+  identifier = "",
+}) => {
   const [dataset, setDataset] = useState([]);
   const [showEmpty, setShowEmpty] = useState(false);
   const [loading, setLoading] = useState(false);
@@ -42,7 +48,10 @@ const HomeAdministrationChart = ({ config, formId, runNow, nextCall }) => {
       api[type === "CRITERIA" ? "post" : "get"](
         url,
         type === "CRITERIA"
-          ? options.map((o) => ({ name: o.name, options: o.options }))
+          ? {
+              data: options.map((o) => ({ name: o.name, options: o.options })),
+              cache: `${identifier}-${title}`,
+            }
           : {}
       )
         .then((res) => {
@@ -89,7 +98,18 @@ const HomeAdministrationChart = ({ config, formId, runNow, nextCall }) => {
           setLoading(false);
         });
     }
-  }, [formId, id, notify, options, stack?.options, type, runNow, nextCall]);
+  }, [
+    formId,
+    id,
+    title,
+    identifier,
+    notify,
+    options,
+    stack?.options,
+    type,
+    runNow,
+    nextCall,
+  ]);
 
   const transformDataset = useMemo(() => {
     if (isStack) {