Feature/1404 data generator jobs

backend/api/v1/v1_data/views.py

@@ -1,4 +1,8 @@
 # Create your views here.
+import pandas as pd
+import os
+import pathlib
+
 from math import ceil
 from collections import defaultdict
 from datetime import datetime, date
@@ -96,7 +100,7 @@
 )
 from utils.custom_serializer_fields import validate_serializers_message
 from utils.default_serializers import DefaultResponseSerializer
-from utils.export_form import generate_excel
+from utils.export_form import blank_data_template
 
 period_length = 60 * 15
 
@@ -1276,7 +1280,16 @@ def get(self, request, batch_id, version):
 @permission_classes([IsAuthenticated])
 def export_form_data(request, version, form_id):
     form = get_object_or_404(Forms, pk=form_id)
-    filepath = generate_excel(form=form)
+    form_name = form.name
+    filename = f"{form.id}-{form_name}"
+    directory = "tmp"
+    pathlib.Path(directory).mkdir(parents=True, exist_ok=True)
+    filepath = f"./{directory}/{filename}.xlsx"
+    if os.path.exists(filepath):
+        os.remove(filepath)
+    writer = pd.ExcelWriter(filepath, engine="xlsxwriter")
+    blank_data_template(form=form, writer=writer)
+    writer.save()
     filename = filepath.split("/")[-1].replace(" ", "-")
     zip_file = open(filepath, "rb")
     response = HttpResponse(

backend/api/v1/v1_jobs/job.py

@@ -31,7 +31,9 @@
 from utils.email_helper import send_email, EmailTypes
 from utils.export_form import (
     generate_definition_sheet,
-    rearrange_definition_columns
+    get_question_names,
+    blank_data_template,
+    meta_columns,
 )
 from utils.functions import update_date_time_format
 from utils.storage import upload
@@ -40,7 +42,7 @@
 logger = logging.getLogger(__name__)
 
 
-def download(
+def download_data(
     form: Forms,
     administration_ids,
     download_type="all",
@@ -67,7 +69,34 @@ def download(
     return [d.to_data_frame for d in data.order_by('id')]
 
 
-def job_generate_download(job_id, **kwargs):
+def generate_data_sheet(
+    writer: pd.ExcelWriter,
+    form: Forms,
+    administration_ids=None,
+    submission_type: SubmissionTypes = None,
+    download_type: str = "all",
+) -> None:
+    question_names = get_question_names(form=form)
+    data = download_data(
+        form=form,
+        administration_ids=None,
+        submission_type=submission_type,
+        download_type=download_type
+    )
+    if len(data):
+        df = pd.DataFrame(data)
+        for question_name in question_names:
+            if question_name not in df:
+                df[question_name] = None
+        # Reorder columns
+        df = df[meta_columns + question_names]
+        df.to_excel(writer, sheet_name='data', index=False)
+        generate_definition_sheet(form=form, writer=writer)
+    else:
+        blank_data_template(form=form, writer=writer)
+
+
+def job_generate_data_download(job_id, **kwargs):
     job = Jobs.objects.get(pk=job_id)
     file_path = './tmp/{0}'.format(job.result)
     if os.path.exists(file_path):
@@ -95,18 +124,16 @@ def job_generate_download(job_id, **kwargs):
     form = Forms.objects.get(pk=job.info.get('form_id'))
     download_type = kwargs.get('download_type')
     submission_type = kwargs.get('submission_type')
-    data = download(
+    writer = pd.ExcelWriter(file_path, engine='xlsxwriter')
+
+    generate_data_sheet(
+        writer=writer,
         form=form,
         administration_ids=administration_ids,
-        download_type=download_type,
         submission_type=submission_type,
+        download_type=download_type,
     )
-    df = pd.DataFrame(data)
-    col_names = rearrange_definition_columns(list(df))
-    df = df[col_names]
-    writer = pd.ExcelWriter(file_path, engine='xlsxwriter')
-    df.to_excel(writer, sheet_name='data', index=False)
-    generate_definition_sheet(form=form, writer=writer)
+
     context = [{
         "context": "Form Name",
         "value": form.name
@@ -145,7 +172,7 @@ def job_generate_download(job_id, **kwargs):
     return url
 
 
-def job_generate_download_result(task):
+def job_generate_data_download_result(task):
     job = Jobs.objects.get(task_id=task.id)
     job.attempt = job.attempt + 1
     if task.success:

backend/api/v1/v1_jobs/management/commands/generate_excel_data.py

@@ -3,13 +3,8 @@
 from django.core.management.base import BaseCommand
 
 from api.v1.v1_forms.models import Forms, SubmissionTypes
-from api.v1.v1_jobs.job import (
-    download,
-    rearrange_definition_columns,
-    generate_definition_sheet,
-)
+from api.v1.v1_jobs.job import generate_data_sheet
 from utils.storage import upload
-from utils.export_form import generate_excel
 
 CRONJOB_RESULT_DIR = "cronjob_results"
 submission_types_obj = {
@@ -50,24 +45,16 @@ def handle(self, *args, **options):
             exit()
         form = Forms.objects.get(pk=form_id)
         form_name = form.name.replace(" ", "_").lower()
-
-        data = download(
+        process_file = f"process-{form_name}.xlsx"
+        writer = pd.ExcelWriter(process_file, engine="xlsxwriter")
+        generate_data_sheet(
+            writer=writer,
             form=form,
             administration_ids=None,
             submission_type=submission_type,
             download_type=download_type
         )
-        process_file = f"process-{form_name}.xlsx"
-        if len(data):
-            df = pd.DataFrame(data)
-            col_names = rearrange_definition_columns(list(df))
-            df = df[col_names]
-            writer = pd.ExcelWriter(process_file, engine='xlsxwriter')
-            df.to_excel(writer, sheet_name='data', index=False)
-            generate_definition_sheet(form=form, writer=writer)
-            writer.save()
-        else:
-            process_file = generate_excel(form=form)
+        writer.save()
 
         out_file = "-".join(list(filter(lambda x: x, [
             form_name,

backend/api/v1/v1_jobs/management/commands/job_download.py

@@ -54,10 +54,10 @@ def handle(self, *args, **options):
             result=out_file,
         )
         task_id = async_task(
-            "api.v1.v1_jobs.job.job_generate_download",
+            "api.v1.v1_jobs.job.job_generate_data_download",
             job.id,
             **info,
-            hook="api.v1.v1_jobs.job.job_generate_download_result"
+            hook="api.v1.v1_jobs.job.job_generate_data_download_result"
         )
         job.task_id = task_id
         job.save()

backend/api/v1/v1_jobs/tests/tests_bulk_data_download.py

@@ -5,7 +5,7 @@
 from django.test.utils import override_settings
 from api.v1.v1_forms.models import Questions, Forms
 from api.v1.v1_data.models import FormData, SubmissionTypes
-from api.v1.v1_jobs.job import download, generate_definition_sheet
+from api.v1.v1_jobs.job import download_data, generate_definition_sheet
 
 
 @override_settings(USE_TZ=False)
@@ -27,7 +27,7 @@ def test_data_download_list_of_columns(self):
         self.assertTrue(form_data)
         form_data = FormData.objects.first()
         administration = form_data.administration
-        download_response = download(form_data.form, [administration.id])
+        download_response = download_data(form_data.form, [administration.id])
         self.assertTrue(download_response)
         download_columns = list(download_response[0].keys())
         questions = Questions.objects.filter(form=form_data.form).values_list(
@@ -41,7 +41,7 @@ def test_data_download_list_of_columns(self):
         self.assertEqual(list(columns).sort(), list(questions).sort())
 
         # test if the download recent data is successful
-        download_response = download(
+        download_response = download_data(
             form_data.form,
             [administration.id],
             "recent")
@@ -63,7 +63,7 @@ def test_data_download_for_certification(self):
         self.assertTrue(form_data.count())
         form_data = form_data.first()
         administration = form_data.administration
-        download_response = download(
+        download_response = download_data(
             form_data.form,
             [administration.id],
             download_type="all",
@@ -79,7 +79,7 @@ def test_data_download_for_empty_verification(self):
         self.assertFalse(form_data.count())
 
         form = Forms.objects.get(pk=1)
-        download_response = download(
+        download_response = download_data(
             form=form,
             administration_ids=None,
             download_type="all",

backend/api/v1/v1_jobs/tests/tests_generate_excel_data_command.py

@@ -1,3 +1,4 @@
+import pandas as pd
 from django.core.management import call_command
 from django.test import TestCase
 from django.test.utils import override_settings
@@ -43,7 +44,12 @@ def test_download_all_data(self):
         )
 
         result_file = f"{CRONJOB_RESULT_DIR}/{form_name}-routine-recent.xlsx"
-        self.assertTrue(storage.check(result_file))
+        # self.assertTrue(storage.check(result_file))
+
+        download_file = storage.download(result_file)
+        df = pd.read_excel(download_file)
+        self.assertTrue(df.shape[0])
+
         storage.delete(result_file)
 
     def test_download_data_by_submission_type(self):

backend/api/v1/v1_jobs/tests/tests_job_download_command.py

@@ -3,7 +3,9 @@
 from django.test.utils import override_settings
 from api.v1.v1_forms.models import Forms
 from api.v1.v1_jobs.models import Jobs
-from api.v1.v1_jobs.job import job_generate_download
+from api.v1.v1_jobs.job import (
+    job_generate_data_download,
+)
 from api.v1.v1_users.models import SystemUser
 from api.v1.v1_profile.constants import UserRoleTypes
 
@@ -36,8 +38,10 @@ def test_download_all_data(self):
             "-t",
             "all",
         )
-        job = Jobs.objects.get(pk=result)
         self.assertTrue(result)
+
+        job = Jobs.objects.get(pk=result)
         self.assertEqual(job.info.get("download_type"), "all")
-        url = job_generate_download(job_id=job.id, **job.info)
+
+        url = job_generate_data_download(job_id=job.id, **job.info)
         self.assertTrue("download-test_form" in url)

backend/api/v1/v1_jobs/views.py

@@ -39,7 +39,6 @@
 )
 from utils import storage
 from utils.custom_serializer_fields import validate_serializers_message
-from utils.storage import download
 
 
 @extend_schema(
@@ -158,7 +157,7 @@ def download_file(request, version, file_name):
     job = get_object_or_404(Jobs, result=file_name)
     type = request.GET.get("type") if request.GET.get("type") else "download"
     url = f"{type}/{job.result}"
-    filepath = download(url)
+    filepath = storage.download(url)
     filename = job.result
     zip_file = open(filepath, "rb")
     response = HttpResponse(

backend/api/v1/v1_profile/views.py

@@ -34,7 +34,7 @@
 from api.v1.v1_profile.job import create_download_job
 from api.v1.v1_users.models import SystemUser
 from api.v1.v1_jobs.constants import JobTypes
-from utils.upload_administration import generate_excel
+from utils.upload_administration import generate_administration_excel
 from utils.custom_helper import clean_array_param, maybe_int
 from utils.default_serializers import DefaultResponseSerializer
 from utils.custom_pagination import Pagination
@@ -277,7 +277,9 @@ def export_administrations_template(request: Request, version):
     attributes = clean_array_param(
         request.query_params.get("attributes", ""), maybe_int
     )
-    filepath = generate_excel(cast(SystemUser, request.user), attributes)
+    filepath = generate_administration_excel(
+        cast(SystemUser, request.user), attributes
+    )
     filename = filepath.split("/")[-1].replace(" ", "-")
     with open(filepath, "rb") as template_file:
         response = HttpResponse(

backend/utils/export_form.py

@@ -1,6 +1,3 @@
-import os
-import pathlib
-
 import pandas as pd
 
 from api.v1.v1_forms.models import Forms, Questions
@@ -11,6 +8,18 @@
                 "geolocation", "submission_type"]
 
 
+def get_question_names(form: Forms):
+    questions = []
+    question_groups = form.form_question_group.all().order_by("order")
+    for q in question_groups:
+        questions.extend(
+            q.question_group_question.all().order_by(
+                "order").values_list(
+                    "name", flat=True)
+        )
+    return questions
+
+
 def get_definition(form: Forms):
     questions = (
         Questions.objects.filter(form=form)
@@ -115,7 +124,7 @@ def rearrange_definition_columns(col_names: list):
     return col_names
 
 
-def generate_excel(form: Forms):
+def blank_data_template(form: Forms, writer: pd.ExcelWriter):
     questions = questions = (
         Questions.objects.filter(form=form)
         .order_by("question_group__order", "order")
@@ -128,14 +137,6 @@ def generate_excel(form: Forms):
     cols = list(data)
     col_names = rearrange_definition_columns(cols)
     data = data[col_names]
-    form_name = form.name
-    filename = f"{form.id}-{form_name}"
-    directory = "tmp"
-    pathlib.Path(directory).mkdir(parents=True, exist_ok=True)
-    filepath = f"./{directory}/{filename}.xlsx"
-    if os.path.exists(filepath):
-        os.remove(filepath)
-    writer = pd.ExcelWriter(filepath, engine="xlsxwriter")
     data.to_excel(
         writer, sheet_name="data", startrow=1, header=False, index=False
     )
@@ -147,5 +148,3 @@ def generate_excel(form: Forms):
     for col_num, value in enumerate(data.columns.values):
         worksheet.write(0, col_num, value, header_format)
     generate_definition_sheet(form=form, writer=writer)
-    writer.save()
-    return filepath

backend/utils/upload_administration.py

@@ -44,7 +44,7 @@ def generate_template(
     writer.save()
 
 
-def generate_excel(
+def generate_administration_excel(
     user: SystemUser,
     attributes: List[int] = [],
 ):
@@ -62,12 +62,12 @@ def generate_excel(
 
 
 def generate_administration_template(
-    job_result: str,
+    file_path: str,
     attributes: List[int] = [],
     level: int = None,
     adm_id: int = None,
 ):
-    file_path = "./tmp/{0}".format(job_result.replace("/", "_"))
+    file_path = "./tmp/{0}".format(file_path.replace("/", "_"))
     if os.path.exists(file_path):
         os.remove(file_path)
     level_headers = [