Feature/82 batch data upload

backend/api/v1/v1_data/management/commands/fake_pending_data_seeder.py

@@ -0,0 +1,159 @@
+from datetime import timedelta
+
+import pandas as pd
+from django.core.management import BaseCommand
+from django.utils import timezone
+from faker import Faker
+
+from api.v1.v1_data.models import PendingFormData, \
+    PendingAnswers
+from api.v1.v1_forms.constants import QuestionTypes
+from api.v1.v1_forms.models import Forms
+from api.v1.v1_profile.models import Administration, Levels
+from api.v1.v1_users.models import SystemUser
+
+fake = Faker()
+
+
+def set_answer_data(data, question):
+    name = None
+    value = None
+    option = None
+
+    if question.type == QuestionTypes.geo:
+        option = data.geo
+    elif question.type == QuestionTypes.administration:
+        name = data.administration.name
+        value = data.administration.id
+    elif question.type == QuestionTypes.text:
+        name = fake.company() if question.meta else fake.sentence(nb_words=3)
+    elif question.type == QuestionTypes.number:
+        value = fake.random_int(min=10, max=50)
+    elif question.type == QuestionTypes.option:
+        option = [
+            question.question_question_options.order_by('?').first().name
+        ]
+    elif question.type == QuestionTypes.multiple_option:
+        option = list(
+            question.question_question_options.order_by('?').values_list(
+                'name', flat=True)[0:fake.random_int(min=1, max=3)])
+    elif question.type == QuestionTypes.photo:
+        name = fake.image_url()
+    elif question.type == QuestionTypes.date:
+        name = fake.date_between_dates(
+            date_start=timezone.datetime.now().date() - timedelta(days=90),
+            date_end=timezone.datetime.now().date())
+    elif question.type == QuestionTypes.date:
+        name = fake.date_between_dates(
+            date_start=timezone.datetime.now().date() - timedelta(days=90),
+            date_end=timezone.datetime.now().date())
+    else:
+        pass
+    return name, value, option
+
+
+def add_fake_answers(data: PendingFormData):
+    form = data.form
+    meta_name = []
+    for question in form.form_questions.all().order_by('order'):
+        name, value, option = set_answer_data(data, question)
+        if question.meta:
+            if name:
+                meta_name.append(name)
+            elif option and question.type != QuestionTypes.geo:
+                meta_name.append(','.join(option))
+            elif value and question.type != QuestionTypes.administration:
+                meta_name.append(str(value))
+            else:
+                pass
+
+        if question.type == QuestionTypes.administration:
+            name = None
+
+        PendingAnswers.objects.create(
+            pending_data=data,
+            question=question,
+            name=name,
+            value=value,
+            options=option,
+            created_by=SystemUser.objects.order_by('?').first())
+    data.name = ' - '.join(meta_name)
+    data.save()
+
+
+def seed_data(form, fake_geo, level_names, repeat, test):
+    for i in range(repeat):
+        geo = fake_geo.iloc[i].to_dict()
+        data = PendingFormData.objects.create(
+            name=fake.pystr_format(),
+            geo=[geo["X"], geo["Y"]],
+            form=form,
+            administration=Administration.objects.first(),
+            created_by=SystemUser.objects.order_by('?').first())
+        level_id = 1
+        # admin_ids = []
+        if not test:
+            for level_name in level_names:
+                level = level_name.split("_")
+                administration = Administration.objects.filter(
+                    parent_id=level_id,
+                    level=Levels.objects.filter(level=level[1]).first(),
+                    name=geo[level_name]).first()
+                level_id = administration.id
+                data.administration = administration
+                data.save()
+        add_fake_answers(data)
+
+
+class Command(BaseCommand):
+    def add_arguments(self, parser):
+        parser.add_argument("-r",
+                            "--repeat",
+                            nargs="?",
+                            const=20,
+                            default=20,
+                            type=int)
+        parser.add_argument("-t",
+                            "--test",
+                            nargs="?",
+                            const=False,
+                            default=False,
+                            type=bool)
+
+    def handle(self, *args, **options):
+        PendingFormData.objects.all().delete()
+        fake_geo = pd.read_csv("./source/kenya_random_points.csv")
+        level_names = list(
+            filter(lambda x: True if "NAME_" in x else False, list(fake_geo)))
+        for form in Forms.objects.all():
+            print(f"Seeding - {form.name}")
+            seed_data(form, fake_geo, level_names, options.get("repeat"),
+                      options.get("test"))
+
+
+"""
+Form1 - 1
+Form2 - 2
+
+Admin - 3 level 3
+Admin - 5 level 4
+Admin - 7 level 4
+
+Admin - 4 level 4
+Admin - 6 level 3
+
+Form1:
+ - User1 admin 3 1.3
+ - User2 admin 5
+ - User2 admin 7 1.3
+
+Form1:
+ - User3 admin 6
+ - User4 admin 4
+
+PendingFormData
+form - Form1
+data - data1
+administration - Admin 7 parent=3 [1.3.5.7.]
+
+"""

backend/api/v1/v1_data/management/commands/form_approval_assignment_seeder.py

@@ -0,0 +1,45 @@
+from django.core.management import BaseCommand
+from faker import Faker
+
+from api.v1.v1_forms.models import FormApprovalRule, FormApprovalAssignment
+from api.v1.v1_profile.constants import UserRoleTypes
+from api.v1.v1_profile.models import Administration, Access
+from api.v1.v1_users.models import SystemUser
+
+
+class Command(BaseCommand):
+
+    def handle(self, *args, **options):
+        FormApprovalAssignment.objects.all().delete()
+        for rule in FormApprovalRule.objects.all():
+            previous = None
+            for level in rule.levels.all().order_by('level'):
+                if previous:
+                    filter_path = '{0}{1}.'.format(previous.path,
+                                                   previous.id)
+                else:
+                    filter_path = '{0}{1}.'.format(
+                        rule.administration.path,
+                        rule.administration.id)
+                administration = Administration.objects.filter(
+                    path__startswith=filter_path,
+                    level=level).first()
+                previous = administration
+                user = SystemUser.objects.filter(
+                    user_access__administration=administration).first()
+                if not user:
+                    fake = Faker()
+                    profile = fake.profile()
+                    name = profile.get("name").split(" ")
+                    user = SystemUser.objects.create_user(
+                        email=profile.get("mail"),
+                        first_name=name[0],
+                        last_name=name[1])
+                    Access.objects.create(user=user,
+                                          role=UserRoleTypes.approver,
+                                          administration=administration)
+                FormApprovalAssignment.objects.create(
+                    form=rule.form,
+                    administration=administration,
+                    user=user
+                )

backend/api/v1/v1_data/migrations/0002_pendingformdata_batch.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.0.1 on 2022-02-23 09:10
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('v1_data', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='pendingformdata',
+            name='batch',
+            field=models.UUIDField(default=None, null=True),
+        ),
+    ]

backend/api/v1/v1_data/migrations/0003_pendingdataapproval_level.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.0.1 on 2022-02-23 09:12
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('v1_profile', '0004_alter_administration_path'),
+        ('v1_data', '0002_pendingformdata_batch'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='pendingdataapproval',
+            name='level',
+            field=models.ForeignKey(default=1,
+                                    on_delete=django.db.models.deletion.CASCADE,
+                                    related_name='level_assigned_pending_data',
+                                    to='v1_profile.levels'),
+            preserve_default=False,
+        ),
+    ]

backend/api/v1/v1_data/models.py

@@ -3,7 +3,7 @@
 # Create your models here.
 from api.v1.v1_data.constants import DataApprovalStatus
 from api.v1.v1_forms.models import Forms, Questions
-from api.v1.v1_profile.models import Administration
+from api.v1.v1_profile.models import Administration, Levels
 from api.v1.v1_users.models import SystemUser
 
 
@@ -49,6 +49,7 @@ class PendingFormData(models.Model):
         related_name='administration_pending_form_data')  # noqa
     geo = models.JSONField(null=True, default=None)
     approved = models.BooleanField(default=False)
+    batch = models.UUIDField(default=None, null=True)
     created_by = models.ForeignKey(to=SystemUser,
                                    on_delete=models.CASCADE,
                                    related_name='pending_form_data_created')
@@ -69,6 +70,9 @@ class PendingDataApproval(models.Model):
     user = models.ForeignKey(to=SystemUser,
                              on_delete=models.CASCADE,
                              related_name='user_assigned_pending_data')
+    level = models.ForeignKey(to=Levels,
+                              on_delete=models.CASCADE,
+                              related_name='level_assigned_pending_data')
     status = models.IntegerField(choices=DataApprovalStatus.FieldStr.items(),
                                  default=DataApprovalStatus.pending)
 

backend/api/v1/v1_forms/management/commands/form_approval_seeder.py

@@ -3,22 +3,32 @@
 from django.core.management import BaseCommand
 
 from api.v1.v1_forms.models import Forms, FormApprovalRule
+from api.v1.v1_forms.constants import FormTypes
 from api.v1.v1_profile.models import Access, Levels
 
 
 class Command(BaseCommand):
     def handle(self, *args, **options):
         FormApprovalRule.objects.all().delete()
         for form in Forms.objects.all():
-            for user in Access.objects.filter(
-                    administration__level=Levels.objects.filter(
-                        level=1).first()):
-                randoms = Levels.objects.filter(level__gt=1).count()
-                randoms = [n + 1 for n in range(randoms)]
-                limit = random.choices(randoms)
-                levels = Levels.objects.filter(
-                    level__gt=1).order_by('?')[:limit[0]]
-                rule = FormApprovalRule.objects.create(
-                    form=form, administration=user.administration)
-                rule.levels.set(levels)
-                rule.save()
+            if form.type == FormTypes.county:
+                for user in Access.objects.filter(
+                        administration__level=Levels.objects.filter(
+                            level=1).first()).distinct('administration_id'):
+                    randoms = Levels.objects.filter(level__gt=1).count()
+                    randoms = [n + 1 for n in range(randoms)]
+                    limit = random.choices(randoms)
+                    levels = Levels.objects.filter(
+                        level__gt=1).order_by('?')[:limit[0]]
+                    rule = FormApprovalRule.objects.create(
+                        form=form, administration=user.administration)
+                    rule.levels.set(levels)
+                    rule.save()
+            else:
+                for user in Access.objects.filter(
+                        administration__level=Levels.objects.filter(
+                            level=0).first()).distinct('administration_id'):
+                    rule = FormApprovalRule.objects.create(
+                        form=form, administration=user.administration)
+                    rule.levels.set(Levels.objects.filter(level=0))
+                    rule.save()

backend/api/v1/v1_profile/management/commands/assign_access.py

@@ -37,9 +37,14 @@ def handle(self, *args, **options):
                 level=Levels.objects.filter(
                     level=1).first()).order_by('?').first()
             role = UserRoleTypes.admin
-        Access.objects.create(user=user,
-                              role=role,
-                              administration=administration)
+        access = Access.objects.filter(user=user).first()
+        if not access:
+            Access.objects.create(user=user,
+                                  role=role,
+                                  administration=administration)
+        else:
+            access.administration = administration
+            access.save()
         if not options.get("test"):
             self.stdout.write(
                 f"{user.email} now has access to {administration.name}")