Merge branch 'develop' into feature/24-incorporating-new-questionnaires

backend/api/v1/v1_categories/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class V1CategoriesConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'api.v1.v1_categories'

backend/api/v1/v1_categories/functions.py

@@ -0,0 +1,117 @@
+import json
+import pandas as pd
+from nwmis.settings import MASTER_DATA
+from django.db import transaction, connection
+
+
+@transaction.atomic
+def refresh_data_category_views():
+    with connection.cursor() as cursor:
+        cursor.execute(
+            """
+            REFRESH MATERIALIZED VIEW data_category;
+            """
+        )
+
+
+def validate_number(q, answer):
+    aw = float(answer[0])
+    op = q.get("number")
+    ok = False
+    if "greater_than" in op:
+        ok = aw > op.get("greater_than")
+    if "less_than" in op:
+        ok = aw < op.get("less_than")
+    if "equal" in op:
+        ok = aw == op.get("equal")
+    if "greater_than_equal" in op:
+        ok = aw >= op.get("greater_than_equal")
+    if "less_than_equal" in op:
+        ok = aw <= op.get("less_than_equal")
+    return ok
+
+
+def get_valid_list(opt, c, category):
+    validator = [q["id"] for q in c["questions"]]
+    valid = []
+    exit = False
+    for q in c["questions"]:
+        if exit:
+            continue
+        answer = opt.get(str(q["id"]))
+        if not answer:
+            opt.update({str(q["id"]): None})
+            continue
+        if q.get("number"):
+            is_valid = validate_number(q, answer)
+            if is_valid:
+                valid.append(q["id"])
+            else:
+                elses = q.get("else")
+                category = elses.get("name")
+                exit = True
+        if q.get("options"):
+            if len(set(q["options"]).intersection(answer)):
+                valid.append(q["id"])
+            # TODO Merge else with above
+            else:
+                if q.get("else"):
+                    elses = q.get("else")
+                    if elses.get("name"):
+                        category = elses.get("name")
+                        exit = True
+                    if elses.get("ignore"):
+                        validator = list(
+                            filter(
+                                lambda x: x not in elses.get("ignore"),
+                                validator,
+                            )
+                        )
+                        valid.append(q["id"])
+                if q.get("other"):
+                    for o in q.get("other"):
+                        if len(set(o["options"]).intersection(answer)):
+                            exit = True
+                            if len(o.get("questions")):
+                                category = get_valid_list(opt, o, category)
+                            else:
+                                category = o.get("name")
+    if len(valid) >= len(validator):
+        conditions = [v if v in valid else False for v in validator]
+        conditions = list(filter(lambda x: x is not False, conditions))
+        if sorted(conditions) == sorted(validator):
+            category = c["name"]
+    if sorted(valid) == sorted(validator):
+        category = c["name"]
+    return category
+
+
+def get_category(opt: dict):
+    file_config = f"{MASTER_DATA}/config/category.json"
+    with open(file_config) as config_file:
+        configs = json.load(config_file)
+    category = False
+    for config in configs:
+        for c in config["categories"]:
+            category = get_valid_list(opt, c, category)
+    return category
+
+
+def get_category_results(data):
+    categories = [d.serialize for d in data]
+    df = pd.DataFrame(categories)
+    results = df.to_dict("records")
+    for d in results:
+        d.update({"category": get_category(d["opt"])})
+    res = pd.DataFrame(results)
+    res = pd.concat([res.drop("opt", axis=1), pd.DataFrame(df["opt"].tolist())], axis=1)
+    res = res[
+        [
+            "id",
+            "data",
+            "form",
+            "name",
+            "category",
+        ]
+    ]
+    return res.to_dict("records")

backend/api/v1/v1_categories/management/commands/generate_views.py

@@ -0,0 +1,72 @@
+import os
+import json
+from django.core.management import BaseCommand
+from nwmis.settings import MASTER_DATA
+from django.db import connection
+
+
+def drop_schema():
+    with connection.cursor() as cursor:
+        cursor.execute(
+            """
+                SELECT EXISTS(
+                SELECT relname FROM pg_class
+                WHERE relkind = 'm' AND relname = %s)
+                """, ['data_category'])
+        exists = cursor.fetchone()[0]
+        if exists:
+            cursor.execute("DROP MATERIALIZED VIEW data_category;")
+
+
+def get_question_config(config: dict, cl: list):
+    for q in config.get("questions"):
+        cl.append(str(q["id"]))
+        if q.get("other"):
+            for o in q.get("other"):
+                cl = get_question_config(config=o, cl=cl)
+    return cl
+
+
+def generate_schema() -> str:
+    file_config = open(f"{MASTER_DATA}/config/category.json")
+    configs = json.load(file_config)
+    file_config.close()
+
+    mview = "CREATE MATERIALIZED VIEW data_category AS\n"
+    mview += "SELECT ROW_NUMBER() OVER (PARTITION BY TRUE) as id, *\n"
+    mview += "FROM (\n"
+    for main_union, config in enumerate(configs):
+        question_config = []
+        for c in config["categories"]:
+            question_config = get_question_config(config=c, cl=question_config)
+        ql = ",".join(question_config)
+        mview += (f"SELECT q.form_id, a.data_id, '{config['name']}' as name,"
+                  " jsonb_object_agg(a.question_id,COALESCE(a.options,"
+                  " to_jsonb(array[a.value]))) as options \n")
+        mview += "FROM answer a \n"
+        mview += "LEFT JOIN question q ON q.id = a.question_id \n"
+        mview += "WHERE (a.value IS NOT NULL OR a.options IS NOT NULL) \n"
+        mview += f"AND q.id IN ({ql}) GROUP BY q.form_id, a.data_id\n"
+        if main_union < len(configs) - 1:
+            mview += " UNION "
+    mview += ") as categories;"
+    return mview
+
+
+def category_json_availability(apps, schema_editor):
+    if os.path.exists("{MASTER_DATA}/config/category.json"):
+        return [('v1_categories', 'category_json_is_available')]
+    else:
+        return []
+
+
+class Command(BaseCommand):
+
+    def handle(self, *args, **options):
+        try:
+            views = generate_schema()
+            drop_schema()
+            with connection.cursor() as cursor:
+                cursor.execute(views)
+        except FileNotFoundError:
+            print(f"{MASTER_DATA}/category.json is not exist")

backend/api/v1/v1_categories/management/commands/refresh_views.py

@@ -0,0 +1,7 @@
+from django.core.management import BaseCommand
+from api.v1.v1_categories.functions import refresh_data_category_views
+
+
+class Command(BaseCommand):
+    def handle(self, *args, **options):
+        refresh_data_category_views()

backend/api/v1/v1_categories/migrations/0001_initial.py

@@ -0,0 +1,32 @@
+# Generated by Django 4.0.4 on 2023-04-07 04:19
+
+from django.db import migrations, models
+from ..management.commands.generate_views import generate_schema
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('v1_data', '0023_viewjmpcount')
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='DataCategory',
+            fields=[
+                ('id', models.IntegerField(primary_key=True, serialize=False)),
+                ('name', models.CharField(max_length=50)),
+                ('options', models.JSONField()),
+            ],
+            options={
+                'db_table': 'data_category',
+                'managed': False,
+            },
+        ),
+        migrations.RunSQL(
+            generate_schema(), """
+            DROP MATERIALIZED VIEW data_category;
+            """)
+    ]

backend/api/v1/v1_categories/models.py

@@ -0,0 +1,29 @@
+from django.db import models
+from api.v1.v1_forms.models import Forms
+from api.v1.v1_data.models import FormData
+
+
+class DataCategory(models.Model):
+    id = models.IntegerField(primary_key=True)
+    name = models.CharField(max_length=50)
+    data = models.ForeignKey(to=FormData,
+                             on_delete=models.DO_NOTHING,
+                             related_name='data_view_data_category')
+    form = models.ForeignKey(to=Forms,
+                             on_delete=models.DO_NOTHING,
+                             related_name='form_view_data_category')
+    options = models.JSONField()
+
+    @property
+    def serialize(self):
+        return {
+            "id": self.id,
+            "name": self.name,
+            "data": self.data_id,
+            "form": self.form_id,
+            "opt": self.options,
+        }
+
+    class Meta:
+        managed = False
+        db_table = 'data_category'

backend/api/v1/v1_categories/urls.py

@@ -0,0 +1,10 @@
+from django.urls import re_path
+from api.v1.v1_categories.views import get_data_with_category, get_raw_data_point
+
+urlpatterns = [
+    re_path(
+        r"^(?P<version>(v1))/form-data-category/(?P<form_id>[0-9]+)",
+        get_data_with_category,
+    ),
+    re_path(r"^(?P<version>(v1))/raw-data/(?P<form_id>[0-9]+)", get_raw_data_point),
+]