From 19a093d73e8c38732568ff1f09f6d81fb7ed359f Mon Sep 17 00:00:00 2001 From: Daniel Rabstejnek Date: Wed, 9 Aug 2023 23:11:54 -0400 Subject: [PATCH 01/35] Preliminary work on replacing prefilter logic with filtersets --- hawc/apps/common/filterset.py | 2 +- hawc/apps/common/forms.py | 26 +++++ .../common/templates/common/dynamic_form.html | 2 + hawc/apps/common/widgets.py | 33 ++++++ hawc/apps/summary/forms.py | 30 ++++- hawc/apps/summary/prefilters.py | 110 ++++++++++++++++++ .../templates/summary/datapivot_form1.html | 42 +++++++ hawc/apps/summary/urls.py | 5 + hawc/apps/summary/views.py | 17 ++- 9 files changed, 262 insertions(+), 5 deletions(-) create mode 100644 hawc/apps/common/templates/common/dynamic_form.html create mode 100644 hawc/apps/summary/prefilters.py create mode 100644 hawc/apps/summary/templates/summary/datapivot_form1.html diff --git a/hawc/apps/common/filterset.py b/hawc/apps/common/filterset.py index 1594fb3a69..6d535351d2 100644 --- a/hawc/apps/common/filterset.py +++ b/hawc/apps/common/filterset.py @@ -171,7 +171,7 @@ def create_form(self): form = form_class(self.data, prefix=self.form_prefix, **self.form_kwargs) else: form = form_class(prefix=self.form_prefix, **self.form_kwargs) - if form.dynamic_fields: # removes unwanted fields from a filterset if specified + if getattr(form,"dynamic_fields",None): # removes unwanted fields from a filterset if specified for field in list(form.fields.keys()): if field not in form.dynamic_fields and field != "is_expanded": form.fields.pop(field) diff --git a/hawc/apps/common/forms.py b/hawc/apps/common/forms.py index 12a4f6d456..25675ecaa0 100644 --- a/hawc/apps/common/forms.py +++ b/hawc/apps/common/forms.py @@ -452,3 +452,29 @@ def validate(self, value): super().validate(value) if value != self.check_value: raise forms.ValidationError(f'The value of "{self.check_value}" is required.') + +class DynamicFormField(forms.JSONField): + """Field to display dynamic form inline.""" + + default_error_messages = {"invalid": "Invalid input"} + widget = widgets.DynamicFormWidget + + def __init__(self, prefix, form_class, form_kwargs=None, *args, **kwargs): + """Create dynamic form field.""" + self.form_class = form_class + self.form_kwargs = {} if form_kwargs is None else form_kwargs + self.widget = self.widget(prefix, form_class, form_kwargs) + super().__init__(*args, **kwargs) + + def bound_data(self, data, initial): + """Get data to be shown for this field on render.""" + if self.disabled: + return initial + return data + + def validate(self, value): + """Validate inline form.""" + super().validate(value) + form = self.form_class(data=value, **self.form_kwargs) + if not form.is_valid(): + raise forms.ValidationError(self.error_messages["invalid"]) \ No newline at end of file diff --git a/hawc/apps/common/templates/common/dynamic_form.html b/hawc/apps/common/templates/common/dynamic_form.html new file mode 100644 index 0000000000..492689a19e --- /dev/null +++ b/hawc/apps/common/templates/common/dynamic_form.html @@ -0,0 +1,2 @@ +{% load crispy_forms_tags %} +{% crispy widget.value %} diff --git a/hawc/apps/common/widgets.py b/hawc/apps/common/widgets.py index 8618e9f4c3..7d7c2bae98 100644 --- a/hawc/apps/common/widgets.py +++ b/hawc/apps/common/widgets.py @@ -1,4 +1,5 @@ from random import randint +import json from django.conf import settings from django.forms import ValidationError @@ -10,6 +11,7 @@ SelectMultiple, Textarea, TextInput, + Widget ) from django.utils import timezone @@ -124,3 +126,34 @@ def build_attrs(self, base_attrs, extra_attrs=None): class_name = attrs.get("class") attrs["class"] = class_name + " quilltext" if class_name else "quilltext" return attrs + +class DynamicFormWidget(Widget): + """Widget to display dynamic form inline.""" + + template_name = "common/dynamic_form.html" + + def __init__(self, prefix, form_class, form_kwargs=None, *args, **kwargs): + """Create dynamic form widget.""" + super().__init__(*args, **kwargs) + self.prefix = prefix + self.form_class = form_class + if form_kwargs is None: + form_kwargs = {} + self.form_kwargs = {"prefix": prefix, **form_kwargs} + + def add_prefix(self, field_name): + """Add prefix in the same way Django forms add prefixes.""" + return f"{self.prefix}-{field_name}" + + def format_value(self, value): + """Value used in rendering.""" + value = json.loads(value) + if value: + value = {self.add_prefix(k): v for k, v in value.items()} + return self.form_class(data=value, **self.form_kwargs) + + def value_from_datadict(self, data, files, name): + """Parse value from POST request.""" + form = self.form_class(data=data, **self.form_kwargs) + form.full_clean() + return form.cleaned_data \ No newline at end of file diff --git a/hawc/apps/summary/forms.py b/hawc/apps/summary/forms.py index aa892f6510..367e396eaf 100644 --- a/hawc/apps/summary/forms.py +++ b/hawc/apps/summary/forms.py @@ -13,7 +13,7 @@ from ..assessment.models import DoseUnits, EffectTag from ..common import validators from ..common.autocomplete import AutocompleteChoiceField -from ..common.forms import BaseFormHelper, QuillField, check_unique_for_assessment +from ..common.forms import BaseFormHelper, QuillField, check_unique_for_assessment, DynamicFormField from ..common.helper import new_window_a from ..common.validators import validate_html_tags, validate_hyperlinks, validate_json_pydantic from ..epi.models import Outcome @@ -21,7 +21,7 @@ from ..lit.models import ReferenceFilterTag from ..study.autocomplete import StudyAutocomplete from ..study.models import Study -from . import autocomplete, constants, models +from . import autocomplete, constants, models, prefilters class PrefilterMixin: @@ -1078,6 +1078,32 @@ def clean(self): self.add_error("excel_file", "Must contain at least 2 columns.") +class DataPivotQueryForm1(DataPivotForm): + class Meta: + model = models.DataPivotQuery + fields = ( + "title", + "slug", + "evidence_type", + "export_style", + "preferred_units", + "settings", + "caption", + "published", + "published_only", + "prefilters", + ) + + def _get_prefilter_form(self,data,**form_kwargs): + prefix = form_kwargs.pop("prefix",None) + return prefilters.BioassayPrefilter(data=data,prefix=prefix,assessment=self.instance.assessment,form_kwargs=form_kwargs).form + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields["prefilters"] = DynamicFormField(prefix="prefilters",form_class=self._get_prefilter_form) + + + class DataPivotQueryForm(PrefilterMixin, DataPivotForm): prefilter_include = ("study", "bioassay", "epi", "invitro", "eco", "effect_tags") diff --git a/hawc/apps/summary/prefilters.py b/hawc/apps/summary/prefilters.py new file mode 100644 index 0000000000..cdcc768bc8 --- /dev/null +++ b/hawc/apps/summary/prefilters.py @@ -0,0 +1,110 @@ +import django_filters as df +from django.forms.widgets import CheckboxInput + +from ..animal.models import Endpoint +from ..common.filterset import BaseFilterSet, filter_noop +from ..study.models import Study + + +class BioassayPrefilter(BaseFilterSet): + # studies + published_only = df.BooleanFilter( + field_name="animal_group__experiment__study__published", + widget=CheckboxInput(), + label="Published studies only", + help_text="Only present data from studies which have been marked as " + '"published" in HAWC.', + ) + prefilter_study = df.BooleanFilter( + method=filter_noop, + widget=CheckboxInput(), + label="Prefilter by study", + help_text="Prefilter endpoints to include only selected studies.", + ) + studies = df.ModelMultipleChoiceFilter( + field_name="animal_group__experiment__study", + queryset=Study.objects.all(), + label="Studies to include", + help_text="""Select one or more studies to include in the plot. + If no study is selected, no endpoints will be available.""", + ) + # bioassay + prefilter_system = df.BooleanFilter( + method=filter_noop, + widget=CheckboxInput(), + label="Prefilter by system", + help_text="Prefilter endpoints on plot to include selected systems.", + ) + systems = df.MultipleChoiceFilter( + field_name="system", + label="Systems to include", + help_text="""Select one or more systems to include in the plot. + If no system is selected, no endpoints will be available.""", + ) + prefilter_organ = df.BooleanFilter( + method=filter_noop, + widget=CheckboxInput(), + label="Prefilter by organ", + help_text="Prefilter endpoints on plot to include selected organs.", + ) + organs = df.MultipleChoiceFilter( + field_name="organ", + label="Organs to include", + help_text="""Select one or more organs to include in the plot. + If no organ is selected, no endpoints will be available.""", + ) + prefilter_effect = df.BooleanFilter( + method=filter_noop, + widget=CheckboxInput(), + label="Prefilter by effect", + help_text="Prefilter endpoints on plot to include selected effects.", + ) + effects = df.MultipleChoiceFilter( + field_name="effect", + label="Effects to include", + help_text="""Select one or more effects to include in the plot. + If no effect is selected, no endpoints will be available.""", + ) + prefilter_effect_subtype = df.BooleanFilter( + method=filter_noop, + widget=CheckboxInput(), + label="Prefilter by effect sub-type", + help_text="Prefilter endpoints on plot to include selected effects.", + ) + effect_subtypes = df.MultipleChoiceFilter( + field_name="effect_subtype", + label="Effect Sub-Types to include", + help_text="""Select one or more effect sub-types to include in the plot. + If no effect sub-type is selected, no endpoints will be available.""", + ) + + class Meta: + model = Endpoint + fields = [ + "published_only", + "prefilter_study", + "studies", + "prefilter_system", + "systems", + "prefilter_organ", + "organs", + "prefilter_effect", + "effects", + "prefilter_effect_subtype", + "effect_subtypes", + ] + + def create_form(self): + form = super().create_form() + form.fields["studies"].queryset = Study.objects.filter(assessment=self.assessment) + form.fields["systems"].choices = Endpoint.objects.get_system_choices(self.assessment.pk) + form.fields["organs"].choices = Endpoint.objects.get_organ_choices(self.assessment.pk) + form.fields["effects"].choices = Endpoint.objects.get_effect_choices(self.assessment.pk) + form.fields["effect_subtypes"].choices = Endpoint.objects.get_effect_subtype_choices( + self.assessment.pk + ) + return form + + +class InvitroPrefilter(BaseFilterSet): + pass diff --git a/hawc/apps/summary/templates/summary/datapivot_form1.html b/hawc/apps/summary/templates/summary/datapivot_form1.html new file mode 100644 index 0000000000..894545d718 --- /dev/null +++ b/hawc/apps/summary/templates/summary/datapivot_form1.html @@ -0,0 +1,42 @@ +{% extends 'assessment-rooted.html' %} + +{% load crispy_forms_tags %} +{% load add_class %} + +{% block content %} + {% include "assessment/preferred_dose_units_widget.html" %} + {% crispy form %} + {% include "summary/_smartTagEditModal.html" with form=smart_tag_form only %} +{% endblock %} + +{% block extrajs %} + {{ smart_tag_form.media }} + +{% endblock extrajs %} diff --git a/hawc/apps/summary/urls.py b/hawc/apps/summary/urls.py index 6aa56e84f1..427b457f12 100644 --- a/hawc/apps/summary/urls.py +++ b/hawc/apps/summary/urls.py @@ -119,6 +119,11 @@ views.DataPivotQueryNew.as_view(), name="dp_new-query", ), + path( + "data-pivot/assessment//create/query/1/", + views.DataPivotQueryNew1.as_view(), + name="dp_new-query1", + ), path( "data-pivot/assessment//create/file/", views.DataPivotFileNew.as_view(), diff --git a/hawc/apps/summary/views.py b/hawc/apps/summary/views.py index e64b69c8e0..3009c34397 100644 --- a/hawc/apps/summary/views.py +++ b/hawc/apps/summary/views.py @@ -15,9 +15,9 @@ from ..assessment.views import check_published_status from ..common.crumbs import Breadcrumb from ..common.helper import WebappConfig -from ..common.views import BaseCreate, BaseDelete, BaseDetail, BaseFilterList, BaseList, BaseUpdate +from ..common.views import BaseCreate, BaseDelete, BaseDetail, BaseFilterList, BaseList, BaseUpdate, FilterSetMixin from ..riskofbias.models import RiskOfBiasMetric -from . import constants, filterset, forms, models, serializers +from . import constants, filterset, forms, models, serializers, prefilters def get_visual_list_crumb(assessment) -> Breadcrumb: @@ -642,6 +642,19 @@ def get_context_data(self, **kwargs): ) return context +class DataPivotQueryNew1(DataPivotNew): + model = models.DataPivotQuery + form_class = forms.DataPivotQueryForm1 + template_name = "summary/datapivot_form1.html" + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + context["file_loader"] = False + context["smart_tag_form"] = forms.SmartTagForm(assessment_id=self.assessment.id) + context["breadcrumbs"].insert( + len(context["breadcrumbs"]) - 1, get_visual_list_crumb(self.assessment) + ) + return context class DataPivotFileNew(DataPivotNew): model = models.DataPivotUpload From 98f69fceb706015e823991206450158d29e3f4e4 Mon Sep 17 00:00:00 2001 From: Daniel Rabstejnek Date: Mon, 21 Aug 2023 02:32:28 -0400 Subject: [PATCH 02/35] Data pivots largely done --- hawc/apps/summary/forms.py | 38 ++- hawc/apps/summary/migrations/0043_new.py | 50 +++ hawc/apps/summary/migrations/0044_newest.py | 47 +++ hawc/apps/summary/migrations/0045_newer.py | 47 +++ hawc/apps/summary/models.py | 67 +--- hawc/apps/summary/prefilters.py | 285 ++++++++++++++---- .../templates/summary/datapivot_detail.html | 2 +- .../templates/summary/datapivot_form1.html | 8 +- hawc/apps/summary/urls.py | 7 +- hawc/apps/summary/views.py | 27 ++ 10 files changed, 465 insertions(+), 113 deletions(-) create mode 100644 hawc/apps/summary/migrations/0043_new.py create mode 100644 hawc/apps/summary/migrations/0044_newest.py create mode 100644 hawc/apps/summary/migrations/0045_newer.py diff --git a/hawc/apps/summary/forms.py b/hawc/apps/summary/forms.py index 367e396eaf..53e58ebc66 100644 --- a/hawc/apps/summary/forms.py +++ b/hawc/apps/summary/forms.py @@ -1090,18 +1090,49 @@ class Meta: "settings", "caption", "published", - "published_only", "prefilters", ) def _get_prefilter_form(self,data,**form_kwargs): prefix = form_kwargs.pop("prefix",None) - return prefilters.BioassayPrefilter(data=data,prefix=prefix,assessment=self.instance.assessment,form_kwargs=form_kwargs).form + #data = json.loads(data) # TODO migrate to json + return self.prefilter(data=data,prefix=prefix,assessment=self.instance.assessment,form_kwargs=form_kwargs).form def __init__(self, *args, **kwargs): + evidence_type = kwargs.pop("evidence_type",None) super().__init__(*args, **kwargs) - self.fields["prefilters"] = DynamicFormField(prefix="prefilters",form_class=self._get_prefilter_form) + if evidence_type is not None: + self.instance.evidence_type = evidence_type + self.fields["evidence_type"].initial = self.instance.evidence_type + self.fields["evidence_type"].disabled = True + + self.prefilter = prefilters.Prefilter.from_study_type(self.instance.evidence_type,self.instance.assessment).value + self.fields["prefilters"] = DynamicFormField(prefix="prefilters",form_class=self._get_prefilter_form,label="") + self.fields["preferred_units"].required = False + self.js_units_choices = json.dumps( + [ + {"id": obj.id, "name": obj.name} + for obj in DoseUnits.objects.get_animal_units(self.instance.assessment) + ] + ) + self.helper = self.setHelper() + + def save(self, commit=True): + self.instance.preferred_units = self.cleaned_data.get("preferred_units", []) + return super().save(commit=commit) + + def clean_export_style(self): + evidence_type = self.cleaned_data["evidence_type"] + export_style = self.cleaned_data["export_style"] + if ( + evidence_type not in (constants.StudyType.IN_VITRO, constants.StudyType.BIOASSAY) + and export_style != constants.ExportStyle.EXPORT_GROUP + ): + raise forms.ValidationError( + "Outcome/Result level export not implemented for this data-type." + ) + return export_style class DataPivotQueryForm(PrefilterMixin, DataPivotForm): @@ -1118,7 +1149,6 @@ class Meta: "settings", "caption", "published", - "published_only", "prefilters", ) diff --git a/hawc/apps/summary/migrations/0043_new.py b/hawc/apps/summary/migrations/0043_new.py new file mode 100644 index 0000000000..70dd3b90dc --- /dev/null +++ b/hawc/apps/summary/migrations/0043_new.py @@ -0,0 +1,50 @@ +# Generated by Django 4.2.3 on 2023-08-21 03:49 +import json + +from django.db import migrations + +from hawc.apps.summary import constants +from hawc.apps.assessment.constants import EpiVersion + +def published_only_prefilters(apps, schema_editor): + # add published_only field to prefilters + DataPivotQuery = apps.get_model("summary", "DataPivotQuery") + objs = DataPivotQuery.objects.all().select_related("assessment") + for obj in objs: + prefilters = json.loads(obj.prefilters) + epi_version = obj.assessment.epi_version + + if obj.evidence_type == constants.StudyType.BIOASSAY: + prefilters["animal_group__experiment__study__published"] = obj.published_only + elif obj.evidence_type == constants.StudyType.EPI and epi_version == EpiVersion.V1: + prefilters["study_population__study__published"] = obj.published_only + elif obj.evidence_type == constants.StudyType.EPI and epi_version == EpiVersion.V2: + prefilters["design__study__published"] = obj.published_only + elif obj.evidence_type == constants.StudyType.EPI_META: + prefilters["protocol__study__published"] = obj.published_only + elif obj.evidence_type == constants.StudyType.IN_VITRO: + prefilters["experiment__study__published"] = obj.published_only + elif obj.evidence_type == constants.StudyType.ECO: + prefilters["design__study__published"] = obj.published_only + + obj.prefilters = json.dumps(prefilters) + DataPivotQuery.objects.bulk_update(objs,["prefilters"]) + + +def reverse_published_only_prefilters(apps, schema_editor): + # TODO + return + + +class Migration(migrations.Migration): + dependencies = [ + ("summary", "0042_summarytable_interactive"), + ] + + operations = [ + migrations.RunPython(published_only_prefilters, reverse_code=reverse_published_only_prefilters), + migrations.RemoveField( + model_name="datapivotquery", + name="published_only", + ), + ] diff --git a/hawc/apps/summary/migrations/0044_newest.py b/hawc/apps/summary/migrations/0044_newest.py new file mode 100644 index 0000000000..4165b5c2e4 --- /dev/null +++ b/hawc/apps/summary/migrations/0044_newest.py @@ -0,0 +1,47 @@ +# Generated by Django 4.2.3 on 2023-08-21 03:49 +import json + +from django.db import migrations, models + + + +def prefilters_dict(apps, schema_editor): + # load prefilters textfield into temp jsonfield + DataPivotQuery = apps.get_model("summary", "DataPivotQuery") + objs = DataPivotQuery.objects.all() + for obj in objs: + obj.temp = json.loads(obj.prefilters) + DataPivotQuery.objects.bulk_update(objs,["temp"]) + + +def reverse_prefilters_dict(apps, schema_editor): + # dump temp jsonfield into prefilters textfield + DataPivotQuery = apps.get_model("summary", "DataPivotQuery") + objs = DataPivotQuery.objects.all() + for obj in objs: + obj.prefilters = json.dumps(obj.temp) + DataPivotQuery.objects.bulk_update(objs,["prefilters"]) + +class Migration(migrations.Migration): + dependencies = [ + ("summary", "0043_new"), + ] + + operations = [ + # change prefilters textfield into jsonfield + migrations.AddField( + model_name="datapivotquery", + name="temp", + field=models.JSONField(default=dict), + ), + migrations.RunPython(prefilters_dict, reverse_code=reverse_prefilters_dict), + migrations.RemoveField( + model_name="datapivotquery", + name="prefilters", + ), + migrations.RenameField( + model_name="datapivotquery", + old_name="temp", + new_name="prefilters", + ), + ] diff --git a/hawc/apps/summary/migrations/0045_newer.py b/hawc/apps/summary/migrations/0045_newer.py new file mode 100644 index 0000000000..741a65ea50 --- /dev/null +++ b/hawc/apps/summary/migrations/0045_newer.py @@ -0,0 +1,47 @@ +# Generated by Django 4.2.3 on 2023-08-21 03:49 +import json + +from django.db import migrations, models + + + +def prefilters_dict(apps, schema_editor): + # load prefilters textfield into temp jsonfield + Visual = apps.get_model("summary", "Visual") + objs = Visual.objects.all() + for obj in objs: + obj.temp = json.loads(obj.prefilters) + Visual.objects.bulk_update(objs,["temp"]) + + +def reverse_prefilters_dict(apps, schema_editor): + # dump temp jsonfield into prefilters textfield + Visual = apps.get_model("summary", "Visual") + objs = Visual.objects.all() + for obj in objs: + obj.prefilters = json.dumps(obj.temp) + Visual.objects.bulk_update(objs,["prefilters"]) + +class Migration(migrations.Migration): + dependencies = [ + ("summary", "0044_newest"), + ] + + operations = [ + # change prefilters textfield into jsonfield + migrations.AddField( + model_name="visual", + name="temp", + field=models.JSONField(default=dict), + ), + migrations.RunPython(prefilters_dict, reverse_code=reverse_prefilters_dict), + migrations.RemoveField( + model_name="visual", + name="prefilters", + ), + migrations.RenameField( + model_name="visual", + old_name="temp", + new_name="prefilters", + ), + ] diff --git a/hawc/apps/summary/models.py b/hawc/apps/summary/models.py index ff59fd38b9..2019cb4eea 100644 --- a/hawc/apps/summary/models.py +++ b/hawc/apps/summary/models.py @@ -48,7 +48,7 @@ from ..invitro.models import IVEndpoint from ..riskofbias.serializers import AssessmentRiskOfBiasSerializer from ..study.models import Study -from . import constants, managers +from . import constants, managers, prefilters logger = logging.getLogger(__name__) @@ -747,13 +747,8 @@ class DataPivotQuery(DataPivot): "percent-response, where dose-units are not needed, or for " "creating one plot similar, but not identical, dose-units.", ) - prefilters = models.TextField(default="{}") - published_only = models.BooleanField( - default=True, - verbose_name="Published studies only", - help_text="Only present data from studies which have been marked as " - '"published" in HAWC.', - ) + prefilters = models.JSONField(default=dict) + def clean(self): count = self.get_queryset().count() @@ -776,62 +771,30 @@ def clean(self): ) raise ValidationError(err) - def _get_dataset_filters(self): - filters = {} + def _refine_queryset(self, qs): epi_version = self.assessment.epi_version if self.evidence_type == constants.StudyType.BIOASSAY: - filters["assessment_id"] = self.assessment_id - if self.published_only: - filters["animal_group__experiment__study__published"] = True + qs = qs.filter(assessment_id = self.assessment_id) if self.preferred_units: - filters["animal_group__dosing_regime__doses__dose_units__in"] = self.preferred_units + qs = qs.filter(animal_group__dosing_regime__doses__dose_units__in = self.preferred_units) elif self.evidence_type == constants.StudyType.EPI and epi_version == EpiVersion.V1: - filters["assessment_id"] = self.assessment_id - if self.published_only: - filters["study_population__study__published"] = True + qs = qs.filter(assessment_id = self.assessment_id) elif self.evidence_type == constants.StudyType.EPI and epi_version == EpiVersion.V2: - filters["design__study__assessment_id"] = self.assessment_id - if self.published_only: - filters["design__study__published"] = True + qs = qs.filter(design__study__assessment_id = self.assessment_id) elif self.evidence_type == constants.StudyType.EPI_META: - filters["protocol__study__assessment_id"] = self.assessment_id - if self.published_only: - filters["protocol__study__published"] = True + qs = qs.filter(protocol__study__assessment_id = self.assessment_id) elif self.evidence_type == constants.StudyType.IN_VITRO: - filters["assessment_id"] = self.assessment_id - if self.published_only: - filters["experiment__study__published"] = True + qs = qs.filter(assessment_id = self.assessment_id) elif self.evidence_type == constants.StudyType.ECO: - filters["design__study__assessment_id"] = self.assessment_id - if self.published_only: - filters["design__study__published"] = True + qs = qs.filter(design__study__assessment_id = self.assessment_id) - Prefilter.setFiltersFromObj(filters, self.prefilters) - return filters - - def _get_dataset_queryset(self, filters): - epi_version = self.assessment.epi_version - if self.evidence_type == constants.StudyType.BIOASSAY: - qs = Endpoint.objects.filter(**filters) - elif self.evidence_type == constants.StudyType.EPI and epi_version == EpiVersion.V1: - qs = Outcome.objects.filter(**filters) - elif self.evidence_type == constants.StudyType.EPI and epi_version == EpiVersion.V2: - qs = DataExtraction.objects.filter(**filters) - elif self.evidence_type == constants.StudyType.EPI_META: - qs = MetaResult.objects.filter(**filters) - elif self.evidence_type == constants.StudyType.IN_VITRO: - qs = IVEndpoint.objects.filter(**filters) - elif self.evidence_type == constants.StudyType.ECO: - qs = Result.objects.filter(**filters) - else: - raise ValueError("Invalid data type") - return qs.order_by("id") + return qs def _get_dataset_exporter(self, qs): if self.evidence_type == constants.StudyType.BIOASSAY: @@ -893,8 +856,10 @@ def _get_dataset_exporter(self, qs): return exporter def get_queryset(self): - filters = self._get_dataset_filters() - return self._get_dataset_queryset(filters) + fs = prefilters.Prefilter.from_study_type(self.evidence_type,self.assessment).value + qs = fs(data=self.prefilters,assessment=self.assessment).qs + qs = self._refine_queryset(qs) + return qs.order_by("id") def get_dataset(self) -> FlatExport: qs = self.get_queryset() diff --git a/hawc/apps/summary/prefilters.py b/hawc/apps/summary/prefilters.py index cdcc768bc8..ed88ba2bbc 100644 --- a/hawc/apps/summary/prefilters.py +++ b/hawc/apps/summary/prefilters.py @@ -1,110 +1,285 @@ import django_filters as df from django.forms.widgets import CheckboxInput +from django import forms +from enum import Enum +from ..assessment.constants import EpiVersion +from ..assessment.models import Assessment, EffectTag from ..animal.models import Endpoint from ..common.filterset import BaseFilterSet, filter_noop +from ..common.forms import BaseFormHelper from ..study.models import Study +from ..epi.models import Outcome +from ..epiv2.models import DataExtraction +from ..epimeta.models import MetaResult +from ..invitro.models import IVEndpoint, IVEndpointCategory, IVChemical +from ..eco.models import Result + +from .constants import StudyType + +class TestForm(forms.Form): + + @property + def helper(self): + helper = BaseFormHelper(self) + helper.form_tag = False + + return helper + +def filter_published_only(queryset, name, value): + if not value: + return queryset + return queryset.filter(**{name:True}) class BioassayPrefilter(BaseFilterSet): # studies - published_only = df.BooleanFilter( - field_name="animal_group__experiment__study__published", + animal_group__experiment__study__published = df.BooleanFilter( + method=filter_published_only, widget=CheckboxInput(), label="Published studies only", help_text="Only present data from studies which have been marked as " '"published" in HAWC.', ) - prefilter_study = df.BooleanFilter( - method=filter_noop, - widget=CheckboxInput(), - label="Prefilter by study", - help_text="Prefilter endpoints to include only selected studies.", - ) - studies = df.ModelMultipleChoiceFilter( + animal_group__experiment__study__in = df.MultipleChoiceFilter( field_name="animal_group__experiment__study", - queryset=Study.objects.all(), label="Studies to include", help_text="""Select one or more studies to include in the plot. If no study is selected, no endpoints will be available.""", ) # bioassay - prefilter_system = df.BooleanFilter( - method=filter_noop, - widget=CheckboxInput(), - label="Prefilter by system", - help_text="Prefilter endpoints on plot to include selected systems.", - ) - systems = df.MultipleChoiceFilter( + system__in = df.MultipleChoiceFilter( field_name="system", label="Systems to include", help_text="""Select one or more systems to include in the plot. If no system is selected, no endpoints will be available.""", ) - prefilter_organ = df.BooleanFilter( - method=filter_noop, - widget=CheckboxInput(), - label="Prefilter by organ", - help_text="Prefilter endpoints on plot to include selected organs.", - ) - organs = df.MultipleChoiceFilter( + organ__in = df.MultipleChoiceFilter( field_name="organ", label="Organs to include", help_text="""Select one or more organs to include in the plot. If no organ is selected, no endpoints will be available.""", ) - prefilter_effect = df.BooleanFilter( - method=filter_noop, - widget=CheckboxInput(), - label="Prefilter by effect", - help_text="Prefilter endpoints on plot to include selected effects.", - ) - effects = df.MultipleChoiceFilter( + effect__in = df.MultipleChoiceFilter( field_name="effect", label="Effects to include", help_text="""Select one or more effects to include in the plot. If no effect is selected, no endpoints will be available.""", ) - prefilter_effect_subtype = df.BooleanFilter( - method=filter_noop, - widget=CheckboxInput(), - label="Prefilter by effect sub-type", - help_text="Prefilter endpoints on plot to include selected effects.", - ) - effect_subtypes = df.MultipleChoiceFilter( + effect_subtype__in = df.MultipleChoiceFilter( field_name="effect_subtype", label="Effect Sub-Types to include", help_text="""Select one or more effect sub-types to include in the plot. If no effect sub-type is selected, no endpoints will be available.""", ) + effects__in = df.MultipleChoiceFilter( + field_name="effects", + label="Tags to include", + help_text="""Select one or more effect-tags to include in the plot. + If no study is selected, no endpoints will be available.""", + ) class Meta: model = Endpoint fields = [ - "published_only", - "prefilter_study", - "studies", - "prefilter_system", - "systems", - "prefilter_organ", - "organs", - "prefilter_effect", - "effects", - "prefilter_effect_subtype", - "effect_subtypes", + "animal_group__experiment__study__published", + "animal_group__experiment__study__in", + "system__in", + "organ__in", + "effect__in", + "effect_subtype__in","effects__in", ] + form = TestForm def create_form(self): form = super().create_form() - form.fields["studies"].queryset = Study.objects.filter(assessment=self.assessment) - form.fields["systems"].choices = Endpoint.objects.get_system_choices(self.assessment.pk) - form.fields["organs"].choices = Endpoint.objects.get_organ_choices(self.assessment.pk) - form.fields["effects"].choices = Endpoint.objects.get_effect_choices(self.assessment.pk) - form.fields["effect_subtypes"].choices = Endpoint.objects.get_effect_subtype_choices( + form.fields["animal_group__experiment__study__in"].choices = Study.objects.get_choices(self.assessment.pk) + form.fields["system__in"].choices = Endpoint.objects.get_system_choices(self.assessment.pk) + form.fields["organ__in"].choices = Endpoint.objects.get_organ_choices(self.assessment.pk) + form.fields["effect__in"].choices = Endpoint.objects.get_effect_choices(self.assessment.pk) + form.fields["effect_subtype__in"].choices = Endpoint.objects.get_effect_subtype_choices( self.assessment.pk ) + form.fields["effects__in"].choices = EffectTag.objects.get_choices(self.assessment.pk) + return form + + + +class EpiV1Prefilter(BaseFilterSet): + # studies + study_population__study__published = df.BooleanFilter( + method=filter_published_only, + widget=CheckboxInput(), + label="Published studies only", + help_text="Only present data from studies which have been marked as " + '"published" in HAWC.', + ) + study_population__study__in = df.MultipleChoiceFilter( + field_name="study_population__study", + label="Studies to include", + help_text="""Select one or more studies to include in the plot. + If no study is selected, no endpoints will be available.""", + ) + # epi + system__in = df.MultipleChoiceFilter( + field_name="system", + label="Systems to include", + help_text="""Select one or more systems to include in the plot. + If no system is selected, no endpoints will be available.""", + ) + effect__in = df.MultipleChoiceFilter( + field_name="effect", + label="Effects to include", + help_text="""Select one or more effects to include in the plot. + If no effect is selected, no endpoints will be available.""", + ) + effects__in = df.MultipleChoiceFilter( + field_name="effects", + label="Tags to include", + help_text="""Select one or more effect-tags to include in the plot. + If no study is selected, no endpoints will be available.""", + ) + + class Meta: + model = Outcome + fields = [ + "study_population__study__published", + "study_population__study__in","system__in","effect__in","effects__in", + ] + form = TestForm + + def create_form(self): + form = super().create_form() + form.fields["study_population__study__in"].choices = Study.objects.get_choices(self.assessment.pk) + form.fields["system__in"].choices = Outcome.objects.get_system_choices(self.assessment.pk) + form.fields["effect__in"].choices = Outcome.objects.get_effect_choices(self.assessment.pk) + form.fields["effects__in"].choices = EffectTag.objects.get_choices(self.assessment.pk) + return form + +class EpiV2Prefilter(BaseFilterSet): + # studies + design__study__published = df.BooleanFilter( + method=filter_published_only, + widget=CheckboxInput(), + label="Published studies only", + help_text="Only present data from studies which have been marked as " + '"published" in HAWC.', + ) + design__study__in = df.MultipleChoiceFilter( + field_name="design__study", + label="Studies to include", + help_text="""Select one or more studies to include in the plot. + If no study is selected, no endpoints will be available.""", + ) + + class Meta: + model = DataExtraction + fields = [ + "design__study__published", + "design__study__in", + ] + form = TestForm + + def create_form(self): + form = super().create_form() + form.fields["design__study__in"].choices = Study.objects.get_choices(self.assessment.pk) return form +class EpiMetaPrefilter(BaseFilterSet): + # studies + protocol__study__published = df.BooleanFilter( + method=filter_published_only, + widget=CheckboxInput(), + label="Published studies only", + help_text="Only present data from studies which have been marked as " + '"published" in HAWC.', + ) + protocol__study__in = df.MultipleChoiceFilter( + field_name="protocol__study", + label="Studies to include", + help_text="""Select one or more studies to include in the plot. + If no study is selected, no endpoints will be available.""", + ) + + class Meta: + model = MetaResult + fields = [ + "protocol__study__published", + "protocol__study__in", + ] + form = TestForm + + def create_form(self): + form = super().create_form() + form.fields["protocol__study__in"].choices = Study.objects.get_choices(self.assessment.pk) + return form class InvitroPrefilter(BaseFilterSet): - pass + # studies + experiment__study__published = df.BooleanFilter( + method=filter_published_only, + widget=CheckboxInput(), + label="Published studies only", + help_text="Only present data from studies which have been marked as " + '"published" in HAWC.', + ) + experiment__study__in = df.MultipleChoiceFilter( + field_name="experiment__study", + label="Studies to include", + help_text="""Select one or more studies to include in the plot. + If no study is selected, no endpoints will be available.""", + ) + # invitro + category__in = df.MultipleChoiceFilter( + field_name="category", + label="Categories to include", + help_text="""Select one or more categories to include in the plot. + If no study is selected, no endpoints will be available.""", + ) + chemical__name__in = df.MultipleChoiceFilter( + field_name="chemical__name", + label="Chemicals to include", + help_text="""Select one or more chemicals to include in the plot. + If no study is selected, no endpoints will be available.""", + ) + effects__in = df.MultipleChoiceFilter( + field_name="effects", + label="Tags to include", + help_text="""Select one or more effect-tags to include in the plot. + If no study is selected, no endpoints will be available.""", + ) + + class Meta: + model = IVEndpoint + fields = [ + "experiment__study__published", + "experiment__study__in","category__in","chemical__name__in","effects__in", + ] + form = TestForm + + def create_form(self): + form = super().create_form() + form.fields["experiment__study__in"].choices = Study.objects.get_choices(self.assessment.pk) + form.fields["category__in"].choices = IVEndpointCategory.get_choices(self.assessment.pk) + form.fields["chemical__name__in"].choices = IVChemical.objects.get_choices(self.assessment.pk) + form.fields["effects__in"].choices = EffectTag.objects.get_choices(self.assessment.pk) + return form + + +class Prefilter(Enum): + BIOASSAY = BioassayPrefilter + EPIV1 = EpiV1Prefilter + EPIV2 = EpiV2Prefilter + EPI_META = EpiMetaPrefilter + IN_VITRO = InvitroPrefilter + + @classmethod + def from_study_type(cls,study_type:StudyType,assessment:Assessment): + study_type = StudyType(study_type) + name = study_type.name + if study_type == StudyType.EPI: + if assessment.epi_version == EpiVersion.V1: + name = "EPIV1" + elif assessment.epi_version == EpiVersion.V2: + name = "EPIV2" + return cls[name] diff --git a/hawc/apps/summary/templates/summary/datapivot_detail.html b/hawc/apps/summary/templates/summary/datapivot_detail.html index 3c47cab50e..9e2b8493b1 100644 --- a/hawc/apps/summary/templates/summary/datapivot_detail.html +++ b/hawc/apps/summary/templates/summary/datapivot_detail.html @@ -10,7 +10,7 @@ {% if obj_perms.edit %}
Edit display settings - Edit other settings + Edit other settings Delete Pivot {% endif %} diff --git a/hawc/apps/summary/templates/summary/datapivot_form1.html b/hawc/apps/summary/templates/summary/datapivot_form1.html index 894545d718..c5bdec594c 100644 --- a/hawc/apps/summary/templates/summary/datapivot_form1.html +++ b/hawc/apps/summary/templates/summary/datapivot_form1.html @@ -24,7 +24,7 @@ {% endif %} window.app.startup("assessmentStartup", function(app){ - var doseWidget = new app.DoseUnitsWidget($('form'), { + new app.DoseUnitsWidget($('form'), { choices: js_units_choices, el: '#id_preferred_units', }); @@ -37,6 +37,12 @@ ); }) + // determine which fields to display depending on data-type + const value = {{form.instance.evidence_type}}, + aniOnlyDivs = $("#div_id_preferred_units"), + aniIvOnlyDivs = $("#div_id_export_style"); + (value === 0) ? aniOnlyDivs.show() : aniOnlyDivs.hide(); + (value == 0 || value == 2) ? aniIvOnlyDivs.show() : aniIvOnlyDivs.hide(); }); {% endblock extrajs %} diff --git a/hawc/apps/summary/urls.py b/hawc/apps/summary/urls.py index 427b457f12..7424b6861e 100644 --- a/hawc/apps/summary/urls.py +++ b/hawc/apps/summary/urls.py @@ -120,7 +120,7 @@ name="dp_new-query", ), path( - "data-pivot/assessment//create/query/1/", + "data-pivot/assessment//create/query//", views.DataPivotQueryNew1.as_view(), name="dp_new-query1", ), @@ -154,6 +154,11 @@ views.DataPivotUpdateQuery.as_view(), name="dp_query-update", ), + path( + "data-pivot/assessment///query-update1/", + views.DataPivotUpdateQuery1.as_view(), + name="dp_query-update1", + ), path( "data-pivot/assessment///file-update/", views.DataPivotUpdateFile.as_view(), diff --git a/hawc/apps/summary/views.py b/hawc/apps/summary/views.py index 3009c34397..fb95608237 100644 --- a/hawc/apps/summary/views.py +++ b/hawc/apps/summary/views.py @@ -647,6 +647,19 @@ class DataPivotQueryNew1(DataPivotNew): form_class = forms.DataPivotQueryForm1 template_name = "summary/datapivot_form1.html" + def get_form_kwargs(self): + kwargs = super().get_form_kwargs() + try: + # get study type enum + study_type = constants.StudyType(self.kwargs.get("study_type")) + # make sure prefilter exists for study type + prefilters.Prefilter.from_study_type(study_type,self.assessment) + # pass study type to form + kwargs["evidence_type"] = study_type + except (KeyError,ValueError): + raise Http404 + return kwargs + def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context["file_loader"] = False @@ -769,6 +782,20 @@ def get_context_data(self, **kwargs): ) return context +class DataPivotUpdateQuery1(GetDataPivotObjectMixin, BaseUpdate): + success_message = "Data Pivot updated." + model = models.DataPivotQuery + form_class = forms.DataPivotQueryForm1 + template_name = "summary/datapivot_form.html" + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + context["file_loader"] = False + context["smart_tag_form"] = forms.SmartTagForm(assessment_id=self.assessment.id) + context["breadcrumbs"].insert( + len(context["breadcrumbs"]) - 2, get_visual_list_crumb(self.assessment) + ) + return context class DataPivotUpdateFile(GetDataPivotObjectMixin, BaseUpdate): success_message = "Data Pivot updated." From cdede6fca5a0cd6f0111fe2e4a08f52ebd8938db Mon Sep 17 00:00:00 2001 From: Daniel Rabstejnek Date: Mon, 21 Aug 2023 07:17:01 -0400 Subject: [PATCH 03/35] Visuals largely done --- hawc/apps/summary/forms.py | 45 ++++++++++++++++++++++++++++---- hawc/apps/summary/models.py | 46 +++++++++++++++++++++------------ hawc/apps/summary/prefilters.py | 15 +++++++++-- hawc/apps/summary/views.py | 2 +- 4 files changed, 83 insertions(+), 25 deletions(-) diff --git a/hawc/apps/summary/forms.py b/hawc/apps/summary/forms.py index 53e58ebc66..51a56e0111 100644 --- a/hawc/apps/summary/forms.py +++ b/hawc/apps/summary/forms.py @@ -681,6 +681,24 @@ class Meta: model = models.Visual exclude = ("assessment", "visual_type", "endpoints", "studies") +class CrossviewForm1(VisualForm): + + def _get_prefilter_form(self,data,**form_kwargs): + prefix = form_kwargs.pop("prefix",None) + return self.prefilter(data=data,prefix=prefix,assessment=self.instance.assessment,form_kwargs=form_kwargs).form + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields["dose_units"].queryset = DoseUnits.objects.get_animal_units( + self.instance.assessment + ) + self.prefilter = prefilters.VisualTypePrefilter.from_visual_type(constants.VisualType.BIOASSAY_CROSSVIEW).value + self.fields["prefilters"] = DynamicFormField(prefix="prefilters",form_class=self._get_prefilter_form,label="") + self.helper = self.setHelper() + + class Meta: + model = models.Visual + exclude = ("assessment", "visual_type", "endpoints", "studies") class RoBForm(PrefilterMixin, VisualForm): prefilter_include = ("bioassay",) @@ -696,6 +714,24 @@ class Meta: model = models.Visual exclude = ("assessment", "visual_type", "dose_units", "endpoints") +class RoBForm1(VisualForm): + + def _get_prefilter_form(self,data,**form_kwargs): + prefix = form_kwargs.pop("prefix",None) + return self.prefilter(data=data,prefix=prefix,assessment=self.instance.assessment,form_kwargs=form_kwargs).form + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields["studies"].queryset = self.fields["studies"].queryset.filter( + assessment=self.instance.assessment + ) + self.prefilter = prefilters.VisualTypePrefilter.from_visual_type(constants.VisualType.ROB_BARCHART).value + self.fields["prefilters"] = DynamicFormField(prefix="prefilters",form_class=self._get_prefilter_form,label="") + self.helper = self.setHelper() + + class Meta: + model = models.Visual + exclude = ("assessment", "visual_type", "dose_units", "endpoints") class TagtreeForm(VisualForm): root_node = forms.TypedChoiceField( @@ -979,9 +1015,9 @@ def get_visual_form(visual_type): try: return { constants.VisualType.BIOASSAY_AGGREGATION: EndpointAggregationForm, - constants.VisualType.BIOASSAY_CROSSVIEW: CrossviewForm, - constants.VisualType.ROB_HEATMAP: RoBForm, - constants.VisualType.ROB_BARCHART: RoBForm, + constants.VisualType.BIOASSAY_CROSSVIEW: CrossviewForm1, + constants.VisualType.ROB_HEATMAP: RoBForm1, + constants.VisualType.ROB_BARCHART: RoBForm1, constants.VisualType.LITERATURE_TAGTREE: TagtreeForm, constants.VisualType.EXTERNAL_SITE: ExternalSiteForm, constants.VisualType.EXPLORE_HEATMAP: ExploreHeatmapForm, @@ -1095,7 +1131,6 @@ class Meta: def _get_prefilter_form(self,data,**form_kwargs): prefix = form_kwargs.pop("prefix",None) - #data = json.loads(data) # TODO migrate to json return self.prefilter(data=data,prefix=prefix,assessment=self.instance.assessment,form_kwargs=form_kwargs).form def __init__(self, *args, **kwargs): @@ -1107,7 +1142,7 @@ def __init__(self, *args, **kwargs): self.fields["evidence_type"].initial = self.instance.evidence_type self.fields["evidence_type"].disabled = True - self.prefilter = prefilters.Prefilter.from_study_type(self.instance.evidence_type,self.instance.assessment).value + self.prefilter = prefilters.StudyTypePrefilter.from_study_type(self.instance.evidence_type,self.instance.assessment).value self.fields["prefilters"] = DynamicFormField(prefix="prefilters",form_class=self._get_prefilter_form,label="") self.fields["preferred_units"].required = False self.js_units_choices = json.dumps( diff --git a/hawc/apps/summary/models.py b/hawc/apps/summary/models.py index 2019cb4eea..dcf7bf7c82 100644 --- a/hawc/apps/summary/models.py +++ b/hawc/apps/summary/models.py @@ -289,7 +289,7 @@ class Visual(models.Model): assessment = models.ForeignKey(Assessment, on_delete=models.CASCADE, related_name="visuals") visual_type = models.PositiveSmallIntegerField(choices=constants.VisualType.choices) dose_units = models.ForeignKey(DoseUnits, on_delete=models.SET_NULL, blank=True, null=True) - prefilters = models.TextField(default="{}") + prefilters = models.JSONField(default=dict) endpoints = models.ManyToManyField( BaseEndpoint, related_name="visuals", @@ -459,6 +459,17 @@ def get_dose_units(): def get_json(self, json_encode=True): return SerializerHelper.get_serialized(self, json=json_encode) + def get_filterset(self): + return prefilters.VisualTypePrefilter.from_visual_type(self.visual_type).value + + def get_request_prefilters(self,request): + # TODO move get_editing_dataset out of models + # so that we can utilize the forms + + # find all keys that start with "prefilters-" prefix + prefix = "prefilters-" + return {key[len(prefix):]:value for key,value in request.POST.lists() if key.startswith(prefix)} + def get_endpoints(self, request=None): qs = Endpoint.objects.none() filters = {"assessment_id": self.assessment_id} @@ -473,18 +484,17 @@ def get_endpoints(self, request=None): qs = Endpoint.objects.filter(**filters) elif self.visual_type == constants.VisualType.BIOASSAY_CROSSVIEW: + fs = self.get_filterset() if request: dose_id = tryParseInt(request.POST.get("dose_units"), -1) - Prefilter.setFiltersFromForm( - self.assessment, filters, request.POST, self.visual_type - ) + qs = fs(data=self.get_request_prefilters(request),assessment=self.assessment).qs else: dose_id = self.dose_units_id - Prefilter.setFiltersFromObj(filters, self.prefilters) + qs = fs(data=self.prefilters,assessment=self.assessment).qs filters["animal_group__dosing_regime__doses__dose_units_id"] = dose_id - qs = Endpoint.objects.filter(**filters).distinct("id") + qs = qs.filter(**filters).distinct("id") return qs @@ -501,14 +511,13 @@ def get_studies(self, request=None): constants.VisualType.ROB_HEATMAP, constants.VisualType.ROB_BARCHART, ]: + fs = self.get_filterset() if request: - efilters = {"assessment_id": self.assessment_id} - Prefilter.setFiltersFromForm( - self.assessment, efilters, request.POST, self.visual_type - ) - if len(efilters) > 1: + prefilters = self.get_request_prefilters(request) + if any(value for value in prefilters.values()): + endpoint_qs = fs(data=prefilters,assessment=self.assessment).qs filters["id__in"] = set( - Endpoint.objects.filter(**efilters).values_list( + endpoint_qs.filter(assessment_id=self.assessment_id).values_list( "animal_group__experiment__study_id", flat=True ) ) @@ -518,11 +527,10 @@ def get_studies(self, request=None): qs = Study.objects.filter(**filters) else: - if self.prefilters != "{}": - efilters = {"assessment_id": self.assessment_id} - Prefilter.setFiltersFromObj(efilters, self.prefilters) + if any(value for value in self.prefilters.values()): + endpoint_qs = fs(data=self.prefilters,assessment=self.assessment).qs filters["id__in"] = set( - Endpoint.objects.filter(**efilters).values_list( + endpoint_qs.filter(assessment_id=self.assessment_id).values_list( "animal_group__experiment__study_id", flat=True ) ) @@ -854,9 +862,13 @@ def _get_dataset_exporter(self, qs): ) return exporter + + def get_filterset(self): + return prefilters.StudyTypePrefilter.from_study_type(self.evidence_type,self.assessment).value + def get_queryset(self): - fs = prefilters.Prefilter.from_study_type(self.evidence_type,self.assessment).value + fs = self.get_filterset() qs = fs(data=self.prefilters,assessment=self.assessment).qs qs = self._refine_queryset(qs) return qs.order_by("id") diff --git a/hawc/apps/summary/prefilters.py b/hawc/apps/summary/prefilters.py index ed88ba2bbc..dafdb57ea7 100644 --- a/hawc/apps/summary/prefilters.py +++ b/hawc/apps/summary/prefilters.py @@ -15,7 +15,7 @@ from ..invitro.models import IVEndpoint, IVEndpointCategory, IVChemical from ..eco.models import Result -from .constants import StudyType +from .constants import StudyType, VisualType class TestForm(forms.Form): @@ -266,7 +266,7 @@ def create_form(self): return form -class Prefilter(Enum): +class StudyTypePrefilter(Enum): BIOASSAY = BioassayPrefilter EPIV1 = EpiV1Prefilter EPIV2 = EpiV2Prefilter @@ -283,3 +283,14 @@ def from_study_type(cls,study_type:StudyType,assessment:Assessment): elif assessment.epi_version == EpiVersion.V2: name = "EPIV2" return cls[name] + +class VisualTypePrefilter(Enum): + BIOASSAY_CROSSVIEW = BioassayPrefilter + ROB_HEATMAP = BioassayPrefilter + ROB_BARCHART = BioassayPrefilter + + @classmethod + def from_visual_type(cls,visual_type:VisualType): + visual_type = VisualType(visual_type) + name = visual_type.name + return cls[name] \ No newline at end of file diff --git a/hawc/apps/summary/views.py b/hawc/apps/summary/views.py index fb95608237..007321af28 100644 --- a/hawc/apps/summary/views.py +++ b/hawc/apps/summary/views.py @@ -653,7 +653,7 @@ def get_form_kwargs(self): # get study type enum study_type = constants.StudyType(self.kwargs.get("study_type")) # make sure prefilter exists for study type - prefilters.Prefilter.from_study_type(study_type,self.assessment) + prefilters.StudyTypePrefilter.from_study_type(study_type,self.assessment) # pass study type to form kwargs["evidence_type"] = study_type except (KeyError,ValueError): From 7cee82bc663392903004b4a904eabf56b23318dd Mon Sep 17 00:00:00 2001 From: Daniel Rabstejnek Date: Tue, 22 Aug 2023 02:33:57 -0400 Subject: [PATCH 04/35] Cleanup, linting --- hawc/apps/common/filterset.py | 4 +- hawc/apps/common/forms.py | 3 +- hawc/apps/common/widgets.py | 7 +- hawc/apps/summary/forms.py | 569 ++---------------- hawc/apps/summary/migrations/0043_new.py | 11 +- hawc/apps/summary/migrations/0044_newest.py | 6 +- hawc/apps/summary/migrations/0045_newer.py | 6 +- hawc/apps/summary/models.py | 181 ++---- hawc/apps/summary/prefilters.py | 86 +-- .../templates/summary/datapivot_form.html | 63 +- .../templates/summary/datapivot_form1.html | 48 -- hawc/apps/summary/urls.py | 12 +- hawc/apps/summary/views.py | 45 +- tests/hawc/apps/summary/test_views.py | 2 +- 14 files changed, 194 insertions(+), 849 deletions(-) delete mode 100644 hawc/apps/summary/templates/summary/datapivot_form1.html diff --git a/hawc/apps/common/filterset.py b/hawc/apps/common/filterset.py index 6d535351d2..697e6cb98b 100644 --- a/hawc/apps/common/filterset.py +++ b/hawc/apps/common/filterset.py @@ -171,7 +171,9 @@ def create_form(self): form = form_class(self.data, prefix=self.form_prefix, **self.form_kwargs) else: form = form_class(prefix=self.form_prefix, **self.form_kwargs) - if getattr(form,"dynamic_fields",None): # removes unwanted fields from a filterset if specified + if getattr( + form, "dynamic_fields", None + ): # removes unwanted fields from a filterset if specified for field in list(form.fields.keys()): if field not in form.dynamic_fields and field != "is_expanded": form.fields.pop(field) diff --git a/hawc/apps/common/forms.py b/hawc/apps/common/forms.py index 25675ecaa0..c01e223dbd 100644 --- a/hawc/apps/common/forms.py +++ b/hawc/apps/common/forms.py @@ -453,6 +453,7 @@ def validate(self, value): if value != self.check_value: raise forms.ValidationError(f'The value of "{self.check_value}" is required.') + class DynamicFormField(forms.JSONField): """Field to display dynamic form inline.""" @@ -477,4 +478,4 @@ def validate(self, value): super().validate(value) form = self.form_class(data=value, **self.form_kwargs) if not form.is_valid(): - raise forms.ValidationError(self.error_messages["invalid"]) \ No newline at end of file + raise forms.ValidationError(self.error_messages["invalid"]) diff --git a/hawc/apps/common/widgets.py b/hawc/apps/common/widgets.py index 7d7c2bae98..424a4e221f 100644 --- a/hawc/apps/common/widgets.py +++ b/hawc/apps/common/widgets.py @@ -1,5 +1,5 @@ -from random import randint import json +from random import randint from django.conf import settings from django.forms import ValidationError @@ -11,7 +11,7 @@ SelectMultiple, Textarea, TextInput, - Widget + Widget, ) from django.utils import timezone @@ -127,6 +127,7 @@ def build_attrs(self, base_attrs, extra_attrs=None): attrs["class"] = class_name + " quilltext" if class_name else "quilltext" return attrs + class DynamicFormWidget(Widget): """Widget to display dynamic form inline.""" @@ -156,4 +157,4 @@ def value_from_datadict(self, data, files, name): """Parse value from POST request.""" form = self.form_class(data=data, **self.form_kwargs) form.full_clean() - return form.cleaned_data \ No newline at end of file + return form.cleaned_data diff --git a/hawc/apps/summary/forms.py b/hawc/apps/summary/forms.py index 51a56e0111..954a91b5fd 100644 --- a/hawc/apps/summary/forms.py +++ b/hawc/apps/summary/forms.py @@ -9,440 +9,17 @@ from openpyxl.utils.exceptions import InvalidFileException from ..animal.autocomplete import EndpointAutocomplete -from ..animal.models import Endpoint -from ..assessment.models import DoseUnits, EffectTag +from ..assessment.models import DoseUnits from ..common import validators from ..common.autocomplete import AutocompleteChoiceField -from ..common.forms import BaseFormHelper, QuillField, check_unique_for_assessment, DynamicFormField +from ..common.forms import BaseFormHelper, DynamicFormField, QuillField, check_unique_for_assessment from ..common.helper import new_window_a from ..common.validators import validate_html_tags, validate_hyperlinks, validate_json_pydantic -from ..epi.models import Outcome -from ..invitro.models import IVChemical, IVEndpointCategory from ..lit.models import ReferenceFilterTag from ..study.autocomplete import StudyAutocomplete -from ..study.models import Study from . import autocomplete, constants, models, prefilters -class PrefilterMixin: - PREFILTER_COMBO_FIELDS = [ - "studies", - "systems", - "organs", - "effects", - "effect_subtypes", - "episystems", - "epieffects", - "iv_categories", - "iv_chemicals", - "effect_tags", - ] - - def createFields(self): - fields = dict() - epi_version = self.instance.assessment.epi_version - - if "study" in self.prefilter_include: - fields.update( - [ - ( - "published_only", - forms.BooleanField( - required=False, - initial=True, - label="Published studies only", - help_text="Only present data from studies which have been marked as " - '"published" in HAWC.', - ), - ), - ( - "prefilter_study", - forms.BooleanField( - required=False, - label="Prefilter by study", - help_text="Prefilter endpoints to include only selected studies.", - ), - ), - ( - "studies", - forms.MultipleChoiceField( - required=False, - widget=forms.SelectMultiple, - label="Studies to include", - help_text="""Select one or more studies to include in the plot. - If no study is selected, no endpoints will be available.""", - ), - ), - ] - ) - - if "bioassay" in self.prefilter_include: - fields.update( - [ - ( - "prefilter_system", - forms.BooleanField( - required=False, - label="Prefilter by system", - help_text="Prefilter endpoints on plot to include selected systems.", - ), - ), - ( - "systems", - forms.MultipleChoiceField( - required=False, - widget=forms.SelectMultiple, - label="Systems to include", - help_text="""Select one or more systems to include in the plot. - If no system is selected, no endpoints will be available.""", - ), - ), - ( - "prefilter_organ", - forms.BooleanField( - required=False, - label="Prefilter by organ", - help_text="Prefilter endpoints on plot to include selected organs.", - ), - ), - ( - "organs", - forms.MultipleChoiceField( - required=False, - widget=forms.SelectMultiple, - label="Organs to include", - help_text="""Select one or more organs to include in the plot. - If no organ is selected, no endpoints will be available.""", - ), - ), - ( - "prefilter_effect", - forms.BooleanField( - required=False, - label="Prefilter by effect", - help_text="Prefilter endpoints on plot to include selected effects.", - ), - ), - ( - "effects", - forms.MultipleChoiceField( - required=False, - widget=forms.SelectMultiple, - label="Effects to include", - help_text="""Select one or more effects to include in the plot. - If no effect is selected, no endpoints will be available.""", - ), - ), - ( - "prefilter_effect_subtype", - forms.BooleanField( - required=False, - label="Prefilter by effect sub-type", - help_text="Prefilter endpoints on plot to include selected effects.", - ), - ), - ( - "effect_subtypes", - forms.MultipleChoiceField( - required=False, - widget=forms.SelectMultiple, - label="Effect Sub-Types to include", - help_text="""Select one or more effect sub-types to include in the plot. - If no effect sub-type is selected, no endpoints will be available.""", - ), - ), - ] - ) - - if "epi" in self.prefilter_include and epi_version == 1: - fields.update( - [ - ( - "prefilter_episystem", - forms.BooleanField( - required=False, - label="Prefilter by system", - help_text="Prefilter endpoints on plot to include selected systems.", - ), - ), - ( - "episystems", - forms.MultipleChoiceField( - required=False, - widget=forms.SelectMultiple, - label="Systems to include", - help_text="""Select one or more systems to include in the plot. - If no system is selected, no endpoints will be available.""", - ), - ), - ( - "prefilter_epieffect", - forms.BooleanField( - required=False, - label="Prefilter by effect", - help_text="Prefilter endpoints on plot to include selected effects.", - ), - ), - ( - "epieffects", - forms.MultipleChoiceField( - required=False, - widget=forms.SelectMultiple, - label="Effects to include", - help_text="""Select one or more effects to include in the plot. - If no effect is selected, no endpoints will be available.""", - ), - ), - ] - ) - - if "invitro" in self.prefilter_include: - fields.update( - [ - ( - "prefilter_iv_category", - forms.BooleanField( - required=False, - label="Prefilter by category", - help_text="Prefilter endpoints to include only selected category.", - ), - ), - ( - "iv_categories", - forms.MultipleChoiceField( - required=False, - widget=forms.SelectMultiple, - label="Categories to include", - help_text="""Select one or more categories to include in the plot. - If no study is selected, no endpoints will be available.""", - ), - ), - ( - "prefilter_iv_chemical", - forms.BooleanField( - required=False, - label="Prefilter by chemical", - help_text="Prefilter endpoints to include only selected chemicals.", - ), - ), - ( - "iv_chemicals", - forms.MultipleChoiceField( - required=False, - widget=forms.SelectMultiple, - label="Chemicals to include", - help_text="""Select one or more chemicals to include in the plot. - If no study is selected, no endpoints will be available.""", - ), - ), - ] - ) - - if "effect_tags" in self.prefilter_include: - fields.update( - [ - ( - "prefilter_effect_tag", - forms.BooleanField( - required=False, - label="Prefilter by effect-tag", - help_text="Prefilter endpoints to include only selected effect-tags.", - ), - ), - ( - "effect_tags", - forms.MultipleChoiceField( - required=False, - widget=forms.SelectMultiple, - label="Tags to include", - help_text="""Select one or more effect-tags to include in the plot. - If no study is selected, no endpoints will be available.""", - ), - ), - ] - ) - - for k, v in fields.items(): - self.fields[k] = v - - def setInitialValues(self): - is_new = self.initial == {} - try: - prefilters = json.loads(self.initial.get("prefilters", "{}")) - except ValueError: - prefilters = {} - - if type(self.instance) is models.Visual: - evidence_type = constants.StudyType.BIOASSAY - else: - evidence_type = self.initial.get("evidence_type") or self.instance.evidence_type - for k, v in prefilters.items(): - if k == "system__in": - if evidence_type == constants.StudyType.BIOASSAY: - self.fields["prefilter_system"].initial = True - self.fields["systems"].initial = v - elif evidence_type == constants.StudyType.EPI: - self.fields["prefilter_episystem"].initial = True - self.fields["episystems"].initial = v - - if k == "organ__in": - self.fields["prefilter_organ"].initial = True - self.fields["organs"].initial = v - - if k == "effect__in": - if evidence_type == constants.StudyType.BIOASSAY: - self.fields["prefilter_effect"].initial = True - self.fields["effects"].initial = v - elif evidence_type == constants.StudyType.EPI: - self.fields["prefilter_epieffect"].initial = True - self.fields["epieffects"].initial = v - - if k == "effect_subtype__in": - self.fields["prefilter_effect_subtype"].initial = True - self.fields["effect_subtypes"].initial = v - - if k == "effects__in": - self.fields["prefilter_effect_tag"].initial = True - self.fields["effect_tags"].initial = v - - if k == "category__in": - self.fields["prefilter_iv_category"].initial = True - self.fields["iv_categories"].initial = v - - if k == "chemical__name__in": - self.fields["prefilter_iv_chemical"].initial = True - self.fields["iv_chemicals"].initial = v - - if k in [ - "animal_group__experiment__study__in", - "study_population__study__in", - "experiment__study__in", - "protocol__study__in", - "design__study__in", - ]: - self.fields["prefilter_study"].initial = True - self.fields["studies"].initial = v - - if self.__class__.__name__ == "CrossviewForm": - published_only = prefilters.get("animal_group__experiment__study__published", False) - if is_new: - published_only = True - self.fields["published_only"].initial = published_only - - for fldname in self.PREFILTER_COMBO_FIELDS: - field = self.fields.get(fldname) - if field: - field.choices = self.getPrefilterQueryset(fldname) - - def getPrefilterQueryset(self, field_name): - assessment_id = self.instance.assessment_id - choices = None - - if field_name == "systems": - choices = Endpoint.objects.get_system_choices(assessment_id) - elif field_name == "organs": - choices = Endpoint.objects.get_organ_choices(assessment_id) - elif field_name == "effects": - choices = Endpoint.objects.get_effect_choices(assessment_id) - elif field_name == "effect_subtypes": - choices = Endpoint.objects.get_effect_subtype_choices(assessment_id) - elif field_name == "iv_categories": - choices = IVEndpointCategory.get_choices(assessment_id) - elif field_name == "iv_chemicals": - choices = IVChemical.objects.get_choices(assessment_id) - elif field_name == "effect_tags": - choices = EffectTag.objects.get_choices(assessment_id) - elif field_name == "studies": - choices = Study.objects.get_choices(assessment_id) - elif field_name == "episystems": - choices = Outcome.objects.get_system_choices(assessment_id) - elif field_name == "epieffects": - choices = Outcome.objects.get_effect_choices(assessment_id) - else: - raise ValueError(f"Unknown field name: {field_name}") - - return choices - - def setFieldStyles(self): - if self.fields.get("prefilters"): - self.fields["prefilters"].widget = forms.HiddenInput() - - for fldname in self.PREFILTER_COMBO_FIELDS: - field = self.fields.get(fldname) - if field: - field.widget.attrs["size"] = 10 - - def setPrefilters(self, data): - prefilters = {} - epi_version = self.instance.assessment.epi_version - - if data.get("prefilter_study") is True: - studies = data.get("studies", []) - - evidence_type = data.get("evidence_type", None) - if self.__class__.__name__ == "CrossviewForm": - evidence_type = 0 - - if evidence_type == constants.StudyType.BIOASSAY: - prefilters["animal_group__experiment__study__in"] = studies - elif evidence_type == constants.StudyType.IN_VITRO: - prefilters["experiment__study__in"] = studies - elif evidence_type == constants.StudyType.EPI: - if epi_version == 1: - prefilters["study_population__study__in"] = studies - elif epi_version == 2: - prefilters["design__study__in"] = studies - else: - raise ValueError("Invalid epi_version") - elif evidence_type == constants.StudyType.EPI_META: - prefilters["protocol__study__in"] = studies - else: - raise ValueError("Unknown evidence type") - - if data.get("prefilter_system") is True: - prefilters["system__in"] = data.get("systems", []) - - if data.get("prefilter_organ") is True: - prefilters["organ__in"] = data.get("organs", []) - - if data.get("prefilter_effect") is True: - prefilters["effect__in"] = data.get("effects", []) - - if data.get("prefilter_effect_subtype") is True: - prefilters["effect_subtype__in"] = data.get("effect_subtypes", []) - - if data.get("prefilter_episystem") is True: - prefilters["system__in"] = data.get("episystems", []) - - if data.get("prefilter_epieffect") is True: - prefilters["effect__in"] = data.get("epieffects", []) - - if data.get("prefilter_iv_category") is True: - prefilters["category__in"] = data.get("iv_categories", []) - - if data.get("prefilter_iv_chemical") is True: - prefilters["chemical__name__in"] = data.get("iv_chemicals", []) - - if data.get("prefilter_effect_tag") is True: - prefilters["effects__in"] = data.get("effect_tags", []) - - if self.__class__.__name__ == "CrossviewForm" and data.get("published_only") is True: - prefilters["animal_group__experiment__study__published"] = True - - return json.dumps(prefilters) - - def clean(self): - cleaned_data = super().clean() - cleaned_data["prefilters"] = self.setPrefilters(cleaned_data) - return cleaned_data - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.createFields() - self.setInitialValues() - self.setFieldStyles() - - class SummaryTextForm(forms.ModelForm): class Meta: model = models.SummaryText @@ -667,72 +244,56 @@ class Meta: exclude = ("assessment", "visual_type", "prefilters", "studies", "sort_order") -class CrossviewForm(PrefilterMixin, VisualForm): - prefilter_include = ("study", "bioassay", "effect_tags") +class CrossviewForm(VisualForm): + def _get_prefilter_form(self, data, **form_kwargs): + prefix = form_kwargs.pop("prefix", None) + return self.prefilter( + data=data, prefix=prefix, assessment=self.instance.assessment, form_kwargs=form_kwargs + ).form def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.fields["dose_units"].queryset = DoseUnits.objects.get_animal_units( self.instance.assessment ) - self.helper = self.setHelper() - - class Meta: - model = models.Visual - exclude = ("assessment", "visual_type", "endpoints", "studies") - -class CrossviewForm1(VisualForm): - - def _get_prefilter_form(self,data,**form_kwargs): - prefix = form_kwargs.pop("prefix",None) - return self.prefilter(data=data,prefix=prefix,assessment=self.instance.assessment,form_kwargs=form_kwargs).form - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.fields["dose_units"].queryset = DoseUnits.objects.get_animal_units( - self.instance.assessment + self.prefilter = prefilters.VisualTypePrefilter.from_visual_type( + constants.VisualType.BIOASSAY_CROSSVIEW + ).value + self.fields["prefilters"] = DynamicFormField( + prefix="prefilters", form_class=self._get_prefilter_form, label="" ) - self.prefilter = prefilters.VisualTypePrefilter.from_visual_type(constants.VisualType.BIOASSAY_CROSSVIEW).value - self.fields["prefilters"] = DynamicFormField(prefix="prefilters",form_class=self._get_prefilter_form,label="") self.helper = self.setHelper() class Meta: model = models.Visual exclude = ("assessment", "visual_type", "endpoints", "studies") -class RoBForm(PrefilterMixin, VisualForm): - prefilter_include = ("bioassay",) - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.fields["studies"].queryset = self.fields["studies"].queryset.filter( - assessment=self.instance.assessment - ) - self.helper = self.setHelper() - - class Meta: - model = models.Visual - exclude = ("assessment", "visual_type", "dose_units", "endpoints") - -class RoBForm1(VisualForm): - def _get_prefilter_form(self,data,**form_kwargs): - prefix = form_kwargs.pop("prefix",None) - return self.prefilter(data=data,prefix=prefix,assessment=self.instance.assessment,form_kwargs=form_kwargs).form +class RoBForm(VisualForm): + def _get_prefilter_form(self, data, **form_kwargs): + prefix = form_kwargs.pop("prefix", None) + return self.prefilter( + data=data, prefix=prefix, assessment=self.instance.assessment, form_kwargs=form_kwargs + ).form def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.fields["studies"].queryset = self.fields["studies"].queryset.filter( assessment=self.instance.assessment ) - self.prefilter = prefilters.VisualTypePrefilter.from_visual_type(constants.VisualType.ROB_BARCHART).value - self.fields["prefilters"] = DynamicFormField(prefix="prefilters",form_class=self._get_prefilter_form,label="") + self.prefilter = prefilters.VisualTypePrefilter.from_visual_type( + constants.VisualType.ROB_BARCHART + ).value + self.fields["prefilters"] = DynamicFormField( + prefix="prefilters", form_class=self._get_prefilter_form, label="" + ) self.helper = self.setHelper() class Meta: model = models.Visual exclude = ("assessment", "visual_type", "dose_units", "endpoints") + class TagtreeForm(VisualForm): root_node = forms.TypedChoiceField( coerce=int, @@ -1015,9 +576,9 @@ def get_visual_form(visual_type): try: return { constants.VisualType.BIOASSAY_AGGREGATION: EndpointAggregationForm, - constants.VisualType.BIOASSAY_CROSSVIEW: CrossviewForm1, - constants.VisualType.ROB_HEATMAP: RoBForm1, - constants.VisualType.ROB_BARCHART: RoBForm1, + constants.VisualType.BIOASSAY_CROSSVIEW: CrossviewForm, + constants.VisualType.ROB_HEATMAP: RoBForm, + constants.VisualType.ROB_BARCHART: RoBForm, constants.VisualType.LITERATURE_TAGTREE: TagtreeForm, constants.VisualType.EXTERNAL_SITE: ExternalSiteForm, constants.VisualType.EXPLORE_HEATMAP: ExploreHeatmapForm, @@ -1114,7 +675,7 @@ def clean(self): self.add_error("excel_file", "Must contain at least 2 columns.") -class DataPivotQueryForm1(DataPivotForm): +class DataPivotQueryForm(DataPivotForm): class Meta: model = models.DataPivotQuery fields = ( @@ -1129,12 +690,14 @@ class Meta: "prefilters", ) - def _get_prefilter_form(self,data,**form_kwargs): - prefix = form_kwargs.pop("prefix",None) - return self.prefilter(data=data,prefix=prefix,assessment=self.instance.assessment,form_kwargs=form_kwargs).form - + def _get_prefilter_form(self, data, **form_kwargs): + prefix = form_kwargs.pop("prefix", None) + return self.prefilter( + data=data, prefix=prefix, assessment=self.instance.assessment, form_kwargs=form_kwargs + ).form + def __init__(self, *args, **kwargs): - evidence_type = kwargs.pop("evidence_type",None) + evidence_type = kwargs.pop("evidence_type", None) super().__init__(*args, **kwargs) if evidence_type is not None: @@ -1142,59 +705,11 @@ def __init__(self, *args, **kwargs): self.fields["evidence_type"].initial = self.instance.evidence_type self.fields["evidence_type"].disabled = True - self.prefilter = prefilters.StudyTypePrefilter.from_study_type(self.instance.evidence_type,self.instance.assessment).value - self.fields["prefilters"] = DynamicFormField(prefix="prefilters",form_class=self._get_prefilter_form,label="") - self.fields["preferred_units"].required = False - self.js_units_choices = json.dumps( - [ - {"id": obj.id, "name": obj.name} - for obj in DoseUnits.objects.get_animal_units(self.instance.assessment) - ] - ) - self.helper = self.setHelper() - - def save(self, commit=True): - self.instance.preferred_units = self.cleaned_data.get("preferred_units", []) - return super().save(commit=commit) - - def clean_export_style(self): - evidence_type = self.cleaned_data["evidence_type"] - export_style = self.cleaned_data["export_style"] - if ( - evidence_type not in (constants.StudyType.IN_VITRO, constants.StudyType.BIOASSAY) - and export_style != constants.ExportStyle.EXPORT_GROUP - ): - raise forms.ValidationError( - "Outcome/Result level export not implemented for this data-type." - ) - return export_style - - -class DataPivotQueryForm(PrefilterMixin, DataPivotForm): - prefilter_include = ("study", "bioassay", "epi", "invitro", "eco", "effect_tags") - - class Meta: - model = models.DataPivotQuery - fields = ( - "title", - "slug", - "evidence_type", - "export_style", - "preferred_units", - "settings", - "caption", - "published", - "prefilters", - ) - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.fields["evidence_type"].choices = ( - (constants.StudyType.BIOASSAY, "Animal Bioassay"), - (constants.StudyType.EPI, "Epidemiology"), - (constants.StudyType.EPI_META, "Epidemiology meta-analysis/pooled analysis"), - (constants.StudyType.IN_VITRO, "In vitro"), - (constants.StudyType.ECO, "Ecology"), + self.prefilter = prefilters.StudyTypePrefilter.from_study_type( + self.instance.evidence_type, self.instance.assessment + ).value + self.fields["prefilters"] = DynamicFormField( + prefix="prefilters", form_class=self._get_prefilter_form, label="" ) self.fields["preferred_units"].required = False self.js_units_choices = json.dumps( diff --git a/hawc/apps/summary/migrations/0043_new.py b/hawc/apps/summary/migrations/0043_new.py index 70dd3b90dc..aa85056054 100644 --- a/hawc/apps/summary/migrations/0043_new.py +++ b/hawc/apps/summary/migrations/0043_new.py @@ -3,8 +3,9 @@ from django.db import migrations -from hawc.apps.summary import constants from hawc.apps.assessment.constants import EpiVersion +from hawc.apps.summary import constants + def published_only_prefilters(apps, schema_editor): # add published_only field to prefilters @@ -28,8 +29,8 @@ def published_only_prefilters(apps, schema_editor): prefilters["design__study__published"] = obj.published_only obj.prefilters = json.dumps(prefilters) - DataPivotQuery.objects.bulk_update(objs,["prefilters"]) - + DataPivotQuery.objects.bulk_update(objs, ["prefilters"]) + def reverse_published_only_prefilters(apps, schema_editor): # TODO @@ -42,7 +43,9 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RunPython(published_only_prefilters, reverse_code=reverse_published_only_prefilters), + migrations.RunPython( + published_only_prefilters, reverse_code=reverse_published_only_prefilters + ), migrations.RemoveField( model_name="datapivotquery", name="published_only", diff --git a/hawc/apps/summary/migrations/0044_newest.py b/hawc/apps/summary/migrations/0044_newest.py index 4165b5c2e4..73974dac4b 100644 --- a/hawc/apps/summary/migrations/0044_newest.py +++ b/hawc/apps/summary/migrations/0044_newest.py @@ -4,14 +4,13 @@ from django.db import migrations, models - def prefilters_dict(apps, schema_editor): # load prefilters textfield into temp jsonfield DataPivotQuery = apps.get_model("summary", "DataPivotQuery") objs = DataPivotQuery.objects.all() for obj in objs: obj.temp = json.loads(obj.prefilters) - DataPivotQuery.objects.bulk_update(objs,["temp"]) + DataPivotQuery.objects.bulk_update(objs, ["temp"]) def reverse_prefilters_dict(apps, schema_editor): @@ -20,7 +19,8 @@ def reverse_prefilters_dict(apps, schema_editor): objs = DataPivotQuery.objects.all() for obj in objs: obj.prefilters = json.dumps(obj.temp) - DataPivotQuery.objects.bulk_update(objs,["prefilters"]) + DataPivotQuery.objects.bulk_update(objs, ["prefilters"]) + class Migration(migrations.Migration): dependencies = [ diff --git a/hawc/apps/summary/migrations/0045_newer.py b/hawc/apps/summary/migrations/0045_newer.py index 741a65ea50..debb367260 100644 --- a/hawc/apps/summary/migrations/0045_newer.py +++ b/hawc/apps/summary/migrations/0045_newer.py @@ -4,14 +4,13 @@ from django.db import migrations, models - def prefilters_dict(apps, schema_editor): # load prefilters textfield into temp jsonfield Visual = apps.get_model("summary", "Visual") objs = Visual.objects.all() for obj in objs: obj.temp = json.loads(obj.prefilters) - Visual.objects.bulk_update(objs,["temp"]) + Visual.objects.bulk_update(objs, ["temp"]) def reverse_prefilters_dict(apps, schema_editor): @@ -20,7 +19,8 @@ def reverse_prefilters_dict(apps, schema_editor): objs = Visual.objects.all() for obj in objs: obj.prefilters = json.dumps(obj.temp) - Visual.objects.bulk_update(objs,["prefilters"]) + Visual.objects.bulk_update(objs, ["prefilters"]) + class Migration(migrations.Migration): dependencies = [ diff --git a/hawc/apps/summary/models.py b/hawc/apps/summary/models.py index dcf7bf7c82..06241b1902 100644 --- a/hawc/apps/summary/models.py +++ b/hawc/apps/summary/models.py @@ -37,15 +37,10 @@ from ..common.models import get_model_copy_name from ..common.validators import validate_html_tags, validate_hyperlinks from ..eco.exports import EcoFlatComplete -from ..eco.models import Result from ..epi.exports import OutcomeDataPivot -from ..epi.models import Outcome from ..epimeta.exports import MetaResultFlatDataPivot -from ..epimeta.models import MetaResult from ..epiv2.exports import EpiFlatComplete -from ..epiv2.models import DataExtraction from ..invitro import exports as ivexports -from ..invitro.models import IVEndpoint from ..riskofbias.serializers import AssessmentRiskOfBiasSerializer from ..study.models import Study from . import constants, managers, prefilters @@ -459,16 +454,23 @@ def get_dose_units(): def get_json(self, json_encode=True): return SerializerHelper.get_serialized(self, json=json_encode) - def get_filterset(self): + def get_filterset_class(self): return prefilters.VisualTypePrefilter.from_visual_type(self.visual_type).value - def get_request_prefilters(self,request): + def get_filterset(self, data, assessment, **kwargs): + return self.get_filterset_class()(data=data, assessment=assessment, **kwargs) + + def get_request_prefilters(self, request): # TODO move get_editing_dataset out of models # so that we can utilize the forms # find all keys that start with "prefilters-" prefix prefix = "prefilters-" - return {key[len(prefix):]:value for key,value in request.POST.lists() if key.startswith(prefix)} + return { + key[len(prefix) :]: value + for key, value in request.POST.lists() + if key.startswith(prefix) + } def get_endpoints(self, request=None): qs = Endpoint.objects.none() @@ -484,17 +486,14 @@ def get_endpoints(self, request=None): qs = Endpoint.objects.filter(**filters) elif self.visual_type == constants.VisualType.BIOASSAY_CROSSVIEW: - fs = self.get_filterset() - if request: - dose_id = tryParseInt(request.POST.get("dose_units"), -1) - qs = fs(data=self.get_request_prefilters(request),assessment=self.assessment).qs - - else: - dose_id = self.dose_units_id - qs = fs(data=self.prefilters,assessment=self.assessment).qs + dose_id = ( + tryParseInt(request.POST.get("dose_units"), -1) if request else self.dose_units_id + ) + prefilters = self.get_request_prefilters(request) if request else self.prefilters + fs = self.get_filterset(prefilters, self.assessment) filters["animal_group__dosing_regime__doses__dose_units_id"] = dose_id - qs = qs.filter(**filters).distinct("id") + qs = fs.qs.filter(**filters).distinct("id") return qs @@ -511,32 +510,31 @@ def get_studies(self, request=None): constants.VisualType.ROB_HEATMAP, constants.VisualType.ROB_BARCHART, ]: - fs = self.get_filterset() - if request: - prefilters = self.get_request_prefilters(request) - if any(value for value in prefilters.values()): - endpoint_qs = fs(data=prefilters,assessment=self.assessment).qs - filters["id__in"] = set( - endpoint_qs.filter(assessment_id=self.assessment_id).values_list( - "animal_group__experiment__study_id", flat=True - ) - ) - else: - filters["id__in"] = request.POST.getlist("studies") - - qs = Study.objects.filter(**filters) - + prefilters = self.get_request_prefilters(request) if request else self.prefilters + fs = self.get_filterset(prefilters, self.assessment) + fs.form.is_valid() + cleaned_prefilters = fs.form.cleaned_data + + study_fields = [ + "animal_group__experiment__study__published", + "animal_group__experiment__study__in", + ] + endpoint_prefilters = { + k: v for k, v in cleaned_prefilters.items() if k not in study_fields + } + + if any(value for value in endpoint_prefilters.values()): + endpoint_qs = fs.qs + filters["id__in"] = set( + endpoint_qs.values_list("animal_group__experiment__study_id", flat=True) + ) else: - if any(value for value in self.prefilters.values()): - endpoint_qs = fs(data=self.prefilters,assessment=self.assessment).qs - filters["id__in"] = set( - endpoint_qs.filter(assessment_id=self.assessment_id).values_list( - "animal_group__experiment__study_id", flat=True - ) - ) - qs = Study.objects.filter(**filters) - else: - qs = self.studies.all() + if f := cleaned_prefilters.pop(study_fields[0], False): + filters["published"] = f + if f := cleaned_prefilters.get(study_fields[1], []): + filters["id__in"] = f + + qs = Study.objects.filter(**filters) if self.sort_order: if self.sort_order == "overall_confidence": @@ -560,6 +558,7 @@ def get_editing_dataset(self, request): return { "assessment": self.assessment_id, + "assessment_rob_name": self.assessment.get_rob_name_display(), "title": request.POST.get("title"), "slug": request.POST.get("slug"), "caption": request.POST.get("caption"), @@ -757,7 +756,6 @@ class DataPivotQuery(DataPivot): ) prefilters = models.JSONField(default=dict) - def clean(self): count = self.get_queryset().count() @@ -783,24 +781,26 @@ def _refine_queryset(self, qs): epi_version = self.assessment.epi_version if self.evidence_type == constants.StudyType.BIOASSAY: - qs = qs.filter(assessment_id = self.assessment_id) + qs = qs.filter(assessment_id=self.assessment_id) if self.preferred_units: - qs = qs.filter(animal_group__dosing_regime__doses__dose_units__in = self.preferred_units) + qs = qs.filter( + animal_group__dosing_regime__doses__dose_units__in=self.preferred_units + ) elif self.evidence_type == constants.StudyType.EPI and epi_version == EpiVersion.V1: - qs = qs.filter(assessment_id = self.assessment_id) + qs = qs.filter(assessment_id=self.assessment_id) elif self.evidence_type == constants.StudyType.EPI and epi_version == EpiVersion.V2: - qs = qs.filter(design__study__assessment_id = self.assessment_id) + qs = qs.filter(design__study__assessment_id=self.assessment_id) elif self.evidence_type == constants.StudyType.EPI_META: - qs = qs.filter(protocol__study__assessment_id = self.assessment_id) + qs = qs.filter(protocol__study__assessment_id=self.assessment_id) elif self.evidence_type == constants.StudyType.IN_VITRO: - qs = qs.filter(assessment_id = self.assessment_id) + qs = qs.filter(assessment_id=self.assessment_id) elif self.evidence_type == constants.StudyType.ECO: - qs = qs.filter(design__study__assessment_id = self.assessment_id) + qs = qs.filter(design__study__assessment_id=self.assessment_id) return qs @@ -862,14 +862,17 @@ def _get_dataset_exporter(self, qs): ) return exporter - - def get_filterset(self): - return prefilters.StudyTypePrefilter.from_study_type(self.evidence_type,self.assessment).value + def get_filterset_class(self): + return prefilters.StudyTypePrefilter.from_study_type( + self.evidence_type, self.assessment + ).value + + def get_filterset(self, data, assessment, **kwargs): + return self.get_filterset_class()(data=data, assessment=assessment, **kwargs) def get_queryset(self): - fs = self.get_filterset() - qs = fs(data=self.prefilters,assessment=self.assessment).qs + qs = self.get_filterset(self.prefilters, self.assessment).qs qs = self._refine_queryset(qs) return qs.order_by("id") @@ -930,74 +933,6 @@ def _update_settings_across_assessments(self, cw: dict) -> str: return json.dumps(settings) -class Prefilter: - """ - Helper-object to deal with DataPivot and Visual prefilters fields. - """ - - @staticmethod - def setFiltersFromForm(assessment, filters, d, visual_type): - evidence_type = d.get("evidence_type") - epi_version = assessment.epi_version - - if visual_type == constants.VisualType.BIOASSAY_CROSSVIEW: - evidence_type = constants.StudyType.BIOASSAY - - if d.get("prefilter_system"): - filters["system__in"] = d.getlist("systems") - - if d.get("prefilter_organ"): - filters["organ__in"] = d.getlist("organs") - - if d.get("prefilter_effect"): - filters["effect__in"] = d.getlist("effects") - - if d.get("prefilter_effect_subtype"): - filters["effect_subtype__in"] = d.getlist("effect_subtypes") - - if d.get("prefilter_effect_tag"): - filters["effects__in"] = d.getlist("effect_tags") - - if d.get("prefilter_episystem"): - filters["system__in"] = d.getlist("episystems") - - if d.get("prefilter_epieffect"): - filters["effect__in"] = d.getlist("epieffects") - - if d.get("prefilter_study"): - studies = d.getlist("studies", []) - if evidence_type == constants.StudyType.BIOASSAY: - filters["animal_group__experiment__study__in"] = studies - elif evidence_type == constants.StudyType.EPI and epi_version == 1: - filters["study_population__study__in"] = studies - elif evidence_type == constants.StudyType.EPI and epi_version == 2: - filters["design__study__in"] = studies - elif evidence_type == constants.StudyType.IN_VITRO: - filters["experiment__study__in"] = studies - elif evidence_type == constants.StudyType.EPI_META: - filters["protocol__study__in"] = studies - else: - raise ValueError("Unknown evidence type") - - if d.get("published_only"): - if evidence_type == constants.StudyType.BIOASSAY: - filters["animal_group__experiment__study__published"] = True - elif evidence_type == constants.StudyType.EPI and epi_version == 1: - filters["study_population__study__published"] = True - elif evidence_type == constants.StudyType.EPI and epi_version == 2: - filters["design__study__published"] = True - elif evidence_type == constants.StudyType.IN_VITRO: - filters["experiment__study__published"] = True - elif evidence_type == constants.StudyType.EPI_META: - filters["protocol__study__published"] = True - else: - raise ValueError("Unknown evidence type") - - @staticmethod - def setFiltersFromObj(filters, prefilters): - filters.update(json.loads(prefilters)) - - reversion.register(SummaryText) reversion.register(SummaryTable) reversion.register(DataPivot) diff --git a/hawc/apps/summary/prefilters.py b/hawc/apps/summary/prefilters.py index dafdb57ea7..bca293eb0c 100644 --- a/hawc/apps/summary/prefilters.py +++ b/hawc/apps/summary/prefilters.py @@ -1,24 +1,23 @@ +from enum import Enum + import django_filters as df -from django.forms.widgets import CheckboxInput from django import forms -from enum import Enum +from django.forms.widgets import CheckboxInput +from ..animal.models import Endpoint from ..assessment.constants import EpiVersion from ..assessment.models import Assessment, EffectTag -from ..animal.models import Endpoint -from ..common.filterset import BaseFilterSet, filter_noop +from ..common.filterset import BaseFilterSet from ..common.forms import BaseFormHelper -from ..study.models import Study from ..epi.models import Outcome -from ..epiv2.models import DataExtraction from ..epimeta.models import MetaResult -from ..invitro.models import IVEndpoint, IVEndpointCategory, IVChemical -from ..eco.models import Result - +from ..epiv2.models import DataExtraction +from ..invitro.models import IVChemical, IVEndpoint, IVEndpointCategory +from ..study.models import Study from .constants import StudyType, VisualType + class TestForm(forms.Form): - @property def helper(self): helper = BaseFormHelper(self) @@ -26,10 +25,11 @@ def helper(self): return helper + def filter_published_only(queryset, name, value): if not value: return queryset - return queryset.filter(**{name:True}) + return queryset.filter(**{name: True}) class BioassayPrefilter(BaseFilterSet): @@ -74,8 +74,8 @@ class BioassayPrefilter(BaseFilterSet): ) effects__in = df.MultipleChoiceFilter( field_name="effects", - label="Tags to include", - help_text="""Select one or more effect-tags to include in the plot. + label="Tags to include", + help_text="""Select one or more effect-tags to include in the plot. If no study is selected, no endpoints will be available.""", ) @@ -87,13 +87,16 @@ class Meta: "system__in", "organ__in", "effect__in", - "effect_subtype__in","effects__in", + "effect_subtype__in", + "effects__in", ] form = TestForm def create_form(self): form = super().create_form() - form.fields["animal_group__experiment__study__in"].choices = Study.objects.get_choices(self.assessment.pk) + form.fields["animal_group__experiment__study__in"].choices = Study.objects.get_choices( + self.assessment.pk + ) form.fields["system__in"].choices = Endpoint.objects.get_system_choices(self.assessment.pk) form.fields["organ__in"].choices = Endpoint.objects.get_organ_choices(self.assessment.pk) form.fields["effect__in"].choices = Endpoint.objects.get_effect_choices(self.assessment.pk) @@ -104,7 +107,6 @@ def create_form(self): return form - class EpiV1Prefilter(BaseFilterSet): # studies study_population__study__published = df.BooleanFilter( @@ -123,20 +125,20 @@ class EpiV1Prefilter(BaseFilterSet): # epi system__in = df.MultipleChoiceFilter( field_name="system", - label="Systems to include", - help_text="""Select one or more systems to include in the plot. + label="Systems to include", + help_text="""Select one or more systems to include in the plot. If no system is selected, no endpoints will be available.""", ) effect__in = df.MultipleChoiceFilter( field_name="effect", - label="Effects to include", - help_text="""Select one or more effects to include in the plot. + label="Effects to include", + help_text="""Select one or more effects to include in the plot. If no effect is selected, no endpoints will be available.""", ) effects__in = df.MultipleChoiceFilter( field_name="effects", - label="Tags to include", - help_text="""Select one or more effect-tags to include in the plot. + label="Tags to include", + help_text="""Select one or more effect-tags to include in the plot. If no study is selected, no endpoints will be available.""", ) @@ -144,18 +146,24 @@ class Meta: model = Outcome fields = [ "study_population__study__published", - "study_population__study__in","system__in","effect__in","effects__in", + "study_population__study__in", + "system__in", + "effect__in", + "effects__in", ] form = TestForm def create_form(self): form = super().create_form() - form.fields["study_population__study__in"].choices = Study.objects.get_choices(self.assessment.pk) + form.fields["study_population__study__in"].choices = Study.objects.get_choices( + self.assessment.pk + ) form.fields["system__in"].choices = Outcome.objects.get_system_choices(self.assessment.pk) form.fields["effect__in"].choices = Outcome.objects.get_effect_choices(self.assessment.pk) form.fields["effects__in"].choices = EffectTag.objects.get_choices(self.assessment.pk) return form + class EpiV2Prefilter(BaseFilterSet): # studies design__study__published = df.BooleanFilter( @@ -185,6 +193,7 @@ def create_form(self): form.fields["design__study__in"].choices = Study.objects.get_choices(self.assessment.pk) return form + class EpiMetaPrefilter(BaseFilterSet): # studies protocol__study__published = df.BooleanFilter( @@ -214,6 +223,7 @@ def create_form(self): form.fields["protocol__study__in"].choices = Study.objects.get_choices(self.assessment.pk) return form + class InvitroPrefilter(BaseFilterSet): # studies experiment__study__published = df.BooleanFilter( @@ -232,20 +242,20 @@ class InvitroPrefilter(BaseFilterSet): # invitro category__in = df.MultipleChoiceFilter( field_name="category", - label="Categories to include", - help_text="""Select one or more categories to include in the plot. + label="Categories to include", + help_text="""Select one or more categories to include in the plot. If no study is selected, no endpoints will be available.""", ) chemical__name__in = df.MultipleChoiceFilter( field_name="chemical__name", - label="Chemicals to include", - help_text="""Select one or more chemicals to include in the plot. + label="Chemicals to include", + help_text="""Select one or more chemicals to include in the plot. If no study is selected, no endpoints will be available.""", ) effects__in = df.MultipleChoiceFilter( field_name="effects", - label="Tags to include", - help_text="""Select one or more effect-tags to include in the plot. + label="Tags to include", + help_text="""Select one or more effect-tags to include in the plot. If no study is selected, no endpoints will be available.""", ) @@ -253,7 +263,10 @@ class Meta: model = IVEndpoint fields = [ "experiment__study__published", - "experiment__study__in","category__in","chemical__name__in","effects__in", + "experiment__study__in", + "category__in", + "chemical__name__in", + "effects__in", ] form = TestForm @@ -261,7 +274,9 @@ def create_form(self): form = super().create_form() form.fields["experiment__study__in"].choices = Study.objects.get_choices(self.assessment.pk) form.fields["category__in"].choices = IVEndpointCategory.get_choices(self.assessment.pk) - form.fields["chemical__name__in"].choices = IVChemical.objects.get_choices(self.assessment.pk) + form.fields["chemical__name__in"].choices = IVChemical.objects.get_choices( + self.assessment.pk + ) form.fields["effects__in"].choices = EffectTag.objects.get_choices(self.assessment.pk) return form @@ -274,7 +289,7 @@ class StudyTypePrefilter(Enum): IN_VITRO = InvitroPrefilter @classmethod - def from_study_type(cls,study_type:StudyType,assessment:Assessment): + def from_study_type(cls, study_type: int | StudyType, assessment: Assessment): study_type = StudyType(study_type) name = study_type.name if study_type == StudyType.EPI: @@ -284,13 +299,14 @@ def from_study_type(cls,study_type:StudyType,assessment:Assessment): name = "EPIV2" return cls[name] + class VisualTypePrefilter(Enum): BIOASSAY_CROSSVIEW = BioassayPrefilter ROB_HEATMAP = BioassayPrefilter ROB_BARCHART = BioassayPrefilter @classmethod - def from_visual_type(cls,visual_type:VisualType): + def from_visual_type(cls, visual_type: int | VisualType): visual_type = VisualType(visual_type) name = visual_type.name - return cls[name] \ No newline at end of file + return cls[name] diff --git a/hawc/apps/summary/templates/summary/datapivot_form.html b/hawc/apps/summary/templates/summary/datapivot_form.html index 853e3a51dc..c5bdec594c 100644 --- a/hawc/apps/summary/templates/summary/datapivot_form.html +++ b/hawc/apps/summary/templates/summary/datapivot_form.html @@ -24,7 +24,7 @@ {% endif %} window.app.startup("assessmentStartup", function(app){ - var doseWidget = new app.DoseUnitsWidget($('form'), { + new app.DoseUnitsWidget($('form'), { choices: js_units_choices, el: '#id_preferred_units', }); @@ -37,63 +37,12 @@ ); }) - var togglePrefilterSelectorVisibility = function(d){ - var fields = [ - ["study", "studies"], - ["system", "systems"], - ["organ", "organs"], - ["effect", "effects"], - ["effect_subtype", "effect_subtypes"], - ["episystem", "episystems"], - ["epieffect", "epieffects"], - ["iv_category", "iv_categories"], - ["iv_chemical", "iv_chemicals"], - ["effect_tag", "effect_tags"], - ]; - _.each(fields, function(d){ - $(printf('#id_prefilter_{0}', d[0])).on('change', function(){ - var div = $(printf('#div_id_{0}', d[1])); - ($(this).prop('checked')) ? div.show(1000) : div.hide(0); - }).trigger('change'); - }); - } - // determine which fields to display depending on data-type - $('#id_evidence_type').on('change', function(){ - const value = parseInt($('#id_evidence_type').val()), - aniOnlyDivs = $([ - "#div_id_preferred_units", - "#div_id_prefilter_system", - "#div_id_systems", - "#div_id_prefilter_organ", - "#div_id_organs", - "#div_id_prefilter_effect", - "#div_id_prefilter_effect_subtype", - "#div_id_effects" - ].join(",")), - epiOnlyDivs = $([ - "#div_id_prefilter_episystem", - "#div_id_episystems", - "#div_id_prefilter_epieffect", - "#div_id_epieffects" - ].join(",")), - ivOnlyDivs = $([ - "#div_id_prefilter_iv_category", - "#div_id_iv_categories", - "#div_id_prefilter_iv_chemical", - "#div_id_iv_chemicals", - ].join(",")), - aniIvOnlyDivs = $("#div_id_export_style"), - notEpiV2Divs = $("#div_id_prefilter_effect_tag"); - (value === 0) ? aniOnlyDivs.show() : aniOnlyDivs.hide(); - (value === 1 && epi_version === 1) ? epiOnlyDivs.show() : epiOnlyDivs.hide(); - (value === 1 && epi_version === 1) ? notEpiV2Divs.show() : notEpiV2Divs.hide(); - (value === 2) ? ivOnlyDivs.show() : ivOnlyDivs.hide(); - (value == 0 || value == 2) ? aniIvOnlyDivs.show() : aniIvOnlyDivs.hide(); - togglePrefilterSelectorVisibility(); - }).trigger('change'); - - togglePrefilterSelectorVisibility(); + const value = {{form.instance.evidence_type}}, + aniOnlyDivs = $("#div_id_preferred_units"), + aniIvOnlyDivs = $("#div_id_export_style"); + (value === 0) ? aniOnlyDivs.show() : aniOnlyDivs.hide(); + (value == 0 || value == 2) ? aniIvOnlyDivs.show() : aniIvOnlyDivs.hide(); }); {% endblock extrajs %} diff --git a/hawc/apps/summary/templates/summary/datapivot_form1.html b/hawc/apps/summary/templates/summary/datapivot_form1.html deleted file mode 100644 index c5bdec594c..0000000000 --- a/hawc/apps/summary/templates/summary/datapivot_form1.html +++ /dev/null @@ -1,48 +0,0 @@ -{% extends 'assessment-rooted.html' %} - -{% load crispy_forms_tags %} -{% load add_class %} - -{% block content %} - {% include "assessment/preferred_dose_units_widget.html" %} - {% crispy form %} - {% include "summary/_smartTagEditModal.html" with form=smart_tag_form only %} -{% endblock %} - -{% block extrajs %} - {{ smart_tag_form.media }} - -{% endblock extrajs %} diff --git a/hawc/apps/summary/urls.py b/hawc/apps/summary/urls.py index 7424b6861e..fd3630e463 100644 --- a/hawc/apps/summary/urls.py +++ b/hawc/apps/summary/urls.py @@ -115,15 +115,10 @@ name="dp_new-prompt", ), path( - "data-pivot/assessment//create/query/", + "data-pivot/assessment//create/query//", views.DataPivotQueryNew.as_view(), name="dp_new-query", ), - path( - "data-pivot/assessment//create/query//", - views.DataPivotQueryNew1.as_view(), - name="dp_new-query1", - ), path( "data-pivot/assessment//create/file/", views.DataPivotFileNew.as_view(), @@ -154,11 +149,6 @@ views.DataPivotUpdateQuery.as_view(), name="dp_query-update", ), - path( - "data-pivot/assessment///query-update1/", - views.DataPivotUpdateQuery1.as_view(), - name="dp_query-update1", - ), path( "data-pivot/assessment///file-update/", views.DataPivotUpdateFile.as_view(), diff --git a/hawc/apps/summary/views.py b/hawc/apps/summary/views.py index 007321af28..7f55dc0d83 100644 --- a/hawc/apps/summary/views.py +++ b/hawc/apps/summary/views.py @@ -15,9 +15,16 @@ from ..assessment.views import check_published_status from ..common.crumbs import Breadcrumb from ..common.helper import WebappConfig -from ..common.views import BaseCreate, BaseDelete, BaseDetail, BaseFilterList, BaseList, BaseUpdate, FilterSetMixin +from ..common.views import ( + BaseCreate, + BaseDelete, + BaseDetail, + BaseFilterList, + BaseList, + BaseUpdate, +) from ..riskofbias.models import RiskOfBiasMetric -from . import constants, filterset, forms, models, serializers, prefilters +from . import constants, filterset, forms, models, prefilters, serializers def get_visual_list_crumb(assessment) -> Breadcrumb: @@ -632,20 +639,7 @@ def get_form_kwargs(self): class DataPivotQueryNew(DataPivotNew): model = models.DataPivotQuery form_class = forms.DataPivotQueryForm - - def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) - context["file_loader"] = False - context["smart_tag_form"] = forms.SmartTagForm(assessment_id=self.assessment.id) - context["breadcrumbs"].insert( - len(context["breadcrumbs"]) - 1, get_visual_list_crumb(self.assessment) - ) - return context - -class DataPivotQueryNew1(DataPivotNew): - model = models.DataPivotQuery - form_class = forms.DataPivotQueryForm1 - template_name = "summary/datapivot_form1.html" + template_name = "summary/datapivot_form.html" def get_form_kwargs(self): kwargs = super().get_form_kwargs() @@ -653,10 +647,10 @@ def get_form_kwargs(self): # get study type enum study_type = constants.StudyType(self.kwargs.get("study_type")) # make sure prefilter exists for study type - prefilters.StudyTypePrefilter.from_study_type(study_type,self.assessment) + prefilters.StudyTypePrefilter.from_study_type(study_type, self.assessment) # pass study type to form kwargs["evidence_type"] = study_type - except (KeyError,ValueError): + except (KeyError, ValueError): raise Http404 return kwargs @@ -669,6 +663,7 @@ def get_context_data(self, **kwargs): ) return context + class DataPivotFileNew(DataPivotNew): model = models.DataPivotUpload form_class = forms.DataPivotUploadForm @@ -782,20 +777,6 @@ def get_context_data(self, **kwargs): ) return context -class DataPivotUpdateQuery1(GetDataPivotObjectMixin, BaseUpdate): - success_message = "Data Pivot updated." - model = models.DataPivotQuery - form_class = forms.DataPivotQueryForm1 - template_name = "summary/datapivot_form.html" - - def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) - context["file_loader"] = False - context["smart_tag_form"] = forms.SmartTagForm(assessment_id=self.assessment.id) - context["breadcrumbs"].insert( - len(context["breadcrumbs"]) - 2, get_visual_list_crumb(self.assessment) - ) - return context class DataPivotUpdateFile(GetDataPivotObjectMixin, BaseUpdate): success_message = "Data Pivot updated." diff --git a/tests/hawc/apps/summary/test_views.py b/tests/hawc/apps/summary/test_views.py index c3a9c23036..66df75cd70 100644 --- a/tests/hawc/apps/summary/test_views.py +++ b/tests/hawc/apps/summary/test_views.py @@ -14,7 +14,7 @@ def test_initial_settings(self): c = Client() assert c.login(username="pm@hawcproject.org", password="pw") is True - url = reverse("summary:dp_new-query", args=(1,)) + url = reverse("summary:dp_new-query", args=(1, 0)) # no initial settings or invalid settings for args in ["", "?initial=-1", "?initial=-1&reset_row_overrides=1"]: From 89d2f588b7923d4422221598f2dbe277d8f34367 Mon Sep 17 00:00:00 2001 From: Daniel Rabstejnek Date: Wed, 23 Aug 2023 12:13:11 -0400 Subject: [PATCH 05/35] Improvements --- hawc/apps/summary/admin.py | 2 +- hawc/apps/summary/forms.py | 3 - .../0043_datapivotquery_prefilters_json.py | 92 +++++++++ hawc/apps/summary/migrations/0043_new.py | 53 ----- hawc/apps/summary/migrations/0044_newest.py | 47 ----- .../migrations/0044_visual_prefilters_json.py | 89 +++++++++ ...pivotquery_published_only_to_prefilters.py | 38 ++++ hawc/apps/summary/migrations/0045_newer.py | 47 ----- .../0046_visual_studies_to_prefilters.py | 33 ++++ hawc/apps/summary/models.py | 32 +-- hawc/apps/summary/prefilters.py | 183 +++++++++++------- hawc/apps/summary/serializers.py | 5 +- .../templates/summary/datapivot_detail.html | 2 +- 13 files changed, 367 insertions(+), 259 deletions(-) create mode 100644 hawc/apps/summary/migrations/0043_datapivotquery_prefilters_json.py delete mode 100644 hawc/apps/summary/migrations/0043_new.py delete mode 100644 hawc/apps/summary/migrations/0044_newest.py create mode 100644 hawc/apps/summary/migrations/0044_visual_prefilters_json.py create mode 100644 hawc/apps/summary/migrations/0045_datapivotquery_published_only_to_prefilters.py delete mode 100644 hawc/apps/summary/migrations/0045_newer.py create mode 100644 hawc/apps/summary/migrations/0046_visual_studies_to_prefilters.py diff --git a/hawc/apps/summary/admin.py b/hawc/apps/summary/admin.py index b4b5f15bde..a384ffece6 100644 --- a/hawc/apps/summary/admin.py +++ b/hawc/apps/summary/admin.py @@ -22,7 +22,7 @@ class VisualAdmin(admin.ModelAdmin): list_filter = ("visual_type", "published", ("assessment", admin.RelatedOnlyFieldListFilter)) search_fields = ("assessment__name", "title") - raw_id_fields = ("endpoints", "studies") + raw_id_fields = ("endpoints",) @admin.display(description="URL") def show_url(self, obj): diff --git a/hawc/apps/summary/forms.py b/hawc/apps/summary/forms.py index 954a91b5fd..d7e292f326 100644 --- a/hawc/apps/summary/forms.py +++ b/hawc/apps/summary/forms.py @@ -278,9 +278,6 @@ def _get_prefilter_form(self, data, **form_kwargs): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.fields["studies"].queryset = self.fields["studies"].queryset.filter( - assessment=self.instance.assessment - ) self.prefilter = prefilters.VisualTypePrefilter.from_visual_type( constants.VisualType.ROB_BARCHART ).value diff --git a/hawc/apps/summary/migrations/0043_datapivotquery_prefilters_json.py b/hawc/apps/summary/migrations/0043_datapivotquery_prefilters_json.py new file mode 100644 index 0000000000..9aa68008c2 --- /dev/null +++ b/hawc/apps/summary/migrations/0043_datapivotquery_prefilters_json.py @@ -0,0 +1,92 @@ +# Generated by Django 4.2.3 on 2023-08-21 03:49 +import json + +from django.db import migrations, models + +from hawc.apps.summary.prefilters import StudyTypePrefilter + +mapping = { + StudyTypePrefilter.BIOASSAY:{ + "published_only":"animal_group__experiment__study__published", + "studies":"animal_group__experiment__study__in", + "systems":"system__in", + "organs":"organ__in", + "effects":"effect__in", + "effect_subtypes":"effect_subtype__in", + "effect_tags":"effects__in" + }, + StudyTypePrefilter.EPIV1:{ + "published_only":"study_population__study__published", + "studies":"study_population__study__in", + "systems":"system__in", + "effects":"effect__in", + "effect_tags":"effects__in" + }, + StudyTypePrefilter.EPIV2:{ + "published_only":"design__study__published", + "studies":"design__study__in" + }, + StudyTypePrefilter.EPI_META:{ + "published_only":"protocol__study__published", + "studies":"protocol__study__in" + }, + StudyTypePrefilter.IN_VITRO:{ + "published_only":"experiment__study__published", + "studies":"experiment__study__in", + "categories":"category__in", + "chemicals":"chemical__name__in", + "effect_tags":"effects__in" + } +} + +def prefilters_dict(apps, schema_editor): + # load prefilters textfield into temp jsonfield + DataPivotQuery = apps.get_model("summary", "DataPivotQuery") + objs = DataPivotQuery.objects.all().select_related("assessment") + for obj in objs: + data = json.loads(obj.prefilters) + if not data: + continue + key_map = mapping[StudyTypePrefilter.from_study_type(obj.evidence_type,obj.assessment)] + key_map = {v:k for k,v in key_map.items()} + data = {key_map[k]:v for k,v in data.items()} + obj.temp = data + DataPivotQuery.objects.bulk_update(objs, ["temp"]) + + +def reverse_prefilters_dict(apps, schema_editor): + # dump temp jsonfield into prefilters textfield + DataPivotQuery = apps.get_model("summary", "DataPivotQuery") + objs = DataPivotQuery.objects.all().select_related("assessment") + for obj in objs: + if not obj.temp: + continue + key_map = mapping[StudyTypePrefilter.from_study_type(obj.evidence_type,obj.assessment)] + data = {key_map[k]:v for k,v in obj.temp.items()} + obj.prefilters = json.dumps(data) + DataPivotQuery.objects.bulk_update(objs, ["prefilters"]) + + +class Migration(migrations.Migration): + dependencies = [ + ("summary", "0042_summarytable_interactive"), + ] + + operations = [ + # change prefilters textfield into jsonfield + migrations.AddField( + model_name="datapivotquery", + name="temp", + field=models.JSONField(default=dict), + ), + migrations.RunPython(prefilters_dict, reverse_code=reverse_prefilters_dict), + migrations.RemoveField( + model_name="datapivotquery", + name="prefilters", + ), + migrations.RenameField( + model_name="datapivotquery", + old_name="temp", + new_name="prefilters", + ), + ] diff --git a/hawc/apps/summary/migrations/0043_new.py b/hawc/apps/summary/migrations/0043_new.py deleted file mode 100644 index aa85056054..0000000000 --- a/hawc/apps/summary/migrations/0043_new.py +++ /dev/null @@ -1,53 +0,0 @@ -# Generated by Django 4.2.3 on 2023-08-21 03:49 -import json - -from django.db import migrations - -from hawc.apps.assessment.constants import EpiVersion -from hawc.apps.summary import constants - - -def published_only_prefilters(apps, schema_editor): - # add published_only field to prefilters - DataPivotQuery = apps.get_model("summary", "DataPivotQuery") - objs = DataPivotQuery.objects.all().select_related("assessment") - for obj in objs: - prefilters = json.loads(obj.prefilters) - epi_version = obj.assessment.epi_version - - if obj.evidence_type == constants.StudyType.BIOASSAY: - prefilters["animal_group__experiment__study__published"] = obj.published_only - elif obj.evidence_type == constants.StudyType.EPI and epi_version == EpiVersion.V1: - prefilters["study_population__study__published"] = obj.published_only - elif obj.evidence_type == constants.StudyType.EPI and epi_version == EpiVersion.V2: - prefilters["design__study__published"] = obj.published_only - elif obj.evidence_type == constants.StudyType.EPI_META: - prefilters["protocol__study__published"] = obj.published_only - elif obj.evidence_type == constants.StudyType.IN_VITRO: - prefilters["experiment__study__published"] = obj.published_only - elif obj.evidence_type == constants.StudyType.ECO: - prefilters["design__study__published"] = obj.published_only - - obj.prefilters = json.dumps(prefilters) - DataPivotQuery.objects.bulk_update(objs, ["prefilters"]) - - -def reverse_published_only_prefilters(apps, schema_editor): - # TODO - return - - -class Migration(migrations.Migration): - dependencies = [ - ("summary", "0042_summarytable_interactive"), - ] - - operations = [ - migrations.RunPython( - published_only_prefilters, reverse_code=reverse_published_only_prefilters - ), - migrations.RemoveField( - model_name="datapivotquery", - name="published_only", - ), - ] diff --git a/hawc/apps/summary/migrations/0044_newest.py b/hawc/apps/summary/migrations/0044_newest.py deleted file mode 100644 index 73974dac4b..0000000000 --- a/hawc/apps/summary/migrations/0044_newest.py +++ /dev/null @@ -1,47 +0,0 @@ -# Generated by Django 4.2.3 on 2023-08-21 03:49 -import json - -from django.db import migrations, models - - -def prefilters_dict(apps, schema_editor): - # load prefilters textfield into temp jsonfield - DataPivotQuery = apps.get_model("summary", "DataPivotQuery") - objs = DataPivotQuery.objects.all() - for obj in objs: - obj.temp = json.loads(obj.prefilters) - DataPivotQuery.objects.bulk_update(objs, ["temp"]) - - -def reverse_prefilters_dict(apps, schema_editor): - # dump temp jsonfield into prefilters textfield - DataPivotQuery = apps.get_model("summary", "DataPivotQuery") - objs = DataPivotQuery.objects.all() - for obj in objs: - obj.prefilters = json.dumps(obj.temp) - DataPivotQuery.objects.bulk_update(objs, ["prefilters"]) - - -class Migration(migrations.Migration): - dependencies = [ - ("summary", "0043_new"), - ] - - operations = [ - # change prefilters textfield into jsonfield - migrations.AddField( - model_name="datapivotquery", - name="temp", - field=models.JSONField(default=dict), - ), - migrations.RunPython(prefilters_dict, reverse_code=reverse_prefilters_dict), - migrations.RemoveField( - model_name="datapivotquery", - name="prefilters", - ), - migrations.RenameField( - model_name="datapivotquery", - old_name="temp", - new_name="prefilters", - ), - ] diff --git a/hawc/apps/summary/migrations/0044_visual_prefilters_json.py b/hawc/apps/summary/migrations/0044_visual_prefilters_json.py new file mode 100644 index 0000000000..99314938b6 --- /dev/null +++ b/hawc/apps/summary/migrations/0044_visual_prefilters_json.py @@ -0,0 +1,89 @@ +# Generated by Django 4.2.3 on 2023-08-21 03:49 +import json + +from django.db import migrations, models + +from hawc.apps.summary.prefilters import VisualTypePrefilter + +mapping = { + VisualTypePrefilter.BIOASSAY_CROSSVIEW: { + "published_only": "animal_group__experiment__study__published", + "studies": "animal_group__experiment__study__in", + "systems": "system__in", + "organs": "organ__in", + "effects": "effect__in", + "effect_subtypes": "effect_subtype__in", + "effect_tags": "effects__in", + }, + VisualTypePrefilter.ROB_HEATMAP: { + "published_only": "animal_group__experiment__study__published", + "studies": "animal_group__experiment__study__in", + "systems": "system__in", + "organs": "organ__in", + "effects": "effect__in", + "effect_subtypes": "effect_subtype__in", + "effect_tags": "effects__in", + }, + VisualTypePrefilter.ROB_BARCHART: { + "published_only": "animal_group__experiment__study__published", + "studies": "animal_group__experiment__study__in", + "systems": "system__in", + "organs": "organ__in", + "effects": "effect__in", + "effect_subtypes": "effect_subtype__in", + "effect_tags": "effects__in", + }, +} + + +def prefilters_dict(apps, schema_editor): + # load prefilters textfield into temp jsonfield + Visual = apps.get_model("summary", "Visual") + objs = Visual.objects.all() + for obj in objs: + data = json.loads(obj.prefilters) + if not data: + continue + key_map = mapping[VisualTypePrefilter.from_visual_type(obj.visual_type)] + key_map = {v: k for k, v in key_map.items()} + data = {key_map[k]: v for k, v in data.items()} + obj.temp = data + Visual.objects.bulk_update(objs, ["temp"]) + + +def reverse_prefilters_dict(apps, schema_editor): + # dump temp jsonfield into prefilters textfield + Visual = apps.get_model("summary", "Visual") + objs = Visual.objects.all() + for obj in objs: + if not obj.temp: + continue + key_map = mapping[VisualTypePrefilter.from_visual_type(obj.visual_type)] + data = {key_map[k]: v for k, v in obj.temp.items()} + obj.prefilters = json.dumps(data) + Visual.objects.bulk_update(objs, ["prefilters"]) + + +class Migration(migrations.Migration): + dependencies = [ + ("summary", "0043_datapivotquery_prefilters_json"), + ] + + operations = [ + # change prefilters textfield into jsonfield + migrations.AddField( + model_name="visual", + name="temp", + field=models.JSONField(default=dict), + ), + migrations.RunPython(prefilters_dict, reverse_code=reverse_prefilters_dict), + migrations.RemoveField( + model_name="visual", + name="prefilters", + ), + migrations.RenameField( + model_name="visual", + old_name="temp", + new_name="prefilters", + ), + ] diff --git a/hawc/apps/summary/migrations/0045_datapivotquery_published_only_to_prefilters.py b/hawc/apps/summary/migrations/0045_datapivotquery_published_only_to_prefilters.py new file mode 100644 index 0000000000..25612b5b69 --- /dev/null +++ b/hawc/apps/summary/migrations/0045_datapivotquery_published_only_to_prefilters.py @@ -0,0 +1,38 @@ +# Generated by Django 4.2.3 on 2023-08-21 03:49 + + +from django.db import migrations + + +def published_only_prefilters(apps, schema_editor): + # add published_only field to prefilters + DataPivotQuery = apps.get_model("summary", "DataPivotQuery") + objs = DataPivotQuery.objects.all() + for obj in objs: + obj.prefilters["published_only"] = obj.published_only + DataPivotQuery.objects.bulk_update(objs, ["prefilters"]) + + +def reverse_published_only_prefilters(apps, schema_editor): + # separate published_only field from prefilters + DataPivotQuery = apps.get_model("summary", "DataPivotQuery") + objs = DataPivotQuery.objects.all() + for obj in objs: + obj.published_only = obj.prefilters["published_only"] + DataPivotQuery.objects.bulk_update(objs, ["published_only"]) + + +class Migration(migrations.Migration): + dependencies = [ + ("summary", "0044_visual_prefilters_json"), + ] + + operations = [ + migrations.RunPython( + published_only_prefilters, reverse_code=reverse_published_only_prefilters + ), + migrations.RemoveField( + model_name="datapivotquery", + name="published_only", + ), + ] diff --git a/hawc/apps/summary/migrations/0045_newer.py b/hawc/apps/summary/migrations/0045_newer.py deleted file mode 100644 index debb367260..0000000000 --- a/hawc/apps/summary/migrations/0045_newer.py +++ /dev/null @@ -1,47 +0,0 @@ -# Generated by Django 4.2.3 on 2023-08-21 03:49 -import json - -from django.db import migrations, models - - -def prefilters_dict(apps, schema_editor): - # load prefilters textfield into temp jsonfield - Visual = apps.get_model("summary", "Visual") - objs = Visual.objects.all() - for obj in objs: - obj.temp = json.loads(obj.prefilters) - Visual.objects.bulk_update(objs, ["temp"]) - - -def reverse_prefilters_dict(apps, schema_editor): - # dump temp jsonfield into prefilters textfield - Visual = apps.get_model("summary", "Visual") - objs = Visual.objects.all() - for obj in objs: - obj.prefilters = json.dumps(obj.temp) - Visual.objects.bulk_update(objs, ["prefilters"]) - - -class Migration(migrations.Migration): - dependencies = [ - ("summary", "0044_newest"), - ] - - operations = [ - # change prefilters textfield into jsonfield - migrations.AddField( - model_name="visual", - name="temp", - field=models.JSONField(default=dict), - ), - migrations.RunPython(prefilters_dict, reverse_code=reverse_prefilters_dict), - migrations.RemoveField( - model_name="visual", - name="prefilters", - ), - migrations.RenameField( - model_name="visual", - old_name="temp", - new_name="prefilters", - ), - ] diff --git a/hawc/apps/summary/migrations/0046_visual_studies_to_prefilters.py b/hawc/apps/summary/migrations/0046_visual_studies_to_prefilters.py new file mode 100644 index 0000000000..ed0e18bfb9 --- /dev/null +++ b/hawc/apps/summary/migrations/0046_visual_studies_to_prefilters.py @@ -0,0 +1,33 @@ +# Generated by Django 4.2.3 on 2023-08-22 06:47 + +from django.db import migrations + + +def studies_prefilters(apps, schema_editor): + # add studies field to prefilters + Visual = apps.get_model("summary", "Visual") + objs = Visual.objects.all().prefetch_related("studies") + for obj in objs: + obj.prefilters["studies"] = list(obj.studies.all().values_list("pk", flat=True)) + Visual.objects.bulk_update(objs, ["prefilters"]) + + +def reverse_studies_prefilters(apps, schema_editor): + # separate studies field from prefilters + # TODO easiest way is probably to get the through table + # and create records there using pks + return + + +class Migration(migrations.Migration): + dependencies = [ + ("summary", "0045_datapivotquery_published_only_to_prefilters"), + ] + + operations = [ + migrations.RunPython(studies_prefilters, reverse_code=reverse_studies_prefilters), + migrations.RemoveField( + model_name="visual", + name="studies", + ), + ] diff --git a/hawc/apps/summary/models.py b/hawc/apps/summary/models.py index 06241b1902..0dea20b1a8 100644 --- a/hawc/apps/summary/models.py +++ b/hawc/apps/summary/models.py @@ -291,12 +291,6 @@ class Visual(models.Model): help_text="Endpoints to be included in visualization", blank=True, ) - studies = models.ManyToManyField( - Study, - related_name="visuals", - help_text="Studies to be included in visualization", - blank=True, - ) settings = models.TextField(default="{}") caption = models.TextField(blank=True) published = models.BooleanField( @@ -461,9 +455,6 @@ def get_filterset(self, data, assessment, **kwargs): return self.get_filterset_class()(data=data, assessment=assessment, **kwargs) def get_request_prefilters(self, request): - # TODO move get_editing_dataset out of models - # so that we can utilize the forms - # find all keys that start with "prefilters-" prefix prefix = "prefilters-" return { @@ -474,7 +465,7 @@ def get_request_prefilters(self, request): def get_endpoints(self, request=None): qs = Endpoint.objects.none() - filters = {"assessment_id": self.assessment_id} + filters = {} if self.visual_type == constants.VisualType.BIOASSAY_AGGREGATION: if request: @@ -504,7 +495,7 @@ def get_studies(self, request=None): to the model. """ qs = Study.objects.none() - filters = {"assessment_id": self.assessment_id} + filters = {} if self.visual_type in [ constants.VisualType.ROB_HEATMAP, @@ -516,8 +507,8 @@ def get_studies(self, request=None): cleaned_prefilters = fs.form.cleaned_data study_fields = [ - "animal_group__experiment__study__published", - "animal_group__experiment__study__in", + "published_only", + "studies", ] endpoint_prefilters = { k: v for k, v in cleaned_prefilters.items() if k not in study_fields @@ -778,27 +769,12 @@ def clean(self): raise ValidationError(err) def _refine_queryset(self, qs): - epi_version = self.assessment.epi_version - if self.evidence_type == constants.StudyType.BIOASSAY: - qs = qs.filter(assessment_id=self.assessment_id) if self.preferred_units: qs = qs.filter( animal_group__dosing_regime__doses__dose_units__in=self.preferred_units ) - elif self.evidence_type == constants.StudyType.EPI and epi_version == EpiVersion.V1: - qs = qs.filter(assessment_id=self.assessment_id) - - elif self.evidence_type == constants.StudyType.EPI and epi_version == EpiVersion.V2: - qs = qs.filter(design__study__assessment_id=self.assessment_id) - - elif self.evidence_type == constants.StudyType.EPI_META: - qs = qs.filter(protocol__study__assessment_id=self.assessment_id) - - elif self.evidence_type == constants.StudyType.IN_VITRO: - qs = qs.filter(assessment_id=self.assessment_id) - elif self.evidence_type == constants.StudyType.ECO: qs = qs.filter(design__study__assessment_id=self.assessment_id) diff --git a/hawc/apps/summary/prefilters.py b/hawc/apps/summary/prefilters.py index bca293eb0c..f77b6270f6 100644 --- a/hawc/apps/summary/prefilters.py +++ b/hawc/apps/summary/prefilters.py @@ -26,53 +26,47 @@ def helper(self): return helper -def filter_published_only(queryset, name, value): - if not value: - return queryset - return queryset.filter(**{name: True}) - - class BioassayPrefilter(BaseFilterSet): # studies - animal_group__experiment__study__published = df.BooleanFilter( - method=filter_published_only, + published_only = df.BooleanFilter( + method="filter_published_only", widget=CheckboxInput(), label="Published studies only", help_text="Only present data from studies which have been marked as " '"published" in HAWC.', ) - animal_group__experiment__study__in = df.MultipleChoiceFilter( + studies = df.MultipleChoiceFilter( field_name="animal_group__experiment__study", label="Studies to include", help_text="""Select one or more studies to include in the plot. If no study is selected, no endpoints will be available.""", ) # bioassay - system__in = df.MultipleChoiceFilter( + systems = df.MultipleChoiceFilter( field_name="system", label="Systems to include", help_text="""Select one or more systems to include in the plot. If no system is selected, no endpoints will be available.""", ) - organ__in = df.MultipleChoiceFilter( + organs = df.MultipleChoiceFilter( field_name="organ", label="Organs to include", help_text="""Select one or more organs to include in the plot. If no organ is selected, no endpoints will be available.""", ) - effect__in = df.MultipleChoiceFilter( + effects = df.MultipleChoiceFilter( field_name="effect", label="Effects to include", help_text="""Select one or more effects to include in the plot. If no effect is selected, no endpoints will be available.""", ) - effect_subtype__in = df.MultipleChoiceFilter( + effect_subtypes = df.MultipleChoiceFilter( field_name="effect_subtype", label="Effect Sub-Types to include", help_text="""Select one or more effect sub-types to include in the plot. If no effect sub-type is selected, no endpoints will be available.""", ) - effects__in = df.MultipleChoiceFilter( + effect_tags = df.MultipleChoiceFilter( field_name="effects", label="Tags to include", help_text="""Select one or more effect-tags to include in the plot. @@ -82,60 +76,67 @@ class BioassayPrefilter(BaseFilterSet): class Meta: model = Endpoint fields = [ - "animal_group__experiment__study__published", - "animal_group__experiment__study__in", - "system__in", - "organ__in", - "effect__in", - "effect_subtype__in", - "effects__in", + "published_only", + "studies", + "systems", + "organs", + "effects", + "effect_subtypes", + "effect_tags", ] form = TestForm + def filter_published_only(self, queryset, name, value): + if not value: + return queryset + return queryset.filter(animal_group__experiment__study__published=True) + + def filter_queryset(self, queryset): + queryset = queryset.filter(assessment_id=self.assessment.pk) + return super().filter_queryset(queryset) + def create_form(self): form = super().create_form() - form.fields["animal_group__experiment__study__in"].choices = Study.objects.get_choices( + form.fields["studies"].choices = Study.objects.get_choices(self.assessment.pk) + form.fields["systems"].choices = Endpoint.objects.get_system_choices(self.assessment.pk) + form.fields["organs"].choices = Endpoint.objects.get_organ_choices(self.assessment.pk) + form.fields["effects"].choices = Endpoint.objects.get_effect_choices(self.assessment.pk) + form.fields["effect_subtypes"].choices = Endpoint.objects.get_effect_subtype_choices( self.assessment.pk ) - form.fields["system__in"].choices = Endpoint.objects.get_system_choices(self.assessment.pk) - form.fields["organ__in"].choices = Endpoint.objects.get_organ_choices(self.assessment.pk) - form.fields["effect__in"].choices = Endpoint.objects.get_effect_choices(self.assessment.pk) - form.fields["effect_subtype__in"].choices = Endpoint.objects.get_effect_subtype_choices( - self.assessment.pk - ) - form.fields["effects__in"].choices = EffectTag.objects.get_choices(self.assessment.pk) + form.fields["effect_tags"].choices = EffectTag.objects.get_choices(self.assessment.pk) return form class EpiV1Prefilter(BaseFilterSet): # studies - study_population__study__published = df.BooleanFilter( - method=filter_published_only, + published_only = df.BooleanFilter( + method="filter_published_only", widget=CheckboxInput(), label="Published studies only", help_text="Only present data from studies which have been marked as " '"published" in HAWC.', ) - study_population__study__in = df.MultipleChoiceFilter( + studies = df.MultipleChoiceFilter( field_name="study_population__study", label="Studies to include", help_text="""Select one or more studies to include in the plot. If no study is selected, no endpoints will be available.""", ) # epi - system__in = df.MultipleChoiceFilter( + systems = df.MultipleChoiceFilter( field_name="system", label="Systems to include", help_text="""Select one or more systems to include in the plot. If no system is selected, no endpoints will be available.""", ) - effect__in = df.MultipleChoiceFilter( + effects = df.MultipleChoiceFilter( field_name="effect", label="Effects to include", help_text="""Select one or more effects to include in the plot. If no effect is selected, no endpoints will be available.""", ) - effects__in = df.MultipleChoiceFilter( + effect_tags = df.MultipleChoiceFilter( field_name="effects", label="Tags to include", help_text="""Select one or more effect-tags to include in the plot. @@ -145,35 +146,42 @@ class EpiV1Prefilter(BaseFilterSet): class Meta: model = Outcome fields = [ - "study_population__study__published", - "study_population__study__in", - "system__in", - "effect__in", - "effects__in", + "published_only", + "studies", + "systems", + "effects", + "effect_tags", ] form = TestForm + def filter_published_only(self, queryset, name, value): + if not value: + return queryset + return queryset.filter(study_population__study__published=True) + + def filter_queryset(self, queryset): + queryset = queryset.filter(assessment_id=self.assessment.pk) + return super().filter_queryset(queryset) + def create_form(self): form = super().create_form() - form.fields["study_population__study__in"].choices = Study.objects.get_choices( - self.assessment.pk - ) - form.fields["system__in"].choices = Outcome.objects.get_system_choices(self.assessment.pk) - form.fields["effect__in"].choices = Outcome.objects.get_effect_choices(self.assessment.pk) - form.fields["effects__in"].choices = EffectTag.objects.get_choices(self.assessment.pk) + form.fields["studies"].choices = Study.objects.get_choices(self.assessment.pk) + form.fields["systems"].choices = Outcome.objects.get_system_choices(self.assessment.pk) + form.fields["effects"].choices = Outcome.objects.get_effect_choices(self.assessment.pk) + form.fields["effect_tags"].choices = EffectTag.objects.get_choices(self.assessment.pk) return form class EpiV2Prefilter(BaseFilterSet): # studies - design__study__published = df.BooleanFilter( - method=filter_published_only, + published_only = df.BooleanFilter( + method="filter_published_only", widget=CheckboxInput(), label="Published studies only", help_text="Only present data from studies which have been marked as " '"published" in HAWC.', ) - design__study__in = df.MultipleChoiceFilter( + studies = df.MultipleChoiceFilter( field_name="design__study", label="Studies to include", help_text="""Select one or more studies to include in the plot. @@ -183,27 +191,36 @@ class EpiV2Prefilter(BaseFilterSet): class Meta: model = DataExtraction fields = [ - "design__study__published", - "design__study__in", + "published_only", + "studies", ] form = TestForm + def filter_published_only(self, queryset, name, value): + if not value: + return queryset + return queryset.filter(design__study__published=True) + + def filter_queryset(self, queryset): + queryset = queryset.filter(design__study__assessment_id=self.assessment.pk) + return super().filter_queryset(queryset) + def create_form(self): form = super().create_form() - form.fields["design__study__in"].choices = Study.objects.get_choices(self.assessment.pk) + form.fields["studies"].choices = Study.objects.get_choices(self.assessment.pk) return form class EpiMetaPrefilter(BaseFilterSet): # studies - protocol__study__published = df.BooleanFilter( - method=filter_published_only, + published_only = df.BooleanFilter( + method="filter_published_only", widget=CheckboxInput(), label="Published studies only", help_text="Only present data from studies which have been marked as " '"published" in HAWC.', ) - protocol__study__in = df.MultipleChoiceFilter( + studies = df.MultipleChoiceFilter( field_name="protocol__study", label="Studies to include", help_text="""Select one or more studies to include in the plot. @@ -213,46 +230,55 @@ class EpiMetaPrefilter(BaseFilterSet): class Meta: model = MetaResult fields = [ - "protocol__study__published", - "protocol__study__in", + "published_only", + "studies", ] form = TestForm + def filter_published_only(self, queryset, name, value): + if not value: + return queryset + return queryset.filter(protocol__study__published=True) + + def filter_queryset(self, queryset): + queryset = queryset.filter(protocol__study__assessment_id=self.assessment.pk) + return super().filter_queryset(queryset) + def create_form(self): form = super().create_form() - form.fields["protocol__study__in"].choices = Study.objects.get_choices(self.assessment.pk) + form.fields["studies"].choices = Study.objects.get_choices(self.assessment.pk) return form class InvitroPrefilter(BaseFilterSet): # studies - experiment__study__published = df.BooleanFilter( - method=filter_published_only, + published_only = df.BooleanFilter( + method="filter_published_only", widget=CheckboxInput(), label="Published studies only", help_text="Only present data from studies which have been marked as " '"published" in HAWC.', ) - experiment__study__in = df.MultipleChoiceFilter( + studies = df.MultipleChoiceFilter( field_name="experiment__study", label="Studies to include", help_text="""Select one or more studies to include in the plot. If no study is selected, no endpoints will be available.""", ) # invitro - category__in = df.MultipleChoiceFilter( + categories = df.MultipleChoiceFilter( field_name="category", label="Categories to include", help_text="""Select one or more categories to include in the plot. If no study is selected, no endpoints will be available.""", ) - chemical__name__in = df.MultipleChoiceFilter( + chemicals = df.MultipleChoiceFilter( field_name="chemical__name", label="Chemicals to include", help_text="""Select one or more chemicals to include in the plot. If no study is selected, no endpoints will be available.""", ) - effects__in = df.MultipleChoiceFilter( + effect_tags = df.MultipleChoiceFilter( field_name="effects", label="Tags to include", help_text="""Select one or more effect-tags to include in the plot. @@ -262,22 +288,29 @@ class InvitroPrefilter(BaseFilterSet): class Meta: model = IVEndpoint fields = [ - "experiment__study__published", - "experiment__study__in", - "category__in", - "chemical__name__in", - "effects__in", + "published_only", + "studies", + "categories", + "chemicals", + "effect_tags", ] form = TestForm + def filter_published_only(self, queryset, name, value): + if not value: + return queryset + return queryset.filter(experiment__study__published=True) + + def filter_queryset(self, queryset): + queryset = queryset.filter(assessment_id=self.assessment.pk) + return super().filter_queryset(queryset) + def create_form(self): form = super().create_form() - form.fields["experiment__study__in"].choices = Study.objects.get_choices(self.assessment.pk) - form.fields["category__in"].choices = IVEndpointCategory.get_choices(self.assessment.pk) - form.fields["chemical__name__in"].choices = IVChemical.objects.get_choices( - self.assessment.pk - ) - form.fields["effects__in"].choices = EffectTag.objects.get_choices(self.assessment.pk) + form.fields["studies"].choices = Study.objects.get_choices(self.assessment.pk) + form.fields["categories"].choices = IVEndpointCategory.get_choices(self.assessment.pk) + form.fields["chemicals"].choices = IVChemical.objects.get_choices(self.assessment.pk) + form.fields["effect_tags"].choices = EffectTag.objects.get_choices(self.assessment.pk) return form diff --git a/hawc/apps/summary/serializers.py b/hawc/apps/summary/serializers.py index aaab5dad88..a30b937a6e 100644 --- a/hawc/apps/summary/serializers.py +++ b/hawc/apps/summary/serializers.py @@ -48,10 +48,7 @@ def to_representation(self, instance): class Meta: model = models.Visual - exclude = ( - "endpoints", - "studies", - ) + exclude = ("endpoints",) class VisualSerializer(CollectionVisualSerializer): diff --git a/hawc/apps/summary/templates/summary/datapivot_detail.html b/hawc/apps/summary/templates/summary/datapivot_detail.html index 9e2b8493b1..3c47cab50e 100644 --- a/hawc/apps/summary/templates/summary/datapivot_detail.html +++ b/hawc/apps/summary/templates/summary/datapivot_detail.html @@ -10,7 +10,7 @@ {% if obj_perms.edit %}
Edit display settings - Edit other settings + Edit other settings Delete Pivot {% endif %} From 228d8766c96ca30f110cf82c2e00bc59b6bd6525 Mon Sep 17 00:00:00 2001 From: Daniel Rabstejnek Date: Wed, 23 Aug 2023 12:38:25 -0400 Subject: [PATCH 06/35] Fix tests --- .../data/api/api-summary-table-set-data.json | 2 +- tests/data/api/api-visual-barchart.json | 6 +- .../api/api-visual-bioassay-aggregation.json | 2 +- tests/data/api/api-visual-crossview.json | 4 +- .../data/api/api-visual-embedded-tableau.json | 2 +- .../api/api-visual-exploratory-heatmap.json | 2 +- tests/data/api/api-visual-rob-heatmap.json | 6 +- tests/data/api/api-visual-tagtree.json | 2 +- tests/data/fixtures/db.yaml | 73 +++++++++---------- 9 files changed, 52 insertions(+), 47 deletions(-) diff --git a/tests/data/api/api-summary-table-set-data.json b/tests/data/api/api-summary-table-set-data.json index 56fbf4a61c..bdc64ee884 100644 --- a/tests/data/api/api-summary-table-set-data.json +++ b/tests/data/api/api-summary-table-set-data.json @@ -111,4 +111,4 @@ "study_id": 1 } ] -} +} \ No newline at end of file diff --git a/tests/data/api/api-visual-barchart.json b/tests/data/api/api-visual-barchart.json index c6eec86537..9a14d8cd09 100644 --- a/tests/data/api/api-visual-barchart.json +++ b/tests/data/api/api-visual-barchart.json @@ -8,7 +8,11 @@ "endpoints": [], "id": 5, "last_updated": "2020-05-08T15:38:48.445082-04:00", - "prefilters": "{}", + "prefilters": { + "studies": [ + 7 + ] + }, "published": true, "rob_settings": { "assessment_id": 2, diff --git a/tests/data/api/api-visual-bioassay-aggregation.json b/tests/data/api/api-visual-bioassay-aggregation.json index e0b42a7608..d3bafa528f 100644 --- a/tests/data/api/api-visual-bioassay-aggregation.json +++ b/tests/data/api/api-visual-bioassay-aggregation.json @@ -361,7 +361,7 @@ ], "id": 10, "last_updated": "2020-11-25T13:35:00.094667-05:00", - "prefilters": "{}", + "prefilters": {}, "published": true, "settings": {}, "slug": "bioassay-aggregation", diff --git a/tests/data/api/api-visual-crossview.json b/tests/data/api/api-visual-crossview.json index 7b81e50dc3..0b90f74615 100644 --- a/tests/data/api/api-visual-crossview.json +++ b/tests/data/api/api-visual-crossview.json @@ -3246,7 +3246,9 @@ ], "id": 4, "last_updated": "2020-05-08T15:37:19.255730-04:00", - "prefilters": "{\"animal_group__experiment__study__published\": true}", + "prefilters": { + "published_only": true + }, "published": true, "settings": { "colorBase": "#cccccc", diff --git a/tests/data/api/api-visual-embedded-tableau.json b/tests/data/api/api-visual-embedded-tableau.json index bbcd8bee02..127de60e75 100644 --- a/tests/data/api/api-visual-embedded-tableau.json +++ b/tests/data/api/api-visual-embedded-tableau.json @@ -8,7 +8,7 @@ "endpoints": [], "id": 7, "last_updated": "2020-05-08T15:45:29.985848-04:00", - "prefilters": "{}", + "prefilters": {}, "published": true, "settings": { "external_url": "https://public.tableau.com/views/Iris_15675445278420/Iris-Actual", diff --git a/tests/data/api/api-visual-exploratory-heatmap.json b/tests/data/api/api-visual-exploratory-heatmap.json index 6223c23d67..b81e90c4e6 100644 --- a/tests/data/api/api-visual-exploratory-heatmap.json +++ b/tests/data/api/api-visual-exploratory-heatmap.json @@ -8,7 +8,7 @@ "endpoints": [], "id": 8, "last_updated": "2020-11-25T09:29:33.991892-05:00", - "prefilters": "{}", + "prefilters": {}, "published": true, "settings": { "autorotate_tick_labels": true, diff --git a/tests/data/api/api-visual-rob-heatmap.json b/tests/data/api/api-visual-rob-heatmap.json index 6c42b509d8..dba2236c57 100644 --- a/tests/data/api/api-visual-rob-heatmap.json +++ b/tests/data/api/api-visual-rob-heatmap.json @@ -8,7 +8,11 @@ "endpoints": [], "id": 3, "last_updated": "2020-05-08T15:36:41.324440-04:00", - "prefilters": "{}", + "prefilters": { + "studies": [ + 7 + ] + }, "published": true, "rob_settings": { "assessment_id": 2, diff --git a/tests/data/api/api-visual-tagtree.json b/tests/data/api/api-visual-tagtree.json index 6d8d8794eb..43819225f7 100644 --- a/tests/data/api/api-visual-tagtree.json +++ b/tests/data/api/api-visual-tagtree.json @@ -8,7 +8,7 @@ "endpoints": [], "id": 6, "last_updated": "2020-05-08T15:43:09.448621-04:00", - "prefilters": "{}", + "prefilters": {}, "published": true, "settings": { "height": 500, diff --git a/tests/data/fixtures/db.yaml b/tests/data/fixtures/db.yaml index 2acad68dbc..2e8e1d79c0 100644 --- a/tests/data/fixtures/db.yaml +++ b/tests/data/fixtures/db.yaml @@ -9284,7 +9284,9 @@ assessment: 1 visual_type: 2 dose_units: null - prefilters: '{}' + prefilters: + studies: + - 1 settings: '{"title":"","xAxisLabel":"","yAxisLabel":"","padding_top":20,"cell_size":40,"padding_right":400,"padding_bottom":35,"padding_left":20,"x_field":"study","study_label_field":"short_citation","included_metrics":[1,2],"excluded_score_ids":[],"show_legend":true,"show_na_legend":true,"show_nr_legend":true,"legend_x":239,"legend_y":17}' caption: '' published: false @@ -9292,8 +9294,6 @@ created: 2020-02-27 20:32:30.901597+00:00 last_updated: 2020-02-27 20:33:06.585979+00:00 endpoints: [] - studies: - - 1 - model: summary.visual pk: 2 fields: @@ -9302,7 +9302,9 @@ assessment: 1 visual_type: 3 dose_units: null - prefilters: '{}' + prefilters: + studies: + - 1 settings: '{"title":"Title","xAxisLabel":"Percent of studies","yAxisLabel":"","plot_width":400,"row_height":30,"padding_top":40,"padding_right":400,"padding_bottom":40,"padding_left":70,"show_values":true,"included_metrics":[1,2],"show_legend":true,"show_na_legend":true,"legend_x":640,"legend_y":10}' caption: '' published: false @@ -9310,8 +9312,6 @@ created: 2020-02-27 20:34:17.606164+00:00 last_updated: 2020-02-27 20:34:17.606186+00:00 endpoints: [] - studies: - - 1 - model: summary.visual pk: 3 fields: @@ -9320,7 +9320,9 @@ assessment: 2 visual_type: 2 dose_units: null - prefilters: '{}' + prefilters: + studies: + - 7 settings: '{"title":"","xAxisLabel":"","yAxisLabel":"","padding_top":20,"cell_size":40,"padding_right":300,"padding_bottom":35,"padding_left":20,"x_field":"study","study_label_field":"short_citation","included_metrics":[14,15],"excluded_score_ids":[],"show_legend":true,"show_na_legend":true,"show_nr_legend":true,"legend_x":231,"legend_y":30}' caption:

caption

published: true @@ -9328,8 +9330,6 @@ created: 2020-05-08 19:35:41.634588+00:00 last_updated: 2020-05-08 19:36:41.324440+00:00 endpoints: [] - studies: - - 7 - model: summary.visual pk: 4 fields: @@ -9338,7 +9338,8 @@ assessment: 2 visual_type: 1 dose_units: 1 - prefilters: '{"animal_group__experiment__study__published": true}' + prefilters: + published_only: true settings: '{"title":"Title","xAxisLabel":"Dose ()","yAxisLabel":"% change from control (continuous), % incidence (dichotomous)","width":1100,"height":600,"inner_width":940,"inner_height":520,"padding_left":75,"padding_top":45,"dose_isLog":true,"dose_range":"","response_range":"","title_x":0,"title_y":0,"xlabel_x":0,"xlabel_y":0,"ylabel_x":0,"ylabel_y":0,"filters":[{"name":"study","headerName":"Study","allValues":true,"values":null,"columns":1,"x":null,"y":null}],"reflines_dose":[{"value":null,"title":"","style":"base"}],"refranges_dose":[{"lower":null,"upper":null,"title":"","style":"base"}],"reflines_response":[{"value":null,"title":"","style":"base"}],"refranges_response":[{"lower":null,"upper":null,"title":"","style":"base"}],"labels":[{"caption":"","style":"base","max_width":null,"x":null,"y":null}],"colorBase":"#cccccc","colorHover":"#ff4040","colorSelected":"#6495ed","colorFilters":[],"colorFilterLegend":true,"colorFilterLegendLabel":"Color filters","colorFilterLegendX":0,"colorFilterLegendY":0,"endpointFilters":[],"endpointFilterLogic":"and"}' @@ -9348,7 +9349,6 @@ created: 2020-05-08 19:37:19.255703+00:00 last_updated: 2020-05-08 19:37:19.255730+00:00 endpoints: [] - studies: [] - model: summary.visual pk: 5 fields: @@ -9357,7 +9357,9 @@ assessment: 2 visual_type: 3 dose_units: null - prefilters: '{}' + prefilters: + studies: + - 7 settings: '{"title":"Title","xAxisLabel":"Percent of studies","yAxisLabel":"","plot_width":400,"row_height":30,"padding_top":40,"padding_right":300,"padding_bottom":40,"padding_left":70,"show_values":true,"included_metrics":[14,15],"show_legend":true,"show_na_legend":true,"legend_x":574,"legend_y":10}' caption:

caption

published: true @@ -9365,8 +9367,6 @@ created: 2020-05-08 19:37:44.283008+00:00 last_updated: 2020-05-08 19:38:48.445082+00:00 endpoints: [] - studies: - - 7 - model: summary.visual pk: 6 fields: @@ -9375,7 +9375,7 @@ assessment: 2 visual_type: 4 dose_units: null - prefilters: '{}' + prefilters: {} settings: '{"root_node": 11, "required_tags": [], "pruned_tags": [], "hide_empty_tag_nodes": false, "height": 500, "width": 1280, "show_legend": true, "show_counts": true}' caption:

caption

published: true @@ -9383,7 +9383,6 @@ created: 2020-05-08 19:43:09.448597+00:00 last_updated: 2020-05-08 19:43:09.448621+00:00 endpoints: [] - studies: [] - model: summary.visual pk: 7 fields: @@ -9392,7 +9391,7 @@ assessment: 2 visual_type: 5 dose_units: null - prefilters: '{}' + prefilters: {} settings: '{"external_url": "https://public.tableau.com/views/Iris_15675445278420/Iris-Actual", "external_url_hostname": "https://public.tableau.com", "external_url_path": "/views/Iris_15675445278420/Iris-Actual", "external_url_query_args": [":showVizHome=no", @@ -9403,7 +9402,6 @@ created: 2020-05-08 19:45:29.985823+00:00 last_updated: 2020-05-08 19:45:29.985848+00:00 endpoints: [] - studies: [] - model: summary.visual pk: 8 fields: @@ -9412,7 +9410,7 @@ assessment: 2 visual_type: 6 dose_units: null - prefilters: '{}' + prefilters: {} settings: '{"cell_height": 50, "cell_width": 50, "color_range": ["#ffffff", "#cc4700"], "compress_x": true, "compress_y": true, "data_url": "/ani/api/assessment/2/endpoint-heatmap/", "hawc_interactivity": true, "filter_widgets": [{"column": "species", "delimiter": "", "on_click_event": "---", "header": ""}, {"column": "strain", "delimiter": "", "on_click_event": "---", "header": ""}], "padding": {"top": 30, "left": 30, "bottom": 30, "right": 30}, "show_axis_border": true, "show_grid": true, "show_tooltip": @@ -9429,7 +9427,6 @@ created: 2020-11-25 14:25:27.528283+00:00 last_updated: 2020-11-25 14:29:33.991892+00:00 endpoints: [] - studies: [] - model: summary.visual pk: 9 fields: @@ -9438,7 +9435,7 @@ assessment: 2 visual_type: 7 dose_units: null - prefilters: '{}' + prefilters: {} settings: '{"data": [{"orientation": "h", "x": [1, 2, 3], "xaxis": "x", "y": [0, 1, 2], "yaxis": "y", "type": "bar"}], "layout": {"title":{"text":"test"}}}' caption: '' published: true @@ -9446,7 +9443,6 @@ created: 2023-05-19 14:25:27.528283+00:00 last_updated: 2023-05-19 14:25:27.528283+00:00 endpoints: [] - studies: [] - model: summary.visual pk: 10 fields: @@ -9455,7 +9451,7 @@ assessment: 2 visual_type: 0 dose_units: 1 - prefilters: '{}' + prefilters: {} settings: '{}' caption: '' published: true @@ -9464,7 +9460,6 @@ last_updated: 2020-11-25 18:35:00.094667+00:00 endpoints: - 3 - studies: [] - model: summary.visual pk: 11 fields: @@ -9473,7 +9468,9 @@ assessment: 2 visual_type: 2 dose_units: null - prefilters: '{}' + prefilters: + studies: + - 7 settings: '{"title":"","xAxisLabel":"","yAxisLabel":"","padding_top":20,"cell_size":40,"padding_right":190,"padding_bottom":35,"padding_left":20,"x_field":"study","study_label_field":"short_citation","included_metrics":[14,15],"excluded_score_ids":[],"show_legend":true,"show_na_legend":true,"show_nr_legend":true,"legend_x":226,"legend_y":16}' caption:

This is a study, an endpoint, @@ -9490,8 +9487,6 @@ created: 2020-11-26 03:21:03.883675+00:00 last_updated: 2020-11-26 03:25:56.660522+00:00 endpoints: [] - studies: - - 7 - model: summary.datapivot pk: 1 fields: @@ -10159,8 +10154,8 @@ evidence_type: 0 export_style: 1 preferred_units: '["1"]' - prefilters: '{}' - published_only: true + prefilters: + published_only: true - model: summary.datapivotquery pk: 1 fields: @@ -10168,8 +10163,8 @@ evidence_type: 0 export_style: 0 preferred_units: '["1"]' - prefilters: '{}' - published_only: true + prefilters: + published_only: true - model: summary.datapivotquery pk: 4 fields: @@ -10177,8 +10172,8 @@ evidence_type: 1 export_style: 0 preferred_units: '[]' - prefilters: '{}' - published_only: true + prefilters: + published_only: true - model: summary.datapivotquery pk: 3 fields: @@ -10186,8 +10181,8 @@ evidence_type: 4 export_style: 0 preferred_units: '[]' - prefilters: '{}' - published_only: true + prefilters: + published_only: true - model: summary.datapivotquery pk: 5 fields: @@ -10195,8 +10190,8 @@ evidence_type: 2 export_style: 1 preferred_units: '[]' - prefilters: '{}' - published_only: true + prefilters: + published_only: true - model: summary.datapivotquery pk: 6 fields: @@ -10204,8 +10199,8 @@ evidence_type: 2 export_style: 0 preferred_units: '[]' - prefilters: '{}' - published_only: true + prefilters: + published_only: true - model: mgmt.task pk: 1 fields: From ca90cf198062950777b9442774aef1b98feea451 Mon Sep 17 00:00:00 2001 From: munnsmunns Date: Thu, 24 Aug 2023 17:27:20 -0400 Subject: [PATCH 07/35] create initial form library classes --- hawc/apps/common/dynamic_forms/__init__.py | 0 hawc/apps/common/dynamic_forms/forms.py | 6 ++++++ hawc/apps/common/dynamic_forms/models.py | 9 +++++++++ hawc/apps/common/dynamic_forms/views.py | 7 +++++++ 4 files changed, 22 insertions(+) create mode 100644 hawc/apps/common/dynamic_forms/__init__.py create mode 100644 hawc/apps/common/dynamic_forms/forms.py create mode 100644 hawc/apps/common/dynamic_forms/models.py create mode 100644 hawc/apps/common/dynamic_forms/views.py diff --git a/hawc/apps/common/dynamic_forms/__init__.py b/hawc/apps/common/dynamic_forms/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hawc/apps/common/dynamic_forms/forms.py b/hawc/apps/common/dynamic_forms/forms.py new file mode 100644 index 0000000000..20489a4a63 --- /dev/null +++ b/hawc/apps/common/dynamic_forms/forms.py @@ -0,0 +1,6 @@ +from django import forms +from models import CustomDataExtraction + + +class CustomDataExtractionForm(forms.ModelForm): + model = CustomDataExtraction diff --git a/hawc/apps/common/dynamic_forms/models.py b/hawc/apps/common/dynamic_forms/models.py new file mode 100644 index 0000000000..8c4fe49200 --- /dev/null +++ b/hawc/apps/common/dynamic_forms/models.py @@ -0,0 +1,9 @@ +from django.db import models + + +# TODO: is this a good name? i changed it from dynamic form because it makes the naming scheme weird +# for the actual form class +class CustomDataExtraction(models.Model): + name = models.CharField() + description = models.TextField() + schema = models.JSONField() diff --git a/hawc/apps/common/dynamic_forms/views.py b/hawc/apps/common/dynamic_forms/views.py new file mode 100644 index 0000000000..1411d3a76d --- /dev/null +++ b/hawc/apps/common/dynamic_forms/views.py @@ -0,0 +1,7 @@ +from forms import CustomDataExtractionForm + +from hawc.apps.common import views + + +class CreateDataExtractionView(views.BaseCreate): + form_class = CustomDataExtractionForm From c2175de46fe1ba6487e617789859a3c594ae33b1 Mon Sep 17 00:00:00 2001 From: munnsmunns Date: Mon, 28 Aug 2023 16:34:12 -0400 Subject: [PATCH 08/35] migrate needed files and add new app --- hawc/apps/common/dynamic_forms/__init__.py | 5 + hawc/apps/common/dynamic_forms/constants.py | 71 +++++++++ hawc/apps/common/dynamic_forms/fields.py | 143 ++++++++++++++++++ hawc/apps/common/dynamic_forms/forms.py | 125 ++++++++++++++- hawc/apps/common/dynamic_forms/schemas.py | 105 +++++++++++++ hawc/apps/form_library/__init__.py | 0 hawc/apps/form_library/forms.py | 6 + .../dynamic_forms => form_library}/models.py | 2 +- .../dynamic_forms => form_library}/views.py | 0 9 files changed, 453 insertions(+), 4 deletions(-) create mode 100644 hawc/apps/common/dynamic_forms/constants.py create mode 100644 hawc/apps/common/dynamic_forms/fields.py create mode 100644 hawc/apps/common/dynamic_forms/schemas.py create mode 100644 hawc/apps/form_library/__init__.py create mode 100644 hawc/apps/form_library/forms.py rename hawc/apps/{common/dynamic_forms => form_library}/models.py (85%) rename hawc/apps/{common/dynamic_forms => form_library}/views.py (100%) diff --git a/hawc/apps/common/dynamic_forms/__init__.py b/hawc/apps/common/dynamic_forms/__init__.py index e69de29bb2..6d497a04ca 100644 --- a/hawc/apps/common/dynamic_forms/__init__.py +++ b/hawc/apps/common/dynamic_forms/__init__.py @@ -0,0 +1,5 @@ +from ..forms import DynamicFormField +from .forms import DynamicForm +from .schemas import Schema + +__all__ = ["DynamicForm", "DynamicFormField", "Schema"] diff --git a/hawc/apps/common/dynamic_forms/constants.py b/hawc/apps/common/dynamic_forms/constants.py new file mode 100644 index 0000000000..95c450471e --- /dev/null +++ b/hawc/apps/common/dynamic_forms/constants.py @@ -0,0 +1,71 @@ +"""Django form enums.""" +from enum import Enum + +from django import forms + +from ..forms import InlineRadioChoiceField + + +class FormField(Enum): + """Django form field enumeration. + + When removing access to a member, please + comment out instead of omitting. + """ + + BOOLEAN = forms.BooleanField + CHAR = forms.CharField + CHOICE = forms.ChoiceField + DATE = forms.DateField + DATE_TIME = forms.DateTimeField + DECIMAL = forms.DecimalField + DURATION = forms.DurationField + EMAIL = forms.EmailField + FILE = forms.FileField + FILEPATH = forms.FilePathField + FLOAT = forms.FloatField + GENERIC_IP_ADDRESS = forms.GenericIPAddressField + IMAGE = forms.ImageField + INTEGER = forms.IntegerField + JSON = forms.JSONField + MULTIPLE_CHOICE = forms.MultipleChoiceField + NULL_BOOLEAN = forms.NullBooleanField + REGEX = forms.RegexField + SLUG = forms.SlugField + TIME = forms.TimeField + TYPED_CHOICE = forms.TypedChoiceField + TYPED_MULTIPLE_CHOICE = forms.TypedMultipleChoiceField + URL = forms.URLField + UUID = forms.UUIDField + YES_NO = InlineRadioChoiceField + + +class Widget(Enum): + """Django widget enumeration. + + When removing access to a member, please + comment out instead of omitting. + """ + + TEXT_INPUT = forms.TextInput + NUMBER_INPUT = forms.NumberInput + EMAIL_INPUT = forms.EmailInput + URL_INPUT = forms.URLInput + PASSWORD_INPUT = forms.PasswordInput + HIDDEN_INPUT = forms.HiddenInput + DATE_INPUT = forms.DateInput + DATE_TIME_INPUT = forms.DateTimeInput + TIME_INPUT = forms.TimeInput + TEXTAREA = forms.Textarea + CHECKBOX_INPUT = forms.CheckboxInput + SELECT = forms.Select + NULL_BOOLEAN_SELECT = forms.NullBooleanSelect + SELECT_MULTIPLE = forms.SelectMultiple + RADIO_SELECT = forms.RadioSelect + CHECKBOX_SELECT_MULTIPLE = forms.CheckboxSelectMultiple + FILE_INPUT = forms.FileInput + CLEARABLE_FILE_INPUT = forms.ClearableFileInput + MULTIPLE_HIDDEN_INPUT = forms.MultipleHiddenInput + SPLIT_DATE_TIME = forms.SplitDateTimeWidget + SPLIT_HIDDEN_DATE_TIME = forms.SplitHiddenDateTimeWidget + SELECT_DATE = forms.SelectDateWidget diff --git a/hawc/apps/common/dynamic_forms/fields.py b/hawc/apps/common/dynamic_forms/fields.py new file mode 100644 index 0000000000..2855d5808b --- /dev/null +++ b/hawc/apps/common/dynamic_forms/fields.py @@ -0,0 +1,143 @@ +"""Dynamic Django form fields.""" +from typing import Annotated, Any, Literal + +from django import forms +from django.utils.html import conditional_escape +from pydantic import BaseModel, validator +from pydantic import Field as PydanticField + +from . import constants + + +class _Field(BaseModel): + """Base class for Django form field schemas. + + This class should only act as a superclass for each specific Django form field. + + Each subclass should add a 'type' property that corresponds with a key + in constants.FormField, and a 'widget' property that corresponds with keys + in constants.Widget. + """ + + name: str # the variable name in the form; extra validation for no whitespace etc? + required: bool | None + label: str | None + label_suffix: str | None + initial: Any = None + help_text: str | None + css_class: str | None + # error_messages + # validators + # localize + # disabled + + class Config: + """Schema config.""" + + underscore_attrs_are_private = True + use_enum_values = True + + @validator("help_text") + def ensure_safe_string(cls, v): + """Sanitize help_text values.""" + if v is None: + return v + return conditional_escape(v) + + def get_form_field_kwargs(self) -> dict: + """Get keyword arguments for Django form field.""" + kwargs = self.dict( + exclude={"type", "widget", "name", "css_class"}, exclude_none=True + ) + kwargs["widget"] = constants.Widget[self.widget.upper()].value + return kwargs + + def get_form_field(self) -> forms.Field: + """Get Django form field.""" + form_field_cls = constants.FormField[self.type.upper()].value + form_field_kwargs = self.get_form_field_kwargs() + return form_field_cls(**form_field_kwargs) + + def get_verbose_name(self) -> str: + """Get reader friendly name for schema.""" + return self.label or self.name.replace("_", " ").title() + + +class BooleanField(_Field): + """Boolean field.""" + + type: Literal["boolean"] = "boolean" # noqa: A003 + widget: Literal["checkbox_input"] = "checkbox_input" + + +class CharField(_Field): + """Character field.""" + + type: Literal["char"] = "char" # noqa: A003 + widget: Literal["text_input", "textarea"] = "text_input" + + max_length: int | None + min_length: int | None + strip: bool | None + empty_value: str | None + + +class IntegerField(_Field): + """Integer field.""" + + type: Literal["integer"] = "integer" # noqa: A003 + widget: Literal["number_input"] = "number_input" + + min_value: int | None + max_value: int | None + + +class FloatField(_Field): + """Float field.""" + + type: Literal["float"] = "float" # noqa: A003 + widget: Literal["number_input"] = "number_input" + + min_value: int | None + max_value: int | None + + +class ChoiceField(_Field): + """Choice field.""" + + type: Literal["choice"] = "choice" # noqa: A003 + widget: Literal["select", "radio_select"] = "select" + + choices: list[tuple[str, str]] + + +class YesNoChoiceField(_Field): + """Yes No field.""" + + type: Literal["yes_no"] = "yes_no" # noqa: A003 + widget: Literal["radio_select"] = PydanticField("radio_select", const=True) + + choices: list[tuple[str, str]] = PydanticField( + [("yes", "Yes"), ("no", "No")], const=True + ) + + +class MultipleChoiceField(_Field): + """Multiple choice field.""" + + type: Literal["multiple_choice"] = "multiple_choice" # noqa: A003 + widget: Literal["select_multiple", "checkbox_select_multiple"] = "select_multiple" + + choices: list[tuple[str, str]] + + +Field = Annotated[ + BooleanField + | CharField + | ChoiceField + | YesNoChoiceField + | FloatField + | IntegerField + | MultipleChoiceField, + PydanticField(discriminator="type"), +] diff --git a/hawc/apps/common/dynamic_forms/forms.py b/hawc/apps/common/dynamic_forms/forms.py index 20489a4a63..aea21744d1 100644 --- a/hawc/apps/common/dynamic_forms/forms.py +++ b/hawc/apps/common/dynamic_forms/forms.py @@ -1,6 +1,125 @@ +"""Dynamic Django forms.""" +import json + +from crispy_forms import layout as cfl from django import forms -from models import CustomDataExtraction +from django.core.exceptions import ValidationError + +from ..forms import BaseFormHelper +from .schemas import Behavior, Schema + + +class DynamicForm(forms.Form): + """Dynamic Django form. + + This is built from a custom schema that defines fields and conditional logic. + """ + + def __init__(self, schema: Schema, *args, **kwargs): + """Create dynamic form.""" + super().__init__(*args, **kwargs) + self.schema = schema + fields = {f.name: f.get_form_field() for f in self.schema.fields} + self.fields.update(fields) + + @property + def helper(self): + """Django crispy form helper.""" + helper = DynamicFormHelper(self) + + # wrap up field inputs with bootstrap grid + helper.auto_wrap_fields() + + # expose serialized conditions w/ unique id to template + helper.conditions = [ + { + "subject_id": self[condition.subject].auto_id, + "observer_ids": [self[observer].auto_id for observer in condition.observers], + "comparison": condition.comparison, + "comparison_value": condition.comparison_value, + "behavior": condition.behavior, + } + for condition in self.schema.conditions + ] + helper.conditions_id = f"conditions-{hash(json.dumps(helper.conditions))}" + helper.layout.append(cfl.HTML("{{ conditions|json_script:conditions_id }}")) + + return helper + + def full_clean(self): + """Overridden full_clean that handles conditional logic from schema.""" + # handle conditions + if self.is_bound: + fields = self.fields.copy() # copy to restore after clean + data = self.data.copy() # copy in case it's immutable + for condition in self.schema.conditions: + bf = self[condition.subject] + value = bf.field.to_python(bf.value()) + check = condition.comparison.compare(value, condition.comparison_value) + show = check if condition.behavior == Behavior.SHOW else not check + if not show: + for observer in condition.observers: + # remove data that should be hidden + data.pop(observer, None) + # remove fields that should be hidden; + # this is restored after clean + self.fields.pop(observer) + self.data = data + + # run default full_clean + super().full_clean() + + # restore fields + if self.is_bound: + self.fields = fields + + +class DynamicFormHelper(BaseFormHelper): + """Django crispy form helper.""" + + form_tag = False + + def auto_wrap_fields(self): + """Wrap fields in bootstrap classes.""" + if len(self.form.schema.fields) == 0: + return + + for field in self.form.schema.fields: + index = self.layout.index(field.name) + css_class = field.css_class or "col-12" + self[index].wrap(cfl.Column, css_class=css_class) + + self[:].wrap_together(cfl.Row, id="row_id_dynamic_form") + self.add_field_wraps() + + +class DynamicFormWidget(forms.Widget): + """Widget to display dynamic form inline.""" + + template_name = "common/widgets/dynamic_form.html" + + def __init__(self, prefix, form_class, form_kwargs=None, *args, **kwargs): + """Create dynamic form widget.""" + super().__init__(*args, **kwargs) + self.prefix = prefix + self.form_class = form_class + if form_kwargs is None: + form_kwargs = {} + self.form_kwargs = {"prefix": prefix, **form_kwargs} + + def add_prefix(self, field_name): + """Add prefix in the same way Django forms add prefixes.""" + return f"{self.prefix}-{field_name}" + def format_value(self, value): + """Value used in rendering.""" + value = json.loads(value) + if value: + value = {self.add_prefix(k): v for k, v in value.items()} + return self.form_class(data=value, **self.form_kwargs) -class CustomDataExtractionForm(forms.ModelForm): - model = CustomDataExtraction + def value_from_datadict(self, data, files, name): + """Parse value from POST request.""" + form = self.form_class(data=data, **self.form_kwargs) + form.full_clean() + return form.cleaned_data diff --git a/hawc/apps/common/dynamic_forms/schemas.py b/hawc/apps/common/dynamic_forms/schemas.py new file mode 100644 index 0000000000..4e857f2b8d --- /dev/null +++ b/hawc/apps/common/dynamic_forms/schemas.py @@ -0,0 +1,105 @@ +"""Schemas to build dynamic Django forms.""" +from enum import Enum + +from django.forms import HiddenInput, JSONField +from pydantic import BaseModel, conlist, root_validator, validator + +from . import fields, forms + + +class Comparison(str, Enum): + """Enum for comparisons.""" + + __slots__ = () + + EQUALS = "equals" + IN = "in" + CONTAINS = "contains" + + def _equals(self, x, y) -> bool: + # x equals y + return x == y + + def _in(self, x, y) -> bool: + # x (or a subset of x) is in y + return any(_ in y for _ in x) + + def _contains(self, x, y) -> bool: + # x contains y + return set(x) >= set(y) + + def compare(self, x, y) -> bool: + """Perform comparison based on enum value.""" + x = x if isinstance(x, list) else [x] + y = y if isinstance(y, list) else [y] + return getattr(self, f"_{self.value.lower()}")(x, y) + + +class Behavior(str, Enum): + """Enum for form field behavior; behavior applies when condition is true.""" + + __slots__ = () + + SHOW = "show" + HIDE = "hide" + + +class Condition(BaseModel): + """Condition that affects the visibility of fields.""" + + subject: str + observers: list[str] + comparison: Comparison = Comparison.EQUALS + comparison_value: bool | str | int | conlist(bool | str | int, min_items=1) + behavior: Behavior = Behavior.SHOW + + class Config: + """Schema config.""" + + smart_union = True + + +class Schema(BaseModel): + """Schema for dynamic form.""" + + fields: list[fields.Field] + conditions: list[Condition] = [] + + @root_validator(skip_on_failure=True) + def validate_conditions(cls, values): + """Validate conditions.""" + # condition subjects and observers should be existing fields + fields = values["fields"] + field_names = {field.name for field in fields} + conditions = values["conditions"] + subjects = {condition.subject for condition in conditions} + observers = { + observer for condition in conditions for observer in condition.observers + } + if bad_subjects := (subjects - field_names): + raise ValueError(f"Invalid condition subject(s): {', '.join(bad_subjects)}") + if bad_observers := (observers - field_names): + raise ValueError( + f"Invalid condition observer(s): {', '.join(bad_observers)}" + ) + return values + + @validator("fields") + def unique_field_names(cls, v): + """Validate field names.""" + unique_names = {field.name for field in v} + if len(unique_names) != len(v): + raise ValueError("Duplicate field name(s)") + return v + + def to_form(self, *args, **kwargs): + """Get dynamic form for this schema.""" + return forms.DynamicForm(self, *args, **kwargs) + + def to_form_field(self, prefix, form_kwargs=None, *args, **kwargs): + """Get dynamic form field for this schema.""" + if len(self.fields) == 0: + return JSONField(widget=HiddenInput(), required=False) + return forms.DynamicFormField( + prefix, self.to_form, form_kwargs, *args, **kwargs + ) diff --git a/hawc/apps/form_library/__init__.py b/hawc/apps/form_library/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hawc/apps/form_library/forms.py b/hawc/apps/form_library/forms.py new file mode 100644 index 0000000000..20489a4a63 --- /dev/null +++ b/hawc/apps/form_library/forms.py @@ -0,0 +1,6 @@ +from django import forms +from models import CustomDataExtraction + + +class CustomDataExtractionForm(forms.ModelForm): + model = CustomDataExtraction diff --git a/hawc/apps/common/dynamic_forms/models.py b/hawc/apps/form_library/models.py similarity index 85% rename from hawc/apps/common/dynamic_forms/models.py rename to hawc/apps/form_library/models.py index 8c4fe49200..168162b114 100644 --- a/hawc/apps/common/dynamic_forms/models.py +++ b/hawc/apps/form_library/models.py @@ -2,7 +2,7 @@ # TODO: is this a good name? i changed it from dynamic form because it makes the naming scheme weird -# for the actual form class +# for the actual form class (DynamicFormForm) class CustomDataExtraction(models.Model): name = models.CharField() description = models.TextField() diff --git a/hawc/apps/common/dynamic_forms/views.py b/hawc/apps/form_library/views.py similarity index 100% rename from hawc/apps/common/dynamic_forms/views.py rename to hawc/apps/form_library/views.py From 14a4d6474c0f60351730b5dd2b9f0a696ff2675d Mon Sep 17 00:00:00 2001 From: munnsmunns Date: Mon, 28 Aug 2023 17:06:59 -0400 Subject: [PATCH 09/35] add to apps --- hawc/apps/form_library/apps.py | 7 +++++++ hawc/constants.py | 1 + hawc/main/settings/base.py | 4 ++++ 3 files changed, 12 insertions(+) create mode 100644 hawc/apps/form_library/apps.py diff --git a/hawc/apps/form_library/apps.py b/hawc/apps/form_library/apps.py new file mode 100644 index 0000000000..849142b68d --- /dev/null +++ b/hawc/apps/form_library/apps.py @@ -0,0 +1,7 @@ +from django.apps import AppConfig + + +class FormLibraryConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "hawc.apps.form_library" + verbose_name = "Form Library" diff --git a/hawc/constants.py b/hawc/constants.py index 7f573f9a0a..b897361026 100644 --- a/hawc/constants.py +++ b/hawc/constants.py @@ -17,6 +17,7 @@ class FeatureFlags(BaseModel): ANONYMOUS_ACCOUNT_CREATION: bool = True ENABLE_BMDS_33 = False ENABLE_PLOTLY_VISUAL: bool = False + ENABLE_DYNAMIC_FORMS: bool = False @classmethod def from_env(cls, variable) -> "FeatureFlags": diff --git a/hawc/main/settings/base.py b/hawc/main/settings/base.py index ad5487f49a..d353855b60 100644 --- a/hawc/main/settings/base.py +++ b/hawc/main/settings/base.py @@ -126,6 +126,10 @@ "hawc.apps.materialized", "hawc.apps.epiv2", ) + +if HAWC_FEATURES.ENABLE_DYNAMIC_FORMS: + INSTALLED_APPS = (*INSTALLED_APPS, "hawc.apps.form_library") + # DB settings DATABASES = { "default": { From ab0e7eb900585178dd50fc331bc1f42405405711 Mon Sep 17 00:00:00 2001 From: munnsmunns Date: Mon, 28 Aug 2023 17:07:46 -0400 Subject: [PATCH 10/35] add validation to form --- hawc/apps/common/forms.py | 14 ++++++++++++++ hawc/apps/form_library/forms.py | 12 +++++++++++- hawc/apps/form_library/models.py | 2 ++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/hawc/apps/common/forms.py b/hawc/apps/common/forms.py index c01e223dbd..1ea8cf291e 100644 --- a/hawc/apps/common/forms.py +++ b/hawc/apps/common/forms.py @@ -8,6 +8,7 @@ from django.template.loader import render_to_string from . import autocomplete, validators, widgets +from .helper import PydanticToDjangoError ASSESSMENT_UNIQUE_MESSAGE = "Must be unique for assessment (current value already exists)." @@ -479,3 +480,16 @@ def validate(self, value): form = self.form_class(data=value, **self.form_kwargs) if not form.is_valid(): raise forms.ValidationError(self.error_messages["invalid"]) + + +class PydanticValidator: + """JSON field validator that uses a pydantic model.""" + + def __init__(self, schema): + """Set the schema.""" + self.schema = schema + + def __call__(self, value): + """Validate the field with the pydantic model.""" + with PydanticToDjangoError(include_field=False): + self.schema.parse_obj(value) diff --git a/hawc/apps/form_library/forms.py b/hawc/apps/form_library/forms.py index 20489a4a63..e749527480 100644 --- a/hawc/apps/form_library/forms.py +++ b/hawc/apps/form_library/forms.py @@ -1,6 +1,16 @@ from django import forms from models import CustomDataExtraction +from hawc.apps.common.dynamic_forms.schemas import Schema +from hawc.apps.common.forms import PydanticValidator + class CustomDataExtractionForm(forms.ModelForm): - model = CustomDataExtraction + schema = forms.JSONField( + initial=Schema(fields=[]).dict(), + validators=[PydanticValidator(Schema)], + ) + + class Meta: + model = CustomDataExtraction + exclude = ("created", "last_updated") diff --git a/hawc/apps/form_library/models.py b/hawc/apps/form_library/models.py index 168162b114..4ec84b1cd8 100644 --- a/hawc/apps/form_library/models.py +++ b/hawc/apps/form_library/models.py @@ -7,3 +7,5 @@ class CustomDataExtraction(models.Model): name = models.CharField() description = models.TextField() schema = models.JSONField() + created = models.DateTimeField(auto_now_add=True) + last_updated = models.DateTimeField(auto_now=True) From 66e00311ee6f68c6cbb9da88820ae4bcf13d0426 Mon Sep 17 00:00:00 2001 From: munnsmunns Date: Tue, 29 Aug 2023 12:56:07 -0400 Subject: [PATCH 11/35] add migration and admin site --- hawc/apps/form_library/admin.py | 11 ++++++++ .../form_library/migrations/0001_initial.py | 28 +++++++++++++++++++ hawc/apps/form_library/migrations/__init__.py | 0 hawc/apps/form_library/models.py | 4 +++ hawc/main/settings/base.py | 2 +- 5 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 hawc/apps/form_library/admin.py create mode 100644 hawc/apps/form_library/migrations/0001_initial.py create mode 100644 hawc/apps/form_library/migrations/__init__.py diff --git a/hawc/apps/form_library/admin.py b/hawc/apps/form_library/admin.py new file mode 100644 index 0000000000..1fbbf59c01 --- /dev/null +++ b/hawc/apps/form_library/admin.py @@ -0,0 +1,11 @@ +from django.contrib import admin + +from . import models + + +class CustomDataExtractionInline(admin.TabularInline): + model = models.CustomDataExtraction + extra = 0 + + +admin.site.register(models.CustomDataExtraction) diff --git a/hawc/apps/form_library/migrations/0001_initial.py b/hawc/apps/form_library/migrations/0001_initial.py new file mode 100644 index 0000000000..0f2cbc7323 --- /dev/null +++ b/hawc/apps/form_library/migrations/0001_initial.py @@ -0,0 +1,28 @@ +# Generated by Django 4.2.4 on 2023-08-29 16:17 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="CustomDataExtraction", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("name", models.CharField()), + ("description", models.TextField()), + ("schema", models.JSONField()), + ("created", models.DateTimeField(auto_now_add=True)), + ("last_updated", models.DateTimeField(auto_now=True)), + ], + ), + ] diff --git a/hawc/apps/form_library/migrations/__init__.py b/hawc/apps/form_library/migrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hawc/apps/form_library/models.py b/hawc/apps/form_library/models.py index 4ec84b1cd8..7f2f57882e 100644 --- a/hawc/apps/form_library/models.py +++ b/hawc/apps/form_library/models.py @@ -1,3 +1,4 @@ +import reversion from django.db import models @@ -9,3 +10,6 @@ class CustomDataExtraction(models.Model): schema = models.JSONField() created = models.DateTimeField(auto_now_add=True) last_updated = models.DateTimeField(auto_now=True) + + +reversion.register(CustomDataExtraction) diff --git a/hawc/main/settings/base.py b/hawc/main/settings/base.py index d353855b60..ec96afe97a 100644 --- a/hawc/main/settings/base.py +++ b/hawc/main/settings/base.py @@ -128,7 +128,7 @@ ) if HAWC_FEATURES.ENABLE_DYNAMIC_FORMS: - INSTALLED_APPS = (*INSTALLED_APPS, "hawc.apps.form_library") + INSTALLED_APPS += ("hawc.apps.form_library",) # DB settings DATABASES = { From b4e1e37cd73c63db064a4b00ed3c947334eee7cc Mon Sep 17 00:00:00 2001 From: munnsmunns Date: Tue, 29 Aug 2023 14:43:28 -0400 Subject: [PATCH 12/35] add url and get form view running --- hawc/apps/common/dynamic_forms/forms.py | 1 - hawc/apps/common/forms.py | 8 ++++++++ hawc/apps/form_library/forms.py | 3 ++- .../form_library/custom_data_extraction_form.html | 7 +++++++ hawc/apps/form_library/urls.py | 13 +++++++++++++ hawc/apps/form_library/views.py | 7 ++++--- hawc/main/urls.py | 2 ++ 7 files changed, 36 insertions(+), 5 deletions(-) create mode 100644 hawc/apps/form_library/templates/form_library/custom_data_extraction_form.html create mode 100644 hawc/apps/form_library/urls.py diff --git a/hawc/apps/common/dynamic_forms/forms.py b/hawc/apps/common/dynamic_forms/forms.py index aea21744d1..2944dd96df 100644 --- a/hawc/apps/common/dynamic_forms/forms.py +++ b/hawc/apps/common/dynamic_forms/forms.py @@ -3,7 +3,6 @@ from crispy_forms import layout as cfl from django import forms -from django.core.exceptions import ValidationError from ..forms import BaseFormHelper from .schemas import Behavior, Schema diff --git a/hawc/apps/common/forms.py b/hawc/apps/common/forms.py index 1ea8cf291e..615d37adce 100644 --- a/hawc/apps/common/forms.py +++ b/hawc/apps/common/forms.py @@ -5,6 +5,7 @@ from crispy_forms import layout as cfl from crispy_forms.utils import TEMPLATE_PACK, flatatt from django import forms +from django.forms.widgets import RadioSelect from django.template.loader import render_to_string from . import autocomplete, validators, widgets @@ -482,6 +483,13 @@ def validate(self, value): raise forms.ValidationError(self.error_messages["invalid"]) +class InlineRadioChoiceField(forms.ChoiceField): + """Choice widget that uses radio buttons that are inline.""" + + widget = RadioSelect + crispy_field_class = cfb.InlineRadios + + class PydanticValidator: """JSON field validator that uses a pydantic model.""" diff --git a/hawc/apps/form_library/forms.py b/hawc/apps/form_library/forms.py index e749527480..c6ae58fe30 100644 --- a/hawc/apps/form_library/forms.py +++ b/hawc/apps/form_library/forms.py @@ -1,9 +1,10 @@ from django import forms -from models import CustomDataExtraction from hawc.apps.common.dynamic_forms.schemas import Schema from hawc.apps.common.forms import PydanticValidator +from .models import CustomDataExtraction + class CustomDataExtractionForm(forms.ModelForm): schema = forms.JSONField( diff --git a/hawc/apps/form_library/templates/form_library/custom_data_extraction_form.html b/hawc/apps/form_library/templates/form_library/custom_data_extraction_form.html new file mode 100644 index 0000000000..6dba6335c0 --- /dev/null +++ b/hawc/apps/form_library/templates/form_library/custom_data_extraction_form.html @@ -0,0 +1,7 @@ +{% extends 'assessment-rooted.html' %} + +{% load crispy_forms_tags %} + +{% block content %} +{% crispy form %} +{% endblock %} diff --git a/hawc/apps/form_library/urls.py b/hawc/apps/form_library/urls.py new file mode 100644 index 0000000000..4fa2d2b66f --- /dev/null +++ b/hawc/apps/form_library/urls.py @@ -0,0 +1,13 @@ +from django.urls import path + +from . import views + +app_name = "form_library" +urlpatterns = [ + # Create a Data Extraction form + path( + "form/create/", + views.CreateDataExtractionView.as_view(), + name="form_create", + ), +] diff --git a/hawc/apps/form_library/views.py b/hawc/apps/form_library/views.py index 1411d3a76d..5fe24cdd4c 100644 --- a/hawc/apps/form_library/views.py +++ b/hawc/apps/form_library/views.py @@ -1,7 +1,8 @@ -from forms import CustomDataExtractionForm +from django.views.generic.edit import CreateView -from hawc.apps.common import views +from .forms import CustomDataExtractionForm -class CreateDataExtractionView(views.BaseCreate): +class CreateDataExtractionView(CreateView): + template_name = "form_library/custom_data_extraction_form.html" form_class = CustomDataExtractionForm diff --git a/hawc/main/urls.py b/hawc/main/urls.py index aa930cae3c..82a6f288c3 100644 --- a/hawc/main/urls.py +++ b/hawc/main/urls.py @@ -10,6 +10,7 @@ import hawc.apps.epi.urls import hawc.apps.epimeta.urls import hawc.apps.epiv2.urls +import hawc.apps.form_library.urls import hawc.apps.hawc_admin.urls import hawc.apps.invitro.urls import hawc.apps.lit.urls @@ -43,6 +44,7 @@ path("epidemiology/", include("hawc.apps.epiv2.urls")), path("epi-meta/", include("hawc.apps.epimeta.urls")), path("in-vitro/", include("hawc.apps.invitro.urls")), + path("form-library/", include("hawc.apps.form_library.urls")), path("bmd/", include("hawc.apps.bmd.urls")), path("lit/", include("hawc.apps.lit.urls")), path("summary/", include("hawc.apps.summary.urls")), From 249b7515bc2679ddebd918aaf2f05233bdb6f49b Mon Sep 17 00:00:00 2001 From: munnsmunns Date: Tue, 29 Aug 2023 15:00:38 -0400 Subject: [PATCH 13/35] improve form layout --- hawc/apps/form_library/forms.py | 14 +++++++++++++- .../form_library/custom_data_extraction_form.html | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/hawc/apps/form_library/forms.py b/hawc/apps/form_library/forms.py index c6ae58fe30..c98383e440 100644 --- a/hawc/apps/form_library/forms.py +++ b/hawc/apps/form_library/forms.py @@ -1,7 +1,8 @@ from django import forms +from django.urls import reverse from hawc.apps.common.dynamic_forms.schemas import Schema -from hawc.apps.common.forms import PydanticValidator +from hawc.apps.common.forms import BaseFormHelper, PydanticValidator from .models import CustomDataExtraction @@ -15,3 +16,14 @@ class CustomDataExtractionForm(forms.ModelForm): class Meta: model = CustomDataExtraction exclude = ("created", "last_updated") + + @property + def helper(self): + cancel_url = reverse("portal") # TODO: replace temp cancel url + helper = BaseFormHelper( + self, + legend_text="Create a custom data extraction form", + cancel_url=cancel_url, + submit_text="Create Form", + ) + return helper diff --git a/hawc/apps/form_library/templates/form_library/custom_data_extraction_form.html b/hawc/apps/form_library/templates/form_library/custom_data_extraction_form.html index 6dba6335c0..50d16031c8 100644 --- a/hawc/apps/form_library/templates/form_library/custom_data_extraction_form.html +++ b/hawc/apps/form_library/templates/form_library/custom_data_extraction_form.html @@ -1,4 +1,4 @@ -{% extends 'assessment-rooted.html' %} +{% extends "crumbless.html" %} {% load crispy_forms_tags %} From 7d74a7d51a6e91312a9de5a788496c6b75e82a0a Mon Sep 17 00:00:00 2001 From: munnsmunns Date: Tue, 29 Aug 2023 15:05:13 -0400 Subject: [PATCH 14/35] migrate tests from hero --- .../apps/common/dynamic_forms/__init__.py | 0 .../apps/common/dynamic_forms/conftest.py | 80 ++++++++++++++++++ .../apps/common/dynamic_forms/test_forms.py | 80 ++++++++++++++++++ .../apps/common/dynamic_forms/test_schemas.py | 82 +++++++++++++++++++ 4 files changed, 242 insertions(+) create mode 100644 tests/hawc/apps/common/dynamic_forms/__init__.py create mode 100644 tests/hawc/apps/common/dynamic_forms/conftest.py create mode 100644 tests/hawc/apps/common/dynamic_forms/test_forms.py create mode 100644 tests/hawc/apps/common/dynamic_forms/test_schemas.py diff --git a/tests/hawc/apps/common/dynamic_forms/__init__.py b/tests/hawc/apps/common/dynamic_forms/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/hawc/apps/common/dynamic_forms/conftest.py b/tests/hawc/apps/common/dynamic_forms/conftest.py new file mode 100644 index 0000000000..5dfc7cd350 --- /dev/null +++ b/tests/hawc/apps/common/dynamic_forms/conftest.py @@ -0,0 +1,80 @@ +import pytest + + +@pytest.fixture() +def complete_schema() -> dict: + return { + "fields": [ + { + "name": "textbox", + "type": "char", + "required": False, + "help_text": "Help text", + "css_class": "col-3", + }, + { + "name": "checkbox", + "type": "boolean", + "required": False, + "help_text": "Help text", + "css_class": "col-3", + }, + { + "name": "integer", + "type": "integer", + "required": False, + "help_text": "Help text", + "css_class": "col-3", + }, + { + "name": "float", + "type": "float", + "required": False, + "help_text": "Help text", + "css_class": "col-3", + }, + { + "name": "select", + "type": "choice", + "choices": [["1", "Item 1"], ["2", "Item 2"], ["3", "Item 3"]], + "required": False, + "help_text": "Help text", + "css_class": "col-6", + }, + { + "name": "select_multiple", + "type": "multiple_choice", + "choices": [["1", "Item 1"], ["2", "Item 2"], ["3", "Item 3"]], + "required": False, + "help_text": "Help text", + "css_class": "col-6", + }, + { + "name": "radio", + "type": "choice", + "widget": "radio_select", + "choices": [["1", "Item 1"], ["2", "Item 2"], ["3", "Item 3"]], + "required": False, + "help_text": "Help text", + "css_class": "col-6", + }, + { + "name": "yesno", + "type": "yes_no", + "label": "Yes/no field?", + "required": False, + "help_text": "Help text", + "css_class": "col-6", + }, + { + "name": "checkbox_multiple", + "type": "multiple_choice", + "widget": "checkbox_select_multiple", + "choices": [["1", "Item 1"], ["2", "Item 2"], ["3", "Item 3"]], + "required": False, + "help_text": "Help text", + "css_class": "col-6", + }, + ], + "conditions": [], + } diff --git a/tests/hawc/apps/common/dynamic_forms/test_forms.py b/tests/hawc/apps/common/dynamic_forms/test_forms.py new file mode 100644 index 0000000000..343da3541b --- /dev/null +++ b/tests/hawc/apps/common/dynamic_forms/test_forms.py @@ -0,0 +1,80 @@ +from copy import deepcopy + +from crispy_forms.utils import render_crispy_form +from pytest_django.asserts import assertInHTML + +from hawc.apps.common.dynamic_forms import Schema + + +class TestDynamicForm: + def test_field_rendering(self, complete_schema): + # ensure schema with all field types can render without error + schema = Schema.parse_obj(complete_schema) + form_rendering = render_crispy_form(schema.to_form({})) + assert len(form_rendering) > 0 + + def test_yesno_rendering(self, complete_schema): + # ensure yesno field with inline styles renders as expected + yesno = deepcopy(complete_schema) + yesno["fields"] = [field for field in yesno["fields"] if field["name"] == "yesno"] + schema = Schema.parse_obj(yesno) + form_rendering = render_crispy_form(schema.to_form({})) + expected = """

+
+
+
+ + +
+
+ + +
+ Help text +
""" # noqa: E501 + assertInHTML(expected, form_rendering) + + def test_validation(self): + schema_dict = {"fields": [{"name": "integer", "type": "integer", "required": True}]} + schema = Schema.parse_obj(schema_dict) + # check required + form_data = {} + form = schema.to_form(form_data) + assert form.errors == {"integer": ["This field is required."]} + # check types + form_data = {"integer": "text"} + form = schema.to_form(form_data) + assert form.errors == {"integer": ["Enter a whole number."]} + + def test_conditions(self): + schema_dict = { + "fields": [ + {"name": "field1", "type": "char", "required": True}, + {"name": "field2", "type": "char", "required": True}, + {"name": "field3", "type": "char", "required": True}, + ], + "conditions": [ + { + "subject": "field1", + "observers": ["field2", "field3"], + "comparison": "equals", + "comparison_value": "value", + "behavior": "hide", + } + ], + } + schema = Schema.parse_obj(schema_dict) + # required should remain required when shown + form_data = {"field1": "not value"} + form = schema.to_form(form_data) + assert "field2" in form.errors + assert "field3" in form.errors + # required should become optional when hidden + form_data = {"field1": "value"} + form = schema.to_form(form_data) + assert form.is_valid() + # hidden fields are left out of data + form_data = {"field1": "value", "field2": "hidden", "field3": "hidden"} + form = schema.to_form(form_data) + assert form.is_valid() + assert form.cleaned_data == {"field1": "value"} diff --git a/tests/hawc/apps/common/dynamic_forms/test_schemas.py b/tests/hawc/apps/common/dynamic_forms/test_schemas.py new file mode 100644 index 0000000000..3771972aa1 --- /dev/null +++ b/tests/hawc/apps/common/dynamic_forms/test_schemas.py @@ -0,0 +1,82 @@ +import pytest +from pydantic import ValidationError as PydanticError + +from hawc.apps.common.dynamic_forms import Schema + + +class TestSchema: + def test_field_validation(self): + # all fields should have unique names + schema_dict = { + "fields": [ + {"name": "field1", "type": "char"}, + {"name": "field1", "type": "integer"}, + ] + } + with pytest.raises(PydanticError, match="Duplicate field name"): + Schema.parse_obj(schema_dict) + + # all fields should have a valid type + schema_dict = { + "fields": [ + {"name": "field1"}, + ] + } + with pytest.raises(PydanticError, match="Discriminator 'type' is missing in value"): + Schema.parse_obj(schema_dict) + schema_dict = { + "fields": [ + {"name": "field1", "type": "not a type"}, + ] + } + with pytest.raises( + PydanticError, + match="No match for discriminator 'type' and value 'not a type'", + ): + Schema.parse_obj(schema_dict) + + # if a widget is given, it should be valid for the type + schema_dict = { + "fields": [ + {"name": "field1", "type": "integer", "widget": "text_input"}, + ] + } + with pytest.raises(PydanticError, match="unexpected value; permitted: 'number_input'"): + Schema.parse_obj(schema_dict) + + def test_condition_validation(self): + # condition subjects should correspond with a field + schema_dict = { + "fields": [ + {"name": "field1", "type": "char"}, + {"name": "field2", "type": "char"}, + {"name": "field3", "type": "char"}, + ], + "conditions": [ + { + "subject": "not_a_field", + "observers": ["field2", "field3"], + "comparison_value": "value", + } + ], + } + with pytest.raises(PydanticError, match="Invalid condition subject"): + Schema.parse_obj(schema_dict) + + # condition observers should correspond with a field + schema_dict = { + "fields": [ + {"name": "field1", "type": "char"}, + {"name": "field2", "type": "char"}, + {"name": "field3", "type": "char"}, + ], + "conditions": [ + { + "subject": "field1", + "observers": ["not_a_field", "field3"], + "comparison_value": "value", + } + ], + } + with pytest.raises(PydanticError, match="Invalid condition observer"): + Schema.parse_obj(schema_dict) From aaf615948ee2866434c7f798ee8d2f09d1e20a1b Mon Sep 17 00:00:00 2001 From: munnsmunns Date: Tue, 29 Aug 2023 15:17:10 -0400 Subject: [PATCH 15/35] ruff --- hawc/apps/common/dynamic_forms/fields.py | 22 ++++++++----------- hawc/apps/common/dynamic_forms/schemas.py | 12 +++------- .../apps/common/dynamic_forms/test_forms.py | 2 +- 3 files changed, 13 insertions(+), 23 deletions(-) diff --git a/hawc/apps/common/dynamic_forms/fields.py b/hawc/apps/common/dynamic_forms/fields.py index 2855d5808b..68696234e2 100644 --- a/hawc/apps/common/dynamic_forms/fields.py +++ b/hawc/apps/common/dynamic_forms/fields.py @@ -46,9 +46,7 @@ def ensure_safe_string(cls, v): def get_form_field_kwargs(self) -> dict: """Get keyword arguments for Django form field.""" - kwargs = self.dict( - exclude={"type", "widget", "name", "css_class"}, exclude_none=True - ) + kwargs = self.dict(exclude={"type", "widget", "name", "css_class"}, exclude_none=True) kwargs["widget"] = constants.Widget[self.widget.upper()].value return kwargs @@ -66,14 +64,14 @@ def get_verbose_name(self) -> str: class BooleanField(_Field): """Boolean field.""" - type: Literal["boolean"] = "boolean" # noqa: A003 + type: Literal["boolean"] = "boolean" widget: Literal["checkbox_input"] = "checkbox_input" class CharField(_Field): """Character field.""" - type: Literal["char"] = "char" # noqa: A003 + type: Literal["char"] = "char" widget: Literal["text_input", "textarea"] = "text_input" max_length: int | None @@ -85,7 +83,7 @@ class CharField(_Field): class IntegerField(_Field): """Integer field.""" - type: Literal["integer"] = "integer" # noqa: A003 + type: Literal["integer"] = "integer" widget: Literal["number_input"] = "number_input" min_value: int | None @@ -95,7 +93,7 @@ class IntegerField(_Field): class FloatField(_Field): """Float field.""" - type: Literal["float"] = "float" # noqa: A003 + type: Literal["float"] = "float" widget: Literal["number_input"] = "number_input" min_value: int | None @@ -105,7 +103,7 @@ class FloatField(_Field): class ChoiceField(_Field): """Choice field.""" - type: Literal["choice"] = "choice" # noqa: A003 + type: Literal["choice"] = "choice" widget: Literal["select", "radio_select"] = "select" choices: list[tuple[str, str]] @@ -114,18 +112,16 @@ class ChoiceField(_Field): class YesNoChoiceField(_Field): """Yes No field.""" - type: Literal["yes_no"] = "yes_no" # noqa: A003 + type: Literal["yes_no"] = "yes_no" widget: Literal["radio_select"] = PydanticField("radio_select", const=True) - choices: list[tuple[str, str]] = PydanticField( - [("yes", "Yes"), ("no", "No")], const=True - ) + choices: list[tuple[str, str]] = PydanticField([("yes", "Yes"), ("no", "No")], const=True) class MultipleChoiceField(_Field): """Multiple choice field.""" - type: Literal["multiple_choice"] = "multiple_choice" # noqa: A003 + type: Literal["multiple_choice"] = "multiple_choice" widget: Literal["select_multiple", "checkbox_select_multiple"] = "select_multiple" choices: list[tuple[str, str]] diff --git a/hawc/apps/common/dynamic_forms/schemas.py b/hawc/apps/common/dynamic_forms/schemas.py index 4e857f2b8d..a5facb3f4c 100644 --- a/hawc/apps/common/dynamic_forms/schemas.py +++ b/hawc/apps/common/dynamic_forms/schemas.py @@ -73,15 +73,11 @@ def validate_conditions(cls, values): field_names = {field.name for field in fields} conditions = values["conditions"] subjects = {condition.subject for condition in conditions} - observers = { - observer for condition in conditions for observer in condition.observers - } + observers = {observer for condition in conditions for observer in condition.observers} if bad_subjects := (subjects - field_names): raise ValueError(f"Invalid condition subject(s): {', '.join(bad_subjects)}") if bad_observers := (observers - field_names): - raise ValueError( - f"Invalid condition observer(s): {', '.join(bad_observers)}" - ) + raise ValueError(f"Invalid condition observer(s): {', '.join(bad_observers)}") return values @validator("fields") @@ -100,6 +96,4 @@ def to_form_field(self, prefix, form_kwargs=None, *args, **kwargs): """Get dynamic form field for this schema.""" if len(self.fields) == 0: return JSONField(widget=HiddenInput(), required=False) - return forms.DynamicFormField( - prefix, self.to_form, form_kwargs, *args, **kwargs - ) + return forms.DynamicFormField(prefix, self.to_form, form_kwargs, *args, **kwargs) diff --git a/tests/hawc/apps/common/dynamic_forms/test_forms.py b/tests/hawc/apps/common/dynamic_forms/test_forms.py index 343da3541b..b4d55332c8 100644 --- a/tests/hawc/apps/common/dynamic_forms/test_forms.py +++ b/tests/hawc/apps/common/dynamic_forms/test_forms.py @@ -31,7 +31,7 @@ def test_yesno_rendering(self, complete_schema): Help text - """ # noqa: E501 + """ assertInHTML(expected, form_rendering) def test_validation(self): From 0c8a9bca9673807f51a6ba436886a2226f59b325 Mon Sep 17 00:00:00 2001 From: munnsmunns Date: Tue, 29 Aug 2023 15:31:21 -0400 Subject: [PATCH 16/35] enable feature flag for tests --- hawc/main/settings/unittest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hawc/main/settings/unittest.py b/hawc/main/settings/unittest.py index 210f86b402..7b12e5d6a5 100644 --- a/hawc/main/settings/unittest.py +++ b/hawc/main/settings/unittest.py @@ -10,6 +10,7 @@ # enable feature flags for tests HAWC_FEATURES.ENABLE_BMDS_33 = True +HAWC_FEATURES.ENABLE_DYNAMIC_FORMS = True # remove toolbar for integration tests INSTALLED_APPS = [app for app in INSTALLED_APPS if app != "debug_toolbar"] From 7cb09df8f0c8cce1978e6d17879a6912dbb1a995 Mon Sep 17 00:00:00 2001 From: munnsmunns Date: Tue, 29 Aug 2023 15:42:04 -0400 Subject: [PATCH 17/35] fix settings file --- hawc/main/settings/base.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/hawc/main/settings/base.py b/hawc/main/settings/base.py index ec96afe97a..f592b86825 100644 --- a/hawc/main/settings/base.py +++ b/hawc/main/settings/base.py @@ -125,11 +125,8 @@ "hawc.apps.hawc_admin", "hawc.apps.materialized", "hawc.apps.epiv2", + "hawc.apps.form_library", ) - -if HAWC_FEATURES.ENABLE_DYNAMIC_FORMS: - INSTALLED_APPS += ("hawc.apps.form_library",) - # DB settings DATABASES = { "default": { From 3d1ae6d138e62132f957f4b528a58032c2ced021 Mon Sep 17 00:00:00 2001 From: munnsmunns Date: Tue, 29 Aug 2023 16:15:24 -0400 Subject: [PATCH 18/35] skip failing test for now --- tests/hawc/apps/common/dynamic_forms/test_forms.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/hawc/apps/common/dynamic_forms/test_forms.py b/tests/hawc/apps/common/dynamic_forms/test_forms.py index b4d55332c8..fd53a7a3fc 100644 --- a/tests/hawc/apps/common/dynamic_forms/test_forms.py +++ b/tests/hawc/apps/common/dynamic_forms/test_forms.py @@ -1,5 +1,6 @@ from copy import deepcopy +import pytest from crispy_forms.utils import render_crispy_form from pytest_django.asserts import assertInHTML @@ -13,6 +14,7 @@ def test_field_rendering(self, complete_schema): form_rendering = render_crispy_form(schema.to_form({})) assert len(form_rendering) > 0 + @pytest.mark.skip def test_yesno_rendering(self, complete_schema): # ensure yesno field with inline styles renders as expected yesno = deepcopy(complete_schema) From 34aab55238c251d64f499427669b9c521769a361 Mon Sep 17 00:00:00 2001 From: munnsmunns Date: Thu, 31 Aug 2023 10:57:08 -0400 Subject: [PATCH 19/35] add more fields from review --- ...2_customdataextraction_creator_and_more.py | 53 +++++++++++++++++++ hawc/apps/form_library/models.py | 17 +++++- 2 files changed, 68 insertions(+), 2 deletions(-) create mode 100644 hawc/apps/form_library/migrations/0002_customdataextraction_creator_and_more.py diff --git a/hawc/apps/form_library/migrations/0002_customdataextraction_creator_and_more.py b/hawc/apps/form_library/migrations/0002_customdataextraction_creator_and_more.py new file mode 100644 index 0000000000..de5271fe9f --- /dev/null +++ b/hawc/apps/form_library/migrations/0002_customdataextraction_creator_and_more.py @@ -0,0 +1,53 @@ +# Generated by Django 4.2.4 on 2023-08-31 14:55 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ("form_library", "0001_initial"), + ] + + operations = [ + migrations.AddField( + model_name="customdataextraction", + name="creator", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="created_forms", + to=settings.AUTH_USER_MODEL, + ), + ), + migrations.AddField( + model_name="customdataextraction", + name="editors", + field=models.ManyToManyField( + blank=True, related_name="editable_forms", to=settings.AUTH_USER_MODEL + ), + ), + migrations.AddField( + model_name="customdataextraction", + name="parent_form", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="child_forms", + to="form_library.customdataextraction", + ), + ), + migrations.AlterField( + model_name="customdataextraction", + name="description", + field=models.TextField(blank=True), + ), + migrations.AlterField( + model_name="customdataextraction", + name="name", + field=models.CharField(max_length=128), + ), + ] diff --git a/hawc/apps/form_library/models.py b/hawc/apps/form_library/models.py index 7f2f57882e..804c538990 100644 --- a/hawc/apps/form_library/models.py +++ b/hawc/apps/form_library/models.py @@ -1,15 +1,28 @@ import reversion from django.db import models +from ..myuser.models import HAWCUser + # TODO: is this a good name? i changed it from dynamic form because it makes the naming scheme weird # for the actual form class (DynamicFormForm) class CustomDataExtraction(models.Model): - name = models.CharField() - description = models.TextField() + name = models.CharField(max_length=128) + description = models.TextField(blank=True) schema = models.JSONField() created = models.DateTimeField(auto_now_add=True) last_updated = models.DateTimeField(auto_now=True) + creator = models.ForeignKey( + HAWCUser, on_delete=models.SET_NULL, null=True, related_name="created_forms" + ) + editors = models.ManyToManyField(HAWCUser, blank=True, related_name="editable_forms") + parent_form = models.ForeignKey( + "form_library.CustomDataExtraction", + blank=True, + null=True, + on_delete=models.SET_NULL, + related_name="child_forms", + ) reversion.register(CustomDataExtraction) From b13a478f80d4cfec8be6aa78ebc30cfdf3d048c5 Mon Sep 17 00:00:00 2001 From: munnsmunns Date: Thu, 31 Aug 2023 14:16:00 -0400 Subject: [PATCH 20/35] add fields to form --- hawc/apps/form_library/forms.py | 14 +++++++++++++- hawc/apps/form_library/models.py | 4 ++-- hawc/apps/form_library/views.py | 13 ++++++++++++- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/hawc/apps/form_library/forms.py b/hawc/apps/form_library/forms.py index c98383e440..47113deda9 100644 --- a/hawc/apps/form_library/forms.py +++ b/hawc/apps/form_library/forms.py @@ -1,8 +1,10 @@ from django import forms from django.urls import reverse +from hawc.apps.common.autocomplete.forms import AutocompleteSelectMultipleWidget from hawc.apps.common.dynamic_forms.schemas import Schema from hawc.apps.common.forms import BaseFormHelper, PydanticValidator +from hawc.apps.myuser.autocomplete import UserAutocomplete from .models import CustomDataExtraction @@ -13,12 +15,22 @@ class CustomDataExtractionForm(forms.ModelForm): validators=[PydanticValidator(Schema)], ) + def __init__(self, *args, **kwargs): + self.user = kwargs.pop("user") + super().__init__(*args, **kwargs) + self.fields["creator"].initial = self.user + class Meta: model = CustomDataExtraction - exclude = ("created", "last_updated") + exclude = ("parent_form", "created", "last_updated") + widgets = { + "editors": AutocompleteSelectMultipleWidget(UserAutocomplete), + "creator": forms.HiddenInput, + } @property def helper(self): + self.fields["description"].widget.attrs["rows"] = 3 cancel_url = reverse("portal") # TODO: replace temp cancel url helper = BaseFormHelper( self, diff --git a/hawc/apps/form_library/models.py b/hawc/apps/form_library/models.py index 804c538990..b4d8920d14 100644 --- a/hawc/apps/form_library/models.py +++ b/hawc/apps/form_library/models.py @@ -10,8 +10,6 @@ class CustomDataExtraction(models.Model): name = models.CharField(max_length=128) description = models.TextField(blank=True) schema = models.JSONField() - created = models.DateTimeField(auto_now_add=True) - last_updated = models.DateTimeField(auto_now=True) creator = models.ForeignKey( HAWCUser, on_delete=models.SET_NULL, null=True, related_name="created_forms" ) @@ -23,6 +21,8 @@ class CustomDataExtraction(models.Model): on_delete=models.SET_NULL, related_name="child_forms", ) + created = models.DateTimeField(auto_now_add=True) + last_updated = models.DateTimeField(auto_now=True) reversion.register(CustomDataExtraction) diff --git a/hawc/apps/form_library/views.py b/hawc/apps/form_library/views.py index 5fe24cdd4c..5cc88ce2a4 100644 --- a/hawc/apps/form_library/views.py +++ b/hawc/apps/form_library/views.py @@ -1,8 +1,19 @@ +from django.urls import reverse from django.views.generic.edit import CreateView +from hawc.apps.common.views import LoginRequiredMixin + from .forms import CustomDataExtractionForm -class CreateDataExtractionView(CreateView): +class CreateDataExtractionView(LoginRequiredMixin, CreateView): template_name = "form_library/custom_data_extraction_form.html" form_class = CustomDataExtractionForm + + def get_form_kwargs(self): + kwargs = super().get_form_kwargs() + kwargs.update(user=self.request.user) + return kwargs + + def get_success_url(self) -> str: + return reverse("portal") From cc8829c3340e7f0ccef6c1bc1fbda84979cd0e56 Mon Sep 17 00:00:00 2001 From: munnsmunns Date: Thu, 31 Aug 2023 16:12:45 -0400 Subject: [PATCH 21/35] lint --- .../migrations/0002_customdataextraction_creator_and_more.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hawc/apps/form_library/migrations/0002_customdataextraction_creator_and_more.py b/hawc/apps/form_library/migrations/0002_customdataextraction_creator_and_more.py index de5271fe9f..8db48db224 100644 --- a/hawc/apps/form_library/migrations/0002_customdataextraction_creator_and_more.py +++ b/hawc/apps/form_library/migrations/0002_customdataextraction_creator_and_more.py @@ -1,8 +1,8 @@ # Generated by Django 4.2.4 on 2023-08-31 14:55 +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion class Migration(migrations.Migration): From b64f95971412e7de9d4ac7a63cddedcf8ab810a2 Mon Sep 17 00:00:00 2001 From: Andy Shapiro Date: Thu, 7 Sep 2023 16:34:48 -0400 Subject: [PATCH 22/35] fix test --- .../apps/common/dynamic_forms/test_forms.py | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/hawc/apps/common/dynamic_forms/test_forms.py b/tests/hawc/apps/common/dynamic_forms/test_forms.py index fd53a7a3fc..e1444a28c2 100644 --- a/tests/hawc/apps/common/dynamic_forms/test_forms.py +++ b/tests/hawc/apps/common/dynamic_forms/test_forms.py @@ -1,6 +1,5 @@ from copy import deepcopy -import pytest from crispy_forms.utils import render_crispy_form from pytest_django.asserts import assertInHTML @@ -14,26 +13,27 @@ def test_field_rendering(self, complete_schema): form_rendering = render_crispy_form(schema.to_form({})) assert len(form_rendering) > 0 - @pytest.mark.skip def test_yesno_rendering(self, complete_schema): # ensure yesno field with inline styles renders as expected yesno = deepcopy(complete_schema) yesno["fields"] = [field for field in yesno["fields"] if field["name"] == "yesno"] schema = Schema.parse_obj(yesno) form_rendering = render_crispy_form(schema.to_form({})) - expected = """
-
-
-
- - -
-
- - -
- Help text -
""" + expected = """ +
+ +
+
+ + +
+
+ + +
+ Help text +
+
""" assertInHTML(expected, form_rendering) def test_validation(self): From 441f6b5425e0cb6cfd2ceec0473dc8afae0241b2 Mon Sep 17 00:00:00 2001 From: Andy Shapiro Date: Thu, 7 Sep 2023 18:53:09 -0400 Subject: [PATCH 23/35] set the creator outside with data from request, not form fields --- hawc/apps/form_library/forms.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hawc/apps/form_library/forms.py b/hawc/apps/form_library/forms.py index 47113deda9..1d2b81fd0c 100644 --- a/hawc/apps/form_library/forms.py +++ b/hawc/apps/form_library/forms.py @@ -16,16 +16,16 @@ class CustomDataExtractionForm(forms.ModelForm): ) def __init__(self, *args, **kwargs): - self.user = kwargs.pop("user") + user = kwargs.pop("user") super().__init__(*args, **kwargs) - self.fields["creator"].initial = self.user + if self.instance.id is None: + self.instance.creator = user class Meta: model = CustomDataExtraction - exclude = ("parent_form", "created", "last_updated") + exclude = ("parent_form", "creator", "created", "last_updated") widgets = { "editors": AutocompleteSelectMultipleWidget(UserAutocomplete), - "creator": forms.HiddenInput, } @property From b3ee0a8748f55a7c4fd0584b13e653d5bd563944 Mon Sep 17 00:00:00 2001 From: munnsmunns Date: Tue, 12 Sep 2023 10:59:15 -0400 Subject: [PATCH 24/35] updates from review --- hawc/apps/form_library/admin.py | 4 ++-- hawc/apps/form_library/forms.py | 6 +++--- hawc/apps/form_library/models.py | 21 +++++++++++-------- ...ata_extraction_form.html => udf_form.html} | 0 hawc/apps/form_library/urls.py | 6 +++--- hawc/apps/form_library/views.py | 8 +++---- hawc/main/urls.py | 2 +- 7 files changed, 25 insertions(+), 22 deletions(-) rename hawc/apps/form_library/templates/form_library/{custom_data_extraction_form.html => udf_form.html} (100%) diff --git a/hawc/apps/form_library/admin.py b/hawc/apps/form_library/admin.py index 1fbbf59c01..b0423e8343 100644 --- a/hawc/apps/form_library/admin.py +++ b/hawc/apps/form_library/admin.py @@ -4,8 +4,8 @@ class CustomDataExtractionInline(admin.TabularInline): - model = models.CustomDataExtraction + model = models.UserDefinedForm extra = 0 -admin.site.register(models.CustomDataExtraction) +admin.site.register(models.UserDefinedForm) diff --git a/hawc/apps/form_library/forms.py b/hawc/apps/form_library/forms.py index 1d2b81fd0c..113c8d9d1f 100644 --- a/hawc/apps/form_library/forms.py +++ b/hawc/apps/form_library/forms.py @@ -6,10 +6,10 @@ from hawc.apps.common.forms import BaseFormHelper, PydanticValidator from hawc.apps.myuser.autocomplete import UserAutocomplete -from .models import CustomDataExtraction +from .models import UserDefinedForm -class CustomDataExtractionForm(forms.ModelForm): +class UDFForm(forms.ModelForm): schema = forms.JSONField( initial=Schema(fields=[]).dict(), validators=[PydanticValidator(Schema)], @@ -22,7 +22,7 @@ def __init__(self, *args, **kwargs): self.instance.creator = user class Meta: - model = CustomDataExtraction + model = UserDefinedForm exclude = ("parent_form", "creator", "created", "last_updated") widgets = { "editors": AutocompleteSelectMultipleWidget(UserAutocomplete), diff --git a/hawc/apps/form_library/models.py b/hawc/apps/form_library/models.py index b4d8920d14..c9338cd9b7 100644 --- a/hawc/apps/form_library/models.py +++ b/hawc/apps/form_library/models.py @@ -6,23 +6,26 @@ # TODO: is this a good name? i changed it from dynamic form because it makes the naming scheme weird # for the actual form class (DynamicFormForm) -class CustomDataExtraction(models.Model): +class UserDefinedForm(models.Model): name = models.CharField(max_length=128) - description = models.TextField(blank=True) + description = models.TextField() schema = models.JSONField() - creator = models.ForeignKey( - HAWCUser, on_delete=models.SET_NULL, null=True, related_name="created_forms" - ) + creator = models.ForeignKey(HAWCUser, on_delete=models.DO_NOTHING, related_name="created_forms") editors = models.ManyToManyField(HAWCUser, blank=True, related_name="editable_forms") - parent_form = models.ForeignKey( - "form_library.CustomDataExtraction", + parent = models.ForeignKey( + "form_library.UserDefinedForm", blank=True, null=True, on_delete=models.SET_NULL, - related_name="child_forms", + related_name="children", ) + deprecated = models.BooleanField(default=False) created = models.DateTimeField(auto_now_add=True) last_updated = models.DateTimeField(auto_now=True) + class Meta: + unique_together = ["creator", "name"] + ordering = ["-last_updated"] + -reversion.register(CustomDataExtraction) +reversion.register(UserDefinedForm) diff --git a/hawc/apps/form_library/templates/form_library/custom_data_extraction_form.html b/hawc/apps/form_library/templates/form_library/udf_form.html similarity index 100% rename from hawc/apps/form_library/templates/form_library/custom_data_extraction_form.html rename to hawc/apps/form_library/templates/form_library/udf_form.html diff --git a/hawc/apps/form_library/urls.py b/hawc/apps/form_library/urls.py index 4fa2d2b66f..5e3738fe0b 100644 --- a/hawc/apps/form_library/urls.py +++ b/hawc/apps/form_library/urls.py @@ -4,10 +4,10 @@ app_name = "form_library" urlpatterns = [ - # Create a Data Extraction form + # Create a user defined form path( - "form/create/", - views.CreateDataExtractionView.as_view(), + "create/", + views.CreateUDFView.as_view(), name="form_create", ), ] diff --git a/hawc/apps/form_library/views.py b/hawc/apps/form_library/views.py index 5cc88ce2a4..bc9cd37688 100644 --- a/hawc/apps/form_library/views.py +++ b/hawc/apps/form_library/views.py @@ -3,12 +3,12 @@ from hawc.apps.common.views import LoginRequiredMixin -from .forms import CustomDataExtractionForm +from .forms import UDFForm -class CreateDataExtractionView(LoginRequiredMixin, CreateView): - template_name = "form_library/custom_data_extraction_form.html" - form_class = CustomDataExtractionForm +class CreateUDFView(LoginRequiredMixin, CreateView): + template_name = "form_library/udf_form.html" + form_class = UDFForm def get_form_kwargs(self): kwargs = super().get_form_kwargs() diff --git a/hawc/main/urls.py b/hawc/main/urls.py index 82a6f288c3..68fc6814e6 100644 --- a/hawc/main/urls.py +++ b/hawc/main/urls.py @@ -44,7 +44,7 @@ path("epidemiology/", include("hawc.apps.epiv2.urls")), path("epi-meta/", include("hawc.apps.epimeta.urls")), path("in-vitro/", include("hawc.apps.invitro.urls")), - path("form-library/", include("hawc.apps.form_library.urls")), + path("forms/", include("hawc.apps.form_library.urls")), path("bmd/", include("hawc.apps.bmd.urls")), path("lit/", include("hawc.apps.lit.urls")), path("summary/", include("hawc.apps.summary.urls")), From 9c8dfac9a22d75e81d3466d0eeec9753cba89628 Mon Sep 17 00:00:00 2001 From: munnsmunns Date: Tue, 12 Sep 2023 10:59:55 -0400 Subject: [PATCH 25/35] remove comment --- hawc/apps/form_library/models.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/hawc/apps/form_library/models.py b/hawc/apps/form_library/models.py index c9338cd9b7..b27e799227 100644 --- a/hawc/apps/form_library/models.py +++ b/hawc/apps/form_library/models.py @@ -4,8 +4,6 @@ from ..myuser.models import HAWCUser -# TODO: is this a good name? i changed it from dynamic form because it makes the naming scheme weird -# for the actual form class (DynamicFormForm) class UserDefinedForm(models.Model): name = models.CharField(max_length=128) description = models.TextField() From 300736ca05c2d8a3ff696a951cbf6535999491e2 Mon Sep 17 00:00:00 2001 From: munnsmunns Date: Tue, 12 Sep 2023 12:18:46 -0400 Subject: [PATCH 26/35] fix urls --- hawc/apps/form_library/urls.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/hawc/apps/form_library/urls.py b/hawc/apps/form_library/urls.py index 5e3738fe0b..f947ab939e 100644 --- a/hawc/apps/form_library/urls.py +++ b/hawc/apps/form_library/urls.py @@ -1,13 +1,18 @@ +from django.conf import settings from django.urls import path from . import views app_name = "form_library" -urlpatterns = [ - # Create a user defined form - path( - "create/", - views.CreateUDFView.as_view(), - name="form_create", - ), -] +urlpatterns = ( + [ + # Create a user defined form + path( + "create/", + views.CreateUDFView.as_view(), + name="form_create", + ), + ] + if settings.HAWC_FEATURES.ENABLE_DYNAMIC_FORMS + else [] +) From 3729afa1820f3de175bc3fd66ab16b59a64c940b Mon Sep 17 00:00:00 2001 From: Andy Shapiro Date: Sat, 16 Sep 2023 09:52:54 -0400 Subject: [PATCH 27/35] vendor dynamic forms --- hawc/apps/common/dynamic_forms/__init__.py | 5 + hawc/apps/common/dynamic_forms/constants.py | 71 +++++++++ hawc/apps/common/dynamic_forms/fields.py | 139 ++++++++++++++++++ hawc/apps/common/dynamic_forms/forms.py | 124 ++++++++++++++++ hawc/apps/common/dynamic_forms/schemas.py | 99 +++++++++++++ hawc/apps/common/filterset.py | 3 +- hawc/apps/common/forms.py | 49 ++++++ .../common/templates/common/dynamic_form.html | 2 + hawc/apps/common/widgets.py | 34 +++++ .../apps/common/dynamic_forms/__init__.py | 0 .../apps/common/dynamic_forms/conftest.py | 80 ++++++++++ .../apps/common/dynamic_forms/test_forms.py | 82 +++++++++++ .../apps/common/dynamic_forms/test_schemas.py | 82 +++++++++++ 13 files changed, 769 insertions(+), 1 deletion(-) create mode 100644 hawc/apps/common/dynamic_forms/__init__.py create mode 100644 hawc/apps/common/dynamic_forms/constants.py create mode 100644 hawc/apps/common/dynamic_forms/fields.py create mode 100644 hawc/apps/common/dynamic_forms/forms.py create mode 100644 hawc/apps/common/dynamic_forms/schemas.py create mode 100644 hawc/apps/common/templates/common/dynamic_form.html create mode 100644 tests/hawc/apps/common/dynamic_forms/__init__.py create mode 100644 tests/hawc/apps/common/dynamic_forms/conftest.py create mode 100644 tests/hawc/apps/common/dynamic_forms/test_forms.py create mode 100644 tests/hawc/apps/common/dynamic_forms/test_schemas.py diff --git a/hawc/apps/common/dynamic_forms/__init__.py b/hawc/apps/common/dynamic_forms/__init__.py new file mode 100644 index 0000000000..6d497a04ca --- /dev/null +++ b/hawc/apps/common/dynamic_forms/__init__.py @@ -0,0 +1,5 @@ +from ..forms import DynamicFormField +from .forms import DynamicForm +from .schemas import Schema + +__all__ = ["DynamicForm", "DynamicFormField", "Schema"] diff --git a/hawc/apps/common/dynamic_forms/constants.py b/hawc/apps/common/dynamic_forms/constants.py new file mode 100644 index 0000000000..95c450471e --- /dev/null +++ b/hawc/apps/common/dynamic_forms/constants.py @@ -0,0 +1,71 @@ +"""Django form enums.""" +from enum import Enum + +from django import forms + +from ..forms import InlineRadioChoiceField + + +class FormField(Enum): + """Django form field enumeration. + + When removing access to a member, please + comment out instead of omitting. + """ + + BOOLEAN = forms.BooleanField + CHAR = forms.CharField + CHOICE = forms.ChoiceField + DATE = forms.DateField + DATE_TIME = forms.DateTimeField + DECIMAL = forms.DecimalField + DURATION = forms.DurationField + EMAIL = forms.EmailField + FILE = forms.FileField + FILEPATH = forms.FilePathField + FLOAT = forms.FloatField + GENERIC_IP_ADDRESS = forms.GenericIPAddressField + IMAGE = forms.ImageField + INTEGER = forms.IntegerField + JSON = forms.JSONField + MULTIPLE_CHOICE = forms.MultipleChoiceField + NULL_BOOLEAN = forms.NullBooleanField + REGEX = forms.RegexField + SLUG = forms.SlugField + TIME = forms.TimeField + TYPED_CHOICE = forms.TypedChoiceField + TYPED_MULTIPLE_CHOICE = forms.TypedMultipleChoiceField + URL = forms.URLField + UUID = forms.UUIDField + YES_NO = InlineRadioChoiceField + + +class Widget(Enum): + """Django widget enumeration. + + When removing access to a member, please + comment out instead of omitting. + """ + + TEXT_INPUT = forms.TextInput + NUMBER_INPUT = forms.NumberInput + EMAIL_INPUT = forms.EmailInput + URL_INPUT = forms.URLInput + PASSWORD_INPUT = forms.PasswordInput + HIDDEN_INPUT = forms.HiddenInput + DATE_INPUT = forms.DateInput + DATE_TIME_INPUT = forms.DateTimeInput + TIME_INPUT = forms.TimeInput + TEXTAREA = forms.Textarea + CHECKBOX_INPUT = forms.CheckboxInput + SELECT = forms.Select + NULL_BOOLEAN_SELECT = forms.NullBooleanSelect + SELECT_MULTIPLE = forms.SelectMultiple + RADIO_SELECT = forms.RadioSelect + CHECKBOX_SELECT_MULTIPLE = forms.CheckboxSelectMultiple + FILE_INPUT = forms.FileInput + CLEARABLE_FILE_INPUT = forms.ClearableFileInput + MULTIPLE_HIDDEN_INPUT = forms.MultipleHiddenInput + SPLIT_DATE_TIME = forms.SplitDateTimeWidget + SPLIT_HIDDEN_DATE_TIME = forms.SplitHiddenDateTimeWidget + SELECT_DATE = forms.SelectDateWidget diff --git a/hawc/apps/common/dynamic_forms/fields.py b/hawc/apps/common/dynamic_forms/fields.py new file mode 100644 index 0000000000..68696234e2 --- /dev/null +++ b/hawc/apps/common/dynamic_forms/fields.py @@ -0,0 +1,139 @@ +"""Dynamic Django form fields.""" +from typing import Annotated, Any, Literal + +from django import forms +from django.utils.html import conditional_escape +from pydantic import BaseModel, validator +from pydantic import Field as PydanticField + +from . import constants + + +class _Field(BaseModel): + """Base class for Django form field schemas. + + This class should only act as a superclass for each specific Django form field. + + Each subclass should add a 'type' property that corresponds with a key + in constants.FormField, and a 'widget' property that corresponds with keys + in constants.Widget. + """ + + name: str # the variable name in the form; extra validation for no whitespace etc? + required: bool | None + label: str | None + label_suffix: str | None + initial: Any = None + help_text: str | None + css_class: str | None + # error_messages + # validators + # localize + # disabled + + class Config: + """Schema config.""" + + underscore_attrs_are_private = True + use_enum_values = True + + @validator("help_text") + def ensure_safe_string(cls, v): + """Sanitize help_text values.""" + if v is None: + return v + return conditional_escape(v) + + def get_form_field_kwargs(self) -> dict: + """Get keyword arguments for Django form field.""" + kwargs = self.dict(exclude={"type", "widget", "name", "css_class"}, exclude_none=True) + kwargs["widget"] = constants.Widget[self.widget.upper()].value + return kwargs + + def get_form_field(self) -> forms.Field: + """Get Django form field.""" + form_field_cls = constants.FormField[self.type.upper()].value + form_field_kwargs = self.get_form_field_kwargs() + return form_field_cls(**form_field_kwargs) + + def get_verbose_name(self) -> str: + """Get reader friendly name for schema.""" + return self.label or self.name.replace("_", " ").title() + + +class BooleanField(_Field): + """Boolean field.""" + + type: Literal["boolean"] = "boolean" + widget: Literal["checkbox_input"] = "checkbox_input" + + +class CharField(_Field): + """Character field.""" + + type: Literal["char"] = "char" + widget: Literal["text_input", "textarea"] = "text_input" + + max_length: int | None + min_length: int | None + strip: bool | None + empty_value: str | None + + +class IntegerField(_Field): + """Integer field.""" + + type: Literal["integer"] = "integer" + widget: Literal["number_input"] = "number_input" + + min_value: int | None + max_value: int | None + + +class FloatField(_Field): + """Float field.""" + + type: Literal["float"] = "float" + widget: Literal["number_input"] = "number_input" + + min_value: int | None + max_value: int | None + + +class ChoiceField(_Field): + """Choice field.""" + + type: Literal["choice"] = "choice" + widget: Literal["select", "radio_select"] = "select" + + choices: list[tuple[str, str]] + + +class YesNoChoiceField(_Field): + """Yes No field.""" + + type: Literal["yes_no"] = "yes_no" + widget: Literal["radio_select"] = PydanticField("radio_select", const=True) + + choices: list[tuple[str, str]] = PydanticField([("yes", "Yes"), ("no", "No")], const=True) + + +class MultipleChoiceField(_Field): + """Multiple choice field.""" + + type: Literal["multiple_choice"] = "multiple_choice" + widget: Literal["select_multiple", "checkbox_select_multiple"] = "select_multiple" + + choices: list[tuple[str, str]] + + +Field = Annotated[ + BooleanField + | CharField + | ChoiceField + | YesNoChoiceField + | FloatField + | IntegerField + | MultipleChoiceField, + PydanticField(discriminator="type"), +] diff --git a/hawc/apps/common/dynamic_forms/forms.py b/hawc/apps/common/dynamic_forms/forms.py new file mode 100644 index 0000000000..2944dd96df --- /dev/null +++ b/hawc/apps/common/dynamic_forms/forms.py @@ -0,0 +1,124 @@ +"""Dynamic Django forms.""" +import json + +from crispy_forms import layout as cfl +from django import forms + +from ..forms import BaseFormHelper +from .schemas import Behavior, Schema + + +class DynamicForm(forms.Form): + """Dynamic Django form. + + This is built from a custom schema that defines fields and conditional logic. + """ + + def __init__(self, schema: Schema, *args, **kwargs): + """Create dynamic form.""" + super().__init__(*args, **kwargs) + self.schema = schema + fields = {f.name: f.get_form_field() for f in self.schema.fields} + self.fields.update(fields) + + @property + def helper(self): + """Django crispy form helper.""" + helper = DynamicFormHelper(self) + + # wrap up field inputs with bootstrap grid + helper.auto_wrap_fields() + + # expose serialized conditions w/ unique id to template + helper.conditions = [ + { + "subject_id": self[condition.subject].auto_id, + "observer_ids": [self[observer].auto_id for observer in condition.observers], + "comparison": condition.comparison, + "comparison_value": condition.comparison_value, + "behavior": condition.behavior, + } + for condition in self.schema.conditions + ] + helper.conditions_id = f"conditions-{hash(json.dumps(helper.conditions))}" + helper.layout.append(cfl.HTML("{{ conditions|json_script:conditions_id }}")) + + return helper + + def full_clean(self): + """Overridden full_clean that handles conditional logic from schema.""" + # handle conditions + if self.is_bound: + fields = self.fields.copy() # copy to restore after clean + data = self.data.copy() # copy in case it's immutable + for condition in self.schema.conditions: + bf = self[condition.subject] + value = bf.field.to_python(bf.value()) + check = condition.comparison.compare(value, condition.comparison_value) + show = check if condition.behavior == Behavior.SHOW else not check + if not show: + for observer in condition.observers: + # remove data that should be hidden + data.pop(observer, None) + # remove fields that should be hidden; + # this is restored after clean + self.fields.pop(observer) + self.data = data + + # run default full_clean + super().full_clean() + + # restore fields + if self.is_bound: + self.fields = fields + + +class DynamicFormHelper(BaseFormHelper): + """Django crispy form helper.""" + + form_tag = False + + def auto_wrap_fields(self): + """Wrap fields in bootstrap classes.""" + if len(self.form.schema.fields) == 0: + return + + for field in self.form.schema.fields: + index = self.layout.index(field.name) + css_class = field.css_class or "col-12" + self[index].wrap(cfl.Column, css_class=css_class) + + self[:].wrap_together(cfl.Row, id="row_id_dynamic_form") + self.add_field_wraps() + + +class DynamicFormWidget(forms.Widget): + """Widget to display dynamic form inline.""" + + template_name = "common/widgets/dynamic_form.html" + + def __init__(self, prefix, form_class, form_kwargs=None, *args, **kwargs): + """Create dynamic form widget.""" + super().__init__(*args, **kwargs) + self.prefix = prefix + self.form_class = form_class + if form_kwargs is None: + form_kwargs = {} + self.form_kwargs = {"prefix": prefix, **form_kwargs} + + def add_prefix(self, field_name): + """Add prefix in the same way Django forms add prefixes.""" + return f"{self.prefix}-{field_name}" + + def format_value(self, value): + """Value used in rendering.""" + value = json.loads(value) + if value: + value = {self.add_prefix(k): v for k, v in value.items()} + return self.form_class(data=value, **self.form_kwargs) + + def value_from_datadict(self, data, files, name): + """Parse value from POST request.""" + form = self.form_class(data=data, **self.form_kwargs) + form.full_clean() + return form.cleaned_data diff --git a/hawc/apps/common/dynamic_forms/schemas.py b/hawc/apps/common/dynamic_forms/schemas.py new file mode 100644 index 0000000000..a5facb3f4c --- /dev/null +++ b/hawc/apps/common/dynamic_forms/schemas.py @@ -0,0 +1,99 @@ +"""Schemas to build dynamic Django forms.""" +from enum import Enum + +from django.forms import HiddenInput, JSONField +from pydantic import BaseModel, conlist, root_validator, validator + +from . import fields, forms + + +class Comparison(str, Enum): + """Enum for comparisons.""" + + __slots__ = () + + EQUALS = "equals" + IN = "in" + CONTAINS = "contains" + + def _equals(self, x, y) -> bool: + # x equals y + return x == y + + def _in(self, x, y) -> bool: + # x (or a subset of x) is in y + return any(_ in y for _ in x) + + def _contains(self, x, y) -> bool: + # x contains y + return set(x) >= set(y) + + def compare(self, x, y) -> bool: + """Perform comparison based on enum value.""" + x = x if isinstance(x, list) else [x] + y = y if isinstance(y, list) else [y] + return getattr(self, f"_{self.value.lower()}")(x, y) + + +class Behavior(str, Enum): + """Enum for form field behavior; behavior applies when condition is true.""" + + __slots__ = () + + SHOW = "show" + HIDE = "hide" + + +class Condition(BaseModel): + """Condition that affects the visibility of fields.""" + + subject: str + observers: list[str] + comparison: Comparison = Comparison.EQUALS + comparison_value: bool | str | int | conlist(bool | str | int, min_items=1) + behavior: Behavior = Behavior.SHOW + + class Config: + """Schema config.""" + + smart_union = True + + +class Schema(BaseModel): + """Schema for dynamic form.""" + + fields: list[fields.Field] + conditions: list[Condition] = [] + + @root_validator(skip_on_failure=True) + def validate_conditions(cls, values): + """Validate conditions.""" + # condition subjects and observers should be existing fields + fields = values["fields"] + field_names = {field.name for field in fields} + conditions = values["conditions"] + subjects = {condition.subject for condition in conditions} + observers = {observer for condition in conditions for observer in condition.observers} + if bad_subjects := (subjects - field_names): + raise ValueError(f"Invalid condition subject(s): {', '.join(bad_subjects)}") + if bad_observers := (observers - field_names): + raise ValueError(f"Invalid condition observer(s): {', '.join(bad_observers)}") + return values + + @validator("fields") + def unique_field_names(cls, v): + """Validate field names.""" + unique_names = {field.name for field in v} + if len(unique_names) != len(v): + raise ValueError("Duplicate field name(s)") + return v + + def to_form(self, *args, **kwargs): + """Get dynamic form for this schema.""" + return forms.DynamicForm(self, *args, **kwargs) + + def to_form_field(self, prefix, form_kwargs=None, *args, **kwargs): + """Get dynamic form field for this schema.""" + if len(self.fields) == 0: + return JSONField(widget=HiddenInput(), required=False) + return forms.DynamicFormField(prefix, self.to_form, form_kwargs, *args, **kwargs) diff --git a/hawc/apps/common/filterset.py b/hawc/apps/common/filterset.py index e96241b4a1..b2fa7b7ea5 100644 --- a/hawc/apps/common/filterset.py +++ b/hawc/apps/common/filterset.py @@ -171,7 +171,8 @@ def create_form(self): form = form_class(self.data, prefix=self.form_prefix, **self.form_kwargs) else: form = form_class(prefix=self.form_prefix, **self.form_kwargs) - if form.dynamic_fields: # removes unwanted fields from a filterset if specified + # removes unwanted fields from a filterset if specified, not empty, and not N one + if getattr(form, "dynamic_fields", None): for field in list(form.fields.keys()): if field not in form.dynamic_fields and field != "is_expanded": form.fields.pop(field) diff --git a/hawc/apps/common/forms.py b/hawc/apps/common/forms.py index 12a4f6d456..615d37adce 100644 --- a/hawc/apps/common/forms.py +++ b/hawc/apps/common/forms.py @@ -5,9 +5,11 @@ from crispy_forms import layout as cfl from crispy_forms.utils import TEMPLATE_PACK, flatatt from django import forms +from django.forms.widgets import RadioSelect from django.template.loader import render_to_string from . import autocomplete, validators, widgets +from .helper import PydanticToDjangoError ASSESSMENT_UNIQUE_MESSAGE = "Must be unique for assessment (current value already exists)." @@ -452,3 +454,50 @@ def validate(self, value): super().validate(value) if value != self.check_value: raise forms.ValidationError(f'The value of "{self.check_value}" is required.') + + +class DynamicFormField(forms.JSONField): + """Field to display dynamic form inline.""" + + default_error_messages = {"invalid": "Invalid input"} + widget = widgets.DynamicFormWidget + + def __init__(self, prefix, form_class, form_kwargs=None, *args, **kwargs): + """Create dynamic form field.""" + self.form_class = form_class + self.form_kwargs = {} if form_kwargs is None else form_kwargs + self.widget = self.widget(prefix, form_class, form_kwargs) + super().__init__(*args, **kwargs) + + def bound_data(self, data, initial): + """Get data to be shown for this field on render.""" + if self.disabled: + return initial + return data + + def validate(self, value): + """Validate inline form.""" + super().validate(value) + form = self.form_class(data=value, **self.form_kwargs) + if not form.is_valid(): + raise forms.ValidationError(self.error_messages["invalid"]) + + +class InlineRadioChoiceField(forms.ChoiceField): + """Choice widget that uses radio buttons that are inline.""" + + widget = RadioSelect + crispy_field_class = cfb.InlineRadios + + +class PydanticValidator: + """JSON field validator that uses a pydantic model.""" + + def __init__(self, schema): + """Set the schema.""" + self.schema = schema + + def __call__(self, value): + """Validate the field with the pydantic model.""" + with PydanticToDjangoError(include_field=False): + self.schema.parse_obj(value) diff --git a/hawc/apps/common/templates/common/dynamic_form.html b/hawc/apps/common/templates/common/dynamic_form.html new file mode 100644 index 0000000000..492689a19e --- /dev/null +++ b/hawc/apps/common/templates/common/dynamic_form.html @@ -0,0 +1,2 @@ +{% load crispy_forms_tags %} +{% crispy widget.value %} diff --git a/hawc/apps/common/widgets.py b/hawc/apps/common/widgets.py index 8618e9f4c3..424a4e221f 100644 --- a/hawc/apps/common/widgets.py +++ b/hawc/apps/common/widgets.py @@ -1,3 +1,4 @@ +import json from random import randint from django.conf import settings @@ -10,6 +11,7 @@ SelectMultiple, Textarea, TextInput, + Widget, ) from django.utils import timezone @@ -124,3 +126,35 @@ def build_attrs(self, base_attrs, extra_attrs=None): class_name = attrs.get("class") attrs["class"] = class_name + " quilltext" if class_name else "quilltext" return attrs + + +class DynamicFormWidget(Widget): + """Widget to display dynamic form inline.""" + + template_name = "common/dynamic_form.html" + + def __init__(self, prefix, form_class, form_kwargs=None, *args, **kwargs): + """Create dynamic form widget.""" + super().__init__(*args, **kwargs) + self.prefix = prefix + self.form_class = form_class + if form_kwargs is None: + form_kwargs = {} + self.form_kwargs = {"prefix": prefix, **form_kwargs} + + def add_prefix(self, field_name): + """Add prefix in the same way Django forms add prefixes.""" + return f"{self.prefix}-{field_name}" + + def format_value(self, value): + """Value used in rendering.""" + value = json.loads(value) + if value: + value = {self.add_prefix(k): v for k, v in value.items()} + return self.form_class(data=value, **self.form_kwargs) + + def value_from_datadict(self, data, files, name): + """Parse value from POST request.""" + form = self.form_class(data=data, **self.form_kwargs) + form.full_clean() + return form.cleaned_data diff --git a/tests/hawc/apps/common/dynamic_forms/__init__.py b/tests/hawc/apps/common/dynamic_forms/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/hawc/apps/common/dynamic_forms/conftest.py b/tests/hawc/apps/common/dynamic_forms/conftest.py new file mode 100644 index 0000000000..5dfc7cd350 --- /dev/null +++ b/tests/hawc/apps/common/dynamic_forms/conftest.py @@ -0,0 +1,80 @@ +import pytest + + +@pytest.fixture() +def complete_schema() -> dict: + return { + "fields": [ + { + "name": "textbox", + "type": "char", + "required": False, + "help_text": "Help text", + "css_class": "col-3", + }, + { + "name": "checkbox", + "type": "boolean", + "required": False, + "help_text": "Help text", + "css_class": "col-3", + }, + { + "name": "integer", + "type": "integer", + "required": False, + "help_text": "Help text", + "css_class": "col-3", + }, + { + "name": "float", + "type": "float", + "required": False, + "help_text": "Help text", + "css_class": "col-3", + }, + { + "name": "select", + "type": "choice", + "choices": [["1", "Item 1"], ["2", "Item 2"], ["3", "Item 3"]], + "required": False, + "help_text": "Help text", + "css_class": "col-6", + }, + { + "name": "select_multiple", + "type": "multiple_choice", + "choices": [["1", "Item 1"], ["2", "Item 2"], ["3", "Item 3"]], + "required": False, + "help_text": "Help text", + "css_class": "col-6", + }, + { + "name": "radio", + "type": "choice", + "widget": "radio_select", + "choices": [["1", "Item 1"], ["2", "Item 2"], ["3", "Item 3"]], + "required": False, + "help_text": "Help text", + "css_class": "col-6", + }, + { + "name": "yesno", + "type": "yes_no", + "label": "Yes/no field?", + "required": False, + "help_text": "Help text", + "css_class": "col-6", + }, + { + "name": "checkbox_multiple", + "type": "multiple_choice", + "widget": "checkbox_select_multiple", + "choices": [["1", "Item 1"], ["2", "Item 2"], ["3", "Item 3"]], + "required": False, + "help_text": "Help text", + "css_class": "col-6", + }, + ], + "conditions": [], + } diff --git a/tests/hawc/apps/common/dynamic_forms/test_forms.py b/tests/hawc/apps/common/dynamic_forms/test_forms.py new file mode 100644 index 0000000000..e1444a28c2 --- /dev/null +++ b/tests/hawc/apps/common/dynamic_forms/test_forms.py @@ -0,0 +1,82 @@ +from copy import deepcopy + +from crispy_forms.utils import render_crispy_form +from pytest_django.asserts import assertInHTML + +from hawc.apps.common.dynamic_forms import Schema + + +class TestDynamicForm: + def test_field_rendering(self, complete_schema): + # ensure schema with all field types can render without error + schema = Schema.parse_obj(complete_schema) + form_rendering = render_crispy_form(schema.to_form({})) + assert len(form_rendering) > 0 + + def test_yesno_rendering(self, complete_schema): + # ensure yesno field with inline styles renders as expected + yesno = deepcopy(complete_schema) + yesno["fields"] = [field for field in yesno["fields"] if field["name"] == "yesno"] + schema = Schema.parse_obj(yesno) + form_rendering = render_crispy_form(schema.to_form({})) + expected = """ +
+ +
+
+ + +
+
+ + +
+ Help text +
+
""" + assertInHTML(expected, form_rendering) + + def test_validation(self): + schema_dict = {"fields": [{"name": "integer", "type": "integer", "required": True}]} + schema = Schema.parse_obj(schema_dict) + # check required + form_data = {} + form = schema.to_form(form_data) + assert form.errors == {"integer": ["This field is required."]} + # check types + form_data = {"integer": "text"} + form = schema.to_form(form_data) + assert form.errors == {"integer": ["Enter a whole number."]} + + def test_conditions(self): + schema_dict = { + "fields": [ + {"name": "field1", "type": "char", "required": True}, + {"name": "field2", "type": "char", "required": True}, + {"name": "field3", "type": "char", "required": True}, + ], + "conditions": [ + { + "subject": "field1", + "observers": ["field2", "field3"], + "comparison": "equals", + "comparison_value": "value", + "behavior": "hide", + } + ], + } + schema = Schema.parse_obj(schema_dict) + # required should remain required when shown + form_data = {"field1": "not value"} + form = schema.to_form(form_data) + assert "field2" in form.errors + assert "field3" in form.errors + # required should become optional when hidden + form_data = {"field1": "value"} + form = schema.to_form(form_data) + assert form.is_valid() + # hidden fields are left out of data + form_data = {"field1": "value", "field2": "hidden", "field3": "hidden"} + form = schema.to_form(form_data) + assert form.is_valid() + assert form.cleaned_data == {"field1": "value"} diff --git a/tests/hawc/apps/common/dynamic_forms/test_schemas.py b/tests/hawc/apps/common/dynamic_forms/test_schemas.py new file mode 100644 index 0000000000..3771972aa1 --- /dev/null +++ b/tests/hawc/apps/common/dynamic_forms/test_schemas.py @@ -0,0 +1,82 @@ +import pytest +from pydantic import ValidationError as PydanticError + +from hawc.apps.common.dynamic_forms import Schema + + +class TestSchema: + def test_field_validation(self): + # all fields should have unique names + schema_dict = { + "fields": [ + {"name": "field1", "type": "char"}, + {"name": "field1", "type": "integer"}, + ] + } + with pytest.raises(PydanticError, match="Duplicate field name"): + Schema.parse_obj(schema_dict) + + # all fields should have a valid type + schema_dict = { + "fields": [ + {"name": "field1"}, + ] + } + with pytest.raises(PydanticError, match="Discriminator 'type' is missing in value"): + Schema.parse_obj(schema_dict) + schema_dict = { + "fields": [ + {"name": "field1", "type": "not a type"}, + ] + } + with pytest.raises( + PydanticError, + match="No match for discriminator 'type' and value 'not a type'", + ): + Schema.parse_obj(schema_dict) + + # if a widget is given, it should be valid for the type + schema_dict = { + "fields": [ + {"name": "field1", "type": "integer", "widget": "text_input"}, + ] + } + with pytest.raises(PydanticError, match="unexpected value; permitted: 'number_input'"): + Schema.parse_obj(schema_dict) + + def test_condition_validation(self): + # condition subjects should correspond with a field + schema_dict = { + "fields": [ + {"name": "field1", "type": "char"}, + {"name": "field2", "type": "char"}, + {"name": "field3", "type": "char"}, + ], + "conditions": [ + { + "subject": "not_a_field", + "observers": ["field2", "field3"], + "comparison_value": "value", + } + ], + } + with pytest.raises(PydanticError, match="Invalid condition subject"): + Schema.parse_obj(schema_dict) + + # condition observers should correspond with a field + schema_dict = { + "fields": [ + {"name": "field1", "type": "char"}, + {"name": "field2", "type": "char"}, + {"name": "field3", "type": "char"}, + ], + "conditions": [ + { + "subject": "field1", + "observers": ["not_a_field", "field3"], + "comparison_value": "value", + } + ], + } + with pytest.raises(PydanticError, match="Invalid condition observer"): + Schema.parse_obj(schema_dict) From aa4caba8c414940dcd2101b540fdd44ec0e540c5 Mon Sep 17 00:00:00 2001 From: Andy Shapiro Date: Sun, 17 Sep 2023 07:40:12 -0400 Subject: [PATCH 28/35] fix merge conflicts --- hawc/apps/summary/admin.py | 2 +- hawc/apps/summary/forms.py | 486 ++++++++++++++++-- .../0043_datapivotquery_prefilters_json.py | 92 ---- .../migrations/0044_visual_prefilters_json.py | 89 ---- ...pivotquery_published_only_to_prefilters.py | 38 -- .../0046_visual_studies_to_prefilters.py | 33 -- hawc/apps/summary/models.py | 246 ++++++--- hawc/apps/summary/prefilters.py | 345 ------------- .../templates/summary/datapivot_form.html | 63 ++- hawc/apps/summary/urls.py | 2 +- hawc/apps/summary/views.py | 25 +- .../data/api/api-summary-table-set-data.json | 2 +- .../api/api-visual-bioassay-aggregation.json | 2 +- tests/data/api/api-visual-crossview.json | 2 +- .../data/api/api-visual-embedded-tableau.json | 2 +- .../api/api-visual-exploratory-heatmap.json | 2 +- tests/data/api/api-visual-rob-barchart.json | 2 +- tests/data/api/api-visual-rob-heatmap.json | 2 +- tests/data/api/api-visual-tagtree.json | 2 +- tests/hawc/apps/summary/test_views.py | 2 +- 20 files changed, 690 insertions(+), 749 deletions(-) delete mode 100644 hawc/apps/summary/migrations/0043_datapivotquery_prefilters_json.py delete mode 100644 hawc/apps/summary/migrations/0044_visual_prefilters_json.py delete mode 100644 hawc/apps/summary/migrations/0045_datapivotquery_published_only_to_prefilters.py delete mode 100644 hawc/apps/summary/migrations/0046_visual_studies_to_prefilters.py delete mode 100644 hawc/apps/summary/prefilters.py diff --git a/hawc/apps/summary/admin.py b/hawc/apps/summary/admin.py index a384ffece6..b4b5f15bde 100644 --- a/hawc/apps/summary/admin.py +++ b/hawc/apps/summary/admin.py @@ -22,7 +22,7 @@ class VisualAdmin(admin.ModelAdmin): list_filter = ("visual_type", "published", ("assessment", admin.RelatedOnlyFieldListFilter)) search_fields = ("assessment__name", "title") - raw_id_fields = ("endpoints",) + raw_id_fields = ("endpoints", "studies") @admin.display(description="URL") def show_url(self, obj): diff --git a/hawc/apps/summary/forms.py b/hawc/apps/summary/forms.py index 8d0a52e488..2f3f811e66 100644 --- a/hawc/apps/summary/forms.py +++ b/hawc/apps/summary/forms.py @@ -11,15 +11,437 @@ from ..animal.autocomplete import EndpointAutocomplete from ..animal.forms import MultipleEndpointChoiceField from ..animal.models import Endpoint -from ..assessment.models import DoseUnits +from ..assessment.models import DoseUnits, EffectTag from ..common import validators from ..common.autocomplete import AutocompleteChoiceField -from ..common.forms import BaseFormHelper, DynamicFormField, QuillField, check_unique_for_assessment +from ..common.forms import BaseFormHelper, QuillField, check_unique_for_assessment from ..common.helper import new_window_a from ..common.validators import validate_html_tags, validate_hyperlinks, validate_json_pydantic +from ..epi.models import Outcome +from ..invitro.models import IVChemical, IVEndpointCategory from ..lit.models import ReferenceFilterTag from ..study.autocomplete import StudyAutocomplete -from . import autocomplete, constants, models, prefilters +from ..study.models import Study +from . import autocomplete, constants, models + + +class PrefilterMixin: + PREFILTER_COMBO_FIELDS = [ + "studies", + "systems", + "organs", + "effects", + "effect_subtypes", + "episystems", + "epieffects", + "iv_categories", + "iv_chemicals", + "effect_tags", + ] + + def createFields(self): + fields = dict() + epi_version = self.instance.assessment.epi_version + + if "study" in self.prefilter_include: + fields.update( + [ + ( + "published_only", + forms.BooleanField( + required=False, + initial=True, + label="Published studies only", + help_text="Only present data from studies which have been marked as " + '"published" in HAWC.', + ), + ), + ( + "prefilter_study", + forms.BooleanField( + required=False, + label="Prefilter by study", + help_text="Prefilter endpoints to include only selected studies.", + ), + ), + ( + "studies", + forms.MultipleChoiceField( + required=False, + widget=forms.SelectMultiple, + label="Studies to include", + help_text="""Select one or more studies to include in the plot. + If no study is selected, no endpoints will be available.""", + ), + ), + ] + ) + + if "bioassay" in self.prefilter_include: + fields.update( + [ + ( + "prefilter_system", + forms.BooleanField( + required=False, + label="Prefilter by system", + help_text="Prefilter endpoints on plot to include selected systems.", + ), + ), + ( + "systems", + forms.MultipleChoiceField( + required=False, + widget=forms.SelectMultiple, + label="Systems to include", + help_text="""Select one or more systems to include in the plot. + If no system is selected, no endpoints will be available.""", + ), + ), + ( + "prefilter_organ", + forms.BooleanField( + required=False, + label="Prefilter by organ", + help_text="Prefilter endpoints on plot to include selected organs.", + ), + ), + ( + "organs", + forms.MultipleChoiceField( + required=False, + widget=forms.SelectMultiple, + label="Organs to include", + help_text="""Select one or more organs to include in the plot. + If no organ is selected, no endpoints will be available.""", + ), + ), + ( + "prefilter_effect", + forms.BooleanField( + required=False, + label="Prefilter by effect", + help_text="Prefilter endpoints on plot to include selected effects.", + ), + ), + ( + "effects", + forms.MultipleChoiceField( + required=False, + widget=forms.SelectMultiple, + label="Effects to include", + help_text="""Select one or more effects to include in the plot. + If no effect is selected, no endpoints will be available.""", + ), + ), + ( + "prefilter_effect_subtype", + forms.BooleanField( + required=False, + label="Prefilter by effect sub-type", + help_text="Prefilter endpoints on plot to include selected effects.", + ), + ), + ( + "effect_subtypes", + forms.MultipleChoiceField( + required=False, + widget=forms.SelectMultiple, + label="Effect Sub-Types to include", + help_text="""Select one or more effect sub-types to include in the plot. + If no effect sub-type is selected, no endpoints will be available.""", + ), + ), + ] + ) + + if "epi" in self.prefilter_include and epi_version == 1: + fields.update( + [ + ( + "prefilter_episystem", + forms.BooleanField( + required=False, + label="Prefilter by system", + help_text="Prefilter endpoints on plot to include selected systems.", + ), + ), + ( + "episystems", + forms.MultipleChoiceField( + required=False, + widget=forms.SelectMultiple, + label="Systems to include", + help_text="""Select one or more systems to include in the plot. + If no system is selected, no endpoints will be available.""", + ), + ), + ( + "prefilter_epieffect", + forms.BooleanField( + required=False, + label="Prefilter by effect", + help_text="Prefilter endpoints on plot to include selected effects.", + ), + ), + ( + "epieffects", + forms.MultipleChoiceField( + required=False, + widget=forms.SelectMultiple, + label="Effects to include", + help_text="""Select one or more effects to include in the plot. + If no effect is selected, no endpoints will be available.""", + ), + ), + ] + ) + + if "invitro" in self.prefilter_include: + fields.update( + [ + ( + "prefilter_iv_category", + forms.BooleanField( + required=False, + label="Prefilter by category", + help_text="Prefilter endpoints to include only selected category.", + ), + ), + ( + "iv_categories", + forms.MultipleChoiceField( + required=False, + widget=forms.SelectMultiple, + label="Categories to include", + help_text="""Select one or more categories to include in the plot. + If no study is selected, no endpoints will be available.""", + ), + ), + ( + "prefilter_iv_chemical", + forms.BooleanField( + required=False, + label="Prefilter by chemical", + help_text="Prefilter endpoints to include only selected chemicals.", + ), + ), + ( + "iv_chemicals", + forms.MultipleChoiceField( + required=False, + widget=forms.SelectMultiple, + label="Chemicals to include", + help_text="""Select one or more chemicals to include in the plot. + If no study is selected, no endpoints will be available.""", + ), + ), + ] + ) + + if "effect_tags" in self.prefilter_include: + fields.update( + [ + ( + "prefilter_effect_tag", + forms.BooleanField( + required=False, + label="Prefilter by effect-tag", + help_text="Prefilter endpoints to include only selected effect-tags.", + ), + ), + ( + "effect_tags", + forms.MultipleChoiceField( + required=False, + widget=forms.SelectMultiple, + label="Tags to include", + help_text="""Select one or more effect-tags to include in the plot. + If no study is selected, no endpoints will be available.""", + ), + ), + ] + ) + + for k, v in fields.items(): + self.fields[k] = v + + def setInitialValues(self): + is_new = self.initial == {} + try: + prefilters = json.loads(self.initial.get("prefilters", "{}")) + except ValueError: + prefilters = {} + + if type(self.instance) is models.Visual: + evidence_type = constants.StudyType.BIOASSAY + else: + evidence_type = self.initial.get("evidence_type") or self.instance.evidence_type + for k, v in prefilters.items(): + if k == "system__in": + if evidence_type == constants.StudyType.BIOASSAY: + self.fields["prefilter_system"].initial = True + self.fields["systems"].initial = v + elif evidence_type == constants.StudyType.EPI: + self.fields["prefilter_episystem"].initial = True + self.fields["episystems"].initial = v + + if k == "organ__in": + self.fields["prefilter_organ"].initial = True + self.fields["organs"].initial = v + + if k == "effect__in": + if evidence_type == constants.StudyType.BIOASSAY: + self.fields["prefilter_effect"].initial = True + self.fields["effects"].initial = v + elif evidence_type == constants.StudyType.EPI: + self.fields["prefilter_epieffect"].initial = True + self.fields["epieffects"].initial = v + + if k == "effect_subtype__in": + self.fields["prefilter_effect_subtype"].initial = True + self.fields["effect_subtypes"].initial = v + + if k == "effects__in": + self.fields["prefilter_effect_tag"].initial = True + self.fields["effect_tags"].initial = v + + if k == "category__in": + self.fields["prefilter_iv_category"].initial = True + self.fields["iv_categories"].initial = v + + if k == "chemical__name__in": + self.fields["prefilter_iv_chemical"].initial = True + self.fields["iv_chemicals"].initial = v + + if k in [ + "animal_group__experiment__study__in", + "study_population__study__in", + "experiment__study__in", + "protocol__study__in", + "design__study__in", + ]: + self.fields["prefilter_study"].initial = True + self.fields["studies"].initial = v + + if self.__class__.__name__ == "CrossviewForm": + published_only = prefilters.get("animal_group__experiment__study__published", False) + if is_new: + published_only = True + self.fields["published_only"].initial = published_only + + for fldname in self.PREFILTER_COMBO_FIELDS: + field = self.fields.get(fldname) + if field: + field.choices = self.getPrefilterQueryset(fldname) + + def getPrefilterQueryset(self, field_name): + assessment_id = self.instance.assessment_id + choices = None + + if field_name == "systems": + choices = Endpoint.objects.get_system_choices(assessment_id) + elif field_name == "organs": + choices = Endpoint.objects.get_organ_choices(assessment_id) + elif field_name == "effects": + choices = Endpoint.objects.get_effect_choices(assessment_id) + elif field_name == "effect_subtypes": + choices = Endpoint.objects.get_effect_subtype_choices(assessment_id) + elif field_name == "iv_categories": + choices = IVEndpointCategory.get_choices(assessment_id) + elif field_name == "iv_chemicals": + choices = IVChemical.objects.get_choices(assessment_id) + elif field_name == "effect_tags": + choices = EffectTag.objects.get_choices(assessment_id) + elif field_name == "studies": + choices = Study.objects.get_choices(assessment_id) + elif field_name == "episystems": + choices = Outcome.objects.get_system_choices(assessment_id) + elif field_name == "epieffects": + choices = Outcome.objects.get_effect_choices(assessment_id) + else: + raise ValueError(f"Unknown field name: {field_name}") + + return choices + + def setFieldStyles(self): + if self.fields.get("prefilters"): + self.fields["prefilters"].widget = forms.HiddenInput() + + for fldname in self.PREFILTER_COMBO_FIELDS: + field = self.fields.get(fldname) + if field: + field.widget.attrs["size"] = 10 + + def setPrefilters(self, data): + prefilters = {} + epi_version = self.instance.assessment.epi_version + + if data.get("prefilter_study") is True: + studies = data.get("studies", []) + + evidence_type = data.get("evidence_type", None) + if self.__class__.__name__ == "CrossviewForm": + evidence_type = 0 + + if evidence_type == constants.StudyType.BIOASSAY: + prefilters["animal_group__experiment__study__in"] = studies + elif evidence_type == constants.StudyType.IN_VITRO: + prefilters["experiment__study__in"] = studies + elif evidence_type == constants.StudyType.EPI: + if epi_version == 1: + prefilters["study_population__study__in"] = studies + elif epi_version == 2: + prefilters["design__study__in"] = studies + else: + raise ValueError("Invalid epi_version") + elif evidence_type == constants.StudyType.EPI_META: + prefilters["protocol__study__in"] = studies + else: + raise ValueError("Unknown evidence type") + + if data.get("prefilter_system") is True: + prefilters["system__in"] = data.get("systems", []) + + if data.get("prefilter_organ") is True: + prefilters["organ__in"] = data.get("organs", []) + + if data.get("prefilter_effect") is True: + prefilters["effect__in"] = data.get("effects", []) + + if data.get("prefilter_effect_subtype") is True: + prefilters["effect_subtype__in"] = data.get("effect_subtypes", []) + + if data.get("prefilter_episystem") is True: + prefilters["system__in"] = data.get("episystems", []) + + if data.get("prefilter_epieffect") is True: + prefilters["effect__in"] = data.get("epieffects", []) + + if data.get("prefilter_iv_category") is True: + prefilters["category__in"] = data.get("iv_categories", []) + + if data.get("prefilter_iv_chemical") is True: + prefilters["chemical__name__in"] = data.get("iv_chemicals", []) + + if data.get("prefilter_effect_tag") is True: + prefilters["effects__in"] = data.get("effect_tags", []) + + if self.__class__.__name__ == "CrossviewForm" and data.get("published_only") is True: + prefilters["animal_group__experiment__study__published"] = True + + return json.dumps(prefilters) + + def clean(self): + cleaned_data = super().clean() + cleaned_data["prefilters"] = self.setPrefilters(cleaned_data) + return cleaned_data + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.createFields() + self.setInitialValues() + self.setFieldStyles() class SummaryTextForm(forms.ModelForm): @@ -255,24 +677,14 @@ class Meta: exclude = ("assessment", "visual_type", "settings", "prefilters", "studies", "sort_order") -class CrossviewForm(VisualForm): - def _get_prefilter_form(self, data, **form_kwargs): - prefix = form_kwargs.pop("prefix", None) - return self.prefilter( - data=data, prefix=prefix, assessment=self.instance.assessment, form_kwargs=form_kwargs - ).form +class CrossviewForm(PrefilterMixin, VisualForm): + prefilter_include = ("study", "bioassay", "effect_tags") def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.fields["dose_units"].queryset = DoseUnits.objects.get_animal_units( self.instance.assessment ) - self.prefilter = prefilters.VisualTypePrefilter.from_visual_type( - constants.VisualType.BIOASSAY_CROSSVIEW - ).value - self.fields["prefilters"] = DynamicFormField( - prefix="prefilters", form_class=self._get_prefilter_form, label="" - ) self.helper = self.setHelper() class Meta: @@ -280,20 +692,13 @@ class Meta: exclude = ("assessment", "visual_type", "endpoints", "studies") -class RoBForm(VisualForm): - def _get_prefilter_form(self, data, **form_kwargs): - prefix = form_kwargs.pop("prefix", None) - return self.prefilter( - data=data, prefix=prefix, assessment=self.instance.assessment, form_kwargs=form_kwargs - ).form +class RoBForm(PrefilterMixin, VisualForm): + prefilter_include = ("bioassay",) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.prefilter = prefilters.VisualTypePrefilter.from_visual_type( - constants.VisualType.ROB_BARCHART - ).value - self.fields["prefilters"] = DynamicFormField( - prefix="prefilters", form_class=self._get_prefilter_form, label="" + self.fields["studies"].queryset = self.fields["studies"].queryset.filter( + assessment=self.instance.assessment ) self.helper = self.setHelper() @@ -683,7 +1088,9 @@ def clean(self): self.add_error("excel_file", "Must contain at least 2 columns.") -class DataPivotQueryForm(DataPivotForm): +class DataPivotQueryForm(PrefilterMixin, DataPivotForm): + prefilter_include = ("study", "bioassay", "epi", "invitro", "eco", "effect_tags") + class Meta: model = models.DataPivotQuery fields = ( @@ -695,29 +1102,18 @@ class Meta: "settings", "caption", "published", + "published_only", "prefilters", ) - def _get_prefilter_form(self, data, **form_kwargs): - prefix = form_kwargs.pop("prefix", None) - return self.prefilter( - data=data, prefix=prefix, assessment=self.instance.assessment, form_kwargs=form_kwargs - ).form - def __init__(self, *args, **kwargs): - evidence_type = kwargs.pop("evidence_type", None) super().__init__(*args, **kwargs) - - if evidence_type is not None: - self.instance.evidence_type = evidence_type - self.fields["evidence_type"].initial = self.instance.evidence_type - self.fields["evidence_type"].disabled = True - - self.prefilter = prefilters.StudyTypePrefilter.from_study_type( - self.instance.evidence_type, self.instance.assessment - ).value - self.fields["prefilters"] = DynamicFormField( - prefix="prefilters", form_class=self._get_prefilter_form, label="" + self.fields["evidence_type"].choices = ( + (constants.StudyType.BIOASSAY, "Animal Bioassay"), + (constants.StudyType.EPI, "Epidemiology"), + (constants.StudyType.EPI_META, "Epidemiology meta-analysis/pooled analysis"), + (constants.StudyType.IN_VITRO, "In vitro"), + (constants.StudyType.ECO, "Ecology"), ) self.fields["preferred_units"].required = False self.js_units_choices = json.dumps( diff --git a/hawc/apps/summary/migrations/0043_datapivotquery_prefilters_json.py b/hawc/apps/summary/migrations/0043_datapivotquery_prefilters_json.py deleted file mode 100644 index 9aa68008c2..0000000000 --- a/hawc/apps/summary/migrations/0043_datapivotquery_prefilters_json.py +++ /dev/null @@ -1,92 +0,0 @@ -# Generated by Django 4.2.3 on 2023-08-21 03:49 -import json - -from django.db import migrations, models - -from hawc.apps.summary.prefilters import StudyTypePrefilter - -mapping = { - StudyTypePrefilter.BIOASSAY:{ - "published_only":"animal_group__experiment__study__published", - "studies":"animal_group__experiment__study__in", - "systems":"system__in", - "organs":"organ__in", - "effects":"effect__in", - "effect_subtypes":"effect_subtype__in", - "effect_tags":"effects__in" - }, - StudyTypePrefilter.EPIV1:{ - "published_only":"study_population__study__published", - "studies":"study_population__study__in", - "systems":"system__in", - "effects":"effect__in", - "effect_tags":"effects__in" - }, - StudyTypePrefilter.EPIV2:{ - "published_only":"design__study__published", - "studies":"design__study__in" - }, - StudyTypePrefilter.EPI_META:{ - "published_only":"protocol__study__published", - "studies":"protocol__study__in" - }, - StudyTypePrefilter.IN_VITRO:{ - "published_only":"experiment__study__published", - "studies":"experiment__study__in", - "categories":"category__in", - "chemicals":"chemical__name__in", - "effect_tags":"effects__in" - } -} - -def prefilters_dict(apps, schema_editor): - # load prefilters textfield into temp jsonfield - DataPivotQuery = apps.get_model("summary", "DataPivotQuery") - objs = DataPivotQuery.objects.all().select_related("assessment") - for obj in objs: - data = json.loads(obj.prefilters) - if not data: - continue - key_map = mapping[StudyTypePrefilter.from_study_type(obj.evidence_type,obj.assessment)] - key_map = {v:k for k,v in key_map.items()} - data = {key_map[k]:v for k,v in data.items()} - obj.temp = data - DataPivotQuery.objects.bulk_update(objs, ["temp"]) - - -def reverse_prefilters_dict(apps, schema_editor): - # dump temp jsonfield into prefilters textfield - DataPivotQuery = apps.get_model("summary", "DataPivotQuery") - objs = DataPivotQuery.objects.all().select_related("assessment") - for obj in objs: - if not obj.temp: - continue - key_map = mapping[StudyTypePrefilter.from_study_type(obj.evidence_type,obj.assessment)] - data = {key_map[k]:v for k,v in obj.temp.items()} - obj.prefilters = json.dumps(data) - DataPivotQuery.objects.bulk_update(objs, ["prefilters"]) - - -class Migration(migrations.Migration): - dependencies = [ - ("summary", "0042_summarytable_interactive"), - ] - - operations = [ - # change prefilters textfield into jsonfield - migrations.AddField( - model_name="datapivotquery", - name="temp", - field=models.JSONField(default=dict), - ), - migrations.RunPython(prefilters_dict, reverse_code=reverse_prefilters_dict), - migrations.RemoveField( - model_name="datapivotquery", - name="prefilters", - ), - migrations.RenameField( - model_name="datapivotquery", - old_name="temp", - new_name="prefilters", - ), - ] diff --git a/hawc/apps/summary/migrations/0044_visual_prefilters_json.py b/hawc/apps/summary/migrations/0044_visual_prefilters_json.py deleted file mode 100644 index 99314938b6..0000000000 --- a/hawc/apps/summary/migrations/0044_visual_prefilters_json.py +++ /dev/null @@ -1,89 +0,0 @@ -# Generated by Django 4.2.3 on 2023-08-21 03:49 -import json - -from django.db import migrations, models - -from hawc.apps.summary.prefilters import VisualTypePrefilter - -mapping = { - VisualTypePrefilter.BIOASSAY_CROSSVIEW: { - "published_only": "animal_group__experiment__study__published", - "studies": "animal_group__experiment__study__in", - "systems": "system__in", - "organs": "organ__in", - "effects": "effect__in", - "effect_subtypes": "effect_subtype__in", - "effect_tags": "effects__in", - }, - VisualTypePrefilter.ROB_HEATMAP: { - "published_only": "animal_group__experiment__study__published", - "studies": "animal_group__experiment__study__in", - "systems": "system__in", - "organs": "organ__in", - "effects": "effect__in", - "effect_subtypes": "effect_subtype__in", - "effect_tags": "effects__in", - }, - VisualTypePrefilter.ROB_BARCHART: { - "published_only": "animal_group__experiment__study__published", - "studies": "animal_group__experiment__study__in", - "systems": "system__in", - "organs": "organ__in", - "effects": "effect__in", - "effect_subtypes": "effect_subtype__in", - "effect_tags": "effects__in", - }, -} - - -def prefilters_dict(apps, schema_editor): - # load prefilters textfield into temp jsonfield - Visual = apps.get_model("summary", "Visual") - objs = Visual.objects.all() - for obj in objs: - data = json.loads(obj.prefilters) - if not data: - continue - key_map = mapping[VisualTypePrefilter.from_visual_type(obj.visual_type)] - key_map = {v: k for k, v in key_map.items()} - data = {key_map[k]: v for k, v in data.items()} - obj.temp = data - Visual.objects.bulk_update(objs, ["temp"]) - - -def reverse_prefilters_dict(apps, schema_editor): - # dump temp jsonfield into prefilters textfield - Visual = apps.get_model("summary", "Visual") - objs = Visual.objects.all() - for obj in objs: - if not obj.temp: - continue - key_map = mapping[VisualTypePrefilter.from_visual_type(obj.visual_type)] - data = {key_map[k]: v for k, v in obj.temp.items()} - obj.prefilters = json.dumps(data) - Visual.objects.bulk_update(objs, ["prefilters"]) - - -class Migration(migrations.Migration): - dependencies = [ - ("summary", "0043_datapivotquery_prefilters_json"), - ] - - operations = [ - # change prefilters textfield into jsonfield - migrations.AddField( - model_name="visual", - name="temp", - field=models.JSONField(default=dict), - ), - migrations.RunPython(prefilters_dict, reverse_code=reverse_prefilters_dict), - migrations.RemoveField( - model_name="visual", - name="prefilters", - ), - migrations.RenameField( - model_name="visual", - old_name="temp", - new_name="prefilters", - ), - ] diff --git a/hawc/apps/summary/migrations/0045_datapivotquery_published_only_to_prefilters.py b/hawc/apps/summary/migrations/0045_datapivotquery_published_only_to_prefilters.py deleted file mode 100644 index 25612b5b69..0000000000 --- a/hawc/apps/summary/migrations/0045_datapivotquery_published_only_to_prefilters.py +++ /dev/null @@ -1,38 +0,0 @@ -# Generated by Django 4.2.3 on 2023-08-21 03:49 - - -from django.db import migrations - - -def published_only_prefilters(apps, schema_editor): - # add published_only field to prefilters - DataPivotQuery = apps.get_model("summary", "DataPivotQuery") - objs = DataPivotQuery.objects.all() - for obj in objs: - obj.prefilters["published_only"] = obj.published_only - DataPivotQuery.objects.bulk_update(objs, ["prefilters"]) - - -def reverse_published_only_prefilters(apps, schema_editor): - # separate published_only field from prefilters - DataPivotQuery = apps.get_model("summary", "DataPivotQuery") - objs = DataPivotQuery.objects.all() - for obj in objs: - obj.published_only = obj.prefilters["published_only"] - DataPivotQuery.objects.bulk_update(objs, ["published_only"]) - - -class Migration(migrations.Migration): - dependencies = [ - ("summary", "0044_visual_prefilters_json"), - ] - - operations = [ - migrations.RunPython( - published_only_prefilters, reverse_code=reverse_published_only_prefilters - ), - migrations.RemoveField( - model_name="datapivotquery", - name="published_only", - ), - ] diff --git a/hawc/apps/summary/migrations/0046_visual_studies_to_prefilters.py b/hawc/apps/summary/migrations/0046_visual_studies_to_prefilters.py deleted file mode 100644 index ed0e18bfb9..0000000000 --- a/hawc/apps/summary/migrations/0046_visual_studies_to_prefilters.py +++ /dev/null @@ -1,33 +0,0 @@ -# Generated by Django 4.2.3 on 2023-08-22 06:47 - -from django.db import migrations - - -def studies_prefilters(apps, schema_editor): - # add studies field to prefilters - Visual = apps.get_model("summary", "Visual") - objs = Visual.objects.all().prefetch_related("studies") - for obj in objs: - obj.prefilters["studies"] = list(obj.studies.all().values_list("pk", flat=True)) - Visual.objects.bulk_update(objs, ["prefilters"]) - - -def reverse_studies_prefilters(apps, schema_editor): - # separate studies field from prefilters - # TODO easiest way is probably to get the through table - # and create records there using pks - return - - -class Migration(migrations.Migration): - dependencies = [ - ("summary", "0045_datapivotquery_published_only_to_prefilters"), - ] - - operations = [ - migrations.RunPython(studies_prefilters, reverse_code=reverse_studies_prefilters), - migrations.RemoveField( - model_name="visual", - name="studies", - ), - ] diff --git a/hawc/apps/summary/models.py b/hawc/apps/summary/models.py index 638c564993..1724c42c7b 100644 --- a/hawc/apps/summary/models.py +++ b/hawc/apps/summary/models.py @@ -37,14 +37,19 @@ from ..common.models import get_model_copy_name from ..common.validators import validate_html_tags, validate_hyperlinks from ..eco.exports import EcoFlatComplete +from ..eco.models import Result from ..epi.exports import OutcomeDataPivot +from ..epi.models import Outcome from ..epimeta.exports import MetaResultFlatDataPivot +from ..epimeta.models import MetaResult from ..epiv2.exports import EpiFlatComplete +from ..epiv2.models import DataExtraction from ..invitro import exports as ivexports +from ..invitro.models import IVEndpoint from ..riskofbias.models import RiskOfBiasScore from ..riskofbias.serializers import AssessmentRiskOfBiasSerializer from ..study.models import Study -from . import constants, managers, prefilters +from . import constants, managers logger = logging.getLogger(__name__) @@ -285,13 +290,19 @@ class Visual(models.Model): assessment = models.ForeignKey(Assessment, on_delete=models.CASCADE, related_name="visuals") visual_type = models.PositiveSmallIntegerField(choices=constants.VisualType.choices) dose_units = models.ForeignKey(DoseUnits, on_delete=models.SET_NULL, blank=True, null=True) - prefilters = models.JSONField(default=dict) + prefilters = models.TextField(default="{}") endpoints = models.ManyToManyField( BaseEndpoint, related_name="visuals", help_text="Endpoints to be included in visualization", blank=True, ) + studies = models.ManyToManyField( + Study, + related_name="visuals", + help_text="Studies to be included in visualization", + blank=True, + ) settings = models.TextField(default="{}") caption = models.TextField(blank=True) published = models.BooleanField( @@ -458,24 +469,9 @@ def get_settings(self) -> dict | None: def get_json(self, json_encode=True): return SerializerHelper.get_serialized(self, json=json_encode) - def get_filterset_class(self): - return prefilters.VisualTypePrefilter.from_visual_type(self.visual_type).value - - def get_filterset(self, data, assessment, **kwargs): - return self.get_filterset_class()(data=data, assessment=assessment, **kwargs) - - def get_request_prefilters(self, request): - # find all keys that start with "prefilters-" prefix - prefix = "prefilters-" - return { - key[len(prefix) :]: value - for key, value in request.POST.lists() - if key.startswith(prefix) - } - def get_endpoints(self, request=None): qs = Endpoint.objects.none() - filters = {} + filters = {"assessment_id": self.assessment_id} if self.visual_type == constants.VisualType.BIOASSAY_AGGREGATION: if request: @@ -487,14 +483,18 @@ def get_endpoints(self, request=None): qs = Endpoint.objects.filter(**filters) elif self.visual_type == constants.VisualType.BIOASSAY_CROSSVIEW: - dose_id = ( - tryParseInt(request.POST.get("dose_units"), -1) if request else self.dose_units_id - ) - prefilters = self.get_request_prefilters(request) if request else self.prefilters - fs = self.get_filterset(prefilters, self.assessment) + if request: + dose_id = tryParseInt(request.POST.get("dose_units"), -1) + Prefilter.setFiltersFromForm( + self.assessment, filters, request.POST, self.visual_type + ) + + else: + dose_id = self.dose_units_id + Prefilter.setFiltersFromObj(filters, self.prefilters) filters["animal_group__dosing_regime__doses__dose_units_id"] = dose_id - qs = fs.qs.filter(**filters).distinct("id") + qs = Endpoint.objects.filter(**filters).distinct("id") return qs @@ -505,37 +505,40 @@ def get_studies(self, request=None): to the model. """ qs = Study.objects.none() - filters = {} + filters = {"assessment_id": self.assessment_id} if self.visual_type in [ constants.VisualType.ROB_HEATMAP, constants.VisualType.ROB_BARCHART, ]: - prefilters = self.get_request_prefilters(request) if request else self.prefilters - fs = self.get_filterset(prefilters, self.assessment) - fs.form.is_valid() - cleaned_prefilters = fs.form.cleaned_data - - study_fields = [ - "published_only", - "studies", - ] - endpoint_prefilters = { - k: v for k, v in cleaned_prefilters.items() if k not in study_fields - } - - if any(value for value in endpoint_prefilters.values()): - endpoint_qs = fs.qs - filters["id__in"] = set( - endpoint_qs.values_list("animal_group__experiment__study_id", flat=True) + if request: + efilters = {"assessment_id": self.assessment_id} + Prefilter.setFiltersFromForm( + self.assessment, efilters, request.POST, self.visual_type ) - else: - if f := cleaned_prefilters.pop(study_fields[0], False): - filters["published"] = f - if f := cleaned_prefilters.get(study_fields[1], []): - filters["id__in"] = f + if len(efilters) > 1: + filters["id__in"] = set( + Endpoint.objects.filter(**efilters).values_list( + "animal_group__experiment__study_id", flat=True + ) + ) + else: + filters["id__in"] = request.POST.getlist("studies") - qs = Study.objects.filter(**filters) + qs = Study.objects.filter(**filters) + + else: + if self.prefilters != "{}": + efilters = {"assessment_id": self.assessment_id} + Prefilter.setFiltersFromObj(efilters, self.prefilters) + filters["id__in"] = set( + Endpoint.objects.filter(**efilters).values_list( + "animal_group__experiment__study_id", flat=True + ) + ) + qs = Study.objects.filter(**filters) + else: + qs = self.studies.all() if self.sort_order: if self.sort_order == "overall_confidence": @@ -559,7 +562,6 @@ def get_editing_dataset(self, request): return { "assessment": self.assessment_id, - "assessment_rob_name": self.assessment.get_rob_name_display(), "title": request.POST.get("title"), "slug": request.POST.get("slug"), "caption": request.POST.get("caption"), @@ -778,7 +780,13 @@ class DataPivotQuery(DataPivot): "percent-response, where dose-units are not needed, or for " "creating one plot similar, but not identical, dose-units.", ) - prefilters = models.JSONField(default=dict) + prefilters = models.TextField(default="{}") + published_only = models.BooleanField( + default=True, + verbose_name="Published studies only", + help_text="Only present data from studies which have been marked as " + '"published" in HAWC.', + ) def clean(self): count = self.get_queryset().count() @@ -798,17 +806,62 @@ def clean(self): """ raise ValidationError(err) - def _refine_queryset(self, qs): + def _get_dataset_filters(self): + filters = {} + epi_version = self.assessment.epi_version + if self.evidence_type == constants.StudyType.BIOASSAY: + filters["assessment_id"] = self.assessment_id + if self.published_only: + filters["animal_group__experiment__study__published"] = True if self.preferred_units: - qs = qs.filter( - animal_group__dosing_regime__doses__dose_units__in=self.preferred_units - ) + filters["animal_group__dosing_regime__doses__dose_units__in"] = self.preferred_units + + elif self.evidence_type == constants.StudyType.EPI and epi_version == EpiVersion.V1: + filters["assessment_id"] = self.assessment_id + if self.published_only: + filters["study_population__study__published"] = True + + elif self.evidence_type == constants.StudyType.EPI and epi_version == EpiVersion.V2: + filters["design__study__assessment_id"] = self.assessment_id + if self.published_only: + filters["design__study__published"] = True + + elif self.evidence_type == constants.StudyType.EPI_META: + filters["protocol__study__assessment_id"] = self.assessment_id + if self.published_only: + filters["protocol__study__published"] = True + + elif self.evidence_type == constants.StudyType.IN_VITRO: + filters["assessment_id"] = self.assessment_id + if self.published_only: + filters["experiment__study__published"] = True elif self.evidence_type == constants.StudyType.ECO: - qs = qs.filter(design__study__assessment_id=self.assessment_id) + filters["design__study__assessment_id"] = self.assessment_id + if self.published_only: + filters["design__study__published"] = True - return qs + Prefilter.setFiltersFromObj(filters, self.prefilters) + return filters + + def _get_dataset_queryset(self, filters): + epi_version = self.assessment.epi_version + if self.evidence_type == constants.StudyType.BIOASSAY: + qs = Endpoint.objects.filter(**filters) + elif self.evidence_type == constants.StudyType.EPI and epi_version == EpiVersion.V1: + qs = Outcome.objects.filter(**filters) + elif self.evidence_type == constants.StudyType.EPI and epi_version == EpiVersion.V2: + qs = DataExtraction.objects.filter(**filters) + elif self.evidence_type == constants.StudyType.EPI_META: + qs = MetaResult.objects.filter(**filters) + elif self.evidence_type == constants.StudyType.IN_VITRO: + qs = IVEndpoint.objects.filter(**filters) + elif self.evidence_type == constants.StudyType.ECO: + qs = Result.objects.filter(**filters) + else: + raise ValueError("Invalid data type") + return qs.order_by("id") def _get_dataset_exporter(self, qs): if self.evidence_type == constants.StudyType.BIOASSAY: @@ -869,18 +922,9 @@ def _get_dataset_exporter(self, qs): return exporter - def get_filterset_class(self): - return prefilters.StudyTypePrefilter.from_study_type( - self.evidence_type, self.assessment - ).value - - def get_filterset(self, data, assessment, **kwargs): - return self.get_filterset_class()(data=data, assessment=assessment, **kwargs) - def get_queryset(self): - qs = self.get_filterset(self.prefilters, self.assessment).qs - qs = self._refine_queryset(qs) - return qs.order_by("id") + filters = self._get_dataset_filters() + return self._get_dataset_queryset(filters) def get_dataset(self) -> FlatExport: qs = self.get_queryset() @@ -939,6 +983,74 @@ def _update_settings_across_assessments(self, cw: dict) -> str: return json.dumps(settings) +class Prefilter: + """ + Helper-object to deal with DataPivot and Visual prefilters fields. + """ + + @staticmethod + def setFiltersFromForm(assessment, filters, d, visual_type): + evidence_type = d.get("evidence_type") + epi_version = assessment.epi_version + + if visual_type == constants.VisualType.BIOASSAY_CROSSVIEW: + evidence_type = constants.StudyType.BIOASSAY + + if d.get("prefilter_system"): + filters["system__in"] = d.getlist("systems") + + if d.get("prefilter_organ"): + filters["organ__in"] = d.getlist("organs") + + if d.get("prefilter_effect"): + filters["effect__in"] = d.getlist("effects") + + if d.get("prefilter_effect_subtype"): + filters["effect_subtype__in"] = d.getlist("effect_subtypes") + + if d.get("prefilter_effect_tag"): + filters["effects__in"] = d.getlist("effect_tags") + + if d.get("prefilter_episystem"): + filters["system__in"] = d.getlist("episystems") + + if d.get("prefilter_epieffect"): + filters["effect__in"] = d.getlist("epieffects") + + if d.get("prefilter_study"): + studies = d.getlist("studies", []) + if evidence_type == constants.StudyType.BIOASSAY: + filters["animal_group__experiment__study__in"] = studies + elif evidence_type == constants.StudyType.EPI and epi_version == 1: + filters["study_population__study__in"] = studies + elif evidence_type == constants.StudyType.EPI and epi_version == 2: + filters["design__study__in"] = studies + elif evidence_type == constants.StudyType.IN_VITRO: + filters["experiment__study__in"] = studies + elif evidence_type == constants.StudyType.EPI_META: + filters["protocol__study__in"] = studies + else: + raise ValueError("Unknown evidence type") + + if d.get("published_only"): + if evidence_type == constants.StudyType.BIOASSAY: + filters["animal_group__experiment__study__published"] = True + elif evidence_type == constants.StudyType.EPI and epi_version == 1: + filters["study_population__study__published"] = True + elif evidence_type == constants.StudyType.EPI and epi_version == 2: + filters["design__study__published"] = True + elif evidence_type == constants.StudyType.IN_VITRO: + filters["experiment__study__published"] = True + elif evidence_type == constants.StudyType.EPI_META: + filters["protocol__study__published"] = True + else: + raise ValueError("Unknown evidence type") + + @staticmethod + def setFiltersFromObj(filters, prefilters): + filters.update(json.loads(prefilters)) + + reversion.register(SummaryText) reversion.register(SummaryTable) reversion.register(DataPivot) diff --git a/hawc/apps/summary/prefilters.py b/hawc/apps/summary/prefilters.py deleted file mode 100644 index f77b6270f6..0000000000 --- a/hawc/apps/summary/prefilters.py +++ /dev/null @@ -1,345 +0,0 @@ -from enum import Enum - -import django_filters as df -from django import forms -from django.forms.widgets import CheckboxInput - -from ..animal.models import Endpoint -from ..assessment.constants import EpiVersion -from ..assessment.models import Assessment, EffectTag -from ..common.filterset import BaseFilterSet -from ..common.forms import BaseFormHelper -from ..epi.models import Outcome -from ..epimeta.models import MetaResult -from ..epiv2.models import DataExtraction -from ..invitro.models import IVChemical, IVEndpoint, IVEndpointCategory -from ..study.models import Study -from .constants import StudyType, VisualType - - -class TestForm(forms.Form): - @property - def helper(self): - helper = BaseFormHelper(self) - helper.form_tag = False - - return helper - - -class BioassayPrefilter(BaseFilterSet): - # studies - published_only = df.BooleanFilter( - method="filter_published_only", - widget=CheckboxInput(), - label="Published studies only", - help_text="Only present data from studies which have been marked as " - '"published" in HAWC.', - ) - studies = df.MultipleChoiceFilter( - field_name="animal_group__experiment__study", - label="Studies to include", - help_text="""Select one or more studies to include in the plot. - If no study is selected, no endpoints will be available.""", - ) - # bioassay - systems = df.MultipleChoiceFilter( - field_name="system", - label="Systems to include", - help_text="""Select one or more systems to include in the plot. - If no system is selected, no endpoints will be available.""", - ) - organs = df.MultipleChoiceFilter( - field_name="organ", - label="Organs to include", - help_text="""Select one or more organs to include in the plot. - If no organ is selected, no endpoints will be available.""", - ) - effects = df.MultipleChoiceFilter( - field_name="effect", - label="Effects to include", - help_text="""Select one or more effects to include in the plot. - If no effect is selected, no endpoints will be available.""", - ) - effect_subtypes = df.MultipleChoiceFilter( - field_name="effect_subtype", - label="Effect Sub-Types to include", - help_text="""Select one or more effect sub-types to include in the plot. - If no effect sub-type is selected, no endpoints will be available.""", - ) - effect_tags = df.MultipleChoiceFilter( - field_name="effects", - label="Tags to include", - help_text="""Select one or more effect-tags to include in the plot. - If no study is selected, no endpoints will be available.""", - ) - - class Meta: - model = Endpoint - fields = [ - "published_only", - "studies", - "systems", - "organs", - "effects", - "effect_subtypes", - "effect_tags", - ] - form = TestForm - - def filter_published_only(self, queryset, name, value): - if not value: - return queryset - return queryset.filter(animal_group__experiment__study__published=True) - - def filter_queryset(self, queryset): - queryset = queryset.filter(assessment_id=self.assessment.pk) - return super().filter_queryset(queryset) - - def create_form(self): - form = super().create_form() - form.fields["studies"].choices = Study.objects.get_choices(self.assessment.pk) - form.fields["systems"].choices = Endpoint.objects.get_system_choices(self.assessment.pk) - form.fields["organs"].choices = Endpoint.objects.get_organ_choices(self.assessment.pk) - form.fields["effects"].choices = Endpoint.objects.get_effect_choices(self.assessment.pk) - form.fields["effect_subtypes"].choices = Endpoint.objects.get_effect_subtype_choices( - self.assessment.pk - ) - form.fields["effect_tags"].choices = EffectTag.objects.get_choices(self.assessment.pk) - return form - - -class EpiV1Prefilter(BaseFilterSet): - # studies - published_only = df.BooleanFilter( - method="filter_published_only", - widget=CheckboxInput(), - label="Published studies only", - help_text="Only present data from studies which have been marked as " - '"published" in HAWC.', - ) - studies = df.MultipleChoiceFilter( - field_name="study_population__study", - label="Studies to include", - help_text="""Select one or more studies to include in the plot. - If no study is selected, no endpoints will be available.""", - ) - # epi - systems = df.MultipleChoiceFilter( - field_name="system", - label="Systems to include", - help_text="""Select one or more systems to include in the plot. - If no system is selected, no endpoints will be available.""", - ) - effects = df.MultipleChoiceFilter( - field_name="effect", - label="Effects to include", - help_text="""Select one or more effects to include in the plot. - If no effect is selected, no endpoints will be available.""", - ) - effect_tags = df.MultipleChoiceFilter( - field_name="effects", - label="Tags to include", - help_text="""Select one or more effect-tags to include in the plot. - If no study is selected, no endpoints will be available.""", - ) - - class Meta: - model = Outcome - fields = [ - "published_only", - "studies", - "systems", - "effects", - "effect_tags", - ] - form = TestForm - - def filter_published_only(self, queryset, name, value): - if not value: - return queryset - return queryset.filter(study_population__study__published=True) - - def filter_queryset(self, queryset): - queryset = queryset.filter(assessment_id=self.assessment.pk) - return super().filter_queryset(queryset) - - def create_form(self): - form = super().create_form() - form.fields["studies"].choices = Study.objects.get_choices(self.assessment.pk) - form.fields["systems"].choices = Outcome.objects.get_system_choices(self.assessment.pk) - form.fields["effects"].choices = Outcome.objects.get_effect_choices(self.assessment.pk) - form.fields["effect_tags"].choices = EffectTag.objects.get_choices(self.assessment.pk) - return form - - -class EpiV2Prefilter(BaseFilterSet): - # studies - published_only = df.BooleanFilter( - method="filter_published_only", - widget=CheckboxInput(), - label="Published studies only", - help_text="Only present data from studies which have been marked as " - '"published" in HAWC.', - ) - studies = df.MultipleChoiceFilter( - field_name="design__study", - label="Studies to include", - help_text="""Select one or more studies to include in the plot. - If no study is selected, no endpoints will be available.""", - ) - - class Meta: - model = DataExtraction - fields = [ - "published_only", - "studies", - ] - form = TestForm - - def filter_published_only(self, queryset, name, value): - if not value: - return queryset - return queryset.filter(design__study__published=True) - - def filter_queryset(self, queryset): - queryset = queryset.filter(design__study__assessment_id=self.assessment.pk) - return super().filter_queryset(queryset) - - def create_form(self): - form = super().create_form() - form.fields["studies"].choices = Study.objects.get_choices(self.assessment.pk) - return form - - -class EpiMetaPrefilter(BaseFilterSet): - # studies - published_only = df.BooleanFilter( - method="filter_published_only", - widget=CheckboxInput(), - label="Published studies only", - help_text="Only present data from studies which have been marked as " - '"published" in HAWC.', - ) - studies = df.MultipleChoiceFilter( - field_name="protocol__study", - label="Studies to include", - help_text="""Select one or more studies to include in the plot. - If no study is selected, no endpoints will be available.""", - ) - - class Meta: - model = MetaResult - fields = [ - "published_only", - "studies", - ] - form = TestForm - - def filter_published_only(self, queryset, name, value): - if not value: - return queryset - return queryset.filter(protocol__study__published=True) - - def filter_queryset(self, queryset): - queryset = queryset.filter(protocol__study__assessment_id=self.assessment.pk) - return super().filter_queryset(queryset) - - def create_form(self): - form = super().create_form() - form.fields["studies"].choices = Study.objects.get_choices(self.assessment.pk) - return form - - -class InvitroPrefilter(BaseFilterSet): - # studies - published_only = df.BooleanFilter( - method="filter_published_only", - widget=CheckboxInput(), - label="Published studies only", - help_text="Only present data from studies which have been marked as " - '"published" in HAWC.', - ) - studies = df.MultipleChoiceFilter( - field_name="experiment__study", - label="Studies to include", - help_text="""Select one or more studies to include in the plot. - If no study is selected, no endpoints will be available.""", - ) - # invitro - categories = df.MultipleChoiceFilter( - field_name="category", - label="Categories to include", - help_text="""Select one or more categories to include in the plot. - If no study is selected, no endpoints will be available.""", - ) - chemicals = df.MultipleChoiceFilter( - field_name="chemical__name", - label="Chemicals to include", - help_text="""Select one or more chemicals to include in the plot. - If no study is selected, no endpoints will be available.""", - ) - effect_tags = df.MultipleChoiceFilter( - field_name="effects", - label="Tags to include", - help_text="""Select one or more effect-tags to include in the plot. - If no study is selected, no endpoints will be available.""", - ) - - class Meta: - model = IVEndpoint - fields = [ - "published_only", - "studies", - "categories", - "chemicals", - "effect_tags", - ] - form = TestForm - - def filter_published_only(self, queryset, name, value): - if not value: - return queryset - return queryset.filter(experiment__study__published=True) - - def filter_queryset(self, queryset): - queryset = queryset.filter(assessment_id=self.assessment.pk) - return super().filter_queryset(queryset) - - def create_form(self): - form = super().create_form() - form.fields["studies"].choices = Study.objects.get_choices(self.assessment.pk) - form.fields["categories"].choices = IVEndpointCategory.get_choices(self.assessment.pk) - form.fields["chemicals"].choices = IVChemical.objects.get_choices(self.assessment.pk) - form.fields["effect_tags"].choices = EffectTag.objects.get_choices(self.assessment.pk) - return form - - -class StudyTypePrefilter(Enum): - BIOASSAY = BioassayPrefilter - EPIV1 = EpiV1Prefilter - EPIV2 = EpiV2Prefilter - EPI_META = EpiMetaPrefilter - IN_VITRO = InvitroPrefilter - - @classmethod - def from_study_type(cls, study_type: int | StudyType, assessment: Assessment): - study_type = StudyType(study_type) - name = study_type.name - if study_type == StudyType.EPI: - if assessment.epi_version == EpiVersion.V1: - name = "EPIV1" - elif assessment.epi_version == EpiVersion.V2: - name = "EPIV2" - return cls[name] - - -class VisualTypePrefilter(Enum): - BIOASSAY_CROSSVIEW = BioassayPrefilter - ROB_HEATMAP = BioassayPrefilter - ROB_BARCHART = BioassayPrefilter - - @classmethod - def from_visual_type(cls, visual_type: int | VisualType): - visual_type = VisualType(visual_type) - name = visual_type.name - return cls[name] diff --git a/hawc/apps/summary/templates/summary/datapivot_form.html b/hawc/apps/summary/templates/summary/datapivot_form.html index ee636612f0..ed842b2046 100644 --- a/hawc/apps/summary/templates/summary/datapivot_form.html +++ b/hawc/apps/summary/templates/summary/datapivot_form.html @@ -24,7 +24,7 @@ {% endif %} window.app.startup("assessmentStartup", function(app){ - new app.DoseUnitsWidget($('form'), { + var doseWidget = new app.DoseUnitsWidget($('form'), { choices: js_units_choices, el: '#id_preferred_units', }); @@ -37,12 +37,63 @@ ); }) + var togglePrefilterSelectorVisibility = function(d){ + var fields = [ + ["study", "studies"], + ["system", "systems"], + ["organ", "organs"], + ["effect", "effects"], + ["effect_subtype", "effect_subtypes"], + ["episystem", "episystems"], + ["epieffect", "epieffects"], + ["iv_category", "iv_categories"], + ["iv_chemical", "iv_chemicals"], + ["effect_tag", "effect_tags"], + ]; + _.each(fields, function(d){ + $(printf('#id_prefilter_{0}', d[0])).on('change', function(){ + var div = $(printf('#div_id_{0}', d[1])); + ($(this).prop('checked')) ? div.show(1000) : div.hide(0); + }).trigger('change'); + }); + } + // determine which fields to display depending on data-type - const value = {{form.instance.evidence_type}}, - aniOnlyDivs = $("#div_id_preferred_units"), - aniIvOnlyDivs = $("#div_id_export_style"); - (value === 0) ? aniOnlyDivs.show() : aniOnlyDivs.hide(); - (value == 0 || value == 2) ? aniIvOnlyDivs.show() : aniIvOnlyDivs.hide(); + $('#id_evidence_type').on('change', function(){ + const value = parseInt($('#id_evidence_type').val()), + aniOnlyDivs = $([ + "#div_id_preferred_units", + "#div_id_prefilter_system", + "#div_id_systems", + "#div_id_prefilter_organ", + "#div_id_organs", + "#div_id_prefilter_effect", + "#div_id_prefilter_effect_subtype", + "#div_id_effects" + ].join(",")), + epiOnlyDivs = $([ + "#div_id_prefilter_episystem", + "#div_id_episystems", + "#div_id_prefilter_epieffect", + "#div_id_epieffects" + ].join(",")), + ivOnlyDivs = $([ + "#div_id_prefilter_iv_category", + "#div_id_iv_categories", + "#div_id_prefilter_iv_chemical", + "#div_id_iv_chemicals", + ].join(",")), + aniIvOnlyDivs = $("#div_id_export_style"), + notEpiV2Divs = $("#div_id_prefilter_effect_tag"); + (value === 0) ? aniOnlyDivs.show() : aniOnlyDivs.hide(); + (value === 1 && epi_version === 1) ? epiOnlyDivs.show() : epiOnlyDivs.hide(); + (value === 1 && epi_version === 1) ? notEpiV2Divs.show() : notEpiV2Divs.hide(); + (value === 2) ? ivOnlyDivs.show() : ivOnlyDivs.hide(); + (value == 0 || value == 2) ? aniIvOnlyDivs.show() : aniIvOnlyDivs.hide(); + togglePrefilterSelectorVisibility(); + }).trigger('change'); + + togglePrefilterSelectorVisibility(); }); {% endblock extrajs %} diff --git a/hawc/apps/summary/urls.py b/hawc/apps/summary/urls.py index fd3630e463..6aa56e84f1 100644 --- a/hawc/apps/summary/urls.py +++ b/hawc/apps/summary/urls.py @@ -115,7 +115,7 @@ name="dp_new-prompt", ), path( - "data-pivot/assessment//create/query//", + "data-pivot/assessment//create/query/", views.DataPivotQueryNew.as_view(), name="dp_new-query", ), diff --git a/hawc/apps/summary/views.py b/hawc/apps/summary/views.py index 60a19d67a4..0c25ac2c41 100644 --- a/hawc/apps/summary/views.py +++ b/hawc/apps/summary/views.py @@ -15,16 +15,9 @@ from ..assessment.views import check_published_status from ..common.crumbs import Breadcrumb from ..common.helper import WebappConfig -from ..common.views import ( - BaseCreate, - BaseDelete, - BaseDetail, - BaseFilterList, - BaseList, - BaseUpdate, -) +from ..common.views import BaseCreate, BaseDelete, BaseDetail, BaseFilterList, BaseList, BaseUpdate from ..riskofbias.models import RiskOfBiasMetric -from . import constants, filterset, forms, models, prefilters, serializers +from . import constants, filterset, forms, models, serializers def get_visual_list_crumb(assessment) -> Breadcrumb: @@ -641,20 +634,6 @@ def get_form_kwargs(self): class DataPivotQueryNew(DataPivotNew): model = models.DataPivotQuery form_class = forms.DataPivotQueryForm - template_name = "summary/datapivot_form.html" - - def get_form_kwargs(self): - kwargs = super().get_form_kwargs() - try: - # get study type enum - study_type = constants.StudyType(self.kwargs.get("study_type")) - # make sure prefilter exists for study type - prefilters.StudyTypePrefilter.from_study_type(study_type, self.assessment) - # pass study type to form - kwargs["evidence_type"] = study_type - except (KeyError, ValueError): - raise Http404 - return kwargs def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) diff --git a/tests/data/api/api-summary-table-set-data.json b/tests/data/api/api-summary-table-set-data.json index bdc64ee884..56fbf4a61c 100644 --- a/tests/data/api/api-summary-table-set-data.json +++ b/tests/data/api/api-summary-table-set-data.json @@ -111,4 +111,4 @@ "study_id": 1 } ] -} \ No newline at end of file +} diff --git a/tests/data/api/api-visual-bioassay-aggregation.json b/tests/data/api/api-visual-bioassay-aggregation.json index be2c83dcf4..cae434f705 100644 --- a/tests/data/api/api-visual-bioassay-aggregation.json +++ b/tests/data/api/api-visual-bioassay-aggregation.json @@ -371,4 +371,4 @@ "url_delete": "/summary/visual/assessment/2/bioassay-aggregation/delete/", "url_update": "/summary/visual/assessment/2/bioassay-aggregation/update/", "visual_type": "animal bioassay endpoint aggregation" -} +} \ No newline at end of file diff --git a/tests/data/api/api-visual-crossview.json b/tests/data/api/api-visual-crossview.json index 7cb8f85fda..8372267d69 100644 --- a/tests/data/api/api-visual-crossview.json +++ b/tests/data/api/api-visual-crossview.json @@ -3335,4 +3335,4 @@ "url_delete": "/summary/visual/assessment/2/crossview/delete/", "url_update": "/summary/visual/assessment/2/crossview/update/", "visual_type": "animal bioassay endpoint crossview" -} +} \ No newline at end of file diff --git a/tests/data/api/api-visual-embedded-tableau.json b/tests/data/api/api-visual-embedded-tableau.json index be7f33f1cf..bd43e29661 100644 --- a/tests/data/api/api-visual-embedded-tableau.json +++ b/tests/data/api/api-visual-embedded-tableau.json @@ -27,4 +27,4 @@ "url_delete": "/summary/visual/assessment/2/embedded-tableau/delete/", "url_update": "/summary/visual/assessment/2/embedded-tableau/update/", "visual_type": "embedded external website" -} +} \ No newline at end of file diff --git a/tests/data/api/api-visual-exploratory-heatmap.json b/tests/data/api/api-visual-exploratory-heatmap.json index b3da775020..4472d4fbbe 100644 --- a/tests/data/api/api-visual-exploratory-heatmap.json +++ b/tests/data/api/api-visual-exploratory-heatmap.json @@ -125,4 +125,4 @@ "url_delete": "/summary/visual/assessment/2/exploratory-heatmap/delete/", "url_update": "/summary/visual/assessment/2/exploratory-heatmap/update/", "visual_type": "exploratory heatmap" -} +} \ No newline at end of file diff --git a/tests/data/api/api-visual-rob-barchart.json b/tests/data/api/api-visual-rob-barchart.json index 17de3cee9d..5456a99252 100644 --- a/tests/data/api/api-visual-rob-barchart.json +++ b/tests/data/api/api-visual-rob-barchart.json @@ -483,4 +483,4 @@ "url_delete": "/summary/visual/assessment/2/rob-barchart/delete/", "url_update": "/summary/visual/assessment/2/rob-barchart/update/", "visual_type": "risk of bias barchart" -} +} \ No newline at end of file diff --git a/tests/data/api/api-visual-rob-heatmap.json b/tests/data/api/api-visual-rob-heatmap.json index 9d10242f60..281c6c2c6f 100644 --- a/tests/data/api/api-visual-rob-heatmap.json +++ b/tests/data/api/api-visual-rob-heatmap.json @@ -485,4 +485,4 @@ "url_delete": "/summary/visual/assessment/2/rob-heatmap/delete/", "url_update": "/summary/visual/assessment/2/rob-heatmap/update/", "visual_type": "risk of bias heatmap" -} +} \ No newline at end of file diff --git a/tests/data/api/api-visual-tagtree.json b/tests/data/api/api-visual-tagtree.json index 9046efdb1b..6969f0dcf6 100644 --- a/tests/data/api/api-visual-tagtree.json +++ b/tests/data/api/api-visual-tagtree.json @@ -27,4 +27,4 @@ "url_delete": "/summary/visual/assessment/2/tagtree/delete/", "url_update": "/summary/visual/assessment/2/tagtree/update/", "visual_type": "literature tagtree" -} +} \ No newline at end of file diff --git a/tests/hawc/apps/summary/test_views.py b/tests/hawc/apps/summary/test_views.py index 66df75cd70..c3a9c23036 100644 --- a/tests/hawc/apps/summary/test_views.py +++ b/tests/hawc/apps/summary/test_views.py @@ -14,7 +14,7 @@ def test_initial_settings(self): c = Client() assert c.login(username="pm@hawcproject.org", password="pw") is True - url = reverse("summary:dp_new-query", args=(1, 0)) + url = reverse("summary:dp_new-query", args=(1,)) # no initial settings or invalid settings for args in ["", "?initial=-1", "?initial=-1&reset_row_overrides=1"]: From 9500b808f3b9e3aba921efc10fe910a549b7fa8f Mon Sep 17 00:00:00 2001 From: Andy Shapiro Date: Sun, 17 Sep 2023 07:44:28 -0400 Subject: [PATCH 29/35] revert db fixture --- tests/data/fixtures/db.yaml | 73 ++++++++++++++++++++----------------- 1 file changed, 39 insertions(+), 34 deletions(-) diff --git a/tests/data/fixtures/db.yaml b/tests/data/fixtures/db.yaml index 52b1ed560c..1018f3e623 100644 --- a/tests/data/fixtures/db.yaml +++ b/tests/data/fixtures/db.yaml @@ -9284,9 +9284,7 @@ assessment: 1 visual_type: 2 dose_units: null - prefilters: - studies: - - 1 + prefilters: '{}' settings: '{"title":"","xAxisLabel":"","yAxisLabel":"","padding_top":20,"cell_size":40,"padding_right":400,"padding_bottom":35,"padding_left":20,"x_field":"study","study_label_field":"short_citation","included_metrics":[1,2],"excluded_score_ids":[],"show_legend":true,"show_na_legend":true,"show_nr_legend":true,"legend_x":239,"legend_y":17}' caption: '' published: false @@ -9294,6 +9292,8 @@ created: 2020-02-27 20:32:30.901597+00:00 last_updated: 2020-02-27 20:33:06.585979+00:00 endpoints: [] + studies: + - 1 - model: summary.visual pk: 2 fields: @@ -9302,9 +9302,7 @@ assessment: 1 visual_type: 3 dose_units: null - prefilters: - studies: - - 1 + prefilters: '{}' settings: '{"title":"Title","xAxisLabel":"Percent of studies","yAxisLabel":"","plot_width":400,"row_height":30,"padding_top":40,"padding_right":400,"padding_bottom":40,"padding_left":70,"show_values":true,"included_metrics":[1,2],"show_legend":true,"show_na_legend":true,"legend_x":640,"legend_y":10}' caption: '' published: false @@ -9312,6 +9310,8 @@ created: 2020-02-27 20:34:17.606164+00:00 last_updated: 2020-02-27 20:34:17.606186+00:00 endpoints: [] + studies: + - 1 - model: summary.visual pk: 3 fields: @@ -9320,9 +9320,7 @@ assessment: 2 visual_type: 2 dose_units: null - prefilters: - studies: - - 7 + prefilters: '{}' settings: '{"title":"","xAxisLabel":"","yAxisLabel":"","padding_top":20,"cell_size":40,"padding_right":300,"padding_bottom":35,"padding_left":20,"x_field":"study","study_label_field":"short_citation","included_metrics":[14,15],"excluded_score_ids":[],"show_legend":true,"show_na_legend":true,"show_nr_legend":true,"legend_x":231,"legend_y":30}' caption:

caption

published: true @@ -9330,6 +9328,8 @@ created: 2020-05-08 19:35:41.634588+00:00 last_updated: 2020-05-08 19:36:41.324440+00:00 endpoints: [] + studies: + - 7 - model: summary.visual pk: 4 fields: @@ -9338,8 +9338,7 @@ assessment: 2 visual_type: 1 dose_units: 1 - prefilters: - published_only: true + prefilters: '{"animal_group__experiment__study__published": true}' settings: '{"title":"Title","xAxisLabel":"Dose ()","yAxisLabel":"% change from control (continuous), % incidence (dichotomous)","width":1100,"height":600,"inner_width":940,"inner_height":520,"padding_left":75,"padding_top":45,"dose_isLog":true,"dose_range":"","response_range":"","title_x":0,"title_y":0,"xlabel_x":0,"xlabel_y":0,"ylabel_x":0,"ylabel_y":0,"filters":[{"name":"study","headerName":"Study","allValues":true,"values":null,"columns":1,"x":null,"y":null}],"reflines_dose":[{"value":null,"title":"","style":"base"}],"refranges_dose":[{"lower":null,"upper":null,"title":"","style":"base"}],"reflines_response":[{"value":null,"title":"","style":"base"}],"refranges_response":[{"lower":null,"upper":null,"title":"","style":"base"}],"labels":[{"caption":"","style":"base","max_width":null,"x":null,"y":null}],"colorBase":"#cccccc","colorHover":"#ff4040","colorSelected":"#6495ed","colorFilters":[],"colorFilterLegend":true,"colorFilterLegendLabel":"Color filters","colorFilterLegendX":0,"colorFilterLegendY":0,"endpointFilters":[],"endpointFilterLogic":"and"}' @@ -9349,6 +9348,7 @@ created: 2020-05-08 19:37:19.255703+00:00 last_updated: 2020-05-08 19:37:19.255730+00:00 endpoints: [] + studies: [] - model: summary.visual pk: 5 fields: @@ -9357,9 +9357,7 @@ assessment: 2 visual_type: 3 dose_units: null - prefilters: - studies: - - 7 + prefilters: '{}' settings: '{"title":"Title","xAxisLabel":"Percent of studies","yAxisLabel":"","plot_width":400,"row_height":30,"padding_top":40,"padding_right":300,"padding_bottom":40,"padding_left":70,"show_values":true,"included_metrics":[14,15],"show_legend":true,"show_na_legend":true,"legend_x":574,"legend_y":10}' caption:

caption

published: true @@ -9367,6 +9365,8 @@ created: 2020-05-08 19:37:44.283008+00:00 last_updated: 2020-05-08 19:38:48.445082+00:00 endpoints: [] + studies: + - 7 - model: summary.visual pk: 6 fields: @@ -9375,7 +9375,7 @@ assessment: 2 visual_type: 4 dose_units: null - prefilters: {} + prefilters: '{}' settings: '{"root_node": 11, "required_tags": [], "pruned_tags": [], "hide_empty_tag_nodes": false, "height": 500, "width": 1280, "show_legend": true, "show_counts": true}' caption:

caption

published: true @@ -9383,6 +9383,7 @@ created: 2020-05-08 19:43:09.448597+00:00 last_updated: 2020-05-08 19:43:09.448621+00:00 endpoints: [] + studies: [] - model: summary.visual pk: 7 fields: @@ -9391,7 +9392,7 @@ assessment: 2 visual_type: 5 dose_units: null - prefilters: {} + prefilters: '{}' settings: '{"external_url": "https://public.tableau.com/views/Iris_15675445278420/Iris-Actual", "external_url_hostname": "https://public.tableau.com", "external_url_path": "/views/Iris_15675445278420/Iris-Actual", "external_url_query_args": [":showVizHome=no", @@ -9402,6 +9403,7 @@ created: 2020-05-08 19:45:29.985823+00:00 last_updated: 2020-05-08 19:45:29.985848+00:00 endpoints: [] + studies: [] - model: summary.visual pk: 8 fields: @@ -9410,7 +9412,7 @@ assessment: 2 visual_type: 6 dose_units: null - prefilters: {} + prefilters: '{}' settings: '{"cell_height": 50, "cell_width": 50, "color_range": ["#ffffff", "#cc4700"], "compress_x": true, "compress_y": true, "data_url": "/ani/api/assessment/2/endpoint-heatmap/", "hawc_interactivity": true, "filter_widgets": [{"column": "species", "delimiter": "", "on_click_event": "---", "header": ""}, {"column": "strain", "delimiter": "", "on_click_event": "---", "header": ""}], "padding": {"top": 30, "left": 30, "bottom": 30, "right": 30}, "show_axis_border": true, "show_grid": true, "show_tooltip": @@ -9427,6 +9429,7 @@ created: 2020-11-25 14:25:27.528283+00:00 last_updated: 2020-11-25 14:29:33.991892+00:00 endpoints: [] + studies: [] - model: summary.visual pk: 9 fields: @@ -9435,7 +9438,7 @@ assessment: 2 visual_type: 7 dose_units: null - prefilters: {} + prefilters: '{}' settings: '{"data": [{"orientation": "h", "x": [1, 2, 3], "xaxis": "x", "y": [0, 1, 2], "yaxis": "y", "type": "bar"}], "layout": {"title":{"text":"test"}}}' caption: '' published: true @@ -9443,6 +9446,7 @@ created: 2023-05-19 14:25:27.528283+00:00 last_updated: 2023-05-19 14:25:27.528283+00:00 endpoints: [] + studies: [] - model: summary.visual pk: 10 fields: @@ -9451,7 +9455,7 @@ assessment: 2 visual_type: 0 dose_units: 1 - prefilters: {} + prefilters: '{}' settings: '{}' caption: '' published: true @@ -9460,6 +9464,7 @@ last_updated: 2020-11-25 18:35:00.094667+00:00 endpoints: - 3 + studies: [] - model: summary.visual pk: 11 fields: @@ -9468,9 +9473,7 @@ assessment: 2 visual_type: 2 dose_units: null - prefilters: - studies: - - 7 + prefilters: '{}' settings: '{"title":"","xAxisLabel":"","yAxisLabel":"","padding_top":20,"cell_size":40,"padding_right":190,"padding_bottom":35,"padding_left":20,"x_field":"study","study_label_field":"short_citation","included_metrics":[14,15],"excluded_score_ids":[],"show_legend":true,"show_na_legend":true,"show_nr_legend":true,"legend_x":226,"legend_y":16}' caption:

This is a study, an endpoint, @@ -9487,6 +9490,8 @@ created: 2020-11-26 03:21:03.883675+00:00 last_updated: 2020-11-26 03:25:56.660522+00:00 endpoints: [] + studies: + - 7 - model: summary.datapivot pk: 1 fields: @@ -10154,8 +10159,8 @@ evidence_type: 0 export_style: 1 preferred_units: '["1"]' - prefilters: - published_only: true + prefilters: '{}' + published_only: true - model: summary.datapivotquery pk: 1 fields: @@ -10163,8 +10168,8 @@ evidence_type: 0 export_style: 0 preferred_units: '["1"]' - prefilters: - published_only: true + prefilters: '{}' + published_only: true - model: summary.datapivotquery pk: 4 fields: @@ -10172,8 +10177,8 @@ evidence_type: 1 export_style: 0 preferred_units: '[]' - prefilters: - published_only: true + prefilters: '{}' + published_only: true - model: summary.datapivotquery pk: 3 fields: @@ -10181,8 +10186,8 @@ evidence_type: 4 export_style: 0 preferred_units: '[]' - prefilters: - published_only: true + prefilters: '{}' + published_only: true - model: summary.datapivotquery pk: 5 fields: @@ -10190,8 +10195,8 @@ evidence_type: 2 export_style: 1 preferred_units: '[]' - prefilters: - published_only: true + prefilters: '{}' + published_only: true - model: summary.datapivotquery pk: 6 fields: @@ -10199,8 +10204,8 @@ evidence_type: 2 export_style: 0 preferred_units: '[]' - prefilters: - published_only: true + prefilters: '{}' + published_only: true - model: mgmt.task pk: 1 fields: From 99660b4e5fb5581c952c4ae6ad4cd4a705d3a3db Mon Sep 17 00:00:00 2001 From: Andy Shapiro Date: Sun, 17 Sep 2023 08:23:26 -0400 Subject: [PATCH 30/35] use regex --- hawc/apps/common/dynamic_forms/fields.py | 4 ++-- .../hawc/apps/common/dynamic_forms/test_fields.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 tests/hawc/apps/common/dynamic_forms/test_fields.py diff --git a/hawc/apps/common/dynamic_forms/fields.py b/hawc/apps/common/dynamic_forms/fields.py index 68696234e2..3d6bcfb8bc 100644 --- a/hawc/apps/common/dynamic_forms/fields.py +++ b/hawc/apps/common/dynamic_forms/fields.py @@ -3,7 +3,7 @@ from django import forms from django.utils.html import conditional_escape -from pydantic import BaseModel, validator +from pydantic import BaseModel, constr, validator from pydantic import Field as PydanticField from . import constants @@ -19,7 +19,7 @@ class _Field(BaseModel): in constants.Widget. """ - name: str # the variable name in the form; extra validation for no whitespace etc? + name: str = PydanticField(regex=r"^[a-zA-Z0-9_]+$") required: bool | None label: str | None label_suffix: str | None diff --git a/tests/hawc/apps/common/dynamic_forms/test_fields.py b/tests/hawc/apps/common/dynamic_forms/test_fields.py new file mode 100644 index 0000000000..d05b00eca0 --- /dev/null +++ b/tests/hawc/apps/common/dynamic_forms/test_fields.py @@ -0,0 +1,14 @@ +import pytest +from pydantic import ValidationError + +from hawc.apps.common.dynamic_forms.fields import _Field + + +class TestField: + def test_name_regex(self): + for name in ["valid", "VALID", "aZ_1"]: + _Field(name=name) + + for name in ["a space", "a+plus", "a.dot"]: + with pytest.raises(ValidationError): + _Field(name=name) From 3c3e38442bba30f3d4a00c46b0f6a5333abe05df Mon Sep 17 00:00:00 2001 From: Andy Shapiro Date: Sun, 17 Sep 2023 08:26:02 -0400 Subject: [PATCH 31/35] remove ID --- hawc/apps/common/dynamic_forms/forms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hawc/apps/common/dynamic_forms/forms.py b/hawc/apps/common/dynamic_forms/forms.py index 2944dd96df..e11cd75f73 100644 --- a/hawc/apps/common/dynamic_forms/forms.py +++ b/hawc/apps/common/dynamic_forms/forms.py @@ -88,7 +88,7 @@ def auto_wrap_fields(self): css_class = field.css_class or "col-12" self[index].wrap(cfl.Column, css_class=css_class) - self[:].wrap_together(cfl.Row, id="row_id_dynamic_form") + self[:].wrap_together(cfl.Row) self.add_field_wraps() From 59ae0d4263ce0cf77f3a90124c1b18c792e0e96e Mon Sep 17 00:00:00 2001 From: Andy Shapiro Date: Sun, 17 Sep 2023 08:37:01 -0400 Subject: [PATCH 32/35] rename to udf --- .../form_library/migrations/0001_initial.py | 28 --------- ...2_customdataextraction_creator_and_more.py | 53 ----------------- hawc/apps/{form_library => udf}/__init__.py | 0 hawc/apps/{form_library => udf}/admin.py | 0 hawc/apps/{form_library => udf}/apps.py | 4 +- hawc/apps/{form_library => udf}/forms.py | 0 hawc/apps/udf/migrations/0001_initial.py | 59 +++++++++++++++++++ .../migrations/__init__.py | 0 hawc/apps/{form_library => udf}/models.py | 2 +- .../templates/udf}/udf_form.html | 0 hawc/apps/{form_library => udf}/urls.py | 6 +- hawc/apps/{form_library => udf}/views.py | 7 ++- hawc/constants.py | 2 +- hawc/main/settings/base.py | 2 +- hawc/main/settings/unittest.py | 2 +- hawc/main/urls.py | 4 +- 16 files changed, 74 insertions(+), 95 deletions(-) delete mode 100644 hawc/apps/form_library/migrations/0001_initial.py delete mode 100644 hawc/apps/form_library/migrations/0002_customdataextraction_creator_and_more.py rename hawc/apps/{form_library => udf}/__init__.py (100%) rename hawc/apps/{form_library => udf}/admin.py (100%) rename hawc/apps/{form_library => udf}/apps.py (64%) rename hawc/apps/{form_library => udf}/forms.py (100%) create mode 100644 hawc/apps/udf/migrations/0001_initial.py rename hawc/apps/{form_library => udf}/migrations/__init__.py (100%) rename hawc/apps/{form_library => udf}/models.py (95%) rename hawc/apps/{form_library/templates/form_library => udf/templates/udf}/udf_form.html (100%) rename hawc/apps/{form_library => udf}/urls.py (69%) rename hawc/apps/{form_library => udf}/views.py (62%) diff --git a/hawc/apps/form_library/migrations/0001_initial.py b/hawc/apps/form_library/migrations/0001_initial.py deleted file mode 100644 index 0f2cbc7323..0000000000 --- a/hawc/apps/form_library/migrations/0001_initial.py +++ /dev/null @@ -1,28 +0,0 @@ -# Generated by Django 4.2.4 on 2023-08-29 16:17 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - initial = True - - dependencies = [] - - operations = [ - migrations.CreateModel( - name="CustomDataExtraction", - fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, primary_key=True, serialize=False, verbose_name="ID" - ), - ), - ("name", models.CharField()), - ("description", models.TextField()), - ("schema", models.JSONField()), - ("created", models.DateTimeField(auto_now_add=True)), - ("last_updated", models.DateTimeField(auto_now=True)), - ], - ), - ] diff --git a/hawc/apps/form_library/migrations/0002_customdataextraction_creator_and_more.py b/hawc/apps/form_library/migrations/0002_customdataextraction_creator_and_more.py deleted file mode 100644 index 8db48db224..0000000000 --- a/hawc/apps/form_library/migrations/0002_customdataextraction_creator_and_more.py +++ /dev/null @@ -1,53 +0,0 @@ -# Generated by Django 4.2.4 on 2023-08-31 14:55 - -import django.db.models.deletion -from django.conf import settings -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ("form_library", "0001_initial"), - ] - - operations = [ - migrations.AddField( - model_name="customdataextraction", - name="creator", - field=models.ForeignKey( - null=True, - on_delete=django.db.models.deletion.SET_NULL, - related_name="created_forms", - to=settings.AUTH_USER_MODEL, - ), - ), - migrations.AddField( - model_name="customdataextraction", - name="editors", - field=models.ManyToManyField( - blank=True, related_name="editable_forms", to=settings.AUTH_USER_MODEL - ), - ), - migrations.AddField( - model_name="customdataextraction", - name="parent_form", - field=models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.SET_NULL, - related_name="child_forms", - to="form_library.customdataextraction", - ), - ), - migrations.AlterField( - model_name="customdataextraction", - name="description", - field=models.TextField(blank=True), - ), - migrations.AlterField( - model_name="customdataextraction", - name="name", - field=models.CharField(max_length=128), - ), - ] diff --git a/hawc/apps/form_library/__init__.py b/hawc/apps/udf/__init__.py similarity index 100% rename from hawc/apps/form_library/__init__.py rename to hawc/apps/udf/__init__.py diff --git a/hawc/apps/form_library/admin.py b/hawc/apps/udf/admin.py similarity index 100% rename from hawc/apps/form_library/admin.py rename to hawc/apps/udf/admin.py diff --git a/hawc/apps/form_library/apps.py b/hawc/apps/udf/apps.py similarity index 64% rename from hawc/apps/form_library/apps.py rename to hawc/apps/udf/apps.py index 849142b68d..970b831028 100644 --- a/hawc/apps/form_library/apps.py +++ b/hawc/apps/udf/apps.py @@ -3,5 +3,5 @@ class FormLibraryConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" - name = "hawc.apps.form_library" - verbose_name = "Form Library" + name = "hawc.apps.udf" + verbose_name = "User Defined Forms" diff --git a/hawc/apps/form_library/forms.py b/hawc/apps/udf/forms.py similarity index 100% rename from hawc/apps/form_library/forms.py rename to hawc/apps/udf/forms.py diff --git a/hawc/apps/udf/migrations/0001_initial.py b/hawc/apps/udf/migrations/0001_initial.py new file mode 100644 index 0000000000..75aebda00c --- /dev/null +++ b/hawc/apps/udf/migrations/0001_initial.py @@ -0,0 +1,59 @@ +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="UserDefinedForm", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("name", models.CharField(max_length=128)), + ("description", models.TextField()), + ("schema", models.JSONField()), + ("deprecated", models.BooleanField(default=False)), + ("created", models.DateTimeField(auto_now_add=True)), + ("last_updated", models.DateTimeField(auto_now=True)), + ( + "creator", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + related_name="created_forms", + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "editors", + models.ManyToManyField( + blank=True, related_name="editable_forms", to=settings.AUTH_USER_MODEL + ), + ), + ( + "parent", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="children", + to="udf.userdefinedform", + ), + ), + ], + options={ + "ordering": ["-last_updated"], + "unique_together": {("creator", "name")}, + }, + ), + ] diff --git a/hawc/apps/form_library/migrations/__init__.py b/hawc/apps/udf/migrations/__init__.py similarity index 100% rename from hawc/apps/form_library/migrations/__init__.py rename to hawc/apps/udf/migrations/__init__.py diff --git a/hawc/apps/form_library/models.py b/hawc/apps/udf/models.py similarity index 95% rename from hawc/apps/form_library/models.py rename to hawc/apps/udf/models.py index b27e799227..254dadd392 100644 --- a/hawc/apps/form_library/models.py +++ b/hawc/apps/udf/models.py @@ -11,7 +11,7 @@ class UserDefinedForm(models.Model): creator = models.ForeignKey(HAWCUser, on_delete=models.DO_NOTHING, related_name="created_forms") editors = models.ManyToManyField(HAWCUser, blank=True, related_name="editable_forms") parent = models.ForeignKey( - "form_library.UserDefinedForm", + "udf.UserDefinedForm", blank=True, null=True, on_delete=models.SET_NULL, diff --git a/hawc/apps/form_library/templates/form_library/udf_form.html b/hawc/apps/udf/templates/udf/udf_form.html similarity index 100% rename from hawc/apps/form_library/templates/form_library/udf_form.html rename to hawc/apps/udf/templates/udf/udf_form.html diff --git a/hawc/apps/form_library/urls.py b/hawc/apps/udf/urls.py similarity index 69% rename from hawc/apps/form_library/urls.py rename to hawc/apps/udf/urls.py index f947ab939e..fc611ed523 100644 --- a/hawc/apps/form_library/urls.py +++ b/hawc/apps/udf/urls.py @@ -3,16 +3,16 @@ from . import views -app_name = "form_library" +app_name = "udf" urlpatterns = ( [ # Create a user defined form path( "create/", views.CreateUDFView.as_view(), - name="form_create", + name="udf_create", ), ] - if settings.HAWC_FEATURES.ENABLE_DYNAMIC_FORMS + if settings.HAWC_FEATURES.ENABLE_UDF else [] ) diff --git a/hawc/apps/form_library/views.py b/hawc/apps/udf/views.py similarity index 62% rename from hawc/apps/form_library/views.py rename to hawc/apps/udf/views.py index bc9cd37688..3722307153 100644 --- a/hawc/apps/form_library/views.py +++ b/hawc/apps/udf/views.py @@ -1,14 +1,15 @@ from django.urls import reverse from django.views.generic.edit import CreateView -from hawc.apps.common.views import LoginRequiredMixin +from hawc.apps.common.views import LoginRequiredMixin, MessageMixin from .forms import UDFForm -class CreateUDFView(LoginRequiredMixin, CreateView): - template_name = "form_library/udf_form.html" +class CreateUDFView(LoginRequiredMixin, MessageMixin, CreateView): + template_name = "udf/udf_form.html" form_class = UDFForm + success_message = "Form created." def get_form_kwargs(self): kwargs = super().get_form_kwargs() diff --git a/hawc/constants.py b/hawc/constants.py index b897361026..a3305b36af 100644 --- a/hawc/constants.py +++ b/hawc/constants.py @@ -17,7 +17,7 @@ class FeatureFlags(BaseModel): ANONYMOUS_ACCOUNT_CREATION: bool = True ENABLE_BMDS_33 = False ENABLE_PLOTLY_VISUAL: bool = False - ENABLE_DYNAMIC_FORMS: bool = False + ENABLE_UDF: bool = False @classmethod def from_env(cls, variable) -> "FeatureFlags": diff --git a/hawc/main/settings/base.py b/hawc/main/settings/base.py index 200081c238..aa901fa8eb 100644 --- a/hawc/main/settings/base.py +++ b/hawc/main/settings/base.py @@ -125,7 +125,7 @@ "hawc.apps.hawc_admin", "hawc.apps.materialized", "hawc.apps.epiv2", - "hawc.apps.form_library", + "hawc.apps.udf", ) # DB settings DATABASES = { diff --git a/hawc/main/settings/unittest.py b/hawc/main/settings/unittest.py index 7b12e5d6a5..06c4ca66a9 100644 --- a/hawc/main/settings/unittest.py +++ b/hawc/main/settings/unittest.py @@ -10,7 +10,7 @@ # enable feature flags for tests HAWC_FEATURES.ENABLE_BMDS_33 = True -HAWC_FEATURES.ENABLE_DYNAMIC_FORMS = True +HAWC_FEATURES.ENABLE_UDF = True # remove toolbar for integration tests INSTALLED_APPS = [app for app in INSTALLED_APPS if app != "debug_toolbar"] diff --git a/hawc/main/urls.py b/hawc/main/urls.py index da86e4eff1..496f6095b1 100644 --- a/hawc/main/urls.py +++ b/hawc/main/urls.py @@ -10,7 +10,6 @@ import hawc.apps.epi.urls import hawc.apps.epimeta.urls import hawc.apps.epiv2.urls -import hawc.apps.form_library.urls import hawc.apps.hawc_admin.urls import hawc.apps.invitro.urls import hawc.apps.lit.urls @@ -18,6 +17,7 @@ import hawc.apps.riskofbias.urls import hawc.apps.study.urls import hawc.apps.summary.urls +import hawc.apps.udf.urls import hawc.apps.vocab.urls from hawc.apps.assessment import views from hawc.apps.common.autocomplete import get_autocomplete @@ -44,7 +44,7 @@ path("epidemiology/", include("hawc.apps.epiv2.urls")), path("epi-meta/", include("hawc.apps.epimeta.urls")), path("in-vitro/", include("hawc.apps.invitro.urls")), - path("forms/", include("hawc.apps.form_library.urls")), + path("udf/", include("hawc.apps.udf.urls")), path("bmd/", include("hawc.apps.bmd.urls")), path("lit/", include("hawc.apps.lit.urls")), path("summary/", include("hawc.apps.summary.urls")), From 2978b324785badc29fc817f6dcbc993d39cd069c Mon Sep 17 00:00:00 2001 From: Andy Shapiro Date: Sun, 17 Sep 2023 09:39:10 -0400 Subject: [PATCH 33/35] lint; add test UDF to db schema --- hawc/apps/common/dynamic_forms/fields.py | 2 +- tests/data/fixtures/db.yaml | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/hawc/apps/common/dynamic_forms/fields.py b/hawc/apps/common/dynamic_forms/fields.py index 3d6bcfb8bc..c8dc7d482b 100644 --- a/hawc/apps/common/dynamic_forms/fields.py +++ b/hawc/apps/common/dynamic_forms/fields.py @@ -3,7 +3,7 @@ from django import forms from django.utils.html import conditional_escape -from pydantic import BaseModel, constr, validator +from pydantic import BaseModel, validator from pydantic import Field as PydanticField from . import constants diff --git a/tests/data/fixtures/db.yaml b/tests/data/fixtures/db.yaml index 1018f3e623..9bea5c2e30 100644 --- a/tests/data/fixtures/db.yaml +++ b/tests/data/fixtures/db.yaml @@ -10252,6 +10252,25 @@ due_date: null started: null completed: null +- model: udf.UserDefinedForm + pk: 1 + fields: + name: demo form + description: a demo form + schema: { + fields: [ + {name: "field1", type: "char"}, + {name: "field2", type: "integer"}, + ] + } + creator: + - pm@hawcproject.org + editors: + - - team@hawcproject.org + parent: null + deprecated: false + created: 2023-09-17 04:10:29.880052+00:00 + last_updated: 2023-09-17 04:15:33.309627+00:00 - model: reversion.revision pk: 1 fields: From f853b495940f179564811ecf6cd30425a7eb186d Mon Sep 17 00:00:00 2001 From: Andy Shapiro Date: Sun, 17 Sep 2023 09:49:13 -0400 Subject: [PATCH 34/35] dont hard-code model related models --- hawc/apps/udf/migrations/0001_initial.py | 8 ++++---- hawc/apps/udf/models.py | 15 ++++++++------- hawc/apps/udf/urls.py | 7 +------ 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/hawc/apps/udf/migrations/0001_initial.py b/hawc/apps/udf/migrations/0001_initial.py index 75aebda00c..f69a4663b3 100644 --- a/hawc/apps/udf/migrations/0001_initial.py +++ b/hawc/apps/udf/migrations/0001_initial.py @@ -29,15 +29,15 @@ class Migration(migrations.Migration): ( "creator", models.ForeignKey( - on_delete=django.db.models.deletion.DO_NOTHING, - related_name="created_forms", + on_delete=django.db.models.deletion.PROTECT, + related_name="udf_forms_creator", to=settings.AUTH_USER_MODEL, ), ), ( "editors", models.ManyToManyField( - blank=True, related_name="editable_forms", to=settings.AUTH_USER_MODEL + blank=True, related_name="udf_forms", to=settings.AUTH_USER_MODEL ), ), ( @@ -52,7 +52,7 @@ class Migration(migrations.Migration): ), ], options={ - "ordering": ["-last_updated"], + "ordering": ("-last_updated",), "unique_together": {("creator", "name")}, }, ), diff --git a/hawc/apps/udf/models.py b/hawc/apps/udf/models.py index 254dadd392..b7525e725a 100644 --- a/hawc/apps/udf/models.py +++ b/hawc/apps/udf/models.py @@ -1,17 +1,18 @@ import reversion +from django.conf import settings from django.db import models -from ..myuser.models import HAWCUser - class UserDefinedForm(models.Model): name = models.CharField(max_length=128) description = models.TextField() schema = models.JSONField() - creator = models.ForeignKey(HAWCUser, on_delete=models.DO_NOTHING, related_name="created_forms") - editors = models.ManyToManyField(HAWCUser, blank=True, related_name="editable_forms") + creator = models.ForeignKey( + settings.AUTH_USER_MODEL, on_delete=models.PROTECT, related_name="udf_forms_creator" + ) + editors = models.ManyToManyField(settings.AUTH_USER_MODEL, blank=True, related_name="udf_forms") parent = models.ForeignKey( - "udf.UserDefinedForm", + "self", blank=True, null=True, on_delete=models.SET_NULL, @@ -22,8 +23,8 @@ class UserDefinedForm(models.Model): last_updated = models.DateTimeField(auto_now=True) class Meta: - unique_together = ["creator", "name"] - ordering = ["-last_updated"] + unique_together = (("creator", "name"),) + ordering = ("-last_updated",) reversion.register(UserDefinedForm) diff --git a/hawc/apps/udf/urls.py b/hawc/apps/udf/urls.py index fc611ed523..bede2cdd10 100644 --- a/hawc/apps/udf/urls.py +++ b/hawc/apps/udf/urls.py @@ -6,12 +6,7 @@ app_name = "udf" urlpatterns = ( [ - # Create a user defined form - path( - "create/", - views.CreateUDFView.as_view(), - name="udf_create", - ), + path("create/", views.CreateUDFView.as_view(), name="udf_create"), ] if settings.HAWC_FEATURES.ENABLE_UDF else [] From f7ea9c4882d0abce53d37a993d183d8e9d1de2cf Mon Sep 17 00:00:00 2001 From: Andy Shapiro Date: Sun, 17 Sep 2023 13:49:02 -0400 Subject: [PATCH 35/35] validate field name via regex --- hawc/apps/common/dynamic_forms/fields.py | 2 +- hawc/apps/common/dynamic_forms/forms.py | 2 +- .../hawc/apps/common/dynamic_forms/test_fields.py | 14 ++++++++++++++ 3 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 tests/hawc/apps/common/dynamic_forms/test_fields.py diff --git a/hawc/apps/common/dynamic_forms/fields.py b/hawc/apps/common/dynamic_forms/fields.py index 68696234e2..c8dc7d482b 100644 --- a/hawc/apps/common/dynamic_forms/fields.py +++ b/hawc/apps/common/dynamic_forms/fields.py @@ -19,7 +19,7 @@ class _Field(BaseModel): in constants.Widget. """ - name: str # the variable name in the form; extra validation for no whitespace etc? + name: str = PydanticField(regex=r"^[a-zA-Z0-9_]+$") required: bool | None label: str | None label_suffix: str | None diff --git a/hawc/apps/common/dynamic_forms/forms.py b/hawc/apps/common/dynamic_forms/forms.py index 2944dd96df..e11cd75f73 100644 --- a/hawc/apps/common/dynamic_forms/forms.py +++ b/hawc/apps/common/dynamic_forms/forms.py @@ -88,7 +88,7 @@ def auto_wrap_fields(self): css_class = field.css_class or "col-12" self[index].wrap(cfl.Column, css_class=css_class) - self[:].wrap_together(cfl.Row, id="row_id_dynamic_form") + self[:].wrap_together(cfl.Row) self.add_field_wraps() diff --git a/tests/hawc/apps/common/dynamic_forms/test_fields.py b/tests/hawc/apps/common/dynamic_forms/test_fields.py new file mode 100644 index 0000000000..d05b00eca0 --- /dev/null +++ b/tests/hawc/apps/common/dynamic_forms/test_fields.py @@ -0,0 +1,14 @@ +import pytest +from pydantic import ValidationError + +from hawc.apps.common.dynamic_forms.fields import _Field + + +class TestField: + def test_name_regex(self): + for name in ["valid", "VALID", "aZ_1"]: + _Field(name=name) + + for name in ["a space", "a+plus", "a.dot"]: + with pytest.raises(ValidationError): + _Field(name=name)