Visual data export - rob heatmap/barcharts (#892)

* remove inheritance from summary viz apis

* add download link for rob datasets

* only add some items based on permission

* update helper method

---------

Co-authored-by: munnsmunns <mmunns16@gmail.com>

frontend/summary/summary/RoBHeatmap.js

@@ -25,6 +25,23 @@ class RoBHeatmap extends BaseVisual {
         delete this.data.studies;
     }
 
+    addActionsMenu() {
+        const items = [
+            "Download data file",
+            {url: this.data.data_url + "?format=xlsx", text: "Download (xlsx)"},
+        ];
+        if (window.isEditable) {
+            items.push(
+                ...[
+                    "Visualization editing",
+                    {url: this.data.url_update, text: "Update"},
+                    {url: this.data.url_delete, text: "Delete"},
+                ]
+            );
+        }
+        return HAWCUtils.pageActionsButton(items);
+    }
+
     displayAsPage($el, options) {
         var title = $("<h2>").text(this.data.title),
             captionDiv = $("<div>").html(this.data.caption),
@@ -34,15 +51,13 @@ class RoBHeatmap extends BaseVisual {
 
         options = options || {};
 
-        const actions = window.isEditable ? this.addActionsMenu() : null;
-
         $el.empty().append($plotDiv);
 
         if (!options.visualOnly) {
             var headerRow = $('<div class="d-flex">').append([
                 title,
                 HAWCUtils.unpublished(this.data.published, window.isEditable),
-                actions,
+                this.addActionsMenu(),
             ]);
             $el.prepend(headerRow).append(captionDiv);
         }

hawc/apps/common/models.py

@@ -1,5 +1,6 @@
 import logging
 import math
+from html import unescape
 
 import pandas as pd
 from django.apps import apps
@@ -12,6 +13,7 @@
 from django.db.models import Case, CharField, Choices, Q, QuerySet, URLField, Value, When
 from django.db.models.functions import Coalesce
 from django.template.defaultfilters import slugify as default_slugify
+from django.utils.html import strip_tags
 from treebeard.mp_tree import MP_Node
 
 from . import forms, validators
@@ -592,6 +594,17 @@ def to_display_array(series: pd.Series, Choice: type[Choices], delimiter: str =
     )
 
 
+def pd_strip_tags(df: pd.DataFrame, columns: list[str]):
+    """Remove tags from text columns a DataFrame; in place
+
+    Args:
+        df (pd.DataFrame): the DataFrame to clean
+        columns (list[str]): column names to remove tags
+    """
+    for col in columns:
+        df.loc[:, col] = df[col].apply(lambda txt: unescape(strip_tags(txt)))
+
+
 class NumericTextField(models.CharField):
     generic_help_text = "Non-numeric values can be used if necessary, but should be limited to <, ≤, ≥, >, LOD, LOQ."
     validators = [validators.NumericTextValidator()]

hawc/apps/riskofbias/managers.py

@@ -1,8 +1,11 @@
+import pandas as pd
 from django.contrib.contenttypes.models import ContentType
 from django.db import models
-from django.db.models import Case, Count, IntegerField, Sum, Value, When
+from django.db.models import Case, Count, IntegerField, QuerySet, Sum, Value, When
 
-from ..common.models import BaseManager
+from ..common.models import BaseManager, pd_strip_tags, sql_display
+from ..study.managers import study_df_annotations, study_df_mapping
+from . import constants
 
 
 class RiskOfBiasDomainManager(BaseManager):
@@ -91,9 +94,64 @@ def get_required_robs_for_metric(self, metric):
         return self.get_qs(assessment.id).filter(filters)
 
 
+class RiskOfBiasScoreQuerySet(QuerySet):
+    def df(self) -> pd.DataFrame:
+        mapping = {
+            # study
+            **study_df_mapping("study-", "riskofbias__study__"),
+            # rob
+            "rob-id": "riskofbias_id",
+            "rob-created": "riskofbias__created",
+            "rob-last_updated": "riskofbias__last_updated",
+            # domain
+            "rob-domain_id": "metric__domain_id",
+            "rob-domain_name": "metric__domain__name",
+            "rob-domain_description": "metric__description",
+            # metric
+            "rob-metric_id": "metric_id",
+            "rob-metric_name": "metric__name",
+            "rob-metric_description": "metric__description",
+            # score
+            "rob-score_id": "id",
+            "rob-score_is_default": "is_default",
+            "rob-score_label": "label",
+            "rob-score": "score",
+            "rob-score_display": Value("-"),
+            "rob-score_symbol": Value("-"),
+            "rob-score_shade": Value("-"),
+            "rob-score_bias_direction": "bias_direction",
+            "rob-score_bias_direction_display": sql_display(
+                "bias_direction", constants.BiasDirections
+            ),
+            "rob-score_notes": "notes",
+        }
+        qs = (
+            self.annotate(**study_df_annotations("riskofbias__study__"))
+            .values_list(*list(mapping.values()))
+            .order_by("riskofbias__study__id", "metric_id", "id")
+        )
+        df = pd.DataFrame(data=qs, columns=list(mapping.keys()))
+        df.loc[:, "rob-score_display"] = df["rob-score"].map(constants.SCORE_CHOICES_MAP)
+        df.loc[:, "rob-score_symbol"] = df["rob-score"].map(constants.SCORE_SYMBOLS)
+        df.loc[:, "rob-score_shade"] = df["rob-score"].map(constants.SCORE_SHADES)
+        pd_strip_tags(
+            df,
+            [
+                "study-summary",
+                "rob-domain_description",
+                "rob-metric_description",
+                "rob-score_notes",
+            ],
+        )
+        return df
+
+
 class RiskOfBiasScoreManager(BaseManager):
     assessment_relation = "riskofbias__study__assessment"
 
+    def get_queryset(self):
+        return RiskOfBiasScoreQuerySet(self.model, using=self._db)
+
 
 class RiskOfBiasScoreOverrideObjectManager(BaseManager):
     def get_queryset(self):

hawc/apps/summary/api.py

@@ -5,6 +5,7 @@
 from rest_framework.decorators import action
 from rest_framework.filters import BaseFilterBackend
 from rest_framework.response import Response
+from rest_framework.status import HTTP_400_BAD_REQUEST
 
 from ..assessment.api import (
     AssessmentEditViewSet,
@@ -16,7 +17,7 @@
 from ..assessment.constants import AssessmentViewSetPermissions
 from ..assessment.models import Assessment
 from ..common.api import DisabledPagination
-from ..common.helper import re_digits
+from ..common.helper import FlatExport, re_digits
 from ..common.renderers import DocxRenderer, PandasRenderers
 from ..common.serializers import UnusedSerializer
 from . import models, serializers, table_serializers
@@ -113,6 +114,22 @@ def get_serializer_class(self):
     def get_queryset(self):
         return super().get_queryset().select_related("assessment")
 
+    @action(
+        detail=True,
+        action_perms=AssessmentViewSetPermissions.CAN_VIEW_OBJECT,
+        renderer_classes=PandasRenderers,
+    )
+    def data(self, request, pk):
+        obj = self.get_object()
+        try:
+            df = obj.data_df()
+        except ValueError:
+            return Response(
+                {"error": "Data export not available for this visual type."},
+                status=HTTP_400_BAD_REQUEST,
+            )
+        return FlatExport.api_response(df, obj.slug)
+
 
 class SummaryTextViewSet(EditPermissionsCheckMixin, AssessmentEditViewSet):
     edit_check_keys = ["assessment"]

hawc/apps/summary/models.py

@@ -46,6 +46,7 @@
 from ..epiv2.models import DataExtraction
 from ..invitro import exports as ivexports
 from ..invitro.models import IVEndpoint
+from ..riskofbias.models import RiskOfBiasScore
 from ..riskofbias.serializers import AssessmentRiskOfBiasSerializer
 from ..study.models import Study
 from . import constants, managers
@@ -344,6 +345,9 @@ def get_assessment(self):
     def get_api_detail(self):
         return reverse("summary:api:visual-detail", args=(self.id,))
 
+    def get_data_url(self):
+        return reverse("summary:api:visual-data", args=(self.id,))
+
     def get_api_heatmap_datasets(self):
         return reverse("summary:api:assessment-heatmap-datasets", args=(self.assessment_id,))
 
@@ -456,6 +460,12 @@ def get_heatmap_datasets(cls, assessment: Assessment) -> HeatmapDatasets:
     def get_dose_units():
         return DoseUnits.objects.json_all()
 
+    def get_settings(self) -> dict | None:
+        try:
+            return json.loads(self.settings)
+        except ValueError:
+            return None
+
     def get_json(self, json_encode=True):
         return SerializerHelper.get_serialized(self, json=json_encode)
 
@@ -600,6 +610,31 @@ def get_plotly_from_json(self) -> Figure:
         except ValueError as err:
             raise ValueError(err)
 
+    def _rob_data_qs(self, use_settings: bool = True) -> models.QuerySet:
+        study_ids = list(self.get_studies().values_list("id", flat=True))
+        settings = json.loads(self.settings)
+
+        qs = RiskOfBiasScore.objects.filter(
+            riskofbias__active=True,
+            riskofbias__final=True,
+            riskofbias__study__in=study_ids,
+        )
+
+        # use settings in read-only view; dont use when configuring plot
+        if use_settings:
+            qs = qs.filter(metric__in=settings["included_metrics"]).exclude(
+                id__in=settings.get("excluded_score_ids", [])
+            )
+        return qs
+
+    def data_df(self, use_settings: bool = True) -> pd.DataFrame:
+        if self.visual_type not in [
+            constants.VisualType.ROB_BARCHART,
+            constants.VisualType.ROB_HEATMAP,
+        ]:
+            raise ValueError("Not supported for this visual type")
+        return self._rob_data_qs(use_settings=use_settings).df()
+
 
 class DataPivot(models.Model):
     objects = managers.DataPivotManager()

hawc/apps/summary/serializers.py

@@ -1,5 +1,3 @@
-import json
-
 from django.core.exceptions import ValidationError
 from django.db import transaction
 from rest_framework import serializers
@@ -11,53 +9,46 @@
 
 
 class CollectionDataPivotSerializer(serializers.ModelSerializer):
-    def to_representation(self, instance):
-        ret = super().to_representation(instance)
-        ret["url"] = instance.get_absolute_url()
-        ret["visual_type"] = instance.visual_type
-        return ret
+    url = serializers.CharField(source="get_absolute_url")
+    visual_type = serializers.CharField(source="get_visual_type_display")
 
     class Meta:
         model = models.DataPivot
-        fields = "__all__"
+        fields = ("id", "title", "url", "visual_type")
 
 
-class DataPivotSerializer(CollectionDataPivotSerializer):
-    def to_representation(self, instance):
-        ret = super().to_representation(instance)
-        ret["data_url"] = instance.get_data_url()
-        ret["download_url"] = instance.get_download_url()
-        return ret
+class DataPivotSerializer(serializers.ModelSerializer):
+    url = serializers.CharField(source="get_absolute_url")
+    data_url = serializers.CharField(source="get_data_url")
+    download_url = serializers.CharField(source="get_download_url")
+    visual_type = serializers.CharField(source="get_visual_type_display")
+
+    class Meta:
+        model = models.DataPivot
+        fields = "__all__"
 
 
 class CollectionVisualSerializer(serializers.ModelSerializer):
-    def to_representation(self, instance):
-        ret = super().to_representation(instance)
-        if instance.id != instance.FAKE_INITIAL_ID:
-            ret["url"] = instance.get_absolute_url()
-        ret["visual_type"] = instance.get_rob_visual_type_display(
-            instance.get_visual_type_display()
-        )
-        try:
-            settings = json.loads(instance.settings)
-        except json.JSONDecodeError:
-            settings = {}
-        ret["settings"] = settings
-        return ret
+    url = serializers.CharField(source="get_absolute_url")
+    visual_type = serializers.CharField(source="get_visual_type_display")
+    data_url = serializers.CharField(source="get_data_url")
 
     class Meta:
         model = models.Visual
-        exclude = (
-            "endpoints",
-            "studies",
-        )
+        fields = ("id", "title", "url", "visual_type", "visual_type", "data_url")
 
 
-class VisualSerializer(CollectionVisualSerializer):
+class VisualSerializer(serializers.ModelSerializer):
+    url = serializers.CharField(source="get_absolute_url")
+    visual_type = serializers.CharField(source="get_visual_type_display")
+    settings = serializers.JSONField(source="get_settings")
+    data_url = serializers.CharField(source="get_data_url")
+
     def to_representation(self, instance):
         ret = super().to_representation(instance)
 
         if instance.id != instance.FAKE_INITIAL_ID:
+            ret["url"] = instance.get_absolute_url()
             ret["url_update"] = instance.get_update_url()
             ret["url_delete"] = instance.get_delete_url()
 
@@ -80,6 +71,10 @@ def to_representation(self, instance):
 
         return ret
 
+    class Meta:
+        model = models.Visual
+        exclude = ("slug", "prefilters", "studies", "endpoints")
+
 
 class SummaryTextSerializer(serializers.ModelSerializer):
     parent = serializers.PrimaryKeyRelatedField(

tests/data/api/api-visual-bioassay-aggregation.json

@@ -4,6 +4,7 @@
   "assessment_rob_name": "Risk of bias",
   "caption": "",
   "created": "2020-11-25T13:32:07.260499-05:00",
+  "data_url": "/summary/api/visual/10/data/",
   "dose_units": 1,
   "endpoints": [
     {
@@ -361,10 +362,8 @@
   ],
   "id": 10,
   "last_updated": "2020-11-25T13:35:00.094667-05:00",
-  "prefilters": "{}",
   "published": true,
   "settings": {},
-  "slug": "bioassay-aggregation",
   "sort_order": "short_citation",
   "studies": [],
   "title": "bioassay-aggregation",

tests/data/api/api-visual-crossview.json

@@ -4,6 +4,7 @@
   "assessment_rob_name": "Risk of bias",
   "caption": "<p>example</p>",
   "created": "2020-05-08T15:37:19.255703-04:00",
+  "data_url": "/summary/api/visual/4/data/",
   "dose_units": 1,
   "endpoints": [
     {
@@ -3246,7 +3247,6 @@
   ],
   "id": 4,
   "last_updated": "2020-05-08T15:37:19.255730-04:00",
-  "prefilters": "{\"animal_group__experiment__study__published\": true}",
   "published": true,
   "settings": {
     "colorBase": "#cccccc",
@@ -3328,7 +3328,6 @@
     "ylabel_x": 0,
     "ylabel_y": 0
   },
-  "slug": "crossview",
   "sort_order": "short_citation",
   "studies": [],
   "title": "crossview",