feat: send data embedded in report email

superset-frontend/src/views/CRUD/alert/AlertReportModal.tsx

@@ -1258,6 +1258,7 @@ const AlertReportModal: FunctionComponent<AlertReportModalProps> = ({
                 >
                   <StyledRadio value="PNG">{t('Send as PNG')}</StyledRadio>
                   <StyledRadio value="CSV">{t('Send as CSV')}</StyledRadio>
+                  <StyledRadio value="TEXT">{t('Send as text')}</StyledRadio>
                 </StyledRadioGroup>
               </div>
             )}

superset-frontend/src/views/CRUD/alert/types.ts

@@ -74,7 +74,7 @@ export type AlertObject = {
   owners?: Array<Owner | MetaObject>;
   sql?: string;
   recipients?: Array<Recipient>;
-  report_format?: 'PNG' | 'CSV';
+  report_format?: 'PNG' | 'CSV' | 'TEXT';
   type?: string;
   validator_config_json?: {
     op?: Operator;

superset/models/reports.py

@@ -72,6 +72,7 @@ class ReportState(str, enum.Enum):
 class ReportDataFormat(str, enum.Enum):
     VISUALIZATION = "PNG"
     DATA = "CSV"
+    TEXT = "TEXT"
 
 
 report_schedule_user = Table(

superset/reports/commands/execute.py

@@ -17,9 +17,11 @@
 import json
 import logging
 from datetime import datetime, timedelta
+from io import BytesIO
 from typing import Any, List, Optional
 from uuid import UUID
 
+import pandas as pd
 from celery.exceptions import SoftTimeLimitExceeded
 from flask_appbuilder.security.sqla.models import User
 from sqlalchemy.orm import Session
@@ -207,11 +209,10 @@ def _get_screenshot(self) -> bytes:
         return image_data
 
     def _get_csv_data(self) -> bytes:
-        if self._report_schedule.chart:
-            url = self._get_url(csv=True)
-            auth_cookies = machine_auth_provider_factory.instance.get_auth_cookies(
-                self._get_user()
-            )
+        url = self._get_url(csv=True)
+        auth_cookies = machine_auth_provider_factory.instance.get_auth_cookies(
+            self._get_user()
+        )
         try:
             csv_data = get_chart_csv_data(url, auth_cookies)
         except SoftTimeLimitExceeded:
@@ -222,13 +223,22 @@ def _get_csv_data(self) -> bytes:
             raise ReportScheduleCsvFailedError()
         return csv_data
 
+    def _get_embedded_data(self) -> str:
+        """
+        Return data as an HTML table, to embed in the email.
+        """
+        buf = BytesIO(self._get_csv_data())
+        df = pd.read_csv(buf)
+        return df.to_html(na_rep="null")
+
     def _get_notification_content(self) -> NotificationContent:
         """
         Gets a notification content, this is composed by a title and a screenshot
 
         :raises: ReportScheduleScreenshotFailedError
         """
         csv_data = None
+        embedded_data = None
         error_text = None
         screenshot_data = None
         url = self._get_url(user_friendly=True)
@@ -252,6 +262,12 @@ def _get_notification_content(self) -> NotificationContent:
                     name=self._report_schedule.name, text=error_text
                 )
 
+        if (
+            self._report_schedule.chart
+            and self._report_schedule.report_format == ReportDataFormat.TEXT
+        ):
+            embedded_data = self._get_embedded_data()
+
         if self._report_schedule.chart:
             name = (
                 f"{self._report_schedule.name}: "
@@ -268,6 +284,7 @@ def _get_notification_content(self) -> NotificationContent:
             screenshot=screenshot_data,
             description=self._report_schedule.description,
             csv=csv_data,
+            embedded_data=embedded_data,
         )
 
     def _send(

superset/reports/notifications/base.py

@@ -29,6 +29,7 @@ class NotificationContent:
     text: Optional[str] = None
     description: Optional[str] = ""
     url: Optional[str] = None  # url to chart/dashboard for this screenshot
+    embedded_data: Optional[str] = ""
 
 
 class BaseNotification:  # pylint: disable=too-few-public-methods

superset/reports/notifications/email.py

@@ -21,6 +21,7 @@
 from email.utils import make_msgid, parseaddr
 from typing import Any, Dict, Optional
 
+import bleach
 from flask_babel import gettext as __
 
 from superset import app
@@ -31,6 +32,8 @@
 
 logger = logging.getLogger(__name__)
 
+TABLE_TAGS = ["table", "th", "tr", "td", "thead", "tbody", "tfoot"]
+
 
 @dataclass
 class EmailContent:
@@ -68,14 +71,23 @@ def _get_content(self) -> EmailContent:
         csv_data = None
         domain = self._get_smtp_domain()
         msgid = make_msgid(domain)[1:-1]
+
+        # Strip any malicious HTML from the description
+        description = bleach.clean(self._content.description or "")
+
+        # Strip malicious HTML from embedded data, allowing table elements
+        embedded_data = bleach.clean(self._content.embedded_data or "", tags=TABLE_TAGS)
+
         body = __(
             """
             <p>%(description)s</p>
             <b><a href="%(url)s">Explore in Superset</a></b><p></p>
+            %(embedded_data)s
             %(img_tag)s
             """,
-            description=self._content.description or "",
+            description=description,
             url=self._content.url,
+            embedded_data=embedded_data,
             img_tag='<img width="1000px" src="cid:{}">'.format(msgid)
             if self._content.screenshot
             else "",

superset/viz.py

@@ -21,7 +21,6 @@
 Superset can render.
 """
 import copy
-import inspect
 import logging
 import math
 import re
@@ -53,7 +52,7 @@
 from geopy.point import Point
 from pandas.tseries.frequencies import to_offset
 
-from superset import app, db, is_feature_enabled
+from superset import app, is_feature_enabled
 from superset.constants import NULL_STRING
 from superset.errors import ErrorLevel, SupersetError, SupersetErrorType
 from superset.exceptions import (
@@ -64,7 +63,6 @@
     SupersetSecurityException,
 )
 from superset.extensions import cache_manager, security_manager
-from superset.models.cache import CacheKey
 from superset.models.helpers import QueryResult
 from superset.typing import Metric, QueryObjectDict, VizData, VizPayload
 from superset.utils import core as utils, csv
@@ -619,12 +617,27 @@ def data(self) -> Dict[str, Any]:
 
     def get_csv(self) -> Optional[str]:
         df = self.get_df_payload()["df"]  # leverage caching logic
+        df = self.post_process(df)
         include_index = not isinstance(df.index, pd.RangeIndex)
         return csv.df_to_escaped_csv(df, index=include_index, **config["CSV_EXPORT"])
 
     def get_data(self, df: pd.DataFrame) -> VizData:
         return df.to_dict(orient="records")
 
+    def post_process(self, df: pd.DataFrame) -> pd.DataFrame:
+        """
+        Post-process data to return same format as visualization.
+
+        Some visualizations post-process the data in the frontend before presenting
+        it to the user. Eg, the t-test visualization will compute p-values and
+        significance in Javascript, while the pivot table will pivot the data.
+
+        When we produce a report we want to send to the user the exact same data
+        that they see in the chart. For visualizations with post-processing we need
+        to compute the perform the same processing in Python.
+        """
+        return df
+
     @property
     def json_data(self) -> str:
         return json.dumps(self.data)
@@ -896,7 +909,7 @@ def _format_datetime(value: Union[pd.Timestamp, datetime, date, str]) -> str:
         # fallback in case something incompatible is returned
         return cast(str, value)
 
-    def get_data(self, df: pd.DataFrame) -> VizData:
+    def _pivot_data(self, df: pd.DataFrame) -> Optional[pd.DataFrame]:
         if df.empty:
             return None
 
@@ -934,6 +947,11 @@ def get_data(self, df: pd.DataFrame) -> VizData:
         # Display metrics side by side with each column
         if self.form_data.get("combine_metric"):
             df = df.stack(0).unstack().reindex(level=-1, columns=metrics)
+
+        return df
+
+    def get_data(self, df: pd.DataFrame) -> VizData:
+        df = self._pivot_data(df)
         return dict(
             columns=list(df.columns),
             html=df.to_html(
@@ -945,6 +963,9 @@ def get_data(self, df: pd.DataFrame) -> VizData:
             ),
         )
 
+    def post_process(self, df: pd.DataFrame) -> pd.DataFrame:
+        return self._pivot_data(df)
+
 
 class TreemapViz(BaseViz):