diff --git a/superset/utils/csv.py b/superset/utils/csv.py index 0dc84ff36a3d..7981c9e872dd 100644 --- a/superset/utils/csv.py +++ b/superset/utils/csv.py @@ -14,6 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +import logging import re import urllib.request from typing import Any, Dict, Optional @@ -22,6 +23,10 @@ import pandas as pd import simplejson +from superset.utils.core import GenericDataType + +logger = logging.getLogger(__name__) + negative_number_re = re.compile(r"^-[0-9.]+$") # This regex will match if the string starts with: @@ -97,11 +102,22 @@ def get_chart_dataframe( return None result = simplejson.loads(content.decode("utf-8")) - # need to convert float value to string to show full long number pd.set_option("display.float_format", lambda x: str(x)) df = pd.DataFrame.from_dict(result["result"][0]["data"]) + try: + # if any column type is equal to 2, need to convert data into + # datetime timestamp for that column. + if GenericDataType.TEMPORAL in result["result"][0]["coltypes"]: + for i in range(len(result["result"][0]["coltypes"])): + if result["result"][0]["coltypes"][i] == GenericDataType.TEMPORAL: + df[result["result"][0]["colnames"][i]] = df[ + result["result"][0]["colnames"][i] + ].astype("datetime64[ms]") + except BaseException as err: + logger.error(err) + # rebuild hierarchical columns and index df.columns = pd.MultiIndex.from_tuples( tuple(colname) if isinstance(colname, list) else (colname,) diff --git a/tests/integration_tests/reports/commands_tests.py b/tests/integration_tests/reports/commands_tests.py index dd23d291fd69..63a54ff56182 100644 --- a/tests/integration_tests/reports/commands_tests.py +++ b/tests/integration_tests/reports/commands_tests.py @@ -16,7 +16,7 @@ # under the License. import json from contextlib import contextmanager -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from typing import Any, Dict, List, Optional from unittest.mock import Mock, patch from uuid import uuid4 @@ -960,6 +960,8 @@ def test_email_chart_report_schedule_with_text( mock_open.return_value = response mock_urlopen.return_value = response mock_urlopen.return_value.getcode.return_value = 200 + + # test without date type. response.read.return_value = json.dumps( { "result": [ @@ -971,6 +973,7 @@ def test_email_chart_report_schedule_with_text( }, "colnames": [("t1",), ("t2",), ("t3__sum",)], "indexnames": [(0,), (1,)], + "coltypes": [1, 1], }, ], } @@ -1011,6 +1014,59 @@ def test_email_chart_report_schedule_with_text( # Assert logs are correct assert_log(ReportState.SUCCESS) + # test with date type. + dt = datetime(2022, 1, 1).replace(tzinfo=timezone.utc) + ts = datetime.timestamp(dt) * 1000 + response.read.return_value = json.dumps( + { + "result": [ + { + "data": { + "t1": {0: "c11", 1: "c21"}, + "t2__date": {0: ts, 1: ts}, + "t3__sum": {0: "c13", 1: "c23"}, + }, + "colnames": [("t1",), ("t2__date",), ("t3__sum",)], + "indexnames": [(0,), (1,)], + "coltypes": [1, 2], + }, + ], + } + ).encode("utf-8") + + with freeze_time("2020-01-01T00:00:00Z"): + AsyncExecuteReportScheduleCommand( + TEST_ID, create_report_email_chart_with_text.id, datetime.utcnow() + ).run() + + # assert that the data is embedded correctly + table_html = """ + + + + + + + + + + + + + + + + + + + + + + +
t1t2__datet3__sum
0c112022-01-01c13
1c212022-01-01c23
""" + + assert table_html in email_mock.call_args[0][2] + @pytest.mark.usefixtures( "load_birth_names_dashboard_with_slices", "create_report_email_dashboard"