From 1c21505177a57ba619b14cb4502f930877d026b1 Mon Sep 17 00:00:00 2001 From: aniaan Date: Sat, 9 Oct 2021 16:02:23 +0800 Subject: [PATCH 01/11] fix(elasticsearch): cast does not take effect for time zone settings --- superset/connectors/sqla/models.py | 35 ++++++++++++++++------- superset/connectors/sqla/utils.py | 4 ++- superset/db_engine_specs/athena.py | 4 ++- superset/db_engine_specs/base.py | 8 ++++-- superset/db_engine_specs/bigquery.py | 4 ++- superset/db_engine_specs/clickhouse.py | 6 ++-- superset/db_engine_specs/crate.py | 6 ++-- superset/db_engine_specs/databricks.py | 6 ++-- superset/db_engine_specs/dremio.py | 6 ++-- superset/db_engine_specs/drill.py | 6 ++-- superset/db_engine_specs/druid.py | 4 ++- superset/db_engine_specs/elasticsearch.py | 23 +++++++++++---- superset/db_engine_specs/firebird.py | 6 ++-- superset/db_engine_specs/firebolt.py | 6 ++-- superset/db_engine_specs/hana.py | 6 ++-- superset/db_engine_specs/hive.py | 4 ++- superset/db_engine_specs/impala.py | 6 ++-- superset/db_engine_specs/kylin.py | 6 ++-- superset/db_engine_specs/mssql.py | 4 ++- superset/db_engine_specs/mysql.py | 5 +++- superset/db_engine_specs/oracle.py | 6 ++-- superset/db_engine_specs/postgres.py | 7 +++-- superset/db_engine_specs/presto.py | 5 +++- superset/db_engine_specs/rockset.py | 6 ++-- superset/db_engine_specs/snowflake.py | 4 ++- superset/db_engine_specs/sqlite.py | 4 ++- superset/db_engine_specs/trino.py | 4 ++- superset/models/sql_types/base.py | 12 ++++++-- tests/integration_tests/model_tests.py | 4 ++- 29 files changed, 148 insertions(+), 59 deletions(-) diff --git a/superset/connectors/sqla/models.py b/superset/connectors/sqla/models.py index a27a4a1631e2..66d67dc5e9eb 100644 --- a/superset/connectors/sqla/models.py +++ b/superset/connectors/sqla/models.py @@ -247,17 +247,23 @@ def is_temporal(self) -> bool: def db_engine_spec(self) -> Type[BaseEngineSpec]: return self.table.db_engine_spec + @property + def db_extra(self) -> Dict[str, Any]: + return self.table.database.get_extra() + @property def type_generic(self) -> Optional[utils.GenericDataType]: if self.is_dttm: return GenericDataType.TEMPORAL - column_spec = self.db_engine_spec.get_column_spec(self.type) + column_spec = self.db_engine_spec.get_column_spec( + self.type, db_extra=self.db_extra + ) return column_spec.generic_type if column_spec else None def get_sqla_col(self, label: Optional[str] = None) -> Column: label = label or self.column_name db_engine_spec = self.db_engine_spec - column_spec = db_engine_spec.get_column_spec(self.type) + column_spec = db_engine_spec.get_column_spec(self.type, db_extra=self.db_extra) type_ = column_spec.sqla_type if column_spec else None if self.expression: tp = self.table.get_template_processor() @@ -312,7 +318,9 @@ def get_timestamp_expression( pdf = self.python_date_format is_epoch = pdf in ("epoch_s", "epoch_ms") - column_spec = self.db_engine_spec.get_column_spec(self.type) + column_spec = self.db_engine_spec.get_column_spec( + self.type, db_extra=self.db_extra + ) type_ = column_spec.sqla_type if column_spec else DateTime if not self.expression and not time_grain and not is_epoch: sqla_col = column(self.column_name, type_=type_) @@ -335,7 +343,11 @@ def dttm_sql_literal( ) -> str: """Convert datetime object to a SQL expression string""" dttm_type = self.type or ("DATETIME" if self.is_dttm else None) - sql = self.db_engine_spec.convert_dttm(dttm_type, dttm) if dttm_type else None + sql = ( + self.db_engine_spec.convert_dttm(dttm_type, dttm, **self.db_extra) + if dttm_type + else None + ) if sql: return sql @@ -348,10 +360,8 @@ def dttm_sql_literal( utils.TimeRangeEndpoint.INCLUSIVE, utils.TimeRangeEndpoint.EXCLUSIVE, ): - tf = ( - self.table.database.get_extra() - .get("python_date_format_by_column_name", {}) - .get(self.column_name) + tf = self.db_extra.get("python_date_format_by_column_name", {}).get( + self.column_name ) if tf: @@ -1174,7 +1184,9 @@ def get_sqla_query( # pylint: disable=too-many-arguments,too-many-locals,too-ma sqla_col = col_obj.get_timestamp_expression(filter_grain) else: sqla_col = col_obj.get_sqla_col() - col_spec = db_engine_spec.get_column_spec(col_obj.type) + col_spec = db_engine_spec.get_column_spec( + col_obj.type, db_extra=self.database.get_extra() + ) is_list_target = op in ( utils.FilterOperator.IN.value, utils.FilterOperator.NOT_IN.value, @@ -1421,6 +1433,7 @@ def _get_series_orderby( def _get_top_groups( self, df: pd.DataFrame, dimensions: List[str], groupby_exprs: Dict[str, Any], ) -> ColumnElement: + db_extra: Dict[str, Any] = self.database.get_extra() column_map = {column.column_name: column for column in self.columns} groups = [] for _unused, row in df.iterrows(): @@ -1434,7 +1447,9 @@ def _get_top_groups( # string into a timestamp. if column_map[dimension].is_temporal and isinstance(value, str): dttm = dateutil.parser.parse(value) - value = text(self.db_engine_spec.convert_dttm("TIMESTAMP", dttm)) + value = text( + self.db_engine_spec.convert_dttm("TIMESTAMP", dttm, **db_extra) + ) group.append(groupby_exprs[dimension] == value) groups.append(and_(*group)) diff --git a/superset/connectors/sqla/utils.py b/superset/connectors/sqla/utils.py index 0a95364e777c..e5209e08dcf6 100644 --- a/superset/connectors/sqla/utils.py +++ b/superset/connectors/sqla/utils.py @@ -57,7 +57,9 @@ def get_physical_table_metadata( db_type = db_engine_spec.column_datatype_to_string( col["type"], db_dialect ) - type_spec = db_engine_spec.get_column_spec(db_type) + type_spec = db_engine_spec.get_column_spec( + db_type, db_extra=database.get_extra() + ) col.update( { "type": db_type, diff --git a/superset/db_engine_specs/athena.py b/superset/db_engine_specs/athena.py index 2952d8267772..ff96692f9197 100644 --- a/superset/db_engine_specs/athena.py +++ b/superset/db_engine_specs/athena.py @@ -61,7 +61,9 @@ class AthenaEngineSpec(BaseEngineSpec): } @classmethod - def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]: + def convert_dttm( + cls, target_type: str, dttm: datetime, **kwargs: Any + ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.DATE: return f"from_iso8601_date('{dttm.date().isoformat()}')" diff --git a/superset/db_engine_specs/base.py b/superset/db_engine_specs/base.py index 74b451ec2da6..d88dd2c342e1 100644 --- a/superset/db_engine_specs/base.py +++ b/superset/db_engine_specs/base.py @@ -686,13 +686,14 @@ def df_to_sql( @classmethod def convert_dttm( # pylint: disable=unused-argument - cls, target_type: str, dttm: datetime, + cls, target_type: str, dttm: datetime, **kwargs: Any, ) -> Optional[str]: """ Convert Python datetime object to a SQL expression :param target_type: The target type of expression :param dttm: The datetime object + :param kwargs: The database extra object :return: The SQL expression """ return None @@ -1280,10 +1281,10 @@ def is_select_query(cls, parsed_query: ParsedQuery) -> bool: return parsed_query.is_select() @classmethod - @memoized def get_column_spec( # pylint: disable=unused-argument cls, native_type: Optional[str], + db_extra: Optional[Dict[str, Any]] = None, source: utils.ColumnTypeSource = utils.ColumnTypeSource.GET_TABLE, column_type_mappings: Tuple[ Tuple[ @@ -1298,6 +1299,7 @@ def get_column_spec( # pylint: disable=unused-argument Converts native database type to sqlalchemy column type. :param native_type: Native database typee :param source: Type coming from the database table or cursor description + :param db_extra: The database extra object :return: ColumnSpec object """ col_types = cls.get_sqla_column_type( @@ -1309,7 +1311,7 @@ def get_column_spec( # pylint: disable=unused-argument # using datetimes if generic_type == GenericDataType.TEMPORAL: column_type = literal_dttm_type_factory( - column_type, cls, native_type or "" + column_type, cls, native_type or "", db_extra=db_extra or {} ) is_dttm = generic_type == GenericDataType.TEMPORAL return ColumnSpec( diff --git a/superset/db_engine_specs/bigquery.py b/superset/db_engine_specs/bigquery.py index a862df58d9f3..4c86eb127441 100644 --- a/superset/db_engine_specs/bigquery.py +++ b/superset/db_engine_specs/bigquery.py @@ -184,7 +184,9 @@ class BigQueryEngineSpec(BaseEngineSpec): } @classmethod - def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]: + def convert_dttm( + cls, target_type: str, dttm: datetime, **kwargs: Any + ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.DATE: return f"CAST('{dttm.date().isoformat()}' AS DATE)" diff --git a/superset/db_engine_specs/clickhouse.py b/superset/db_engine_specs/clickhouse.py index 60a3584baf46..59fc8e59e0f0 100644 --- a/superset/db_engine_specs/clickhouse.py +++ b/superset/db_engine_specs/clickhouse.py @@ -16,7 +16,7 @@ # under the License. import logging from datetime import datetime -from typing import Dict, List, Optional, Type, TYPE_CHECKING +from typing import Any, Dict, List, Optional, Type, TYPE_CHECKING from urllib3.exceptions import NewConnectionError @@ -72,7 +72,9 @@ def get_dbapi_mapped_exception(cls, exception: Exception) -> Exception: return new_exception(str(exception)) @classmethod - def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]: + def convert_dttm( + cls, target_type: str, dttm: datetime, **kwargs: Any + ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.DATE: return f"toDate('{dttm.date().isoformat()}')" diff --git a/superset/db_engine_specs/crate.py b/superset/db_engine_specs/crate.py index a55d72fd0632..976e7985e9a9 100644 --- a/superset/db_engine_specs/crate.py +++ b/superset/db_engine_specs/crate.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. from datetime import datetime -from typing import Optional, TYPE_CHECKING +from typing import Any, Dict, Optional, TYPE_CHECKING from superset.db_engine_specs.base import BaseEngineSpec from superset.utils import core as utils @@ -50,7 +50,9 @@ def epoch_ms_to_dttm(cls) -> str: return "{col}" @classmethod - def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]: + def convert_dttm( + cls, target_type: str, dttm: datetime, **kwargs: Any + ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.TIMESTAMP: return f"{dttm.timestamp() * 1000}" diff --git a/superset/db_engine_specs/databricks.py b/superset/db_engine_specs/databricks.py index 7f0e44f78556..a6e3bab1afa7 100644 --- a/superset/db_engine_specs/databricks.py +++ b/superset/db_engine_specs/databricks.py @@ -16,7 +16,7 @@ # under the License.o from datetime import datetime -from typing import Optional +from typing import Any, Dict, Optional from superset.db_engine_specs.base import BaseEngineSpec from superset.db_engine_specs.hive import HiveEngineSpec @@ -40,7 +40,9 @@ class DatabricksODBCEngineSpec(BaseEngineSpec): _time_grain_expressions = HiveEngineSpec._time_grain_expressions @classmethod - def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]: + def convert_dttm( + cls, target_type: str, dttm: datetime, **kwargs: Any + ) -> Optional[str]: return HiveEngineSpec.convert_dttm(target_type, dttm) @classmethod diff --git a/superset/db_engine_specs/dremio.py b/superset/db_engine_specs/dremio.py index a76909bf3a8e..5710fd144638 100644 --- a/superset/db_engine_specs/dremio.py +++ b/superset/db_engine_specs/dremio.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. from datetime import datetime -from typing import Optional +from typing import Any, Dict, Optional from superset.db_engine_specs.base import BaseEngineSpec from superset.utils import core as utils @@ -43,7 +43,9 @@ def epoch_to_dttm(cls) -> str: return "TO_DATE({col})" @classmethod - def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]: + def convert_dttm( + cls, target_type: str, dttm: datetime, **kwargs: Any + ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.DATE: return f"TO_DATE('{dttm.date().isoformat()}', 'YYYY-MM-DD')" diff --git a/superset/db_engine_specs/drill.py b/superset/db_engine_specs/drill.py index 6578965778b6..43456896675a 100644 --- a/superset/db_engine_specs/drill.py +++ b/superset/db_engine_specs/drill.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. from datetime import datetime -from typing import Optional +from typing import Any, Dict, Optional from urllib import parse from sqlalchemy.engine.url import URL @@ -55,7 +55,9 @@ def epoch_ms_to_dttm(cls) -> str: return "TO_DATE({col})" @classmethod - def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]: + def convert_dttm( + cls, target_type: str, dttm: datetime, **kwargs: Any + ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.DATE: return f"TO_DATE('{dttm.date().isoformat()}', 'yyyy-MM-dd')" diff --git a/superset/db_engine_specs/druid.py b/superset/db_engine_specs/druid.py index 58545d4f8d73..4b2d664fb6cc 100644 --- a/superset/db_engine_specs/druid.py +++ b/superset/db_engine_specs/druid.py @@ -94,7 +94,9 @@ def get_extra_params(database: "Database") -> Dict[str, Any]: return extra @classmethod - def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]: + def convert_dttm( + cls, target_type: str, dttm: datetime, **kwargs: Any + ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.DATE: return f"CAST(TIME_PARSE('{dttm.date().isoformat()}') AS DATE)" diff --git a/superset/db_engine_specs/elasticsearch.py b/superset/db_engine_specs/elasticsearch.py index 65042ed05c48..06e90d83c477 100644 --- a/superset/db_engine_specs/elasticsearch.py +++ b/superset/db_engine_specs/elasticsearch.py @@ -15,7 +15,8 @@ # specific language governing permissions and limitations # under the License. from datetime import datetime -from typing import Dict, Optional, Type +from distutils.version import StrictVersion +from typing import Any, Dict, Optional, Type from superset.db_engine_specs.base import BaseEngineSpec from superset.db_engine_specs.exceptions import ( @@ -59,10 +60,20 @@ def get_dbapi_exception_mapping(cls) -> Dict[Type[Exception], Type[Exception]]: } @classmethod - def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]: - if target_type.upper() == utils.TemporalType.DATETIME: + def convert_dttm( + cls, target_type: str, dttm: datetime, **kwargs: Any + ) -> Optional[str]: + + if target_type.upper() != utils.TemporalType.DATETIME: + return None + + es_version = kwargs.get("version") + + if es_version and StrictVersion(es_version) >= StrictVersion("7.8"): + datetime_formatted = dttm.isoformat(sep=" ", timespec="seconds") + return f"""DATETIME_PARSE('{datetime_formatted}', 'yyyy-MM-dd HH:mm:ss')""" + else: return f"""CAST('{dttm.isoformat(timespec="seconds")}' AS DATETIME)""" - return None class OpenDistroEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method @@ -87,7 +98,9 @@ class OpenDistroEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method engine_name = "ElasticSearch (OpenDistro SQL)" @classmethod - def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]: + def convert_dttm( + cls, target_type: str, dttm: datetime, **kwargs: Any + ) -> Optional[str]: if target_type.upper() == utils.TemporalType.DATETIME: return f"""'{dttm.isoformat(timespec="seconds")}'""" return None diff --git a/superset/db_engine_specs/firebird.py b/superset/db_engine_specs/firebird.py index 72b462ab4560..af7790161329 100644 --- a/superset/db_engine_specs/firebird.py +++ b/superset/db_engine_specs/firebird.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. from datetime import datetime -from typing import Optional +from typing import Any, Dict, Optional from superset.db_engine_specs.base import BaseEngineSpec, LimitMethod from superset.utils import core as utils @@ -70,7 +70,9 @@ def epoch_to_dttm(cls) -> str: return "DATEADD(second, {col}, CAST('00:00:00' AS TIMESTAMP))" @classmethod - def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]: + def convert_dttm( + cls, target_type: str, dttm: datetime, **kwargs: Any + ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.TIMESTAMP: dttm_formatted = dttm.isoformat(sep=" ") diff --git a/superset/db_engine_specs/firebolt.py b/superset/db_engine_specs/firebolt.py index ea5091f69ffa..0d896c3a3f94 100644 --- a/superset/db_engine_specs/firebolt.py +++ b/superset/db_engine_specs/firebolt.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. from datetime import datetime -from typing import Optional +from typing import Any, Dict, Optional from superset.db_engine_specs.base import BaseEngineSpec from superset.utils import core as utils @@ -41,7 +41,9 @@ class FireboltEngineSpec(BaseEngineSpec): } @classmethod - def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]: + def convert_dttm( + cls, target_type: str, dttm: datetime, **kwargs: Any + ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.DATE: return f"CAST('{dttm.date().isoformat()}' AS DATE)" diff --git a/superset/db_engine_specs/hana.py b/superset/db_engine_specs/hana.py index 11e49d1896ab..faccac7d467f 100644 --- a/superset/db_engine_specs/hana.py +++ b/superset/db_engine_specs/hana.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. from datetime import datetime -from typing import Optional +from typing import Any, Dict, Optional from superset.db_engine_specs.base import LimitMethod from superset.db_engine_specs.postgres import PostgresBaseEngineSpec @@ -43,7 +43,9 @@ class HanaEngineSpec(PostgresBaseEngineSpec): } @classmethod - def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]: + def convert_dttm( + cls, target_type: str, dttm: datetime, **kwargs: Any + ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.DATE: return f"TO_DATE('{dttm.date().isoformat()}', 'YYYY-MM-DD')" diff --git a/superset/db_engine_specs/hive.py b/superset/db_engine_specs/hive.py index 061ab7f52373..b9d3e0610530 100644 --- a/superset/db_engine_specs/hive.py +++ b/superset/db_engine_specs/hive.py @@ -248,7 +248,9 @@ def _get_hive_type(dtype: np.dtype) -> str: ) @classmethod - def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]: + def convert_dttm( + cls, target_type: str, dttm: datetime, **kwargs: Any + ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.DATE: return f"CAST('{dttm.date().isoformat()}' AS DATE)" diff --git a/superset/db_engine_specs/impala.py b/superset/db_engine_specs/impala.py index 9d8dc91374f4..291a349ab168 100644 --- a/superset/db_engine_specs/impala.py +++ b/superset/db_engine_specs/impala.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. from datetime import datetime -from typing import List, Optional +from typing import Any, Dict, List, Optional from sqlalchemy.engine.reflection import Inspector @@ -45,7 +45,9 @@ def epoch_to_dttm(cls) -> str: return "from_unixtime({col})" @classmethod - def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]: + def convert_dttm( + cls, target_type: str, dttm: datetime, **kwargs: Any + ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.DATE: return f"CAST('{dttm.date().isoformat()}' AS DATE)" diff --git a/superset/db_engine_specs/kylin.py b/superset/db_engine_specs/kylin.py index ffa7f105b43c..79ad30dd4bd7 100644 --- a/superset/db_engine_specs/kylin.py +++ b/superset/db_engine_specs/kylin.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. from datetime import datetime -from typing import Optional +from typing import Any, Dict, Optional from superset.db_engine_specs.base import BaseEngineSpec from superset.utils import core as utils @@ -40,7 +40,9 @@ class KylinEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method } @classmethod - def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]: + def convert_dttm( + cls, target_type: str, dttm: datetime, **kwargs: Any + ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.DATE: return f"CAST('{dttm.date().isoformat()}' AS DATE)" diff --git a/superset/db_engine_specs/mssql.py b/superset/db_engine_specs/mssql.py index ff6b5326d393..c1875d40ac1f 100644 --- a/superset/db_engine_specs/mssql.py +++ b/superset/db_engine_specs/mssql.py @@ -98,7 +98,9 @@ def epoch_to_dttm(cls) -> str: return "dateadd(S, {col}, '1970-01-01')" @classmethod - def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]: + def convert_dttm( + cls, target_type: str, dttm: datetime, **kwargs: Any + ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.DATE: return f"CONVERT(DATE, '{dttm.date().isoformat()}', 23)" diff --git a/superset/db_engine_specs/mysql.py b/superset/db_engine_specs/mysql.py index 2fb1e9737bb6..971c18a73810 100644 --- a/superset/db_engine_specs/mysql.py +++ b/superset/db_engine_specs/mysql.py @@ -151,7 +151,9 @@ class MySQLEngineSpec(BaseEngineSpec, BasicParametersMixin): } @classmethod - def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]: + def convert_dttm( + cls, target_type: str, dttm: datetime, **kwargs: Any + ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.DATE: return f"STR_TO_DATE('{dttm.date().isoformat()}', '%Y-%m-%d')" @@ -204,6 +206,7 @@ def _extract_error_message(cls, ex: Exception) -> str: def get_column_spec( cls, native_type: Optional[str], + db_extra: Optional[Dict[str, Any]] = None, source: utils.ColumnTypeSource = utils.ColumnTypeSource.GET_TABLE, column_type_mappings: Tuple[ Tuple[ diff --git a/superset/db_engine_specs/oracle.py b/superset/db_engine_specs/oracle.py index 47775973f11c..88b1832c58f3 100644 --- a/superset/db_engine_specs/oracle.py +++ b/superset/db_engine_specs/oracle.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. from datetime import datetime -from typing import Any, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple from superset.db_engine_specs.base import BaseEngineSpec, LimitMethod from superset.utils import core as utils @@ -41,7 +41,9 @@ class OracleEngineSpec(BaseEngineSpec): } @classmethod - def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]: + def convert_dttm( + cls, target_type: str, dttm: datetime, **kwargs: Any + ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.DATE: return f"TO_DATE('{dttm.date().isoformat()}', 'YYYY-MM-DD')" diff --git a/superset/db_engine_specs/postgres.py b/superset/db_engine_specs/postgres.py index 4d1936310d4d..d5ce8d0b64bc 100644 --- a/superset/db_engine_specs/postgres.py +++ b/superset/db_engine_specs/postgres.py @@ -90,7 +90,7 @@ class FixedOffsetTimezone(_FixedOffset): class PostgresBaseEngineSpec(BaseEngineSpec): - """ Abstract class for Postgres 'like' databases """ + """Abstract class for Postgres 'like' databases""" engine = "" engine_name = "PostgreSQL" @@ -242,7 +242,9 @@ def get_table_names( return sorted(tables) @classmethod - def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]: + def convert_dttm( + cls, target_type: str, dttm: datetime, **kwargs: Any + ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.DATE: return f"TO_DATE('{dttm.date().isoformat()}', 'YYYY-MM-DD')" @@ -279,6 +281,7 @@ def get_extra_params(database: "Database") -> Dict[str, Any]: def get_column_spec( cls, native_type: Optional[str], + db_extra: Optional[Dict[str, Any]] = None, source: utils.ColumnTypeSource = utils.ColumnTypeSource.GET_TABLE, column_type_mappings: Tuple[ Tuple[ diff --git a/superset/db_engine_specs/presto.py b/superset/db_engine_specs/presto.py index 9be73045e4ed..d31abda9830c 100644 --- a/superset/db_engine_specs/presto.py +++ b/superset/db_engine_specs/presto.py @@ -743,7 +743,9 @@ def adjust_database_uri( uri.database = database @classmethod - def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]: + def convert_dttm( + cls, target_type: str, dttm: datetime, **kwargs: Any + ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.DATE: return f"""from_iso8601_date('{dttm.date().isoformat()}')""" @@ -1215,6 +1217,7 @@ def is_readonly_query(cls, parsed_query: ParsedQuery) -> bool: def get_column_spec( cls, native_type: Optional[str], + db_extra: Optional[Dict[str, Any]] = None, source: utils.ColumnTypeSource = utils.ColumnTypeSource.GET_TABLE, column_type_mappings: Tuple[ Tuple[ diff --git a/superset/db_engine_specs/rockset.py b/superset/db_engine_specs/rockset.py index 7cb3fb0640f7..2ab1272034cf 100644 --- a/superset/db_engine_specs/rockset.py +++ b/superset/db_engine_specs/rockset.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. from datetime import datetime -from typing import Optional, TYPE_CHECKING +from typing import Any, Dict, Optional, TYPE_CHECKING from superset.db_engine_specs.base import BaseEngineSpec from superset.utils import core as utils @@ -50,7 +50,9 @@ def epoch_ms_to_dttm(cls) -> str: return "{col}" @classmethod - def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]: + def convert_dttm( + cls, target_type: str, dttm: datetime, **kwargs: Any + ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.DATE: return f"DATE '{dttm.date().isoformat()}'" diff --git a/superset/db_engine_specs/snowflake.py b/superset/db_engine_specs/snowflake.py index 6dd85706562b..7d02c8505b74 100644 --- a/superset/db_engine_specs/snowflake.py +++ b/superset/db_engine_specs/snowflake.py @@ -104,7 +104,9 @@ def epoch_ms_to_dttm(cls) -> str: return "DATEADD(MS, {col}, '1970-01-01')" @classmethod - def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]: + def convert_dttm( + cls, target_type: str, dttm: datetime, **kwargs: Any + ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.DATE: return f"TO_DATE('{dttm.date().isoformat()}')" diff --git a/superset/db_engine_specs/sqlite.py b/superset/db_engine_specs/sqlite.py index d2b87a488950..8f3aa7d3b05e 100644 --- a/superset/db_engine_specs/sqlite.py +++ b/superset/db_engine_specs/sqlite.py @@ -97,7 +97,9 @@ def get_all_datasource_names( raise Exception(f"Unsupported datasource_type: {datasource_type}") @classmethod - def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]: + def convert_dttm( + cls, target_type: str, dttm: datetime, **kwargs: Any + ) -> Optional[str]: tt = target_type.upper() if tt in (utils.TemporalType.TEXT, utils.TemporalType.DATETIME): return f"""'{dttm.isoformat(sep=" ", timespec="microseconds")}'""" diff --git a/superset/db_engine_specs/trino.py b/superset/db_engine_specs/trino.py index 7d28cfbaa604..bd321bb6611b 100644 --- a/superset/db_engine_specs/trino.py +++ b/superset/db_engine_specs/trino.py @@ -46,7 +46,9 @@ class TrinoEngineSpec(BaseEngineSpec): } @classmethod - def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]: + def convert_dttm( + cls, target_type: str, dttm: datetime, **kwargs: Any + ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.DATE: value = dttm.date().isoformat() diff --git a/superset/models/sql_types/base.py b/superset/models/sql_types/base.py index 669181b3a481..ebd46c73d20f 100644 --- a/superset/models/sql_types/base.py +++ b/superset/models/sql_types/base.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. from datetime import datetime -from typing import Any, Callable, Type, TYPE_CHECKING +from typing import Any, Callable, Dict, Type, TYPE_CHECKING from flask_babel import gettext as __ from sqlalchemy import types @@ -26,7 +26,10 @@ def literal_dttm_type_factory( - sqla_type: types.TypeEngine, db_engine_spec: Type["BaseEngineSpec"], col_type: str, + sqla_type: types.TypeEngine, + db_engine_spec: Type["BaseEngineSpec"], + col_type: str, + db_extra: Dict[str, Any], ) -> types.TypeEngine: """ Create a custom SQLAlchemy type that supports datetime literal binds. @@ -34,6 +37,7 @@ def literal_dttm_type_factory( :param sqla_type: Base type to extend :param db_engine_spec: Database engine spec which supports `convert_dttm` method :param col_type: native column type as defined in table metadata + :param db_extra: The database extra object :return: SQLAlchemy type that supports using datetima as literal bind """ # pylint: disable=too-few-public-methods @@ -42,7 +46,9 @@ class TemporalWrapperType(type(sqla_type)): # type: ignore def literal_processor(self, dialect: Dialect) -> Callable[[Any], Any]: def process(value: Any) -> Any: if isinstance(value, datetime): - ts_expression = db_engine_spec.convert_dttm(col_type, value) + ts_expression = db_engine_spec.convert_dttm( + col_type, value, **db_extra + ) if ts_expression is None: raise NotImplementedError( __( diff --git a/tests/integration_tests/model_tests.py b/tests/integration_tests/model_tests.py index e314a1371b41..cbab550eba58 100644 --- a/tests/integration_tests/model_tests.py +++ b/tests/integration_tests/model_tests.py @@ -539,6 +539,8 @@ def test_data_for_slices(self): def test_literal_dttm_type_factory(): orig_type = DateTime() - new_type = literal_dttm_type_factory(orig_type, PostgresEngineSpec, "TIMESTAMP") + new_type = literal_dttm_type_factory( + orig_type, PostgresEngineSpec, "TIMESTAMP", db_extra={} + ) assert type(new_type).__name__ == "TemporalWrapperType" assert str(new_type) == str(orig_type) From a97af180e6119fa02c16dbb1c003ea8139d42b29 Mon Sep 17 00:00:00 2001 From: aniaan Date: Sat, 9 Oct 2021 16:30:26 +0800 Subject: [PATCH 02/11] test(elasticsearch): add test --- superset/db_engine_specs/base.py | 1 - superset/db_engine_specs/crate.py | 2 +- superset/db_engine_specs/databricks.py | 2 +- superset/db_engine_specs/dremio.py | 2 +- superset/db_engine_specs/drill.py | 2 +- superset/db_engine_specs/elasticsearch.py | 4 ++-- superset/db_engine_specs/firebird.py | 2 +- superset/db_engine_specs/firebolt.py | 2 +- superset/db_engine_specs/hana.py | 2 +- superset/db_engine_specs/impala.py | 2 +- superset/db_engine_specs/kylin.py | 2 +- superset/db_engine_specs/oracle.py | 2 +- superset/db_engine_specs/rockset.py | 2 +- .../db_engine_specs/elasticsearch_tests.py | 13 +++++++++++++ 14 files changed, 26 insertions(+), 14 deletions(-) diff --git a/superset/db_engine_specs/base.py b/superset/db_engine_specs/base.py index d88dd2c342e1..063a88073d79 100644 --- a/superset/db_engine_specs/base.py +++ b/superset/db_engine_specs/base.py @@ -64,7 +64,6 @@ from superset.utils import core as utils from superset.utils.core import ColumnSpec, GenericDataType from superset.utils.hashing import md5_sha_from_str -from superset.utils.memoized import memoized from superset.utils.network import is_hostname_valid, is_port_open if TYPE_CHECKING: diff --git a/superset/db_engine_specs/crate.py b/superset/db_engine_specs/crate.py index 976e7985e9a9..6e509c5b74bc 100644 --- a/superset/db_engine_specs/crate.py +++ b/superset/db_engine_specs/crate.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. from datetime import datetime -from typing import Any, Dict, Optional, TYPE_CHECKING +from typing import Any, Optional, TYPE_CHECKING from superset.db_engine_specs.base import BaseEngineSpec from superset.utils import core as utils diff --git a/superset/db_engine_specs/databricks.py b/superset/db_engine_specs/databricks.py index a6e3bab1afa7..2acfee23cb92 100644 --- a/superset/db_engine_specs/databricks.py +++ b/superset/db_engine_specs/databricks.py @@ -16,7 +16,7 @@ # under the License.o from datetime import datetime -from typing import Any, Dict, Optional +from typing import Any, Optional from superset.db_engine_specs.base import BaseEngineSpec from superset.db_engine_specs.hive import HiveEngineSpec diff --git a/superset/db_engine_specs/dremio.py b/superset/db_engine_specs/dremio.py index 5710fd144638..801b2ef1c8a5 100644 --- a/superset/db_engine_specs/dremio.py +++ b/superset/db_engine_specs/dremio.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. from datetime import datetime -from typing import Any, Dict, Optional +from typing import Any, Optional from superset.db_engine_specs.base import BaseEngineSpec from superset.utils import core as utils diff --git a/superset/db_engine_specs/drill.py b/superset/db_engine_specs/drill.py index 43456896675a..2e3330c57b9c 100644 --- a/superset/db_engine_specs/drill.py +++ b/superset/db_engine_specs/drill.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. from datetime import datetime -from typing import Any, Dict, Optional +from typing import Any, Optional from urllib import parse from sqlalchemy.engine.url import URL diff --git a/superset/db_engine_specs/elasticsearch.py b/superset/db_engine_specs/elasticsearch.py index 06e90d83c477..d84c827fc671 100644 --- a/superset/db_engine_specs/elasticsearch.py +++ b/superset/db_engine_specs/elasticsearch.py @@ -72,8 +72,8 @@ def convert_dttm( if es_version and StrictVersion(es_version) >= StrictVersion("7.8"): datetime_formatted = dttm.isoformat(sep=" ", timespec="seconds") return f"""DATETIME_PARSE('{datetime_formatted}', 'yyyy-MM-dd HH:mm:ss')""" - else: - return f"""CAST('{dttm.isoformat(timespec="seconds")}' AS DATETIME)""" + + return f"""CAST('{dttm.isoformat(timespec="seconds")}' AS DATETIME)""" class OpenDistroEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method diff --git a/superset/db_engine_specs/firebird.py b/superset/db_engine_specs/firebird.py index af7790161329..2063355d70b7 100644 --- a/superset/db_engine_specs/firebird.py +++ b/superset/db_engine_specs/firebird.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. from datetime import datetime -from typing import Any, Dict, Optional +from typing import Any, Optional from superset.db_engine_specs.base import BaseEngineSpec, LimitMethod from superset.utils import core as utils diff --git a/superset/db_engine_specs/firebolt.py b/superset/db_engine_specs/firebolt.py index 0d896c3a3f94..88e6e59b6527 100644 --- a/superset/db_engine_specs/firebolt.py +++ b/superset/db_engine_specs/firebolt.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. from datetime import datetime -from typing import Any, Dict, Optional +from typing import Any, Optional from superset.db_engine_specs.base import BaseEngineSpec from superset.utils import core as utils diff --git a/superset/db_engine_specs/hana.py b/superset/db_engine_specs/hana.py index faccac7d467f..f6805feaabf9 100644 --- a/superset/db_engine_specs/hana.py +++ b/superset/db_engine_specs/hana.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. from datetime import datetime -from typing import Any, Dict, Optional +from typing import Any, Optional from superset.db_engine_specs.base import LimitMethod from superset.db_engine_specs.postgres import PostgresBaseEngineSpec diff --git a/superset/db_engine_specs/impala.py b/superset/db_engine_specs/impala.py index 291a349ab168..f1cf539af6bc 100644 --- a/superset/db_engine_specs/impala.py +++ b/superset/db_engine_specs/impala.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. from datetime import datetime -from typing import Any, Dict, List, Optional +from typing import Any, List, Optional from sqlalchemy.engine.reflection import Inspector diff --git a/superset/db_engine_specs/kylin.py b/superset/db_engine_specs/kylin.py index 79ad30dd4bd7..334953b93639 100644 --- a/superset/db_engine_specs/kylin.py +++ b/superset/db_engine_specs/kylin.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. from datetime import datetime -from typing import Any, Dict, Optional +from typing import Any, Optional from superset.db_engine_specs.base import BaseEngineSpec from superset.utils import core as utils diff --git a/superset/db_engine_specs/oracle.py b/superset/db_engine_specs/oracle.py index 88b1832c58f3..6df35bfba461 100644 --- a/superset/db_engine_specs/oracle.py +++ b/superset/db_engine_specs/oracle.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. from datetime import datetime -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, List, Optional, Tuple from superset.db_engine_specs.base import BaseEngineSpec, LimitMethod from superset.utils import core as utils diff --git a/superset/db_engine_specs/rockset.py b/superset/db_engine_specs/rockset.py index 2ab1272034cf..d3dd3fa72b41 100644 --- a/superset/db_engine_specs/rockset.py +++ b/superset/db_engine_specs/rockset.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. from datetime import datetime -from typing import Any, Dict, Optional, TYPE_CHECKING +from typing import Any, Optional, TYPE_CHECKING from superset.db_engine_specs.base import BaseEngineSpec from superset.utils import core as utils diff --git a/tests/integration_tests/db_engine_specs/elasticsearch_tests.py b/tests/integration_tests/db_engine_specs/elasticsearch_tests.py index 92c25aa84351..02bb7223406f 100644 --- a/tests/integration_tests/db_engine_specs/elasticsearch_tests.py +++ b/tests/integration_tests/db_engine_specs/elasticsearch_tests.py @@ -34,6 +34,19 @@ def test_convert_dttm(self): "CAST('2019-01-02T03:04:05' AS DATETIME)", ) + def test_convert_dttm2(self): + """ + ES 7.8 and above versions need to use the DATETIME_PARSE function to + solve the time zone problem + """ + dttm = self.get_dttm() + db_extra = {"version": "7.8"} + + self.assertEqual( + ElasticSearchEngineSpec.convert_dttm("DATETIME", dttm, **db_extra), + "DATETIME_PARSE('2019-01-02T03:04:05', 'yyyy-MM-dd HH:mm:ss')", + ) + def test_opendistro_convert_dttm(self): """ DB Eng Specs (opendistro): Test convert_dttm From 356dbe3ea1457f6fef60b7fa014e34fd43e0394b Mon Sep 17 00:00:00 2001 From: aniaan Date: Sat, 9 Oct 2021 16:55:28 +0800 Subject: [PATCH 03/11] fix(test): fix typo --- tests/integration_tests/db_engine_specs/elasticsearch_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration_tests/db_engine_specs/elasticsearch_tests.py b/tests/integration_tests/db_engine_specs/elasticsearch_tests.py index 02bb7223406f..f3cffe624e2a 100644 --- a/tests/integration_tests/db_engine_specs/elasticsearch_tests.py +++ b/tests/integration_tests/db_engine_specs/elasticsearch_tests.py @@ -44,7 +44,7 @@ def test_convert_dttm2(self): self.assertEqual( ElasticSearchEngineSpec.convert_dttm("DATETIME", dttm, **db_extra), - "DATETIME_PARSE('2019-01-02T03:04:05', 'yyyy-MM-dd HH:mm:ss')", + "DATETIME_PARSE('2019-01-02 03:04:05', 'yyyy-MM-dd HH:mm:ss')", ) def test_opendistro_convert_dttm(self): From 2e36740511f980fcde498eb69a027a294473c24e Mon Sep 17 00:00:00 2001 From: aniaan Date: Mon, 11 Oct 2021 22:52:41 +0800 Subject: [PATCH 04/11] docs(elasticsearch): add annotation --- superset/db_engine_specs/elasticsearch.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/superset/db_engine_specs/elasticsearch.py b/superset/db_engine_specs/elasticsearch.py index d84c827fc671..c05534707740 100644 --- a/superset/db_engine_specs/elasticsearch.py +++ b/superset/db_engine_specs/elasticsearch.py @@ -64,16 +64,20 @@ def convert_dttm( cls, target_type: str, dttm: datetime, **kwargs: Any ) -> Optional[str]: - if target_type.upper() != utils.TemporalType.DATETIME: - return None - - es_version = kwargs.get("version") - - if es_version and StrictVersion(es_version) >= StrictVersion("7.8"): - datetime_formatted = dttm.isoformat(sep=" ", timespec="seconds") - return f"""DATETIME_PARSE('{datetime_formatted}', 'yyyy-MM-dd HH:mm:ss')""" + if target_type.upper() == utils.TemporalType.DATETIME: + es_version = kwargs.get("version") + # The elasticsearch CAST function does not take effect for the time zone + # setting. In elasticsearch7.8 and above, we can use the DATETIME_PARSE + # function to solve this problem. + if es_version and StrictVersion(es_version) >= StrictVersion("7.8"): + datetime_formatted = dttm.isoformat(sep=" ", timespec="seconds") + return ( + f"""DATETIME_PARSE('{datetime_formatted}', 'yyyy-MM-dd HH:mm:ss')""" + ) + + return f"""CAST('{dttm.isoformat(timespec="seconds")}' AS DATETIME)""" - return f"""CAST('{dttm.isoformat(timespec="seconds")}' AS DATETIME)""" + return None class OpenDistroEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method From 71ed93183ac8c113029e7155411924df978a340c Mon Sep 17 00:00:00 2001 From: aniaan Date: Mon, 11 Oct 2021 23:44:09 +0800 Subject: [PATCH 05/11] docs(elasticsearch): add time_zone desc --- .../Connecting to Databases/elasticsearch.mdx | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/src/pages/docs/Connecting to Databases/elasticsearch.mdx b/docs/src/pages/docs/Connecting to Databases/elasticsearch.mdx index 1704412baa5d..d8b88b969df3 100644 --- a/docs/src/pages/docs/Connecting to Databases/elasticsearch.mdx +++ b/docs/src/pages/docs/Connecting to Databases/elasticsearch.mdx @@ -48,3 +48,24 @@ POST /_aliases ``` Then register your table with the alias name logstasg_all + +**Time zone** + +By default, Superset uses UTC time zone for elasticsearch query. If you need to specify a time zone, +please edit your Database and enter the settings of your specified time zone in the Other > ENGINE PARAMETERS: + + +``` +{ + "connect_args": { + "time_zone": "Asia/Shanghai" + } +} +``` + +Another issue to note about the time zone problem is that before elasticsearch7.8, if you want to convert a string into a `DATETIME` object, +you need to use the `CAST` function,but this function does not support our `time_zone` setting. So it is recommended to upgrade to the version after elasticsearch7.8. +After elasticsearch7.8, you can use the `DATETIME_PARSE` function to solve this problem. +The DATETIME_PARSE function is to support our `time_zone` setting, and here you need to fill in your elasticsearch version number in the Other > VERSION setting. +the superset will use the `DATETIME_PARSE` function for conversion. + From 82ed2cd5526e5a918dea7e77a566b7652405b133 Mon Sep 17 00:00:00 2001 From: aniaan Date: Mon, 11 Oct 2021 23:47:19 +0800 Subject: [PATCH 06/11] docs(elasticsearch): fix typo --- docs/src/pages/docs/Connecting to Databases/elasticsearch.mdx | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/src/pages/docs/Connecting to Databases/elasticsearch.mdx b/docs/src/pages/docs/Connecting to Databases/elasticsearch.mdx index d8b88b969df3..720964a36703 100644 --- a/docs/src/pages/docs/Connecting to Databases/elasticsearch.mdx +++ b/docs/src/pages/docs/Connecting to Databases/elasticsearch.mdx @@ -68,4 +68,3 @@ you need to use the `CAST` function,but this function does not support our `time After elasticsearch7.8, you can use the `DATETIME_PARSE` function to solve this problem. The DATETIME_PARSE function is to support our `time_zone` setting, and here you need to fill in your elasticsearch version number in the Other > VERSION setting. the superset will use the `DATETIME_PARSE` function for conversion. - From a54147fcaab48a24c62760db79226bdab8255fb0 Mon Sep 17 00:00:00 2001 From: aniaan Date: Thu, 11 Nov 2021 18:19:07 +0800 Subject: [PATCH 07/11] refactor(db_engine): change convert_dttm signature --- superset/connectors/sqla/models.py | 11 +++++++---- superset/db_engine_specs/athena.py | 2 +- superset/db_engine_specs/base.py | 2 +- superset/db_engine_specs/bigquery.py | 2 +- superset/db_engine_specs/clickhouse.py | 2 +- superset/db_engine_specs/crate.py | 4 ++-- superset/db_engine_specs/databricks.py | 6 +++--- superset/db_engine_specs/dremio.py | 4 ++-- superset/db_engine_specs/drill.py | 4 ++-- superset/db_engine_specs/druid.py | 2 +- superset/db_engine_specs/elasticsearch.py | 7 ++++--- superset/db_engine_specs/firebird.py | 4 ++-- superset/db_engine_specs/firebolt.py | 4 ++-- superset/db_engine_specs/hana.py | 4 ++-- superset/db_engine_specs/hive.py | 2 +- superset/db_engine_specs/impala.py | 4 ++-- superset/db_engine_specs/kylin.py | 4 ++-- superset/db_engine_specs/mssql.py | 2 +- superset/db_engine_specs/mysql.py | 2 +- superset/db_engine_specs/oracle.py | 4 ++-- superset/db_engine_specs/postgres.py | 2 +- superset/db_engine_specs/presto.py | 2 +- superset/db_engine_specs/rockset.py | 4 ++-- superset/db_engine_specs/snowflake.py | 2 +- superset/db_engine_specs/sqlite.py | 2 +- superset/db_engine_specs/trino.py | 2 +- superset/models/sql_types/base.py | 2 +- tests/integration_tests/charts/api_tests.py | 2 +- .../db_engine_specs/base_engine_spec_tests.py | 2 +- .../db_engine_specs/elasticsearch_tests.py | 6 +++--- 30 files changed, 53 insertions(+), 49 deletions(-) diff --git a/superset/connectors/sqla/models.py b/superset/connectors/sqla/models.py index a58cda99b443..5c116939b191 100644 --- a/superset/connectors/sqla/models.py +++ b/superset/connectors/sqla/models.py @@ -353,7 +353,11 @@ def dttm_sql_literal( ], ) -> str: """Convert datetime object to a SQL expression string""" - sql = self.db_engine_spec.convert_dttm(self.type, dttm,**self.db_extra) if self.type else None + sql = ( + self.db_engine_spec.convert_dttm(self.type, dttm, db_extra=self.db_extra) + if self.type + else None + ) if sql: return sql @@ -1486,10 +1490,11 @@ def _normalize_prequery_result_type( value = value.item() column_ = columns_by_name[dimension] + db_extra: Dict[str, Any] = self.database.get_extra() if column_.type and column_.is_temporal and isinstance(value, str): sql = self.db_engine_spec.convert_dttm( - column_.type, dateutil.parser.parse(value), + column_.type, dateutil.parser.parse(value), db_extra=db_extra ) if sql: @@ -1504,8 +1509,6 @@ def _get_top_groups( groupby_exprs: Dict[str, Any], columns_by_name: Dict[str, TableColumn], ) -> ColumnElement: - db_extra: Dict[str, Any] = self.database.get_extra() - column_map = {column.column_name: column for column in self.columns} groups = [] for _unused, row in df.iterrows(): group = [] diff --git a/superset/db_engine_specs/athena.py b/superset/db_engine_specs/athena.py index 8b65c43855e3..a33d08f2bf45 100644 --- a/superset/db_engine_specs/athena.py +++ b/superset/db_engine_specs/athena.py @@ -62,7 +62,7 @@ class AthenaEngineSpec(BaseEngineSpec): @classmethod def convert_dttm( - cls, target_type: str, dttm: datetime, **kwargs: Any + cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.DATE: diff --git a/superset/db_engine_specs/base.py b/superset/db_engine_specs/base.py index 88c09360d373..e795d7ed9405 100644 --- a/superset/db_engine_specs/base.py +++ b/superset/db_engine_specs/base.py @@ -691,7 +691,7 @@ def df_to_sql( @classmethod def convert_dttm( # pylint: disable=unused-argument - cls, target_type: str, dttm: datetime, **kwargs: Any, + cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None ) -> Optional[str]: """ Convert Python datetime object to a SQL expression diff --git a/superset/db_engine_specs/bigquery.py b/superset/db_engine_specs/bigquery.py index afec0146331b..30e04c4f2fe9 100644 --- a/superset/db_engine_specs/bigquery.py +++ b/superset/db_engine_specs/bigquery.py @@ -187,7 +187,7 @@ class BigQueryEngineSpec(BaseEngineSpec): @classmethod def convert_dttm( - cls, target_type: str, dttm: datetime, **kwargs: Any + cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.DATE: diff --git a/superset/db_engine_specs/clickhouse.py b/superset/db_engine_specs/clickhouse.py index 3d22b657c0a2..4f34d2a5543c 100644 --- a/superset/db_engine_specs/clickhouse.py +++ b/superset/db_engine_specs/clickhouse.py @@ -73,7 +73,7 @@ def get_dbapi_mapped_exception(cls, exception: Exception) -> Exception: @classmethod def convert_dttm( - cls, target_type: str, dttm: datetime, **kwargs: Any + cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.DATE: diff --git a/superset/db_engine_specs/crate.py b/superset/db_engine_specs/crate.py index f998fa223ec5..4d934c448c35 100644 --- a/superset/db_engine_specs/crate.py +++ b/superset/db_engine_specs/crate.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. from datetime import datetime -from typing import Any, Optional, TYPE_CHECKING +from typing import Any, Dict, Optional, TYPE_CHECKING from superset.db_engine_specs.base import BaseEngineSpec from superset.utils import core as utils @@ -51,7 +51,7 @@ def epoch_ms_to_dttm(cls) -> str: @classmethod def convert_dttm( - cls, target_type: str, dttm: datetime, **kwargs: Any + cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.TIMESTAMP: diff --git a/superset/db_engine_specs/databricks.py b/superset/db_engine_specs/databricks.py index 2acfee23cb92..50ea59bae289 100644 --- a/superset/db_engine_specs/databricks.py +++ b/superset/db_engine_specs/databricks.py @@ -16,7 +16,7 @@ # under the License.o from datetime import datetime -from typing import Any, Optional +from typing import Any, Dict, Optional from superset.db_engine_specs.base import BaseEngineSpec from superset.db_engine_specs.hive import HiveEngineSpec @@ -41,9 +41,9 @@ class DatabricksODBCEngineSpec(BaseEngineSpec): @classmethod def convert_dttm( - cls, target_type: str, dttm: datetime, **kwargs: Any + cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None ) -> Optional[str]: - return HiveEngineSpec.convert_dttm(target_type, dttm) + return HiveEngineSpec.convert_dttm(target_type, dttm, db_extra=db_extra) @classmethod def epoch_to_dttm(cls) -> str: diff --git a/superset/db_engine_specs/dremio.py b/superset/db_engine_specs/dremio.py index a8c3c5874511..fddba00b5fea 100644 --- a/superset/db_engine_specs/dremio.py +++ b/superset/db_engine_specs/dremio.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. from datetime import datetime -from typing import Any, Optional +from typing import Any, Dict, Optional from superset.db_engine_specs.base import BaseEngineSpec from superset.utils import core as utils @@ -44,7 +44,7 @@ def epoch_to_dttm(cls) -> str: @classmethod def convert_dttm( - cls, target_type: str, dttm: datetime, **kwargs: Any + cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.DATE: diff --git a/superset/db_engine_specs/drill.py b/superset/db_engine_specs/drill.py index a41c66bcf89b..3c06017fa2da 100644 --- a/superset/db_engine_specs/drill.py +++ b/superset/db_engine_specs/drill.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. from datetime import datetime -from typing import Any, Optional +from typing import Any, Dict, Optional from urllib import parse from sqlalchemy.engine.url import URL @@ -56,7 +56,7 @@ def epoch_ms_to_dttm(cls) -> str: @classmethod def convert_dttm( - cls, target_type: str, dttm: datetime, **kwargs: Any + cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.DATE: diff --git a/superset/db_engine_specs/druid.py b/superset/db_engine_specs/druid.py index d23d6f98fbda..d193daf5844b 100644 --- a/superset/db_engine_specs/druid.py +++ b/superset/db_engine_specs/druid.py @@ -97,7 +97,7 @@ def get_extra_params(database: "Database") -> Dict[str, Any]: @classmethod def convert_dttm( - cls, target_type: str, dttm: datetime, **kwargs: Any + cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.DATE: diff --git a/superset/db_engine_specs/elasticsearch.py b/superset/db_engine_specs/elasticsearch.py index c05534707740..0d64f93d14c7 100644 --- a/superset/db_engine_specs/elasticsearch.py +++ b/superset/db_engine_specs/elasticsearch.py @@ -61,11 +61,12 @@ def get_dbapi_exception_mapping(cls) -> Dict[Type[Exception], Type[Exception]]: @classmethod def convert_dttm( - cls, target_type: str, dttm: datetime, **kwargs: Any + cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None ) -> Optional[str]: + db_extra = db_extra if db_extra else {} if target_type.upper() == utils.TemporalType.DATETIME: - es_version = kwargs.get("version") + es_version = db_extra.get("version") # The elasticsearch CAST function does not take effect for the time zone # setting. In elasticsearch7.8 and above, we can use the DATETIME_PARSE # function to solve this problem. @@ -103,7 +104,7 @@ class OpenDistroEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method @classmethod def convert_dttm( - cls, target_type: str, dttm: datetime, **kwargs: Any + cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None ) -> Optional[str]: if target_type.upper() == utils.TemporalType.DATETIME: return f"""'{dttm.isoformat(timespec="seconds")}'""" diff --git a/superset/db_engine_specs/firebird.py b/superset/db_engine_specs/firebird.py index 2063355d70b7..9254a3f2aa31 100644 --- a/superset/db_engine_specs/firebird.py +++ b/superset/db_engine_specs/firebird.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. from datetime import datetime -from typing import Any, Optional +from typing import Any, Dict, Optional from superset.db_engine_specs.base import BaseEngineSpec, LimitMethod from superset.utils import core as utils @@ -71,7 +71,7 @@ def epoch_to_dttm(cls) -> str: @classmethod def convert_dttm( - cls, target_type: str, dttm: datetime, **kwargs: Any + cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.TIMESTAMP: diff --git a/superset/db_engine_specs/firebolt.py b/superset/db_engine_specs/firebolt.py index 3ba3e5278cee..04f48b612a45 100644 --- a/superset/db_engine_specs/firebolt.py +++ b/superset/db_engine_specs/firebolt.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. from datetime import datetime -from typing import Any, Optional +from typing import Any, Dict, Optional from superset.db_engine_specs.base import BaseEngineSpec from superset.utils import core as utils @@ -42,7 +42,7 @@ class FireboltEngineSpec(BaseEngineSpec): @classmethod def convert_dttm( - cls, target_type: str, dttm: datetime, **kwargs: Any + cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.DATE: diff --git a/superset/db_engine_specs/hana.py b/superset/db_engine_specs/hana.py index d0d3e5360d66..0cc55d08d3f6 100644 --- a/superset/db_engine_specs/hana.py +++ b/superset/db_engine_specs/hana.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. from datetime import datetime -from typing import Any, Optional +from typing import Any, Dict, Optional from superset.db_engine_specs.base import LimitMethod from superset.db_engine_specs.postgres import PostgresBaseEngineSpec @@ -44,7 +44,7 @@ class HanaEngineSpec(PostgresBaseEngineSpec): @classmethod def convert_dttm( - cls, target_type: str, dttm: datetime, **kwargs: Any + cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.DATE: diff --git a/superset/db_engine_specs/hive.py b/superset/db_engine_specs/hive.py index 066d49cf79a9..bf7ebcce48c4 100644 --- a/superset/db_engine_specs/hive.py +++ b/superset/db_engine_specs/hive.py @@ -249,7 +249,7 @@ def _get_hive_type(dtype: np.dtype) -> str: @classmethod def convert_dttm( - cls, target_type: str, dttm: datetime, **kwargs: Any + cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.DATE: diff --git a/superset/db_engine_specs/impala.py b/superset/db_engine_specs/impala.py index 041f81f41680..048588c046fd 100644 --- a/superset/db_engine_specs/impala.py +++ b/superset/db_engine_specs/impala.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. from datetime import datetime -from typing import Any, List, Optional +from typing import Any, Dict, List, Optional from sqlalchemy.engine.reflection import Inspector @@ -46,7 +46,7 @@ def epoch_to_dttm(cls) -> str: @classmethod def convert_dttm( - cls, target_type: str, dttm: datetime, **kwargs: Any + cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.DATE: diff --git a/superset/db_engine_specs/kylin.py b/superset/db_engine_specs/kylin.py index 244312070453..dc3836c7373e 100644 --- a/superset/db_engine_specs/kylin.py +++ b/superset/db_engine_specs/kylin.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. from datetime import datetime -from typing import Any, Optional +from typing import Any, Dict, Optional from superset.db_engine_specs.base import BaseEngineSpec from superset.utils import core as utils @@ -41,7 +41,7 @@ class KylinEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method @classmethod def convert_dttm( - cls, target_type: str, dttm: datetime, **kwargs: Any + cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.DATE: diff --git a/superset/db_engine_specs/mssql.py b/superset/db_engine_specs/mssql.py index fc49cefe6e24..9db02186981f 100644 --- a/superset/db_engine_specs/mssql.py +++ b/superset/db_engine_specs/mssql.py @@ -99,7 +99,7 @@ def epoch_to_dttm(cls) -> str: @classmethod def convert_dttm( - cls, target_type: str, dttm: datetime, **kwargs: Any + cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.DATE: diff --git a/superset/db_engine_specs/mysql.py b/superset/db_engine_specs/mysql.py index 20d71f291b97..0d7493c386e2 100644 --- a/superset/db_engine_specs/mysql.py +++ b/superset/db_engine_specs/mysql.py @@ -152,7 +152,7 @@ class MySQLEngineSpec(BaseEngineSpec, BasicParametersMixin): @classmethod def convert_dttm( - cls, target_type: str, dttm: datetime, **kwargs: Any + cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.DATE: diff --git a/superset/db_engine_specs/oracle.py b/superset/db_engine_specs/oracle.py index 841d646cafac..ee04e49ffc64 100644 --- a/superset/db_engine_specs/oracle.py +++ b/superset/db_engine_specs/oracle.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. from datetime import datetime -from typing import Any, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple from superset.db_engine_specs.base import BaseEngineSpec, LimitMethod from superset.utils import core as utils @@ -42,7 +42,7 @@ class OracleEngineSpec(BaseEngineSpec): @classmethod def convert_dttm( - cls, target_type: str, dttm: datetime, **kwargs: Any + cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.DATE: diff --git a/superset/db_engine_specs/postgres.py b/superset/db_engine_specs/postgres.py index 2cb2aee88a8f..3222d00c2526 100644 --- a/superset/db_engine_specs/postgres.py +++ b/superset/db_engine_specs/postgres.py @@ -243,7 +243,7 @@ def get_table_names( @classmethod def convert_dttm( - cls, target_type: str, dttm: datetime, **kwargs: Any + cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.DATE: diff --git a/superset/db_engine_specs/presto.py b/superset/db_engine_specs/presto.py index 16bab05adf6b..93d39adae2d2 100644 --- a/superset/db_engine_specs/presto.py +++ b/superset/db_engine_specs/presto.py @@ -744,7 +744,7 @@ def adjust_database_uri( @classmethod def convert_dttm( - cls, target_type: str, dttm: datetime, **kwargs: Any + cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.DATE: diff --git a/superset/db_engine_specs/rockset.py b/superset/db_engine_specs/rockset.py index a210705c45de..606b860a5e64 100644 --- a/superset/db_engine_specs/rockset.py +++ b/superset/db_engine_specs/rockset.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. from datetime import datetime -from typing import Any, Optional, TYPE_CHECKING +from typing import Any, Dict, Optional, TYPE_CHECKING from superset.db_engine_specs.base import BaseEngineSpec from superset.utils import core as utils @@ -51,7 +51,7 @@ def epoch_ms_to_dttm(cls) -> str: @classmethod def convert_dttm( - cls, target_type: str, dttm: datetime, **kwargs: Any + cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.DATE: diff --git a/superset/db_engine_specs/snowflake.py b/superset/db_engine_specs/snowflake.py index 49736d4aa78b..058ca89c6af2 100644 --- a/superset/db_engine_specs/snowflake.py +++ b/superset/db_engine_specs/snowflake.py @@ -131,7 +131,7 @@ def epoch_ms_to_dttm(cls) -> str: @classmethod def convert_dttm( - cls, target_type: str, dttm: datetime, **kwargs: Any + cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.DATE: diff --git a/superset/db_engine_specs/sqlite.py b/superset/db_engine_specs/sqlite.py index 385868e9789d..23512b3cb492 100644 --- a/superset/db_engine_specs/sqlite.py +++ b/superset/db_engine_specs/sqlite.py @@ -98,7 +98,7 @@ def get_all_datasource_names( @classmethod def convert_dttm( - cls, target_type: str, dttm: datetime, **kwargs: Any + cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None ) -> Optional[str]: tt = target_type.upper() if tt in (utils.TemporalType.TEXT, utils.TemporalType.DATETIME): diff --git a/superset/db_engine_specs/trino.py b/superset/db_engine_specs/trino.py index 991440a0d8f9..8a56e8fb0ad3 100644 --- a/superset/db_engine_specs/trino.py +++ b/superset/db_engine_specs/trino.py @@ -47,7 +47,7 @@ class TrinoEngineSpec(BaseEngineSpec): @classmethod def convert_dttm( - cls, target_type: str, dttm: datetime, **kwargs: Any + cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None ) -> Optional[str]: tt = target_type.upper() if tt == utils.TemporalType.DATE: diff --git a/superset/models/sql_types/base.py b/superset/models/sql_types/base.py index ebd46c73d20f..597631a37216 100644 --- a/superset/models/sql_types/base.py +++ b/superset/models/sql_types/base.py @@ -47,7 +47,7 @@ def literal_processor(self, dialect: Dialect) -> Callable[[Any], Any]: def process(value: Any) -> Any: if isinstance(value, datetime): ts_expression = db_engine_spec.convert_dttm( - col_type, value, **db_extra + col_type, value, db_extra=db_extra ) if ts_expression is None: raise NotImplementedError( diff --git a/tests/integration_tests/charts/api_tests.py b/tests/integration_tests/charts/api_tests.py index f0c685ba4c44..2c63e61d6d87 100644 --- a/tests/integration_tests/charts/api_tests.py +++ b/tests/integration_tests/charts/api_tests.py @@ -1403,7 +1403,7 @@ def test_chart_data_dttm_filter(self): dttm_col = col if dttm_col: dttm_expression = table.database.db_engine_spec.convert_dttm( - dttm_col.type, dttm, + dttm_col.type, dttm, db_extra=table.database.get_extra() ) self.assertIn(dttm_expression, result["query"]) else: diff --git a/tests/integration_tests/db_engine_specs/base_engine_spec_tests.py b/tests/integration_tests/db_engine_specs/base_engine_spec_tests.py index 6ee8c4c84735..294c6801e753 100644 --- a/tests/integration_tests/db_engine_specs/base_engine_spec_tests.py +++ b/tests/integration_tests/db_engine_specs/base_engine_spec_tests.py @@ -251,7 +251,7 @@ def test_column_datatype_to_string(self): def test_convert_dttm(self): dttm = self.get_dttm() - self.assertIsNone(BaseEngineSpec.convert_dttm("", dttm)) + self.assertIsNone(BaseEngineSpec.convert_dttm("", dttm, db_extra=None)) def test_pyodbc_rows_to_tuples(self): # Test for case when pyodbc.Row is returned (odbc driver) diff --git a/tests/integration_tests/db_engine_specs/elasticsearch_tests.py b/tests/integration_tests/db_engine_specs/elasticsearch_tests.py index f3cffe624e2a..7416edfa94ec 100644 --- a/tests/integration_tests/db_engine_specs/elasticsearch_tests.py +++ b/tests/integration_tests/db_engine_specs/elasticsearch_tests.py @@ -30,7 +30,7 @@ def test_convert_dttm(self): dttm = self.get_dttm() self.assertEqual( - ElasticSearchEngineSpec.convert_dttm("DATETIME", dttm), + ElasticSearchEngineSpec.convert_dttm("DATETIME", dttm, db_extra=None), "CAST('2019-01-02T03:04:05' AS DATETIME)", ) @@ -43,7 +43,7 @@ def test_convert_dttm2(self): db_extra = {"version": "7.8"} self.assertEqual( - ElasticSearchEngineSpec.convert_dttm("DATETIME", dttm, **db_extra), + ElasticSearchEngineSpec.convert_dttm("DATETIME", dttm, db_extra=db_extra), "DATETIME_PARSE('2019-01-02 03:04:05', 'yyyy-MM-dd HH:mm:ss')", ) @@ -54,7 +54,7 @@ def test_opendistro_convert_dttm(self): dttm = self.get_dttm() self.assertEqual( - OpenDistroEngineSpec.convert_dttm("DATETIME", dttm), + OpenDistroEngineSpec.convert_dttm("DATETIME", dttm, db_extra=None), "'2019-01-02T03:04:05'", ) From 5e43a5ca8221d490f9b56717fd763501d3ac9f81 Mon Sep 17 00:00:00 2001 From: aniaan Date: Thu, 11 Nov 2021 18:32:29 +0800 Subject: [PATCH 08/11] fix(test): fix test --- superset/db_engine_specs/base.py | 2 +- tests/integration_tests/sqla_models_tests.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/superset/db_engine_specs/base.py b/superset/db_engine_specs/base.py index e795d7ed9405..93974900e360 100644 --- a/superset/db_engine_specs/base.py +++ b/superset/db_engine_specs/base.py @@ -698,7 +698,7 @@ def convert_dttm( # pylint: disable=unused-argument :param target_type: The target type of expression :param dttm: The datetime object - :param kwargs: The database extra object + :param db_extra: The database extra object :return: The SQL expression """ return None diff --git a/tests/integration_tests/sqla_models_tests.py b/tests/integration_tests/sqla_models_tests.py index 03f2d88a0790..53bf031bf53b 100644 --- a/tests/integration_tests/sqla_models_tests.py +++ b/tests/integration_tests/sqla_models_tests.py @@ -521,7 +521,9 @@ def test__normalize_prequery_result_type( dimension: str, result: Any, ) -> None: - def _convert_dttm(target_type: str, dttm: datetime) -> Optional[str]: + def _convert_dttm( + target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None + ) -> Optional[str]: if target_type.upper() == TemporalType.TIMESTAMP: return f"""TIME_PARSE('{dttm.isoformat(timespec="seconds")}')""" From cca6a6fc7a2e6b8a8ced99f93784520209440465 Mon Sep 17 00:00:00 2001 From: aniaan Date: Tue, 16 Nov 2021 21:38:09 +0800 Subject: [PATCH 09/11] fix(es): add try catch --- superset/db_engine_specs/elasticsearch.py | 17 +++++++++++++++-- .../db_engine_specs/elasticsearch_tests.py | 16 ++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/superset/db_engine_specs/elasticsearch.py b/superset/db_engine_specs/elasticsearch.py index 0d64f93d14c7..a8dc391d204a 100644 --- a/superset/db_engine_specs/elasticsearch.py +++ b/superset/db_engine_specs/elasticsearch.py @@ -14,6 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +import logging from datetime import datetime from distutils.version import StrictVersion from typing import Any, Dict, Optional, Type @@ -26,6 +27,8 @@ ) from superset.utils import core as utils +logger = logging.getLogger() + class ElasticSearchEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method engine = "elasticsearch" @@ -64,13 +67,23 @@ def convert_dttm( cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None ) -> Optional[str]: - db_extra = db_extra if db_extra else {} + db_extra = db_extra or {} if target_type.upper() == utils.TemporalType.DATETIME: es_version = db_extra.get("version") # The elasticsearch CAST function does not take effect for the time zone # setting. In elasticsearch7.8 and above, we can use the DATETIME_PARSE # function to solve this problem. - if es_version and StrictVersion(es_version) >= StrictVersion("7.8"): + supports_dttm_parse = False + try: + if es_version: + supports_dttm_parse = StrictVersion(es_version) >= StrictVersion( + "7.8" + ) + except Exception as ex: + logger.error("Unexpected error while convert es_version", exc_info=True) + logger.exception(ex) + + if supports_dttm_parse: datetime_formatted = dttm.isoformat(sep=" ", timespec="seconds") return ( f"""DATETIME_PARSE('{datetime_formatted}', 'yyyy-MM-dd HH:mm:ss')""" diff --git a/tests/integration_tests/db_engine_specs/elasticsearch_tests.py b/tests/integration_tests/db_engine_specs/elasticsearch_tests.py index 7416edfa94ec..53c70f3aa0f5 100644 --- a/tests/integration_tests/db_engine_specs/elasticsearch_tests.py +++ b/tests/integration_tests/db_engine_specs/elasticsearch_tests.py @@ -47,6 +47,22 @@ def test_convert_dttm2(self): "DATETIME_PARSE('2019-01-02 03:04:05', 'yyyy-MM-dd HH:mm:ss')", ) + def test_convert_dttm3(self, caplog): + dttm = self.get_dttm() + db_extra = {"version": 7.8} + + self.assertEqual( + ElasticSearchEngineSpec.convert_dttm("DATETIME", dttm, db_extra=None), + "CAST('2019-01-02T03:04:05' AS DATETIME)", + ) + + self.assertNotEqual( + ElasticSearchEngineSpec.convert_dttm("DATETIME", dttm, db_extra=db_extra), + "DATETIME_PARSE('2019-01-02 03:04:05', 'yyyy-MM-dd HH:mm:ss')", + ) + + self.assertIn("Unexpected error while convert es_version", caplog.text) + def test_opendistro_convert_dttm(self): """ DB Eng Specs (opendistro): Test convert_dttm From 6441f4d65a5dfd1a27a62828313f21a4a4919486 Mon Sep 17 00:00:00 2001 From: aniaan Date: Tue, 16 Nov 2021 22:05:52 +0800 Subject: [PATCH 10/11] fix(test): fix caplog --- superset/db_engine_specs/elasticsearch.py | 2 +- .../db_engine_specs/elasticsearch_tests.py | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/superset/db_engine_specs/elasticsearch.py b/superset/db_engine_specs/elasticsearch.py index a8dc391d204a..12a5e21e225d 100644 --- a/superset/db_engine_specs/elasticsearch.py +++ b/superset/db_engine_specs/elasticsearch.py @@ -79,7 +79,7 @@ def convert_dttm( supports_dttm_parse = StrictVersion(es_version) >= StrictVersion( "7.8" ) - except Exception as ex: + except Exception as ex: # pylint: disable=broad-except logger.error("Unexpected error while convert es_version", exc_info=True) logger.exception(ex) diff --git a/tests/integration_tests/db_engine_specs/elasticsearch_tests.py b/tests/integration_tests/db_engine_specs/elasticsearch_tests.py index 53c70f3aa0f5..c7adc4d683ad 100644 --- a/tests/integration_tests/db_engine_specs/elasticsearch_tests.py +++ b/tests/integration_tests/db_engine_specs/elasticsearch_tests.py @@ -23,9 +23,14 @@ OpenDistroEngineSpec, ) from tests.integration_tests.db_engine_specs.base_tests import TestDbEngineSpec +import pytest class TestElasticSearchDbEngineSpec(TestDbEngineSpec): + @pytest.fixture(autouse=True) + def inject_fixtures(self, caplog): + self._caplog = caplog + def test_convert_dttm(self): dttm = self.get_dttm() @@ -52,7 +57,7 @@ def test_convert_dttm3(self, caplog): db_extra = {"version": 7.8} self.assertEqual( - ElasticSearchEngineSpec.convert_dttm("DATETIME", dttm, db_extra=None), + ElasticSearchEngineSpec.convert_dttm("DATETIME", dttm, db_extra=db_extra), "CAST('2019-01-02T03:04:05' AS DATETIME)", ) @@ -61,7 +66,7 @@ def test_convert_dttm3(self, caplog): "DATETIME_PARSE('2019-01-02 03:04:05', 'yyyy-MM-dd HH:mm:ss')", ) - self.assertIn("Unexpected error while convert es_version", caplog.text) + self.assertIn("Unexpected error while convert es_version", self._caplog.text) def test_opendistro_convert_dttm(self): """ From aa600a63e4b1316823b8369285320732861a163d Mon Sep 17 00:00:00 2001 From: aniaan Date: Tue, 16 Nov 2021 22:18:34 +0800 Subject: [PATCH 11/11] fix(test): fix typo --- .../integration_tests/db_engine_specs/elasticsearch_tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration_tests/db_engine_specs/elasticsearch_tests.py b/tests/integration_tests/db_engine_specs/elasticsearch_tests.py index c7adc4d683ad..7dd5157792ac 100644 --- a/tests/integration_tests/db_engine_specs/elasticsearch_tests.py +++ b/tests/integration_tests/db_engine_specs/elasticsearch_tests.py @@ -16,6 +16,7 @@ # under the License. from unittest.mock import MagicMock +import pytest from sqlalchemy import column from superset.db_engine_specs.elasticsearch import ( @@ -23,7 +24,6 @@ OpenDistroEngineSpec, ) from tests.integration_tests.db_engine_specs.base_tests import TestDbEngineSpec -import pytest class TestElasticSearchDbEngineSpec(TestDbEngineSpec): @@ -52,7 +52,7 @@ def test_convert_dttm2(self): "DATETIME_PARSE('2019-01-02 03:04:05', 'yyyy-MM-dd HH:mm:ss')", ) - def test_convert_dttm3(self, caplog): + def test_convert_dttm3(self): dttm = self.get_dttm() db_extra = {"version": 7.8}