From 7325a4fb4ba08f554454534fe9efe3d0eea5a6ce Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Thu, 10 Nov 2016 23:54:51 +0000 Subject: [PATCH 1/6] [hotfix] table view not group by without orderby fails --- superset/config.py | 1 + superset/viz.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/superset/config.py b/superset/config.py index b8dbd62f226d..dcd40c39b982 100644 --- a/superset/config.py +++ b/superset/config.py @@ -247,6 +247,7 @@ class CeleryConfig(object): try: from superset_config import * # noqa + print('Loaded your LOCAL configuration') except ImportError: pass diff --git a/superset/viz.py b/superset/viz.py index aca2ac740d46..ce4ddecd26ab 100755 --- a/superset/viz.py +++ b/superset/viz.py @@ -441,7 +441,8 @@ def query_obj(self): if fd.get('all_columns'): d['columns'] = fd.get('all_columns') d['groupby'] = [] - d['orderby'] = [json.loads(t) for t in fd.get('order_by_cols', [])] + order_by_cols = fd.get('order_by_cols', []) or [] + d['orderby'] = [json.loads(t) for t in order_by_cols] return d def get_df(self, query_obj=None): From d6bc354ff3e2f24aeb459dbc1413371f2b072306 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Fri, 11 Nov 2016 00:39:20 +0000 Subject: [PATCH 2/6] [hotfix] fix support for presto DATE and TIMESTAMP type --- superset/db_engine_specs.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/superset/db_engine_specs.py b/superset/db_engine_specs.py index f5e19107b9e9..d1175352e9a6 100644 --- a/superset/db_engine_specs.py +++ b/superset/db_engine_specs.py @@ -174,8 +174,11 @@ class PrestoEngineSpec(BaseEngineSpec): @classmethod def convert_dttm(cls, target_type, dttm): - if target_type.upper() in ('DATE', 'DATETIME'): - return "from_iso8601_date('{}')".format(dttm.isoformat()) + tt = target_type.upper() + if tt == 'DATE': + return "from_iso8601_date('{}')".format(dttm.isoformat()[:10]) + if tt == 'TIMESTAMP': + return "from_iso8601_timestamp('{}')".format(dttm.isoformat()) return "'{}'".format(dttm.strftime('%Y-%m-%d %H:%M:%S')) @classmethod From 96d32dd11f29afa3590b79d8683aaddd05f48a02 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Fri, 11 Nov 2016 09:46:48 -0800 Subject: [PATCH 3/6] Improve Druid metadata fetching resilience (#1584) --- superset/bin/superset | 12 ++++++++---- superset/models.py | 42 ++++++++++++++++++++++++++++++++++-------- 2 files changed, 42 insertions(+), 12 deletions(-) diff --git a/superset/bin/superset b/superset/bin/superset index 3b5b52906ffd..3d711fc826ed 100755 --- a/superset/bin/superset +++ b/superset/bin/superset @@ -112,14 +112,18 @@ def load_examples(load_test_data): print("Loading [Unicode test data]") data.load_unicode_test_data() -@manager.command -def refresh_druid(): - """Refresh all druid datasources""" +@manager.option( + '-d', '--datasource', + help=( + "Specify which datasource name to load, if omitted, all " + "datasources will be refreshed")) +def refresh_druid(datasource): + """Refresh druid datasources""" session = db.session() from superset import models for cluster in session.query(models.DruidCluster).all(): try: - cluster.refresh_datasources() + cluster.refresh_datasources(datasource_name=datasource) except Exception as e: print( "Error while processing cluster '{}'\n{}".format( diff --git a/superset/models.py b/superset/models.py index c91bfff3224d..4557317851c1 100644 --- a/superset/models.py +++ b/superset/models.py @@ -1522,11 +1522,16 @@ def get_druid_version(self): ).format(obj=self) return json.loads(requests.get(endpoint).text)['version'] - def refresh_datasources(self): + def refresh_datasources(self, datasource_name=None): + """Refresh metadata of all datasources in the cluster + + If ``datasource_name`` is specified, only that datasource is updated + """ self.druid_version = self.get_druid_version() for datasource in self.get_datasources(): if datasource not in config.get('DRUID_DATA_SOURCE_BLACKLIST'): - DruidDatasource.sync_to_db(datasource, self) + if not datasource_name or datasource_name == datasource: + DruidDatasource.sync_to_db(datasource, self) @property def perm(self): @@ -1670,15 +1675,35 @@ def latest_metadata(self): # we need to set this interval to more than 1 day ago to exclude # realtime segments, which trigged a bug (fixed in druid 0.8.2). # https://groups.google.com/forum/#!topic/druid-user/gVCqqspHqOQ - start = (0 if self.version_higher(self.cluster.druid_version, '0.8.2') else 1) - intervals = (max_time - timedelta(days=7)).isoformat() + '/' - intervals += (max_time - timedelta(days=start)).isoformat() - segment_metadata = client.segment_metadata( - datasource=self.datasource_name, - intervals=intervals) + lbound = (max_time - timedelta(days=7)).isoformat() + rbound = max_time.isoformat() + if not self.version_higher(self.cluster.druid_version, '0.8.2'): + rbound = (max_time - timedelta(1)).isoformat() + segment_metadata = None + try: + segment_metadata = client.segment_metadata( + datasource=self.datasource_name, + intervals=lbound + '/' + rbound) + except Exception as e: + logging.warning("Failed first attempt to get latest segment") + logging.exception(e) + if not segment_metadata: + # if no segments in the past 7 days, look at all segments + lbound = datetime(1901, 1, 1).isoformat()[:10] + rbound = datetime(2050, 1, 1).isoformat()[:10] + if not self.version_higher(self.cluster.druid_version, '0.8.2'): + rbound = datetime.now().isoformat()[:10] + try: + segment_metadata = client.segment_metadata( + datasource=self.datasource_name, + intervals=lbound + '/' + rbound) + except Exception as e: + logging.warning("Failed 2nd attempt to get latest segment") + logging.exception(e) if segment_metadata: return segment_metadata[-1]['columns'] + def generate_metrics(self): for col in self.columns: col.generate_metrics() @@ -1774,6 +1799,7 @@ def sync_to_db(cls, name, cluster): cols = datasource.latest_metadata() if not cols: + logging.error("Failed at fetching the latest segment") return for col in cols: col_obj = ( From d33874bd3d9ffffca7f4726a29c3eb9de2a68d42 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Fri, 11 Nov 2016 09:57:34 -0800 Subject: [PATCH 4/6] [hotfix] postgres issue when slice_id is missing --- superset/views.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/superset/views.py b/superset/views.py index 6ea81cb4f902..841c968b4376 100755 --- a/superset/views.py +++ b/superset/views.py @@ -1307,7 +1307,9 @@ def import_dashboards(self): def explore(self, datasource_type, datasource_id): viz_type = request.args.get("viz_type") slice_id = request.args.get('slice_id') - slc = db.session.query(models.Slice).filter_by(id=slice_id).first() + slc = None + if slice_id: + slc = db.session.query(models.Slice).filter_by(id=slice_id).first() error_redirect = '/slicemodelview/list/' datasource_class = SourceRegistry.sources[datasource_type] From c064d6d8475b07a63a3b5ca7b4dbd248a437f6ed Mon Sep 17 00:00:00 2001 From: Mazdak B Date: Sat, 12 Nov 2016 03:50:04 +0330 Subject: [PATCH 5/6] Correct part_fields variable name (#1586) --- superset/jinja_context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/superset/jinja_context.py b/superset/jinja_context.py index 830466702178..94f56328b247 100644 --- a/superset/jinja_context.py +++ b/superset/jinja_context.py @@ -171,7 +171,7 @@ def latest_sub_partition(self, table_name, **kwargs): indexes = self.database.get_indexes(table_name, schema) part_fields = indexes[0]['column_names'] for k in kwargs.keys(): - if k not in k in part_field: + if k not in k in part_fields: msg = "Field [{k}] is not part of the partionning key" raise SupersetTemplateException(msg) if len(kwargs.keys()) != len(part_fields) - 1: From fdbb2bbdab5e5dbb2f3496b67eb227dc3dc5f2a7 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Fri, 11 Nov 2016 16:24:57 -0800 Subject: [PATCH 6/6] fixing the build --- superset/assets/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/superset/assets/package.json b/superset/assets/package.json index 4f1400307495..33588b25f8f8 100644 --- a/superset/assets/package.json +++ b/superset/assets/package.json @@ -55,6 +55,7 @@ "font-awesome": "^4.6.3", "gridster": "^0.5.6", "immutability-helper": "^2.0.0", + "immutable": "^3.8.1", "jquery": "^2.2.1", "jquery-ui": "1.10.5", "mapbox-gl": "^0.26.0",