Merge branch 'master' into vliu_dashboard_standalone_v2

README.md

@@ -113,8 +113,7 @@ Resources
 -------------
 * [Superset Google Group](https://groups.google.com/forum/#!forum/airbnb_superset)
 * [Gitter (live chat) Channel](https://gitter.im/airbnb/superset)
-* [Docker image 1](https://hub.docker.com/r/kochalex/superset/)
-  [Docker image 2](https://hub.docker.com/r/amancevice/superset/) (community contributed)
+* [Docker image](https://hub.docker.com/r/amancevice/superset/) (community contributed)
 * [Slides from Strata (March 2016)](https://drive.google.com/open?id=0B5PVE0gzO81oOVJkdF9aNkJMSmM)
 
 

superset/assets/javascripts/dashboard/Dashboard.jsx

@@ -19,9 +19,11 @@ export function getInitialState(dashboardData, context) {
   dashboard.firstLoad = true;
 
   dashboard.posDict = {};
-  dashboard.position_json.forEach(position => {
-    dashboard.posDict[position.slice_id] = position;
-  });
+  if (dashboard.position_json) {
+    dashboard.position_json.forEach(position => {
+      dashboard.posDict[position.slice_id] = position;
+    });
+  }
   dashboard.curUserId = dashboard.context.user_id;
   dashboard.refreshTimer = null;
 

superset/assets/javascripts/explorev2/components/FieldSet.jsx

@@ -14,7 +14,7 @@ const propTypes = {
   places: PropTypes.number,
   validators: PropTypes.any,
   onChange: React.PropTypes.func,
-  value: PropTypes.oneOf([PropTypes.string, PropTypes.bool, PropTypes.array]).isRequired,
+  value: PropTypes.oneOfType([PropTypes.string, PropTypes.bool, PropTypes.array]).isRequired,
 };
 
 const defaultProps = {
@@ -40,10 +40,11 @@ export default class FieldSet extends React.Component {
       />);
   }
 
-  renderSelectField() {
+  renderSelectField(selectProps) {
     return (
       <SelectField
         {...this.props}
+        {...selectProps}
       />);
   }
 
@@ -56,18 +57,18 @@ export default class FieldSet extends React.Component {
 
   render() {
     const type = this.props.type;
-    const selectTypes = [
-      'SelectField',
-      'SelectCustomMultiField',
-      'SelectMultipleSortableField',
-      'FreeFormSelectField',
-    ];
+    const selectProps = {
+      SelectCustomMultiField: { multi: true, freeForm: true },
+      SelectMultipleSortableField: { multi: true, freeForm: false },
+      SelectField: { multi: false, freeForm: false },
+      FreeFormSelectField: { multi: false, freeForm: true },
+    };
     let field;
 
     if (type === 'CheckboxField') {
       field = this.renderCheckBoxField();
-    } else if (selectTypes.includes(type)) {
-      field = this.renderSelectField();
+    } else if (Object.keys(selectProps).includes(type)) {
+      field = this.renderSelectField(selectProps[type]);
     } else if (['TextField', 'IntegerField'].includes(type)) {
       field = this.renderTextField();
     } else if (type === 'TextAreaField') {

superset/assets/javascripts/explorev2/components/SelectField.jsx

@@ -1,18 +1,23 @@
 import React, { PropTypes } from 'react';
-import { FormGroup, FormControl } from 'react-bootstrap';
 import ControlLabelWithTooltip from './ControlLabelWithTooltip';
 import { slugify } from '../../modules/utils';
+import Select, { Creatable } from 'react-select';
+
 
 const propTypes = {
   name: PropTypes.string.isRequired,
   choices: PropTypes.array,
-  value: PropTypes.string,
+  value: PropTypes.oneOfType([PropTypes.string, PropTypes.array]),
   label: PropTypes.string,
   description: PropTypes.string,
   onChange: PropTypes.func,
+  multi: PropTypes.bool,
+  freeForm: PropTypes.bool,
 };
 
 const defaultProps = {
+  multi: false,
+  freeForm: false,
   value: '',
   label: null,
   description: null,
@@ -21,25 +26,44 @@ const defaultProps = {
 
 export default class SelectField extends React.Component {
   onChange(opt) {
-    this.props.onChange(this.props.name, opt.target.value);
+    let optionValue = opt ? opt.value : null;
+    // if multi, return options values as an array
+    if (this.props.multi) {
+      optionValue = opt ? opt.map((o) => o.value) : null;
+    }
+    this.props.onChange(this.props.name, optionValue);
   }
-
   render() {
+    const options = this.props.choices.map((c) => ({ value: c[0], label: c[1] }));
+    if (this.props.freeForm) {
+      // For FreeFormSelect, insert value into options if not exist
+      const values = this.props.choices.map((c) => c[0]);
+      if (values.indexOf(this.props.value) === -1) {
+        options.push({ value: this.props.value, label: this.props.value });
+      }
+    }
+
+    const selectProps = {
+      multi: this.props.multi,
+      name: `select-${this.props.name}`,
+      placeholder: `Select (${this.props.choices.length})`,
+      options,
+      value: this.props.value,
+      autosize: false,
+      onChange: this.onChange.bind(this),
+    };
+    //  Tab, comma or Enter will trigger a new option created for FreeFormSelect
+    const selectWrap = this.props.freeForm ?
+      (<Creatable {...selectProps} />) : (<Select {...selectProps} />);
+
     return (
-      <FormGroup controlId={`formControlsSelect-${slugify(this.props.label)}`}>
+      <div id={`formControlsSelect-${slugify(this.props.label)}`}>
         <ControlLabelWithTooltip
           label={this.props.label}
           description={this.props.description}
         />
-        <FormControl
-          componentClass="select"
-          placeholder="select"
-          onChange={this.onChange.bind(this)}
-          value={this.props.value}
-        >
-          {this.props.choices.map((c) => <option key={c[0]} value={c[0]}>{c[1]}</option>)}
-        </FormControl>
-      </FormGroup>
+        {selectWrap}
+      </div>
     );
   }
 }

superset/assets/javascripts/explorev2/stores/store.js

@@ -1278,23 +1278,23 @@ export const fields = {
   },
 
   series_height: {
-    type: 'SelectCustomMultiField',
+    type: 'FreeFormSelectField',
     label: 'Series Height',
     default: 25,
     choices: formatSelectOptions([10, 25, 40, 50, 75, 100, 150, 200]),
     description: 'Pixel height of each series',
   },
 
   x_axis_format: {
-    type: 'SelectCustomMultiField',
+    type: 'FreeFormSelectField',
     label: 'X axis format',
     default: 'smart_date',
     choices: TIME_STAMP_OPTIONS,
     description: D3_FORMAT_DOCS,
   },
 
   y_axis_format: {
-    type: 'SelectCustomMultiField',
+    type: 'FreeFormSelectField',
     label: 'Y axis format',
     default: '.3s',
     choices: D3_TIME_FORMAT_OPTIONS,

superset/assets/package.json

@@ -55,6 +55,7 @@
     "font-awesome": "^4.6.3",
     "gridster": "^0.5.6",
     "immutability-helper": "^2.0.0",
+    "immutable": "^3.8.1",
     "jquery": "^2.2.1",
     "jquery-ui": "1.10.5",
     "mapbox-gl": "^0.26.0",

superset/assets/spec/javascripts/explorev2/components/SelectField_spec.js

@@ -1,6 +1,6 @@
 /* eslint-disable no-unused-expressions */
 import React from 'react';
-import { FormControl } from 'react-bootstrap';
+import Select, { Creatable } from 'react-select';
 import sinon from 'sinon';
 import { expect } from 'chai';
 import { describe, it, beforeEach } from 'mocha';
@@ -21,13 +21,18 @@ describe('SelectField', () => {
     wrapper = shallow(<SelectField {...defaultProps} />);
   });
 
-  it('renders a FormControl', () => {
-    expect(wrapper.find(FormControl)).to.have.lengthOf(1);
+  it('renders a Select', () => {
+    expect(wrapper.find(Select)).to.have.lengthOf(1);
   });
 
   it('calls onChange when toggled', () => {
-    const select = wrapper.find(FormControl);
-    select.simulate('change', { target: { value: 50 } });
+    const select = wrapper.find(Select);
+    select.simulate('change', { value: 50 });
     expect(defaultProps.onChange.calledWith('row_limit', 50)).to.be.true;
   });
+
+  it('renders a Creatable for freeform', () => {
+    wrapper = shallow(<SelectField {...defaultProps} freeForm />);
+    expect(wrapper.find(Creatable)).to.have.lengthOf(1);
+  });
 });

superset/bin/superset

@@ -112,14 +112,18 @@ def load_examples(load_test_data):
         print("Loading [Unicode test data]")
         data.load_unicode_test_data()
 
-@manager.command
-def refresh_druid():
-    """Refresh all druid datasources"""
+@manager.option(
+    '-d', '--datasource',
+    help=(
+        "Specify which datasource name to load, if omitted, all "
+        "datasources will be refreshed"))
+def refresh_druid(datasource):
+    """Refresh druid datasources"""
     session = db.session()
     from superset import models
     for cluster in session.query(models.DruidCluster).all():
         try:
-            cluster.refresh_datasources()
+            cluster.refresh_datasources(datasource_name=datasource)
         except Exception as e:
             print(
                 "Error while processing cluster '{}'\n{}".format(

superset/config.py

@@ -247,6 +247,7 @@ class CeleryConfig(object):
 
 try:
     from superset_config import *  # noqa
+    print('Loaded your LOCAL configuration')
 except ImportError:
     pass
 

superset/db_engine_specs.py

@@ -174,8 +174,11 @@ class PrestoEngineSpec(BaseEngineSpec):
 
     @classmethod
     def convert_dttm(cls, target_type, dttm):
-        if target_type.upper() in ('DATE', 'DATETIME'):
-            return "from_iso8601_date('{}')".format(dttm.isoformat())
+        tt = target_type.upper()
+        if tt == 'DATE':
+            return "from_iso8601_date('{}')".format(dttm.isoformat()[:10])
+        if tt == 'TIMESTAMP':
+            return "from_iso8601_timestamp('{}')".format(dttm.isoformat())
         return "'{}'".format(dttm.strftime('%Y-%m-%d %H:%M:%S'))
 
     @classmethod

superset/jinja_context.py

@@ -171,7 +171,7 @@ def latest_sub_partition(self, table_name, **kwargs):
         indexes = self.database.get_indexes(table_name, schema)
         part_fields = indexes[0]['column_names']
         for k in kwargs.keys():
-            if k not in k in part_field:
+            if k not in k in part_fields:
                 msg = "Field [{k}] is not part of the partionning key"
                 raise SupersetTemplateException(msg)
         if len(kwargs.keys()) != len(part_fields) - 1:

superset/models.py

@@ -1522,11 +1522,16 @@ def get_druid_version(self):
         ).format(obj=self)
         return json.loads(requests.get(endpoint).text)['version']
 
-    def refresh_datasources(self):
+    def refresh_datasources(self, datasource_name=None):
+        """Refresh metadata of all datasources in the cluster
+
+        If ``datasource_name`` is specified, only that datasource is updated
+        """
         self.druid_version = self.get_druid_version()
         for datasource in self.get_datasources():
             if datasource not in config.get('DRUID_DATA_SOURCE_BLACKLIST'):
-                DruidDatasource.sync_to_db(datasource, self)
+                if not datasource_name or datasource_name == datasource:
+                    DruidDatasource.sync_to_db(datasource, self)
 
     @property
     def perm(self):
@@ -1670,15 +1675,35 @@ def latest_metadata(self):
         # we need to set this interval to more than 1 day ago to exclude
         # realtime segments, which trigged a bug (fixed in druid 0.8.2).
         # https://groups.google.com/forum/#!topic/druid-user/gVCqqspHqOQ
-        start = (0 if self.version_higher(self.cluster.druid_version, '0.8.2') else 1)
-        intervals = (max_time - timedelta(days=7)).isoformat() + '/'
-        intervals += (max_time - timedelta(days=start)).isoformat()
-        segment_metadata = client.segment_metadata(
-            datasource=self.datasource_name,
-            intervals=intervals)
+        lbound = (max_time - timedelta(days=7)).isoformat()
+        rbound = max_time.isoformat()
+        if not self.version_higher(self.cluster.druid_version, '0.8.2'):
+            rbound = (max_time - timedelta(1)).isoformat()
+        segment_metadata = None
+        try:
+            segment_metadata = client.segment_metadata(
+                datasource=self.datasource_name,
+                intervals=lbound + '/' + rbound)
+        except Exception as e:
+            logging.warning("Failed first attempt to get latest segment")
+            logging.exception(e)
+        if not segment_metadata:
+            # if no segments in the past 7 days, look at all segments
+            lbound = datetime(1901, 1, 1).isoformat()[:10]
+            rbound = datetime(2050, 1, 1).isoformat()[:10]
+            if not self.version_higher(self.cluster.druid_version, '0.8.2'):
+                rbound = datetime.now().isoformat()[:10]
+            try:
+                segment_metadata = client.segment_metadata(
+                    datasource=self.datasource_name,
+                    intervals=lbound + '/' + rbound)
+            except Exception as e:
+                logging.warning("Failed 2nd attempt to get latest segment")
+                logging.exception(e)
         if segment_metadata:
             return segment_metadata[-1]['columns']
 
+
     def generate_metrics(self):
         for col in self.columns:
             col.generate_metrics()
@@ -1774,6 +1799,7 @@ def sync_to_db(cls, name, cluster):
 
         cols = datasource.latest_metadata()
         if not cols:
+            logging.error("Failed at fetching the latest segment")
             return
         for col in cols:
             col_obj = (

superset/views.py

@@ -1307,7 +1307,9 @@ def import_dashboards(self):
     def explore(self, datasource_type, datasource_id):
         viz_type = request.args.get("viz_type")
         slice_id = request.args.get('slice_id')
-        slc = db.session.query(models.Slice).filter_by(id=slice_id).first()
+        slc = None
+        if slice_id:
+            slc = db.session.query(models.Slice).filter_by(id=slice_id).first()
 
         error_redirect = '/slicemodelview/list/'
         datasource_class = SourceRegistry.sources[datasource_type]

superset/viz.py

@@ -441,7 +441,8 @@ def query_obj(self):
         if fd.get('all_columns'):
             d['columns'] = fd.get('all_columns')
             d['groupby'] = []
-            d['orderby'] = [json.loads(t) for t in fd.get('order_by_cols', [])]
+            order_by_cols = fd.get('order_by_cols', []) or []
+            d['orderby'] = [json.loads(t) for t in order_by_cols]
         return d
 
     def get_df(self, query_obj=None):