More improvements to SQL Lab (#1104)

* Handling timeouts

* Fixing timer on non-utc server

* Allowing async with results

* [bugfix] database is not selected

* Making sure the session is up and running

* Cleaning up query results and query objects

* Picking a groupby and metric field on visualize flow

* Showing local time in query history

* Using pull-left pull-right instead of grid layout for table metdata

Long column name were looking weird and icons were wrapping oddly

* Linting

* Eliminating east buttons under the sql editor

* Sort database dropdown by name

* Linting

* Allowing non-SELECT statements to run

* Adding a db config

* Making sqla checkout check cross-db

.codeclimate.yml

@@ -31,3 +31,4 @@ exclude_paths:
 - "caravel/assets/vendor/"
 - "caravel/assets/node_modules/"
 - "caravel/assets/javascripts/dist/"
+- "caravel/migrations"

caravel/__init__.py

@@ -10,6 +10,7 @@
 
 from flask import Flask, redirect
 from flask_appbuilder import SQLA, AppBuilder, IndexView
+from sqlalchemy import event, exc
 from flask_appbuilder.baseviews import expose
 from flask_cache import Cache
 from flask_migrate import Migrate
@@ -27,6 +28,35 @@
 
 db = SQLA(app)
 
+
+@event.listens_for(db.engine, 'checkout')
+def checkout(dbapi_con, con_record, con_proxy):
+    """
+    Making sure the connection is live, and preventing against:
+    'MySQL server has gone away'
+
+    Copied from:
+    http://stackoverflow.com/questions/30630120/mysql-keeps-losing-connection-during-celery-tasks
+    """
+    try:
+        try:
+            if hasattr(dbapi_con, 'ping'):
+                dbapi_con.ping(False)
+            else:
+                cursor = dbapi_con.cursor()
+                cursor.execute("SELECT 1")
+        except TypeError:
+            app.logger.debug('MySQL connection died. Restoring...')
+            dbapi_con.ping()
+    except dbapi_con.OperationalError as e:
+        app.logger.warning(e)
+        if e.args[0] in (2006, 2013, 2014, 2045, 2055):
+            raise exc.DisconnectionError()
+        else:
+            raise
+    return db
+
+
 cache = Cache(app, config=app.config.get('CACHE_CONFIG'))
 
 migrate = Migrate(app, db, directory=APP_DIR + "/migrations")

caravel/assets/javascripts/SqlLab/components/QueryTable.jsx

@@ -41,7 +41,7 @@ class QueryTable extends React.Component {
       if (q.endDttm) {
         q.duration = fDuration(q.startDttm, q.endDttm);
       }
-      q.started = moment.utc(q.startDttm).format('HH:mm:ss');
+      q.started = moment(q.startDttm).format('HH:mm:ss');
       const source = (q.ctas) ? q.executedSql : q.sql;
       q.sql = (
         <SqlShrink sql={source} />

caravel/assets/javascripts/SqlLab/components/ResultSet.jsx

@@ -60,7 +60,7 @@ class ResultSet extends React.Component {
         </div>
       );
     }
-    if (results && results.data.length > 0) {
+    if (results && results.data && results.data.length > 0) {
       return (
         <div>
           <VisualizeModal

caravel/assets/javascripts/SqlLab/components/SqlEditor.jsx

@@ -9,9 +9,7 @@ import {
   InputGroup,
   Form,
   FormControl,
-  DropdownButton,
   Label,
-  MenuItem,
   OverlayTrigger,
   Row,
   Tooltip,
@@ -27,7 +25,6 @@ import { connect } from 'react-redux';
 import * as Actions from '../actions';
 
 import shortid from 'shortid';
-import ButtonWithTooltip from './ButtonWithTooltip';
 import SouthPane from './SouthPane';
 import Timer from './Timer';
 
@@ -52,8 +49,8 @@ class SqlEditor extends React.Component {
       this.startQuery();
     }
   }
-  runQuery() {
-    this.startQuery();
+  runQuery(runAsync = false) {
+    this.startQuery(runAsync);
   }
   startQuery(runAsync = false, ctas = false) {
     const that = this;
@@ -76,10 +73,10 @@ class SqlEditor extends React.Component {
 
     const sqlJsonUrl = '/caravel/sql_json/';
     const sqlJsonRequest = {
-      async: runAsync,
       client_id: query.id,
       database_id: this.props.queryEditor.dbId,
       json: true,
+      runAsync,
       schema: this.props.queryEditor.schema,
       select_as_cta: ctas,
       sql: this.props.queryEditor.sql,
@@ -149,17 +146,36 @@ class SqlEditor extends React.Component {
   }
 
   render() {
-    let runButtons = (
-      <ButtonGroup bsSize="small" className="inline m-r-5 pull-left">
+    let runButtons = [];
+    if (this.props.database && this.props.database.allow_run_sync) {
+      runButtons.push(
         <Button
           bsSize="small"
           bsStyle="primary"
           style={{ width: '100px' }}
-          onClick={this.runQuery.bind(this)}
+          onClick={this.runQuery.bind(this, false)}
           disabled={!(this.props.queryEditor.dbId)}
         >
           <i className="fa fa-table" /> Run Query
         </Button>
+      );
+    }
+    if (this.props.database && this.props.database.allow_run_async) {
+      runButtons.push(
+        <Button
+          bsSize="small"
+          bsStyle="primary"
+          style={{ width: '100px' }}
+          onClick={this.runQuery.bind(this, true)}
+          disabled={!(this.props.queryEditor.dbId)}
+        >
+          <i className="fa fa-table" /> Run Async
+        </Button>
+      );
+    }
+    runButtons = (
+      <ButtonGroup bsSize="small" className="inline m-r-5 pull-left">
+        {runButtons}
       </ButtonGroup>
     );
     if (this.props.latestQuery && this.props.latestQuery.state === 'running') {
@@ -176,35 +192,6 @@ class SqlEditor extends React.Component {
         </ButtonGroup>
       );
     }
-    const rightButtons = (
-      <ButtonGroup className="inlineblock">
-        <ButtonWithTooltip
-          tooltip="Save this query in your workspace"
-          placement="left"
-          bsSize="small"
-          onClick={this.addWorkspaceQuery.bind(this)}
-        >
-          <i className="fa fa-save" />&nbsp;
-        </ButtonWithTooltip>
-        <DropdownButton
-          id="ddbtn-export"
-          pullRight
-          bsSize="small"
-          title={<i className="fa fa-file-o" />}
-        >
-          <MenuItem
-            onClick={this.notImplemented}
-          >
-            <i className="fa fa-file-text-o" /> export to .csv
-          </MenuItem>
-          <MenuItem
-            onClick={this.notImplemented}
-          >
-            <i className="fa fa-file-code-o" /> export to .json
-          </MenuItem>
-        </DropdownButton>
-      </ButtonGroup>
-    );
     let limitWarning = null;
     const rowLimit = 1000;
     if (this.props.latestQuery && this.props.latestQuery.rows === rowLimit) {
@@ -256,7 +243,6 @@ class SqlEditor extends React.Component {
         <div className="pull-right">
           {limitWarning}
           <Timer query={this.props.latestQuery} />
-          {rightButtons}
         </div>
       </div>
     );

caravel/assets/javascripts/SqlLab/components/SqlEditorLeft.jsx

@@ -77,7 +77,12 @@ class SqlEditorTopToolbar extends React.Component {
   }
   fetchDatabaseOptions() {
     this.setState({ databaseLoading: true });
-    const url = '/databaseasync/api/read?_flt_0_expose_in_sqllab=1';
+    const url = (
+      '/databaseasync/api/read?' +
+      '_flt_0_expose_in_sqllab=1&' +
+      '_oc_DatabaseAsync=database_name&' +
+      '_od_DatabaseAsync=asc'
+    );
     $.get(url, (data) => {
       const options = data.result.map((db) => ({ value: db.id, label: db.database_name }));
       this.props.actions.setDatabases(data.result);

caravel/assets/javascripts/SqlLab/components/TableElement.jsx

@@ -63,12 +63,12 @@ class TableElement extends React.Component {
       metadata = (
         <div>
           {this.props.table.columns.map((col) => (
-            <div className="row">
-              <div className="col-sm-8">
-                <div className="m-l-5">{col.name}</div>
+            <div className="clearfix">
+              <div className="pull-left m-l-10">
+                {col.name}
               </div>
-              <div className="col-sm-4">
-                <div className="pull-right text-muted"><small>{col.type}</small></div>
+              <div className="pull-right text-muted">
+                <small> {col.type}</small>
               </div>
             </div>
           ))}
@@ -88,11 +88,11 @@ class TableElement extends React.Component {
     }
     return (
       <div>
-        <div className="row">
-          <div className="col-sm-9 m-b-10">
+        <div className="clearfix">
+          <div className="pull-left">
             {buttonToggle}
           </div>
-          <div className="col-sm-3">
+          <div className="pull-right">
             <ButtonGroup className="ws-el-controls pull-right">
               <Link
                 className="fa fa-pencil pull-left m-l-2"

caravel/assets/javascripts/SqlLab/components/Timer.jsx

@@ -27,8 +27,8 @@ class Timer extends React.Component {
   }
   stopwatch() {
     if (this.props && this.props.query) {
-      const since = this.props.query.endDttm || now();
-      const clockStr = fDuration(this.props.query.startDttm, since);
+      const endDttm = this.props.query.endDttm || now();
+      const clockStr = fDuration(this.props.query.startDttm, endDttm);
       this.setState({ clockStr });
       if (this.props.query.state !== 'running') {
         this.stopTimer();

caravel/assets/javascripts/SqlLab/main.css

@@ -239,6 +239,3 @@ div.tablePopover:hover {
 .SouthPane .tab-content {
     padding-top: 10px;
 }
-.SqlEditor textarea {
-    display: none;
-}

caravel/assets/javascripts/SqlLab/reducers.js

@@ -36,7 +36,7 @@ function addToObject(state, arrKey, obj) {
 
 function alterInObject(state, arrKey, obj, alterations) {
   const newObject = Object.assign({}, state[arrKey]);
-  newObject[obj.id] = (Object.assign({}, newObject[obj.id], alterations));
+  newObject[obj.id] = Object.assign({}, newObject[obj.id], alterations);
   return Object.assign({}, state, { [arrKey]: newObject });
 }
 
@@ -65,6 +65,16 @@ function removeFromArr(state, arrKey, obj, idKey = 'id') {
   return Object.assign({}, state, { [arrKey]: newArr });
 }
 
+function getFromArr(arr, id) {
+  let obj;
+  arr.forEach((o) => {
+    if (o.id === id) {
+      obj = o;
+    }
+  });
+  return obj;
+}
+
 function addToArr(state, arrKey, obj) {
   const newObj = Object.assign({}, obj);
   if (!newObj.id) {
@@ -87,9 +97,16 @@ export const sqlLabReducer = function (state, action) {
       let newState = removeFromArr(state, 'queryEditors', action.queryEditor);
       // List of remaining queryEditor ids
       const qeIds = newState.queryEditors.map((qe) => qe.id);
-      let th = state.tabHistory.slice();
-      th = th.filter((id) => qeIds.includes(id));
-      newState = Object.assign({}, newState, { tabHistory: th });
+      const queries = {};
+      Object.keys(state.queries).forEach((k) => {
+        const query = state.queries[k];
+        if (qeIds.includes(query.sqlEditorId)) {
+          queries[k] = query;
+        }
+      });
+      let tabHistory = state.tabHistory.slice();
+      tabHistory = tabHistory.filter((id) => qeIds.includes(id));
+      newState = Object.assign({}, newState, { tabHistory, queries });
       return newState;
     },
     [actions.REMOVE_QUERY]() {
@@ -113,20 +130,31 @@ export const sqlLabReducer = function (state, action) {
       return removeFromArr(state, 'tables', action.table);
     },
     [actions.START_QUERY]() {
-      const newState = addToObject(state, 'queries', action.query);
+      const qe = getFromArr(state.queryEditors, action.query.sqlEditorId);
+      let newState = Object.assign({}, state);
+      if (qe.latestQueryId) {
+        const q = Object.assign({}, state.queries[qe.latestQueryId], { results: null });
+        const queries = Object.assign({}, state.queries, { [q.id]: q });
+        newState = Object.assign({}, state, { queries });
+      }
+      newState = addToObject(newState, 'queries', action.query);
       const sqlEditor = { id: action.query.sqlEditorId };
       return alterInArr(newState, 'queryEditors', sqlEditor, { latestQueryId: action.query.id });
     },
     [actions.STOP_QUERY]() {
       return alterInObject(state, 'queries', action.query, { state: 'stopped' });
     },
     [actions.QUERY_SUCCESS]() {
+      let rows;
+      if (action.results.data) {
+        rows = action.results.data.length;
+      }
       const alts = {
-        state: 'success',
-        results: action.results,
-        rows: action.results.data.length,
-        progress: 100,
         endDttm: now(),
+        progress: 100,
+        results: action.results,
+        rows,
+        state: 'success',
       };
       return alterInObject(state, 'queries', action.query, alts);
     },
@@ -158,12 +186,6 @@ export const sqlLabReducer = function (state, action) {
     [actions.QUERY_EDITOR_SET_AUTORUN]() {
       return alterInArr(state, 'queryEditors', action.queryEditor, { autorun: action.autorun });
     },
-    [actions.ADD_WORKSPACE_QUERY]() {
-      return addToArr(state, 'workspaceQueries', action.query);
-    },
-    [actions.REMOVE_WORKSPACE_QUERY]() {
-      return removeFromArr(state, 'workspaceQueries', action.query);
-    },
     [actions.ADD_ALERT]() {
       return addToArr(state, 'alerts', action.alert);
     },

caravel/assets/visualizations/nvd3_vis.css

@@ -8,7 +8,6 @@ g.caravel path {
 }
 
 text.nv-axislabel {
-  // font-weight: bold;
   font-size: 14px;
 }
 

caravel/config.py

@@ -213,6 +213,9 @@ class CeleryConfig(object):
 # The db id here results in selecting this one as a default in SQL Lab
 DEFAULT_DB_ID = None
 
+# Timeout duration for SQL Lab synchronous queries
+SQLLAB_TIMEOUT = 30
+
 try:
     from caravel_config import *  # noqa
 except ImportError:

caravel/migrations/versions/4500485bde7d_allow_run_sync_async.py

@@ -0,0 +1,28 @@
+"""allow_run_sync_async
+
+Revision ID: 4500485bde7d
+Revises: 41f6a59a61f2
+Create Date: 2016-09-12 23:33:14.789632
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '4500485bde7d'
+down_revision = '41f6a59a61f2'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    op.add_column('dbs', sa.Column('allow_run_async', sa.Boolean(), nullable=True))
+    op.add_column('dbs', sa.Column('allow_run_sync', sa.Boolean(), nullable=True))
+
+
+def downgrade():
+    try:
+        op.drop_column('dbs', 'allow_run_sync')
+        op.drop_column('dbs', 'allow_run_async')
+    except Exception:
+        pass
+