[sql lab] WIP. Autosync queries periodically

caravel/assets/javascripts/SqlLab/components/QueryAutoRefresh.jsx

@@ -16,7 +16,7 @@ class QueryAutoRefresh extends React.Component {
   }
   startTimer() {
     if (!(this.timer)) {
-      this.timer = setInterval(this.stopwatch.bind(this), 2000);
+      this.timer = setInterval(this.stopwatch.bind(this), 1000);
     }
   }
   stopTimer() {
@@ -25,26 +25,24 @@ class QueryAutoRefresh extends React.Component {
   }
   stopwatch() {
     const url = '/caravel/queries/0';
+    // No updates in case of failure.
     $.getJSON(
       url,
       (data, status, xhr) => {
-        if (status == 200) {
-          console.log("Fetched queries")
-          console.log(data)
+        if (status === "success") {
           this.props.actions.refreshQueries(data);
         }
     });
-    console.log(new Date());
   }
   render() {
     return null;
   }
 }
 QueryAutoRefresh.propTypes = {
-  query: React.PropTypes.object,
+  // queries: React.PropTypes.object,
 };
 QueryAutoRefresh.defaultProps = {
-  query: null,
+  // queries: null,
 };
 
 function mapStateToProps() {

caravel/assets/javascripts/SqlLab/components/QueryHistory.jsx

@@ -18,7 +18,7 @@ const QueryHistory = (props) => {
   if (queriesArray.length > 0) {
     return (
       <QueryTable
-        columns={['state', 'started', 'duration', 'rows', 'sql', 'actions']}
+        columns={['state', 'started', 'duration', 'progress', 'rows', 'sql', 'actions']}
         queries={queriesArray}
       />
     );
@@ -31,7 +31,7 @@ const QueryHistory = (props) => {
 };
 
 QueryHistory.defaultProps = {
-  queries: [],
+  queries: {},
 };
 
 QueryHistory.propTypes = {

caravel/assets/javascripts/SqlLab/components/QueryTable.jsx

@@ -36,13 +36,13 @@ class QueryTable extends React.Component {
   render() {
     const data = this.props.queries.map((query) => {
       const q = Object.assign({}, query);
-      const since = (q.endDttm) ? q.endDttm : new Date();
-      let duration = since.valueOf() - q.startDttm.valueOf();
-      duration = moment.utc(duration);
+      // TODO(bkyryliuk): rename ...Dttm into the ...Timestamp.
+      const since = (q.endDttm) ? q.endDttm : new Date().getTime();
+      const duration = moment.utc(since - q.startDttm);
       if (q.endDttm) {
         q.duration = duration.format('HH:mm:ss.SS');
       }
-      q.started = moment(q.startDttm).format('HH:mm:ss');
+      q.started = moment.utc(q.startDttm).format('HH:mm:ss');
       q.sql = <SyntaxHighlighter language="sql" style={github}>{q.sql}</SyntaxHighlighter>;
       q.state = (
         <span
@@ -92,7 +92,7 @@ class QueryTable extends React.Component {
           onHide={this.hideVisualizeModal.bind(this)}
         />
         <Table
-          columns={['state', 'started', 'duration', 'rows', 'sql', 'actions']}
+          columns={['state', 'started', 'duration', 'progress', 'rows', 'sql', 'actions']}
           className="table table-condensed"
           data={data}
         />
@@ -103,10 +103,10 @@ class QueryTable extends React.Component {
 QueryTable.propTypes = {
   columns: React.PropTypes.array,
   actions: React.PropTypes.object,
-  queries: React.PropTypes.object,
+  queries: React.PropTypes.array,
 };
 QueryTable.defaultProps = {
-  columns: ['state', 'started', 'duration', 'rows', 'sql', 'actions'],
+  columns: ['state', 'started', 'duration', 'progress', 'rows', 'sql', 'actions'],
   queries: [],
 };
 

caravel/assets/javascripts/SqlLab/components/ResultSet.jsx

@@ -53,7 +53,7 @@ class ResultSet extends React.Component {
         </div>
       );
     }
-    if (results.data.length > 0) {
+    if (results && results.data.length > 0) {
       return (
         <div>
           <VisualizeModal

caravel/assets/javascripts/SqlLab/components/SqlEditor.jsx

@@ -56,6 +56,7 @@ class SqlEditor extends React.Component {
   }
   startQuery(runAsync = false, ctas = false) {
     const that = this;
+
     const query = {
       id: shortid.generate(),
       sqlEditorId: this.props.queryEditor.id,
@@ -65,21 +66,28 @@ class SqlEditor extends React.Component {
       dbId: this.props.queryEditor.dbId,
       startDttm: new Date(),
     };
-    const url = '/caravel/sql_json/';
-    const data = {
+
+    // Execute the Query
+    that.props.actions.startQuery(query);
+
+    const sqlJsonUrl = '/caravel/sql_json/';
+    const sqlJsonRequest = {
+      json: true,
+      client_id: query.id,
+      async: runAsync,
       sql: this.props.queryEditor.sql,
       database_id: this.props.queryEditor.dbId,
+      sql_editor_id: this.props.queryEditor.id,
       schema: this.props.queryEditor.schema,
+      tab: this.props.queryEditor.title,
       json: true,
-      async: runAsync,
       select_as_cta: ctas,
     };
-    this.props.actions.startQuery(query);
     $.ajax({
       type: 'POST',
       dataType: 'json',
-      url,
-      data,
+      url: sqlJsonUrl,
+      data: sqlJsonRequest,
       success(results) {
         if (runAsync) {
           // TODO nothing?
@@ -130,6 +138,8 @@ class SqlEditor extends React.Component {
         </Button>
       </ButtonGroup>
     );
+    console.log("this.props.latestQuery")
+    console.log(this.props.latestQuery)
     if (this.props.latestQuery && this.props.latestQuery.state === 'running') {
       runButtons = (
         <ButtonGroup className="inline m-r-5 pull-left">

caravel/assets/javascripts/SqlLab/reducers.js

@@ -21,16 +21,6 @@ export const initialState = {
   workspaceQueries: [],
 };
 
-function removeFromObject(state, arrKey, obj, idKey = 'id') {
-  const newObject = {};
-  for (const key in state[arrKey]) {
-    if (!(key === obj[idKey])) {
-      newObject[idKey] = state[arrKey][idKey];
-    }
-  }
-  return Object.assign({}, state, { [arrKey]: newObject });
-}
-
 function addToObject(state, arrKey, obj) {
   const newObject = Object.assign({}, state[arrKey]);
   const copiedObject = Object.assign({}, obj);
@@ -116,7 +106,9 @@ export const sqlLabReducer = function (state, action) {
       return newState;
     },
     [actions.REMOVE_QUERY]() {
-      return removeFromObject(state, 'queries', action.query);
+      const newQueries = Object.assign({}, state['queries']);
+      delete newQueries[action.query.id]
+      return Object.assign({}, state, { queries: newQueries });
     },
     [actions.RESET_STATE]() {
       return Object.assign({}, initialState);
@@ -191,8 +183,13 @@ export const sqlLabReducer = function (state, action) {
       return removeFromArr(state, 'alerts', action.alert);
     },
     [actions.REFRESH_QUERIES]() {
-      queries = action.queries
-      return alterInArr(state, 'queries', queries, { state: 'stopped' });
+      const newQueries = Object.assign({}, state['queries']);
+      // Fetch the updates to the queries present in the store.
+      for (var queryId in state['queries']) {
+        newQueries[queryId] = Object.assign(newQueries[queryId],
+            action.alteredQueries[queryId])
+      }
+      return Object.assign({}, state, { queries: newQueries });
     },
   };
   if (action.type in actionHandlers) {

caravel/config.py

@@ -36,8 +36,8 @@
 SECRET_KEY = '\2\1thisismyscretkey\1\2\e\y\y\h'  # noqa
 
 # The SQLAlchemy connection string.
-# SQLALCHEMY_DATABASE_URI = 'sqlite:///' + os.path.join(DATA_DIR, 'caravel.db')
-SQLALCHEMY_DATABASE_URI = 'mysql://root@localhost/caravel_db'
+SQLALCHEMY_DATABASE_URI = 'sqlite:///' + os.path.join(DATA_DIR, 'caravel.db')
+# SQLALCHEMY_DATABASE_URI = 'mysql://myapp@localhost/myapp'
 # SQLALCHEMY_DATABASE_URI = 'postgresql://root:password@localhost/myapp'
 
 # Flask-WTF flag for CSRF

caravel/migrations/versions/ad82a75afd82_add_query_model.py

@@ -17,9 +17,11 @@
 def upgrade():
     op.create_table('query',
         sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('client_id',  sa.String(length=11), nullable=False),
         sa.Column('database_id', sa.Integer(), nullable=False),
         sa.Column('tmp_table_name', sa.String(length=256), nullable=True),
         sa.Column('tab_name', sa.String(length=256),nullable=True),
+        sa.Column('sql_editor_id', sa.String(length=256), nullable=True),
         sa.Column('user_id', sa.Integer(), nullable=True),
         sa.Column('status', sa.String(length=16), nullable=True),
         sa.Column('name', sa.String(length=256), nullable=True),

caravel/models.py

@@ -1750,8 +1750,8 @@ def from_presto_states(self, presto_status):
 
     SCHEDULED = 'SCHEDULED'
     CANCELLED = 'CANCELLED'
-    IN_PROGRESS = 'IN_PROGRESS'
-    FINISHED = 'FINISHED'
+    IN_PROGRESS = 'RUNNING'
+    FINISHED = 'SUCCESS'
     TIMED_OUT = 'TIMED_OUT'
     FAILED = 'FAILED'
 
@@ -1762,6 +1762,7 @@ class Query(Model):
 
     __tablename__ = 'query'
     id = Column(Integer, primary_key=True)
+    client_id = Column(String(11))
 
     database_id = Column(Integer, ForeignKey('dbs.id'), nullable=False)
 
@@ -1775,6 +1776,7 @@ class Query(Model):
 
     name = Column(String(256))
     tab_name = Column(String(256))
+    sql_editor_id = Column(String(256))
     schema = Column(String(256))
     sql = Column(Text)
     # Query to retrieve the results,
@@ -1803,17 +1805,21 @@ class Query(Model):
 
     def to_dict(self):
         return {
-            'id': self.id,
-            'database_id': self.database_id,
-            'tab_name': self.tab_name,
-            'user_id': self.user_id,
-            'status': self.status,
+            'serverId': self.id,
+            'id': self.client_id,
+            'dbId': self.database_id,
+            'tab': self.tab_name,
+            'sqlEditorId': self.sql_editor_id,
+            'userId': self.user_id,
+            'state': self.status.lower(),
             'schema': self.schema,
             'sql': self.sql,
             'limit': self.limit,
             'progress': self.progress,
-            'error_message': self.error_message,
-            'start_time': self.start_time,
-            'end_time': self.end_time,
-            'changed_on': self.end_time
+            'errorMessage': self.error_message,
+            'startDttm': self.start_time,
+            'endDttm': self.end_time,
+            'changedOn': self.changed_on,
+            'rows': self.rows,
+
         }

caravel/sql_lab.py

@@ -53,7 +53,7 @@ def _sanity_check(self):
 
         if self._query.error_message:
             self._query.status = models.QueryStatus.FAILED
-            self._session.flush()
+            self._session.commit()
             return False
         return True
 
@@ -98,7 +98,9 @@ def run_sql(self):
         self._get_sql_results(engine)
 
         self._query.end_time = datetime.now()
-        self._session.flush()
+        self._session.commit()
+
+        query = self._session.query(models.Query).filter_by(id=self._query.id).first()
         return self._query.status
 
     def _get_sql_results(self, engine):
@@ -113,8 +115,6 @@ def _get_sql_results(self, engine):
         cursor = result_proxy.cursor
         if hasattr(cursor, "poll"):
             query_stats = cursor.poll()
-            self._query.status = models.QueryStatus.IN_PROGRESS
-            self._session.flush()
             # poll returns dict -- JSON status information or ``None``
             # if the query is done
             # https://github.com/dropbox/PyHive/blob/
@@ -128,7 +128,7 @@ def _get_sql_results(self, engine):
                 if progress > self._query.progress:
                     self._query.progress = progress
 
-                self._session.flush()
+                self._session.commit()
                 query_stats = cursor.poll()
                 # TODO(b.kyryliuk): check for the kill signal.
 

caravel/sql_lab_utils.py

@@ -14,7 +14,7 @@ def create_scoped_session():
     engine = create_engine(
         app.config.get('SQLALCHEMY_DATABASE_URI'), convert_unicode=True)
     return scoped_session(sessionmaker(
-        autocommit=True, autoflush=False, bind=engine))
+        autocommit=False, autoflush=False, bind=engine))
 
 
 def fetch_response_from_cursor(result_proxy):