diff --git a/caravel/assets/javascripts/SqlLab/components/QueryAutoRefresh.jsx b/caravel/assets/javascripts/SqlLab/components/QueryAutoRefresh.jsx index cccb6bfb8c19..036579670c97 100644 --- a/caravel/assets/javascripts/SqlLab/components/QueryAutoRefresh.jsx +++ b/caravel/assets/javascripts/SqlLab/components/QueryAutoRefresh.jsx @@ -16,7 +16,7 @@ class QueryAutoRefresh extends React.Component { } startTimer() { if (!(this.timer)) { - this.timer = setInterval(this.stopwatch.bind(this), 2000); + this.timer = setInterval(this.stopwatch.bind(this), 1000); } } stopTimer() { @@ -25,26 +25,24 @@ class QueryAutoRefresh extends React.Component { } stopwatch() { const url = '/caravel/queries/0'; + // No updates in case of failure. $.getJSON( url, (data, status, xhr) => { - if (status == 200) { - console.log("Fetched queries") - console.log(data) + if (status === "success") { this.props.actions.refreshQueries(data); } }); - console.log(new Date()); } render() { return null; } } QueryAutoRefresh.propTypes = { - query: React.PropTypes.object, + // queries: React.PropTypes.object, }; QueryAutoRefresh.defaultProps = { - query: null, + // queries: null, }; function mapStateToProps() { diff --git a/caravel/assets/javascripts/SqlLab/components/QueryHistory.jsx b/caravel/assets/javascripts/SqlLab/components/QueryHistory.jsx index 3d591790a978..af6cb0da0863 100644 --- a/caravel/assets/javascripts/SqlLab/components/QueryHistory.jsx +++ b/caravel/assets/javascripts/SqlLab/components/QueryHistory.jsx @@ -18,7 +18,7 @@ const QueryHistory = (props) => { if (queriesArray.length > 0) { return ( ); @@ -31,7 +31,7 @@ const QueryHistory = (props) => { }; QueryHistory.defaultProps = { - queries: [], + queries: {}, }; QueryHistory.propTypes = { diff --git a/caravel/assets/javascripts/SqlLab/components/QueryTable.jsx b/caravel/assets/javascripts/SqlLab/components/QueryTable.jsx index 028352dc572b..841e50d285f1 100644 --- a/caravel/assets/javascripts/SqlLab/components/QueryTable.jsx +++ b/caravel/assets/javascripts/SqlLab/components/QueryTable.jsx @@ -36,13 +36,13 @@ class QueryTable extends React.Component { render() { const data = this.props.queries.map((query) => { const q = Object.assign({}, query); - const since = (q.endDttm) ? q.endDttm : new Date(); - let duration = since.valueOf() - q.startDttm.valueOf(); - duration = moment.utc(duration); + // TODO(bkyryliuk): rename ...Dttm into the ...Timestamp. + const since = (q.endDttm) ? q.endDttm : new Date().getTime(); + const duration = moment.utc(since - q.startDttm); if (q.endDttm) { q.duration = duration.format('HH:mm:ss.SS'); } - q.started = moment(q.startDttm).format('HH:mm:ss'); + q.started = moment.utc(q.startDttm).format('HH:mm:ss'); q.sql = {q.sql}; q.state = ( @@ -103,10 +103,10 @@ class QueryTable extends React.Component { QueryTable.propTypes = { columns: React.PropTypes.array, actions: React.PropTypes.object, - queries: React.PropTypes.object, + queries: React.PropTypes.array, }; QueryTable.defaultProps = { - columns: ['state', 'started', 'duration', 'rows', 'sql', 'actions'], + columns: ['state', 'started', 'duration', 'progress', 'rows', 'sql', 'actions'], queries: [], }; diff --git a/caravel/assets/javascripts/SqlLab/components/ResultSet.jsx b/caravel/assets/javascripts/SqlLab/components/ResultSet.jsx index 1414d1d4b720..25ed558e6ce2 100644 --- a/caravel/assets/javascripts/SqlLab/components/ResultSet.jsx +++ b/caravel/assets/javascripts/SqlLab/components/ResultSet.jsx @@ -53,7 +53,7 @@ class ResultSet extends React.Component { ); } - if (results.data.length > 0) { + if (results && results.data.length > 0) { return (
); + console.log("this.props.latestQuery") + console.log(this.props.latestQuery) if (this.props.latestQuery && this.props.latestQuery.state === 'running') { runButtons = ( diff --git a/caravel/assets/javascripts/SqlLab/reducers.js b/caravel/assets/javascripts/SqlLab/reducers.js index 730565d84b08..96615e914923 100644 --- a/caravel/assets/javascripts/SqlLab/reducers.js +++ b/caravel/assets/javascripts/SqlLab/reducers.js @@ -21,16 +21,6 @@ export const initialState = { workspaceQueries: [], }; -function removeFromObject(state, arrKey, obj, idKey = 'id') { - const newObject = {}; - for (const key in state[arrKey]) { - if (!(key === obj[idKey])) { - newObject[idKey] = state[arrKey][idKey]; - } - } - return Object.assign({}, state, { [arrKey]: newObject }); -} - function addToObject(state, arrKey, obj) { const newObject = Object.assign({}, state[arrKey]); const copiedObject = Object.assign({}, obj); @@ -116,7 +106,9 @@ export const sqlLabReducer = function (state, action) { return newState; }, [actions.REMOVE_QUERY]() { - return removeFromObject(state, 'queries', action.query); + const newQueries = Object.assign({}, state['queries']); + delete newQueries[action.query.id] + return Object.assign({}, state, { queries: newQueries }); }, [actions.RESET_STATE]() { return Object.assign({}, initialState); @@ -191,8 +183,13 @@ export const sqlLabReducer = function (state, action) { return removeFromArr(state, 'alerts', action.alert); }, [actions.REFRESH_QUERIES]() { - queries = action.queries - return alterInArr(state, 'queries', queries, { state: 'stopped' }); + const newQueries = Object.assign({}, state['queries']); + // Fetch the updates to the queries present in the store. + for (var queryId in state['queries']) { + newQueries[queryId] = Object.assign(newQueries[queryId], + action.alteredQueries[queryId]) + } + return Object.assign({}, state, { queries: newQueries }); }, }; if (action.type in actionHandlers) { diff --git a/caravel/config.py b/caravel/config.py index 6cb76aa09c4d..fd2062e68eca 100644 --- a/caravel/config.py +++ b/caravel/config.py @@ -36,8 +36,8 @@ SECRET_KEY = '\2\1thisismyscretkey\1\2\e\y\y\h' # noqa # The SQLAlchemy connection string. -# SQLALCHEMY_DATABASE_URI = 'sqlite:///' + os.path.join(DATA_DIR, 'caravel.db') -SQLALCHEMY_DATABASE_URI = 'mysql://root@localhost/caravel_db' +SQLALCHEMY_DATABASE_URI = 'sqlite:///' + os.path.join(DATA_DIR, 'caravel.db') +# SQLALCHEMY_DATABASE_URI = 'mysql://myapp@localhost/myapp' # SQLALCHEMY_DATABASE_URI = 'postgresql://root:password@localhost/myapp' # Flask-WTF flag for CSRF diff --git a/caravel/migrations/versions/ad82a75afd82_add_query_model.py b/caravel/migrations/versions/ad82a75afd82_add_query_model.py index 2d5c99551b28..a166bba63746 100644 --- a/caravel/migrations/versions/ad82a75afd82_add_query_model.py +++ b/caravel/migrations/versions/ad82a75afd82_add_query_model.py @@ -17,9 +17,11 @@ def upgrade(): op.create_table('query', sa.Column('id', sa.Integer(), nullable=False), + sa.Column('client_id', sa.String(length=11), nullable=False), sa.Column('database_id', sa.Integer(), nullable=False), sa.Column('tmp_table_name', sa.String(length=256), nullable=True), sa.Column('tab_name', sa.String(length=256),nullable=True), + sa.Column('sql_editor_id', sa.String(length=256), nullable=True), sa.Column('user_id', sa.Integer(), nullable=True), sa.Column('status', sa.String(length=16), nullable=True), sa.Column('name', sa.String(length=256), nullable=True), diff --git a/caravel/models.py b/caravel/models.py index 8f4dfda0cc48..c482b90e42ad 100644 --- a/caravel/models.py +++ b/caravel/models.py @@ -1750,8 +1750,8 @@ def from_presto_states(self, presto_status): SCHEDULED = 'SCHEDULED' CANCELLED = 'CANCELLED' - IN_PROGRESS = 'IN_PROGRESS' - FINISHED = 'FINISHED' + IN_PROGRESS = 'RUNNING' + FINISHED = 'SUCCESS' TIMED_OUT = 'TIMED_OUT' FAILED = 'FAILED' @@ -1762,6 +1762,7 @@ class Query(Model): __tablename__ = 'query' id = Column(Integer, primary_key=True) + client_id = Column(String(11)) database_id = Column(Integer, ForeignKey('dbs.id'), nullable=False) @@ -1775,6 +1776,7 @@ class Query(Model): name = Column(String(256)) tab_name = Column(String(256)) + sql_editor_id = Column(String(256)) schema = Column(String(256)) sql = Column(Text) # Query to retrieve the results, @@ -1803,17 +1805,21 @@ class Query(Model): def to_dict(self): return { - 'id': self.id, - 'database_id': self.database_id, - 'tab_name': self.tab_name, - 'user_id': self.user_id, - 'status': self.status, + 'serverId': self.id, + 'id': self.client_id, + 'dbId': self.database_id, + 'tab': self.tab_name, + 'sqlEditorId': self.sql_editor_id, + 'userId': self.user_id, + 'state': self.status.lower(), 'schema': self.schema, 'sql': self.sql, 'limit': self.limit, 'progress': self.progress, - 'error_message': self.error_message, - 'start_time': self.start_time, - 'end_time': self.end_time, - 'changed_on': self.end_time + 'errorMessage': self.error_message, + 'startDttm': self.start_time, + 'endDttm': self.end_time, + 'changedOn': self.changed_on, + 'rows': self.rows, + } diff --git a/caravel/sql_lab.py b/caravel/sql_lab.py index b7b5c3152ab2..fce615b78397 100644 --- a/caravel/sql_lab.py +++ b/caravel/sql_lab.py @@ -53,7 +53,7 @@ def _sanity_check(self): if self._query.error_message: self._query.status = models.QueryStatus.FAILED - self._session.flush() + self._session.commit() return False return True @@ -98,7 +98,9 @@ def run_sql(self): self._get_sql_results(engine) self._query.end_time = datetime.now() - self._session.flush() + self._session.commit() + + query = self._session.query(models.Query).filter_by(id=self._query.id).first() return self._query.status def _get_sql_results(self, engine): @@ -113,8 +115,6 @@ def _get_sql_results(self, engine): cursor = result_proxy.cursor if hasattr(cursor, "poll"): query_stats = cursor.poll() - self._query.status = models.QueryStatus.IN_PROGRESS - self._session.flush() # poll returns dict -- JSON status information or ``None`` # if the query is done # https://github.com/dropbox/PyHive/blob/ @@ -128,7 +128,7 @@ def _get_sql_results(self, engine): if progress > self._query.progress: self._query.progress = progress - self._session.flush() + self._session.commit() query_stats = cursor.poll() # TODO(b.kyryliuk): check for the kill signal. diff --git a/caravel/sql_lab_utils.py b/caravel/sql_lab_utils.py index 085a953667ac..9e1c21b460fb 100644 --- a/caravel/sql_lab_utils.py +++ b/caravel/sql_lab_utils.py @@ -14,7 +14,7 @@ def create_scoped_session(): engine = create_engine( app.config.get('SQLALCHEMY_DATABASE_URI'), convert_unicode=True) return scoped_session(sessionmaker( - autocommit=True, autoflush=False, bind=engine)) + autocommit=False, autoflush=False, bind=engine)) def fetch_response_from_cursor(result_proxy): diff --git a/caravel/views.py b/caravel/views.py index d93726b9536d..3bc5577b8de9 100755 --- a/caravel/views.py +++ b/caravel/views.py @@ -1273,7 +1273,7 @@ def sqllab_viz(self): table = models.SqlaTable( table_name=table_name, ) - table.database_id = data.get('dbId') + table.database_id = data.get('databaseId') table.sql = data.get('sql') db.session.add(table) cols = [] @@ -1448,40 +1448,35 @@ def theme(self): @log_this def sql_json(self): """Runs arbitrary sql and returns and json""" + async = request.form.get('async') == 'true' + client_id = request.form.get('client_id') sql = request.form.get('sql') database_id = request.form.get('database_id') schema = request.form.get('schema') - tab_name = request.form.get('tab_name') - - async = request.form.get('async') == 'true' + tab_name = request.form.get('tab') + sql_editor_id = request.form.get('sql_editor_id') tmp_table_name = request.form.get('tmp_table_name', None) select_as_cta = request.form.get('select_as_cta') == 'true' - print(request.form.get('async')) - print(type(request.form.get('async'))) + start_time = datetime.now() + query_name = '{}_{}_{}'.format( + g.user.get_id(), tab_name, start_time.strftime('%M:%S:%f')) + + s = db.session() session = db.session() mydb = session.query(models.Database).filter_by(id=database_id).first() - if not mydb: - return Response( - json.dumps({ - 'error': 'Database with id {} is missing.'.format( - database_id), - 'status': models.QueryStatus.FAILED, - }), + return Response(json.dumps({ + 'error': 'Database with id {} is missing.'.format(database_id), + 'status': models.QueryStatus.FAILED, + }), status=500, - mimetype="application/json") - - if not (self.can_access( - 'all_datasource_access', 'all_datasource_access') or - self.can_access('database_access', mydb.perm)): - raise utils.CaravelSecurityException(_( - "SQL Lab requires the `all_datasource_access` or " - "specific DB permission")) - - start_time = datetime.now() - query_name = '{}_{}_{}'.format( - g.user.get_id(), tab_name, start_time.strftime('%M:%S:%f')) + mimetype="application/json" + ) + if not (self.can_access('all_datasource_access', 'all_datasource_access') or + self.can_access('database_access', mydb.perm)): + raise utils.CaravelSecurityException(_( + "SQL Lab requires the `all_datasource_access` or specific DB permission")) query = models.Query( database_id=database_id, @@ -1492,38 +1487,61 @@ def sql_json(self): # TODO(bkyryliuk): consider it being DB property. select_as_cta=select_as_cta, start_time=start_time, - status=models.QueryStatus.SCHEDULED, + status=models.QueryStatus.IN_PROGRESS, tab_name=tab_name, + sql_editor_id=sql_editor_id, tmp_table_name=tmp_table_name, user_id=g.user.get_id(), + client_id=client_id, ) - session.add(query) - session.commit() + s.add(query) + s.commit() + + s = db.session() + query = s.query(models.Query).filter_by(id=int(query.id)).first() + mydb = s.query(models.Database).filter_by(id=query.database_id).first() + + if not mydb: + return Response( + json.dumps({ + 'error': 'Database with id {} is missing.'.format( + database_id), + 'status': models.QueryStatus.FAILED, + }), + status=500, + mimetype="application/json") + + if not (self.can_access( + 'all_datasource_access', 'all_datasource_access') or + self.can_access('database_access', mydb.perm)): + raise utils.CaravelSecurityException(_( + "SQL Lab requires the `all_datasource_access` or specific DB permission")) # Async request. if async: # Ignore the celery future object and the request may time out. sql_lab.get_sql_results.delay(query.id) - return Response(json.dumps( - { - 'query_id': query.id, - 'status': query.status, - }, - default=utils.json_int_dttm_ser, allow_nan=False), + return Response( + json.dumps({'query': query.to_dict()}, + default=utils.json_int_dttm_ser, + allow_nan=False), status=202, # Accepted mimetype="application/json") # Sync request. data = sql_lab.get_sql_results(query.id) + s.close() + s = db.session() + query = s.query(models.Query).filter_by(id=query.id).first() + data['query'] = query.to_dict() if data['status'] == models.QueryStatus.FAILED: return Response( - json.dumps( - data, default=utils.json_int_dttm_ser, allow_nan=False), + json.dumps(data, default=utils.json_int_dttm_ser, allow_nan=False), status=500, - mimetype="application/json") + mimetype="application/json" + ) return Response( - json.dumps( - data, default=utils.json_int_dttm_ser, allow_nan=False), + json.dumps(data, default=utils.json_int_dttm_ser, allow_nan=False), status=200, mimetype="application/json") @@ -1569,7 +1587,6 @@ def csv(self, query_id): @log_this def queries(self, last_updated_ms): """Get the updated queries.""" - if not g.user.get_id(): return Response( json.dumps({'error': "Please login to access the queries."}), @@ -1593,7 +1610,7 @@ def queries(self, last_updated_ms): models.Query.changed_on >= last_updated_dt) if sql_queries: - dict_queries = [q.to_dict() for q in sql_queries] + dict_queries = {q.client_id: q.to_dict() for q in sql_queries} return Response( json.dumps(dict_queries, default=utils.json_int_dttm_ser), status=200,