Skip to content

Commit

Permalink
feat: Scheduling queries from SQL Lab (apache#7416)
Browse files Browse the repository at this point in the history
Merge commit '90eef51' from lyft-release-sp8
  • Loading branch information
betodealmeida authored and Dave Smith committed May 3, 2019
2 parents 8e23f90 + 90eef51 commit 4b5e54f
Show file tree
Hide file tree
Showing 8 changed files with 259 additions and 9 deletions.
78 changes: 78 additions & 0 deletions docs/installation.rst
Original file line number Diff line number Diff line change
Expand Up @@ -811,6 +811,84 @@ in this dictionary are made available for users to use in their SQL.
'my_crazy_macro': lambda x: x*2,
}
**Scheduling queries**

You can optionally allow your users to schedule queries directly in SQL Lab.
This is done by addding extra metadata to saved queries, which are then picked
up by an external scheduled (like [Apache Airflow](https://airflow.apache.org/)).

To allow scheduled queries, add the following to your `config.py`:

.. code-block:: python
FEATURE_FLAGS = {
# Configuration for scheduling queries from SQL Lab. This information is
# collected when the user clicks "Schedule query", and saved into the `extra`
# field of saved queries.
# See: https://github.com/mozilla-services/react-jsonschema-form
'SCHEDULED_QUERIES': {
'JSONSCHEMA': {
'title': 'Schedule',
'description': (
'In order to schedule a query, you need to specify when it '
'should start running, when it should stop running, and how '
'often it should run. You can also optionally specify '
'dependencies that should be met before the query is '
'executed. Please read the documentation for best practices '
'and more information on how to specify dependencies.'
),
'type': 'object',
'properties': {
'output_table': {
'type': 'string',
'title': 'Output table name',
},
'start_date': {
'type': 'string',
'format': 'date-time',
'title': 'Start date',
},
'end_date': {
'type': 'string',
'format': 'date-time',
'title': 'End date',
},
'schedule_interval': {
'type': 'string',
'title': 'Schedule interval',
},
'dependencies': {
'type': 'array',
'title': 'Dependencies',
'items': {
'type': 'string',
},
},
},
},
'UISCHEMA': {
'schedule_interval': {
'ui:placeholder': '@daily, @weekly, etc.',
},
'dependencies': {
'ui:help': (
'Check the documentation for the correct format when '
'defining dependencies.'
),
},
},
},
}
This feature flag is based on [react-jsonschema-form](https://github.com/mozilla-services/react-jsonschema-form),
and will add a button called "Schedule Query" to SQL Lab. When the button is
clicked, a modal will show up where the user can add the metadata required for
scheduling the query.

This information can then be retrieved from the endpoint `/savedqueryviewapi/api/read`
and used to schedule the queries that have `scheduled_queries` in their JSON
metadata. For schedulers other than Airflow, additional fields can be easily
added to the configuration file above.

Celery Flower
-------------
Expand Down
46 changes: 42 additions & 4 deletions superset/assets/package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions superset/assets/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,7 @@
"react-dom": "^16.4.1",
"react-gravatar": "^2.6.1",
"react-hot-loader": "^4.3.6",
"react-jsonschema-form": "^1.2.0",
"react-map-gl": "^4.0.10",
"react-markdown": "^3.3.0",
"react-redux": "^5.0.2",
Expand Down
10 changes: 9 additions & 1 deletion superset/assets/src/SqlLab/components/QueryAutoRefresh.jsx
Original file line number Diff line number Diff line change
Expand Up @@ -41,10 +41,18 @@ class QueryAutoRefresh extends React.PureComponent {
const { queries, queriesLastUpdate } = this.props;
const now = new Date().getTime();

// due to a race condition, queries can be marked as successful before the
// results key is set; this is a workaround until we fix the underlying
// problem
const isQueryRunning = q => (
['running', 'started', 'pending', 'fetching'].indexOf(q.state) >= 0 ||
(q.state === 'success' && q.resultsKey === null)
);

return (
queriesLastUpdate > 0 &&
Object.values(queries).some(
q => ['running', 'started', 'pending', 'fetching'].indexOf(q.state) >= 0 &&
q => isQueryRunning(q) &&
now - q.startDttm < MAX_QUERY_AGE_TO_POLL,
)
);
Expand Down
109 changes: 109 additions & 0 deletions superset/assets/src/SqlLab/components/ScheduleQueryButton.jsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import React from 'react';
import PropTypes from 'prop-types';
import Form from 'react-jsonschema-form';
import { t } from '@superset-ui/translation';

import Button from '../../components/Button';
import ModalTrigger from '../../components/ModalTrigger';

const propTypes = {
defaultLabel: PropTypes.string,
sql: PropTypes.string.isRequired,
schema: PropTypes.string.isRequired,
dbId: PropTypes.number.isRequired,
animation: PropTypes.bool,
onSchedule: PropTypes.func,
};
const defaultProps = {
defaultLabel: t('Undefined'),
animation: true,
onSchedule: () => {},
};

class ScheduleQueryButton extends React.PureComponent {
constructor(props) {
super(props);
this.state = {
description: '',
label: props.defaultLabel,
showSchedule: false,
};
this.toggleSchedule = this.toggleSchedule.bind(this);
this.onSchedule = this.onSchedule.bind(this);
this.onCancel = this.onCancel.bind(this);
this.onLabelChange = this.onLabelChange.bind(this);
this.onDescriptionChange = this.onDescriptionChange.bind(this);
}
onSchedule({ formData }) {
const query = {
label: this.state.label,
description: this.state.description,
db_id: this.props.dbId,
schema: this.props.schema,
sql: this.props.sql,
extra_json: JSON.stringify({ schedule_info: formData }),
};
this.props.onSchedule(query);
this.saveModal.close();
}
onCancel() {
this.saveModal.close();
}
onLabelChange(e) {
this.setState({ label: e.target.value });
}
onDescriptionChange(e) {
this.setState({ description: e.target.value });
}
toggleSchedule(e) {
this.setState({ target: e.target, showSchedule: !this.state.showSchedule });
}
renderModalBody() {
return (
<Form
schema={window.featureFlags.SCHEDULED_QUERIES.JSONSCHEMA}
uiSchema={window.featureFlags.SCHEDULED_QUERIES.UISCHEMA}
onSubmit={this.onSchedule}
/>
);
}
render() {
return (
<span className="ScheduleQueryButton">
<ModalTrigger
ref={(ref) => { this.saveModal = ref; }}
modalTitle={t('Schedule Query')}
modalBody={this.renderModalBody()}
triggerNode={
<Button bsSize="small" className="toggleSchedule" onClick={this.toggleSchedule}>
<i className="fa fa-calendar" /> {t('Schedule Query')}
</Button>
}
bsSize="medium"
/>
</span>
);
}
}
ScheduleQueryButton.propTypes = propTypes;
ScheduleQueryButton.defaultProps = defaultProps;

export default ScheduleQueryButton;
14 changes: 14 additions & 0 deletions superset/assets/src/SqlLab/components/SqlEditor.jsx
Original file line number Diff line number Diff line change
Expand Up @@ -36,13 +36,15 @@ import LimitControl from './LimitControl';
import TemplateParamsEditor from './TemplateParamsEditor';
import SouthPane from './SouthPane';
import SaveQuery from './SaveQuery';
import ScheduleQueryButton from './ScheduleQueryButton';
import ShareSqlLabQuery from './ShareSqlLabQuery';
import Timer from '../../components/Timer';
import Hotkeys from '../../components/Hotkeys';
import SqlEditorLeftBar from './SqlEditorLeftBar';
import AceEditorWrapper from './AceEditorWrapper';
import { STATE_BSSTYLE_MAP } from '../constants';
import RunQueryActionButton from './RunQueryActionButton';
import { FeatureFlag, isFeatureEnabled } from '../../featureFlags';

const SQL_EDITOR_PADDING = 10;
const SQL_TOOLBAR_HEIGHT = 51;
Expand Down Expand Up @@ -313,6 +315,18 @@ class SqlEditor extends React.PureComponent {
sql={this.state.sql}
/>
</span>
{isFeatureEnabled(FeatureFlag.SCHEDULED_QUERIES) &&
<span className="m-r-5">
<ScheduleQueryButton
defaultLabel={qe.title}
sql={qe.sql}
className="m-r-5"
onSchedule={this.props.actions.saveQuery}
schema={qe.schema}
dbId={qe.dbId}
/>
</span>
}
<span className="m-r-5">
<SaveQuery
defaultLabel={qe.title}
Expand Down
3 changes: 2 additions & 1 deletion superset/assets/src/featureFlags.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ export enum FeatureFlag {
SCOPED_FILTER = 'SCOPED_FILTER',
OMNIBAR = 'OMNIBAR',
CLIENT_CACHE = 'CLIENT_CACHE',
SCHEDULED_QUERIES = 'SCHEDULED_QUERIES',
}

export type FeatureFlagMap = {
Expand All @@ -39,5 +40,5 @@ export function initFeatureFlags(featureFlags: FeatureFlagMap) {
}

export function isFeatureEnabled(feature: FeatureFlag) {
return !!window.featureFlags[feature];
return window && window.featureFlags && !!window.featureFlags[feature];
}
7 changes: 4 additions & 3 deletions superset/views/sql_lab.py
Original file line number Diff line number Diff line change
Expand Up @@ -116,9 +116,10 @@ def pre_update(self, obj):

class SavedQueryViewApi(SavedQueryView):
list_columns = [
'label', 'sqlalchemy_uri', 'user_email', 'schema', 'description',
'sql']
show_columns = ['label', 'db_id', 'schema', 'description', 'sql']
'id', 'label', 'sqlalchemy_uri', 'user_email', 'schema', 'description',
'sql', 'extra_json']
show_columns = [
'label', 'db_id', 'schema', 'description', 'sql', 'extra_json']
add_columns = show_columns
edit_columns = add_columns

Expand Down

0 comments on commit 4b5e54f

Please sign in to comment.