Merge apache 72d1011

MANIFEST.in

@@ -18,7 +18,7 @@ include NOTICE
 include LICENSE.txt
 graft licenses/
 include README.md
-recursive-include superset/data *
+recursive-include superset/examples *
 recursive-include superset/migrations *
 recursive-include superset/static *
 recursive-exclude superset/static/assets/docs *

UPDATING.md

@@ -23,6 +23,10 @@ assists people when migrating to a new version.
 
 ## Next Version
 
+* [7848](https://github.com/apache/incubator-superset/pull/7848): If you are 
+running redis with celery, celery bump to 4.3.0 requires redis-py upgrade to
+3.2.0 or later.
+
 * [7667](https://github.com/apache/incubator-superset/pull/7667): a change to
 make all Unix timestamp (which by definition are in UTC) comparisons refer
 to a timestamp in UTC as opposed to local time.

contrib/docker/Dockerfile

@@ -45,11 +45,14 @@ WORKDIR /home/superset
 
 COPY requirements.txt .
 COPY requirements-dev.txt .
+COPY contrib/docker/requirements-extra.txt .
 
 RUN pip install --upgrade setuptools pip \
-    && pip install -r requirements.txt -r requirements-dev.txt \
+    && pip install -r requirements.txt -r requirements-dev.txt -r requirements-extra.txt \
     && rm -rf /root/.cache/pip
 
+RUN pip install gevent
+
 COPY --chown=superset:superset superset superset
 
 ENV PATH=/home/superset/superset/bin:$PATH \

contrib/docker/README.md

@@ -26,7 +26,7 @@ Docker is an easy way to get stated with Superset.
 To initialize the database with a user and example charts, dashboards and datasets run:
 
 ```bash
-SUPERSET_LOAD_EXAMPLES=yes docker-compose run --rm superset ./docker-init.sh
+docker-compose run -e SUPERSET_LOAD_EXAMPLES=yes --rm superset ./docker-init.sh
 ```
 
 This may take a minute.

contrib/docker/docker-compose.yml

@@ -20,7 +20,7 @@ services:
     image: redis:3.2
     restart: unless-stopped
     ports:
-      - 6379:6379
+      - "127.0.0.1:6379:6379"
     volumes:
       - redis:/data
 
@@ -32,7 +32,7 @@ services:
       POSTGRES_PASSWORD: superset
       POSTGRES_USER: superset
     ports:
-      - 5432:5432
+      - "127.0.0.1:5432:5432"
     volumes:
       - postgres:/var/lib/postgresql/data
 
@@ -52,6 +52,7 @@ services:
       # If using production, comment development volume below
       #SUPERSET_ENV: production
       SUPERSET_ENV: development
+    user: root:root
     ports:
       - 8088:8088
     depends_on:

contrib/docker/requirements-extra.txt

@@ -0,0 +1,17 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+gevent==1.4.0

docs/installation.rst

@@ -521,6 +521,24 @@ into your global default defined in ``CACHE_CONFIG``.
         'CACHE_REDIS_URL': 'redis://localhost:6379/0',
     }
 
+It is also possible to pass a custom cache initialization function in the
+config to handle additional caching use cases. The function must return an
+object that is compatible with the `Flask-Cache <https://pythonhosted.org/Flask-Cache/>`_ API.
+
+.. code-block:: python
+
+    from custom_caching import CustomCache
+
+    def init_cache(app):
+        """Takes an app instance and returns a custom cache backend"""
+        config = {
+            'CACHE_DEFAULT_TIMEOUT': 60 * 60 * 24, # 1 day default (in secs)
+            'CACHE_KEY_PREFIX': 'superset_results',
+        }
+        return CustomCache(app, config)
+
+    CACHE_CONFIG = init_cache
+
 Superset has a Celery task that will periodically warm up the cache based on
 different strategies. To use it, add the following to the `CELERYBEAT_SCHEDULE`
 section in `config.py`:

docs/sqllab.rst

@@ -87,6 +87,8 @@ Superset's Jinja context:
 
 .. autofunction:: superset.jinja_context.filter_values
 
+.. autofunction:: superset.jinja_context.CacheKeyWrapper.cache_key_wrapper
+
 .. autoclass:: superset.jinja_context.PrestoTemplateProcessor
     :members:
 

requirements-dev.txt

@@ -30,6 +30,6 @@ pycodestyle==2.5.0
 pyhive==0.6.1
 pylint==1.9.2
 python-dotenv==0.10.1
-redis==2.10.6
+redis==3.2.1
 statsd==3.3.0
 tox==3.11.1

requirements.txt

@@ -5,14 +5,14 @@
 #    pip-compile --output-file=requirements.txt setup.py
 #
 alembic==1.0.0            # via flask-migrate
-amqp==2.3.2               # via kombu
+amqp==2.5.0               # via kombu
 apispec[yaml]==1.2.0      # via flask-appbuilder
 asn1crypto==0.24.0        # via cryptography
 attrs==19.1.0             # via jsonschema
 babel==2.6.0              # via flask-babel
-billiard==3.5.0.4         # via celery
+billiard==3.6.0.0         # via celery
 bleach==3.0.2
-celery==4.2.0
+celery==4.3.0
 certifi==2018.8.24        # via requests
 cffi==1.11.5              # via cryptography
 chardet==3.0.4            # via requests
@@ -44,15 +44,15 @@ isodate==0.6.0
 itsdangerous==0.24        # via flask
 jinja2==2.10.1            # via flask, flask-babel
 jsonschema==3.0.1         # via flask-appbuilder
-kombu==4.2.1              # via celery
+kombu==4.6.3              # via celery
 mako==1.0.7               # via alembic
 markdown==3.0
 markupsafe==1.0           # via jinja2, mako
 marshmallow-enum==1.4.1   # via flask-appbuilder
 marshmallow-sqlalchemy==0.16.2  # via flask-appbuilder
 marshmallow==2.19.2       # via flask-appbuilder, marshmallow-enum, marshmallow-sqlalchemy
 numpy==1.15.2             # via pandas
-pandas==0.23.4
+pandas==0.24.2
 parsedatetime==2.0.0
 pathlib2==2.3.0
 polyline==1.3.2
@@ -78,7 +78,7 @@ sqlalchemy-utils==0.33.11
 sqlalchemy==1.3.5
 sqlparse==0.2.4
 urllib3==1.24.3           # via requests, selenium
-vine==1.1.4               # via amqp
+vine==1.3.0               # via amqp, celery
 webencodings==0.5.1       # via bleach
 werkzeug==0.14.1          # via flask, flask-jwt-extended
 wtforms-json==0.3.3

setup.py

@@ -66,7 +66,7 @@ def get_git_sha():
     scripts=["superset/bin/superset"],
     install_requires=[
         "bleach>=3.0.2, <4.0.0",
-        "celery>=4.2.0, <5.0.0",
+        "celery>=4.3.0, <5.0.0",
         "click>=6.0, <7.0.0",  # `click`>=7 forces "-" instead of "_"
         "colorama",
         "contextlib2",
@@ -85,7 +85,7 @@ def get_git_sha():
         "idna",
         "isodate",
         "markdown>=3.0",
-        "pandas>=0.18.0, <0.24.0",  # `pandas`>=0.24.0 changes datetimelike API
+        "pandas>=0.24.2, <0.25.0",
         "parsedatetime",
         "pathlib2",
         "polyline",

superset/assets/cypress.json

@@ -1,10 +1,11 @@
 {
   "baseUrl": "http://localhost:8081",
-  "videoUploadOnPasses": false,
-  "video": false,
+  "chromeWebSecurity": false,
+  "defaultCommandTimeout": 10000,
   "ignoreTestFiles": ["**/!(*.test.js)"],
   "projectId": "fbf96q",
-  "defaultCommandTimeout": 10000,
+  "video": false,
+  "videoUploadOnPasses": false,
   "viewportWidth": 1280,
   "viewportHeight": 800
 }

superset/assets/cypress/integration/dashboard/save.js

@@ -20,8 +20,6 @@ import readResponseBlob from '../../utils/readResponseBlob';
 import { WORLD_HEALTH_DASHBOARD } from './dashboard.helper';
 
 export default () => describe('save', () => {
-  Cypress.config('chromeWebSecurity', false);
-
   let dashboardId;
   let boxplotChartId;
 

superset/assets/package.json

@@ -75,7 +75,7 @@
     "@superset-ui/legacy-plugin-chart-word-cloud": "^0.10.11",
     "@superset-ui/legacy-plugin-chart-world-map": "^0.10.11",
     "@superset-ui/legacy-preset-chart-big-number": "^0.10.11",
-    "@superset-ui/legacy-preset-chart-nvd3": "^0.10.35",
+    "@superset-ui/legacy-preset-chart-nvd3": "^0.10.36",
     "@superset-ui/number-format": "^0.11.3",
     "@superset-ui/time-format": "^0.11.3",
     "@superset-ui/translation": "^0.11.0",

superset/assets/spec/javascripts/components/TableSelector_spec.jsx

@@ -145,7 +145,6 @@ describe('TableSelector', () => {
     it('should clear table options', () => {
       inst.fetchTables(true);
       expect(wrapper.state().tableOptions).toEqual([]);
-      expect(wrapper.state().filterOptions).toBeNull();
     });
 
     it('should fetch table options', () => {

superset/assets/spec/javascripts/explore/components/MetricsControl_spec.jsx

@@ -333,5 +333,40 @@ describe('MetricsControl', () => {
         'SUM(',
       )).toBe(true);
     });
+
+    it('Removes metrics if savedMetrics changes', () => {
+      const { props, wrapper, onChange } = setup({
+        value: [
+          {
+            expressionType: EXPRESSION_TYPES.SIMPLE,
+            column: { type: 'double', column_name: 'value' },
+            aggregate: AGGREGATES.SUM,
+            label: 'SUM(value)',
+            optionName: 'blahblahblah',
+          },
+        ],
+      });
+      expect(wrapper.state('value')).toHaveLength(1);
+
+      wrapper.setProps({ ...props, columns: [] });
+      expect(onChange.lastCall.args).toEqual([[]]);
+    });
+
+    it('Does not remove custom sql metric if savedMetrics changes', () => {
+      const { props, wrapper, onChange } = setup({
+        value: [
+          {
+            expressionType: EXPRESSION_TYPES.SQL,
+            sqlExpression: 'COUNT(*)',
+            label: 'old label',
+            hasCustomLabel: true,
+          },
+        ],
+      });
+      expect(wrapper.state('value')).toHaveLength(1);
+
+      wrapper.setProps({ ...props, columns: [] });
+      expect(onChange.calledOnce).toEqual(false);
+    });
   });
 });

superset/assets/src/SqlLab/actions/sqlLab.js

@@ -530,8 +530,7 @@ export function createDatasource(vizOptions) {
       endpoint: '/superset/sqllab_viz/',
       postPayload: { data: vizOptions },
     })
-      .then(({ json }) => {
-        const data = JSON.parse(json);
+      .then(({ data }) => {
         dispatch(createDatasourceSuccess(data));
 
         return Promise.resolve(data);

superset/assets/src/SqlLab/components/QuerySearch.jsx

@@ -207,7 +207,7 @@ class QuerySearch extends React.PureComponent {
           <div className="col-sm-2">
             <AsyncSelect
               onChange={this.onChange}
-              dataEndpoint="/databaseasync/api/read?_flt_0_expose_in_sqllab=1"
+              dataEndpoint="/api/v1/database/?q=(filters:!((col:expose_in_sqllab,opr:eq,value:!t)))"
               value={this.state.databaseId}
               mutator={this.dbMutator}
               placeholder={t('Filter by database')}