From 05ac27e21a6393f758ea730885b8966731a9e57b Mon Sep 17 00:00:00 2001 From: John Bodley <4567245+john-bodley@users.noreply.github.com> Date: Fri, 30 Mar 2018 09:28:16 -0700 Subject: [PATCH] [cli] Deprecating gunicorn/flower dependencies (#4451) (cherry picked from commit b3442a7b53e1ca53e893a1433e8f026aee5f9783) --- docs/installation.rst | 33 +++++++++++++++++++-------------- setup.py | 4 ++-- superset/cli.py | 15 ++++++++++++--- superset/config.py | 6 +++--- 4 files changed, 36 insertions(+), 22 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index a2d3a64d4d5d..1dcaf302e0b3 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -125,10 +125,7 @@ Follow these few simple steps to install Superset.:: # Create default roles and permissions superset init - # Start the web server on port 8088, use -p to bind to another port - superset runserver - - # To start a development web server, use the -d switch + # To start a development web server on port 8088, use -p to bind to another port # superset runserver -d @@ -147,12 +144,8 @@ Gunicorn, preferably in **async mode**, which allows for impressive concurrency even and is fairly easy to install and configure. Please refer to the documentation of your preferred technology to set up this Flask WSGI -application in a way that works well in your environment. - -While the `superset runserver` command act as an quick wrapper -around `gunicorn`, it doesn't expose all the options you may need, -so you'll want to craft your own `gunicorn` command in your production -environment. Here's an **async** setup known to work well: :: +application in a way that works well in your environment. Here's an **async** +setup known to work well in production: :: 　gunicorn \ -w 10 \ @@ -165,7 +158,7 @@ environment. Here's an **async** setup known to work well: :: superset:app Refer to the -[Gunicorn documentation](http://docs.gunicorn.org/en/stable/design.html) +`Gunicorn documentation `_ for more information. Note that *gunicorn* does not @@ -225,7 +218,6 @@ of the parameters you can copy / paste in that configuration module: :: # Superset specific config #--------------------------------------------------------- ROW_LIMIT = 5000 - SUPERSET_WORKERS = 4 SUPERSET_WEBSERVER_PORT = 8088 #--------------------------------------------------------- @@ -500,8 +492,8 @@ execute beyond the typical web request's timeout (30-60 seconds), it is necessary to configure an asynchronous backend for Superset which consist of: * one or many Superset worker (which is implemented as a Celery worker), and - can be started with the ``superset worker`` command, run - ``superset worker --help`` to view the related options + can be started with the ``celery worker`` command, run + ``celery worker --help`` to view the related options. * a celery broker (message queue) for which we recommend using Redis or RabbitMQ * a results backend that defines where the worker will persist the query @@ -521,6 +513,10 @@ have the same configuration. CELERY_CONFIG = CeleryConfig +To start a Celery worker to leverage the configuration run: :: + + celery worker --app=superset.sql_lab:celery_app --pool=gevent -Ofair + To setup a result backend, you need to pass an instance of a derivative of ``werkzeug.contrib.cache.BaseCache`` to the ``RESULTS_BACKEND`` configuration key in your ``superset_config.py``. It's possible to use @@ -561,6 +557,15 @@ in this dictionary are made available for users to use in their SQL. } +Flower is a web based tool for monitoring the Celery cluster which you can +install from pip: :: + + pip install flower + +and run via: :: + + celery flower --app=superset.sql_lab:celery_app + Making your own build --------------------- diff --git a/setup.py b/setup.py index f8f785e72150..e635a116a859 100644 --- a/setup.py +++ b/setup.py @@ -63,12 +63,12 @@ def get_git_sha(): 'flask-sqlalchemy==2.1', 'flask-testing==0.7.1', 'flask-wtf==0.14.2', - 'flower==0.9.2', + 'flower==0.9.2', # deprecated 'future>=0.16.0, <0.17', 'geopy==1.11.0', 'python-geohash==0.8.5', 'humanize==0.5.1', - 'gunicorn==19.7.1', + 'gunicorn==19.7.1', # deprecated 'idna==2.6', 'markdown==2.6.11', 'pandas==0.22.0', diff --git a/superset/cli.py b/superset/cli.py index de3a38fb4062..dc11e2cc6982 100755 --- a/superset/cli.py +++ b/superset/cli.py @@ -48,15 +48,15 @@ def init(): @manager.option( '-w', '--workers', default=config.get('SUPERSET_WORKERS', 2), - help='Number of gunicorn web server workers to fire up') + help='Number of gunicorn web server workers to fire up [DEPRECATED]') @manager.option( '-t', '--timeout', default=config.get('SUPERSET_WEBSERVER_TIMEOUT'), - help='Specify the timeout (seconds) for the gunicorn web server') + help='Specify the timeout (seconds) for the gunicorn web server [DEPRECATED]') @manager.option( '-s', '--socket', default=config.get('SUPERSET_WEBSERVER_SOCKET'), help='Path to a UNIX socket as an alternative to address:port, e.g. ' '/var/run/superset.sock. ' - 'Will override the address and port values.') + 'Will override the address and port values. [DEPRECATED]') def runserver(debug, use_reloader, address, port, timeout, workers, socket): """Starts a Superset web server.""" debug = debug or config.get('DEBUG') @@ -75,6 +75,9 @@ def runserver(debug, use_reloader, address, port, timeout, workers, socket): debug=True, use_reloader=use_reloader) else: + logging.info( + "The Gunicorn 'superset runserver' command is deprecated. Please " + "use the 'gunicorn' command instead.") addr_str = ' unix:{socket} ' if socket else' {address}:{port} ' cmd = ( 'gunicorn ' @@ -285,6 +288,9 @@ def update_datasources_cache(): help='Number of celery server workers to fire up') def worker(workers): """Starts a Superset worker for async SQL query execution.""" + logging.info( + "The 'superset worker' command is deprecated. Please use the 'celery " + "worker' command instead.") if workers: celery_app.conf.update(CELERYD_CONCURRENCY=workers) elif config.get('SUPERSET_CELERY_WORKERS'): @@ -315,6 +321,9 @@ def flower(port, address): '--port={port} ' '--address={address} ' ).format(**locals()) + logging.info( + "The 'superset flower' command is deprecated. Please use the 'celery " + "flower' command instead.") print(Fore.GREEN + 'Starting a Celery Flower instance') print(Fore.BLUE + '-=' * 40) print(Fore.YELLOW + cmd) diff --git a/superset/config.py b/superset/config.py index ae81cfcb6e40..d9e238c5cba8 100644 --- a/superset/config.py +++ b/superset/config.py @@ -42,12 +42,12 @@ VIZ_ROW_LIMIT = 10000 # max rows retrieved by filter select auto complete FILTER_SELECT_ROW_LIMIT = 10000 -SUPERSET_WORKERS = 2 -SUPERSET_CELERY_WORKERS = 32 +SUPERSET_WORKERS = 2 # deprecated +SUPERSET_CELERY_WORKERS = 32 # deprecated SUPERSET_WEBSERVER_ADDRESS = '0.0.0.0' SUPERSET_WEBSERVER_PORT = 8088 -SUPERSET_WEBSERVER_TIMEOUT = 60 +SUPERSET_WEBSERVER_TIMEOUT = 60 # deprecated EMAIL_NOTIFICATIONS = False CUSTOM_SECURITY_MANAGER = None SQLALCHEMY_TRACK_MODIFICATIONS = False