From 50da4f8c0708f91ff24c0abd7189a137a1415bf6 Mon Sep 17 00:00:00 2001 From: Benjamin Yolken Date: Thu, 1 Dec 2016 15:18:55 -0800 Subject: [PATCH] Support running superset via pex (#1713) * Support running superset via pex * [superset] Update default port in superset/bin/superset * Fix codeclimate line length issues * Fix another line length issue, in config.py * Add trivial utils test to increase test coverage * Clean up runserver handling --- superset/bin/superset | 79 +++++++++++++++++++++++++++++++++++++++++-- superset/cli.py | 37 -------------------- superset/config.py | 8 +++++ tests/utils_tests.py | 6 ++++ 4 files changed, 91 insertions(+), 39 deletions(-) diff --git a/superset/bin/superset b/superset/bin/superset index 169008efc5ad..95ec5327724f 100755 --- a/superset/bin/superset +++ b/superset/bin/superset @@ -4,7 +4,82 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals -from superset.cli import manager +import argparse +import sys + +import gunicorn.app.base + + +class GunicornSupersetApplication(gunicorn.app.base.BaseApplication): + def __init__(self, address, port, workers, timeout): + self.options = { + 'bind': '%s:%s' % (address, port), + 'workers': workers, + 'timeout': timeout, + 'limit-request-line': 0, + 'limit-request-field_size': 0 + } + + super(GunicornSupersetApplication, self).__init__() + + def load_config(self): + config = dict([(key, value) for key, value in self.options.iteritems() + if key in self.cfg.settings and value is not None]) + for key, value in config.iteritems(): + self.cfg.set(key.lower(), value) + + def load(self): + from superset import app + + return app + + +def run_server(): + parser = argparse.ArgumentParser(description='Run gunicorn for superset') + subparsers = parser.add_subparsers() + gunicorn_parser = subparsers.add_parser('runserver') + + gunicorn_parser.add_argument( + '-d', '--debug', action='store_true', + help='Start the web server in debug mode') + gunicorn_parser.add_argument( + '-a', '--address', type=str, default='127.0.0.1', + help='Specify the address to which to bind the web server') + gunicorn_parser.add_argument( + '-p', '--port', type=int, default=8088, + help='Specify the port on which to run the web server') + gunicorn_parser.add_argument( + '-w', '--workers', type=int, default=4, + help='Number of gunicorn web server workers to fire up') + gunicorn_parser.add_argument( + '-t', '--timeout', type=int, default=30, + help='Specify the timeout (seconds) for the gunicorn web server') + + args = parser.parse_args() + + if args.debug: + from superset import app + + app.run( + host='0.0.0.0', + port=int(args.port), + threaded=True, + debug=True) + else: + gunicorn_app_obj = GunicornSupersetApplication( + args.address, args.port, args.workers, args.timeout) + gunicorn_app_obj.run() + if __name__ == "__main__": - manager.run() + if len(sys.argv) > 1 and sys.argv[1] == 'runserver': + # In the runserver case, don't go through the manager so that superset + # import is deferred until the app is loaded; this allows for the app to be run via pex + # and cleanly forked in the gunicorn case. + # + # TODO: Refactor cli so that gunicorn can be started without first importing superset; + # this will allow us to move the runserver logic back into cli module. + run_server() + else: + from superset.cli import manager + manager.run() diff --git a/superset/cli.py b/superset/cli.py index 1b108b68530e..bd99ee66efc3 100755 --- a/superset/cli.py +++ b/superset/cli.py @@ -21,43 +21,6 @@ manager.add_command('db', MigrateCommand) -@manager.option( - '-d', '--debug', action='store_true', - help="Start the web server in debug mode") -@manager.option( - '-a', '--address', default=config.get("SUPERSET_WEBSERVER_ADDRESS"), - help="Specify the address to which to bind the web server") -@manager.option( - '-p', '--port', default=config.get("SUPERSET_WEBSERVER_PORT"), - help="Specify the port on which to run the web server") -@manager.option( - '-w', '--workers', default=config.get("SUPERSET_WORKERS", 2), - help="Number of gunicorn web server workers to fire up") -@manager.option( - '-t', '--timeout', default=config.get("SUPERSET_WEBSERVER_TIMEOUT"), - help="Specify the timeout (seconds) for the gunicorn web server") -def runserver(debug, address, port, timeout, workers): - """Starts a Superset web server""" - debug = debug or config.get("DEBUG") - if debug: - app.run( - host='0.0.0.0', - port=int(port), - threaded=True, - debug=True) - else: - cmd = ( - "gunicorn " - "-w {workers} " - "--timeout {timeout} " - "-b {address}:{port} " - "--limit-request-line 0 " - "--limit-request-field_size 0 " - "superset:app").format(**locals()) - print("Starting server with command: " + cmd) - Popen(cmd, shell=True).wait() - - @manager.command def init(): """Inits the Superset application""" diff --git a/superset/config.py b/superset/config.py index a6dac3ba9fea..3ee559655814 100644 --- a/superset/config.py +++ b/superset/config.py @@ -9,6 +9,7 @@ from __future__ import print_function from __future__ import unicode_literals +import imp import json import os @@ -248,7 +249,14 @@ class CeleryConfig(object): # by humans. ROBOT_PERMISSION_ROLES = ['Public', 'Gamma', 'Alpha', 'Admin', 'sql_lab'] +CONFIG_PATH_ENV_VAR = 'SUPERSET_CONFIG_PATH' + try: + if CONFIG_PATH_ENV_VAR in os.environ: + # Explicitly import config module that is not in pythonpath; useful + # for case where app is being executed via pex. + imp.load_source('superset_config', os.environ[CONFIG_PATH_ENV_VAR]) + from superset_config import * # noqa print('Loaded your LOCAL configuration') except ImportError: diff --git a/tests/utils_tests.py b/tests/utils_tests.py index af605509700f..0cb61a66233f 100644 --- a/tests/utils_tests.py +++ b/tests/utils_tests.py @@ -2,6 +2,7 @@ from superset import utils import unittest +from mock import Mock, patch class UtilsTestCase(unittest.TestCase): def test_json_int_dttm_ser(self): @@ -16,3 +17,8 @@ def test_json_int_dttm_ser(self): with self.assertRaises(TypeError): utils.json_int_dttm_ser("this is not a date") + + @patch('superset.utils.datetime') + def test_parse_human_timedelta(self, mock_now): + mock_now.return_value = datetime(2016, 12, 1) + self.assertEquals(utils.parse_human_timedelta('now'), timedelta(0))