Support running superset via pex (#1713)

* Support running superset via pex

* [superset] Update default port in superset/bin/superset

* Fix codeclimate line length issues

* Fix another line length issue, in config.py

* Add trivial utils test to increase test coverage

* Clean up runserver handling

superset/bin/superset

@@ -4,7 +4,82 @@ from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
 
-from superset.cli import manager
+import argparse
+import sys
+
+import gunicorn.app.base
+
+
+class GunicornSupersetApplication(gunicorn.app.base.BaseApplication):
+    def __init__(self, address, port, workers, timeout):
+        self.options = {
+            'bind': '%s:%s' % (address, port),
+            'workers': workers,
+            'timeout': timeout,
+            'limit-request-line': 0,
+            'limit-request-field_size': 0
+        }
+
+        super(GunicornSupersetApplication, self).__init__()
+
+    def load_config(self):
+        config = dict([(key, value) for key, value in self.options.iteritems()
+                       if key in self.cfg.settings and value is not None])
+        for key, value in config.iteritems():
+            self.cfg.set(key.lower(), value)
+
+    def load(self):
+        from superset import app
+
+        return app
+
+
+def run_server():
+    parser = argparse.ArgumentParser(description='Run gunicorn for superset')
+    subparsers = parser.add_subparsers()
+    gunicorn_parser = subparsers.add_parser('runserver')
+
+    gunicorn_parser.add_argument(
+        '-d', '--debug', action='store_true',
+        help='Start the web server in debug mode')
+    gunicorn_parser.add_argument(
+        '-a', '--address', type=str, default='127.0.0.1',
+        help='Specify the address to which to bind the web server')
+    gunicorn_parser.add_argument(
+        '-p', '--port', type=int, default=8088,
+        help='Specify the port on which to run the web server')
+    gunicorn_parser.add_argument(
+        '-w', '--workers', type=int, default=4,
+        help='Number of gunicorn web server workers to fire up')
+    gunicorn_parser.add_argument(
+        '-t', '--timeout', type=int, default=30,
+        help='Specify the timeout (seconds) for the gunicorn web server')
+
+    args = parser.parse_args()
+
+    if args.debug:
+        from superset import app
+
+        app.run(
+            host='0.0.0.0',
+            port=int(args.port),
+            threaded=True,
+            debug=True)
+    else:
+        gunicorn_app_obj = GunicornSupersetApplication(
+            args.address, args.port, args.workers, args.timeout)
+        gunicorn_app_obj.run()
+
 
 if __name__ == "__main__":
-    manager.run()
+    if len(sys.argv) > 1 and sys.argv[1] == 'runserver':
+        # In the runserver case, don't go through the manager so that superset
+        # import is deferred until the app is loaded; this allows for the app to be run via pex
+        # and cleanly forked in the gunicorn case.
+        #
+        # TODO: Refactor cli so that gunicorn can be started without first importing superset;
+        # this will allow us to move the runserver logic back into cli module.
+        run_server()
+    else:
+        from superset.cli import manager
+        manager.run()

superset/cli.py

@@ -21,43 +21,6 @@
 manager.add_command('db', MigrateCommand)
 
 
-@manager.option(
-    '-d', '--debug', action='store_true',
-    help="Start the web server in debug mode")
-@manager.option(
-    '-a', '--address', default=config.get("SUPERSET_WEBSERVER_ADDRESS"),
-    help="Specify the address to which to bind the web server")
-@manager.option(
-    '-p', '--port', default=config.get("SUPERSET_WEBSERVER_PORT"),
-    help="Specify the port on which to run the web server")
-@manager.option(
-    '-w', '--workers', default=config.get("SUPERSET_WORKERS", 2),
-    help="Number of gunicorn web server workers to fire up")
-@manager.option(
-    '-t', '--timeout', default=config.get("SUPERSET_WEBSERVER_TIMEOUT"),
-    help="Specify the timeout (seconds) for the gunicorn web server")
-def runserver(debug, address, port, timeout, workers):
-    """Starts a Superset web server"""
-    debug = debug or config.get("DEBUG")
-    if debug:
-        app.run(
-            host='0.0.0.0',
-            port=int(port),
-            threaded=True,
-            debug=True)
-    else:
-        cmd = (
-            "gunicorn "
-            "-w {workers} "
-            "--timeout {timeout} "
-            "-b {address}:{port} "
-            "--limit-request-line 0 "
-            "--limit-request-field_size 0 "
-            "superset:app").format(**locals())
-        print("Starting server with command: " + cmd)
-        Popen(cmd, shell=True).wait()
-
-
 @manager.command
 def init():
     """Inits the Superset application"""

superset/config.py

@@ -9,6 +9,7 @@
 from __future__ import print_function
 from __future__ import unicode_literals
 
+import imp
 import json
 import os
 
@@ -248,7 +249,14 @@ class CeleryConfig(object):
 # by humans.
 ROBOT_PERMISSION_ROLES = ['Public', 'Gamma', 'Alpha', 'Admin', 'sql_lab']
 
+CONFIG_PATH_ENV_VAR = 'SUPERSET_CONFIG_PATH'
+
 try:
+    if CONFIG_PATH_ENV_VAR in os.environ:
+        # Explicitly import config module that is not in pythonpath; useful
+        # for case where app is being executed via pex.
+        imp.load_source('superset_config', os.environ[CONFIG_PATH_ENV_VAR])
+
     from superset_config import *  # noqa
     print('Loaded your LOCAL configuration')
 except ImportError:

tests/utils_tests.py

@@ -2,6 +2,7 @@
 from superset import utils
 import unittest
 
+from mock import Mock, patch
 
 class UtilsTestCase(unittest.TestCase):
     def test_json_int_dttm_ser(self):
@@ -16,3 +17,8 @@ def test_json_int_dttm_ser(self):
 
         with self.assertRaises(TypeError):
             utils.json_int_dttm_ser("this is not a date")
+
+    @patch('superset.utils.datetime')
+    def test_parse_human_timedelta(self, mock_now):
+        mock_now.return_value = datetime(2016, 12, 1)
+        self.assertEquals(utils.parse_human_timedelta('now'), timedelta(0))