Updated as per comments.

sparkmagic/example_config.json

@@ -2,13 +2,15 @@
   "kernel_python_credentials" : {
     "username": "",
     "password": "",
-    "url": "http://localhost:8998"
+    "url": "http://localhost:8998",
+    "auth": "None"
   },
 
   "kernel_scala_credentials" : {
     "username": "",
     "password": "",
-    "url": "http://localhost:8998"
+    "url": "http://localhost:8998",
+    "auth": "None"
   },
 
   "logging_config": {

sparkmagic/requirements.txt

@@ -9,4 +9,5 @@ requests
 ipykernel>=4.2.2,<5
 ipywidgets>5.0.0,<7.0
 notebook>=4.2,<5.0
-tornado>=4
+tornado>=4
+requests_kerberos>=0.8.0

sparkmagic/setup.py

@@ -89,6 +89,6 @@ def version(path):
           'ipywidgets>5.0.0,<7.0',
           'notebook>=4.2,<5.0',
           'tornado>=4',
-          'requests_kerberos'
+          'requests_kerberos>=0.8.0'
       ])
 

sparkmagic/sparkmagic/controllerwidget/addendpointwidget.py

@@ -33,7 +33,7 @@ def __init__(self, spark_controller, ipywidget_factory, ipython_display, endpoin
             value='password',
             width=widget_width
         )
-        self.auth_type = self.ipywidget_factory.get_dropdown(
+        self.auth = self.ipywidget_factory.get_dropdown(
             options={constants.AUTH_KERBEROS: constants.AUTH_KERBEROS, constants.AUTH_BASIC: constants.AUTH_BASIC,
                      constants.NO_AUTH: constants.NO_AUTH},
             description=u"Auth type:"
@@ -44,26 +44,28 @@ def __init__(self, spark_controller, ipywidget_factory, ipython_display, endpoin
             description='Add endpoint'
         )
 
-        self.auth_type.on_trait_change(self.update_endpoint)
+        self.auth.on_trait_change(self._show_correct_endpoint_fields)
 
         self.children = [self.ipywidget_factory.get_html(value="<br/>", width=widget_width),
-                         self.address_widget, self.auth_type, self.user_widget, self.password_widget,
+                         self.address_widget, self.auth, self.user_widget, self.password_widget,
                          self.ipywidget_factory.get_html(value="<br/>", width=widget_width), self.submit_widget]
 
         for child in self.children:
             child.parent_widget = self
 
+        self._show_correct_endpoint_fields()
+
     def run(self):
-        endpoint = Endpoint(self.address_widget.value, self.auth_type.value, self.user_widget.value, self.password_widget.value)
+        endpoint = Endpoint(self.address_widget.value, self.auth.value, self.user_widget.value, self.password_widget.value)
         self.endpoints[self.address_widget.value] = endpoint
         self.ipython_display.writeln("Added endpoint {}".format(self.address_widget.value))
 
         # We need to call the refresh method because drop down in Tab 2 for endpoints wouldn't refresh with the new
         # value otherwise.
         self.refresh_method()
 
-    def update_endpoint(self):
-        if self.auth_type.value == constants.NO_AUTH or self.auth_type.value == constants.AUTH_KERBEROS:
+    def _show_correct_endpoint_fields(self):
+        if self.auth.value == constants.NO_AUTH or self.auth.value == constants.AUTH_KERBEROS:
             self.user_widget.layout.display = 'none'
             self.password_widget.layout.display = 'none'
         else:

sparkmagic/sparkmagic/kernels/kernelmagics.py

@@ -12,6 +12,7 @@
 from hdijupyterutils.utils import generate_uuid
 
 import sparkmagic.utils.configuration as conf
+from sparkmagic.utils import constants
 from sparkmagic.utils.utils import get_livy_kind, parse_argstring_or_throw
 from sparkmagic.utils.sparkevents import SparkEvents
 from sparkmagic.utils.constants import LANGS_SUPPORTED
@@ -349,25 +350,25 @@ def _do_not_call_change_language(self, line, cell="", local_ns=None):
     @argument("-u", "--username", type=str, help="Username to use.")
     @argument("-p", "--password", type=str, help="Password to use.")
     @argument("-s", "--server", type=str, help="Url of server to use.")
-    @argument("-t", "--auth_type", type=str, help="Auth type for authentication")
+    @argument("-t", "--auth", type=str, help="Auth type for authentication")
     @_event
     def _do_not_call_change_endpoint(self, line, cell="", local_ns=None):
         args = parse_argstring_or_throw(self._do_not_call_change_endpoint, line)
         username = args.username
         password = args.password
         server = args.server
-        auth_type = args.auth_type
+        auth = args.auth
 
         if self.session_started:
             error = u"Cannot change the endpoint if a session has been started."
             raise BadUserDataException(error)
 
-        self.endpoint = Endpoint(server, auth_type, username, password)
+        self.endpoint = Endpoint(server, auth, username, password)
 
     def refresh_configuration(self):
         credentials = getattr(conf, 'base64_kernel_' + self.language + '_credentials')()
-        (username, password, url, auth_type) = (credentials['username'], credentials['password'], credentials['url'], credentials['auth_type'])
-        self.endpoint = Endpoint(url, auth_type, username, password)
+        (username, password, auth, url) = (credentials['username'], credentials['password'], credentials['auth'], credentials['url'])
+        self.endpoint = Endpoint(url, auth, username, password)
 
     def get_session_settings(self, line, force):
         line = line.strip()

sparkmagic/sparkmagic/livyclientlib/endpoint.py

@@ -1,15 +1,14 @@
 from .exceptions import BadUserDataException
-import sparkmagic.utils.configuration as conf
 
 
 class Endpoint(object):
-    def __init__(self, url, auth_type=conf.default_livy_endpoint_auth_type(), username="", password=""):
+    def __init__(self, url, auth, username="", password=""):
         if not url:
             raise BadUserDataException(u"URL must not be empty")
         self.url = url.rstrip(u"/")
         self.username = username
         self.password = password
-        self.auth_type = auth_type
+        self.auth = auth
 
     def __eq__(self, other):
         if type(other) is not Endpoint:

sparkmagic/sparkmagic/livyclientlib/reliablehttpclient.py

@@ -4,13 +4,13 @@
 from time import sleep
 import requests
 from requests_kerberos import HTTPKerberosAuth, REQUIRED
-import subprocess
 
 import sparkmagic.utils.configuration as conf
 from sparkmagic.utils.sparklogger import SparkLog
 from sparkmagic.utils.constants import MAGICS_LOGGER_NAME
 import sparkmagic.utils.constants as constants
 from sparkmagic.livyclientlib.exceptions import HttpClientException
+from sparkmagic.livyclientlib.exceptions import BadUserConfigurationException
 
 
 class ReliableHttpClient(object):
@@ -20,10 +20,12 @@ def __init__(self, endpoint, headers, retry_policy):
         self._endpoint = endpoint
         self._headers = headers
         self._retry_policy = retry_policy
-        if self._endpoint.auth_type == constants.AUTH_KERBEROS:
+        if self._endpoint.auth == constants.AUTH_KERBEROS:
             self._auth = HTTPKerberosAuth(mutual_authentication=REQUIRED)
-        elif self._endpoint.auth_type == constants.AUTH_BASIC:
+        elif self._endpoint.auth == constants.AUTH_BASIC:
             self._auth = (self._endpoint.username, self._endpoint.password)
+        elif self._endpoint.auth != constants.NO_AUTH:
+            raise BadUserConfigurationException(u"Unsupported auth %s" %self._endpoint.auth)
 
         self.logger = SparkLog(u"ReliableHttpClient")
 
@@ -54,7 +56,7 @@ def _send_request(self, relative_url, accepted_status_codes, function, data=None
     def _send_request_helper(self, url, accepted_status_codes, function, data, retry_count):
         while True:
             try:
-                if self._endpoint.auth_type is constants.NO_AUTH:
+                if self._endpoint.auth == constants.NO_AUTH:
                     if data is None:
                         r = function(url, headers=self._headers, verify=self.verify_ssl)
                     else:

sparkmagic/sparkmagic/magics/remotesparkmagics.py

@@ -79,7 +79,7 @@ def spark(self, line, cell="", local_ns=None):
            add
                Add a Livy session given a session name (-s), language (-l), and endpoint credentials.
                The -k argument, if present, will skip adding this session if it already exists.
-               e.g. `%spark add -s test -l python -u https://sparkcluster.net/livy -a u -p -k`
+               e.g. `%spark add -s test -l python -u https://sparkcluster.net/livy -t Kerberos -a u -p -k`
            config
                Override the livy session properties sent to Livy on session creation. All session creations will
                contain these config settings from then on.

sparkmagic/sparkmagic/serverextension/handlers.py

@@ -8,6 +8,7 @@
 
 from sparkmagic.kernels.kernelmagics import KernelMagics
 import sparkmagic.utils.configuration as conf
+from sparkmagic.utils import constants
 from sparkmagic.utils.sparkevents import SparkEvents
 from sparkmagic.utils.sparklogger import SparkLog
 
@@ -38,6 +39,12 @@ def post(self):
             username = self._get_argument_or_raise(data, 'username')
             password = self._get_argument_or_raise(data, 'password')
             endpoint = self._get_argument_or_raise(data, 'endpoint')
+            auth = self._get_argument_if_exists(data, 'auth')
+            if auth is None:
+                if username == '' and password == '':
+                    auth = constants.NO_AUTH
+                else:
+                    auth = constants.AUTH_BASIC
         except MissingArgumentError as e:
             self.set_status(400)
             self.finish(str(e))
@@ -52,7 +59,7 @@ def post(self):
 
         # Execute code
         client = kernel_manager.client()
-        code = '%{} -s {} -u {} -p {}'.format(KernelMagics._do_not_call_change_endpoint.__name__, endpoint, username, password)
+        code = '%{} -s {} -u {} -p {} -t {}'.format(KernelMagics._do_not_call_change_endpoint.__name__, endpoint, username, password, auth)
         response_id = client.execute(code, silent=False, store_history=False)
         msg = client.get_shell_msg(response_id)
 

sparkmagic/sparkmagic/tests/test_configuration.py

@@ -4,7 +4,7 @@
 
 import sparkmagic.utils.configuration as conf
 from sparkmagic.livyclientlib.exceptions import BadUserConfigurationException
-from sparkmagic.utils.constants import AUTH_BASIC
+from sparkmagic.utils.constants import AUTH_BASIC, NO_AUTH
 
 
 def _setup():
@@ -13,19 +13,43 @@ def _setup():
 
 @with_setup(_setup)
 def test_configuration_override_base64_password():
-    kpc = { 'username': 'U', 'password': 'P', 'base64_password': 'cGFzc3dvcmQ=', 'url': 'L', "auth_type": AUTH_BASIC }
+    kpc = { 'username': 'U', 'password': 'P', 'base64_password': 'cGFzc3dvcmQ=', 'url': 'L', "auth": AUTH_BASIC }
     overrides = { conf.kernel_python_credentials.__name__: kpc }
     conf.override_all(overrides)
     conf.override(conf.status_sleep_seconds.__name__, 1)
     assert_equals(conf.d, { conf.kernel_python_credentials.__name__: kpc,
                                      conf.status_sleep_seconds.__name__: 1 })
     assert_equals(conf.status_sleep_seconds(), 1)
-    assert_equals(conf.base64_kernel_python_credentials(), { 'username': 'U', 'password': 'password', 'url': 'L', 'auth_type': AUTH_BASIC })
+    assert_equals(conf.base64_kernel_python_credentials(), { 'username': 'U', 'password': 'password', 'url': 'L', 'auth': AUTH_BASIC })
+
+
+@with_setup(_setup)
+def test_configuration_auth_missing_basic_auth():
+    kpc = { 'username': 'U', 'password': 'P', 'url': 'L'}
+    overrides = { conf.kernel_python_credentials.__name__: kpc }
+    conf.override_all(overrides)
+    conf.override(conf.status_sleep_seconds.__name__, 1)
+    assert_equals(conf.d, { conf.kernel_python_credentials.__name__: kpc,
+                                     conf.status_sleep_seconds.__name__: 1 })
+    assert_equals(conf.status_sleep_seconds(), 1)
+    assert_equals(conf.base64_kernel_python_credentials(), { 'username': 'U', 'password': 'P', 'url': 'L', 'auth': AUTH_BASIC })
+
+
+@with_setup(_setup)
+def test_configuration_auth_missing_no_auth():
+    kpc = { 'username': '', 'password': '', 'url': 'L'}
+    overrides = { conf.kernel_python_credentials.__name__: kpc }
+    conf.override_all(overrides)
+    conf.override(conf.status_sleep_seconds.__name__, 1)
+    assert_equals(conf.d, { conf.kernel_python_credentials.__name__: kpc,
+                                     conf.status_sleep_seconds.__name__: 1 })
+    assert_equals(conf.status_sleep_seconds(), 1)
+    assert_equals(conf.base64_kernel_python_credentials(), { 'username': '', 'password': '', 'url': 'L', 'auth': NO_AUTH })
 
 
 @with_setup(_setup)
 def test_configuration_override_fallback_to_password():
-    kpc = { 'username': 'U', 'password': 'P', 'url': 'L', 'auth_type': None }
+    kpc = { 'username': 'U', 'password': 'P', 'url': 'L', 'auth': NO_AUTH }
     overrides = { conf.kernel_python_credentials.__name__: kpc }
     conf.override_all(overrides)
     conf.override(conf.status_sleep_seconds.__name__, 1)
@@ -37,14 +61,14 @@ def test_configuration_override_fallback_to_password():
 
 @with_setup(_setup)
 def test_configuration_override_work_with_empty_password():
-    kpc = { 'username': 'U', 'base64_password': '', 'password': '', 'url': '', 'auth_type': AUTH_BASIC }
+    kpc = { 'username': 'U', 'base64_password': '', 'password': '', 'url': '', 'auth': AUTH_BASIC }
     overrides = { conf.kernel_python_credentials.__name__: kpc }
     conf.override_all(overrides)
     conf.override(conf.status_sleep_seconds.__name__, 1)
     assert_equals(conf.d, { conf.kernel_python_credentials.__name__: kpc,
                                      conf.status_sleep_seconds.__name__: 1 })
     assert_equals(conf.status_sleep_seconds(), 1)
-    assert_equals(conf.base64_kernel_python_credentials(),  { 'username': 'U', 'password': '', 'url': '', 'auth_type': AUTH_BASIC })
+    assert_equals(conf.base64_kernel_python_credentials(),  { 'username': 'U', 'password': '', 'url': '', 'auth': AUTH_BASIC })
 
 
 @raises(BadUserConfigurationException)
@@ -59,6 +83,6 @@ def test_configuration_raise_error_for_bad_base64_password():
 
 @with_setup(_setup)
 def test_share_config_between_pyspark_and_pyspark3():
-    kpc = { 'username': 'U', 'password': 'P', 'base64_password': 'cGFzc3dvcmQ=', 'url': 'L' }
+    kpc = { 'username': 'U', 'password': 'P', 'base64_password': 'cGFzc3dvcmQ=', 'url': 'L', 'auth': AUTH_BASIC }
     overrides = { conf.kernel_python_credentials.__name__: kpc }
     assert_equals(conf.base64_kernel_python3_credentials(), conf.base64_kernel_python_credentials())

sparkmagic/sparkmagic/tests/test_handlers.py

@@ -9,6 +9,7 @@
 from sparkmagic.serverextension.handlers import ReconnectHandler
 from sparkmagic.kernels.kernelmagics import KernelMagics
 import sparkmagic.utils.configuration as conf
+from sparkmagic.utils import constants
 
 
 class SimpleObject(object):
@@ -30,6 +31,7 @@ class TestSparkMagicHandler(AsyncTestCase):
     username = 'username'
     password = 'password'
     endpoint = 'http://endpoint.com'
+    auth = constants.AUTH_BASIC
     response_id = '0'
     good_msg = dict(content=dict(status='ok'))
     bad_msg = dict(content=dict(status='error', ename='SyntaxError', evalue='oh no!'))
@@ -62,7 +64,7 @@ def setUp(self):
 
         # Mock request
         self.request = MagicMock()
-        self.request.body = json.dumps({"path": self.path, "username": self.username, "password": self.password, "endpoint": self.endpoint})
+        self.request.body = json.dumps({"path": self.path, "username": self.username, "password": self.password, "endpoint": self.endpoint, "auth": self.auth})
 
         # Create mocked reconnect_handler
         ReconnectHandler.__bases__ = (SimpleObject,)
@@ -124,7 +126,7 @@ def test_post_existing_kernel(self, _get_kernel_manager):
         res = yield self.reconnect_handler.post()
         assert_equals(res, None)
 
-        code = '%{} -s {} -u {} -p {}'.format(KernelMagics._do_not_call_change_endpoint.__name__, self.endpoint, self.username, self.password)
+        code = '%{} -s {} -u {} -p {} -t {}'.format(KernelMagics._do_not_call_change_endpoint.__name__, self.endpoint, self.username, self.password, self.auth)
         self.client.execute.assert_called_once_with(code, silent=False, store_history=False)
         self.reconnect_handler.set_status.assert_called_once_with(200)
         self.reconnect_handler.finish.assert_called_once_with('{"error": null, "success": true}')
@@ -141,7 +143,7 @@ def test_post_existing_kernel_failed(self, _get_kernel_manager):
         res = yield self.reconnect_handler.post()
         assert_equals(res, None)
 
-        code = '%{} -s {} -u {} -p {}'.format(KernelMagics._do_not_call_change_endpoint.__name__, self.endpoint, self.username, self.password)
+        code = '%{} -s {} -u {} -p {} -t {}'.format(KernelMagics._do_not_call_change_endpoint.__name__, self.endpoint, self.username, self.password, self.auth)
         self.client.execute.assert_called_once_with(code, silent=False, store_history=False)
         self.reconnect_handler.set_status.assert_called_once_with(500)
         self.reconnect_handler.finish.assert_called_once_with('{"error": "SyntaxError:\\noh no!", "success": false}')

sparkmagic/sparkmagic/tests/test_kernel_magics.py

@@ -25,10 +25,10 @@ def __init__(self, shell, data=None, spark_events=None):
         super(TestKernelMagics, self).__init__(shell, spark_events=spark_events)
 
         self.language = constants.LANG_PYTHON
-        self.endpoint = Endpoint("url")
+        self.endpoint = Endpoint("url", NO_AUTH)
 
     def refresh_configuration(self):
-        self.endpoint = Endpoint("new_url")
+        self.endpoint = Endpoint("new_url", NO_AUTH)
 
 
 def _setup():
@@ -132,7 +132,7 @@ def test_change_language():
     magic._do_not_call_change_language(line)
 
     assert_equals(constants.LANG_SCALA, magic.language)
-    assert_equals(Endpoint("new_url"), magic.endpoint)
+    assert_equals(Endpoint("new_url", NO_AUTH), magic.endpoint)
 
 
 @with_setup(_setup, _teardown)
@@ -145,7 +145,7 @@ def test_change_language_session_started():
 
     assert_equals(ipython_display.send_error.call_count, 1)
     assert_equals(constants.LANG_PYTHON, magic.language)
-    assert_equals(Endpoint("url"), magic.endpoint)
+    assert_equals(Endpoint("url", NO_AUTH), magic.endpoint)
 
 
 @with_setup(_setup, _teardown)
@@ -157,7 +157,7 @@ def test_change_language_not_valid():
 
     assert_equals(ipython_display.send_error.call_count, 1)
     assert_equals(constants.LANG_PYTHON, magic.language)
-    assert_equals(Endpoint("url"), magic.endpoint)
+    assert_equals(Endpoint("url", NO_AUTH), magic.endpoint)
 
 
 @with_setup(_setup, _teardown)