Kerberos authentication support

README.md

@@ -68,11 +68,12 @@ Request Body example:
                 'username': 'username',
                 'password': 'password',
                 'endpoint': 'url',
+                'auth': 'Kerberos',
                 'kernelname': 'pysparkkernel'
         }
         ```
 
-*Note that the kernelname parameter is optional and defaults to the one specified on the config file or pysparkkernel if not on the config file.*
+*Note that the auth can be either None, Basic_Access or Kerberos based on the authentication enabled in livy. The kernelname parameter is optional and defaults to the one specified on the config file or pysparkkernel if not on the config file.*
 Returns `200` if successful; `400` if body is not JSON string or key is not found; `500` if error is encountered changing clusters.
 
 Reply Body example:
@@ -107,6 +108,18 @@ However, there are some important limitations to note:
 2. Since all code is run on a remote driver through Livy, all structured data must be serialized to JSON and parsed by the Sparkmagic library so that it can be manipulated and visualized on the client side.
 In practice this means that you must use Python for client-side data manipulation in `%%local` mode.
 
+## Authentication Methods
+
+Sparkmagic supports:
+
+* No auth
+* Basic authentication
+* Kerberos
+
+Kerberos support is implemented via the [requests-kerberos](https://github.com/requests/requests-kerberos) package. Sparkmagic expects a kerberos ticket to be available in the system. Requests-kerberos will pick up the kerberos ticket from a cache file. For the ticket to be available, the user needs to have run [kinit](https://web.mit.edu/kerberos/krb5-1.12/doc/user/user_commands/kinit.html) to create the kerberos ticket.
+
+Currently, sparkmagic does not support passing a kerberos principal/token, but we welcome pull requests.
+
 ## Contributing
 
 We welcome contributions from everyone. 

sparkmagic/example_config.json

@@ -2,13 +2,15 @@
   "kernel_python_credentials" : {
     "username": "",
     "password": "",
-    "url": "http://localhost:8998"
+    "url": "http://localhost:8998",
+    "auth": "None"
   },
 
   "kernel_scala_credentials" : {
     "username": "",
     "password": "",
-    "url": "http://localhost:8998"
+    "url": "http://localhost:8998",
+    "auth": "None"
   },
 
   "logging_config": {

sparkmagic/requirements.txt

@@ -9,4 +9,5 @@ requests
 ipykernel>=4.2.2,<5
 ipywidgets>5.0.0,<7.0
 notebook>=4.2,<5.0
-tornado>=4
+tornado>=4
+requests_kerberos>=0.8.0

sparkmagic/setup.py

@@ -88,6 +88,7 @@ def version(path):
           'ipykernel>=4.2.2,<5',		
           'ipywidgets>5.0.0,<7.0',
           'notebook>=4.2,<5.0',
-          'tornado>=4'
+          'tornado>=4',
+          'requests_kerberos>=0.8.0'
       ])
 

sparkmagic/sparkmagic/controllerwidget/addendpointwidget.py

@@ -2,6 +2,7 @@
 # Distributed under the terms of the Modified BSD License.
 from sparkmagic.controllerwidget.abstractmenuwidget import AbstractMenuWidget
 from sparkmagic.livyclientlib.endpoint import Endpoint
+import sparkmagic.utils.constants as constants
 
 
 class AddEndpointWidget(AbstractMenuWidget):
@@ -32,24 +33,42 @@ def __init__(self, spark_controller, ipywidget_factory, ipython_display, endpoin
             value='password',
             width=widget_width
         )
+        self.auth = self.ipywidget_factory.get_dropdown(
+            options={constants.AUTH_KERBEROS: constants.AUTH_KERBEROS, constants.AUTH_BASIC: constants.AUTH_BASIC,
+                     constants.NO_AUTH: constants.NO_AUTH},
+            description=u"Auth type:"
+        )
 
         # Submit widget
         self.submit_widget = self.ipywidget_factory.get_submit_button(
             description='Add endpoint'
         )
 
+        self.auth.on_trait_change(self._show_correct_endpoint_fields)
+
         self.children = [self.ipywidget_factory.get_html(value="<br/>", width=widget_width),
-                         self.address_widget, self.user_widget, self.password_widget,
+                         self.address_widget, self.auth, self.user_widget, self.password_widget,
                          self.ipywidget_factory.get_html(value="<br/>", width=widget_width), self.submit_widget]
 
         for child in self.children:
             child.parent_widget = self
 
+        self._show_correct_endpoint_fields()
+
     def run(self):
-        endpoint = Endpoint(self.address_widget.value, self.user_widget.value, self.password_widget.value)
+        endpoint = Endpoint(self.address_widget.value, self.auth.value, self.user_widget.value, self.password_widget.value)
         self.endpoints[self.address_widget.value] = endpoint
         self.ipython_display.writeln("Added endpoint {}".format(self.address_widget.value))
 
         # We need to call the refresh method because drop down in Tab 2 for endpoints wouldn't refresh with the new
         # value otherwise.
         self.refresh_method()
+
+    def _show_correct_endpoint_fields(self):
+        if self.auth.value == constants.NO_AUTH or self.auth.value == constants.AUTH_KERBEROS:
+            self.user_widget.layout.display = 'none'
+            self.password_widget.layout.display = 'none'
+        else:
+            self.user_widget.layout.display = 'flex'
+            self.password_widget.layout.display = 'flex'
+

sparkmagic/sparkmagic/kernels/kernelmagics.py

@@ -12,6 +12,7 @@
 from hdijupyterutils.utils import generate_uuid
 
 import sparkmagic.utils.configuration as conf
+from sparkmagic.utils import constants
 from sparkmagic.utils.utils import get_livy_kind, parse_argstring_or_throw
 from sparkmagic.utils.sparkevents import SparkEvents
 from sparkmagic.utils.constants import LANGS_SUPPORTED
@@ -349,23 +350,25 @@ def _do_not_call_change_language(self, line, cell="", local_ns=None):
     @argument("-u", "--username", type=str, help="Username to use.")
     @argument("-p", "--password", type=str, help="Password to use.")
     @argument("-s", "--server", type=str, help="Url of server to use.")
+    @argument("-t", "--auth", type=str, help="Auth type for authentication")
     @_event
     def _do_not_call_change_endpoint(self, line, cell="", local_ns=None):
         args = parse_argstring_or_throw(self._do_not_call_change_endpoint, line)
         username = args.username
         password = args.password
         server = args.server
+        auth = args.auth
 
         if self.session_started:
             error = u"Cannot change the endpoint if a session has been started."
             raise BadUserDataException(error)
 
-        self.endpoint = Endpoint(server, username, password)
+        self.endpoint = Endpoint(server, auth, username, password)
 
     def refresh_configuration(self):
         credentials = getattr(conf, 'base64_kernel_' + self.language + '_credentials')()
-        (username, password, url) = (credentials['username'], credentials['password'], credentials['url'])
-        self.endpoint = Endpoint(url, username, password)
+        (username, password, auth, url) = (credentials['username'], credentials['password'], credentials['auth'], credentials['url'])
+        self.endpoint = Endpoint(url, auth, username, password)
 
     def get_session_settings(self, line, force):
         line = line.strip()

sparkmagic/sparkmagic/livyclientlib/endpoint.py

@@ -2,13 +2,13 @@
 
 
 class Endpoint(object):
-    def __init__(self, url, username="", password=""):
+    def __init__(self, url, auth, username="", password=""):
         if not url:
             raise BadUserDataException(u"URL must not be empty")
         self.url = url.rstrip(u"/")
         self.username = username
         self.password = password
-        self.authenticate = False if username == '' and password == '' else True
+        self.auth = auth
 
     def __eq__(self, other):
         if type(other) is not Endpoint:

sparkmagic/sparkmagic/livyclientlib/reliablehttpclient.py

@@ -3,11 +3,14 @@
 import json
 from time import sleep
 import requests
+from requests_kerberos import HTTPKerberosAuth, REQUIRED
 
 import sparkmagic.utils.configuration as conf
 from sparkmagic.utils.sparklogger import SparkLog
 from sparkmagic.utils.constants import MAGICS_LOGGER_NAME
+import sparkmagic.utils.constants as constants
 from sparkmagic.livyclientlib.exceptions import HttpClientException
+from sparkmagic.livyclientlib.exceptions import BadUserConfigurationException
 
 
 class ReliableHttpClient(object):
@@ -17,6 +20,13 @@ def __init__(self, endpoint, headers, retry_policy):
         self._endpoint = endpoint
         self._headers = headers
         self._retry_policy = retry_policy
+        if self._endpoint.auth == constants.AUTH_KERBEROS:
+            self._auth = HTTPKerberosAuth(mutual_authentication=REQUIRED)
+        elif self._endpoint.auth == constants.AUTH_BASIC:
+            self._auth = (self._endpoint.username, self._endpoint.password)
+        elif self._endpoint.auth != constants.NO_AUTH:
+            raise BadUserConfigurationException(u"Unsupported auth %s" %self._endpoint.auth)
+
         self.logger = SparkLog(u"ReliableHttpClient")
 
         self.verify_ssl = not conf.ignore_ssl_errors()
@@ -46,17 +56,16 @@ def _send_request(self, relative_url, accepted_status_codes, function, data=None
     def _send_request_helper(self, url, accepted_status_codes, function, data, retry_count):
         while True:
             try:
-                if not self._endpoint.authenticate:
+                if self._endpoint.auth == constants.NO_AUTH:
                     if data is None:
                         r = function(url, headers=self._headers, verify=self.verify_ssl)
                     else:
                         r = function(url, headers=self._headers, data=json.dumps(data), verify=self.verify_ssl)
                 else:
                     if data is None:
-                        r = function(url, headers=self._headers, auth=(self._endpoint.username, self._endpoint.password),
-                                     verify=self.verify_ssl)
+                        r = function(url, headers=self._headers, auth=self._auth, verify=self.verify_ssl)
                     else:
-                        r = function(url, headers=self._headers, auth=(self._endpoint.username, self._endpoint.password),
+                        r = function(url, headers=self._headers, auth=self._auth,
                                      data=json.dumps(data), verify=self.verify_ssl)
             except requests.exceptions.RequestException as e:
                 error = True

sparkmagic/sparkmagic/magics/remotesparkmagics.py

@@ -54,6 +54,7 @@ def manage_spark(self, line, local_ns=None):
     @argument("-u", "--url", type=str, default=None, help="URL for Livy endpoint")
     @argument("-a", "--user", type=str, default="", help="Username for HTTP access to Livy endpoint")
     @argument("-p", "--password", type=str, default="", help="Password for HTTP access to Livy endpoint")
+    @argument("-t", "--auth", type=str, help="Auth type for HTTP access to Livy endpoint. [Kerberos, None, Basic Auth]")
     @argument("-l", "--language", type=str, default=None,
               help="Language for Livy session; one of {}".format(', '.join([LANG_PYTHON, LANG_SCALA, LANG_R])))
     @argument("command", type=str, default=[""], nargs="*", help="Commands to execute.")
@@ -78,7 +79,7 @@ def spark(self, line, cell="", local_ns=None):
            add
                Add a Livy session given a session name (-s), language (-l), and endpoint credentials.
                The -k argument, if present, will skip adding this session if it already exists.
-               e.g. `%spark add -s test -l python -u https://sparkcluster.net/livy -a u -p -k`
+               e.g. `%spark add -s test -l python -u https://sparkcluster.net/livy -t Kerberos -a u -p -k`
            config
                Override the livy session properties sent to Livy on session creation. All session creations will
                contain these config settings from then on.
@@ -112,7 +113,7 @@ def spark(self, line, cell="", local_ns=None):
         # info
         if subcommand == "info":
             if args.url is not None:
-                endpoint = Endpoint(args.url, args.user, args.password)
+                endpoint = Endpoint(args.url, args.auth, args.user, args.password)
                 info_sessions = self.spark_controller.get_all_sessions_endpoint_info(endpoint)
                 self._print_endpoint_info(info_sessions)
             else:
@@ -128,7 +129,7 @@ def spark(self, line, cell="", local_ns=None):
 
             name = args.session
             language = args.language
-            endpoint = Endpoint(args.url, args.user, args.password)
+            endpoint = Endpoint(args.url, args.auth, args.user, args.password)
             skip = args.skip
 
             properties = conf.get_session_properties(language)
@@ -142,15 +143,15 @@ def spark(self, line, cell="", local_ns=None):
                 if args.id is None:
                     self.ipython_display.send_error("Must provide --id or -i option to delete session at endpoint from URL")
                     return
-                endpoint = Endpoint(args.url, args.user, args.password)
+                endpoint = Endpoint(args.url, args.auth, args.user, args.password)
                 session_id = args.id
                 self.spark_controller.delete_session_by_id(endpoint, session_id)
             else:
                 self.ipython_display.send_error("Subcommand 'delete' requires a session name or a URL and session ID")
         # cleanup
         elif subcommand == "cleanup":
             if args.url is not None:
-                endpoint = Endpoint(args.url, args.user, args.password)
+                endpoint = Endpoint(args.url, args.auth, args.user, args.password)
                 self.spark_controller.cleanup_endpoint(endpoint)
             else:
                 self.spark_controller.cleanup()

sparkmagic/sparkmagic/serverextension/handlers.py

@@ -8,6 +8,7 @@
 
 from sparkmagic.kernels.kernelmagics import KernelMagics
 import sparkmagic.utils.configuration as conf
+from sparkmagic.utils import constants
 from sparkmagic.utils.sparkevents import SparkEvents
 from sparkmagic.utils.sparklogger import SparkLog
 
@@ -38,6 +39,12 @@ def post(self):
             username = self._get_argument_or_raise(data, 'username')
             password = self._get_argument_or_raise(data, 'password')
             endpoint = self._get_argument_or_raise(data, 'endpoint')
+            auth = self._get_argument_if_exists(data, 'auth')
+            if auth is None:
+                if username == '' and password == '':
+                    auth = constants.NO_AUTH
+                else:
+                    auth = constants.AUTH_BASIC
         except MissingArgumentError as e:
             self.set_status(400)
             self.finish(str(e))
@@ -52,7 +59,7 @@ def post(self):
 
         # Execute code
         client = kernel_manager.client()
-        code = '%{} -s {} -u {} -p {}'.format(KernelMagics._do_not_call_change_endpoint.__name__, endpoint, username, password)
+        code = '%{} -s {} -u {} -p {} -t {}'.format(KernelMagics._do_not_call_change_endpoint.__name__, endpoint, username, password, auth)
         response_id = client.execute(code, silent=False, store_history=False)
         msg = client.get_shell_msg(response_id)
 

sparkmagic/sparkmagic/tests/test_configuration.py

@@ -4,6 +4,7 @@
 
 import sparkmagic.utils.configuration as conf
 from sparkmagic.livyclientlib.exceptions import BadUserConfigurationException
+from sparkmagic.utils.constants import AUTH_BASIC, NO_AUTH
 
 
 def _setup():
@@ -12,19 +13,35 @@ def _setup():
 
 @with_setup(_setup)
 def test_configuration_override_base64_password():
-    kpc = { 'username': 'U', 'password': 'P', 'base64_password': 'cGFzc3dvcmQ=', 'url': 'L' }
+    kpc = { 'username': 'U', 'password': 'P', 'base64_password': 'cGFzc3dvcmQ=', 'url': 'L', "auth": AUTH_BASIC }
     overrides = { conf.kernel_python_credentials.__name__: kpc }
     conf.override_all(overrides)
     conf.override(conf.status_sleep_seconds.__name__, 1)
     assert_equals(conf.d, { conf.kernel_python_credentials.__name__: kpc,
                                      conf.status_sleep_seconds.__name__: 1 })
     assert_equals(conf.status_sleep_seconds(), 1)
-    assert_equals(conf.base64_kernel_python_credentials(), { 'username': 'U', 'password': 'password', 'url': 'L' })
+    assert_equals(conf.base64_kernel_python_credentials(), { 'username': 'U', 'password': 'password', 'url': 'L', 'auth': AUTH_BASIC })
+
+
+@with_setup(_setup)
+def test_configuration_auth_missing_basic_auth():
+    kpc = { 'username': 'U', 'password': 'P', 'url': 'L'}
+    overrides = { conf.kernel_python_credentials.__name__: kpc }
+    conf.override_all(overrides)
+    assert_equals(conf.base64_kernel_python_credentials(), { 'username': 'U', 'password': 'P', 'url': 'L', 'auth': AUTH_BASIC })
+
+
+@with_setup(_setup)
+def test_configuration_auth_missing_no_auth():
+    kpc = { 'username': '', 'password': '', 'url': 'L'}
+    overrides = { conf.kernel_python_credentials.__name__: kpc }
+    conf.override_all(overrides)
+    assert_equals(conf.base64_kernel_python_credentials(), { 'username': '', 'password': '', 'url': 'L', 'auth': NO_AUTH })
 
 
 @with_setup(_setup)
 def test_configuration_override_fallback_to_password():
-    kpc = { 'username': 'U', 'password': 'P', 'url': 'L' }
+    kpc = { 'username': 'U', 'password': 'P', 'url': 'L', 'auth': NO_AUTH }
     overrides = { conf.kernel_python_credentials.__name__: kpc }
     conf.override_all(overrides)
     conf.override(conf.status_sleep_seconds.__name__, 1)
@@ -36,14 +53,14 @@ def test_configuration_override_fallback_to_password():
 
 @with_setup(_setup)
 def test_configuration_override_work_with_empty_password():
-    kpc = { 'username': 'U', 'base64_password': '', 'password': '', 'url': '' }
+    kpc = { 'username': 'U', 'base64_password': '', 'password': '', 'url': '', 'auth': AUTH_BASIC }
     overrides = { conf.kernel_python_credentials.__name__: kpc }
     conf.override_all(overrides)
     conf.override(conf.status_sleep_seconds.__name__, 1)
     assert_equals(conf.d, { conf.kernel_python_credentials.__name__: kpc,
                                      conf.status_sleep_seconds.__name__: 1 })
     assert_equals(conf.status_sleep_seconds(), 1)
-    assert_equals(conf.base64_kernel_python_credentials(),  { 'username': 'U', 'password': '', 'url': '' })
+    assert_equals(conf.base64_kernel_python_credentials(),  { 'username': 'U', 'password': '', 'url': '', 'auth': AUTH_BASIC })
 
 
 @raises(BadUserConfigurationException)
@@ -58,6 +75,6 @@ def test_configuration_raise_error_for_bad_base64_password():
 
 @with_setup(_setup)
 def test_share_config_between_pyspark_and_pyspark3():
-    kpc = { 'username': 'U', 'password': 'P', 'base64_password': 'cGFzc3dvcmQ=', 'url': 'L' }
+    kpc = { 'username': 'U', 'password': 'P', 'base64_password': 'cGFzc3dvcmQ=', 'url': 'L', 'auth': AUTH_BASIC }
     overrides = { conf.kernel_python_credentials.__name__: kpc }
     assert_equals(conf.base64_kernel_python3_credentials(), conf.base64_kernel_python_credentials())