throttles the number of simultaneous kerberos library invocations dur…

…ing authentication

waiter/config-full.edn

@@ -167,7 +167,9 @@
 
                         ;; :kind :kerberos enables authentication using the Kerberos protocol
                         ;:kind :kerberos
-                        :kerberos {:factory-fn waiter.auth.kerberos/kerberos-authenticator}}
+                        :kerberos {:factory-fn waiter.auth.kerberos/kerberos-authenticator
+                                   ;; the number of concurrent Kerberos library (GSSLibStub) invocations allowed
+                                   :concurrency-level 20}}
 
  ;; Waiter supports the run-as-requester feature to launch a service as the requesting user.
  ;; Triggering this feature without passing explicit headers requires providing an explicit consent and storing this in a cookie.

waiter/src/waiter/auth/kerberos.clj

@@ -22,7 +22,8 @@
             [waiter.auth.authentication :as auth]
             [waiter.authorization :as authz]
             [waiter.auth.spnego :as spnego]
-            [waiter.util.utils :as utils]))
+            [waiter.util.utils :as utils])
+  (:import (java.util.concurrent Semaphore)))
 
 (defn get-opt-in-accounts
   "Returns the list of users whose tickets are prestashed on host"
@@ -101,16 +102,19 @@
                          :status 403
                          :user run-as-user}))))))
 
-(defrecord KerberosAuthenticator [password]
+(defrecord KerberosAuthenticator [^Semaphore throttler password]
   auth/Authenticator
   (wrap-auth-handler [_ request-handler]
-    (spnego/require-gss request-handler password)))
+    (spnego/require-gss request-handler throttler password)))
 
 (defn kerberos-authenticator
   "Factory function for creating Kerberos authenticator middleware"
-  [{:keys [password]}]
-  {:pre [(not-empty password)]}
-  (->KerberosAuthenticator password))
+  [{:keys [concurrency-level password]}]
+  {:pre [(not-empty password)
+         (integer? concurrency-level)
+         (pos? concurrency-level)]}
+  (let [throttler (Semaphore. concurrency-level)]
+    (->KerberosAuthenticator throttler password)))
 
 (defrecord KerberosAuthorizer
   [prestash-cache query-chan]

waiter/src/waiter/auth/spnego.clj

@@ -20,6 +20,7 @@
             [clojure.tools.logging :as log]
             [metrics.counters :as counters]
             [metrics.meters :as meters]
+            [metrics.timers :as timers]
             [ring.middleware.cookies :as cookies]
             [ring.util.response :as rr]
             [waiter.auth.authentication :as auth]
@@ -29,7 +30,8 @@
            (org.eclipse.jetty.client.api Authentication$Result Request)
            (org.eclipse.jetty.http HttpHeader)
            (org.ietf.jgss GSSManager GSSCredential GSSContext GSSName Oid)
-           (java.net URI)))
+           (java.net URI)
+           (java.util.concurrent Semaphore)))
 
 (defn decode-input-token
   "Decode the input token from the negotiate line, expects the authorization token to exist"
@@ -73,7 +75,7 @@
     (meters/mark! (metrics/waiter-meter "core" "gss-context-creation"))
     gss))
 
-(defn gss-get-princ
+(defn gss-get-principal
   [^GSSContext gss]
   (str (.getSrcName gss)))
 
@@ -83,7 +85,7 @@
    will be run, otherwise the handler will not be run and 401
    returned instead.  This middleware doesn't handle cookies for
    authentication, but that should be stacked before this handler."
-  [request-handler password]
+  [request-handler ^Semaphore throttler password]
   (fn require-gss-handler [{:keys [headers] :as req}]
     (let [waiter-cookie (auth/get-auth-cookie-value (get headers "cookie"))
           [auth-principal _ :as decoded-auth-cookie] (auth/decode-auth-cookie waiter-cookie password)]
@@ -95,20 +97,30 @@
           (request-handler' req))
         ;; Try and authenticate using kerberos and add cookie in response when valid
         (get-in req [:headers "authorization"])
-        (let [^GSSContext gss_context (gss-context-init)
-              token (do-gss-auth-check gss_context req)]
-          (if (.isEstablished gss_context)
-            (let [principal (gss-get-princ gss_context)
-                  user (first (str/split principal #"@" 2))
-                  resp (auth/handle-request-auth request-handler req user principal password)]
-              (log/debug "added cookies to response")
-              (if token
-                (if (map? resp)
-                  (rr/header resp "WWW-Authenticate" token)
-                  (async/go
-                    (rr/header (async/<! resp) "WWW-Authenticate" token)))
-                resp))
-            (response-401-negotiate)))
+        (let [response-holder (promise)]
+          (try
+            (timers/start-stop-time!
+              (metrics/waiter-timer "core" "spnego" "throttle")
+              (.acquire throttler))
+            (let [^GSSContext gss-context (gss-context-init)
+                  token (do-gss-auth-check gss-context req)]
+              (->> (if (.isEstablished gss-context)
+                     (let [principal (gss-get-principal gss-context)
+                           user (first (str/split principal #"@" 2))
+                           resp (auth/handle-request-auth request-handler req user principal password)]
+                       (log/debug "added cookies to response")
+                       (if token
+                         (if (map? resp)
+                           (rr/header resp "WWW-Authenticate" token)
+                           (async/go
+                             (rr/header (async/<! resp) "WWW-Authenticate" token)))
+                         resp))
+                     (response-401-negotiate))
+                   (deliver response-holder)))
+            (finally
+              (.release throttler)))
+          ;; if no exception has been thrown, response-holder has our response
+          (deref response-holder))
         ;; Default to unauthorized
         :else
         (response-401-negotiate)))))

waiter/src/waiter/settings.clj

@@ -206,6 +206,8 @@
 
 (def settings-defaults
   {:authenticator-config {:kind :one-user
+                          :kerberos {:factory-fn 'waiter.auth.kerberos/kerberos-authenticator
+                                     :concurrency-level 20}
                           :one-user {:factory-fn 'waiter.auth.authentication/one-user-authenticator}}
    :cors-config {:kind :patterns
                  :patterns {:factory-fn 'waiter.cors/pattern-based-validator

waiter/test/waiter/auth/kerberos_test.clj

@@ -110,7 +110,8 @@
 
 (deftest test-kerberos-authenticator
   (with-redefs [start-prestash-cache-maintainer (constantly nil)]
-    (let [config {:password "test-password"
+    (let [config {:concurrency-level 20
+                  :password "test-password"
                   :prestash-cache-refresh-ms 100
                   :prestash-cache-min-refresh-ms 10
                   :prestash-query-host "example.com"}