HIVE-26071: JWT authentication mechanism for Thrift over HTTP in Hive…

…Metastore (#3233) (Sourabh Goyal, reviewed by Yu-wen, Deng and Sai)

What changes were proposed in this pull request?
This PR is a follow up of #3105. It adds a support for JWT authentication in HiveMetastore server when run in HTTP transport mode.

Why are the changes needed?
It supports a new authentication mechanism ie JWT in HiveMetastore server.

Does this PR introduce any user-facing change?
No

How was this patch tested?
Added new unit tests that cover cases like

successfully authenticating valid JWT
failing to authenticate expired, invalid JWTs

* Add JWTValidator and URLBasedJWKSProvider code from HS2

Change-Id: I969f57daf640adb16f228e95b1b522f8ffc24ffe

* Add JWT authentication in HiveMetastore

Change-Id: I6d84517a1ee97df492ad3816ec866c0b785ed5ed

* Better error handling for authentication failures. Added integration tests for validating JWT

Change-Id: I6b9da531db4e4a805d8daa1ba6d941c5643bf514

* Added test JWTs for jwt authentication tests

Change-Id: Ice36a703d8af7d4dbf28a48c9bb96127100fd8c7

* moved jwt test keys under jwt directory

Change-Id: I8bf0b4bbc101a0acb3f69bb1963b9c4bcda5b719

* Fixes failures in metastore jwt unit tests

Change-Id: I2877730a34dff7d3184b100ec04031032611838a

* Addresses review comments

Change-Id: I8498e85212476c663cf735211848a28baaa3bad5

* Addresses nits from review comments

Change-Id: Id67588c106104732a0f6e49e5c983cb5f7287c3e

* Added more comments in the code

Change-Id: Ia51f490362985d109778a6a0aa92a281436d5d21

* removes unsed import statement

Change-Id: I94633bdce0db87a9085968dde79d8ff6cd9bf4a3

standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java

@@ -80,6 +80,7 @@
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.http.HttpException;
+import org.apache.http.HttpHeaders;
 import org.apache.http.HttpRequest;
 import org.apache.http.HttpRequestInterceptor;
 import org.apache.http.impl.client.HttpClientBuilder;
@@ -601,32 +602,52 @@ public void renamePartition(String catName, String dbname, String tableName, Lis
     client.rename_partition_req(req);
   }
 
+  /*
+  Creates a THttpClient if HTTP mode is enabled. If Client auth mode is set to JWT,
+  then the method fetches JWT from environment variable: HMS_JWT and sets in auth
+  header in http request
+   */
   private THttpClient createHttpClient(URI store, boolean useSSL) throws MetaException,
       TTransportException {
     String path = MetaStoreUtils.getHttpPath(MetastoreConf.getVar(conf, ConfVars.THRIFT_HTTP_PATH));
     String httpUrl = (useSSL ? "https://" : "http://") + store.getHost() + ":" + store.getPort() + path;
 
-    String user = MetastoreConf.getVar(conf, ConfVars.METASTORE_CLIENT_PLAIN_USERNAME);
-    if (user == null || user.equals("")) {
-      try {
-        LOG.debug("No username passed in config " + ConfVars.METASTORE_CLIENT_PLAIN_USERNAME.getHiveName() +
-            ". Trying to get the current user from UGI" );
-        user = UserGroupInformation.getCurrentUser().getShortUserName();
-      } catch (IOException e) {
-        throw new MetaException("Failed to get client username from UGI");
+    HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();
+    String authType = MetastoreConf.getVar(conf, ConfVars.METASTORE_CLIENT_AUTH_MODE);
+    if (authType.equalsIgnoreCase("jwt")) {
+      // fetch JWT token from environment and set it in Auth Header in HTTP request
+      String jwtToken = System.getenv("HMS_JWT");
+      if (jwtToken == null || jwtToken.isEmpty()) {
+        LOG.debug("No jwt token set in environment variable: HMS_JWT");
+        throw new MetaException("For auth mode JWT, valid signed jwt token must be provided in the "
+            + "environment variable HMS_JWT");
       }
+      httpClientBuilder.addInterceptorFirst(new HttpRequestInterceptor() {
+        @Override
+        public void process(HttpRequest httpRequest, HttpContext httpContext)
+            throws HttpException, IOException {
+          httpRequest.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + jwtToken);
+        }
+      });
+    } else {
+      String user = MetastoreConf.getVar(conf, ConfVars.METASTORE_CLIENT_PLAIN_USERNAME);
+      if (user == null || user.equals("")) {
+        try {
+          user = UserGroupInformation.getCurrentUser().getShortUserName();
+        } catch (IOException e) {
+          throw new MetaException("Failed to get client username from UGI");
+        }
+      }
+      final String httpUser = user;
+      httpClientBuilder.addInterceptorFirst(new HttpRequestInterceptor() {
+        @Override
+        public void process(HttpRequest httpRequest, HttpContext httpContext)
+            throws HttpException, IOException {
+          httpRequest.addHeader(MetaStoreUtils.USER_NAME_HTTP_HEADER, httpUser);
+        }
+      });
     }
-    final String httpUser = user;
     THttpClient tHttpClient;
-    HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();
-    httpClientBuilder.addInterceptorFirst(new HttpRequestInterceptor() {
-      @Override
-      public void process(HttpRequest httpRequest, HttpContext httpContext)
-          throws HttpException, IOException {
-        httpRequest.addHeader(MetaStoreUtils.USER_NAME_HTTP_HEADER, httpUser);
-      }
-    });
-
     try {
       if (useSSL) {
         String trustStorePath = MetastoreConf.getVar(conf, ConfVars.SSL_TRUSTSTORE_PATH).trim();

standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java

@@ -875,15 +875,20 @@ public enum ConfVars {
             "The special string _HOST will be replaced automatically with the correct host name."),
     THRIFT_METASTORE_AUTHENTICATION("metastore.authentication", "hive.metastore.authentication",
             "NOSASL",
-      new StringSetValidator("NOSASL", "NONE", "LDAP", "KERBEROS", "CUSTOM"),
+      new StringSetValidator("NOSASL", "NONE", "LDAP", "KERBEROS", "CUSTOM", "JWT"),
         "Client authentication types.\n" +
                 "  NONE: no authentication check\n" +
                 "  LDAP: LDAP/AD based authentication\n" +
                 "  KERBEROS: Kerberos/GSSAPI authentication\n" +
                 "  CUSTOM: Custom authentication provider\n" +
                 "          (Use with property metastore.custom.authentication.class)\n" +
                 "  CONFIG: username and password is specified in the config" +
-                "  NOSASL:  Raw transport"),
+                "  NOSASL:  Raw transport" +
+                "  JWT:  JSON Web Token authentication via JWT token. Only supported in Http/Https mode"),
+    THRIFT_METASTORE_AUTHENTICATION_JWT_JWKS_URL("metastore.authentication.jwt.jwks.url",
+        "hive.metastore.authentication.jwt.jwks.url", "", "File URL from where URLBasedJWKSProvider "
+        + "in metastore server will try to load JWKS to match a JWT sent in HTTP request header. Used only when "
+        + "Hive metastore server is running in JWT auth mode"),
     METASTORE_CUSTOM_AUTHENTICATION_CLASS("metastore.custom.authentication.class",
             "hive.metastore.custom.authentication.class",
             "",
@@ -1544,9 +1549,11 @@ public enum ConfVars {
         "If true, the metastore Thrift interface will be secured with SASL. Clients must authenticate with Kerberos."),
     METASTORE_CLIENT_AUTH_MODE("metastore.client.auth.mode",
             "hive.metastore.client.auth.mode", "NOSASL",
-            new StringSetValidator("NOSASL", "PLAIN", "KERBEROS"),
+            new StringSetValidator("NOSASL", "PLAIN", "KERBEROS", "JWT"),
             "If PLAIN, clients will authenticate using plain authentication, by providing username" +
-                    " and password. Any other value is ignored right now but may be used later."),
+                    " and password. Any other value is ignored right now but may be used later."
+                + "If JWT- Supported only in HTTP transport mode. If set, HMS Client will pick the value of JWT from "
+                + "environment variable HMS_JWT and set it in Authorization header in http request"),
     METASTORE_CLIENT_PLAIN_USERNAME("metastore.client.plain.username",
             "hive.metastore.client.plain.username",  "",
         "The username used by the metastore client when " +

standalone-metastore/metastore-server/pom.xml

@@ -311,6 +311,22 @@
       <artifactId>curator-test</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>com.nimbusds</groupId>
+      <artifactId>nimbus-jose-jwt</artifactId>
+      <version>${nimbus-jose-jwt.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.pac4j</groupId>
+      <artifactId>pac4j-core</artifactId>
+      <version>${pac4j-core.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.github.tomakehurst</groupId>
+      <artifactId>wiremock-jre8-standalone</artifactId>
+      <version>2.32.0</version>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
   <profiles>
     <profile>

standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java

@@ -477,7 +477,7 @@ public Thread newThread(@NotNull Runnable r) {
     IHMSHandler handler = newRetryingHMSHandler(baseHandler, conf);
     processor = new ThriftHiveMetastore.Processor<>(handler);
     LOG.info("Starting DB backed MetaStore Server with generic processor");
-    TServlet thriftHttpServlet = new HmsThriftHttpServlet(processor, protocolFactory);
+    TServlet thriftHttpServlet = new HmsThriftHttpServlet(processor, protocolFactory, conf);
 
     boolean directSqlEnabled = MetastoreConf.getBoolVar(conf, ConfVars.TRY_DIRECT_SQL);
     HMSHandler.LOG.info("Direct SQL optimization = {}",  directSqlEnabled);

standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HmsThriftHttpServlet.java

@@ -17,14 +17,23 @@
 
 package org.apache.hadoop.hive.metastore;
 
+import com.google.common.base.Preconditions;
 import java.io.IOException;
 import java.security.PrivilegedExceptionAction;
 import java.util.Enumeration;
-
+import java.util.Optional;
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.auth.HttpAuthenticationException;
+import org.apache.hadoop.hive.metastore.auth.jwt.JWTValidator;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
 import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
+import org.pac4j.core.context.JEEContext;
+import org.pac4j.core.credentials.TokenCredentials;
+import org.pac4j.core.credentials.extractor.BearerAuthExtractor;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -33,81 +42,126 @@
 import org.apache.thrift.protocol.TProtocolFactory;
 import org.apache.thrift.server.TServlet;
 
+/*
+Servlet class used by HiveMetastore server when running in HTTP mode.
+If JWT auth is enabled, then the servlet is also responsible for validating
+JWTs sent in the Authorization header in HTTP request.
+ */
 public class HmsThriftHttpServlet extends TServlet {
-
   private static final Logger LOG = LoggerFactory
       .getLogger(HmsThriftHttpServlet.class);
-
   private static final String X_USER = MetaStoreUtils.USER_NAME_HTTP_HEADER;
-
   private final boolean isSecurityEnabled;
+  private final boolean jwtAuthEnabled;
+  private JWTValidator jwtValidator = null;
+  private Configuration conf;
 
   public HmsThriftHttpServlet(TProcessor processor,
-      TProtocolFactory inProtocolFactory, TProtocolFactory outProtocolFactory) {
-    super(processor, inProtocolFactory, outProtocolFactory);
-    // This should ideally be reveiving an instance of the Configuration which is used for the check
+      TProtocolFactory protocolFactory, Configuration conf) {
+    super(processor, protocolFactory);
+    this.conf = conf;
     isSecurityEnabled = UserGroupInformation.isSecurityEnabled();
+    if (MetastoreConf.getVar(conf,
+        ConfVars.THRIFT_METASTORE_AUTHENTICATION).equalsIgnoreCase("jwt")) {
+      jwtAuthEnabled = true;
+    } else {
+      jwtAuthEnabled = false;
+    }
   }
 
-  public HmsThriftHttpServlet(TProcessor processor,
-      TProtocolFactory protocolFactory) {
-    super(processor, protocolFactory);
-    isSecurityEnabled = UserGroupInformation.isSecurityEnabled();
+  public void init() throws ServletException {
+    super.init();
+    if (jwtAuthEnabled) {
+      try {
+        jwtValidator = new JWTValidator(this.conf);
+      } catch (Exception e) {
+        throw new ServletException("Failed to initialize HmsThriftHttpServlet."
+            + " Error: " + e);
+      }
+    }
   }
 
   @Override
   protected void doPost(HttpServletRequest request,
       HttpServletResponse response) throws ServletException, IOException {
-
-    Enumeration<String> headerNames = request.getHeaderNames();
     if (LOG.isDebugEnabled()) {
-      LOG.debug("Logging headers in request");
+      LOG.debug("Logging headers in doPost request");
+      Enumeration<String> headerNames = request.getHeaderNames();
       while (headerNames.hasMoreElements()) {
         String headerName = headerNames.nextElement();
         LOG.debug("Header: [{}], Value: [{}]", headerName,
             request.getHeader(headerName));
       }
     }
-    String userFromHeader = request.getHeader(X_USER);
-    if (userFromHeader == null || userFromHeader.isEmpty()) {
-      LOG.error("No user header: {} found", X_USER);
-      response.sendError(HttpServletResponse.SC_FORBIDDEN,
-          "Header: " + X_USER + " missing in the request");
-      return;
-    }
-
-    // TODO: These should ideally be in some kind of a Cache with Weak referencse.
-    // If HMS were to set up some kind of a session, this would go into the session by having
-    // this filter work with a custom Processor / or set the username into the session
-    // as is done for HS2.
-    // In case of HMS, it looks like each request is independent, and there is no session
-    // information, so the UGI needs to be set up in the Connection layer itself.
-    UserGroupInformation clientUgi;
-    // Temporary, and useless for now. Here only to allow this to work on an otherwise kerberized
-    // server.
-    if (isSecurityEnabled) {
-      LOG.info("Creating proxy user for: {}", userFromHeader);
-      clientUgi = UserGroupInformation.createProxyUser(userFromHeader, UserGroupInformation.getLoginUser());
-    } else {
-      LOG.info("Creating remote user for: {}", userFromHeader);
-      clientUgi = UserGroupInformation.createRemoteUser(userFromHeader);
+    try {
+      String userFromHeader = extractUserName(request, response);
+      UserGroupInformation clientUgi;
+      // Temporary, and useless for now. Here only to allow this to work on an otherwise kerberized
+      // server.
+      if (isSecurityEnabled) {
+        LOG.info("Creating proxy user for: {}", userFromHeader);
+        clientUgi = UserGroupInformation.createProxyUser(userFromHeader, UserGroupInformation.getLoginUser());
+      } else {
+        LOG.info("Creating remote user for: {}", userFromHeader);
+        clientUgi = UserGroupInformation.createRemoteUser(userFromHeader);
+      }
+      PrivilegedExceptionAction<Void> action = new PrivilegedExceptionAction<Void>() {
+        @Override
+        public Void run() throws Exception {
+          HmsThriftHttpServlet.super.doPost(request, response);
+          return null;
+        }
+      };
+      try {
+        clientUgi.doAs(action);
+      } catch (InterruptedException | RuntimeException e) {
+        LOG.error("Exception when executing http request as user: " + clientUgi.getUserName(),
+            e);
+        throw new ServletException(e);
+      }
+    } catch (HttpAuthenticationException e) {
+      response.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
+      response.getWriter().println("Authentication error: " + e.getMessage());
+      // Also log the error message on server side
+      LOG.error("Authentication error: ", e);
     }
-
-
-    PrivilegedExceptionAction<Void> action = new PrivilegedExceptionAction<Void>() {
-      @Override
-      public Void run() throws Exception {
-        HmsThriftHttpServlet.super.doPost(request, response);
-        return null;
+  }
+  private String extractUserName(HttpServletRequest request, HttpServletResponse response)
+      throws HttpAuthenticationException {
+    if (!jwtAuthEnabled) {
+      String userFromHeader = request.getHeader(X_USER);
+      if (userFromHeader == null || userFromHeader.isEmpty()) {
+        throw new HttpAuthenticationException("User header " + X_USER + " missing in request");
       }
-    };
-
+      return userFromHeader;
+    }
+    String signedJwt = extractBearerToken(request, response);
+    if (signedJwt == null) {
+      throw new HttpAuthenticationException("Couldn't find bearer token in the auth header in the request");
+    }
+    String user;
     try {
-      clientUgi.doAs(action);
-    } catch (InterruptedException | RuntimeException e) {
-      LOG.error("Exception when executing http request as user: " + clientUgi.getUserName(),
-          e);
-      throw new ServletException(e);
+      user = jwtValidator.validateJWTAndExtractUser(signedJwt);
+      Preconditions.checkNotNull(user, "JWT needs to contain the user name as subject");
+      Preconditions.checkState(!user.isEmpty(), "User name should not be empty in JWT");
+      LOG.info("Successfully validated and extracted user name {} from JWT in Auth "
+          + "header in the request", user);
+    } catch (Exception e) {
+      throw new HttpAuthenticationException("Failed to validate JWT from Bearer token in "
+          + "Authentication header", e);
     }
+    return user;
+  }
+
+  /**
+   * Extracts the bearer authorization header from the request. If there is no bearer
+   * authorization token, returns null.
+   */
+  private String extractBearerToken(HttpServletRequest request,
+      HttpServletResponse response) {
+    BearerAuthExtractor extractor = new BearerAuthExtractor();
+    Optional<TokenCredentials> tokenCredentials = extractor.extract(new JEEContext(
+        request, response));
+    return tokenCredentials.map(TokenCredentials::getToken).orElse(null);
   }
 }

standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/auth/HttpAuthenticationException.java

@@ -0,0 +1,47 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+
+package org.apache.hadoop.hive.metastore.auth;
+
+/*
+Encapsulates any exceptions thrown by HiveMetastore server
+when authenticating http requests
+ */
+public class HttpAuthenticationException extends Exception {
+
+  private static final long serialVersionUID = 0;
+
+  /**
+   * @param cause original exception
+   */
+  public HttpAuthenticationException(Throwable cause) {
+    super(cause);
+  }
+
+  /**
+   * @param msg exception message
+   */
+  public HttpAuthenticationException(String msg) {
+    super(msg);
+  }
+
+  /**
+   * @param msg   exception message
+   * @param cause original exception
+   */
+  public HttpAuthenticationException(String msg, Throwable cause) {
+    super(msg, cause);
+  }
+
+}