Skip to content
Permalink
Browse files
HIVE-26071: JWT authentication mechanism for Thrift over HTTP in Hive…
…Metastore (#3233) (Sourabh Goyal, reviewed by Yu-wen, Deng and Sai)

What changes were proposed in this pull request?
This PR is a follow up of #3105. It adds a support for JWT authentication in HiveMetastore server when run in HTTP transport mode.

Why are the changes needed?
It supports a new authentication mechanism ie JWT in HiveMetastore server.

Does this PR introduce any user-facing change?
No

How was this patch tested?
Added new unit tests that cover cases like

successfully authenticating valid JWT
failing to authenticate expired, invalid JWTs

* Add JWTValidator and URLBasedJWKSProvider code from HS2

Change-Id: I969f57daf640adb16f228e95b1b522f8ffc24ffe

* Add JWT authentication in HiveMetastore

Change-Id: I6d84517a1ee97df492ad3816ec866c0b785ed5ed

* Better error handling for authentication failures. Added integration tests for validating JWT

Change-Id: I6b9da531db4e4a805d8daa1ba6d941c5643bf514

* Added test JWTs for jwt authentication tests

Change-Id: Ice36a703d8af7d4dbf28a48c9bb96127100fd8c7

* moved jwt test keys under jwt directory

Change-Id: I8bf0b4bbc101a0acb3f69bb1963b9c4bcda5b719

* Fixes failures in metastore jwt unit tests

Change-Id: I2877730a34dff7d3184b100ec04031032611838a

* Addresses review comments

Change-Id: I8498e85212476c663cf735211848a28baaa3bad5

* Addresses nits from review comments

Change-Id: Id67588c106104732a0f6e49e5c983cb5f7287c3e

* Added more comments in the code

Change-Id: Ia51f490362985d109778a6a0aa92a281436d5d21

* removes unsed import statement

Change-Id: I94633bdce0db87a9085968dde79d8ff6cd9bf4a3
  • Loading branch information
sourabh912 committed May 12, 2022
1 parent a1906b9 commit d30db8cbafba110f6519354df7504b36643a8e60
Showing 13 changed files with 746 additions and 74 deletions.
@@ -80,6 +80,7 @@
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.http.HttpException;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpRequest;
import org.apache.http.HttpRequestInterceptor;
import org.apache.http.impl.client.HttpClientBuilder;
@@ -601,32 +602,52 @@ public void renamePartition(String catName, String dbname, String tableName, Lis
client.rename_partition_req(req);
}

/*
Creates a THttpClient if HTTP mode is enabled. If Client auth mode is set to JWT,
then the method fetches JWT from environment variable: HMS_JWT and sets in auth
header in http request
*/
private THttpClient createHttpClient(URI store, boolean useSSL) throws MetaException,
TTransportException {
String path = MetaStoreUtils.getHttpPath(MetastoreConf.getVar(conf, ConfVars.THRIFT_HTTP_PATH));
String httpUrl = (useSSL ? "https://" : "http://") + store.getHost() + ":" + store.getPort() + path;

String user = MetastoreConf.getVar(conf, ConfVars.METASTORE_CLIENT_PLAIN_USERNAME);
if (user == null || user.equals("")) {
try {
LOG.debug("No username passed in config " + ConfVars.METASTORE_CLIENT_PLAIN_USERNAME.getHiveName() +
". Trying to get the current user from UGI" );
user = UserGroupInformation.getCurrentUser().getShortUserName();
} catch (IOException e) {
throw new MetaException("Failed to get client username from UGI");
HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();
String authType = MetastoreConf.getVar(conf, ConfVars.METASTORE_CLIENT_AUTH_MODE);
if (authType.equalsIgnoreCase("jwt")) {
// fetch JWT token from environment and set it in Auth Header in HTTP request
String jwtToken = System.getenv("HMS_JWT");
if (jwtToken == null || jwtToken.isEmpty()) {
LOG.debug("No jwt token set in environment variable: HMS_JWT");
throw new MetaException("For auth mode JWT, valid signed jwt token must be provided in the "
+ "environment variable HMS_JWT");
}
httpClientBuilder.addInterceptorFirst(new HttpRequestInterceptor() {
@Override
public void process(HttpRequest httpRequest, HttpContext httpContext)
throws HttpException, IOException {
httpRequest.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + jwtToken);
}
});
} else {
String user = MetastoreConf.getVar(conf, ConfVars.METASTORE_CLIENT_PLAIN_USERNAME);
if (user == null || user.equals("")) {
try {
user = UserGroupInformation.getCurrentUser().getShortUserName();
} catch (IOException e) {
throw new MetaException("Failed to get client username from UGI");
}
}
final String httpUser = user;
httpClientBuilder.addInterceptorFirst(new HttpRequestInterceptor() {
@Override
public void process(HttpRequest httpRequest, HttpContext httpContext)
throws HttpException, IOException {
httpRequest.addHeader(MetaStoreUtils.USER_NAME_HTTP_HEADER, httpUser);
}
});
}
final String httpUser = user;
THttpClient tHttpClient;
HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();
httpClientBuilder.addInterceptorFirst(new HttpRequestInterceptor() {
@Override
public void process(HttpRequest httpRequest, HttpContext httpContext)
throws HttpException, IOException {
httpRequest.addHeader(MetaStoreUtils.USER_NAME_HTTP_HEADER, httpUser);
}
});

try {
if (useSSL) {
String trustStorePath = MetastoreConf.getVar(conf, ConfVars.SSL_TRUSTSTORE_PATH).trim();
@@ -875,15 +875,20 @@ public enum ConfVars {
"The special string _HOST will be replaced automatically with the correct host name."),
THRIFT_METASTORE_AUTHENTICATION("metastore.authentication", "hive.metastore.authentication",
"NOSASL",
new StringSetValidator("NOSASL", "NONE", "LDAP", "KERBEROS", "CUSTOM"),
new StringSetValidator("NOSASL", "NONE", "LDAP", "KERBEROS", "CUSTOM", "JWT"),
"Client authentication types.\n" +
" NONE: no authentication check\n" +
" LDAP: LDAP/AD based authentication\n" +
" KERBEROS: Kerberos/GSSAPI authentication\n" +
" CUSTOM: Custom authentication provider\n" +
" (Use with property metastore.custom.authentication.class)\n" +
" CONFIG: username and password is specified in the config" +
" NOSASL: Raw transport"),
" NOSASL: Raw transport" +
" JWT: JSON Web Token authentication via JWT token. Only supported in Http/Https mode"),
THRIFT_METASTORE_AUTHENTICATION_JWT_JWKS_URL("metastore.authentication.jwt.jwks.url",
"hive.metastore.authentication.jwt.jwks.url", "", "File URL from where URLBasedJWKSProvider "
+ "in metastore server will try to load JWKS to match a JWT sent in HTTP request header. Used only when "
+ "Hive metastore server is running in JWT auth mode"),
METASTORE_CUSTOM_AUTHENTICATION_CLASS("metastore.custom.authentication.class",
"hive.metastore.custom.authentication.class",
"",
@@ -1544,9 +1549,11 @@ public enum ConfVars {
"If true, the metastore Thrift interface will be secured with SASL. Clients must authenticate with Kerberos."),
METASTORE_CLIENT_AUTH_MODE("metastore.client.auth.mode",
"hive.metastore.client.auth.mode", "NOSASL",
new StringSetValidator("NOSASL", "PLAIN", "KERBEROS"),
new StringSetValidator("NOSASL", "PLAIN", "KERBEROS", "JWT"),
"If PLAIN, clients will authenticate using plain authentication, by providing username" +
" and password. Any other value is ignored right now but may be used later."),
" and password. Any other value is ignored right now but may be used later."
+ "If JWT- Supported only in HTTP transport mode. If set, HMS Client will pick the value of JWT from "
+ "environment variable HMS_JWT and set it in Authorization header in http request"),
METASTORE_CLIENT_PLAIN_USERNAME("metastore.client.plain.username",
"hive.metastore.client.plain.username", "",
"The username used by the metastore client when " +
@@ -311,6 +311,22 @@
<artifactId>curator-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.nimbusds</groupId>
<artifactId>nimbus-jose-jwt</artifactId>
<version>${nimbus-jose-jwt.version}</version>
</dependency>
<dependency>
<groupId>org.pac4j</groupId>
<artifactId>pac4j-core</artifactId>
<version>${pac4j-core.version}</version>
</dependency>
<dependency>
<groupId>com.github.tomakehurst</groupId>
<artifactId>wiremock-jre8-standalone</artifactId>
<version>2.32.0</version>
<scope>test</scope>
</dependency>
</dependencies>
<profiles>
<profile>
@@ -477,7 +477,7 @@ public Thread newThread(@NotNull Runnable r) {
IHMSHandler handler = newRetryingHMSHandler(baseHandler, conf);
processor = new ThriftHiveMetastore.Processor<>(handler);
LOG.info("Starting DB backed MetaStore Server with generic processor");
TServlet thriftHttpServlet = new HmsThriftHttpServlet(processor, protocolFactory);
TServlet thriftHttpServlet = new HmsThriftHttpServlet(processor, protocolFactory, conf);

boolean directSqlEnabled = MetastoreConf.getBoolVar(conf, ConfVars.TRY_DIRECT_SQL);
HMSHandler.LOG.info("Direct SQL optimization = {}", directSqlEnabled);
@@ -17,14 +17,23 @@

package org.apache.hadoop.hive.metastore;

import com.google.common.base.Preconditions;
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
import java.util.Enumeration;

import java.util.Optional;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.metastore.auth.HttpAuthenticationException;
import org.apache.hadoop.hive.metastore.auth.jwt.JWTValidator;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
import org.pac4j.core.context.JEEContext;
import org.pac4j.core.credentials.TokenCredentials;
import org.pac4j.core.credentials.extractor.BearerAuthExtractor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

@@ -33,81 +42,126 @@
import org.apache.thrift.protocol.TProtocolFactory;
import org.apache.thrift.server.TServlet;

/*
Servlet class used by HiveMetastore server when running in HTTP mode.
If JWT auth is enabled, then the servlet is also responsible for validating
JWTs sent in the Authorization header in HTTP request.
*/
public class HmsThriftHttpServlet extends TServlet {

private static final Logger LOG = LoggerFactory
.getLogger(HmsThriftHttpServlet.class);

private static final String X_USER = MetaStoreUtils.USER_NAME_HTTP_HEADER;

private final boolean isSecurityEnabled;
private final boolean jwtAuthEnabled;
private JWTValidator jwtValidator = null;
private Configuration conf;

public HmsThriftHttpServlet(TProcessor processor,
TProtocolFactory inProtocolFactory, TProtocolFactory outProtocolFactory) {
super(processor, inProtocolFactory, outProtocolFactory);
// This should ideally be reveiving an instance of the Configuration which is used for the check
TProtocolFactory protocolFactory, Configuration conf) {
super(processor, protocolFactory);
this.conf = conf;
isSecurityEnabled = UserGroupInformation.isSecurityEnabled();
if (MetastoreConf.getVar(conf,
ConfVars.THRIFT_METASTORE_AUTHENTICATION).equalsIgnoreCase("jwt")) {
jwtAuthEnabled = true;
} else {
jwtAuthEnabled = false;
}
}

public HmsThriftHttpServlet(TProcessor processor,
TProtocolFactory protocolFactory) {
super(processor, protocolFactory);
isSecurityEnabled = UserGroupInformation.isSecurityEnabled();
public void init() throws ServletException {
super.init();
if (jwtAuthEnabled) {
try {
jwtValidator = new JWTValidator(this.conf);
} catch (Exception e) {
throw new ServletException("Failed to initialize HmsThriftHttpServlet."
+ " Error: " + e);
}
}
}

@Override
protected void doPost(HttpServletRequest request,
HttpServletResponse response) throws ServletException, IOException {

Enumeration<String> headerNames = request.getHeaderNames();
if (LOG.isDebugEnabled()) {
LOG.debug("Logging headers in request");
LOG.debug("Logging headers in doPost request");
Enumeration<String> headerNames = request.getHeaderNames();
while (headerNames.hasMoreElements()) {
String headerName = headerNames.nextElement();
LOG.debug("Header: [{}], Value: [{}]", headerName,
request.getHeader(headerName));
}
}
String userFromHeader = request.getHeader(X_USER);
if (userFromHeader == null || userFromHeader.isEmpty()) {
LOG.error("No user header: {} found", X_USER);
response.sendError(HttpServletResponse.SC_FORBIDDEN,
"Header: " + X_USER + " missing in the request");
return;
}

// TODO: These should ideally be in some kind of a Cache with Weak referencse.
// If HMS were to set up some kind of a session, this would go into the session by having
// this filter work with a custom Processor / or set the username into the session
// as is done for HS2.
// In case of HMS, it looks like each request is independent, and there is no session
// information, so the UGI needs to be set up in the Connection layer itself.
UserGroupInformation clientUgi;
// Temporary, and useless for now. Here only to allow this to work on an otherwise kerberized
// server.
if (isSecurityEnabled) {
LOG.info("Creating proxy user for: {}", userFromHeader);
clientUgi = UserGroupInformation.createProxyUser(userFromHeader, UserGroupInformation.getLoginUser());
} else {
LOG.info("Creating remote user for: {}", userFromHeader);
clientUgi = UserGroupInformation.createRemoteUser(userFromHeader);
try {
String userFromHeader = extractUserName(request, response);
UserGroupInformation clientUgi;
// Temporary, and useless for now. Here only to allow this to work on an otherwise kerberized
// server.
if (isSecurityEnabled) {
LOG.info("Creating proxy user for: {}", userFromHeader);
clientUgi = UserGroupInformation.createProxyUser(userFromHeader, UserGroupInformation.getLoginUser());
} else {
LOG.info("Creating remote user for: {}", userFromHeader);
clientUgi = UserGroupInformation.createRemoteUser(userFromHeader);
}
PrivilegedExceptionAction<Void> action = new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
HmsThriftHttpServlet.super.doPost(request, response);
return null;
}
};
try {
clientUgi.doAs(action);
} catch (InterruptedException | RuntimeException e) {
LOG.error("Exception when executing http request as user: " + clientUgi.getUserName(),
e);
throw new ServletException(e);
}
} catch (HttpAuthenticationException e) {
response.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
response.getWriter().println("Authentication error: " + e.getMessage());
// Also log the error message on server side
LOG.error("Authentication error: ", e);
}


PrivilegedExceptionAction<Void> action = new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
HmsThriftHttpServlet.super.doPost(request, response);
return null;
}
private String extractUserName(HttpServletRequest request, HttpServletResponse response)
throws HttpAuthenticationException {
if (!jwtAuthEnabled) {
String userFromHeader = request.getHeader(X_USER);
if (userFromHeader == null || userFromHeader.isEmpty()) {
throw new HttpAuthenticationException("User header " + X_USER + " missing in request");
}
};

return userFromHeader;
}
String signedJwt = extractBearerToken(request, response);
if (signedJwt == null) {
throw new HttpAuthenticationException("Couldn't find bearer token in the auth header in the request");
}
String user;
try {
clientUgi.doAs(action);
} catch (InterruptedException | RuntimeException e) {
LOG.error("Exception when executing http request as user: " + clientUgi.getUserName(),
e);
throw new ServletException(e);
user = jwtValidator.validateJWTAndExtractUser(signedJwt);
Preconditions.checkNotNull(user, "JWT needs to contain the user name as subject");
Preconditions.checkState(!user.isEmpty(), "User name should not be empty in JWT");
LOG.info("Successfully validated and extracted user name {} from JWT in Auth "
+ "header in the request", user);
} catch (Exception e) {
throw new HttpAuthenticationException("Failed to validate JWT from Bearer token in "
+ "Authentication header", e);
}
return user;
}

/**
* Extracts the bearer authorization header from the request. If there is no bearer
* authorization token, returns null.
*/
private String extractBearerToken(HttpServletRequest request,
HttpServletResponse response) {
BearerAuthExtractor extractor = new BearerAuthExtractor();
Optional<TokenCredentials> tokenCredentials = extractor.extract(new JEEContext(
request, response));
return tokenCredentials.map(TokenCredentials::getToken).orElse(null);
}
}
@@ -0,0 +1,47 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License. See accompanying LICENSE file.
*/

package org.apache.hadoop.hive.metastore.auth;

/*
Encapsulates any exceptions thrown by HiveMetastore server
when authenticating http requests
*/
public class HttpAuthenticationException extends Exception {

private static final long serialVersionUID = 0;

/**
* @param cause original exception
*/
public HttpAuthenticationException(Throwable cause) {
super(cause);
}

/**
* @param msg exception message
*/
public HttpAuthenticationException(String msg) {
super(msg);
}

/**
* @param msg exception message
* @param cause original exception
*/
public HttpAuthenticationException(String msg, Throwable cause) {
super(msg, cause);
}

}

0 comments on commit d30db8c

Please sign in to comment.