diff --git a/LICENSE-binary b/LICENSE-binary index 380c67d0c11..4d3335913bd 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -230,14 +230,6 @@ com.google.guava:failureaccess com.google.guava:guava org.apache.hadoop:hadoop-client-api org.apache.hadoop:hadoop-client-runtime -org.apache.hive:hive-common -org.apache.hive:hive-metastore -org.apache.hive:hive-standalone-metastore -org.apache.hive:hive-llap-client -org.apache.hive:hive-serde -org.apache.hive:hive-service-rpc -org.apache.hive:hive-shims-0.23 -org.apache.hive:hive-shims-common com.google.j2objc:j2objc-annotations com.fasterxml.jackson.core:jackson-annotations com.fasterxml.jackson.core:jackson-core @@ -270,8 +262,6 @@ org.eclipse.jetty:jetty-servlet org.eclipse.jetty:jetty-util-ajax org.eclipse.jetty:jetty-util org.eclipse.jetty:jetty-proxy -org.apache.thrift:libfb303 -org.apache.thrift:libthrift org.apache.logging.log4j:log4j-1.2-api org.apache.logging.log4j:log4j-api org.apache.logging.log4j:log4j-core diff --git a/NOTICE-binary b/NOTICE-binary index 747b61fff65..b8104d39fad 100644 --- a/NOTICE-binary +++ b/NOTICE-binary @@ -198,33 +198,12 @@ Copyright 2022 The Apache Software Foundation Hive JDBC Copyright 2022 The Apache Software Foundation -Hive Llap Client -Copyright 2022 The Apache Software Foundation - -Hive Metastore -Copyright 2022 The Apache Software Foundation - -Hive Serde -Copyright 2022 The Apache Software Foundation - Hive Service Copyright 2022 The Apache Software Foundation Hive Service RPC Copyright 2022 The Apache Software Foundation -Hive Shims 0.23 -Copyright 2022 The Apache Software Foundation - -Hive Shims Common -Copyright 2022 The Apache Software Foundation - -Hive Standalone Metastore -Copyright 2022 The Apache Software Foundation - -Hive Storage API -Copyright 2020 The Apache Software Foundation - Apache HttpClient Copyright 1999-2020 The Apache Software Foundation @@ -798,9 +777,6 @@ Permission to use, copy, modify and distribute UnixCrypt for non-commercial or commercial purposes and without fee is granted provided that the copyright notice appears in all copies. -Apache Thrift -Copyright 2006-2010 The Apache Software Foundation. - Apache Log4j 1.x Compatibility API Copyright 1999-2022 The Apache Software Foundation diff --git a/dev/dependencyList b/dev/dependencyList index 4143b511ace..394568cf3d1 100644 --- a/dev/dependencyList +++ b/dev/dependencyList @@ -51,14 +51,6 @@ gson/2.10.1//gson-2.10.1.jar guava/32.0.1-jre//guava-32.0.1-jre.jar hadoop-client-api/3.3.6//hadoop-client-api-3.3.6.jar hadoop-client-runtime/3.3.6//hadoop-client-runtime-3.3.6.jar -hive-common/3.1.3//hive-common-3.1.3.jar -hive-metastore/3.1.3//hive-metastore-3.1.3.jar -hive-serde/3.1.3//hive-serde-3.1.3.jar -hive-service-rpc/3.1.3//hive-service-rpc-3.1.3.jar -hive-shims-0.23/3.1.3//hive-shims-0.23-3.1.3.jar -hive-shims-common/3.1.3//hive-shims-common-3.1.3.jar -hive-standalone-metastore/3.1.3//hive-standalone-metastore-3.1.3.jar -hive-storage-api/2.7.0//hive-storage-api-2.7.0.jar hk2-api/2.6.1//hk2-api-2.6.1.jar hk2-locator/2.6.1//hk2-locator-2.6.1.jar hk2-utils/2.6.1//hk2-utils-2.6.1.jar @@ -133,8 +125,6 @@ kubernetes-model-rbac/6.8.1//kubernetes-model-rbac-6.8.1.jar kubernetes-model-resource/6.8.1//kubernetes-model-resource-6.8.1.jar kubernetes-model-scheduling/6.8.1//kubernetes-model-scheduling-6.8.1.jar kubernetes-model-storageclass/6.8.1//kubernetes-model-storageclass-6.8.1.jar -libfb303/0.9.3//libfb303-0.9.3.jar -libthrift/0.9.3//libthrift-0.9.3.jar log4j-1.2-api/2.20.0//log4j-1.2-api-2.20.0.jar log4j-api/2.20.0//log4j-api-2.20.0.jar log4j-core/2.20.0//log4j-core-2.20.0.jar diff --git a/kyuubi-server/pom.xml b/kyuubi-server/pom.xml index f187ebff148..cb9c39a11a4 100644 --- a/kyuubi-server/pom.xml +++ b/kyuubi-server/pom.xml @@ -78,6 +78,12 @@ ${project.version} + + org.apache.kyuubi + kyuubi-relocated-hive-metastore-client + ${kyuubi-relocated.version} + + org.antlr antlr4-runtime @@ -107,6 +113,7 @@ org.apache.hive hive-metastore ${hive.version} + test * @@ -119,6 +126,7 @@ org.apache.hive hive-standalone-metastore ${hive.version} + test * @@ -131,6 +139,7 @@ org.apache.hive hive-serde ${hive.version} + test * @@ -143,6 +152,7 @@ org.apache.hive.shims hive-shims-common ${hive.version} + test * @@ -155,6 +165,7 @@ org.apache.hive.shims hive-shims-0.23 ${hive.version} + test * @@ -167,6 +178,7 @@ org.apache.hive hive-common ${hive.version} + test * @@ -179,6 +191,7 @@ org.apache.hive hive-storage-api ${hive.storage-api.version} + test * @@ -190,16 +203,19 @@ org.apache.thrift libfb303 + test org.apache.thrift libthrift + test org.apache.hive hive-service-rpc + test diff --git a/kyuubi-server/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier b/kyuubi-server/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier index 65e2965c025..f1f00d9aaad 100644 --- a/kyuubi-server/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier +++ b/kyuubi-server/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier @@ -15,4 +15,4 @@ # limitations under the License. # -org.apache.hadoop.hive.metastore.security.DelegationTokenIdentifier +org.apache.kyuubi.shaded.hive.metastore.security.DelegationTokenIdentifier diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/credentials/HiveDelegationTokenProvider.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/credentials/HiveDelegationTokenProvider.scala index 128346de55d..f1966f13de7 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/credentials/HiveDelegationTokenProvider.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/credentials/HiveDelegationTokenProvider.scala @@ -18,9 +18,6 @@ package org.apache.kyuubi.credentials import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.hive.conf.HiveConf -import org.apache.hadoop.hive.metastore.{IMetaStoreClient, RetryingMetaStoreClient} -import org.apache.hadoop.hive.metastore.security.DelegationTokenIdentifier import org.apache.hadoop.io.Text import org.apache.hadoop.security.{Credentials, SecurityUtil} import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod @@ -28,6 +25,9 @@ import org.apache.hadoop.security.token.Token import org.apache.kyuubi.Logging import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.shaded.hive.metastore.{IMetaStoreClient, RetryingMetaStoreClient} +import org.apache.kyuubi.shaded.hive.metastore.conf.MetastoreConf +import org.apache.kyuubi.shaded.hive.metastore.security.DelegationTokenIdentifier class HiveDelegationTokenProvider extends HadoopDelegationTokenProvider with Logging { @@ -38,7 +38,7 @@ class HiveDelegationTokenProvider extends HadoopDelegationTokenProvider with Log override def serviceName: String = "hive" override def initialize(hadoopConf: Configuration, kyuubiConf: KyuubiConf): Unit = { - val conf = new HiveConf(hadoopConf, classOf[HiveConf]) + val conf = MetastoreConf.newMetastoreConf(hadoopConf) val metastoreUris = conf.getTrimmed("hive.metastore.uris", "") // SQL engine requires token alias to be `hive.metastore.uris` tokenAlias = new Text(metastoreUris) @@ -51,7 +51,7 @@ class HiveDelegationTokenProvider extends HadoopDelegationTokenProvider with Log principal = conf.getTrimmed(principalKey, "") require(principal.nonEmpty, s"Hive principal $principalKey undefined") - client = Some(RetryingMetaStoreClient.getProxy(conf, false)) + client = Some(RetryingMetaStoreClient.getProxy(conf)) info(s"Created HiveMetaStoreClient with metastore uris $metastoreUris") } } diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/http/ThriftHttpServlet.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/http/ThriftHttpServlet.scala index 980f35d70b5..d781faf3bf9 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/http/ThriftHttpServlet.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/http/ThriftHttpServlet.scala @@ -26,8 +26,6 @@ import javax.ws.rs.core.NewCookie import scala.collection.mutable -import org.apache.hadoop.hive.shims.Utils - import org.apache.kyuubi.Logging import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.config.KyuubiConf.FRONTEND_PROXY_HTTP_CLIENT_IP_HEADER @@ -57,6 +55,8 @@ class ThriftHttpServlet( private var isHttpOnlyCookie = false private val X_FORWARDED_FOR_HEADER = "X-Forwarded-For" private val authenticationFilter = new AuthenticationFilter(conf) + private val XSRF_HEADER_DEFAULT = "X-XSRF-HEADER" + private val XSRF_METHODS_TO_IGNORE_DEFAULT = Set("GET", "OPTIONS", "HEAD", "TRACE") override def init(): Unit = { isCookieAuthEnabled = conf.get(KyuubiConf.FRONTEND_THRIFT_HTTP_COOKIE_AUTH_ENABLED) @@ -82,7 +82,7 @@ class ThriftHttpServlet( var requireNewCookie: Boolean = false try { if (conf.get(KyuubiConf.FRONTEND_THRIFT_HTTP_XSRF_FILTER_ENABLED)) { - val continueProcessing = Utils.doXsrfFilter(request, response, null, null) + val continueProcessing = doXsrfFilter(request, response) if (!continueProcessing) { warn("Request did not have valid XSRF header, rejecting.") return @@ -303,4 +303,22 @@ class ThriftHttpServlet( null } + + private def doXsrfFilter( + httpRequest: HttpServletRequest, + response: HttpServletResponse): Boolean = { + if (XSRF_METHODS_TO_IGNORE_DEFAULT.contains(httpRequest.getMethod) + || httpRequest.getHeader(XSRF_HEADER_DEFAULT) != null) { + true + } else { + response.sendError( + HttpServletResponse.SC_BAD_REQUEST, + "Missing Required Header for Vulnerability Protection") + // scalastyle:off println + response.getWriter.println( + "XSRF filter denial, requests must contain header : " + XSRF_HEADER_DEFAULT) + // scalastyle:on println + false + } + } } diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/credentials/HiveDelegationTokenProviderSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/credentials/HiveDelegationTokenProviderSuite.scala index 6c0370f5530..76d3e561463 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/credentials/HiveDelegationTokenProviderSuite.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/credentials/HiveDelegationTokenProviderSuite.scala @@ -31,7 +31,7 @@ import org.apache.hadoop.conf.Configuration import org.apache.hadoop.hive.conf.HiveConf import org.apache.hadoop.hive.conf.HiveConf.ConfVars._ import org.apache.hadoop.hive.metastore.{HiveMetaException, HiveMetaStore} -import org.apache.hadoop.hive.metastore.security.{DelegationTokenIdentifier, HadoopThriftAuthBridge, HadoopThriftAuthBridge23} +import org.apache.hadoop.hive.metastore.security.{HadoopThriftAuthBridge, HadoopThriftAuthBridge23} import org.apache.hadoop.io.Text import org.apache.hadoop.security.{Credentials, UserGroupInformation} import org.apache.hadoop.security.authorize.ProxyUsers @@ -44,6 +44,7 @@ import org.scalatest.time.SpanSugar.convertIntToGrainOfTime import org.apache.kyuubi.{KerberizedTestHelper, Logging, Utils} import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.credentials.LocalMetaServer.defaultHiveConf +import org.apache.kyuubi.shaded.hive.metastore.security.DelegationTokenIdentifier class HiveDelegationTokenProviderSuite extends KerberizedTestHelper {