diff --git a/LICENSE-binary b/LICENSE-binary
index 380c67d0c11..4d3335913bd 100644
--- a/LICENSE-binary
+++ b/LICENSE-binary
@@ -230,14 +230,6 @@ com.google.guava:failureaccess
com.google.guava:guava
org.apache.hadoop:hadoop-client-api
org.apache.hadoop:hadoop-client-runtime
-org.apache.hive:hive-common
-org.apache.hive:hive-metastore
-org.apache.hive:hive-standalone-metastore
-org.apache.hive:hive-llap-client
-org.apache.hive:hive-serde
-org.apache.hive:hive-service-rpc
-org.apache.hive:hive-shims-0.23
-org.apache.hive:hive-shims-common
com.google.j2objc:j2objc-annotations
com.fasterxml.jackson.core:jackson-annotations
com.fasterxml.jackson.core:jackson-core
@@ -270,8 +262,6 @@ org.eclipse.jetty:jetty-servlet
org.eclipse.jetty:jetty-util-ajax
org.eclipse.jetty:jetty-util
org.eclipse.jetty:jetty-proxy
-org.apache.thrift:libfb303
-org.apache.thrift:libthrift
org.apache.logging.log4j:log4j-1.2-api
org.apache.logging.log4j:log4j-api
org.apache.logging.log4j:log4j-core
diff --git a/NOTICE-binary b/NOTICE-binary
index 747b61fff65..b8104d39fad 100644
--- a/NOTICE-binary
+++ b/NOTICE-binary
@@ -198,33 +198,12 @@ Copyright 2022 The Apache Software Foundation
Hive JDBC
Copyright 2022 The Apache Software Foundation
-Hive Llap Client
-Copyright 2022 The Apache Software Foundation
-
-Hive Metastore
-Copyright 2022 The Apache Software Foundation
-
-Hive Serde
-Copyright 2022 The Apache Software Foundation
-
Hive Service
Copyright 2022 The Apache Software Foundation
Hive Service RPC
Copyright 2022 The Apache Software Foundation
-Hive Shims 0.23
-Copyright 2022 The Apache Software Foundation
-
-Hive Shims Common
-Copyright 2022 The Apache Software Foundation
-
-Hive Standalone Metastore
-Copyright 2022 The Apache Software Foundation
-
-Hive Storage API
-Copyright 2020 The Apache Software Foundation
-
Apache HttpClient
Copyright 1999-2020 The Apache Software Foundation
@@ -798,9 +777,6 @@ Permission to use, copy, modify and distribute UnixCrypt
for non-commercial or commercial purposes and without fee is
granted provided that the copyright notice appears in all copies.
-Apache Thrift
-Copyright 2006-2010 The Apache Software Foundation.
-
Apache Log4j 1.x Compatibility API
Copyright 1999-2022 The Apache Software Foundation
diff --git a/dev/dependencyList b/dev/dependencyList
index 4143b511ace..394568cf3d1 100644
--- a/dev/dependencyList
+++ b/dev/dependencyList
@@ -51,14 +51,6 @@ gson/2.10.1//gson-2.10.1.jar
guava/32.0.1-jre//guava-32.0.1-jre.jar
hadoop-client-api/3.3.6//hadoop-client-api-3.3.6.jar
hadoop-client-runtime/3.3.6//hadoop-client-runtime-3.3.6.jar
-hive-common/3.1.3//hive-common-3.1.3.jar
-hive-metastore/3.1.3//hive-metastore-3.1.3.jar
-hive-serde/3.1.3//hive-serde-3.1.3.jar
-hive-service-rpc/3.1.3//hive-service-rpc-3.1.3.jar
-hive-shims-0.23/3.1.3//hive-shims-0.23-3.1.3.jar
-hive-shims-common/3.1.3//hive-shims-common-3.1.3.jar
-hive-standalone-metastore/3.1.3//hive-standalone-metastore-3.1.3.jar
-hive-storage-api/2.7.0//hive-storage-api-2.7.0.jar
hk2-api/2.6.1//hk2-api-2.6.1.jar
hk2-locator/2.6.1//hk2-locator-2.6.1.jar
hk2-utils/2.6.1//hk2-utils-2.6.1.jar
@@ -133,8 +125,6 @@ kubernetes-model-rbac/6.8.1//kubernetes-model-rbac-6.8.1.jar
kubernetes-model-resource/6.8.1//kubernetes-model-resource-6.8.1.jar
kubernetes-model-scheduling/6.8.1//kubernetes-model-scheduling-6.8.1.jar
kubernetes-model-storageclass/6.8.1//kubernetes-model-storageclass-6.8.1.jar
-libfb303/0.9.3//libfb303-0.9.3.jar
-libthrift/0.9.3//libthrift-0.9.3.jar
log4j-1.2-api/2.20.0//log4j-1.2-api-2.20.0.jar
log4j-api/2.20.0//log4j-api-2.20.0.jar
log4j-core/2.20.0//log4j-core-2.20.0.jar
diff --git a/kyuubi-server/pom.xml b/kyuubi-server/pom.xml
index f187ebff148..cb9c39a11a4 100644
--- a/kyuubi-server/pom.xml
+++ b/kyuubi-server/pom.xml
@@ -78,6 +78,12 @@
${project.version}
+
+ org.apache.kyuubi
+ kyuubi-relocated-hive-metastore-client
+ ${kyuubi-relocated.version}
+
+
org.antlr
antlr4-runtime
@@ -107,6 +113,7 @@
org.apache.hive
hive-metastore
${hive.version}
+ test
*
@@ -119,6 +126,7 @@
org.apache.hive
hive-standalone-metastore
${hive.version}
+ test
*
@@ -131,6 +139,7 @@
org.apache.hive
hive-serde
${hive.version}
+ test
*
@@ -143,6 +152,7 @@
org.apache.hive.shims
hive-shims-common
${hive.version}
+ test
*
@@ -155,6 +165,7 @@
org.apache.hive.shims
hive-shims-0.23
${hive.version}
+ test
*
@@ -167,6 +178,7 @@
org.apache.hive
hive-common
${hive.version}
+ test
*
@@ -179,6 +191,7 @@
org.apache.hive
hive-storage-api
${hive.storage-api.version}
+ test
*
@@ -190,16 +203,19 @@
org.apache.thrift
libfb303
+ test
org.apache.thrift
libthrift
+ test
org.apache.hive
hive-service-rpc
+ test
diff --git a/kyuubi-server/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier b/kyuubi-server/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
index 65e2965c025..f1f00d9aaad 100644
--- a/kyuubi-server/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
+++ b/kyuubi-server/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
@@ -15,4 +15,4 @@
# limitations under the License.
#
-org.apache.hadoop.hive.metastore.security.DelegationTokenIdentifier
+org.apache.kyuubi.shaded.hive.metastore.security.DelegationTokenIdentifier
diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/credentials/HiveDelegationTokenProvider.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/credentials/HiveDelegationTokenProvider.scala
index 128346de55d..f1966f13de7 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/credentials/HiveDelegationTokenProvider.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/credentials/HiveDelegationTokenProvider.scala
@@ -18,9 +18,6 @@
package org.apache.kyuubi.credentials
import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.hive.conf.HiveConf
-import org.apache.hadoop.hive.metastore.{IMetaStoreClient, RetryingMetaStoreClient}
-import org.apache.hadoop.hive.metastore.security.DelegationTokenIdentifier
import org.apache.hadoop.io.Text
import org.apache.hadoop.security.{Credentials, SecurityUtil}
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod
@@ -28,6 +25,9 @@ import org.apache.hadoop.security.token.Token
import org.apache.kyuubi.Logging
import org.apache.kyuubi.config.KyuubiConf
+import org.apache.kyuubi.shaded.hive.metastore.{IMetaStoreClient, RetryingMetaStoreClient}
+import org.apache.kyuubi.shaded.hive.metastore.conf.MetastoreConf
+import org.apache.kyuubi.shaded.hive.metastore.security.DelegationTokenIdentifier
class HiveDelegationTokenProvider extends HadoopDelegationTokenProvider with Logging {
@@ -38,7 +38,7 @@ class HiveDelegationTokenProvider extends HadoopDelegationTokenProvider with Log
override def serviceName: String = "hive"
override def initialize(hadoopConf: Configuration, kyuubiConf: KyuubiConf): Unit = {
- val conf = new HiveConf(hadoopConf, classOf[HiveConf])
+ val conf = MetastoreConf.newMetastoreConf(hadoopConf)
val metastoreUris = conf.getTrimmed("hive.metastore.uris", "")
// SQL engine requires token alias to be `hive.metastore.uris`
tokenAlias = new Text(metastoreUris)
@@ -51,7 +51,7 @@ class HiveDelegationTokenProvider extends HadoopDelegationTokenProvider with Log
principal = conf.getTrimmed(principalKey, "")
require(principal.nonEmpty, s"Hive principal $principalKey undefined")
- client = Some(RetryingMetaStoreClient.getProxy(conf, false))
+ client = Some(RetryingMetaStoreClient.getProxy(conf))
info(s"Created HiveMetaStoreClient with metastore uris $metastoreUris")
}
}
diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/http/ThriftHttpServlet.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/http/ThriftHttpServlet.scala
index 980f35d70b5..d781faf3bf9 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/http/ThriftHttpServlet.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/http/ThriftHttpServlet.scala
@@ -26,8 +26,6 @@ import javax.ws.rs.core.NewCookie
import scala.collection.mutable
-import org.apache.hadoop.hive.shims.Utils
-
import org.apache.kyuubi.Logging
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.config.KyuubiConf.FRONTEND_PROXY_HTTP_CLIENT_IP_HEADER
@@ -57,6 +55,8 @@ class ThriftHttpServlet(
private var isHttpOnlyCookie = false
private val X_FORWARDED_FOR_HEADER = "X-Forwarded-For"
private val authenticationFilter = new AuthenticationFilter(conf)
+ private val XSRF_HEADER_DEFAULT = "X-XSRF-HEADER"
+ private val XSRF_METHODS_TO_IGNORE_DEFAULT = Set("GET", "OPTIONS", "HEAD", "TRACE")
override def init(): Unit = {
isCookieAuthEnabled = conf.get(KyuubiConf.FRONTEND_THRIFT_HTTP_COOKIE_AUTH_ENABLED)
@@ -82,7 +82,7 @@ class ThriftHttpServlet(
var requireNewCookie: Boolean = false
try {
if (conf.get(KyuubiConf.FRONTEND_THRIFT_HTTP_XSRF_FILTER_ENABLED)) {
- val continueProcessing = Utils.doXsrfFilter(request, response, null, null)
+ val continueProcessing = doXsrfFilter(request, response)
if (!continueProcessing) {
warn("Request did not have valid XSRF header, rejecting.")
return
@@ -303,4 +303,22 @@ class ThriftHttpServlet(
null
}
+
+ private def doXsrfFilter(
+ httpRequest: HttpServletRequest,
+ response: HttpServletResponse): Boolean = {
+ if (XSRF_METHODS_TO_IGNORE_DEFAULT.contains(httpRequest.getMethod)
+ || httpRequest.getHeader(XSRF_HEADER_DEFAULT) != null) {
+ true
+ } else {
+ response.sendError(
+ HttpServletResponse.SC_BAD_REQUEST,
+ "Missing Required Header for Vulnerability Protection")
+ // scalastyle:off println
+ response.getWriter.println(
+ "XSRF filter denial, requests must contain header : " + XSRF_HEADER_DEFAULT)
+ // scalastyle:on println
+ false
+ }
+ }
}
diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/credentials/HiveDelegationTokenProviderSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/credentials/HiveDelegationTokenProviderSuite.scala
index 6c0370f5530..76d3e561463 100644
--- a/kyuubi-server/src/test/scala/org/apache/kyuubi/credentials/HiveDelegationTokenProviderSuite.scala
+++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/credentials/HiveDelegationTokenProviderSuite.scala
@@ -31,7 +31,7 @@ import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.hive.conf.HiveConf
import org.apache.hadoop.hive.conf.HiveConf.ConfVars._
import org.apache.hadoop.hive.metastore.{HiveMetaException, HiveMetaStore}
-import org.apache.hadoop.hive.metastore.security.{DelegationTokenIdentifier, HadoopThriftAuthBridge, HadoopThriftAuthBridge23}
+import org.apache.hadoop.hive.metastore.security.{HadoopThriftAuthBridge, HadoopThriftAuthBridge23}
import org.apache.hadoop.io.Text
import org.apache.hadoop.security.{Credentials, UserGroupInformation}
import org.apache.hadoop.security.authorize.ProxyUsers
@@ -44,6 +44,7 @@ import org.scalatest.time.SpanSugar.convertIntToGrainOfTime
import org.apache.kyuubi.{KerberizedTestHelper, Logging, Utils}
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.credentials.LocalMetaServer.defaultHiveConf
+import org.apache.kyuubi.shaded.hive.metastore.security.DelegationTokenIdentifier
class HiveDelegationTokenProviderSuite extends KerberizedTestHelper {