Use kyuubi-relocated-hive-metastore-client

jiaoqingbo · Mar 8, 2024 · af17be1 · af17be1
1 parent 02bf4f0
commit af17be1
Show file tree

Hide file tree

Showing 4 changed files with 46 additions and 10 deletions.
diff --git a/kyuubi-server/pom.xml b/kyuubi-server/pom.xml
@@ -78,6 +78,12 @@
             <version>${project.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.kyuubi</groupId>
+            <artifactId>kyuubi-relocated-hive-metastore-client</artifactId>
+            <version>${kyuubi-relocated.version}</version>
+        </dependency>
+
         <dependency>
             <groupId>org.antlr</groupId>
             <artifactId>antlr4-runtime</artifactId>
@@ -107,6 +113,7 @@
             <groupId>org.apache.hive</groupId>
             <artifactId>hive-metastore</artifactId>
             <version>${hive.version}</version>
+            <scope>test</scope>
             <exclusions>
                 <exclusion>
                     <groupId>*</groupId>
@@ -119,6 +126,7 @@
             <groupId>org.apache.hive</groupId>
             <artifactId>hive-standalone-metastore</artifactId>
             <version>${hive.version}</version>
+            <scope>test</scope>
             <exclusions>
                 <exclusion>
                     <groupId>*</groupId>
@@ -131,6 +139,7 @@
             <groupId>org.apache.hive</groupId>
             <artifactId>hive-serde</artifactId>
             <version>${hive.version}</version>
+            <scope>test</scope>
             <exclusions>
                 <exclusion>
                     <groupId>*</groupId>
@@ -143,6 +152,7 @@
             <groupId>org.apache.hive.shims</groupId>
             <artifactId>hive-shims-common</artifactId>
             <version>${hive.version}</version>
+            <scope>test</scope>
             <exclusions>
                 <exclusion>
                     <groupId>*</groupId>
@@ -155,6 +165,7 @@
             <groupId>org.apache.hive.shims</groupId>
             <artifactId>hive-shims-0.23</artifactId>
             <version>${hive.version}</version>
+            <scope>test</scope>
             <exclusions>
                 <exclusion>
                     <groupId>*</groupId>
@@ -167,6 +178,7 @@
             <groupId>org.apache.hive</groupId>
             <artifactId>hive-common</artifactId>
             <version>${hive.version}</version>
+            <scope>test</scope>
             <exclusions>
                 <exclusion>
                     <groupId>*</groupId>
@@ -179,6 +191,7 @@
             <groupId>org.apache.hive</groupId>
             <artifactId>hive-storage-api</artifactId>
             <version>${hive.storage-api.version}</version>
+            <scope>test</scope>
             <exclusions>
                 <exclusion>
                     <groupId>*</groupId>
@@ -190,16 +203,19 @@
         <dependency>
             <groupId>org.apache.thrift</groupId>
             <artifactId>libfb303</artifactId>
+            <scope>test</scope>
         </dependency>
 
         <dependency>
             <groupId>org.apache.thrift</groupId>
             <artifactId>libthrift</artifactId>
+            <scope>test</scope>
         </dependency>
 
         <dependency>
             <groupId>org.apache.hive</groupId>
             <artifactId>hive-service-rpc</artifactId>
+            <scope>test</scope>
         </dependency>
 
         <dependency>

diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/credentials/HiveDelegationTokenProvider.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/credentials/HiveDelegationTokenProvider.scala
@@ -18,16 +18,16 @@
 package org.apache.kyuubi.credentials
 
 import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.hive.conf.HiveConf
-import org.apache.hadoop.hive.metastore.{IMetaStoreClient, RetryingMetaStoreClient}
-import org.apache.hadoop.hive.metastore.security.DelegationTokenIdentifier
 import org.apache.hadoop.io.Text
 import org.apache.hadoop.security.{Credentials, SecurityUtil}
 import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod
 import org.apache.hadoop.security.token.Token
 
 import org.apache.kyuubi.Logging
 import org.apache.kyuubi.config.KyuubiConf
+import org.apache.kyuubi.shaded.hive.metastore.{IMetaStoreClient, RetryingMetaStoreClient}
+import org.apache.kyuubi.shaded.hive.metastore.conf.MetastoreConf
+import org.apache.kyuubi.shaded.hive.metastore.security.DelegationTokenIdentifier
 
 class HiveDelegationTokenProvider extends HadoopDelegationTokenProvider with Logging {
 
@@ -38,7 +38,7 @@ class HiveDelegationTokenProvider extends HadoopDelegationTokenProvider with Log
   override def serviceName: String = "hive"
 
   override def initialize(hadoopConf: Configuration, kyuubiConf: KyuubiConf): Unit = {
-    val conf = new HiveConf(hadoopConf, classOf[HiveConf])
+    val conf = MetastoreConf.newMetastoreConf(hadoopConf)
     val metastoreUris = conf.getTrimmed("hive.metastore.uris", "")
     // SQL engine requires token alias to be `hive.metastore.uris`
     tokenAlias = new Text(metastoreUris)
@@ -51,7 +51,7 @@ class HiveDelegationTokenProvider extends HadoopDelegationTokenProvider with Log
       principal = conf.getTrimmed(principalKey, "")
       require(principal.nonEmpty, s"Hive principal $principalKey undefined")
 
-      client = Some(RetryingMetaStoreClient.getProxy(conf, false))
+      client = Some(RetryingMetaStoreClient.getProxy(conf))
       info(s"Created HiveMetaStoreClient with metastore uris $metastoreUris")
     }
   }

diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/http/ThriftHttpServlet.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/http/ThriftHttpServlet.scala
@@ -26,8 +26,6 @@ import javax.ws.rs.core.NewCookie
 
 import scala.collection.mutable
 
-import org.apache.hadoop.hive.shims.Utils
-
 import org.apache.kyuubi.Logging
 import org.apache.kyuubi.config.KyuubiConf
 import org.apache.kyuubi.config.KyuubiConf.FRONTEND_PROXY_HTTP_CLIENT_IP_HEADER
@@ -57,6 +55,8 @@ class ThriftHttpServlet(
   private var isHttpOnlyCookie = false
   private val X_FORWARDED_FOR_HEADER = "X-Forwarded-For"
   private val authenticationFilter = new AuthenticationFilter(conf)
+  private val XSRF_HEADER_DEFAULT = "X-XSRF-HEADER"
+  private val XSRF_METHODS_TO_IGNORE_DEFAULT = Set("GET", "OPTIONS", "HEAD", "TRACE")
 
   override def init(): Unit = {
     isCookieAuthEnabled = conf.get(KyuubiConf.FRONTEND_THRIFT_HTTP_COOKIE_AUTH_ENABLED)
@@ -82,7 +82,7 @@ class ThriftHttpServlet(
     var requireNewCookie: Boolean = false
     try {
       if (conf.get(KyuubiConf.FRONTEND_THRIFT_HTTP_XSRF_FILTER_ENABLED)) {
-        val continueProcessing = Utils.doXsrfFilter(request, response, null, null)
+        val continueProcessing = doXsrfFilter(request, response)
         if (!continueProcessing) {
           warn("Request did not have valid XSRF header, rejecting.")
           return
@@ -303,4 +303,22 @@ class ThriftHttpServlet(
 
     null
   }
+
+  private def doXsrfFilter(
+      httpRequest: HttpServletRequest,
+      response: HttpServletResponse): Boolean = {
+    if (XSRF_METHODS_TO_IGNORE_DEFAULT.contains(httpRequest.getMethod)
+      || httpRequest.getHeader(XSRF_HEADER_DEFAULT) != null) {
+      true
+    } else {
+      response.sendError(
+        HttpServletResponse.SC_BAD_REQUEST,
+        "Missing Required Header for Vulnerability Protection")
+      // scalastyle:off println
+      response.getWriter.println(
+        "XSRF filter denial, requests must contain header : " + XSRF_HEADER_DEFAULT)
+      // scalastyle:on println
+      false
+    }
+  }
 }
diff --git a/...erver/src/test/scala/org/apache/kyuubi/credentials/HiveDelegationTokenProviderSuite.scala b/...erver/src/test/scala/org/apache/kyuubi/credentials/HiveDelegationTokenProviderSuite.scala
@@ -31,10 +31,11 @@ import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.hive.conf.HiveConf
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars._
 import org.apache.hadoop.hive.metastore.{HiveMetaException, HiveMetaStore}
-import org.apache.hadoop.hive.metastore.security.{DelegationTokenIdentifier, HadoopThriftAuthBridge, HadoopThriftAuthBridge23}
+import org.apache.hadoop.hive.metastore.security.{HadoopThriftAuthBridge, HadoopThriftAuthBridge23}
 import org.apache.hadoop.io.Text
 import org.apache.hadoop.security.{Credentials, UserGroupInformation}
 import org.apache.hadoop.security.authorize.ProxyUsers
+import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier
 import org.apache.thrift.TProcessor
 import org.apache.thrift.protocol.TProtocol
 import org.scalatest.Assertions._
@@ -44,6 +45,7 @@ import org.scalatest.time.SpanSugar.convertIntToGrainOfTime
 import org.apache.kyuubi.{KerberizedTestHelper, Logging, Utils}
 import org.apache.kyuubi.config.KyuubiConf
 import org.apache.kyuubi.credentials.LocalMetaServer.defaultHiveConf
+import org.apache.kyuubi.shaded.hive.metastore.security.DelegationTokenIdentifier
 
 class HiveDelegationTokenProviderSuite extends KerberizedTestHelper {
 
@@ -118,7 +120,7 @@ class HiveDelegationTokenProviderSuite extends KerberizedTestHelper {
       assert(aliasAndToken._2 != null)
 
       val token = aliasAndToken._2
-      val tokenIdent = token.decodeIdentifier().asInstanceOf[DelegationTokenIdentifier]
+      val tokenIdent = token.decodeIdentifier().asInstanceOf[AbstractDelegationTokenIdentifier]
       assertResult(DelegationTokenIdentifier.HIVE_DELEGATION_KIND)(token.getKind)
       assertResult(new Text(owner))(tokenIdent.getOwner)
       val currentUserName = UserGroupInformation.getCurrentUser.getUserName