Merge branch 'master' into KYUUBI-5579

zhaohehuhu · Nov 2, 2023 · 6f634f4 · 6f634f4
2 parents 33282e2 + ea9a78f
commit 6f634f4
Show file tree

Hide file tree

Showing 40 changed files with 559 additions and 350 deletions.
diff --git a/.github/ISSUE_TEMPLATE/code-contrib-task.yml b/.github/ISSUE_TEMPLATE/code-contrib-task.yml
@@ -41,6 +41,19 @@ body:
           have good logical thinking and the ability to solve complex problems, be proficient in programming
           skills or algorithms
 
+  - type: dropdown
+    id: level
+    attributes:
+      label: What's the level of this task?
+      multiple: false
+      options:
+        - TRIVIAL
+        - EASY
+        - MEDIUM
+        - CHALLENGE
+    validations:
+      required: true
+
   - type: checkboxes
     attributes:
       label: Code of Conduct
@@ -65,19 +78,19 @@ body:
       description: Mentor is required for MEDIUM and CHALLENGE tasks, to guide contributors to complete the task.
       options:
         - label: >
-            I have sufficient knowledge and experience of this task, and I volunteer to be the mentor of this task
-            to guide contributors to complete the task.
+            I have sufficient expertise on this task, and I volunteer to be a mentor of this task to guide
+            contributors through the task.
           required: false
 
   - type: textarea
     attributes:
       label: Skill requirements
-      description: Which stills are required for contributors who want to take this task?
+      description: What skills are required for contributors who want to take this task?
       placeholder: |
         e.g.
-        - Basic knowledge on Scala Programing Language
+        - Basic knowledge on Scala programming language
         - Familiar with Apache Maven, Docker and GitHub Action
-        - Basic knowledge on network programing and Apache Thrift RPC framework
+        - Basic knowledge on network programming and Apache Thrift RPC framework
         - Familiar with Apache Spark
         - ...
     validations:
@@ -105,8 +118,9 @@ body:
   - type: textarea
     attributes:
       label: Additional context
-      placeholder: >
-        Anything else that related to this task that the contributors need to know.
+      description: Anything else that related to this task that the contributors need to know.
+      value: |
+        Introduction of [2023 Kyuubi Code Contribution Program](https://github.com/apache/kyuubi/issues/5357)
     validations:
       required: false
 

diff --git a/build/dist b/build/dist
@@ -249,6 +249,7 @@ mkdir -p "$DISTDIR/pid"
 mkdir -p "$DISTDIR/logs"
 mkdir -p "$DISTDIR/work"
 mkdir -p "$DISTDIR/jars"
+mkdir -p "$DISTDIR/db-scripts"
 mkdir -p "$DISTDIR/beeline-jars"
 mkdir -p "$DISTDIR/web-ui"
 mkdir -p "$DISTDIR/externals/engines/flink"
@@ -270,6 +271,9 @@ echo "Build flags: $@" >> "$DISTDIR/RELEASE"
 # Copy kyuubi server jars
 cp -r "$KYUUBI_HOME"/kyuubi-assembly/target/scala-$SCALA_VERSION/jars/*.jar "$DISTDIR/jars/"
 
+# Copy kyuubi database scripts
+cp -r "$KYUUBI_HOME"/kyuubi-server/src/main/resources/sql/* "$DISTDIR/db-scripts/"
+
 # Copy kyuubi beeline jars
 cp "$KYUUBI_HOME"/kyuubi-hive-beeline/target/*.jar "$DISTDIR/beeline-jars/"
 

diff --git a/docs/configuration/settings.md b/docs/configuration/settings.md
@@ -387,22 +387,22 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co
 
 ### Server
 
-|                           Key                            |      Default      |                                                                                                                    Meaning                                                                                                                     |   Type   | Since |
-|----------------------------------------------------------|-------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|-------|
-| kyuubi.server.administrators                                                || Comma-separated list of Kyuubi service administrators. We use this config to grant admin permission to any service accounts.                                                                                                                   | set      | 1.8.0 |
-| kyuubi.server.info.provider                              | ENGINE            | The server information provider name, some clients may rely on this information to check the server compatibilities and functionalities. <li>SERVER: Return Kyuubi server information.</li> <li>ENGINE: Return Kyuubi engine information.</li> | string   | 1.6.1 |
-| kyuubi.server.limit.batch.connections.per.ipaddress      | &lt;undefined&gt; | Maximum kyuubi server batch connections per ipaddress. Any user exceeding this limit will not be allowed to connect.                                                                                                                           | int      | 1.7.0 |
-| kyuubi.server.limit.batch.connections.per.user           | &lt;undefined&gt; | Maximum kyuubi server batch connections per user. Any user exceeding this limit will not be allowed to connect.                                                                                                                                | int      | 1.7.0 |
-| kyuubi.server.limit.batch.connections.per.user.ipaddress | &lt;undefined&gt; | Maximum kyuubi server batch connections per user:ipaddress combination. Any user-ipaddress exceeding this limit will not be allowed to connect.                                                                                                | int      | 1.7.0 |
-| kyuubi.server.limit.client.fetch.max.rows                | &lt;undefined&gt; | Max rows limit for getting result row set operation. If the max rows specified by client-side is larger than the limit, request will fail directly.                                                                                            | int      | 1.8.0 |
-| kyuubi.server.limit.connections.per.ipaddress            | &lt;undefined&gt; | Maximum kyuubi server connections per ipaddress. Any user exceeding this limit will not be allowed to connect.                                                                                                                                 | int      | 1.6.0 |
-| kyuubi.server.limit.connections.per.user                 | &lt;undefined&gt; | Maximum kyuubi server connections per user. Any user exceeding this limit will not be allowed to connect.                                                                                                                                      | int      | 1.6.0 |
-| kyuubi.server.limit.connections.per.user.ipaddress       | &lt;undefined&gt; | Maximum kyuubi server connections per user:ipaddress combination. Any user-ipaddress exceeding this limit will not be allowed to connect.                                                                                                      | int      | 1.6.0 |
-| kyuubi.server.limit.connections.user.deny.list                              || The user in the deny list will be denied to connect to kyuubi server, if the user has configured both user.unlimited.list and user.deny.list, the priority of the latter is higher.                                                            | set      | 1.8.0 |
-| kyuubi.server.limit.connections.user.unlimited.list                         || The maximum connections of the user in the white list will not be limited.                                                                                                                                                                     | set      | 1.7.0 |
-| kyuubi.server.name                                       | &lt;undefined&gt; | The name of Kyuubi Server.                                                                                                                                                                                                                     | string   | 1.5.0 |
-| kyuubi.server.periodicGC.interval                        | PT30M             | How often to trigger a garbage collection.                                                                                                                                                                                                     | duration | 1.7.0 |
-| kyuubi.server.redaction.regex                            | &lt;undefined&gt; | Regex to decide which Kyuubi contain sensitive information. When this regex matches a property key or value, the value is redacted from the various logs.                                                                                                || 1.6.0 |
+|                           Key                            |      Default      |                                                                                                                                 Meaning                                                                                                                                 |   Type   | Since |
+|----------------------------------------------------------|-------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|-------|
+| kyuubi.server.administrators                                                || Comma-separated list of Kyuubi service administrators. We use this config to grant admin permission to any service accounts when security mechanism is enabled. Note, when kyuubi.authentication is configured to NOSASL or NONE, everyone is treated as administrator. | set      | 1.8.0 |
+| kyuubi.server.info.provider                              | ENGINE            | The server information provider name, some clients may rely on this information to check the server compatibilities and functionalities. <li>SERVER: Return Kyuubi server information.</li> <li>ENGINE: Return Kyuubi engine information.</li>                          | string   | 1.6.1 |
+| kyuubi.server.limit.batch.connections.per.ipaddress      | &lt;undefined&gt; | Maximum kyuubi server batch connections per ipaddress. Any user exceeding this limit will not be allowed to connect.                                                                                                                                                    | int      | 1.7.0 |
+| kyuubi.server.limit.batch.connections.per.user           | &lt;undefined&gt; | Maximum kyuubi server batch connections per user. Any user exceeding this limit will not be allowed to connect.                                                                                                                                                         | int      | 1.7.0 |
+| kyuubi.server.limit.batch.connections.per.user.ipaddress | &lt;undefined&gt; | Maximum kyuubi server batch connections per user:ipaddress combination. Any user-ipaddress exceeding this limit will not be allowed to connect.                                                                                                                         | int      | 1.7.0 |
+| kyuubi.server.limit.client.fetch.max.rows                | &lt;undefined&gt; | Max rows limit for getting result row set operation. If the max rows specified by client-side is larger than the limit, request will fail directly.                                                                                                                     | int      | 1.8.0 |
+| kyuubi.server.limit.connections.per.ipaddress            | &lt;undefined&gt; | Maximum kyuubi server connections per ipaddress. Any user exceeding this limit will not be allowed to connect.                                                                                                                                                          | int      | 1.6.0 |
+| kyuubi.server.limit.connections.per.user                 | &lt;undefined&gt; | Maximum kyuubi server connections per user. Any user exceeding this limit will not be allowed to connect.                                                                                                                                                               | int      | 1.6.0 |
+| kyuubi.server.limit.connections.per.user.ipaddress       | &lt;undefined&gt; | Maximum kyuubi server connections per user:ipaddress combination. Any user-ipaddress exceeding this limit will not be allowed to connect.                                                                                                                               | int      | 1.6.0 |
+| kyuubi.server.limit.connections.user.deny.list                              || The user in the deny list will be denied to connect to kyuubi server, if the user has configured both user.unlimited.list and user.deny.list, the priority of the latter is higher.                                                                                     | set      | 1.8.0 |
+| kyuubi.server.limit.connections.user.unlimited.list                         || The maximum connections of the user in the white list will not be limited.                                                                                                                                                                                              | set      | 1.7.0 |
+| kyuubi.server.name                                       | &lt;undefined&gt; | The name of Kyuubi Server.                                                                                                                                                                                                                                              | string   | 1.5.0 |
+| kyuubi.server.periodicGC.interval                        | PT30M             | How often to trigger a garbage collection.                                                                                                                                                                                                                              | duration | 1.7.0 |
+| kyuubi.server.redaction.regex                            | &lt;undefined&gt; | Regex to decide which Kyuubi contain sensitive information. When this regex matches a property key or value, the value is redacted from the various logs.                                                                                                                         || 1.6.0 |
 
 ### Session
 

diff --git a/.../main/resources/META-INF/services/org.apache.kyuubi.plugin.spark.authz.serde.URIExtractor b/.../main/resources/META-INF/services/org.apache.kyuubi.plugin.spark.authz.serde.URIExtractor
@@ -15,5 +15,6 @@
 # limitations under the License.
 #
 
+org.apache.kyuubi.plugin.spark.authz.serde.CatalogStorageFormatURIExtractor
 org.apache.kyuubi.plugin.spark.authz.serde.HadoopFsRelationFileIndexURIExtractor
 org.apache.kyuubi.plugin.spark.authz.serde.StringURIExtractor
diff --git a/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json b/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json
@@ -1097,7 +1097,11 @@
     "fieldName" : "query",
     "fieldExtractor" : "LogicalPlanQueryExtractor"
   } ],
-  "uriDescs" : [ ]
+  "uriDescs" : [ {
+    "fieldName" : "storage",
+    "fieldExtractor" : "CatalogStorageFormatURIExtractor",
+    "isInput" : false
+  } ]
 }, {
   "classname" : "org.apache.spark.sql.execution.command.LoadDataCommand",
   "tableDescs" : [ {
@@ -1370,7 +1374,11 @@
     "fieldName" : "query",
     "fieldExtractor" : "LogicalPlanQueryExtractor"
   } ],
-  "uriDescs" : [ ]
+  "uriDescs" : [ {
+    "fieldName" : "storage",
+    "fieldExtractor" : "CatalogStorageFormatURIExtractor",
+    "isInput" : false
+  } ]
 }, {
   "classname" : "org.apache.spark.sql.hive.execution.InsertIntoHiveTable",
   "tableDescs" : [ {
@@ -1968,19 +1976,4 @@
   "opType" : "QUERY",
   "queryDescs" : [ ],
   "uriDescs" : [ ]
-}, {
-  "classname" : "org.apache.spark.sql.delta.commands.CreateDeltaTableCommand",
-  "tableDescs" : [ {
-    "fieldName" : "table",
-    "fieldExtractor" : "CatalogTableTableExtractor",
-    "columnDesc" : null,
-    "actionTypeDesc" : null,
-    "tableTypeDesc" : null,
-    "catalogDesc" : null,
-    "isInput" : false,
-    "setCurrentDatabaseIfMissing" : false
-  } ],
-  "opType" : "CREATETABLE",
-  "queryDescs" : [ ],
-  "uriDescs" : [ ]
 } ]
diff --git a/...i-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/PrivilegesBuilder.scala b/...i-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/PrivilegesBuilder.scala
@@ -200,15 +200,11 @@ object PrivilegesBuilder {
         }
         spec.uriDescs.foreach { ud =>
           try {
-            val uri = ud.extract(plan)
-            uri match {
-              case Some(uri) =>
-                if (ud.isInput) {
-                  inputObjs += PrivilegeObject(uri)
-                } else {
-                  outputObjs += PrivilegeObject(uri)
-                }
-              case None =>
+            val uris = ud.extract(plan)
+            if (ud.isInput) {
+              inputObjs ++= uris.map(PrivilegeObject(_))
+            } else {
+              outputObjs ++= uris.map(PrivilegeObject(_))
             }
           } catch {
             case e: Exception =>

diff --git a/...bi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/Descriptor.scala b/...bi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/Descriptor.scala
@@ -318,7 +318,7 @@ case class UriDesc(
     fieldName: String,
     fieldExtractor: String,
     isInput: Boolean = false) extends Descriptor {
-  override def extract(v: AnyRef): Option[Uri] = {
+  override def extract(v: AnyRef): Seq[Uri] = {
     val uriVal = invokeAs[AnyRef](v, fieldName)
     val uriExtractor = lookupExtractor[URIExtractor](fieldExtractor)
     uriExtractor(uriVal)

diff --git a/...in/spark/authz/serde/pathExtractors.scala → ...gin/spark/authz/serde/uriExtractors.scala b/...in/spark/authz/serde/pathExtractors.scala → ...gin/spark/authz/serde/uriExtractors.scala
@@ -17,9 +17,10 @@
 
 package org.apache.kyuubi.plugin.spark.authz.serde
 
+import org.apache.spark.sql.catalyst.catalog.CatalogStorageFormat
 import org.apache.spark.sql.execution.datasources.HadoopFsRelation
 
-trait URIExtractor extends (AnyRef => Option[Uri]) with Extractor
+trait URIExtractor extends (AnyRef => Seq[Uri]) with Extractor
 
 object URIExtractor {
   val uriExtractors: Map[String, URIExtractor] = {
@@ -31,8 +32,14 @@ object URIExtractor {
  * String
  */
 class StringURIExtractor extends URIExtractor {
-  override def apply(v1: AnyRef): Option[Uri] = {
-    Some(Uri(v1.asInstanceOf[String]))
+  override def apply(v1: AnyRef): Seq[Uri] = {
+    Seq(Uri(v1.asInstanceOf[String]))
+  }
+}
+
+class CatalogStorageFormatURIExtractor extends URIExtractor {
+  override def apply(v1: AnyRef): Seq[Uri] = {
+    v1.asInstanceOf[CatalogStorageFormat].locationUri.map(uri => Uri(uri.getPath)).toSeq
   }
 }
 

diff --git a/...rk-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/PrivilegesBuilderSuite.scala b/...rk-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/PrivilegesBuilderSuite.scala
@@ -1467,7 +1467,15 @@ class HiveCatalogPrivilegeBuilderSuite extends PrivilegesBuilderSuite {
     val accessType0 = ranger.AccessType(po0, operationType, isInput = true)
     assert(accessType0 === AccessType.SELECT)
 
-    assert(out.isEmpty)
+    assert(out.size == 1)
+    val po1 = out.head
+    assert(po1.actionType === PrivilegeObjectActionType.OTHER)
+    assert(po1.privilegeObjectType === PrivilegeObjectType.DFS_URL)
+    assert(po1.dbname === directory.path)
+    assert(po1.objectName === null)
+    assert(po1.columns === Seq.empty)
+    val accessType1 = ranger.AccessType(po1, operationType, isInput = true)
+    assert(accessType1 == AccessType.SELECT)
   }
 
   test("InsertIntoDataSourceCommand") {
@@ -1591,7 +1599,15 @@ class HiveCatalogPrivilegeBuilderSuite extends PrivilegesBuilderSuite {
     val accessType0 = ranger.AccessType(po0, operationType, isInput = true)
     assert(accessType0 === AccessType.SELECT)
 
-    assert(out.isEmpty)
+    assert(out.size == 1)
+    val po1 = out.head
+    assert(po1.actionType === PrivilegeObjectActionType.OTHER)
+    assert(po1.privilegeObjectType === PrivilegeObjectType.DFS_URL)
+    assert(po1.dbname === directory.path)
+    assert(po1.objectName === null)
+    assert(po1.columns === Seq.empty)
+    val accessType1 = ranger.AccessType(po1, operationType, isInput = true)
+    assert(accessType1 == AccessType.SELECT)
   }
 
   test("InsertIntoHiveDirCommand") {
@@ -1616,7 +1632,15 @@ class HiveCatalogPrivilegeBuilderSuite extends PrivilegesBuilderSuite {
     val accessType0 = ranger.AccessType(po0, operationType, isInput = true)
     assert(accessType0 === AccessType.SELECT)
 
-    assert(out.isEmpty)
+    assert(out.size == 1)
+    val po1 = out.head
+    assert(po1.actionType === PrivilegeObjectActionType.OTHER)
+    assert(po1.privilegeObjectType === PrivilegeObjectType.DFS_URL)
+    assert(po1.dbname === directory.path)
+    assert(po1.objectName === null)
+    assert(po1.columns === Seq.empty)
+    val accessType1 = ranger.AccessType(po1, operationType, isInput = true)
+    assert(accessType1 == AccessType.SELECT)
   }
 
   test("InsertIntoHiveTableCommand") {