Merge branch 'apache:master' into master

.github/ISSUE_TEMPLATE/doc-improvement-report.yml → .github/ISSUE_TEMPLATE/documentation.yml

@@ -15,16 +15,11 @@
 # limitations under the License.
 #
 
-name: Doc Improvement Report
-title: "[DOCS] "
+name: Documentation fixes or improvement
+title: ":memo: Fix/Add <what> for <which> page"
 description: Fix errors, or improve the content or refactor architecture of online documentation
-labels: ["kind:documentation"]
+labels: ["kind:documentation,kind:minor,help wanted,good first issue"]
 body:
-  - type: markdown
-    attributes:
-      value: |
-        Thank you for finding the time to report the problem! We really appreciate the community efforts to improve Kyuubi.
-
   - type: checkboxes
     attributes:
       label: Code of Conduct
@@ -43,22 +38,25 @@ body:
             issues.
           required: true
 
-  - type: textarea
+  - type: dropdown
+    id: priority
     attributes:
-      label: Which parts of the documentation do you think need improvement?
-      description: Please describe the details with documentation you have.
-      placeholder: >
-        Please include links to the documentation that you want to improve and possibly screenshots showing
-        the details. Explain why do you think it needs to improve. Make sure you include view of the target
-        audience of the documentation. Please explain why you think the docs are wrong.
+      label: What type of changes will we make to the documentation?
+      options:
+        - Bugfixes
+        - Usage of New Feature
+        - Showcase
+        - Refactoring
+        - Typo, layout, grammar, spelling, punctuation errors, etc.
+    validations:
+      required: true
 
   - type: input
     id: versions
     attributes:
       label: Affects Version(s)
       description: Which versions of Kyuubi Documentation are affected by this issue?
-      placeholder: >
-        e.g. master/1.5.0/1.4.1/...
+      placeholder: e.g. master/1.5.0/1.4.1/...
     validations:
       required: true
 
@@ -67,20 +65,9 @@ body:
       label: Improving the documentation
       description: How do you think the documentation can be improved?
       placeholder: >
-        Please explain how you think the documentation could be improved. Ideally specify where a new or missing
-        documentation should be added and what kind of information should be included. Sometimes people
-        writing the documentation do not realise that some assumptions they have might not be in the heads
-        of the reader, so try to explain exactly what you would like to see in the docs and why.
-
-  - type: textarea
-    attributes:
-      label: Anything else
-      description: Anything else we need to know?
-      placeholder: >
-        How often does this problem occur? (Once? Every time? Only when certain conditions are met?)
-        Any relevant logs to include? Put them here inside fenced
-        ``` ``` blocks or inside a foldable details tag if it's long:
-        <details><summary>x.log</summary> lots of stuff </details>
+        Please include links to the documentation that you want to improve and possibly screenshots showing
+        the details. Explain why do you think it needs to improve. Make sure you include view of the target
+        audience of the documentation. Please explain why you think the docs are wrong.
 
   - type: checkboxes
     attributes:

.github/PULL_REQUEST_TEMPLATE

@@ -20,4 +20,13 @@ Please clarify why the changes are needed. For instance,
 
 - [ ] Add screenshots for manual tests if appropriate
 
-- [ ] [Run test](https://kyuubi.readthedocs.io/en/master/develop_tools/testing.html#running-tests) locally before make a pull request
+- [ ] [Run test](https://kyuubi.readthedocs.io/en/master/contributing/code/testing.html#running-tests) locally before make a pull request
+
+
+### _Was this patch authored or co-authored using generative AI tooling?_
+<!--
+If a generative AI tooling has been used in the process of authoring this patch, please include
+phrase 'Generated-by: ' followed by the name of the tool and its version.
+If no, write 'No'.
+Please refer to the [ASF Generative Tooling Guidance](https://www.apache.org/legal/generative-tooling.html) for details.
+-->

.github/actions/setup-mvnd/action.yaml → .github/actions/setup-maven/action.yaml

@@ -15,21 +15,17 @@
 # limitations under the License.
 #
 
-name: 'setup-mvnd'
-description: 'Setup Maven and Mvnd'
+name: setup-maven
+description: 'Install and cache maven'
 runs:
   using: composite
   steps:
-    - name: Cache Mvnd
+    - name: Restore cached Maven
       uses: actions/cache@v3
       with:
-        path: |
-          build/maven-mvnd-*
-          build/apache-maven-*
-        key: setup-mvnd-${{ runner.os }}
-    - name: Check Maven
-      run: build/mvn -v
-      shell: bash
-    - name: Check Mvnd
-      run: build/mvnd -v || true
+        path: build/apache-maven-*
+        key: setup-maven-${{ hashFiles('pom.xml') }}
+        restore-keys: setup-maven-
+    - name: Install Maven
       shell: bash
+      run: build/mvn -v

.github/workflows/dep.yml

@@ -26,7 +26,6 @@ on:
       # when pom or dependency workflow changes
       - '**/pom.xml'
       - '.github/workflows/dep.yml'
-      - .github/actions/setup-mvnd/*.yaml
 
 concurrency:
   group: dep-${{ github.head_ref || github.run_id }}
@@ -45,12 +44,12 @@ jobs:
           java-version: 8
           cache: 'maven'
           check-latest: false
-      - name: Setup Maven and Mvnd
-        uses: ./.github/actions/setup-mvnd
+      - name: Setup Maven
+        uses: ./.github/actions/setup-maven
       - name: Check kyuubi modules available
         id: modules-check
         run: >-
-          build/mvnd dependency:resolve validate -q
+          build/mvn dependency:resolve validate
           -DincludeGroupIds="org.apache.kyuubi" -DincludeScope="compile"
           -Pfast -Denforcer.skip=false
           -pl kyuubi-ctl,kyuubi-server,kyuubi-assembly -am
@@ -60,7 +59,7 @@ jobs:
           MAVEN_OPTS: -Dorg.slf4j.simpleLogger.defaultLogLevel=error
         if: steps.modules-check.conclusion == 'success' && steps.modules-check.outcome == 'failure'
         run: >-
-          build/mvnd clean install
+          build/mvn clean install
           -Pflink-provided,spark-provided,hive-provided
           -Dmaven.javadoc.skip=true
           -Drat.skip=true

.github/workflows/license.yml

@@ -42,12 +42,10 @@ jobs:
           java-version: 8
           cache: 'maven'
           check-latest: false
-      - name: Setup Maven and Mvnd
-        uses: ./.github/actions/setup-mvnd
       - run: >-
-          build/mvnd org.apache.rat:apache-rat-plugin:check
+          build/mvn org.apache.rat:apache-rat-plugin:check
           -Ptpcds -Pspark-block-cleaner -Pkubernetes-it
-          -Pspark-3.1 -Pspark-3.2 -Pspark-3.3
+          -Pspark-3.1 -Pspark-3.2 -Pspark-3.3 -Pspark-3.4
       - name: Upload rat report
         if: failure()
         uses: actions/upload-artifact@v3

.github/workflows/master.yml

@@ -32,7 +32,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  MVN_OPT: -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip -Dorg.slf4j.simpleLogger.defaultLogLevel=warn -Pjdbc-shaded -Dmaven.plugin.download.cache.path=/tmp/engine-archives
+  MVN_OPT: -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip -Dorg.slf4j.simpleLogger.defaultLogLevel=warn -Pjdbc-shaded,gen-policy -Dmaven.plugin.download.cache.path=/tmp/engine-archives
   KUBERNETES_VERSION: v1.26.1
   MINIKUBE_VERSION: v1.29.0
 
@@ -46,24 +46,42 @@ jobs:
         java:
           - 8
           - 11
+          - 17
         spark:
           - '3.1'
           - '3.2'
           - '3.3'
+          - '3.4'
         spark-archive: [""]
         exclude-tags: [""]
         comment: ["normal"]
         include:
           - java: 8
-            spark: '3.3'
-            spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.1.3 -Dspark.archive.name=spark-3.1.3-bin-hadoop3.2.tgz'
-            exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.IcebergTest'
+            spark: '3.4'
+            spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.1.3 -Dspark.archive.name=spark-3.1.3-bin-hadoop3.2.tgz -Pzookeeper-3.6'
+            exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
             comment: 'verify-on-spark-3.1-binary'
           - java: 8
-            spark: '3.3'
-            spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.2.3 -Dspark.archive.name=spark-3.2.3-bin-hadoop3.2.tgz'
-            exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.IcebergTest'
+            spark: '3.4'
+            spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.2.4 -Dspark.archive.name=spark-3.2.4-bin-hadoop3.2.tgz -Pzookeeper-3.6'
+            exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
             comment: 'verify-on-spark-3.2-binary'
+          - java: 8
+            spark: '3.4'
+            spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.3.3 -Dspark.archive.name=spark-3.3.3-bin-hadoop3.tgz -Pzookeeper-3.6'
+            exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
+            comment: 'verify-on-spark-3.3-binary'
+          - java: 8
+            spark: '3.4'
+            spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.5.0 -Dspark.archive.name=spark-3.5.0-bin-hadoop3.tgz -Pzookeeper-3.6'
+            exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
+            comment: 'verify-on-spark-3.5-binary'
+        exclude:
+          # SPARK-33772: Spark supports JDK 17 since 3.3.0
+          - java: 17
+            spark: '3.1'
+          - java: 17
+            spark: '3.2'
     env:
       SPARK_LOCAL_IP: localhost
     steps:
@@ -77,10 +95,10 @@ jobs:
           java-version: ${{ matrix.java }}
           cache: 'maven'
           check-latest: false
-      - name: Setup Maven and Mvnd
-        uses: ./.github/actions/setup-mvnd
       - name: Cache Engine Archives
         uses: ./.github/actions/cache-engine-archives
+      - name: Setup Maven
+        uses: ./.github/actions/setup-maven
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
@@ -93,7 +111,7 @@ jobs:
       - name: Code coverage
         if: |
           matrix.java == 8 &&
-          matrix.spark == '3.2' &&
+          matrix.spark == '3.4' &&
           matrix.spark-archive == ''
         uses: codecov/codecov-action@v3
         with:
@@ -133,8 +151,8 @@ jobs:
           java-version: ${{ matrix.java }}
           cache: 'maven'
           check-latest: false
-      - name: Setup Maven and Mvnd
-        uses: ./.github/actions/setup-mvnd
+      - name: Setup Maven
+        uses: ./.github/actions/setup-maven
       - name: Cache Engine Archives
         uses: ./.github/actions/cache-engine-archives
       - name: Build and test Kyuubi AuthZ with supported Spark versions
@@ -151,6 +169,38 @@ jobs:
             **/target/unit-tests.log
             **/kyuubi-spark-sql-engine.log*
 
+  scala213:
+    name: Scala Compilation Test
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        java:
+          - '8'
+        scala:
+          - '2.13'
+    steps:
+      - uses: actions/checkout@v3
+      - name: Tune Runner VM
+        uses: ./.github/actions/tune-runner-vm
+      - name: Setup JDK ${{ matrix.java }}
+        uses: actions/setup-java@v3
+        with:
+          distribution: temurin
+          java-version: ${{ matrix.java }}
+          cache: 'maven'
+          check-latest: false
+      - name: Setup Maven
+        uses: ./.github/actions/setup-maven
+      - name: Build on Scala ${{ matrix.scala }}
+        run: |
+          MODULES='!externals/kyuubi-flink-sql-engine'
+          ./build/mvn clean install -pl ${MODULES} -am \
+          -DskipTests -Pflink-provided,hive-provided,spark-provided \
+          -Pjava-${{ matrix.java }} \
+          -Pscala-${{ matrix.scala }} \
+          -Pspark-3.3
+
   flink-it:
     name: Flink Test
     runs-on: ubuntu-22.04
@@ -161,15 +211,15 @@ jobs:
           - 8
           - 11
         flink:
-          - '1.15'
           - '1.16'
+          - '1.17'
         flink-archive: [ "" ]
         comment: [ "normal" ]
         include:
           - java: 8
-            flink: '1.16'
-            flink-archive: '-Dflink.archive.mirror=https://archive.apache.org/dist/flink/flink-1.15.4 -Dflink.archive.name=flink-1.15.4-bin-scala_2.12.tgz'
-            comment: 'verify-on-flink-1.15-binary'
+            flink: '1.17'
+            flink-archive: '-Dflink.archive.mirror=https://archive.apache.org/dist/flink/flink-1.16.1 -Dflink.archive.name=flink-1.16.1-bin-scala_2.12.tgz'
+            comment: 'verify-on-flink-1.16-binary'
     steps:
       - uses: actions/checkout@v3
       - name: Tune Runner VM
@@ -181,8 +231,8 @@ jobs:
           java-version: ${{ matrix.java }}
           cache: 'maven'
           check-latest: false
-      - name: Setup Maven and Mvnd
-        uses: ./.github/actions/setup-mvnd
+      - name: Setup Maven
+        uses: ./.github/actions/setup-maven
       - name: Cache Engine Archives
         uses: ./.github/actions/cache-engine-archives
       - name: Build Flink with maven w/o linters
@@ -229,8 +279,8 @@ jobs:
           java-version: ${{ matrix.java }}
           cache: 'maven'
           check-latest: false
-      - name: Setup Maven and Mvnd
-        uses: ./.github/actions/setup-mvnd
+      - name: Setup Maven
+        uses: ./.github/actions/setup-maven
       - name: Cache Engine Archives
         uses: ./.github/actions/cache-engine-archives
       - name: Build and test Hive with maven w/o linters
@@ -268,8 +318,8 @@ jobs:
           java-version: ${{ matrix.java }}
           cache: 'maven'
           check-latest: false
-      - name: Setup Maven and Mvnd
-        uses: ./.github/actions/setup-mvnd
+      - name: Setup Maven
+        uses: ./.github/actions/setup-maven
       - name: Cache Engine Archives
         uses: ./.github/actions/cache-engine-archives
       - name: Build and test JDBC with maven w/o linters
@@ -307,8 +357,8 @@ jobs:
           java-version: ${{ matrix.java }}
           cache: 'maven'
           check-latest: false
-      - name: Setup Maven and Mvnd
-        uses: ./.github/actions/setup-mvnd
+      - name: Setup Maven
+        uses: ./.github/actions/setup-maven
       - name: Cache Engine Archives
         uses: ./.github/actions/cache-engine-archives
       - name: Build and test Trino with maven w/o linters
@@ -341,8 +391,8 @@ jobs:
           java-version: 8
           cache: 'maven'
           check-latest: false
-      - name: Setup Maven and Mvnd
-        uses: ./.github/actions/setup-mvnd
+      - name: Setup Maven
+        uses: ./.github/actions/setup-maven
       - name: Cache Engine Archives
         uses: ./.github/actions/cache-engine-archives
       - name: Run TPC-DS Tests
@@ -480,8 +530,8 @@ jobs:
           java-version: ${{ matrix.java }}
           cache: 'maven'
           check-latest: false
-      - name: Setup Maven and Mvnd
-        uses: ./.github/actions/setup-mvnd
+      - name: Setup Maven
+        uses: ./.github/actions/setup-maven
       - name: Cache Engine Archives
         uses: ./.github/actions/cache-engine-archives
       - name: zookeeper integration tests

.github/workflows/nightly.yml

@@ -43,8 +43,8 @@ jobs:
           java-version: 8
           cache: 'maven'
           check-latest: false
-      - name: Setup Maven and Mvnd
-        uses: ./.github/actions/setup-mvnd
+      - name: Setup Maven
+        uses: ./.github/actions/setup-maven
       - name: Build with Maven
         run: ./build/mvn clean install ${{ matrix.profiles }} -Dmaven.javadoc.skip=true -V
       - name: Upload test logs