[TEST] [DELTA] Refine CI and add test detla 1.0.0 with Spark 3.1

.github/workflows/master.yml

@@ -34,11 +34,9 @@ jobs:
       matrix:
         profiles:
           - ''
-          - '-Pspark-3.0 -Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.1.1 -Dspark.archive.name=spark-3.1.1-bin-hadoop2.7.tgz -Dmaven.plugin.scalatest.exclude.tags=org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DataLakeTest'
+          - '-Pspark-3.0 -Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.1.1 -Dspark.archive.name=spark-3.1.1-bin-hadoop2.7.tgz -Dmaven.plugin.scalatest.exclude.tags=org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest'
           - '-Pspark-3.1 -Pkyuubi-extension-spark_3.1'
-          - '-Pspark-3.1 -Pkyuubi-extension-spark_3.1 -Pspark-hadoop-3.2'
-          - '-Pspark-3.2-snapshot -pl :kyuubi-spark-sql-engine,:kyuubi-common,:kyuubi-ha,:kyuubi-zookeeper -Dmaven.plugin.scalatest.exclude.tags=org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DataLakeTest'
-          - '-DwildcardSuites=org.apache.kyuubi.operation.tpcds.TPCDSOutputSchemaSuite,org.apache.kyuubi.operation.tpcds.TPCDSDDLSuite -Dmaven.plugin.scalatest.exclude.tags=""'
+          - '-Pspark-master -pl :kyuubi-spark-sql-engine,:kyuubi-common,:kyuubi-ha,:kyuubi-zookeeper'
     env:
       SPARK_LOCAL_IP: localhost
     steps:
@@ -72,16 +70,12 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-maven-io-
       - name: Build with Maven
-        if: ${{ ! contains(matrix.profiles, 'wildcardSuites') }}
-        run: mvn clean install ${{ matrix.profiles }} -Dmaven.javadoc.skip=true -V
-      - name: Wild Card Suites Tests
-        if: ${{ contains(matrix.profiles, 'wildcardSuites') }}
-        run: mvn clean install -DskipTests -pl :kyuubi-spark-sql-engine,:kyuubi-common,:kyuubi-ha,:kyuubi-zookeeper && mvn -Dtest=none ${{ matrix.profiles }} test
+        run: mvn clean install --no-transfer-progress ${{ matrix.profiles }} -Dmaven.javadoc.skip=true -V
       - name: Code coverage
         if: ${{ matrix.profiles == '' }}
         run: bash <(curl -s https://codecov.io/bash)
       - name: Detected Dependency List Change
-        if: ${{ ! contains(matrix.profiles, 'spark-3.2-snapshot') && ! contains(matrix.profiles, 'wildcardSuites') }}
+        if: ${{ ! contains(matrix.profiles, 'spark-master') }}
         run: build/dependency.sh
       - name: Upload test logs
         if: failure()
@@ -91,3 +85,43 @@ jobs:
           path: |
             **/target/unit-tests.log
             **/kyuubi-spark-sql-engine.log*
+
+  tpcds:
+    name: TPC-DS Schema Tests
+    runs-on: ubuntu-latest
+    env:
+      SPARK_LOCAL_IP: localhost
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup JDK 1.8
+        uses: actions/setup-java@v1
+        with:
+          java-version: '1.8'
+      - uses: actions/cache@v1
+        with:
+          path: ~/.m2/repository/com
+          key: ${{ runner.os }}-maven-com-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-com-
+      - uses: actions/cache@v1
+        with:
+          path: ~/.m2/repository/org
+          key: ${{ runner.os }}-maven-org-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-org-
+      - uses: actions/cache@v1
+        with:
+          path: ~/.m2/repository/net
+          key: ${{ runner.os }}-maven-net-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-net-
+      - uses: actions/cache@v1
+        with:
+          path: ~/.m2/repository/io
+          key: ${{ runner.os }}-maven-io-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-io-
+      - name: Run TPC-DS Schema Tests
+        run: |
+          mvn clean install --no-transfer-progress -Pspark-3.0 -DskipTests -pl :kyuubi-spark-sql-engine,:kyuubi-common,:kyuubi-ha,:kyuubi-zookeeper
+          mvn test --no-transfer-progress -Pspark-3.0 -Dtest=none -DwildcardSuites=*TPCDSSuite

.travis.yml

@@ -31,36 +31,45 @@ matrix:
     - name: Test Kyuubi w/ Default Profile
       env:
         - PROFILE=""
-        - EXCLUDE_TAGS="org.apache.kyuubi.tags.ExtendedSQLTest"
+        - JOB="build"
+        - CODECOV="true"
     - name: Test Kyuubi w/ -Pspark-3.0 and Spark 3.1 binary
       env:
-        - PROFILE="-Pspark-3.0 -Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.1.1 -Dspark.archive.name=spark-3.1.1-bin-hadoop2.7.tgz"
-        - EXCLUDE_TAGS="org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DataLakeTest"
+        - PROFILE="-Pspark-3.0 -Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.1.1 -Dspark.archive.name=spark-3.1.1-bin-hadoop2.7.tgz -Dmaven.plugin.scalatest.exclude.tags=org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest"
+        - JOB="build"
     - name: Test Kyuubi w/ -Pspark-3.1 -Pkyuubi-extension-spark_3.1
       env:
         - PROFILE="-Pspark-3.1 -Pkyuubi-extension-spark_3.1"
-        - EXCLUDE_TAGS="org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DataLakeTest"
-    - name: Test Kyuubi w/ -Pspark-3.1 -Pspark-hadoop-3.2 -Pkyuubi-extension-spark_3.1
+        - JOB="build"
+    - name: Test Kyuubi w/ -Pspark-master w/ Spark 3.2 nightly build
       env:
-        - PROFILE="-Pspark-3.1 -Dspark-hadoop-3.2 -Pkyuubi-extension-spark_3.1"
-        - EXCLUDE_TAGS="org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DataLakeTest"
-    - name: Test Kyuubi w/ -Pspark-3.2-snapshot w/ Spark 3.2 nightly build
-      env:
-        - PROFILE="-Pspark-3.2-snapshot -pl :kyuubi-spark-sql-engine,:kyuubi-common,:kyuubi-ha,:kyuubi-zookeeper"
-        - EXCLUDE_TAGS="org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DataLakeTest"
+        - PROFILE="-Pspark-master"
+        - JOB="nightly-build"
     - name: Test Kyuubi w/ -Pspark-3.0 TPCDS Tests
       env:
-        - TPCDS_TESTS=true
+        - PROFILE="-Pspark-3.0"
+        - JOB="tpcds-tests"
 
 install:
   - mvn --version
 
 script:
-  - if [[ "$TPCDS_TESTS" == "true" ]]; then mvn clean install -DskipTests -pl :kyuubi-spark-sql-engine,:kyuubi-common,:kyuubi-ha,:kyuubi-zookeeper && mvn -Dtest=none -DwildcardSuites=org.apache.kyuubi.operation.tpcds.TPCDSOutputSchemaSuite,org.apache.kyuubi.operation.tpcds.TPCDSDDLSuite -Dmaven.plugin.scalatest.exclude.tags="" test; fi
-  - if [[ "$TPCDS_TESTS" != "true" ]]; then mvn clean install $PROFILE -Dmaven.plugin.scalatest.exclude.tags=$EXCLUDE_TAGS -Dmaven.javadoc.skip=true -V ;fi
+  - |
+    if [[ "$JOB" == "build" ]]; then
+      mvn clean install --no-transfer-progress $PROFILE -Dmaven.javadoc.skip=true -V
+    fi
+  - |
+    if [[ "$JOB" == "nightly-build" ]]; then
+      mvn clean install --no-transfer-progress $PROFILE -pl :kyuubi-spark-sql-engine,:kyuubi-common,:kyuubi-ha,:kyuubi-zookeeper -Dmaven.javadoc.skip=true -V
+    fi
+  - |
+    if [[ "$JOB" == "tpcds-tests" ]]; then
+      mvn clean install --no-transfer-progress $PROFILE -DskipTests -pl :kyuubi-spark-sql-engine,:kyuubi-common,:kyuubi-ha,:kyuubi-zookeeper
+      mvn test --no-transfer-progress $PROFILE -Dtest=none -DwildcardSuites=*TPCDSSuite
+    fi
 
 after_success:
-  - if [[ -z $EXCLUDE_TAGS ]]; then bash <(curl -s https://codecov.io/bash); fi
+  - if [[ "$CODECOV" == "true" ]]; then bash <(curl -s https://codecov.io/bash); fi
 
 after_failure:
   - echo "==== Build with $PROFILE FAILED ===="

externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkDeltaOperationSuite.scala

@@ -19,9 +19,9 @@ package org.apache.kyuubi.engine.spark.operation
 
 import org.apache.kyuubi.engine.spark.WithSparkSQLEngine
 import org.apache.kyuubi.operation.BasicDeltaJDBCTests
-import org.apache.kyuubi.tags.DataLakeTest
+import org.apache.kyuubi.tags.DeltaTest
 
-@DataLakeTest
+@DeltaTest
 class SparkDeltaOperationSuite extends WithSparkSQLEngine with BasicDeltaJDBCTests {
   override protected def jdbcUrl: String = getJdbcUrl
   override def withKyuubiConf: Map[String, String] = extraConfigs

externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkIcebergOperationSuite.scala

@@ -19,9 +19,9 @@ package org.apache.kyuubi.engine.spark.operation
 
 import org.apache.kyuubi.engine.spark.WithSparkSQLEngine
 import org.apache.kyuubi.operation.BasicIcebergJDBCTests
-import org.apache.kyuubi.tags.DataLakeTest
+import org.apache.kyuubi.tags.IcebergTest
 
-@DataLakeTest
+@IcebergTest
 class SparkIcebergOperationSuite extends WithSparkSQLEngine with BasicIcebergJDBCTests {
   override protected def jdbcUrl: String = getJdbcUrl
   override def withKyuubiConf: Map[String, String] = extraConfigs

kyuubi-common/src/test/java/org/apache/kyuubi/tags/DataLakeTest.java → kyuubi-common/src/test/java/org/apache/kyuubi/tags/DeltaTest.java

@@ -27,4 +27,4 @@
 @TagAnnotation
 @Retention(RetentionPolicy.RUNTIME)
 @Target({ElementType.METHOD, ElementType.TYPE})
-public @interface DataLakeTest {}
+public @interface DeltaTest {}

kyuubi-common/src/test/java/org/apache/kyuubi/tags/IcebergTest.java

@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kyuubi.tags;
+
+import org.scalatest.TagAnnotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@TagAnnotation
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.METHOD, ElementType.TYPE})
+public @interface IcebergTest {}

kyuubi-main/src/test/scala/org/apache/kyuubi/operation/datalake/DeltaOperationSuite.scala

@@ -20,9 +20,9 @@ package org.apache.kyuubi.operation.datalake
 import org.apache.kyuubi.WithKyuubiServer
 import org.apache.kyuubi.config.KyuubiConf
 import org.apache.kyuubi.operation.BasicDeltaJDBCTests
-import org.apache.kyuubi.tags.DataLakeTest
+import org.apache.kyuubi.tags.DeltaTest
 
-@DataLakeTest
+@DeltaTest
 class DeltaOperationSuite extends WithKyuubiServer with BasicDeltaJDBCTests {
   override protected val conf: KyuubiConf = {
     val kyuubiConf = KyuubiConf().set(KyuubiConf.ENGINE_IDLE_TIMEOUT, 20000L)

kyuubi-main/src/test/scala/org/apache/kyuubi/operation/datalake/IcebergOperationSuite.scala

@@ -20,9 +20,9 @@ package org.apache.kyuubi.operation.datalake
 import org.apache.kyuubi.WithKyuubiServer
 import org.apache.kyuubi.config.KyuubiConf
 import org.apache.kyuubi.operation.BasicIcebergJDBCTests
-import org.apache.kyuubi.tags.DataLakeTest
+import org.apache.kyuubi.tags.IcebergTest
 
-@DataLakeTest
+@IcebergTest
 class IcebergOperationSuite extends WithKyuubiServer with BasicIcebergJDBCTests {
   override protected val conf: KyuubiConf = {
     val kyuubiConf = KyuubiConf().set(KyuubiConf.ENGINE_IDLE_TIMEOUT, 20000L)

kyuubi-main/src/test/scala/org/apache/kyuubi/operation/tpcds/TPCDSDDLSuite.scala → kyuubi-main/src/test/scala/org/apache/kyuubi/operation/tpcds/DDLTPCDSSuite.scala

@@ -23,7 +23,7 @@ import org.apache.kyuubi.operation.JDBCTestUtils
 import org.apache.kyuubi.tags.ExtendedSQLTest
 
 @ExtendedSQLTest
-class TPCDSDDLSuite extends WithKyuubiServer with JDBCTestUtils with TPCDSHelper {
+class DDLTPCDSSuite extends WithKyuubiServer with JDBCTestUtils with TPCDSHelper {
   override protected val conf: KyuubiConf = KyuubiConf()
 
   override protected def jdbcUrl: String = getJdbcUrl

kyuubi-main/src/test/scala/org/apache/kyuubi/operation/tpcds/TPCDSOutputSchemaSuite.scala → kyuubi-main/src/test/scala/org/apache/kyuubi/operation/tpcds/OutputSchemaTPCDSSuite.scala

@@ -29,17 +29,17 @@ import org.apache.kyuubi.tags.ExtendedSQLTest
 /**
  * To run this test suite:
  * {{{
- *   build/mvn -Dtest=none -DwildcardSuites=org.apache.kyuubi.operation.tpcds.TPCDSOutputSchemaSuite -Dmaven.plugin.scalatest.exclude.tags="" test
- *  }}}
+ *   build/mvn test -Dtest=none -DwildcardSuites=org.apache.kyuubi.operation.tpcds.OutputSchemaTPCDSSuite
+ * }}}
  *
  * To re-generate golden files for this suite:
  * {{{
- *   KYUUBI_UPDATE=1 build/mvn -Dtest=none -DwildcardSuites=org.apache.kyuubi.operation.tpcds.TPCDSOutputSchemaSuite -Dmaven.plugin.scalatest.exclude.tags="" test
+ *   KYUUBI_UPDATE=1 build/mvn test -Dtest=none -DwildcardSuites=org.apache.kyuubi.operation.tpcds.OutputSchemaTPCDSSuite
  * }}}
  */
 // scalastyle:on line.size.limit
 @ExtendedSQLTest
-class TPCDSOutputSchemaSuite extends WithKyuubiServer with JDBCTestUtils with TPCDSHelper {
+class OutputSchemaTPCDSSuite extends WithKyuubiServer with JDBCTestUtils with TPCDSHelper {
   override protected val conf: KyuubiConf = KyuubiConf()
   override protected def jdbcUrl: String = getJdbcUrl
   override def database: String = this.getClass.getSimpleName

pom.xml

@@ -1104,6 +1104,20 @@
                 <groupId>io.delta</groupId>
                 <artifactId>delta-core_${scala.binary.version}</artifactId>
                 <version>${delta.version}</version>
+                <exclusions>
+                    <!--
+                     Apache Spark already has Antlr and Scala dependencies, and a potential compatibility
+                     issue of Antlr version on delta 1.0.0: https://github.com/delta-io/delta/pull/676
+                     -->
+                    <exclusion>
+                        <groupId>org.antlr</groupId>
+                        <artifactId>*</artifactId>
+                    </exclusion>
+                    <exclusion>
+                        <groupId>org.scala-lang</groupId>
+                        <artifactId>scala-library</artifactId>
+                    </exclusion>
+                </exclusions>
             </dependency>
         </dependencies>
     </dependencyManagement>
@@ -1550,20 +1564,27 @@
 
         <profile>
             <id>spark-3.0</id>
+            <properties>
+                <spark.version>3.0.2</spark.version>
+                <delta.version>0.8.0</delta.version>
+                <maven.plugin.scalatest.exclude.tags></maven.plugin.scalatest.exclude.tags>
+            </properties>
         </profile>
 
         <profile>
-            <id>spark-3.2-snapshot</id>
+            <id>spark-3.1</id>
             <properties>
-                <spark.version>3.2.0-SNAPSHOT</spark.version>
+                <spark.version>3.1.1</spark.version>
+                <delta.version>1.0.0</delta.version>
+                <maven.plugin.scalatest.exclude.tags>org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.IcebergTest</maven.plugin.scalatest.exclude.tags>
             </properties>
         </profile>
 
         <profile>
-            <id>spark-3.1</id>
+            <id>spark-master</id>
             <properties>
-                <spark.version>3.1.1</spark.version>
-                <maven.plugin.scalatest.exclude.tags>org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DataLakeTest</maven.plugin.scalatest.exclude.tags>
+                <spark.version>3.2.0-SNAPSHOT</spark.version>
+                <maven.plugin.scalatest.exclude.tags>org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest</maven.plugin.scalatest.exclude.tags>
             </properties>
         </profile>