[KYUUBI #3239] [Subtask] DorisSQLEngine - Add integration tests

### _Why are the changes needed?_ close #3239 (comment) ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [ ] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request Closes #3240 from zhaomin1423/jdbc_it. Closes #3239 8da64f2 [Min] fix it 24cceba [Min] add dependency b477326 [Min] add server to dependency e74aa93 [Min] fix getJdbcUrl conflict 01e6a30 [Min] [KYUUBI #3239] [Subtask] DorisSQLEngine - Add integration tests Authored-by: Min <zhaomin1423@163.com> Signed-off-by: Cheng Pan <chengpan@apache.org>
apache · Aug 17, 2022 · 3cf5a20 · 3cf5a20
1 parent ecda418
commit 3cf5a20
Show file tree

Hide file tree

Showing 14 changed files with 265 additions and 8 deletions.
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
@@ -208,7 +208,7 @@ jobs:
           check-latest: false
       - name: Build and test JDBC with maven w/o linters
         run: |
-          TEST_MODULES="externals/kyuubi-jdbc-engine" || echo 'TODO integration-tests/kyuubi-jdbc-it'
+          TEST_MODULES="externals/kyuubi-jdbc-engine,integration-tests/kyuubi-jdbc-it"
           ./build/mvn ${MVN_OPT} -pl ${TEST_MODULES} -am clean install -DskipTests
           ./build/mvn ${MVN_OPT} -pl ${TEST_MODULES} test
       - name: Upload test logs

diff --git a/...nals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/WithJdbcEngine.scala b/...nals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/WithJdbcEngine.scala
@@ -55,6 +55,6 @@ trait WithJdbcEngine extends KyuubiFunSuite {
     connectionUrl = engine.frontendServices.head.connectionUrl
   }
 
-  protected def getJdbcUrl: String = s"jdbc:hive2://$connectionUrl/;"
+  protected def jdbcConnectionUrl: String = s"jdbc:hive2://$connectionUrl/;"
 
 }
diff --git a/...-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/doris/DorisOperationSuite.scala b/...-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/doris/DorisOperationSuite.scala
@@ -23,7 +23,7 @@ import scala.collection.mutable.ArrayBuffer
 import org.apache.kyuubi.operation.HiveJDBCTestHelper
 import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant._
 
-class DorisOperationSuite extends WithDorisEngine with HiveJDBCTestHelper {
+abstract class DorisOperationSuite extends WithDorisEngine with HiveJDBCTestHelper {
   test("doris - get tables") {
     case class Table(catalog: String, schema: String, tableName: String, tableType: String)
 
@@ -258,6 +258,4 @@ class DorisOperationSuite extends WithDorisEngine with HiveJDBCTestHelper {
       statement.execute("drop database db2")
     }
   }
-
-  override protected def jdbcUrl: String = getJdbcUrl
 }
diff --git a/...-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/doris/OperationWithEngineSuite.scala b/...-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/doris/OperationWithEngineSuite.scala
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kyuubi.engine.jdbc.doris
+
+import org.apache.kyuubi.operation.HiveJDBCTestHelper
+
+class OperationWithEngineSuite extends DorisOperationSuite with HiveJDBCTestHelper {
+
+  override protected def jdbcUrl: String = jdbcConnectionUrl
+}
diff --git a/.../kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/doris/SessionSuite.scala b/.../kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/doris/SessionSuite.scala
@@ -35,5 +35,5 @@ class SessionSuite extends WithDorisEngine with HiveJDBCTestHelper {
     }
   }
 
-  override protected def jdbcUrl: String = getJdbcUrl
+  override protected def jdbcUrl: String = jdbcConnectionUrl
 }
diff --git a/...yuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/doris/StatementSuite.scala b/...yuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/doris/StatementSuite.scala
@@ -90,5 +90,5 @@ class StatementSuite extends WithDorisEngine with HiveJDBCTestHelper {
     }
   }
 
-  override protected def jdbcUrl: String = getJdbcUrl
+  override protected def jdbcUrl: String = jdbcConnectionUrl
 }
diff --git a/...uubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/doris/WithDorisEngine.scala b/...uubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/doris/WithDorisEngine.scala
@@ -27,5 +27,6 @@ trait WithDorisEngine extends WithJdbcEngine with WithDorisContainer {
     ENGINE_JDBC_CONNECTION_USER.key -> "root",
     ENGINE_JDBC_CONNECTION_PASSWORD.key -> "",
     ENGINE_TYPE.key -> "jdbc",
-    ENGINE_JDBC_SHORT_NAME.key -> "doris")
+    ENGINE_JDBC_SHORT_NAME.key -> "doris",
+    ENGINE_JDBC_DRIVER_CLASS.key -> "com.mysql.cj.jdbc.Driver")
 }
diff --git a/integration-tests/kyuubi-jdbc-it/pom.xml b/integration-tests/kyuubi-jdbc-it/pom.xml
@@ -0,0 +1,85 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>integration-tests</artifactId>
+        <groupId>org.apache.kyuubi</groupId>
+        <version>1.6.0-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>kyuubi-jdbc-it_2.12</artifactId>
+    <name>Kyuubi Test Jdbc IT</name>
+    <url>https://kyuubi.apache.org/</url>
+
+    <dependencies>
+
+        <dependency>
+            <groupId>org.apache.kyuubi</groupId>
+            <artifactId>kyuubi-common_${scala.binary.version}</artifactId>
+            <version>${project.version}</version>
+            <type>test-jar</type>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.kyuubi</groupId>
+            <artifactId>kyuubi-jdbc-engine_${scala.binary.version}</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.kyuubi</groupId>
+            <artifactId>kyuubi-jdbc-engine_${scala.binary.version}</artifactId>
+            <version>${project.version}</version>
+            <type>test-jar</type>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.kyuubi</groupId>
+            <artifactId>kyuubi-server_${scala.binary.version}</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.kyuubi</groupId>
+            <artifactId>kyuubi-server_${scala.binary.version}</artifactId>
+            <version>${project.version}</version>
+            <type>test-jar</type>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.kyuubi</groupId>
+            <artifactId>kyuubi-hive-jdbc-shaded</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>com.dimafeng</groupId>
+            <artifactId>testcontainers-scala-scalatest_${scala.binary.version}</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+    </dependencies>
+
+</project>
diff --git a/integration-tests/kyuubi-jdbc-it/src/test/resources/doris-compose.yml b/integration-tests/kyuubi-jdbc-it/src/test/resources/doris-compose.yml
@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+version: "3"
+services:
+  doris-fe:
+    # https://hub.docker.com/repository/docker/zhaomin1423/doris-fe
+    image: zhaomin1423/doris-fe:1.0.0
+    ports:
+      - "9030"
+
+  doris-be:
+    # https://hub.docker.com/repository/docker/zhaomin1423/doris-be
+    image: zhaomin1423/doris-be:1.0.0-0.1
+    ports:
+      - "8040"
+    depends_on:
+      - doris-fe
+    entrypoint: /opt/entrypoint.sh
+
+
diff --git a/integration-tests/kyuubi-jdbc-it/src/test/resources/log4j2-test.xml b/integration-tests/kyuubi-jdbc-it/src/test/resources/log4j2-test.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<!-- Extra logging related to initialization of Log4j. 
+ Set to debug or trace if log4j initialization is failing. -->
+<Configuration status="WARN">
+    <Appenders>
+        <Console name="stdout" target="SYSTEM_OUT">
+            <PatternLayout pattern="%d{HH:mm:ss.SSS} %p %c: %m%n"/>
+            <Filters>
+                <ThresholdFilter level="FATAL"/>
+                <RegexFilter regex=".*Thrift error occurred during processing of message.*" onMatch="DENY" onMismatch="NEUTRAL"/>
+            </Filters>
+        </Console>
+        <File name="file" fileName="target/unit-tests.log">
+            <PatternLayout pattern="%d{HH:mm:ss.SSS} %t %p %c{1}: %m%n"/>
+            <Filters>
+                <RegexFilter regex=".*Thrift error occurred during processing of message.*" onMatch="DENY" onMismatch="NEUTRAL"/>
+            </Filters>
+        </File>
+    </Appenders>
+    <Loggers>
+        <Root level="INFO">
+            <AppenderRef ref="stdout"/>
+            <AppenderRef ref="file"/>
+        </Root>
+    </Loggers>
+</Configuration>
diff --git a/integration-tests/kyuubi-jdbc-it/src/test/resources/mysql/mysql-connector-java-8.0.30.jar b/integration-tests/kyuubi-jdbc-it/src/test/resources/mysql/mysql-connector-java-8.0.30.jar
diff --git a/...ubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/doris/OperationWithServerSuite.scala b/...ubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/doris/OperationWithServerSuite.scala
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kyuubi.it.jdbc.doris
+
+import org.apache.kyuubi.engine.jdbc.doris.DorisOperationSuite
+
+class OperationWithServerSuite extends DorisOperationSuite with WithKyuubiServerAndDorisContainer {
+
+  override protected def jdbcUrl: String = getJdbcUrl
+
+}
diff --git a/...it/src/test/scala/org/apache/kyuubi/it/jdbc/doris/WithKyuubiServerAndDorisContainer.scala b/...it/src/test/scala/org/apache/kyuubi/it/jdbc/doris/WithKyuubiServerAndDorisContainer.scala
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kyuubi.it.jdbc.doris
+
+import org.apache.kyuubi.{Utils, WithKyuubiServer}
+import org.apache.kyuubi.config.KyuubiConf
+import org.apache.kyuubi.config.KyuubiConf.{ENGINE_JDBC_EXTRA_CLASSPATH, KYUUBI_ENGINE_ENV_PREFIX, KYUUBI_HOME}
+import org.apache.kyuubi.engine.jdbc.doris.WithDorisEngine
+
+trait WithKyuubiServerAndDorisContainer extends WithKyuubiServer with WithDorisEngine {
+
+  private val kyuubiHome: String = Utils
+    .getCodeSourceLocation(getClass).split("integration-tests").head
+
+  override protected val conf: KyuubiConf = {
+    KyuubiConf()
+      .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME", kyuubiHome)
+      .set(
+        ENGINE_JDBC_EXTRA_CLASSPATH,
+        getClass.getClassLoader.getResource(
+          "mysql/mysql-connector-java-8.0.30.jar").toURI.getPath)
+  }
+
+  override def beforeAll(): Unit = {
+    val configs = withKyuubiConf
+    configs.foreach(config => conf.set(config._1, config._2))
+    super.beforeAll()
+  }
+}
diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml
@@ -35,6 +35,7 @@
         <module>kyuubi-flink-it</module>
         <module>kyuubi-hive-it</module>
         <module>kyuubi-trino-it</module>
+        <module>kyuubi-jdbc-it</module>
     </modules>
 
     <profiles>