From b02cc4d7f2fd42033d339b43589bdd61030e260e Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Thu, 5 Sep 2019 13:31:09 +0800 Subject: [PATCH 01/16] Revert "[SPARK-27831][SQL][TEST] Move Hive test jars to maven dependency" This reverts commit d53b61c3 --- pom.xml | 69 ------------------ sql/hive-thriftserver/pom.xml | 9 --- .../sql/hive/thriftserver/CliSuite.scala | 16 ++-- .../HiveThriftServer2Suites.scala | 6 +- sql/hive/pom.xml | 8 -- .../apache/spark/sql/hive/test/TestHive.scala | 13 ++++ .../test/resources/hive-contrib-0.13.1.jar | Bin 0 -> 114878 bytes .../src/test/resources/hive-contrib-2.3.5.jar | Bin 0 -> 125843 bytes .../resources/hive-hcatalog-core-0.13.1.jar | Bin 0 -> 468533 bytes .../resources/hive-hcatalog-core-2.3.5.jar | Bin 0 -> 263956 bytes .../spark/sql/hive/HiveSparkSubmitSuite.scala | 14 ++-- .../sql/hive/execution/HiveQuerySuite.scala | 8 +- .../sql/hive/execution/SQLQuerySuite.scala | 4 +- .../spark/sql/hive/test/HiveTestUtils.scala | 32 -------- 14 files changed, 41 insertions(+), 138 deletions(-) rename sql/hive/src/{test => main}/scala/org/apache/spark/sql/hive/test/TestHive.scala (98%) create mode 100644 sql/hive/src/test/resources/hive-contrib-0.13.1.jar create mode 100644 sql/hive/src/test/resources/hive-contrib-2.3.5.jar create mode 100644 sql/hive/src/test/resources/hive-hcatalog-core-0.13.1.jar create mode 100644 sql/hive/src/test/resources/hive-hcatalog-core-2.3.5.jar delete mode 100644 sql/hive/src/test/scala/org/apache/spark/sql/hive/test/HiveTestUtils.scala diff --git a/pom.xml b/pom.xml index 6544b502af063..741c7598f6811 100644 --- a/pom.xml +++ b/pom.xml @@ -2000,75 +2000,6 @@ - - ${hive.group} - hive-contrib - ${hive.version} - test - - - ${hive.group} - hive-exec - - - ${hive.group} - hive-serde - - - ${hive.group} - hive-shims - - - commons-codec - commons-codec - - - org.slf4j - slf4j-api - - - - - ${hive.group}.hcatalog - hive-hcatalog-core - ${hive.version} - test - - - ${hive.group} - hive-exec - - - ${hive.group} - hive-metastore - - - ${hive.group} - hive-cli - - - ${hive.group} - hive-common - - - com.google.guava - guava - - - org.slf4j - slf4j-api - - - org.codehaus.jackson - jackson-mapper-asl - - - org.apache.hadoop - * - - - - org.apache.orc orc-core diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml index 5b1352adddd89..9dd927084298a 100644 --- a/sql/hive-thriftserver/pom.xml +++ b/sql/hive-thriftserver/pom.xml @@ -77,15 +77,6 @@ ${hive.group} hive-beeline - - - ${hive.group} - hive-contrib - - - ${hive.group}.hcatalog - hive-hcatalog-core - org.eclipse.jetty jetty-server diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala index 6e042ac41d9da..a1e611e122401 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala @@ -27,12 +27,11 @@ import scala.concurrent.Promise import scala.concurrent.duration._ import org.apache.hadoop.hive.conf.HiveConf.ConfVars -import org.apache.hadoop.hive.contrib.udaf.example.UDAFExampleMax import org.scalatest.BeforeAndAfterAll import org.apache.spark.SparkFunSuite import org.apache.spark.internal.Logging -import org.apache.spark.sql.hive.test.HiveTestUtils +import org.apache.spark.sql.hive.HiveUtils import org.apache.spark.sql.test.ProcessTestUtils.ProcessOutputCapturer import org.apache.spark.util.{ThreadUtils, Utils} @@ -202,7 +201,10 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging { } test("Commands using SerDe provided in --jars") { - val jarFile = HiveTestUtils.getHiveHcatalogCoreJar.getCanonicalPath + val jarFile = "../hive/src/test/resources/" + + s"hive-hcatalog-core-${if (HiveUtils.isHive23) "2.3.5" else "0.13.1"}.jar" + .split("/") + .mkString(File.separator) val dataFilePath = Thread.currentThread().getContextClassLoader.getResource("data/files/small_kv.txt") @@ -297,11 +299,15 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging { } test("Support hive.aux.jars.path") { - val hiveContribJar = HiveTestUtils.getHiveContribJar.getCanonicalPath + val hiveContribJar = "../hive/src/test/resources/" + + s"hive-contrib-${if (HiveUtils.isHive23) "2.3.5" else "0.13.1"}.jar" + .split("/") + .mkString(File.separator) runCliWithin( 1.minute, Seq("--conf", s"spark.hadoop.${ConfVars.HIVEAUXJARS}=$hiveContribJar"))( - s"CREATE TEMPORARY FUNCTION example_max AS '${classOf[UDAFExampleMax].getName}';" -> "", + "CREATE TEMPORARY FUNCTION example_max AS " + + "'org.apache.hadoop.hive.contrib.udaf.example.UDAFExampleMax';" -> "", "SELECT example_max(1);" -> "1" ) } diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala index b7185db2f2ae7..3dd4455698250 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala @@ -43,7 +43,6 @@ import org.scalatest.BeforeAndAfterAll import org.apache.spark.{SparkException, SparkFunSuite} import org.apache.spark.internal.Logging import org.apache.spark.sql.hive.HiveUtils -import org.apache.spark.sql.hive.test.HiveTestUtils import org.apache.spark.sql.internal.StaticSQLConf.HIVE_THRIFT_SERVER_SINGLESESSION import org.apache.spark.sql.test.ProcessTestUtils.ProcessOutputCapturer import org.apache.spark.util.{ThreadUtils, Utils} @@ -485,7 +484,10 @@ class HiveThriftBinaryServerSuite extends HiveThriftJdbcTest { withMultipleConnectionJdbcStatement("smallKV", "addJar")( { statement => - val jarFile = HiveTestUtils.getHiveHcatalogCoreJar.getCanonicalPath + val jarFile = "../hive/src/test/resources/" + + s"hive-hcatalog-core-${if (HiveUtils.isHive23) "2.3.5" else "0.13.1"}.jar" + .split("/") + .mkString(File.separator) statement.executeQuery(s"ADD JAR $jarFile") }, diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index d37f0c8573659..f627227aa0380 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -103,14 +103,6 @@ ${hive.group} hive-metastore - - ${hive.group} - hive-contrib - - - ${hive.group}.hcatalog - hive-hcatalog-core -