From d13efd28049c79d12b51312e450b5a9032e0fe6c Mon Sep 17 00:00:00 2001 From: Cheng Lian Date: Thu, 28 Aug 2014 19:37:41 -0700 Subject: [PATCH 1/2] Turns on in-memory columnar compression in HiveCompatibilitySuite --- .../spark/sql/hive/execution/HiveCompatibilitySuite.scala | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala index 035fd3214bd1d..2481edae48d66 100644 --- a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala +++ b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala @@ -22,6 +22,7 @@ import java.util.{Locale, TimeZone} import org.scalatest.BeforeAndAfter +import org.apache.spark.sql.SQLConf import org.apache.spark.sql.hive.test.TestHive /** @@ -34,6 +35,7 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter { var originalTimeZone: TimeZone = _ var originalLocale: Locale = _ + val originalUseCompression = TestHive.useCompression def testCases = hiveQueryDir.listFiles.map(f => f.getName.stripSuffix(".q") -> f) @@ -42,16 +44,19 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter { // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*) originalTimeZone = TimeZone.getDefault TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles")) - + // Add Locale setting originalLocale = Locale.getDefault Locale.setDefault(Locale.US) + + TestHive.setConf(SQLConf.COMPRESS_CACHED, "true") } override def afterAll() { TestHive.cacheTables = false TimeZone.setDefault(originalTimeZone) Locale.setDefault(originalLocale) + TestHive.setConf(SQLConf.COMPRESS_CACHED, originalUseCompression.toString) } /** A list of tests deemed out of scope currently and thus completely disregarded. */ From 88b536c5e969c2aefde585e3714ee5393c70bee6 Mon Sep 17 00:00:00 2001 From: Cheng Lian Date: Thu, 28 Aug 2014 19:42:08 -0700 Subject: [PATCH 2/2] Code cleanup, narrowed field visibility --- .../hive/execution/HiveCompatibilitySuite.scala | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala index 2481edae48d66..b589994bd25fa 100644 --- a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala +++ b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala @@ -30,25 +30,23 @@ import org.apache.spark.sql.hive.test.TestHive */ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter { // TODO: bundle in jar files... get from classpath - lazy val hiveQueryDir = TestHive.getHiveFile("ql" + File.separator + "src" + - File.separator + "test" + File.separator + "queries" + File.separator + "clientpositive") + private lazy val hiveQueryDir = TestHive.getHiveFile( + "ql/src/test/queries/clientpositive".split("/").mkString(File.separator)) - var originalTimeZone: TimeZone = _ - var originalLocale: Locale = _ - val originalUseCompression = TestHive.useCompression + private val originalTimeZone = TimeZone.getDefault + private val originalLocale = Locale.getDefault + private val originalUseCompression = TestHive.useCompression def testCases = hiveQueryDir.listFiles.map(f => f.getName.stripSuffix(".q") -> f) override def beforeAll() { + // Enable in-memory columnar caching TestHive.cacheTables = true // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*) - originalTimeZone = TimeZone.getDefault TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles")) - // Add Locale setting - originalLocale = Locale.getDefault Locale.setDefault(Locale.US) - + // Enable in-memory columnar compression TestHive.setConf(SQLConf.COMPRESS_CACHED, "true") }