From 37a758359d3271a8be8b2445b588283729833a4b Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Mon, 4 Jul 2016 11:39:45 +0530 Subject: [PATCH 01/25] alter_rename_partition added --- ...tition-0-40b47feca125adfc6a303ce445367cad} | 0 ...tition-1-4ddbd74888a207f46e231da902bc2cf1} | 0 ...ition-12-40b47feca125adfc6a303ce445367cad} | 0 ...ition-13-4ddbd74888a207f46e231da902bc2cf1} | 0 .../sql/hive/execution/HiveQuerySuite.scala | 47 +++++++++++++++++++ 5 files changed, 47 insertions(+) rename sql/hive/src/test/resources/golden/{alter_rename_partition-0-1ed18256c5230de3439fe75d925ea73 => alter_rename_partition-0-40b47feca125adfc6a303ce445367cad} (100%) rename sql/hive/src/test/resources/golden/{alter_rename_partition-1-2f79bceed6fc8ada34a670396ee6aada => alter_rename_partition-1-4ddbd74888a207f46e231da902bc2cf1} (100%) rename sql/hive/src/test/resources/golden/{alter_rename_partition-12-1ed18256c5230de3439fe75d925ea73 => alter_rename_partition-12-40b47feca125adfc6a303ce445367cad} (100%) rename sql/hive/src/test/resources/golden/{alter_rename_partition-13-2f79bceed6fc8ada34a670396ee6aada => alter_rename_partition-13-4ddbd74888a207f46e231da902bc2cf1} (100%) diff --git a/sql/hive/src/test/resources/golden/alter_rename_partition-0-1ed18256c5230de3439fe75d925ea73 b/sql/hive/src/test/resources/golden/alter_rename_partition-0-40b47feca125adfc6a303ce445367cad similarity index 100% rename from sql/hive/src/test/resources/golden/alter_rename_partition-0-1ed18256c5230de3439fe75d925ea73 rename to sql/hive/src/test/resources/golden/alter_rename_partition-0-40b47feca125adfc6a303ce445367cad diff --git a/sql/hive/src/test/resources/golden/alter_rename_partition-1-2f79bceed6fc8ada34a670396ee6aada b/sql/hive/src/test/resources/golden/alter_rename_partition-1-4ddbd74888a207f46e231da902bc2cf1 similarity index 100% rename from sql/hive/src/test/resources/golden/alter_rename_partition-1-2f79bceed6fc8ada34a670396ee6aada rename to sql/hive/src/test/resources/golden/alter_rename_partition-1-4ddbd74888a207f46e231da902bc2cf1 diff --git a/sql/hive/src/test/resources/golden/alter_rename_partition-12-1ed18256c5230de3439fe75d925ea73 b/sql/hive/src/test/resources/golden/alter_rename_partition-12-40b47feca125adfc6a303ce445367cad similarity index 100% rename from sql/hive/src/test/resources/golden/alter_rename_partition-12-1ed18256c5230de3439fe75d925ea73 rename to sql/hive/src/test/resources/golden/alter_rename_partition-12-40b47feca125adfc6a303ce445367cad diff --git a/sql/hive/src/test/resources/golden/alter_rename_partition-13-2f79bceed6fc8ada34a670396ee6aada b/sql/hive/src/test/resources/golden/alter_rename_partition-13-4ddbd74888a207f46e231da902bc2cf1 similarity index 100% rename from sql/hive/src/test/resources/golden/alter_rename_partition-13-2f79bceed6fc8ada34a670396ee6aada rename to sql/hive/src/test/resources/golden/alter_rename_partition-13-4ddbd74888a207f46e231da902bc2cf1 diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index f8c55ec45650e..489c2faaae88a 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -1212,6 +1212,53 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { } assertUnsupportedFeature { sql("DROP TEMPORARY MACRO SIGMOID") } } + + // scalastyle:off + createQueryTest("alter_rename_partition", + """ + |-- Cleanup + |DROP TABLE IF EXISTS alter_rename_partition_src; + |DROP TABLE IF EXISTS alter_rename_partition; + |SHOW TABLES; + | + |create table alter_rename_partition_src ( col1 string ) stored as textfile ; + |load data local inpath '../../data/files/test.dat' overwrite into table alter_rename_partition_src ; + | + |create table alter_rename_partition ( col1 string ) partitioned by (pcol1 string , pcol2 string) stored as sequencefile; + | + |insert overwrite table alter_rename_partition partition (pCol1='old_part1:', pcol2='old_part2:') select col1 from alter_rename_partition_src ; + |select * from alter_rename_partition where pcol1='old_part1:' and pcol2='old_part2:'; + | + |alter table alter_rename_partition partition (pCol1='old_part1:', pcol2='old_part2:') rename to partition (pCol1='new_part1:', pcol2='new_part2:'); + |SHOW PARTITIONS alter_rename_partition; + |select * from alter_rename_partition where pcol1='old_part1:' and pcol2='old_part2:'; + |select * from alter_rename_partition where pcol1='new_part1:' and pcol2='new_part2:'; + | + |-- Cleanup + |DROP TABLE IF EXISTS alter_rename_partition_src; + |DROP TABLE IF EXISTS alter_rename_partition; + |SHOW TABLES; + | + |-- With non-default Database + | + |CREATE DATABASE alter_rename_partition_db; + |USE alter_rename_partition_db; + |SHOW TABLES; + | + |CREATE TABLE alter_rename_partition_src (col1 STRING) STORED AS TEXTFILE ; + |LOAD DATA LOCAL INPATH '../../data/files/test.dat' OVERWRITE INTO TABLE alter_rename_partition_src ; + | + |CREATE TABLE alter_rename_partition (col1 STRING) PARTITIONED BY (pcol1 STRING, pcol2 STRING) STORED AS SEQUENCEFILE; + | + |INSERT OVERWRITE TABLE alter_rename_partition PARTITION (pCol1='old_part1:', pcol2='old_part2:') SELECT col1 FROM alter_rename_partition_src ; + |SELECT * FROM alter_rename_partition WHERE pcol1='old_part1:' AND pcol2='old_part2:'; + | + |ALTER TABLE alter_rename_partition PARTITION (pCol1='old_part1:', pcol2='old_part2:') RENAME TO PARTITION (pCol1='new_part1:', pcol2='new_part2:'); + |SHOW PARTITIONS alter_rename_partition; + |SELECT * FROM alter_rename_partition WHERE pcol1='old_part1:' and pcol2='old_part2:'; + |SELECT * FROM alter_rename_partition WHERE pcol1='new_part1:' and pcol2='new_part2:'; + """.stripMargin) + // scalastyle:on } // for SPARK-2180 test From 6b0e3c5b72d4ee7cd5215a7a8c884b6f221d7102 Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Mon, 4 Jul 2016 11:59:17 +0530 Subject: [PATCH 02/25] date_1 not working we need timestamp and date functions --- ...date_1-1-d4c25b848047acb9546eb2def9c33a01} | 0 ...ate_1-12-c43300026ebf9d0f79b58d2bb7255599} | 0 ...ate_1-13-63ae717a423cb85b060ea837431efc23} | 0 ...ate_1-14-67c78afa310d373a6197ba0a3cc6e70d} | 0 ...ate_1-15-ca5bdcb066b100b8c676bce6535a71b1} | 0 ...ate_1-16-79d09da7ad2beb7ee1da1f5889eae0a4} | 0 ...ate_1-17-d4c25b848047acb9546eb2def9c33a01} | 0 ...date_1-3-97f403b3d42b6391934f6a9da6fef792} | 0 ...date_1-6-6724d5d81b68e07f0420a28b9232d464} | 0 ...date_1-9-241dff52c41c266c0e36f7814913326c} | 0 .../sql/hive/execution/HiveQuerySuite.scala | 93 +++++++++++++++++++ 11 files changed, 93 insertions(+) rename sql/hive/src/test/resources/golden/{date_1-1-23edf29bf7376c70d5ecf12720f4b1eb => date_1-1-d4c25b848047acb9546eb2def9c33a01} (100%) rename sql/hive/src/test/resources/golden/{date_1-12-480c5f024a28232b7857be327c992509 => date_1-12-c43300026ebf9d0f79b58d2bb7255599} (100%) rename sql/hive/src/test/resources/golden/{date_1-13-4c0ed7fcb75770d8790575b586bf14f4 => date_1-13-63ae717a423cb85b060ea837431efc23} (100%) rename sql/hive/src/test/resources/golden/{date_1-14-44fc74c1993062c0a9522199ff27fea => date_1-14-67c78afa310d373a6197ba0a3cc6e70d} (100%) rename sql/hive/src/test/resources/golden/{date_1-15-4855a66124b16d1d0d003235995ac06b => date_1-15-ca5bdcb066b100b8c676bce6535a71b1} (100%) rename sql/hive/src/test/resources/golden/{date_1-16-8bc190dba0f641840b5e1e198a14c55b => date_1-16-79d09da7ad2beb7ee1da1f5889eae0a4} (100%) rename sql/hive/src/test/resources/golden/{date_1-17-23edf29bf7376c70d5ecf12720f4b1eb => date_1-17-d4c25b848047acb9546eb2def9c33a01} (100%) rename sql/hive/src/test/resources/golden/{date_1-3-26b5c291400dfde455b3c1b878b71d0 => date_1-3-97f403b3d42b6391934f6a9da6fef792} (100%) rename sql/hive/src/test/resources/golden/{date_1-6-559d01fb0b42c42f0c4927fa0f9deac4 => date_1-6-6724d5d81b68e07f0420a28b9232d464} (100%) rename sql/hive/src/test/resources/golden/{date_1-9-8306558e0eabe936ac33dabaaa17fea4 => date_1-9-241dff52c41c266c0e36f7814913326c} (100%) diff --git a/sql/hive/src/test/resources/golden/date_1-1-23edf29bf7376c70d5ecf12720f4b1eb b/sql/hive/src/test/resources/golden/date_1-1-d4c25b848047acb9546eb2def9c33a01 similarity index 100% rename from sql/hive/src/test/resources/golden/date_1-1-23edf29bf7376c70d5ecf12720f4b1eb rename to sql/hive/src/test/resources/golden/date_1-1-d4c25b848047acb9546eb2def9c33a01 diff --git a/sql/hive/src/test/resources/golden/date_1-12-480c5f024a28232b7857be327c992509 b/sql/hive/src/test/resources/golden/date_1-12-c43300026ebf9d0f79b58d2bb7255599 similarity index 100% rename from sql/hive/src/test/resources/golden/date_1-12-480c5f024a28232b7857be327c992509 rename to sql/hive/src/test/resources/golden/date_1-12-c43300026ebf9d0f79b58d2bb7255599 diff --git a/sql/hive/src/test/resources/golden/date_1-13-4c0ed7fcb75770d8790575b586bf14f4 b/sql/hive/src/test/resources/golden/date_1-13-63ae717a423cb85b060ea837431efc23 similarity index 100% rename from sql/hive/src/test/resources/golden/date_1-13-4c0ed7fcb75770d8790575b586bf14f4 rename to sql/hive/src/test/resources/golden/date_1-13-63ae717a423cb85b060ea837431efc23 diff --git a/sql/hive/src/test/resources/golden/date_1-14-44fc74c1993062c0a9522199ff27fea b/sql/hive/src/test/resources/golden/date_1-14-67c78afa310d373a6197ba0a3cc6e70d similarity index 100% rename from sql/hive/src/test/resources/golden/date_1-14-44fc74c1993062c0a9522199ff27fea rename to sql/hive/src/test/resources/golden/date_1-14-67c78afa310d373a6197ba0a3cc6e70d diff --git a/sql/hive/src/test/resources/golden/date_1-15-4855a66124b16d1d0d003235995ac06b b/sql/hive/src/test/resources/golden/date_1-15-ca5bdcb066b100b8c676bce6535a71b1 similarity index 100% rename from sql/hive/src/test/resources/golden/date_1-15-4855a66124b16d1d0d003235995ac06b rename to sql/hive/src/test/resources/golden/date_1-15-ca5bdcb066b100b8c676bce6535a71b1 diff --git a/sql/hive/src/test/resources/golden/date_1-16-8bc190dba0f641840b5e1e198a14c55b b/sql/hive/src/test/resources/golden/date_1-16-79d09da7ad2beb7ee1da1f5889eae0a4 similarity index 100% rename from sql/hive/src/test/resources/golden/date_1-16-8bc190dba0f641840b5e1e198a14c55b rename to sql/hive/src/test/resources/golden/date_1-16-79d09da7ad2beb7ee1da1f5889eae0a4 diff --git a/sql/hive/src/test/resources/golden/date_1-17-23edf29bf7376c70d5ecf12720f4b1eb b/sql/hive/src/test/resources/golden/date_1-17-d4c25b848047acb9546eb2def9c33a01 similarity index 100% rename from sql/hive/src/test/resources/golden/date_1-17-23edf29bf7376c70d5ecf12720f4b1eb rename to sql/hive/src/test/resources/golden/date_1-17-d4c25b848047acb9546eb2def9c33a01 diff --git a/sql/hive/src/test/resources/golden/date_1-3-26b5c291400dfde455b3c1b878b71d0 b/sql/hive/src/test/resources/golden/date_1-3-97f403b3d42b6391934f6a9da6fef792 similarity index 100% rename from sql/hive/src/test/resources/golden/date_1-3-26b5c291400dfde455b3c1b878b71d0 rename to sql/hive/src/test/resources/golden/date_1-3-97f403b3d42b6391934f6a9da6fef792 diff --git a/sql/hive/src/test/resources/golden/date_1-6-559d01fb0b42c42f0c4927fa0f9deac4 b/sql/hive/src/test/resources/golden/date_1-6-6724d5d81b68e07f0420a28b9232d464 similarity index 100% rename from sql/hive/src/test/resources/golden/date_1-6-559d01fb0b42c42f0c4927fa0f9deac4 rename to sql/hive/src/test/resources/golden/date_1-6-6724d5d81b68e07f0420a28b9232d464 diff --git a/sql/hive/src/test/resources/golden/date_1-9-8306558e0eabe936ac33dabaaa17fea4 b/sql/hive/src/test/resources/golden/date_1-9-241dff52c41c266c0e36f7814913326c similarity index 100% rename from sql/hive/src/test/resources/golden/date_1-9-8306558e0eabe936ac33dabaaa17fea4 rename to sql/hive/src/test/resources/golden/date_1-9-241dff52c41c266c0e36f7814913326c diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index 489c2faaae88a..2466703d271b0 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -1258,6 +1258,99 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { |SELECT * FROM alter_rename_partition WHERE pcol1='old_part1:' and pcol2='old_part2:'; |SELECT * FROM alter_rename_partition WHERE pcol1='new_part1:' and pcol2='new_part2:'; """.stripMargin) + + createQueryTest("date_1", + """ + |set hive.fetch.task.conversion=more; + | + |drop table if exists date_1; + | + |create table date_1 (d date); + | + |insert overwrite table date_1 + | select cast('2011-01-01' as date) from src tablesample (1 rows); + | + |select * from date_1 limit 1; + |select d, count(d) from date_1 group by d; + | + |insert overwrite table date_1 + | select date '2011-01-01' from src tablesample (1 rows); + | + |select * from date_1 limit 1; + |select d, count(d) from date_1 group by d; + | + |insert overwrite table date_1 + | select cast(cast('2011-01-01 00:00:00' as timestamp) as date) from src tablesample (1 rows); + | + |select * from date_1 limit 1; + |select d, count(d) from date_1 group by d; + | + |-- Valid casts + |select + | cast('2012-01-01' as string), + | cast(d as string), + | cast(d as timestamp), + | cast(cast(d as timestamp) as date), + | cast(d as date) + |from date_1 limit 1; + | + |-- Invalid casts. + |select + | cast(d as boolean), + | cast(d as tinyint), + | cast(d as smallint), + | cast(d as int), + | cast(d as bigint), + | cast(d as float), + | cast(d as double) + |from date_1 limit 1; + | + |-- These comparisons should all be true + |select + | date '2011-01-01' = date '2011-01-01', + | unix_timestamp(date '2011-01-01') = unix_timestamp(date '2011-01-01'), + | unix_timestamp(date '2011-01-01') = unix_timestamp(cast(date '2011-01-01' as timestamp)), + | unix_timestamp(date '2011-01-01') = unix_timestamp(cast(cast('2011-01-01 12:13:14' as timestamp) as date)), + | unix_timestamp(date '2011-01-01') < unix_timestamp(cast('2011-01-01 00:00:01' as timestamp)), + | unix_timestamp(date '2011-01-01') = unix_timestamp(cast('2011-01-01 00:00:00' as timestamp)), + | unix_timestamp(date '2011-01-01') > unix_timestamp(cast('2010-12-31 23:59:59' as timestamp)), + | date '2011-01-01' = cast(timestamp('2011-01-01 23:24:25') as date), + | '2011-01-01' = cast(d as string), + | '2011-01-01' = cast(date '2011-01-01' as string) + |from date_1 limit 1; + | + |select + | date('2001-01-28'), + | date('2001-02-28'), + | date('2001-03-28'), + | date('2001-04-28'), + | date('2001-05-28'), + | date('2001-06-28'), + | date('2001-07-28'), + | date('2001-08-28'), + | date('2001-09-28'), + | date('2001-10-28'), + | date('2001-11-28'), + | date('2001-12-28') + |from date_1 limit 1; + | + |select + | unix_timestamp(date('2001-01-28')) = unix_timestamp(cast('2001-01-28 0:0:0' as timestamp)), + | unix_timestamp(date('2001-02-28')) = unix_timestamp(cast('2001-02-28 0:0:0' as timestamp)), + | unix_timestamp(date('2001-03-28')) = unix_timestamp(cast('2001-03-28 0:0:0' as timestamp)), + | unix_timestamp(date('2001-04-28')) = unix_timestamp(cast('2001-04-28 0:0:0' as timestamp)), + | unix_timestamp(date('2001-05-28')) = unix_timestamp(cast('2001-05-28 0:0:0' as timestamp)), + | unix_timestamp(date('2001-06-28')) = unix_timestamp(cast('2001-06-28 0:0:0' as timestamp)), + | unix_timestamp(date('2001-07-28')) = unix_timestamp(cast('2001-07-28 0:0:0' as timestamp)), + | unix_timestamp(date('2001-08-28')) = unix_timestamp(cast('2001-08-28 0:0:0' as timestamp)), + | unix_timestamp(date('2001-09-28')) = unix_timestamp(cast('2001-09-28 0:0:0' as timestamp)), + | unix_timestamp(date('2001-10-28')) = unix_timestamp(cast('2001-10-28 0:0:0' as timestamp)), + | unix_timestamp(date('2001-11-28')) = unix_timestamp(cast('2001-11-28 0:0:0' as timestamp)), + | unix_timestamp(date('2001-12-28')) = unix_timestamp(cast('2001-12-28 0:0:0' as timestamp)) + |from date_1 limit 1; + | + |drop table if exists date_1; + """.stripMargin) // scalastyle:on } From 8ff4d0700a38e4a7dc3e2428ec5aa4fa8dd7d105 Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Mon, 4 Jul 2016 12:03:29 +0530 Subject: [PATCH 03/25] date_4 added --- ...> date_4-1-fc13c9dca90ab0950781c3831f449d9a} | 0 ...> date_4-4-9a75e3611fa1fdeaf939924e328c556b} | 0 ...> date_4-6-fc13c9dca90ab0950781c3831f449d9a} | 0 .../sql/hive/execution/HiveQuerySuite.scala | 17 +++++++++++++++++ 4 files changed, 17 insertions(+) rename sql/hive/src/test/resources/golden/{date_4-1-6272f5e518f6a20bc96a5870ff315c4f => date_4-1-fc13c9dca90ab0950781c3831f449d9a} (100%) rename sql/hive/src/test/resources/golden/{date_4-4-6c4c2941bae77147a4d3d8fcaa1c88c8 => date_4-4-9a75e3611fa1fdeaf939924e328c556b} (100%) rename sql/hive/src/test/resources/golden/{date_4-6-b84f7e931d710dcbe3c5126d998285a8 => date_4-6-fc13c9dca90ab0950781c3831f449d9a} (100%) diff --git a/sql/hive/src/test/resources/golden/date_4-1-6272f5e518f6a20bc96a5870ff315c4f b/sql/hive/src/test/resources/golden/date_4-1-fc13c9dca90ab0950781c3831f449d9a similarity index 100% rename from sql/hive/src/test/resources/golden/date_4-1-6272f5e518f6a20bc96a5870ff315c4f rename to sql/hive/src/test/resources/golden/date_4-1-fc13c9dca90ab0950781c3831f449d9a diff --git a/sql/hive/src/test/resources/golden/date_4-4-6c4c2941bae77147a4d3d8fcaa1c88c8 b/sql/hive/src/test/resources/golden/date_4-4-9a75e3611fa1fdeaf939924e328c556b similarity index 100% rename from sql/hive/src/test/resources/golden/date_4-4-6c4c2941bae77147a4d3d8fcaa1c88c8 rename to sql/hive/src/test/resources/golden/date_4-4-9a75e3611fa1fdeaf939924e328c556b diff --git a/sql/hive/src/test/resources/golden/date_4-6-b84f7e931d710dcbe3c5126d998285a8 b/sql/hive/src/test/resources/golden/date_4-6-fc13c9dca90ab0950781c3831f449d9a similarity index 100% rename from sql/hive/src/test/resources/golden/date_4-6-b84f7e931d710dcbe3c5126d998285a8 rename to sql/hive/src/test/resources/golden/date_4-6-fc13c9dca90ab0950781c3831f449d9a diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index 2466703d271b0..4553772770ee1 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -1351,6 +1351,23 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { | |drop table if exists date_1; """.stripMargin) + + createQueryTest("date_4", + """ + |set hive.fetch.task.conversion=more; + | + |drop table if exists date_4; + | + |create table date_4 (d date); + |alter table date_4 set serde 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'; + | + |-- Test date literal syntax + |insert overwrite table date_4 + | select date '2011-01-01' from src tablesample (1 rows); + |select d, date '2011-01-01' from date_4 limit 1; + | + |drop table if exists date_4; + """.stripMargin) // scalastyle:on } From 601d7cf8b135c4dcad6340517fb647e85adb9733 Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Mon, 4 Jul 2016 12:16:07 +0530 Subject: [PATCH 04/25] date_join1 added --- ..._join1-0-7f9988c6a09cfd7d6a257b3c7c55448d} | 0 ..._join1-3-7b837465673e322a2c8e1d61ec4e7d54} | 0 ..._join1-4-7f9988c6a09cfd7d6a257b3c7c55448d} | 0 .../sql/hive/execution/HiveQuerySuite.scala | 23 +++++++++++++++++++ 4 files changed, 23 insertions(+) rename sql/hive/src/test/resources/golden/{date_join1-0-70b9b49c55699fe94cfde069f5d197c => date_join1-0-7f9988c6a09cfd7d6a257b3c7c55448d} (100%) rename sql/hive/src/test/resources/golden/{date_join1-3-f71c7be760fb4de4eff8225f2c6614b2 => date_join1-3-7b837465673e322a2c8e1d61ec4e7d54} (100%) rename sql/hive/src/test/resources/golden/{date_join1-4-70b9b49c55699fe94cfde069f5d197c => date_join1-4-7f9988c6a09cfd7d6a257b3c7c55448d} (100%) diff --git a/sql/hive/src/test/resources/golden/date_join1-0-70b9b49c55699fe94cfde069f5d197c b/sql/hive/src/test/resources/golden/date_join1-0-7f9988c6a09cfd7d6a257b3c7c55448d similarity index 100% rename from sql/hive/src/test/resources/golden/date_join1-0-70b9b49c55699fe94cfde069f5d197c rename to sql/hive/src/test/resources/golden/date_join1-0-7f9988c6a09cfd7d6a257b3c7c55448d diff --git a/sql/hive/src/test/resources/golden/date_join1-3-f71c7be760fb4de4eff8225f2c6614b2 b/sql/hive/src/test/resources/golden/date_join1-3-7b837465673e322a2c8e1d61ec4e7d54 similarity index 100% rename from sql/hive/src/test/resources/golden/date_join1-3-f71c7be760fb4de4eff8225f2c6614b2 rename to sql/hive/src/test/resources/golden/date_join1-3-7b837465673e322a2c8e1d61ec4e7d54 diff --git a/sql/hive/src/test/resources/golden/date_join1-4-70b9b49c55699fe94cfde069f5d197c b/sql/hive/src/test/resources/golden/date_join1-4-7f9988c6a09cfd7d6a257b3c7c55448d similarity index 100% rename from sql/hive/src/test/resources/golden/date_join1-4-70b9b49c55699fe94cfde069f5d197c rename to sql/hive/src/test/resources/golden/date_join1-4-7f9988c6a09cfd7d6a257b3c7c55448d diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index 4553772770ee1..209e9502a1a16 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -1368,6 +1368,29 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { | |drop table if exists date_4; """.stripMargin) + + createQueryTest("date_join1", + """ + |drop table if exists date_join1; + | + |create table date_join1 ( + | ORIGIN_CITY_NAME string, + | DEST_CITY_NAME string, + | FL_DATE date, + | ARR_DELAY float, + | FL_NUM int + |); + | + |LOAD DATA LOCAL INPATH '../../data/files/flights_join.txt' OVERWRITE INTO TABLE date_join1; + | + |-- Note that there are 2 rows with date 2000-11-28, so we should expect 4 rows with that date in the join results + |select t1.fl_num, t1.fl_date, t2.fl_num, t2.fl_date + | from date_join1 t1 + | join date_join1 t2 + | on (t1.fl_date = t2.fl_date); + | + |drop table if exists date_join1; + """.stripMargin) // scalastyle:on } From 406da2e9a2f7fc08f42b8f0be42866a9f9b2ccf0 Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Mon, 4 Jul 2016 12:21:57 +0530 Subject: [PATCH 05/25] date_serde added --- ..._serde-1-195fed131624dd895a97dbde00c871f7} | 0 ...serde-12-20fdce02d016214f18b34dbd76f44821} | 0 ...serde-17-bc6e92266e0d770c0b214f31fea24183} | 0 ..._serde-2-26db1db4a0c997136eeff4534ceeefd2} | 0 ...serde-22-51ac8110ec1b9ff2cf8570a1d1afe23f} | 0 ...serde-27-fe76d43e21af70b9fcf05f85dc34ac8b} | 0 ..._serde-3-d2fd4c762962f3636a0a5ead6d0d3df3} | 0 ...serde-32-3e5ea6e7683e4cd03566e9fcb9e3726a} | 0 ...serde-35-ca88593bb7ec47fa782145d732100c07} | 0 ...serde-36-195fed131624dd895a97dbde00c871f7} | 0 ...serde-37-26db1db4a0c997136eeff4534ceeefd2} | 0 ...serde-38-d2fd4c762962f3636a0a5ead6d0d3df3} | 0 ...serde-39-81935193b8afc4cc4301471d07a07239} | 0 ..._serde-4-81935193b8afc4cc4301471d07a07239} | 0 ...serde-40-9f3df142f2913859e316540c34412ae2} | 0 ..._serde-5-9f3df142f2913859e316540c34412ae2} | 0 .../sql/hive/execution/HiveQuerySuite.scala | 127 ++++++++++++++++++ 17 files changed, 127 insertions(+) rename sql/hive/src/test/resources/golden/{date_serde-1-36e6041f53433482631018410bb62a99 => date_serde-1-195fed131624dd895a97dbde00c871f7} (100%) rename sql/hive/src/test/resources/golden/{date_serde-12-c3c3275658b89d31fc504db31ae9f99c => date_serde-12-20fdce02d016214f18b34dbd76f44821} (100%) rename sql/hive/src/test/resources/golden/{date_serde-17-a0df43062f8ab676ef728c9968443f12 => date_serde-17-bc6e92266e0d770c0b214f31fea24183} (100%) rename sql/hive/src/test/resources/golden/{date_serde-2-3ddfd8ecb28991aeed588f1ea852c427 => date_serde-2-26db1db4a0c997136eeff4534ceeefd2} (100%) rename sql/hive/src/test/resources/golden/{date_serde-22-45240a488fb708e432d2f45b74ef7e63 => date_serde-22-51ac8110ec1b9ff2cf8570a1d1afe23f} (100%) rename sql/hive/src/test/resources/golden/{date_serde-27-4d86c79f858866acec3c37f6598c2638 => date_serde-27-fe76d43e21af70b9fcf05f85dc34ac8b} (100%) rename sql/hive/src/test/resources/golden/{date_serde-3-e6167e27465514356c557a77d956ea46 => date_serde-3-d2fd4c762962f3636a0a5ead6d0d3df3} (100%) rename sql/hive/src/test/resources/golden/{date_serde-32-bb2f76bd307ed616a3c797f8dd45a8d1 => date_serde-32-3e5ea6e7683e4cd03566e9fcb9e3726a} (100%) rename sql/hive/src/test/resources/golden/{date_serde-35-8651a7c351cbc07fb1af6193f6885de8 => date_serde-35-ca88593bb7ec47fa782145d732100c07} (100%) rename sql/hive/src/test/resources/golden/{date_serde-36-36e6041f53433482631018410bb62a99 => date_serde-36-195fed131624dd895a97dbde00c871f7} (100%) rename sql/hive/src/test/resources/golden/{date_serde-37-3ddfd8ecb28991aeed588f1ea852c427 => date_serde-37-26db1db4a0c997136eeff4534ceeefd2} (100%) rename sql/hive/src/test/resources/golden/{date_serde-38-e6167e27465514356c557a77d956ea46 => date_serde-38-d2fd4c762962f3636a0a5ead6d0d3df3} (100%) rename sql/hive/src/test/resources/golden/{date_serde-39-c1e17c93582656c12970c37bac153bf2 => date_serde-39-81935193b8afc4cc4301471d07a07239} (100%) rename sql/hive/src/test/resources/golden/{date_serde-4-c1e17c93582656c12970c37bac153bf2 => date_serde-4-81935193b8afc4cc4301471d07a07239} (100%) rename sql/hive/src/test/resources/golden/{date_serde-40-4a17944b9ec8999bb20c5ba5d4cb877c => date_serde-40-9f3df142f2913859e316540c34412ae2} (100%) rename sql/hive/src/test/resources/golden/{date_serde-5-4a17944b9ec8999bb20c5ba5d4cb877c => date_serde-5-9f3df142f2913859e316540c34412ae2} (100%) diff --git a/sql/hive/src/test/resources/golden/date_serde-1-36e6041f53433482631018410bb62a99 b/sql/hive/src/test/resources/golden/date_serde-1-195fed131624dd895a97dbde00c871f7 similarity index 100% rename from sql/hive/src/test/resources/golden/date_serde-1-36e6041f53433482631018410bb62a99 rename to sql/hive/src/test/resources/golden/date_serde-1-195fed131624dd895a97dbde00c871f7 diff --git a/sql/hive/src/test/resources/golden/date_serde-12-c3c3275658b89d31fc504db31ae9f99c b/sql/hive/src/test/resources/golden/date_serde-12-20fdce02d016214f18b34dbd76f44821 similarity index 100% rename from sql/hive/src/test/resources/golden/date_serde-12-c3c3275658b89d31fc504db31ae9f99c rename to sql/hive/src/test/resources/golden/date_serde-12-20fdce02d016214f18b34dbd76f44821 diff --git a/sql/hive/src/test/resources/golden/date_serde-17-a0df43062f8ab676ef728c9968443f12 b/sql/hive/src/test/resources/golden/date_serde-17-bc6e92266e0d770c0b214f31fea24183 similarity index 100% rename from sql/hive/src/test/resources/golden/date_serde-17-a0df43062f8ab676ef728c9968443f12 rename to sql/hive/src/test/resources/golden/date_serde-17-bc6e92266e0d770c0b214f31fea24183 diff --git a/sql/hive/src/test/resources/golden/date_serde-2-3ddfd8ecb28991aeed588f1ea852c427 b/sql/hive/src/test/resources/golden/date_serde-2-26db1db4a0c997136eeff4534ceeefd2 similarity index 100% rename from sql/hive/src/test/resources/golden/date_serde-2-3ddfd8ecb28991aeed588f1ea852c427 rename to sql/hive/src/test/resources/golden/date_serde-2-26db1db4a0c997136eeff4534ceeefd2 diff --git a/sql/hive/src/test/resources/golden/date_serde-22-45240a488fb708e432d2f45b74ef7e63 b/sql/hive/src/test/resources/golden/date_serde-22-51ac8110ec1b9ff2cf8570a1d1afe23f similarity index 100% rename from sql/hive/src/test/resources/golden/date_serde-22-45240a488fb708e432d2f45b74ef7e63 rename to sql/hive/src/test/resources/golden/date_serde-22-51ac8110ec1b9ff2cf8570a1d1afe23f diff --git a/sql/hive/src/test/resources/golden/date_serde-27-4d86c79f858866acec3c37f6598c2638 b/sql/hive/src/test/resources/golden/date_serde-27-fe76d43e21af70b9fcf05f85dc34ac8b similarity index 100% rename from sql/hive/src/test/resources/golden/date_serde-27-4d86c79f858866acec3c37f6598c2638 rename to sql/hive/src/test/resources/golden/date_serde-27-fe76d43e21af70b9fcf05f85dc34ac8b diff --git a/sql/hive/src/test/resources/golden/date_serde-3-e6167e27465514356c557a77d956ea46 b/sql/hive/src/test/resources/golden/date_serde-3-d2fd4c762962f3636a0a5ead6d0d3df3 similarity index 100% rename from sql/hive/src/test/resources/golden/date_serde-3-e6167e27465514356c557a77d956ea46 rename to sql/hive/src/test/resources/golden/date_serde-3-d2fd4c762962f3636a0a5ead6d0d3df3 diff --git a/sql/hive/src/test/resources/golden/date_serde-32-bb2f76bd307ed616a3c797f8dd45a8d1 b/sql/hive/src/test/resources/golden/date_serde-32-3e5ea6e7683e4cd03566e9fcb9e3726a similarity index 100% rename from sql/hive/src/test/resources/golden/date_serde-32-bb2f76bd307ed616a3c797f8dd45a8d1 rename to sql/hive/src/test/resources/golden/date_serde-32-3e5ea6e7683e4cd03566e9fcb9e3726a diff --git a/sql/hive/src/test/resources/golden/date_serde-35-8651a7c351cbc07fb1af6193f6885de8 b/sql/hive/src/test/resources/golden/date_serde-35-ca88593bb7ec47fa782145d732100c07 similarity index 100% rename from sql/hive/src/test/resources/golden/date_serde-35-8651a7c351cbc07fb1af6193f6885de8 rename to sql/hive/src/test/resources/golden/date_serde-35-ca88593bb7ec47fa782145d732100c07 diff --git a/sql/hive/src/test/resources/golden/date_serde-36-36e6041f53433482631018410bb62a99 b/sql/hive/src/test/resources/golden/date_serde-36-195fed131624dd895a97dbde00c871f7 similarity index 100% rename from sql/hive/src/test/resources/golden/date_serde-36-36e6041f53433482631018410bb62a99 rename to sql/hive/src/test/resources/golden/date_serde-36-195fed131624dd895a97dbde00c871f7 diff --git a/sql/hive/src/test/resources/golden/date_serde-37-3ddfd8ecb28991aeed588f1ea852c427 b/sql/hive/src/test/resources/golden/date_serde-37-26db1db4a0c997136eeff4534ceeefd2 similarity index 100% rename from sql/hive/src/test/resources/golden/date_serde-37-3ddfd8ecb28991aeed588f1ea852c427 rename to sql/hive/src/test/resources/golden/date_serde-37-26db1db4a0c997136eeff4534ceeefd2 diff --git a/sql/hive/src/test/resources/golden/date_serde-38-e6167e27465514356c557a77d956ea46 b/sql/hive/src/test/resources/golden/date_serde-38-d2fd4c762962f3636a0a5ead6d0d3df3 similarity index 100% rename from sql/hive/src/test/resources/golden/date_serde-38-e6167e27465514356c557a77d956ea46 rename to sql/hive/src/test/resources/golden/date_serde-38-d2fd4c762962f3636a0a5ead6d0d3df3 diff --git a/sql/hive/src/test/resources/golden/date_serde-39-c1e17c93582656c12970c37bac153bf2 b/sql/hive/src/test/resources/golden/date_serde-39-81935193b8afc4cc4301471d07a07239 similarity index 100% rename from sql/hive/src/test/resources/golden/date_serde-39-c1e17c93582656c12970c37bac153bf2 rename to sql/hive/src/test/resources/golden/date_serde-39-81935193b8afc4cc4301471d07a07239 diff --git a/sql/hive/src/test/resources/golden/date_serde-4-c1e17c93582656c12970c37bac153bf2 b/sql/hive/src/test/resources/golden/date_serde-4-81935193b8afc4cc4301471d07a07239 similarity index 100% rename from sql/hive/src/test/resources/golden/date_serde-4-c1e17c93582656c12970c37bac153bf2 rename to sql/hive/src/test/resources/golden/date_serde-4-81935193b8afc4cc4301471d07a07239 diff --git a/sql/hive/src/test/resources/golden/date_serde-40-4a17944b9ec8999bb20c5ba5d4cb877c b/sql/hive/src/test/resources/golden/date_serde-40-9f3df142f2913859e316540c34412ae2 similarity index 100% rename from sql/hive/src/test/resources/golden/date_serde-40-4a17944b9ec8999bb20c5ba5d4cb877c rename to sql/hive/src/test/resources/golden/date_serde-40-9f3df142f2913859e316540c34412ae2 diff --git a/sql/hive/src/test/resources/golden/date_serde-5-4a17944b9ec8999bb20c5ba5d4cb877c b/sql/hive/src/test/resources/golden/date_serde-5-9f3df142f2913859e316540c34412ae2 similarity index 100% rename from sql/hive/src/test/resources/golden/date_serde-5-4a17944b9ec8999bb20c5ba5d4cb877c rename to sql/hive/src/test/resources/golden/date_serde-5-9f3df142f2913859e316540c34412ae2 diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index 209e9502a1a16..386ebed94b0c4 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -1391,6 +1391,133 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { | |drop table if exists date_join1; """.stripMargin) + + createQueryTest("date_serde", + """ + |drop table if exists date_serde_regex; + |drop table if exists date_serde_lb; + |drop table if exists date_serde_ls; + |drop table if exists date_serde_c; + |drop table if exists date_serde_lbc; + |drop table if exists date_serde_orc; + | + | + |-- + |-- RegexSerDe + |-- + |create table date_serde_regex ( + | ORIGIN_CITY_NAME string, + | DEST_CITY_NAME string, + | FL_DATE date, + | ARR_DELAY float, + | FL_NUM int + |) + |row format serde 'org.apache.hadoop.hive.serde2.RegexSerDe' + |with serdeproperties ( + | "input.regex" = "([^]*)([^]*)([^]*)([^]*)([0-9]*)" + |) + |stored as textfile; + | + |load data local inpath '../../data/files/flights_tiny.txt.1' overwrite into table date_serde_regex; + | + |select * from date_serde_regex; + |select fl_date, count(*) from date_serde_regex group by fl_date; + | + |-- + |-- LazyBinary + |-- + |create table date_serde_lb ( + | c1 date, + | c2 int + |); + |alter table date_serde_lb set serde 'org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe'; + | + |insert overwrite table date_serde_lb + | select fl_date, fl_num from date_serde_regex limit 1; + | + |select * from date_serde_lb; + |select c1, sum(c2) from date_serde_lb group by c1; + | + |-- + |-- LazySimple + |-- + |create table date_serde_ls ( + | c1 date, + | c2 int + |); + |alter table date_serde_ls set serde 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'; + | + |insert overwrite table date_serde_ls + | select c1, c2 from date_serde_lb limit 1; + | + |select * from date_serde_ls; + |select c1, sum(c2) from date_serde_ls group by c1; + | + |-- + |-- Columnar + |-- + |create table date_serde_c ( + | c1 date, + | c2 int + |) stored as rcfile; + |alter table date_serde_c set serde 'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe'; + | + |insert overwrite table date_serde_c + | select c1, c2 from date_serde_ls limit 1; + | + |select * from date_serde_c; + |select c1, sum(c2) from date_serde_c group by c1; + | + |-- + |-- LazyBinaryColumnar + |-- + |create table date_serde_lbc ( + | c1 date, + | c2 int + |) stored as rcfile; + |alter table date_serde_lbc set serde 'org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe'; + | + |insert overwrite table date_serde_lbc + | select c1, c2 from date_serde_c limit 1; + | + |select * from date_serde_lbc; + |select c1, sum(c2) from date_serde_lbc group by c1; + | + |-- + |-- ORC + |-- + |create table date_serde_orc ( + | c1 date, + | c2 int + |) stored as orc; + |alter table date_serde_orc set serde 'org.apache.hadoop.hive.ql.io.orc.OrcSerde'; + | + |insert overwrite table date_serde_orc + | select c1, c2 from date_serde_lbc limit 1; + | + |select * from date_serde_orc; + |select c1, sum(c2) from date_serde_orc group by c1; + | + | + | + |drop table if exists date_serde_regex; + |drop table if exists date_serde_lb; + |drop table if exists date_serde_ls; + |drop table if exists date_serde_c; + |drop table if exists date_serde_lbc; + |drop table if exists date_serde_orc; + """.stripMargin) + + createQueryTest("test", + """ + | + """.stripMargin) + + createQueryTest("test11", + """ + | + """.stripMargin) + // scalastyle:on } From 60188631210a345c679bf304cef0403384f4c227 Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Mon, 4 Jul 2016 12:27:17 +0530 Subject: [PATCH 06/25] insert_compressed added --- ...pressed-1-4ef7b64a704c23cb363d0f11a203f72} | 0 ...pressed-9-4ef7b64a704c23cb363d0f11a203f72} | 0 .../sql/hive/execution/HiveQuerySuite.scala | 19 ++++++++++++++----- 3 files changed, 14 insertions(+), 5 deletions(-) rename sql/hive/src/test/resources/golden/{insert_compressed-1-d20899578068ff4adfee5355cddca819 => insert_compressed-1-4ef7b64a704c23cb363d0f11a203f72} (100%) rename sql/hive/src/test/resources/golden/{insert_compressed-9-d20899578068ff4adfee5355cddca819 => insert_compressed-9-4ef7b64a704c23cb363d0f11a203f72} (100%) diff --git a/sql/hive/src/test/resources/golden/insert_compressed-1-d20899578068ff4adfee5355cddca819 b/sql/hive/src/test/resources/golden/insert_compressed-1-4ef7b64a704c23cb363d0f11a203f72 similarity index 100% rename from sql/hive/src/test/resources/golden/insert_compressed-1-d20899578068ff4adfee5355cddca819 rename to sql/hive/src/test/resources/golden/insert_compressed-1-4ef7b64a704c23cb363d0f11a203f72 diff --git a/sql/hive/src/test/resources/golden/insert_compressed-9-d20899578068ff4adfee5355cddca819 b/sql/hive/src/test/resources/golden/insert_compressed-9-4ef7b64a704c23cb363d0f11a203f72 similarity index 100% rename from sql/hive/src/test/resources/golden/insert_compressed-9-d20899578068ff4adfee5355cddca819 rename to sql/hive/src/test/resources/golden/insert_compressed-9-4ef7b64a704c23cb363d0f11a203f72 diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index 386ebed94b0c4..0f68060a40573 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -1508,14 +1508,23 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { |drop table if exists date_serde_orc; """.stripMargin) - createQueryTest("test", + createQueryTest("insert_compressed", """ + |set hive.exec.compress.output=true; | - """.stripMargin) - - createQueryTest("test11", - """ + |drop table if exists insert_compressed; + |create table insert_compressed (key int, value string); + | + |insert overwrite table insert_compressed select * from src; + |select count(*) from insert_compressed; + | + |insert into table insert_compressed select * from src; + |select count(*) from insert_compressed; + | + |insert into table insert_compressed select * from src; + |select count(*) from insert_compressed; | + |drop table if exists insert_compressed; """.stripMargin) // scalastyle:on From 7105d5676d51ec4a7f22d2088d7d5b8c6f78c72c Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Mon, 4 Jul 2016 12:30:08 +0530 Subject: [PATCH 07/25] lateral_view_cp added --- ...ateral_view_cp-0-8b86f01a49d3f64baddfce9a1ed51ec} | 0 .../spark/sql/hive/execution/HiveQuerySuite.scala | 12 ++++++++++++ 2 files changed, 12 insertions(+) rename sql/hive/src/test/resources/golden/{lateral_view_cp-0-6ae48c990343343aad4145203c364f79 => lateral_view_cp-0-8b86f01a49d3f64baddfce9a1ed51ec} (100%) diff --git a/sql/hive/src/test/resources/golden/lateral_view_cp-0-6ae48c990343343aad4145203c364f79 b/sql/hive/src/test/resources/golden/lateral_view_cp-0-8b86f01a49d3f64baddfce9a1ed51ec similarity index 100% rename from sql/hive/src/test/resources/golden/lateral_view_cp-0-6ae48c990343343aad4145203c364f79 rename to sql/hive/src/test/resources/golden/lateral_view_cp-0-8b86f01a49d3f64baddfce9a1ed51ec diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index 0f68060a40573..b3646bf26d9d0 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -1527,6 +1527,18 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { |drop table if exists insert_compressed; """.stripMargin) + createQueryTest("lateral_view_cp", + """ + |--HIVE 3226 + |drop table if exists array_valued_src; + |create table array_valued_src (key string, value array); + |insert overwrite table array_valued_src select key, array(value) from src; + | + |-- replace sel(*) to sel(exprs) for reflecting CP result properly + |explain select count(val) from (select a.key as key, b.value as array_val from src a join array_valued_src b on a.key=b.key) i lateral view explode (array_val) c as val; + |select count(val) from (select a.key as key, b.value as array_val from src a join array_valued_src b on a.key=b.key) i lateral view explode (array_val) c as val; + """.stripMargin) + // scalastyle:on } From 28bf24d78ffdeadc4b1a4a8e58c493720c919470 Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Mon, 4 Jul 2016 12:33:59 +0530 Subject: [PATCH 08/25] leftsemijoin added --- ...mijoin-0-7a640484cc265c7b40223e0f22c7d30d} | 0 ...mijoin-1-e8c10afb0fcae7d24f4bf4ada41e4b4f} | 0 ...ijoin-11-7a640484cc265c7b40223e0f22c7d30d} | 0 ...ijoin-12-e8c10afb0fcae7d24f4bf4ada41e4b4f} | 0 .../sql/hive/execution/HiveQuerySuite.scala | 26 +++++++++++++++++++ 5 files changed, 26 insertions(+) rename sql/hive/src/test/resources/golden/{leftsemijoin-0-80b6466213face7fbcb0de044611e1f5 => leftsemijoin-0-7a640484cc265c7b40223e0f22c7d30d} (100%) rename sql/hive/src/test/resources/golden/{leftsemijoin-1-d1f6a3dea28a5f0fee08026bf33d9129 => leftsemijoin-1-e8c10afb0fcae7d24f4bf4ada41e4b4f} (100%) rename sql/hive/src/test/resources/golden/{leftsemijoin-11-80b6466213face7fbcb0de044611e1f5 => leftsemijoin-11-7a640484cc265c7b40223e0f22c7d30d} (100%) rename sql/hive/src/test/resources/golden/{leftsemijoin-12-d1f6a3dea28a5f0fee08026bf33d9129 => leftsemijoin-12-e8c10afb0fcae7d24f4bf4ada41e4b4f} (100%) diff --git a/sql/hive/src/test/resources/golden/leftsemijoin-0-80b6466213face7fbcb0de044611e1f5 b/sql/hive/src/test/resources/golden/leftsemijoin-0-7a640484cc265c7b40223e0f22c7d30d similarity index 100% rename from sql/hive/src/test/resources/golden/leftsemijoin-0-80b6466213face7fbcb0de044611e1f5 rename to sql/hive/src/test/resources/golden/leftsemijoin-0-7a640484cc265c7b40223e0f22c7d30d diff --git a/sql/hive/src/test/resources/golden/leftsemijoin-1-d1f6a3dea28a5f0fee08026bf33d9129 b/sql/hive/src/test/resources/golden/leftsemijoin-1-e8c10afb0fcae7d24f4bf4ada41e4b4f similarity index 100% rename from sql/hive/src/test/resources/golden/leftsemijoin-1-d1f6a3dea28a5f0fee08026bf33d9129 rename to sql/hive/src/test/resources/golden/leftsemijoin-1-e8c10afb0fcae7d24f4bf4ada41e4b4f diff --git a/sql/hive/src/test/resources/golden/leftsemijoin-11-80b6466213face7fbcb0de044611e1f5 b/sql/hive/src/test/resources/golden/leftsemijoin-11-7a640484cc265c7b40223e0f22c7d30d similarity index 100% rename from sql/hive/src/test/resources/golden/leftsemijoin-11-80b6466213face7fbcb0de044611e1f5 rename to sql/hive/src/test/resources/golden/leftsemijoin-11-7a640484cc265c7b40223e0f22c7d30d diff --git a/sql/hive/src/test/resources/golden/leftsemijoin-12-d1f6a3dea28a5f0fee08026bf33d9129 b/sql/hive/src/test/resources/golden/leftsemijoin-12-e8c10afb0fcae7d24f4bf4ada41e4b4f similarity index 100% rename from sql/hive/src/test/resources/golden/leftsemijoin-12-d1f6a3dea28a5f0fee08026bf33d9129 rename to sql/hive/src/test/resources/golden/leftsemijoin-12-e8c10afb0fcae7d24f4bf4ada41e4b4f diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index b3646bf26d9d0..40574a8f7c6ae 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -1539,6 +1539,32 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { |select count(val) from (select a.key as key, b.value as array_val from src a join array_valued_src b on a.key=b.key) i lateral view explode (array_val) c as val; """.stripMargin) + createQueryTest("leftsemijoin", + """ + |drop table if exists sales; + |drop table if exists things; + | + |set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; + | + |CREATE TABLE sales (name STRING, id INT) + |ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'; + | + |CREATE TABLE things (id INT, name STRING) partitioned by (ds string) + |ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'; + | + |load data local inpath '../../data/files/sales.txt' INTO TABLE sales; + |load data local inpath '../../data/files/things.txt' INTO TABLE things partition(ds='2011-10-23'); + |load data local inpath '../../data/files/things2.txt' INTO TABLE things partition(ds='2011-10-24'); + | + |SELECT name,id FROM sales ORDER BY name ASC, id ASC; + | + |SELECT id,name FROM things ORDER BY id ASC, name ASC; + | + |SELECT name,id FROM sales LEFT SEMI JOIN things ON (sales.id = things.id) ORDER BY name ASC, id ASC; + | + |drop table if exists sales; + |drop table if exists things; + """.stripMargin) // scalastyle:on } From 6782ad94a300b8ca0cb0515e43efcea583df0601 Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Mon, 4 Jul 2016 12:37:24 +0530 Subject: [PATCH 09/25] mapjoin_subquery2 added --- ...bquery2-0-65db5164b6b8a8919d2be7ba991b3c7} | 0 ...query2-1-f8432143e645a1886ae31aa568a16ce1} | 0 ...query2-12-d7bf9bc4192b6b384f6fc2cc40f7761} | 0 ...uery2-13-258c97fa0a1b2ffc01c34ebec1aa95d3} | 0 ...query2-14-65db5164b6b8a8919d2be7ba991b3c7} | 0 ...uery2-15-f8432143e645a1886ae31aa568a16ce1} | 0 ...uery2-16-512cf08aaaa421be72bb38dfa5f552bd} | 0 ...query2-2-512cf08aaaa421be72bb38dfa5f552bd} | 0 .../sql/hive/execution/HiveQuerySuite.scala | 43 +++++++++++++++++++ 9 files changed, 43 insertions(+) rename sql/hive/src/test/resources/golden/{mapjoin_subquery2-0-59fdb2842fbf4f530366f0237ff235e2 => mapjoin_subquery2-0-65db5164b6b8a8919d2be7ba991b3c7} (100%) rename sql/hive/src/test/resources/golden/{mapjoin_subquery2-1-c0a9763a930555f846a2576d003fb517 => mapjoin_subquery2-1-f8432143e645a1886ae31aa568a16ce1} (100%) rename sql/hive/src/test/resources/golden/{mapjoin_subquery2-12-7ebf0bee394756c0e0c78bdd1034f183 => mapjoin_subquery2-12-d7bf9bc4192b6b384f6fc2cc40f7761} (100%) rename sql/hive/src/test/resources/golden/{mapjoin_subquery2-13-c876a518451059f17fc15e29f6f57951 => mapjoin_subquery2-13-258c97fa0a1b2ffc01c34ebec1aa95d3} (100%) rename sql/hive/src/test/resources/golden/{mapjoin_subquery2-14-59fdb2842fbf4f530366f0237ff235e2 => mapjoin_subquery2-14-65db5164b6b8a8919d2be7ba991b3c7} (100%) rename sql/hive/src/test/resources/golden/{mapjoin_subquery2-15-c0a9763a930555f846a2576d003fb517 => mapjoin_subquery2-15-f8432143e645a1886ae31aa568a16ce1} (100%) rename sql/hive/src/test/resources/golden/{mapjoin_subquery2-16-2f8ecc54049960ee4707f5e6f491fdf4 => mapjoin_subquery2-16-512cf08aaaa421be72bb38dfa5f552bd} (100%) rename sql/hive/src/test/resources/golden/{mapjoin_subquery2-2-2f8ecc54049960ee4707f5e6f491fdf4 => mapjoin_subquery2-2-512cf08aaaa421be72bb38dfa5f552bd} (100%) diff --git a/sql/hive/src/test/resources/golden/mapjoin_subquery2-0-59fdb2842fbf4f530366f0237ff235e2 b/sql/hive/src/test/resources/golden/mapjoin_subquery2-0-65db5164b6b8a8919d2be7ba991b3c7 similarity index 100% rename from sql/hive/src/test/resources/golden/mapjoin_subquery2-0-59fdb2842fbf4f530366f0237ff235e2 rename to sql/hive/src/test/resources/golden/mapjoin_subquery2-0-65db5164b6b8a8919d2be7ba991b3c7 diff --git a/sql/hive/src/test/resources/golden/mapjoin_subquery2-1-c0a9763a930555f846a2576d003fb517 b/sql/hive/src/test/resources/golden/mapjoin_subquery2-1-f8432143e645a1886ae31aa568a16ce1 similarity index 100% rename from sql/hive/src/test/resources/golden/mapjoin_subquery2-1-c0a9763a930555f846a2576d003fb517 rename to sql/hive/src/test/resources/golden/mapjoin_subquery2-1-f8432143e645a1886ae31aa568a16ce1 diff --git a/sql/hive/src/test/resources/golden/mapjoin_subquery2-12-7ebf0bee394756c0e0c78bdd1034f183 b/sql/hive/src/test/resources/golden/mapjoin_subquery2-12-d7bf9bc4192b6b384f6fc2cc40f7761 similarity index 100% rename from sql/hive/src/test/resources/golden/mapjoin_subquery2-12-7ebf0bee394756c0e0c78bdd1034f183 rename to sql/hive/src/test/resources/golden/mapjoin_subquery2-12-d7bf9bc4192b6b384f6fc2cc40f7761 diff --git a/sql/hive/src/test/resources/golden/mapjoin_subquery2-13-c876a518451059f17fc15e29f6f57951 b/sql/hive/src/test/resources/golden/mapjoin_subquery2-13-258c97fa0a1b2ffc01c34ebec1aa95d3 similarity index 100% rename from sql/hive/src/test/resources/golden/mapjoin_subquery2-13-c876a518451059f17fc15e29f6f57951 rename to sql/hive/src/test/resources/golden/mapjoin_subquery2-13-258c97fa0a1b2ffc01c34ebec1aa95d3 diff --git a/sql/hive/src/test/resources/golden/mapjoin_subquery2-14-59fdb2842fbf4f530366f0237ff235e2 b/sql/hive/src/test/resources/golden/mapjoin_subquery2-14-65db5164b6b8a8919d2be7ba991b3c7 similarity index 100% rename from sql/hive/src/test/resources/golden/mapjoin_subquery2-14-59fdb2842fbf4f530366f0237ff235e2 rename to sql/hive/src/test/resources/golden/mapjoin_subquery2-14-65db5164b6b8a8919d2be7ba991b3c7 diff --git a/sql/hive/src/test/resources/golden/mapjoin_subquery2-15-c0a9763a930555f846a2576d003fb517 b/sql/hive/src/test/resources/golden/mapjoin_subquery2-15-f8432143e645a1886ae31aa568a16ce1 similarity index 100% rename from sql/hive/src/test/resources/golden/mapjoin_subquery2-15-c0a9763a930555f846a2576d003fb517 rename to sql/hive/src/test/resources/golden/mapjoin_subquery2-15-f8432143e645a1886ae31aa568a16ce1 diff --git a/sql/hive/src/test/resources/golden/mapjoin_subquery2-16-2f8ecc54049960ee4707f5e6f491fdf4 b/sql/hive/src/test/resources/golden/mapjoin_subquery2-16-512cf08aaaa421be72bb38dfa5f552bd similarity index 100% rename from sql/hive/src/test/resources/golden/mapjoin_subquery2-16-2f8ecc54049960ee4707f5e6f491fdf4 rename to sql/hive/src/test/resources/golden/mapjoin_subquery2-16-512cf08aaaa421be72bb38dfa5f552bd diff --git a/sql/hive/src/test/resources/golden/mapjoin_subquery2-2-2f8ecc54049960ee4707f5e6f491fdf4 b/sql/hive/src/test/resources/golden/mapjoin_subquery2-2-512cf08aaaa421be72bb38dfa5f552bd similarity index 100% rename from sql/hive/src/test/resources/golden/mapjoin_subquery2-2-2f8ecc54049960ee4707f5e6f491fdf4 rename to sql/hive/src/test/resources/golden/mapjoin_subquery2-2-512cf08aaaa421be72bb38dfa5f552bd diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index 40574a8f7c6ae..278273e8a9bba 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -1565,6 +1565,49 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { |drop table if exists sales; |drop table if exists things; """.stripMargin) + + createQueryTest("mapjoin_subquery2", + """ + |drop table if exists x; + |drop table if exists y; + |drop table if exists z; + | + |CREATE TABLE x (name STRING, id INT) + |ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'; + | + |CREATE TABLE y (id INT, name STRING) + |ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'; + | + |CREATE TABLE z (id INT, name STRING) + |ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'; + | + |load data local inpath '../../data/files/x.txt' INTO TABLE x; + |load data local inpath '../../data/files/y.txt' INTO TABLE y; + |load data local inpath '../../data/files/z.txt' INTO TABLE z; + | + |set hive.auto.convert.join=true; + |set hive.auto.convert.join.noconditionaltask=true; + |set hive.auto.convert.join.noconditionaltask.size=10000; + | + |-- Since the inputs are small, it should be automatically converted to mapjoin + | + |EXPLAIN + |SELECT subq.key1, subq.value1, subq.key2, subq.value2, z.id, z.name + |FROM + |(SELECT x.id as key1, x.name as value1, y.id as key2, y.name as value2 + | FROM y JOIN x ON (x.id = y.id)) subq + | JOIN z ON (subq.key1 = z.id); + | + |SELECT subq.key1, subq.value1, subq.key2, subq.value2, z.id, z.name + |FROM + |(SELECT x.id as key1, x.name as value1, y.id as key2, y.name as value2 + | FROM y JOIN x ON (x.id = y.id)) subq + | JOIN z ON (subq.key1 = z.id); + | + |drop table if exists x; + |drop table if exists y; + |drop table if exists z; + """.stripMargin) // scalastyle:on } From ca171c6038f47f29b6664c05f7c8cc8a8d41d279 Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Mon, 4 Jul 2016 12:40:08 +0530 Subject: [PATCH 10/25] nomore_ambiguous_table_col added --- ...s_table_col-0-7df4c4b6396d5f2a3a817384f37a8bf2} | 0 ...s_table_col-3-e47dfc804ca7645058093293b9b0725b} | 0 .../spark/sql/hive/execution/HiveQuerySuite.scala | 14 ++++++++++++++ 3 files changed, 14 insertions(+) rename sql/hive/src/test/resources/golden/{nomore_ambiguous_table_col-0-535a12e87c72793bfba96520a0ea251b => nomore_ambiguous_table_col-0-7df4c4b6396d5f2a3a817384f37a8bf2} (100%) rename sql/hive/src/test/resources/golden/{nomore_ambiguous_table_col-3-29a7783c88e234ba32eaf1401ca8cc22 => nomore_ambiguous_table_col-3-e47dfc804ca7645058093293b9b0725b} (100%) diff --git a/sql/hive/src/test/resources/golden/nomore_ambiguous_table_col-0-535a12e87c72793bfba96520a0ea251b b/sql/hive/src/test/resources/golden/nomore_ambiguous_table_col-0-7df4c4b6396d5f2a3a817384f37a8bf2 similarity index 100% rename from sql/hive/src/test/resources/golden/nomore_ambiguous_table_col-0-535a12e87c72793bfba96520a0ea251b rename to sql/hive/src/test/resources/golden/nomore_ambiguous_table_col-0-7df4c4b6396d5f2a3a817384f37a8bf2 diff --git a/sql/hive/src/test/resources/golden/nomore_ambiguous_table_col-3-29a7783c88e234ba32eaf1401ca8cc22 b/sql/hive/src/test/resources/golden/nomore_ambiguous_table_col-3-e47dfc804ca7645058093293b9b0725b similarity index 100% rename from sql/hive/src/test/resources/golden/nomore_ambiguous_table_col-3-29a7783c88e234ba32eaf1401ca8cc22 rename to sql/hive/src/test/resources/golden/nomore_ambiguous_table_col-3-e47dfc804ca7645058093293b9b0725b diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index 278273e8a9bba..436a8d7fa0f87 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -1608,6 +1608,20 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { |drop table if exists y; |drop table if exists z; """.stripMargin) + + createQueryTest("nomore_ambiguous_table_col", + """ + |-- was negative/ambiguous_table_col.q + | + |drop table if exists ambiguous; + |create table ambiguous (key string, value string); + | + |FROM src key + |INSERT OVERWRITE TABLE ambiguous SELECT key.key, key.value WHERE key.value < 'val_100'; + | + |drop table if exists ambiguous; + """.stripMargin) + // scalastyle:on } From 5925fa94365bbafde27f99b7cfbee752bc68147f Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Mon, 4 Jul 2016 12:48:48 +0530 Subject: [PATCH 11/25] partition_date added --- ...n_date-0-682e294d62b359eb69745ab9748f6d8c} | 0 ..._date-24-682e294d62b359eb69745ab9748f6d8c} | 0 ...n_date-5-5006463ce1f8455e99b23d872d0f7c59} | 0 ...n_date-6-c24618a05b069d06366328cd86a8d9c7} | 0 .../sql/hive/execution/HiveQuerySuite.scala | 62 +++++++++++++++++++ 5 files changed, 62 insertions(+) rename sql/hive/src/test/resources/golden/{partition_date-0-7ec1f3a845e2c49191460e15af30aa30 => partition_date-0-682e294d62b359eb69745ab9748f6d8c} (100%) rename sql/hive/src/test/resources/golden/{partition_date-24-7ec1f3a845e2c49191460e15af30aa30 => partition_date-24-682e294d62b359eb69745ab9748f6d8c} (100%) rename sql/hive/src/test/resources/golden/{partition_date-5-59355f4e222dcd4f77f51d15bd896f11 => partition_date-5-5006463ce1f8455e99b23d872d0f7c59} (100%) rename sql/hive/src/test/resources/golden/{partition_date-6-416ab10ac818d432cd3a81870d6e5164 => partition_date-6-c24618a05b069d06366328cd86a8d9c7} (100%) diff --git a/sql/hive/src/test/resources/golden/partition_date-0-7ec1f3a845e2c49191460e15af30aa30 b/sql/hive/src/test/resources/golden/partition_date-0-682e294d62b359eb69745ab9748f6d8c similarity index 100% rename from sql/hive/src/test/resources/golden/partition_date-0-7ec1f3a845e2c49191460e15af30aa30 rename to sql/hive/src/test/resources/golden/partition_date-0-682e294d62b359eb69745ab9748f6d8c diff --git a/sql/hive/src/test/resources/golden/partition_date-24-7ec1f3a845e2c49191460e15af30aa30 b/sql/hive/src/test/resources/golden/partition_date-24-682e294d62b359eb69745ab9748f6d8c similarity index 100% rename from sql/hive/src/test/resources/golden/partition_date-24-7ec1f3a845e2c49191460e15af30aa30 rename to sql/hive/src/test/resources/golden/partition_date-24-682e294d62b359eb69745ab9748f6d8c diff --git a/sql/hive/src/test/resources/golden/partition_date-5-59355f4e222dcd4f77f51d15bd896f11 b/sql/hive/src/test/resources/golden/partition_date-5-5006463ce1f8455e99b23d872d0f7c59 similarity index 100% rename from sql/hive/src/test/resources/golden/partition_date-5-59355f4e222dcd4f77f51d15bd896f11 rename to sql/hive/src/test/resources/golden/partition_date-5-5006463ce1f8455e99b23d872d0f7c59 diff --git a/sql/hive/src/test/resources/golden/partition_date-6-416ab10ac818d432cd3a81870d6e5164 b/sql/hive/src/test/resources/golden/partition_date-6-c24618a05b069d06366328cd86a8d9c7 similarity index 100% rename from sql/hive/src/test/resources/golden/partition_date-6-416ab10ac818d432cd3a81870d6e5164 rename to sql/hive/src/test/resources/golden/partition_date-6-c24618a05b069d06366328cd86a8d9c7 diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index 436a8d7fa0f87..5e6ec10745bb6 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -1622,6 +1622,68 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { |drop table if exists ambiguous; """.stripMargin) + createQueryTest("partition_date", + """ + |drop table if exists partition_date_1; + | + |create table partition_date_1 (key string, value string) partitioned by (dt date, region string); + | + |insert overwrite table partition_date_1 partition(dt='2000-01-01', region= '1') + | select * from src tablesample (10 rows); + |insert overwrite table partition_date_1 partition(dt='2000-01-01', region= '2') + | select * from src tablesample (5 rows); + |insert overwrite table partition_date_1 partition(dt='2013-12-10', region= '2020-20-20') + | select * from src tablesample (5 rows); + |insert overwrite table partition_date_1 partition(dt='2013-08-08', region= '1') + | select * from src tablesample (20 rows); + |insert overwrite table partition_date_1 partition(dt='2013-08-08', region= '10') + | select * from src tablesample (11 rows); + | + | + |select distinct dt from partition_date_1; + |select * from partition_date_1 where dt = '2000-01-01' and region = '2' order by key,value; + | + |-- 15 + |select count(*) from partition_date_1 where dt = date '2000-01-01'; + |-- 15. Also try with string value in predicate + |select count(*) from partition_date_1 where dt = '2000-01-01'; + |-- 5 + |select count(*) from partition_date_1 where dt = date '2000-01-01' and region = '2'; + |-- 11 + |select count(*) from partition_date_1 where dt = date '2013-08-08' and region = '10'; + |-- 30 + |select count(*) from partition_date_1 where region = '1'; + |-- 0 + |select count(*) from partition_date_1 where dt = date '2000-01-01' and region = '3'; + |-- 0 + |select count(*) from partition_date_1 where dt = date '1999-01-01'; + | + |-- Try other comparison operations + | + |-- 20 + |select count(*) from partition_date_1 where dt > date '2000-01-01' and region = '1'; + |-- 10 + |select count(*) from partition_date_1 where dt < date '2000-01-02' and region = '1'; + |-- 20 + |select count(*) from partition_date_1 where dt >= date '2000-01-02' and region = '1'; + |-- 10 + |select count(*) from partition_date_1 where dt <= date '2000-01-01' and region = '1'; + |-- 20 + |select count(*) from partition_date_1 where dt <> date '2000-01-01' and region = '1'; + |-- 10 + |select count(*) from partition_date_1 where dt between date '1999-12-30' and date '2000-01-03' and region = '1'; + | + | + |-- Try a string key with date-like strings + | + |-- 5 + |select count(*) from partition_date_1 where region = '2020-20-20'; + |-- 5 + |select count(*) from partition_date_1 where region > '2010-01-01'; + | + |drop table if exists partition_date_1; + """.stripMargin) + // scalastyle:on } From ab8955e8edb771220be5b982a5c6c17c2ac763cc Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Mon, 4 Jul 2016 12:51:58 +0530 Subject: [PATCH 12/25] partition_varchar1 added --- ...rchar1-0-cc382e7ea447854adeffae818d668592} | 0 ...char1-19-cc382e7ea447854adeffae818d668592} | 0 .../sql/hive/execution/HiveQuerySuite.scala | 47 +++++++++++++++++++ 3 files changed, 47 insertions(+) rename sql/hive/src/test/resources/golden/{partition_varchar1-0-ec359154c4bad3bfce1b7ab4914d1554 => partition_varchar1-0-cc382e7ea447854adeffae818d668592} (100%) rename sql/hive/src/test/resources/golden/{partition_varchar1-19-ec359154c4bad3bfce1b7ab4914d1554 => partition_varchar1-19-cc382e7ea447854adeffae818d668592} (100%) diff --git a/sql/hive/src/test/resources/golden/partition_varchar1-0-ec359154c4bad3bfce1b7ab4914d1554 b/sql/hive/src/test/resources/golden/partition_varchar1-0-cc382e7ea447854adeffae818d668592 similarity index 100% rename from sql/hive/src/test/resources/golden/partition_varchar1-0-ec359154c4bad3bfce1b7ab4914d1554 rename to sql/hive/src/test/resources/golden/partition_varchar1-0-cc382e7ea447854adeffae818d668592 diff --git a/sql/hive/src/test/resources/golden/partition_varchar1-19-ec359154c4bad3bfce1b7ab4914d1554 b/sql/hive/src/test/resources/golden/partition_varchar1-19-cc382e7ea447854adeffae818d668592 similarity index 100% rename from sql/hive/src/test/resources/golden/partition_varchar1-19-ec359154c4bad3bfce1b7ab4914d1554 rename to sql/hive/src/test/resources/golden/partition_varchar1-19-cc382e7ea447854adeffae818d668592 diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index 5e6ec10745bb6..93e3233e0416c 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -1684,6 +1684,53 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { |drop table if exists partition_date_1; """.stripMargin) + createQueryTest("partition_varchar1", + """ + |drop table if exists partition_varchar_1; + | + |create table partition_varchar_1 (key string, value varchar(20)) partitioned by (dt varchar(10), region int); + | + |insert overwrite table partition_varchar_1 partition(dt='2000-01-01', region=1) + | select * from src tablesample (10 rows); + |insert overwrite table partition_varchar_1 partition(dt='2000-01-01', region=2) + | select * from src tablesample (5 rows); + |insert overwrite table partition_varchar_1 partition(dt='2013-08-08', region=1) + | select * from src tablesample (20 rows); + |insert overwrite table partition_varchar_1 partition(dt='2013-08-08', region=10) + | select * from src tablesample (11 rows); + | + |select distinct dt from partition_varchar_1; + |select * from partition_varchar_1 where dt = '2000-01-01' and region = 2 order by key,value; + | + |-- 15 + |select count(*) from partition_varchar_1 where dt = '2000-01-01'; + |-- 5 + |select count(*) from partition_varchar_1 where dt = '2000-01-01' and region = 2; + |-- 11 + |select count(*) from partition_varchar_1 where dt = '2013-08-08' and region = 10; + |-- 30 + |select count(*) from partition_varchar_1 where region = 1; + |-- 0 + |select count(*) from partition_varchar_1 where dt = '2000-01-01' and region = 3; + |-- 0 + |select count(*) from partition_varchar_1 where dt = '1999-01-01'; + | + |-- Try other comparison operations + | + |-- 20 + |select count(*) from partition_varchar_1 where dt > '2000-01-01' and region = 1; + |-- 10 + |select count(*) from partition_varchar_1 where dt < '2000-01-02' and region = 1; + |-- 20 + |select count(*) from partition_varchar_1 where dt >= '2000-01-02' and region = 1; + |-- 10 + |select count(*) from partition_varchar_1 where dt <= '2000-01-01' and region = 1; + |-- 20 + |select count(*) from partition_varchar_1 where dt <> '2000-01-01' and region = 1; + | + |drop table if exists partition_varchar_1; + """.stripMargin) + // scalastyle:on } From 93f245b941464fd8bf70637b29731b0bfb2cf371 Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Mon, 4 Jul 2016 12:54:36 +0530 Subject: [PATCH 13/25] ppd_repeated_alias added --- ..._alias-0-7e8b9270313e4bf7d0ae60ef91ec0b33} | 0 ..._alias-1-26261aae41a7234afc96b372a469b4b3} | 0 ..._alias-8-7e8b9270313e4bf7d0ae60ef91ec0b33} | 0 ..._alias-9-26261aae41a7234afc96b372a469b4b3} | 0 .../sql/hive/execution/HiveQuerySuite.scala | 36 +++++++++++++++++++ 5 files changed, 36 insertions(+) rename sql/hive/src/test/resources/golden/{ppd_repeated_alias-0-89696914fad2d7b7bfc5b7729a7e7c34 => ppd_repeated_alias-0-7e8b9270313e4bf7d0ae60ef91ec0b33} (100%) rename sql/hive/src/test/resources/golden/{ppd_repeated_alias-1-a299c8b1a9f8c2772989a5454574f4e5 => ppd_repeated_alias-1-26261aae41a7234afc96b372a469b4b3} (100%) rename sql/hive/src/test/resources/golden/{ppd_repeated_alias-8-89696914fad2d7b7bfc5b7729a7e7c34 => ppd_repeated_alias-8-7e8b9270313e4bf7d0ae60ef91ec0b33} (100%) rename sql/hive/src/test/resources/golden/{ppd_repeated_alias-9-a299c8b1a9f8c2772989a5454574f4e5 => ppd_repeated_alias-9-26261aae41a7234afc96b372a469b4b3} (100%) diff --git a/sql/hive/src/test/resources/golden/ppd_repeated_alias-0-89696914fad2d7b7bfc5b7729a7e7c34 b/sql/hive/src/test/resources/golden/ppd_repeated_alias-0-7e8b9270313e4bf7d0ae60ef91ec0b33 similarity index 100% rename from sql/hive/src/test/resources/golden/ppd_repeated_alias-0-89696914fad2d7b7bfc5b7729a7e7c34 rename to sql/hive/src/test/resources/golden/ppd_repeated_alias-0-7e8b9270313e4bf7d0ae60ef91ec0b33 diff --git a/sql/hive/src/test/resources/golden/ppd_repeated_alias-1-a299c8b1a9f8c2772989a5454574f4e5 b/sql/hive/src/test/resources/golden/ppd_repeated_alias-1-26261aae41a7234afc96b372a469b4b3 similarity index 100% rename from sql/hive/src/test/resources/golden/ppd_repeated_alias-1-a299c8b1a9f8c2772989a5454574f4e5 rename to sql/hive/src/test/resources/golden/ppd_repeated_alias-1-26261aae41a7234afc96b372a469b4b3 diff --git a/sql/hive/src/test/resources/golden/ppd_repeated_alias-8-89696914fad2d7b7bfc5b7729a7e7c34 b/sql/hive/src/test/resources/golden/ppd_repeated_alias-8-7e8b9270313e4bf7d0ae60ef91ec0b33 similarity index 100% rename from sql/hive/src/test/resources/golden/ppd_repeated_alias-8-89696914fad2d7b7bfc5b7729a7e7c34 rename to sql/hive/src/test/resources/golden/ppd_repeated_alias-8-7e8b9270313e4bf7d0ae60ef91ec0b33 diff --git a/sql/hive/src/test/resources/golden/ppd_repeated_alias-9-a299c8b1a9f8c2772989a5454574f4e5 b/sql/hive/src/test/resources/golden/ppd_repeated_alias-9-26261aae41a7234afc96b372a469b4b3 similarity index 100% rename from sql/hive/src/test/resources/golden/ppd_repeated_alias-9-a299c8b1a9f8c2772989a5454574f4e5 rename to sql/hive/src/test/resources/golden/ppd_repeated_alias-9-26261aae41a7234afc96b372a469b4b3 diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index 93e3233e0416c..c8a9c486d987e 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -1731,6 +1731,42 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { |drop table if exists partition_varchar_1; """.stripMargin) + createQueryTest("ppd_repeated_alias", + """ + |drop table if exists pokes; + |drop table if exists pokes2; + |create table pokes (foo int, bar int, blah int); + |create table pokes2 (foo int, bar int, blah int); + | + |-- Q1: predicate should not be pushed on the right side of a left outer join + |explain + |SELECT a.foo as foo1, b.foo as foo2, b.bar + |FROM pokes a LEFT OUTER JOIN pokes2 b + |ON a.foo=b.foo + |WHERE b.bar=3; + | + |-- Q2: predicate should not be pushed on the right side of a left outer join + |explain + |SELECT * FROM + | (SELECT a.foo as foo1, b.foo as foo2, b.bar + | FROM pokes a LEFT OUTER JOIN pokes2 b + | ON a.foo=b.foo) a + |WHERE a.bar=3; + | + |-- Q3: predicate should be pushed + |explain + |SELECT * FROM + | (SELECT a.foo as foo1, b.foo as foo2, a.bar + | FROM pokes a JOIN pokes2 b + | ON a.foo=b.foo) a + |WHERE a.bar=3; + | + |-- Q4: here, the filter c.bar should be created under the first join but above the second + |explain select c.foo, d.bar from (select c.foo, b.bar, c.blah from pokes c left outer join pokes b on c.foo=b.foo) c left outer join pokes d where d.foo=1 and c.bar=2; + | + |drop table if exists pokes; + |drop table if exists pokes2; + """.stripMargin) // scalastyle:on } From 42e261df78243f4b5ba6cd9200779bcdc76bb1d2 Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Mon, 4 Jul 2016 12:57:52 +0530 Subject: [PATCH 14/25] push_or added --- ...5 => push_or-0-603322fe089469a7298d1ca8f1515842} | 0 .../spark/sql/hive/execution/HiveQuerySuite.scala | 13 +++++++++++++ 2 files changed, 13 insertions(+) rename sql/hive/src/test/resources/golden/{push_or-0-82eac35d076df1ee8020d57832f0f895 => push_or-0-603322fe089469a7298d1ca8f1515842} (100%) diff --git a/sql/hive/src/test/resources/golden/push_or-0-82eac35d076df1ee8020d57832f0f895 b/sql/hive/src/test/resources/golden/push_or-0-603322fe089469a7298d1ca8f1515842 similarity index 100% rename from sql/hive/src/test/resources/golden/push_or-0-82eac35d076df1ee8020d57832f0f895 rename to sql/hive/src/test/resources/golden/push_or-0-603322fe089469a7298d1ca8f1515842 diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index c8a9c486d987e..d1fa8f17dfa3b 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -1767,6 +1767,19 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { |drop table if exists pokes; |drop table if exists pokes2; """.stripMargin) + + createQueryTest("push_or", + """ + |drop table if exists push_or; + | + |create table push_or (key int, value string) partitioned by (ds string); + | + |insert overwrite table push_or partition (ds='2000-04-08') select * from src where key < 20 order by key; + |insert overwrite table push_or partition (ds='2000-04-09') select * from src where key < 20 order by key; + | + |explain extended select key, value, ds from push_or where ds='2000-04-09' or key=5 order by key, ds; + |select key, value, ds from push_or where ds='2000-04-09' or key=5 order by key, ds; + """.stripMargin) // scalastyle:on } From fa9a2cab44c832e9df051b48b750820f4acef8b4 Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Mon, 4 Jul 2016 13:01:52 +0530 Subject: [PATCH 15/25] reducesink_dedup added --- ..._dedup-0-b98f3142905595af658c90b4ad54f3dd} | 0 ..._dedup-1-d5d830d5b07e5a460162e15cc6f2b6bf} | 0 ..._dedup-2-5e57ee02c743e9a6f18db969e4cacd0c} | 0 .../sql/hive/execution/HiveQuerySuite.scala | 24 +++++++++++++++++++ 4 files changed, 24 insertions(+) rename sql/hive/src/test/resources/golden/{reducesink_dedup-0-d3f50875bd5dff172cf813fdb7d738eb => reducesink_dedup-0-b98f3142905595af658c90b4ad54f3dd} (100%) rename sql/hive/src/test/resources/golden/{reducesink_dedup-1-dda16565b98926fc3587de937b9401c7 => reducesink_dedup-1-d5d830d5b07e5a460162e15cc6f2b6bf} (100%) rename sql/hive/src/test/resources/golden/{reducesink_dedup-2-971f12c90619b02484c8e7db1b32f050 => reducesink_dedup-2-5e57ee02c743e9a6f18db969e4cacd0c} (100%) diff --git a/sql/hive/src/test/resources/golden/reducesink_dedup-0-d3f50875bd5dff172cf813fdb7d738eb b/sql/hive/src/test/resources/golden/reducesink_dedup-0-b98f3142905595af658c90b4ad54f3dd similarity index 100% rename from sql/hive/src/test/resources/golden/reducesink_dedup-0-d3f50875bd5dff172cf813fdb7d738eb rename to sql/hive/src/test/resources/golden/reducesink_dedup-0-b98f3142905595af658c90b4ad54f3dd diff --git a/sql/hive/src/test/resources/golden/reducesink_dedup-1-dda16565b98926fc3587de937b9401c7 b/sql/hive/src/test/resources/golden/reducesink_dedup-1-d5d830d5b07e5a460162e15cc6f2b6bf similarity index 100% rename from sql/hive/src/test/resources/golden/reducesink_dedup-1-dda16565b98926fc3587de937b9401c7 rename to sql/hive/src/test/resources/golden/reducesink_dedup-1-d5d830d5b07e5a460162e15cc6f2b6bf diff --git a/sql/hive/src/test/resources/golden/reducesink_dedup-2-971f12c90619b02484c8e7db1b32f050 b/sql/hive/src/test/resources/golden/reducesink_dedup-2-5e57ee02c743e9a6f18db969e4cacd0c similarity index 100% rename from sql/hive/src/test/resources/golden/reducesink_dedup-2-971f12c90619b02484c8e7db1b32f050 rename to sql/hive/src/test/resources/golden/reducesink_dedup-2-5e57ee02c743e9a6f18db969e4cacd0c diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index d1fa8f17dfa3b..97a95f758d882 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -1780,6 +1780,30 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { |explain extended select key, value, ds from push_or where ds='2000-04-09' or key=5 order by key, ds; |select key, value, ds from push_or where ds='2000-04-09' or key=5 order by key, ds; """.stripMargin) + + createQueryTest("reducesink_dedup", + """ + |DROP TABLE IF EXISTS part; + | + |-- data setup + |CREATE TABLE part( + | p_partkey INT, + | p_name STRING, + | p_mfgr STRING, + | p_brand STRING, + | p_type STRING, + | p_size INT, + | p_container STRING, + | p_retailprice DOUBLE, + | p_comment STRING + |); + | + | + |select p_name + |from (select p_name from part distribute by 1 sort by 1) p + |distribute by 1 sort by 1 + |; + """.stripMargin) // scalastyle:on } From 7ad0d4d59614bd2e49731ce6db8b858e9333f3a3 Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Mon, 4 Jul 2016 13:09:22 +0530 Subject: [PATCH 16/25] subquery_in added --- ...ery_in-0-b98f3142905595af658c90b4ad54f3dd} | 0 ...ery_in-1-d5d830d5b07e5a460162e15cc6f2b6bf} | 0 ...ry_in-10-b5a4815c7fbd8db27cbcae924cb03f04} | 0 ...ry_in-11-db542fa09971125b8d0c08b86cbd78a2} | 0 ...ry_in-12-c02f884fcdb510a8a4f96dba2a3f1125} | 0 ...ry_in-13-b1ce6eac7e503201bdb1a772ec65f2bd} | 0 ...ry_in-14-4fd9d26eccc76c067a9f261c579b3e45} | 0 ...ry_in-15-8af71a3ea0be3a1ac9ad731adeca3c4b} | 0 ...ry_in-16-91fe44cd9d37b71be1f48bf87de90aa8} | 0 ...ry_in-17-ce25b95a93f415dc014ec27bd31e6c86} | 0 ...ry_in-18-846993459b4a24baf5824a6b3e71fcee} | 0 ...ry_in-19-4b766f1d5705c2c6d85b79cd35a66002} | 0 ...ery_in-3-af32648a8ed9ca6193a27d7b4a13d2d6} | 0 ...uery_in-6-cd9225b1dc79bf9bb0cd863c907291d} | 0 ...ery_in-7-5014a6f25ea5b5d3a9b8861c0e32da51} | 0 ...uery_in-8-f9d61a4665b37eadd78f1481b5325dc} | 0 ...ery_in-9-c5a406131bad355e8a2986acb49cf8a7} | 0 .../sql/hive/execution/HiveQuerySuite.scala | 167 ++++++++++++++++++ 18 files changed, 167 insertions(+) rename sql/hive/src/test/resources/golden/{subquery_in-0-d3f50875bd5dff172cf813fdb7d738eb => subquery_in-0-b98f3142905595af658c90b4ad54f3dd} (100%) rename sql/hive/src/test/resources/golden/{subquery_in-1-dda16565b98926fc3587de937b9401c7 => subquery_in-1-d5d830d5b07e5a460162e15cc6f2b6bf} (100%) rename sql/hive/src/test/resources/golden/{subquery_in-10-3cd5ddc0f57e69745cbca1d5a8dd87c4 => subquery_in-10-b5a4815c7fbd8db27cbcae924cb03f04} (100%) rename sql/hive/src/test/resources/golden/{subquery_in-11-21659892bff071ffb0dec9134dd465a8 => subquery_in-11-db542fa09971125b8d0c08b86cbd78a2} (100%) rename sql/hive/src/test/resources/golden/{subquery_in-12-79fc971b8a399c25e1e2a1a30e08f336 => subquery_in-12-c02f884fcdb510a8a4f96dba2a3f1125} (100%) rename sql/hive/src/test/resources/golden/{subquery_in-13-f17e8105a6efd193ef1065110d1145a6 => subquery_in-13-b1ce6eac7e503201bdb1a772ec65f2bd} (100%) rename sql/hive/src/test/resources/golden/{subquery_in-14-df6d4aad4f4c5d0675b1fbceac367fe2 => subquery_in-14-4fd9d26eccc76c067a9f261c579b3e45} (100%) rename sql/hive/src/test/resources/golden/{subquery_in-15-bacf85b0769b4030514a6f96c64d1ff7 => subquery_in-15-8af71a3ea0be3a1ac9ad731adeca3c4b} (100%) rename sql/hive/src/test/resources/golden/{subquery_in-16-d51e0128520c31dbe041ffa4ae22dd4b => subquery_in-16-91fe44cd9d37b71be1f48bf87de90aa8} (100%) rename sql/hive/src/test/resources/golden/{subquery_in-17-5f132cdb7fc12e6389d620472df5ba7f => subquery_in-17-ce25b95a93f415dc014ec27bd31e6c86} (100%) rename sql/hive/src/test/resources/golden/{subquery_in-18-f80281d529559f7f35ee5b42d53dd2ca => subquery_in-18-846993459b4a24baf5824a6b3e71fcee} (100%) rename sql/hive/src/test/resources/golden/{subquery_in-19-466013b596cc4160456daab670684af6 => subquery_in-19-4b766f1d5705c2c6d85b79cd35a66002} (100%) rename sql/hive/src/test/resources/golden/{subquery_in-3-42f922e862f882b9927abf566fe43050 => subquery_in-3-af32648a8ed9ca6193a27d7b4a13d2d6} (100%) rename sql/hive/src/test/resources/golden/{subquery_in-6-8b37b644ebdb9007c609043c6c855cb0 => subquery_in-6-cd9225b1dc79bf9bb0cd863c907291d} (100%) rename sql/hive/src/test/resources/golden/{subquery_in-7-208c9201161f60c2c7e521b0b33f0b19 => subquery_in-7-5014a6f25ea5b5d3a9b8861c0e32da51} (100%) rename sql/hive/src/test/resources/golden/{subquery_in-8-d7212bf1f2c9e019b7142314b823a979 => subquery_in-8-f9d61a4665b37eadd78f1481b5325dc} (100%) rename sql/hive/src/test/resources/golden/{subquery_in-9-3d9f3ef5aa4fbb982a28109af8db9805 => subquery_in-9-c5a406131bad355e8a2986acb49cf8a7} (100%) diff --git a/sql/hive/src/test/resources/golden/subquery_in-0-d3f50875bd5dff172cf813fdb7d738eb b/sql/hive/src/test/resources/golden/subquery_in-0-b98f3142905595af658c90b4ad54f3dd similarity index 100% rename from sql/hive/src/test/resources/golden/subquery_in-0-d3f50875bd5dff172cf813fdb7d738eb rename to sql/hive/src/test/resources/golden/subquery_in-0-b98f3142905595af658c90b4ad54f3dd diff --git a/sql/hive/src/test/resources/golden/subquery_in-1-dda16565b98926fc3587de937b9401c7 b/sql/hive/src/test/resources/golden/subquery_in-1-d5d830d5b07e5a460162e15cc6f2b6bf similarity index 100% rename from sql/hive/src/test/resources/golden/subquery_in-1-dda16565b98926fc3587de937b9401c7 rename to sql/hive/src/test/resources/golden/subquery_in-1-d5d830d5b07e5a460162e15cc6f2b6bf diff --git a/sql/hive/src/test/resources/golden/subquery_in-10-3cd5ddc0f57e69745cbca1d5a8dd87c4 b/sql/hive/src/test/resources/golden/subquery_in-10-b5a4815c7fbd8db27cbcae924cb03f04 similarity index 100% rename from sql/hive/src/test/resources/golden/subquery_in-10-3cd5ddc0f57e69745cbca1d5a8dd87c4 rename to sql/hive/src/test/resources/golden/subquery_in-10-b5a4815c7fbd8db27cbcae924cb03f04 diff --git a/sql/hive/src/test/resources/golden/subquery_in-11-21659892bff071ffb0dec9134dd465a8 b/sql/hive/src/test/resources/golden/subquery_in-11-db542fa09971125b8d0c08b86cbd78a2 similarity index 100% rename from sql/hive/src/test/resources/golden/subquery_in-11-21659892bff071ffb0dec9134dd465a8 rename to sql/hive/src/test/resources/golden/subquery_in-11-db542fa09971125b8d0c08b86cbd78a2 diff --git a/sql/hive/src/test/resources/golden/subquery_in-12-79fc971b8a399c25e1e2a1a30e08f336 b/sql/hive/src/test/resources/golden/subquery_in-12-c02f884fcdb510a8a4f96dba2a3f1125 similarity index 100% rename from sql/hive/src/test/resources/golden/subquery_in-12-79fc971b8a399c25e1e2a1a30e08f336 rename to sql/hive/src/test/resources/golden/subquery_in-12-c02f884fcdb510a8a4f96dba2a3f1125 diff --git a/sql/hive/src/test/resources/golden/subquery_in-13-f17e8105a6efd193ef1065110d1145a6 b/sql/hive/src/test/resources/golden/subquery_in-13-b1ce6eac7e503201bdb1a772ec65f2bd similarity index 100% rename from sql/hive/src/test/resources/golden/subquery_in-13-f17e8105a6efd193ef1065110d1145a6 rename to sql/hive/src/test/resources/golden/subquery_in-13-b1ce6eac7e503201bdb1a772ec65f2bd diff --git a/sql/hive/src/test/resources/golden/subquery_in-14-df6d4aad4f4c5d0675b1fbceac367fe2 b/sql/hive/src/test/resources/golden/subquery_in-14-4fd9d26eccc76c067a9f261c579b3e45 similarity index 100% rename from sql/hive/src/test/resources/golden/subquery_in-14-df6d4aad4f4c5d0675b1fbceac367fe2 rename to sql/hive/src/test/resources/golden/subquery_in-14-4fd9d26eccc76c067a9f261c579b3e45 diff --git a/sql/hive/src/test/resources/golden/subquery_in-15-bacf85b0769b4030514a6f96c64d1ff7 b/sql/hive/src/test/resources/golden/subquery_in-15-8af71a3ea0be3a1ac9ad731adeca3c4b similarity index 100% rename from sql/hive/src/test/resources/golden/subquery_in-15-bacf85b0769b4030514a6f96c64d1ff7 rename to sql/hive/src/test/resources/golden/subquery_in-15-8af71a3ea0be3a1ac9ad731adeca3c4b diff --git a/sql/hive/src/test/resources/golden/subquery_in-16-d51e0128520c31dbe041ffa4ae22dd4b b/sql/hive/src/test/resources/golden/subquery_in-16-91fe44cd9d37b71be1f48bf87de90aa8 similarity index 100% rename from sql/hive/src/test/resources/golden/subquery_in-16-d51e0128520c31dbe041ffa4ae22dd4b rename to sql/hive/src/test/resources/golden/subquery_in-16-91fe44cd9d37b71be1f48bf87de90aa8 diff --git a/sql/hive/src/test/resources/golden/subquery_in-17-5f132cdb7fc12e6389d620472df5ba7f b/sql/hive/src/test/resources/golden/subquery_in-17-ce25b95a93f415dc014ec27bd31e6c86 similarity index 100% rename from sql/hive/src/test/resources/golden/subquery_in-17-5f132cdb7fc12e6389d620472df5ba7f rename to sql/hive/src/test/resources/golden/subquery_in-17-ce25b95a93f415dc014ec27bd31e6c86 diff --git a/sql/hive/src/test/resources/golden/subquery_in-18-f80281d529559f7f35ee5b42d53dd2ca b/sql/hive/src/test/resources/golden/subquery_in-18-846993459b4a24baf5824a6b3e71fcee similarity index 100% rename from sql/hive/src/test/resources/golden/subquery_in-18-f80281d529559f7f35ee5b42d53dd2ca rename to sql/hive/src/test/resources/golden/subquery_in-18-846993459b4a24baf5824a6b3e71fcee diff --git a/sql/hive/src/test/resources/golden/subquery_in-19-466013b596cc4160456daab670684af6 b/sql/hive/src/test/resources/golden/subquery_in-19-4b766f1d5705c2c6d85b79cd35a66002 similarity index 100% rename from sql/hive/src/test/resources/golden/subquery_in-19-466013b596cc4160456daab670684af6 rename to sql/hive/src/test/resources/golden/subquery_in-19-4b766f1d5705c2c6d85b79cd35a66002 diff --git a/sql/hive/src/test/resources/golden/subquery_in-3-42f922e862f882b9927abf566fe43050 b/sql/hive/src/test/resources/golden/subquery_in-3-af32648a8ed9ca6193a27d7b4a13d2d6 similarity index 100% rename from sql/hive/src/test/resources/golden/subquery_in-3-42f922e862f882b9927abf566fe43050 rename to sql/hive/src/test/resources/golden/subquery_in-3-af32648a8ed9ca6193a27d7b4a13d2d6 diff --git a/sql/hive/src/test/resources/golden/subquery_in-6-8b37b644ebdb9007c609043c6c855cb0 b/sql/hive/src/test/resources/golden/subquery_in-6-cd9225b1dc79bf9bb0cd863c907291d similarity index 100% rename from sql/hive/src/test/resources/golden/subquery_in-6-8b37b644ebdb9007c609043c6c855cb0 rename to sql/hive/src/test/resources/golden/subquery_in-6-cd9225b1dc79bf9bb0cd863c907291d diff --git a/sql/hive/src/test/resources/golden/subquery_in-7-208c9201161f60c2c7e521b0b33f0b19 b/sql/hive/src/test/resources/golden/subquery_in-7-5014a6f25ea5b5d3a9b8861c0e32da51 similarity index 100% rename from sql/hive/src/test/resources/golden/subquery_in-7-208c9201161f60c2c7e521b0b33f0b19 rename to sql/hive/src/test/resources/golden/subquery_in-7-5014a6f25ea5b5d3a9b8861c0e32da51 diff --git a/sql/hive/src/test/resources/golden/subquery_in-8-d7212bf1f2c9e019b7142314b823a979 b/sql/hive/src/test/resources/golden/subquery_in-8-f9d61a4665b37eadd78f1481b5325dc similarity index 100% rename from sql/hive/src/test/resources/golden/subquery_in-8-d7212bf1f2c9e019b7142314b823a979 rename to sql/hive/src/test/resources/golden/subquery_in-8-f9d61a4665b37eadd78f1481b5325dc diff --git a/sql/hive/src/test/resources/golden/subquery_in-9-3d9f3ef5aa4fbb982a28109af8db9805 b/sql/hive/src/test/resources/golden/subquery_in-9-c5a406131bad355e8a2986acb49cf8a7 similarity index 100% rename from sql/hive/src/test/resources/golden/subquery_in-9-3d9f3ef5aa4fbb982a28109af8db9805 rename to sql/hive/src/test/resources/golden/subquery_in-9-c5a406131bad355e8a2986acb49cf8a7 diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index 97a95f758d882..dc33e9ba90ac0 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -1804,6 +1804,173 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { |distribute by 1 sort by 1 |; """.stripMargin) + + createQueryTest("subquery_in", + """ + |DROP TABLE IF EXISTS part; + | + |-- data setup + |CREATE TABLE part( + | p_partkey INT, + | p_name STRING, + | p_mfgr STRING, + | p_brand STRING, + | p_type STRING, + | p_size INT, + | p_container STRING, + | p_retailprice DOUBLE, + | p_comment STRING + |); + | + |LOAD DATA LOCAL INPATH '../../data/files/part_tiny.txt' overwrite into table part; + | + |DROP TABLE IF EXISTS lineitem; + |CREATE TABLE lineitem (L_ORDERKEY INT, + | L_PARTKEY INT, + | L_SUPPKEY INT, + | L_LINENUMBER INT, + | L_QUANTITY DOUBLE, + | L_EXTENDEDPRICE DOUBLE, + | L_DISCOUNT DOUBLE, + | L_TAX DOUBLE, + | L_RETURNFLAG STRING, + | L_LINESTATUS STRING, + | l_shipdate STRING, + | L_COMMITDATE STRING, + | L_RECEIPTDATE STRING, + | L_SHIPINSTRUCT STRING, + | L_SHIPMODE STRING, + | L_COMMENT STRING) + |ROW FORMAT DELIMITED + |FIELDS TERMINATED BY '|'; + | + |LOAD DATA LOCAL INPATH '../../data/files/lineitem.txt' OVERWRITE INTO TABLE lineitem; + | + |-- non agg, non corr + |explain + | select * + |from src + |where src.key in (select key from src s1 where s1.key > '9') + |; + | + |select * + |from src + |where src.key in (select key from src s1 where s1.key > '9') + |order by key + |; + | + |-- non agg, corr + |explain + |select * + |from src b + |where b.key in + | (select a.key + | from src a + | where b.value = a.value and a.key > '9' + | ) + |; + | + |select * + |from src b + |where b.key in + | (select a.key + | from src a + | where b.value = a.value and a.key > '9' + | ) + |order by b.key + |; + | + |-- agg, non corr + |explain + |select p_name, p_size + |from + |part where part.p_size in + | (select avg(p_size) + | from (select p_size, rank() over(partition by p_mfgr order by p_size) as r from part) a + | where r <= 2 + | ) + |; + |select p_name, p_size + |from + |part where part.p_size in + | (select avg(p_size) + | from (select p_size, rank() over(partition by p_mfgr order by p_size) as r from part) a + | where r <= 2 + | ) + |order by p_name + |; + | + |-- agg, corr + |explain + |select p_mfgr, p_name, p_size + |from part b where b.p_size in + | (select min(p_size) + | from (select p_mfgr, p_size, rank() over(partition by p_mfgr order by p_size) as r from part) a + | where r <= 2 and b.p_mfgr = a.p_mfgr + | ) + |; + | + |select p_mfgr, p_name, p_size + |from part b where b.p_size in + | (select min(p_size) + | from (select p_mfgr, p_size, rank() over(partition by p_mfgr order by p_size) as r from part) a + | where r <= 2 and b.p_mfgr = a.p_mfgr + | ) + |order by p_mfgr, p_name, p_size + |; + | + |-- distinct, corr + |explain + |select * + |from src b + |where b.key in + | (select distinct a.key + | from src a + | where b.value = a.value and a.key > '9' + | ) + |; + | + |select * + |from src b + |where b.key in + | (select distinct a.key + | from src a + | where b.value = a.value and a.key > '9' + | ) + |order by b.key + |; + | + |-- non agg, non corr, windowing + |select p_mfgr, p_name, p_size + |from part + |where part.p_size in + | (select first_value(p_size) over(partition by p_mfgr order by p_size) from part) + |order by p_mfgr, p_name, p_size + |; + | + |-- non agg, non corr, with join in Parent Query + |explain + |select p.p_partkey, li.l_suppkey + |from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li on p.p_partkey = li.l_partkey + |where li.l_linenumber = 1 and + | li.l_orderkey in (select l_orderkey from lineitem where l_shipmode = 'AIR') + |; + | + |select p.p_partkey, li.l_suppkey + |from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li on p.p_partkey = li.l_partkey + |where li.l_linenumber = 1 and + | li.l_orderkey in (select l_orderkey from lineitem where l_shipmode = 'AIR') + |order by p.p_partkey, li.l_suppkey + |; + | + |-- non agg, corr, with join in Parent Query + |select p.p_partkey, li.l_suppkey + |from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li on p.p_partkey = li.l_partkey + |where li.l_linenumber = 1 and + | li.l_orderkey in (select l_orderkey from lineitem where l_shipmode = 'AIR' and l_linenumber = li.l_linenumber) + |order by p.p_partkey, li.l_suppkey + |; + """.stripMargin) // scalastyle:on } From 135eb840e21da17c4ea793f3de917765a978d34b Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Mon, 4 Jul 2016 13:17:16 +0530 Subject: [PATCH 17/25] subquery_notin_having added --- ...having-0-b98f3142905595af658c90b4ad54f3dd} | 0 ...having-1-d5d830d5b07e5a460162e15cc6f2b6bf} | 0 ...having-3-6560240323f5f079cff9186da8697694} | 0 ..._having-4-9653402378a230fbe6bcea1919ca8b7} | 0 ...having-5-dfd7fa583955e2e66fc988086fc7e53c} | 0 ...having-6-86fcb369bf5d4c301c9b980ed78d4772} | 0 ...having-7-5d9271b588bd7795483bf590f17d8bac} | 0 .../sql/hive/execution/HiveQuerySuite.scala | 78 +++++++++++++++++++ 8 files changed, 78 insertions(+) rename sql/hive/src/test/resources/golden/{subquery_notin_having-0-d3f50875bd5dff172cf813fdb7d738eb => subquery_notin_having-0-b98f3142905595af658c90b4ad54f3dd} (100%) rename sql/hive/src/test/resources/golden/{subquery_notin_having-1-dda16565b98926fc3587de937b9401c7 => subquery_notin_having-1-d5d830d5b07e5a460162e15cc6f2b6bf} (100%) rename sql/hive/src/test/resources/golden/{subquery_notin_having-3-21a44539fd357dc260687003554fe02a => subquery_notin_having-3-6560240323f5f079cff9186da8697694} (100%) rename sql/hive/src/test/resources/golden/{subquery_notin_having-4-dea2fabba75cc13e7fa8df072f6b557b => subquery_notin_having-4-9653402378a230fbe6bcea1919ca8b7} (100%) rename sql/hive/src/test/resources/golden/{subquery_notin_having-5-341feddde788c15197d08d7969dafe19 => subquery_notin_having-5-dfd7fa583955e2e66fc988086fc7e53c} (100%) rename sql/hive/src/test/resources/golden/{subquery_notin_having-6-7ed33e3bcdc0728a69995ef0b2fa54a5 => subquery_notin_having-6-86fcb369bf5d4c301c9b980ed78d4772} (100%) rename sql/hive/src/test/resources/golden/{subquery_notin_having-7-44bdb73da0c1f4089b6edb43614e3e04 => subquery_notin_having-7-5d9271b588bd7795483bf590f17d8bac} (100%) diff --git a/sql/hive/src/test/resources/golden/subquery_notin_having-0-d3f50875bd5dff172cf813fdb7d738eb b/sql/hive/src/test/resources/golden/subquery_notin_having-0-b98f3142905595af658c90b4ad54f3dd similarity index 100% rename from sql/hive/src/test/resources/golden/subquery_notin_having-0-d3f50875bd5dff172cf813fdb7d738eb rename to sql/hive/src/test/resources/golden/subquery_notin_having-0-b98f3142905595af658c90b4ad54f3dd diff --git a/sql/hive/src/test/resources/golden/subquery_notin_having-1-dda16565b98926fc3587de937b9401c7 b/sql/hive/src/test/resources/golden/subquery_notin_having-1-d5d830d5b07e5a460162e15cc6f2b6bf similarity index 100% rename from sql/hive/src/test/resources/golden/subquery_notin_having-1-dda16565b98926fc3587de937b9401c7 rename to sql/hive/src/test/resources/golden/subquery_notin_having-1-d5d830d5b07e5a460162e15cc6f2b6bf diff --git a/sql/hive/src/test/resources/golden/subquery_notin_having-3-21a44539fd357dc260687003554fe02a b/sql/hive/src/test/resources/golden/subquery_notin_having-3-6560240323f5f079cff9186da8697694 similarity index 100% rename from sql/hive/src/test/resources/golden/subquery_notin_having-3-21a44539fd357dc260687003554fe02a rename to sql/hive/src/test/resources/golden/subquery_notin_having-3-6560240323f5f079cff9186da8697694 diff --git a/sql/hive/src/test/resources/golden/subquery_notin_having-4-dea2fabba75cc13e7fa8df072f6b557b b/sql/hive/src/test/resources/golden/subquery_notin_having-4-9653402378a230fbe6bcea1919ca8b7 similarity index 100% rename from sql/hive/src/test/resources/golden/subquery_notin_having-4-dea2fabba75cc13e7fa8df072f6b557b rename to sql/hive/src/test/resources/golden/subquery_notin_having-4-9653402378a230fbe6bcea1919ca8b7 diff --git a/sql/hive/src/test/resources/golden/subquery_notin_having-5-341feddde788c15197d08d7969dafe19 b/sql/hive/src/test/resources/golden/subquery_notin_having-5-dfd7fa583955e2e66fc988086fc7e53c similarity index 100% rename from sql/hive/src/test/resources/golden/subquery_notin_having-5-341feddde788c15197d08d7969dafe19 rename to sql/hive/src/test/resources/golden/subquery_notin_having-5-dfd7fa583955e2e66fc988086fc7e53c diff --git a/sql/hive/src/test/resources/golden/subquery_notin_having-6-7ed33e3bcdc0728a69995ef0b2fa54a5 b/sql/hive/src/test/resources/golden/subquery_notin_having-6-86fcb369bf5d4c301c9b980ed78d4772 similarity index 100% rename from sql/hive/src/test/resources/golden/subquery_notin_having-6-7ed33e3bcdc0728a69995ef0b2fa54a5 rename to sql/hive/src/test/resources/golden/subquery_notin_having-6-86fcb369bf5d4c301c9b980ed78d4772 diff --git a/sql/hive/src/test/resources/golden/subquery_notin_having-7-44bdb73da0c1f4089b6edb43614e3e04 b/sql/hive/src/test/resources/golden/subquery_notin_having-7-5d9271b588bd7795483bf590f17d8bac similarity index 100% rename from sql/hive/src/test/resources/golden/subquery_notin_having-7-44bdb73da0c1f4089b6edb43614e3e04 rename to sql/hive/src/test/resources/golden/subquery_notin_having-7-5d9271b588bd7795483bf590f17d8bac diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index dc33e9ba90ac0..c9635b47a13e3 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -1971,6 +1971,84 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { |order by p.p_partkey, li.l_suppkey |; """.stripMargin) + + createQueryTest("subquery_notin_having", + """ + |DROP TABLE IF EXISTS part; + | + |-- data setup + |CREATE TABLE part( + | p_partkey INT, + | p_name STRING, + | p_mfgr STRING, + | p_brand STRING, + | p_type STRING, + | p_size INT, + | p_container STRING, + | p_retailprice DOUBLE, + | p_comment STRING + |); + | + |LOAD DATA LOCAL INPATH '../../data/files/part_tiny.txt' overwrite into table part; + | + | + |-- non agg, non corr + |explain + |select key, count(*) + |from src + |group by key + |having key not in + | ( select key from src s1 + | where s1.key > '12' + | ) + |; + | + |-- non agg, corr + |explain + |select b.p_mfgr, min(p_retailprice) + |from part b + |group by b.p_mfgr + |having b.p_mfgr not in + | (select p_mfgr + | from (select p_mfgr, min(p_retailprice) l, max(p_retailprice) r, avg(p_retailprice) a from part group by p_mfgr) a + | where min(p_retailprice) = l and r - l > 600 + | ) + |; + | + |select b.p_mfgr, min(p_retailprice) + |from part b + |group by b.p_mfgr + |having b.p_mfgr not in + | (select p_mfgr + | from (select p_mfgr, min(p_retailprice) l, max(p_retailprice) r, avg(p_retailprice) a from part group by p_mfgr) a + | where min(p_retailprice) = l and r - l > 600 + | ) + |; + | + |-- agg, non corr + |explain + |select b.p_mfgr, min(p_retailprice) + |from part b + |group by b.p_mfgr + |having b.p_mfgr not in + | (select p_mfgr + | from part a + | group by p_mfgr + | having max(p_retailprice) - min(p_retailprice) > 600 + | ) + |; + | + |select b.p_mfgr, min(p_retailprice) + |from part b + |group by b.p_mfgr + |having b.p_mfgr not in + | (select p_mfgr + | from part a + | group by p_mfgr + | having max(p_retailprice) - min(p_retailprice) > 600 + | ) + |; + """.stripMargin) // scalastyle:on } From e423a9bf7e5a7a22588b87f01098f3b5c828f5c9 Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Mon, 4 Jul 2016 13:21:32 +0530 Subject: [PATCH 18/25] timestamp_3 added --- ...tamp_3-1-615fd1da17e48f40e60b28a81c688251} | 0 ...amp_3-14-615fd1da17e48f40e60b28a81c688251} | 0 .../sql/hive/execution/HiveQuerySuite.scala | 26 +++++++++++++++++++ 3 files changed, 26 insertions(+) rename sql/hive/src/test/resources/golden/{timestamp_3-1-165256158e3db1ce19c3c9db3c8011d2 => timestamp_3-1-615fd1da17e48f40e60b28a81c688251} (100%) rename sql/hive/src/test/resources/golden/{timestamp_3-14-165256158e3db1ce19c3c9db3c8011d2 => timestamp_3-14-615fd1da17e48f40e60b28a81c688251} (100%) diff --git a/sql/hive/src/test/resources/golden/timestamp_3-1-165256158e3db1ce19c3c9db3c8011d2 b/sql/hive/src/test/resources/golden/timestamp_3-1-615fd1da17e48f40e60b28a81c688251 similarity index 100% rename from sql/hive/src/test/resources/golden/timestamp_3-1-165256158e3db1ce19c3c9db3c8011d2 rename to sql/hive/src/test/resources/golden/timestamp_3-1-615fd1da17e48f40e60b28a81c688251 diff --git a/sql/hive/src/test/resources/golden/timestamp_3-14-165256158e3db1ce19c3c9db3c8011d2 b/sql/hive/src/test/resources/golden/timestamp_3-14-615fd1da17e48f40e60b28a81c688251 similarity index 100% rename from sql/hive/src/test/resources/golden/timestamp_3-14-165256158e3db1ce19c3c9db3c8011d2 rename to sql/hive/src/test/resources/golden/timestamp_3-14-615fd1da17e48f40e60b28a81c688251 diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index c9635b47a13e3..6860c16051483 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -2049,6 +2049,32 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { | ) |; """.stripMargin) + + createQueryTest("timestamp_3", + """ + |set hive.fetch.task.conversion=more; + | + |drop table if exists timestamp_3; + | + |create table timestamp_3 (t timestamp); + |alter table timestamp_3 set serde 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'; + | + |insert overwrite table timestamp_3 + | select cast(cast('1.3041352164485E9' as double) as timestamp) from src tablesample (1 rows); + |select cast(t as boolean) from timestamp_3 limit 1; + |select cast(t as tinyint) from timestamp_3 limit 1; + |select cast(t as smallint) from timestamp_3 limit 1; + |select cast(t as int) from timestamp_3 limit 1; + |select cast(t as bigint) from timestamp_3 limit 1; + |select cast(t as float) from timestamp_3 limit 1; + |select cast(t as double) from timestamp_3 limit 1; + |select cast(t as string) from timestamp_3 limit 1; + | + |select t, sum(t), count(*), sum(t)/count(*), avg(t) from timestamp_3 group by t; + | + |drop table if exists timestamp_3; + """.stripMargin) + // scalastyle:on } From 784409304dc59c760bc912eccd026a5c3ed6c96e Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Mon, 4 Jul 2016 13:27:58 +0530 Subject: [PATCH 19/25] timestamp_lazy added --- ... timestamp_lazy-0-1bb39555e70b1cc130f061ff4dab7d87} | 0 .../spark/sql/hive/execution/HiveQuerySuite.scala | 10 ++++++++++ 2 files changed, 10 insertions(+) rename sql/hive/src/test/resources/golden/{timestamp_lazy-0-c0b1d928c9dea9b247ace0a056586594 => timestamp_lazy-0-1bb39555e70b1cc130f061ff4dab7d87} (100%) diff --git a/sql/hive/src/test/resources/golden/timestamp_lazy-0-c0b1d928c9dea9b247ace0a056586594 b/sql/hive/src/test/resources/golden/timestamp_lazy-0-1bb39555e70b1cc130f061ff4dab7d87 similarity index 100% rename from sql/hive/src/test/resources/golden/timestamp_lazy-0-c0b1d928c9dea9b247ace0a056586594 rename to sql/hive/src/test/resources/golden/timestamp_lazy-0-1bb39555e70b1cc130f061ff4dab7d87 diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index 6860c16051483..73d71a1475483 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -2075,6 +2075,16 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { |drop table if exists timestamp_3; """.stripMargin) + createQueryTest("timestamp_lazy", + """ + |drop table if exists timestamp_lazy; + |create table timestamp_lazy (t timestamp, key string, value string); + |insert overwrite table timestamp_lazy select cast('2011-01-01 01:01:01' as timestamp), key, value from src tablesample (5 rows); + | + |select t,key,value from timestamp_lazy ORDER BY key ASC, value ASC; + |select t,key,value from timestamp_lazy distribute by t sort by key ASC, value ASC; + """.stripMargin) + // scalastyle:on } From 770e796679fae1e2ff306ef565a3af53d4e14cd5 Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Mon, 4 Jul 2016 13:30:04 +0530 Subject: [PATCH 20/25] udaf_covar_pop added --- ...ar_pop-0-754f4bdfb468eaa73e2b7edb9631fecb} | 0 ...r_pop-10-754f4bdfb468eaa73e2b7edb9631fecb} | 0 .../sql/hive/execution/HiveQuerySuite.scala | 20 +++++++++++++++++++ 3 files changed, 20 insertions(+) rename sql/hive/src/test/resources/golden/{udaf_covar_pop-0-2f783a39471a7f1df9a824d741c443a9 => udaf_covar_pop-0-754f4bdfb468eaa73e2b7edb9631fecb} (100%) rename sql/hive/src/test/resources/golden/{udaf_covar_pop-10-2f783a39471a7f1df9a824d741c443a9 => udaf_covar_pop-10-754f4bdfb468eaa73e2b7edb9631fecb} (100%) diff --git a/sql/hive/src/test/resources/golden/udaf_covar_pop-0-2f783a39471a7f1df9a824d741c443a9 b/sql/hive/src/test/resources/golden/udaf_covar_pop-0-754f4bdfb468eaa73e2b7edb9631fecb similarity index 100% rename from sql/hive/src/test/resources/golden/udaf_covar_pop-0-2f783a39471a7f1df9a824d741c443a9 rename to sql/hive/src/test/resources/golden/udaf_covar_pop-0-754f4bdfb468eaa73e2b7edb9631fecb diff --git a/sql/hive/src/test/resources/golden/udaf_covar_pop-10-2f783a39471a7f1df9a824d741c443a9 b/sql/hive/src/test/resources/golden/udaf_covar_pop-10-754f4bdfb468eaa73e2b7edb9631fecb similarity index 100% rename from sql/hive/src/test/resources/golden/udaf_covar_pop-10-2f783a39471a7f1df9a824d741c443a9 rename to sql/hive/src/test/resources/golden/udaf_covar_pop-10-754f4bdfb468eaa73e2b7edb9631fecb diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index 73d71a1475483..9df1d5e6b5582 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -2085,6 +2085,26 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { |select t,key,value from timestamp_lazy distribute by t sort by key ASC, value ASC; """.stripMargin) + createQueryTest("udaf_covar_pop", + """ + |DROP TABLE IF EXISTS covar_tab; + |CREATE TABLE covar_tab (a INT, b INT, c INT) + |ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' + |STORED AS TEXTFILE; + |LOAD DATA LOCAL INPATH '../../data/files/covar_tab.txt' OVERWRITE + |INTO TABLE covar_tab; + | + |DESCRIBE FUNCTION covar_pop; + |DESCRIBE FUNCTION EXTENDED covar_pop; + |SELECT covar_pop(b, c) FROM covar_tab WHERE a < 1; + |SELECT covar_pop(b, c) FROM covar_tab WHERE a < 3; + |SELECT covar_pop(b, c) FROM covar_tab WHERE a = 3; + |SELECT a, covar_pop(b, c) FROM covar_tab GROUP BY a ORDER BY a; + |SELECT covar_pop(b, c) FROM covar_tab; + | + |DROP TABLE IF EXISTS covar_tab; + """.stripMargin) + // scalastyle:on } From e418cb6cc98c53ed0c3299d32019f122402e49f6 Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Mon, 4 Jul 2016 13:34:30 +0530 Subject: [PATCH 21/25] union31 added --- ...nion31-0-1bd046af25b8f8115a900b83e0021a90} | 0 ...nion31-1-bc9dedf5e776ee6fc3644b75916a6597} | 0 ...ion31-16-1bd046af25b8f8115a900b83e0021a90} | 0 ...ion31-17-bc9dedf5e776ee6fc3644b75916a6597} | 0 .../sql/hive/execution/HiveQuerySuite.scala | 101 ++++++++++++++++++ 5 files changed, 101 insertions(+) rename sql/hive/src/test/resources/golden/{union31-0-ca16024e6f5399b1d035f5b9fd665163 => union31-0-1bd046af25b8f8115a900b83e0021a90} (100%) rename sql/hive/src/test/resources/golden/{union31-1-b12aa87aaf64b573ce0be7013117651b => union31-1-bc9dedf5e776ee6fc3644b75916a6597} (100%) rename sql/hive/src/test/resources/golden/{union31-16-ca16024e6f5399b1d035f5b9fd665163 => union31-16-1bd046af25b8f8115a900b83e0021a90} (100%) rename sql/hive/src/test/resources/golden/{union31-17-b12aa87aaf64b573ce0be7013117651b => union31-17-bc9dedf5e776ee6fc3644b75916a6597} (100%) diff --git a/sql/hive/src/test/resources/golden/union31-0-ca16024e6f5399b1d035f5b9fd665163 b/sql/hive/src/test/resources/golden/union31-0-1bd046af25b8f8115a900b83e0021a90 similarity index 100% rename from sql/hive/src/test/resources/golden/union31-0-ca16024e6f5399b1d035f5b9fd665163 rename to sql/hive/src/test/resources/golden/union31-0-1bd046af25b8f8115a900b83e0021a90 diff --git a/sql/hive/src/test/resources/golden/union31-1-b12aa87aaf64b573ce0be7013117651b b/sql/hive/src/test/resources/golden/union31-1-bc9dedf5e776ee6fc3644b75916a6597 similarity index 100% rename from sql/hive/src/test/resources/golden/union31-1-b12aa87aaf64b573ce0be7013117651b rename to sql/hive/src/test/resources/golden/union31-1-bc9dedf5e776ee6fc3644b75916a6597 diff --git a/sql/hive/src/test/resources/golden/union31-16-ca16024e6f5399b1d035f5b9fd665163 b/sql/hive/src/test/resources/golden/union31-16-1bd046af25b8f8115a900b83e0021a90 similarity index 100% rename from sql/hive/src/test/resources/golden/union31-16-ca16024e6f5399b1d035f5b9fd665163 rename to sql/hive/src/test/resources/golden/union31-16-1bd046af25b8f8115a900b83e0021a90 diff --git a/sql/hive/src/test/resources/golden/union31-17-b12aa87aaf64b573ce0be7013117651b b/sql/hive/src/test/resources/golden/union31-17-bc9dedf5e776ee6fc3644b75916a6597 similarity index 100% rename from sql/hive/src/test/resources/golden/union31-17-b12aa87aaf64b573ce0be7013117651b rename to sql/hive/src/test/resources/golden/union31-17-bc9dedf5e776ee6fc3644b75916a6597 diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index 9df1d5e6b5582..8e869bd524b45 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -2105,6 +2105,107 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { |DROP TABLE IF EXISTS covar_tab; """.stripMargin) + createQueryTest("union31", + """ + |drop table if exists t1; + |drop table if exists t2; + | + | + |create table t1 as select * from src where key < 10; + |create table t2 as select * from src where key < 10; + | + |create table t3(key string, cnt int); + |create table t4(value string, cnt int); + | + |explain + |from + |(select * from t1 + | union all + | select * from t2 + |) x + |insert overwrite table t3 + | select key, count(1) group by key + |insert overwrite table t4 + | select value, count(1) group by value; + | + |from + |(select * from t1 + | union all + | select * from t2 + |) x + |insert overwrite table t3 + | select key, count(1) group by key + |insert overwrite table t4 + | select value, count(1) group by value; + | + |select * from t3 order by key; + |select * from t4 order by value; + | + |create table t5(c1 string, cnt int); + |create table t6(c1 string, cnt int); + | + |explain + |from + |( + | select key as c1, count(1) as cnt from t1 group by key + | union all + | select key as c1, count(1) as cnt from t2 group by key + |) x + |insert overwrite table t5 + | select c1, sum(cnt) group by c1 + |insert overwrite table t6 + | select c1, sum(cnt) group by c1; + | + |from + |( + | select key as c1, count(1) as cnt from t1 group by key + | union all + | select key as c1, count(1) as cnt from t2 group by key + |) x + |insert overwrite table t5 + | select c1, sum(cnt) group by c1 + |insert overwrite table t6 + | select c1, sum(cnt) group by c1; + | + |select * from t5 order by c1; + |select * from t6 order by c1; + | + |drop table if exists t1; + |drop table if exists t2; + | + |create table t1 as select * from src where key < 10; + |create table t2 as select key, count(1) as cnt from src where key < 10 group by key; + | + |create table t7(c1 string, cnt int); + |create table t8(c1 string, cnt int); + | + |explain + |from + |( + | select key as c1, count(1) as cnt from t1 group by key + | union all + | select key as c1, cnt from t2 + |) x + |insert overwrite table t7 + | select c1, count(1) group by c1 + |insert overwrite table t8 + | select c1, count(1) group by c1; + | + |from + |( + | select key as c1, count(1) as cnt from t1 group by key + | union all + | select key as c1, cnt from t2 + |) x + |insert overwrite table t7 + | select c1, count(1) group by c1 + |insert overwrite table t8 + | select c1, count(1) group by c1; + | + |select * from t7 order by c1; + |select * from t8 order by c1; + """.stripMargin) + // scalastyle:on } From cc689104f803c2bdf1a257b7edb06f126a9ab918 Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Mon, 4 Jul 2016 13:37:03 +0530 Subject: [PATCH 22/25] union_date added --- ...n_date-0-34010242fb916b1ab3d4bcde2d1b0733} | 0 ...n_date-1-e3c8c7139cd5165c8c8207ba9f8eeafe} | 0 ...n_date-7-34010242fb916b1ab3d4bcde2d1b0733} | 0 ...n_date-8-e3c8c7139cd5165c8c8207ba9f8eeafe} | 0 .../sql/hive/execution/HiveQuerySuite.scala | 34 +++++++++++++++++++ 5 files changed, 34 insertions(+) rename sql/hive/src/test/resources/golden/{union_date-0-a0bade1c77338d4f72962389a1f5bea2 => union_date-0-34010242fb916b1ab3d4bcde2d1b0733} (100%) rename sql/hive/src/test/resources/golden/{union_date-1-21306adbd8be8ad75174ad9d3e42b73c => union_date-1-e3c8c7139cd5165c8c8207ba9f8eeafe} (100%) rename sql/hive/src/test/resources/golden/{union_date-7-a0bade1c77338d4f72962389a1f5bea2 => union_date-7-34010242fb916b1ab3d4bcde2d1b0733} (100%) rename sql/hive/src/test/resources/golden/{union_date-8-21306adbd8be8ad75174ad9d3e42b73c => union_date-8-e3c8c7139cd5165c8c8207ba9f8eeafe} (100%) diff --git a/sql/hive/src/test/resources/golden/union_date-0-a0bade1c77338d4f72962389a1f5bea2 b/sql/hive/src/test/resources/golden/union_date-0-34010242fb916b1ab3d4bcde2d1b0733 similarity index 100% rename from sql/hive/src/test/resources/golden/union_date-0-a0bade1c77338d4f72962389a1f5bea2 rename to sql/hive/src/test/resources/golden/union_date-0-34010242fb916b1ab3d4bcde2d1b0733 diff --git a/sql/hive/src/test/resources/golden/union_date-1-21306adbd8be8ad75174ad9d3e42b73c b/sql/hive/src/test/resources/golden/union_date-1-e3c8c7139cd5165c8c8207ba9f8eeafe similarity index 100% rename from sql/hive/src/test/resources/golden/union_date-1-21306adbd8be8ad75174ad9d3e42b73c rename to sql/hive/src/test/resources/golden/union_date-1-e3c8c7139cd5165c8c8207ba9f8eeafe diff --git a/sql/hive/src/test/resources/golden/union_date-7-a0bade1c77338d4f72962389a1f5bea2 b/sql/hive/src/test/resources/golden/union_date-7-34010242fb916b1ab3d4bcde2d1b0733 similarity index 100% rename from sql/hive/src/test/resources/golden/union_date-7-a0bade1c77338d4f72962389a1f5bea2 rename to sql/hive/src/test/resources/golden/union_date-7-34010242fb916b1ab3d4bcde2d1b0733 diff --git a/sql/hive/src/test/resources/golden/union_date-8-21306adbd8be8ad75174ad9d3e42b73c b/sql/hive/src/test/resources/golden/union_date-8-e3c8c7139cd5165c8c8207ba9f8eeafe similarity index 100% rename from sql/hive/src/test/resources/golden/union_date-8-21306adbd8be8ad75174ad9d3e42b73c rename to sql/hive/src/test/resources/golden/union_date-8-e3c8c7139cd5165c8c8207ba9f8eeafe diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index 8e869bd524b45..ae8e08a929e0c 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -2206,6 +2206,40 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { |select * from t8 order by c1; """.stripMargin) + createQueryTest("union_date", + """ + |drop table if exists union_date_1; + |drop table if exists union_date_2; + | + |create table union_date_1 ( + | ORIGIN_CITY_NAME string, + | DEST_CITY_NAME string, + | FL_DATE date, + | ARR_DELAY float, + | FL_NUM int + |); + | + |create table union_date_2 ( + | ORIGIN_CITY_NAME string, + | DEST_CITY_NAME string, + | FL_DATE date, + | ARR_DELAY float, + | FL_NUM int + |); + | + |LOAD DATA LOCAL INPATH '../../data/files/flights_join.txt' OVERWRITE INTO TABLE union_date_1; + |LOAD DATA LOCAL INPATH '../../data/files/flights_join.txt' OVERWRITE INTO TABLE union_date_2; + | + |select * from ( + | select fl_num, fl_date from union_date_1 + | union all + | select fl_num, fl_date from union_date_2 + |) union_result order by fl_date, fl_num; + | + |drop table if exists union_date_1; + |drop table if exists union_date_2; + """.stripMargin) + // scalastyle:on } From 250271bf48373d0eb2a302a8ad854dd85a6da6b1 Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Mon, 4 Jul 2016 13:40:49 +0530 Subject: [PATCH 23/25] varchar_2 added --- ...char_2-0-149988ea7c4fcefd5b0c14d9e3afb384} | 0 ...char_2-7-149988ea7c4fcefd5b0c14d9e3afb384} | 0 .../sql/hive/execution/HiveQuerySuite.scala | 40 +++++++++++++++++++ 3 files changed, 40 insertions(+) rename sql/hive/src/test/resources/golden/{varchar_2-0-fbced67562c536a7e9b6d3928bd9d0af => varchar_2-0-149988ea7c4fcefd5b0c14d9e3afb384} (100%) rename sql/hive/src/test/resources/golden/{varchar_2-7-fbced67562c536a7e9b6d3928bd9d0af => varchar_2-7-149988ea7c4fcefd5b0c14d9e3afb384} (100%) diff --git a/sql/hive/src/test/resources/golden/varchar_2-0-fbced67562c536a7e9b6d3928bd9d0af b/sql/hive/src/test/resources/golden/varchar_2-0-149988ea7c4fcefd5b0c14d9e3afb384 similarity index 100% rename from sql/hive/src/test/resources/golden/varchar_2-0-fbced67562c536a7e9b6d3928bd9d0af rename to sql/hive/src/test/resources/golden/varchar_2-0-149988ea7c4fcefd5b0c14d9e3afb384 diff --git a/sql/hive/src/test/resources/golden/varchar_2-7-fbced67562c536a7e9b6d3928bd9d0af b/sql/hive/src/test/resources/golden/varchar_2-7-149988ea7c4fcefd5b0c14d9e3afb384 similarity index 100% rename from sql/hive/src/test/resources/golden/varchar_2-7-fbced67562c536a7e9b6d3928bd9d0af rename to sql/hive/src/test/resources/golden/varchar_2-7-149988ea7c4fcefd5b0c14d9e3afb384 diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index ae8e08a929e0c..ebbe6d24e3ed7 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -2240,6 +2240,46 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { |drop table if exists union_date_2; """.stripMargin) + createQueryTest("varchar_2", + """ + |drop table if exists varchar_2; + | + |create table varchar_2 ( + | key varchar(10), + | value varchar(20) + |); + | + |insert overwrite table varchar_2 select * from src; + | + |select value, sum(cast(key as int)), count(*) numrows + |from src + |group by value + |order by value asc + |limit 5; + | + |-- should match the query from src + |select value, sum(cast(key as int)), count(*) numrows + |from varchar_2 + |group by value + |order by value asc + |limit 5; + | + |select value, sum(cast(key as int)), count(*) numrows + |from src + |group by value + |order by value desc + |limit 5; + | + |-- should match the query from src + |select value, sum(cast(key as int)), count(*) numrows + |from varchar_2 + |group by value + |order by value desc + |limit 5; + | + |drop table if exists varchar_2; + """.stripMargin) + // scalastyle:on } From 7ff3301c36be24e6178ad1939559250e08bee910 Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Mon, 4 Jul 2016 13:43:50 +0530 Subject: [PATCH 24/25] varchar_join1 added --- ..._join1-0-ddcd66e13377e03bcc49b84eacdd43a6} | 0 ..._join1-1-37b8031f02bc25c86302c558645afc66} | 0 ...join1-12-ddcd66e13377e03bcc49b84eacdd43a6} | 0 ...join1-13-37b8031f02bc25c86302c558645afc66} | 0 ...join1-14-213185af55d5d94aadb0a0dd0773d984} | 0 ..._join1-2-213185af55d5d94aadb0a0dd0773d984} | 0 .../sql/hive/execution/HiveQuerySuite.scala | 38 +++++++++++++++++++ 7 files changed, 38 insertions(+) rename sql/hive/src/test/resources/golden/{varchar_join1-0-f170a8c3d5db4cbf4e7c10aee1ef652e => varchar_join1-0-ddcd66e13377e03bcc49b84eacdd43a6} (100%) rename sql/hive/src/test/resources/golden/{varchar_join1-1-8318072ee410cae36186c9880989a1a5 => varchar_join1-1-37b8031f02bc25c86302c558645afc66} (100%) rename sql/hive/src/test/resources/golden/{varchar_join1-12-f170a8c3d5db4cbf4e7c10aee1ef652e => varchar_join1-12-ddcd66e13377e03bcc49b84eacdd43a6} (100%) rename sql/hive/src/test/resources/golden/{varchar_join1-13-8318072ee410cae36186c9880989a1a5 => varchar_join1-13-37b8031f02bc25c86302c558645afc66} (100%) rename sql/hive/src/test/resources/golden/{varchar_join1-14-8bbaae82c89553165b975c5b8ca3aee4 => varchar_join1-14-213185af55d5d94aadb0a0dd0773d984} (100%) rename sql/hive/src/test/resources/golden/{varchar_join1-2-8bbaae82c89553165b975c5b8ca3aee4 => varchar_join1-2-213185af55d5d94aadb0a0dd0773d984} (100%) diff --git a/sql/hive/src/test/resources/golden/varchar_join1-0-f170a8c3d5db4cbf4e7c10aee1ef652e b/sql/hive/src/test/resources/golden/varchar_join1-0-ddcd66e13377e03bcc49b84eacdd43a6 similarity index 100% rename from sql/hive/src/test/resources/golden/varchar_join1-0-f170a8c3d5db4cbf4e7c10aee1ef652e rename to sql/hive/src/test/resources/golden/varchar_join1-0-ddcd66e13377e03bcc49b84eacdd43a6 diff --git a/sql/hive/src/test/resources/golden/varchar_join1-1-8318072ee410cae36186c9880989a1a5 b/sql/hive/src/test/resources/golden/varchar_join1-1-37b8031f02bc25c86302c558645afc66 similarity index 100% rename from sql/hive/src/test/resources/golden/varchar_join1-1-8318072ee410cae36186c9880989a1a5 rename to sql/hive/src/test/resources/golden/varchar_join1-1-37b8031f02bc25c86302c558645afc66 diff --git a/sql/hive/src/test/resources/golden/varchar_join1-12-f170a8c3d5db4cbf4e7c10aee1ef652e b/sql/hive/src/test/resources/golden/varchar_join1-12-ddcd66e13377e03bcc49b84eacdd43a6 similarity index 100% rename from sql/hive/src/test/resources/golden/varchar_join1-12-f170a8c3d5db4cbf4e7c10aee1ef652e rename to sql/hive/src/test/resources/golden/varchar_join1-12-ddcd66e13377e03bcc49b84eacdd43a6 diff --git a/sql/hive/src/test/resources/golden/varchar_join1-13-8318072ee410cae36186c9880989a1a5 b/sql/hive/src/test/resources/golden/varchar_join1-13-37b8031f02bc25c86302c558645afc66 similarity index 100% rename from sql/hive/src/test/resources/golden/varchar_join1-13-8318072ee410cae36186c9880989a1a5 rename to sql/hive/src/test/resources/golden/varchar_join1-13-37b8031f02bc25c86302c558645afc66 diff --git a/sql/hive/src/test/resources/golden/varchar_join1-14-8bbaae82c89553165b975c5b8ca3aee4 b/sql/hive/src/test/resources/golden/varchar_join1-14-213185af55d5d94aadb0a0dd0773d984 similarity index 100% rename from sql/hive/src/test/resources/golden/varchar_join1-14-8bbaae82c89553165b975c5b8ca3aee4 rename to sql/hive/src/test/resources/golden/varchar_join1-14-213185af55d5d94aadb0a0dd0773d984 diff --git a/sql/hive/src/test/resources/golden/varchar_join1-2-8bbaae82c89553165b975c5b8ca3aee4 b/sql/hive/src/test/resources/golden/varchar_join1-2-213185af55d5d94aadb0a0dd0773d984 similarity index 100% rename from sql/hive/src/test/resources/golden/varchar_join1-2-8bbaae82c89553165b975c5b8ca3aee4 rename to sql/hive/src/test/resources/golden/varchar_join1-2-213185af55d5d94aadb0a0dd0773d984 diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index ebbe6d24e3ed7..9300464b90e49 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -2280,6 +2280,44 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { |drop table if exists varchar_2; """.stripMargin) + createQueryTest("varchar_join1", + """ + |drop table if exists varchar_join1_vc1; + |drop table if exists varchar_join1_vc2; + |drop table if exists varchar_join1_str; + | + |create table varchar_join1_vc1 ( + | c1 int, + | c2 varchar(10) + |); + | + |create table varchar_join1_vc2 ( + | c1 int, + | c2 varchar(20) + |); + | + |create table varchar_join1_str ( + | c1 int, + | c2 string + |); + | + |load data local inpath '../../data/files/vc1.txt' into table varchar_join1_vc1; + |load data local inpath '../../data/files/vc1.txt' into table varchar_join1_vc2; + |load data local inpath '../../data/files/vc1.txt' into table varchar_join1_str; + | + |-- Join varchar with same length varchar + |select * from varchar_join1_vc1 a join varchar_join1_vc1 b on (a.c2 = b.c2) order by a.c1; + | + |-- Join varchar with different length varchar + |select * from varchar_join1_vc1 a join varchar_join1_vc2 b on (a.c2 = b.c2) order by a.c1; + | + |-- Join varchar with string + |select * from varchar_join1_vc1 a join varchar_join1_str b on (a.c2 = b.c2) order by a.c1; + | + |drop table if exists varchar_join1_vc1; + |drop table if exists varchar_join1_vc2; + |drop table if exists varchar_join1_str; + """.stripMargin) // scalastyle:on } From 30b6012241173b1c395ffce4f0fe375350cd9bb9 Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Mon, 4 Jul 2016 13:51:15 +0530 Subject: [PATCH 25/25] revert date_1 --- ...date_1-1-23edf29bf7376c70d5ecf12720f4b1eb} | 0 ...ate_1-12-480c5f024a28232b7857be327c992509} | 0 ...ate_1-13-4c0ed7fcb75770d8790575b586bf14f4} | 0 ...date_1-14-44fc74c1993062c0a9522199ff27fea} | 0 ...ate_1-15-4855a66124b16d1d0d003235995ac06b} | 0 ...ate_1-16-8bc190dba0f641840b5e1e198a14c55b} | 0 ...ate_1-17-23edf29bf7376c70d5ecf12720f4b1eb} | 0 ... date_1-3-26b5c291400dfde455b3c1b878b71d0} | 0 ...date_1-6-559d01fb0b42c42f0c4927fa0f9deac4} | 0 ...date_1-9-8306558e0eabe936ac33dabaaa17fea4} | 0 .../sql/hive/execution/HiveQuerySuite.scala | 93 ------------------- 11 files changed, 93 deletions(-) rename sql/hive/src/test/resources/golden/{date_1-1-d4c25b848047acb9546eb2def9c33a01 => date_1-1-23edf29bf7376c70d5ecf12720f4b1eb} (100%) rename sql/hive/src/test/resources/golden/{date_1-12-c43300026ebf9d0f79b58d2bb7255599 => date_1-12-480c5f024a28232b7857be327c992509} (100%) rename sql/hive/src/test/resources/golden/{date_1-13-63ae717a423cb85b060ea837431efc23 => date_1-13-4c0ed7fcb75770d8790575b586bf14f4} (100%) rename sql/hive/src/test/resources/golden/{date_1-14-67c78afa310d373a6197ba0a3cc6e70d => date_1-14-44fc74c1993062c0a9522199ff27fea} (100%) rename sql/hive/src/test/resources/golden/{date_1-15-ca5bdcb066b100b8c676bce6535a71b1 => date_1-15-4855a66124b16d1d0d003235995ac06b} (100%) rename sql/hive/src/test/resources/golden/{date_1-16-79d09da7ad2beb7ee1da1f5889eae0a4 => date_1-16-8bc190dba0f641840b5e1e198a14c55b} (100%) rename sql/hive/src/test/resources/golden/{date_1-17-d4c25b848047acb9546eb2def9c33a01 => date_1-17-23edf29bf7376c70d5ecf12720f4b1eb} (100%) rename sql/hive/src/test/resources/golden/{date_1-3-97f403b3d42b6391934f6a9da6fef792 => date_1-3-26b5c291400dfde455b3c1b878b71d0} (100%) rename sql/hive/src/test/resources/golden/{date_1-6-6724d5d81b68e07f0420a28b9232d464 => date_1-6-559d01fb0b42c42f0c4927fa0f9deac4} (100%) rename sql/hive/src/test/resources/golden/{date_1-9-241dff52c41c266c0e36f7814913326c => date_1-9-8306558e0eabe936ac33dabaaa17fea4} (100%) diff --git a/sql/hive/src/test/resources/golden/date_1-1-d4c25b848047acb9546eb2def9c33a01 b/sql/hive/src/test/resources/golden/date_1-1-23edf29bf7376c70d5ecf12720f4b1eb similarity index 100% rename from sql/hive/src/test/resources/golden/date_1-1-d4c25b848047acb9546eb2def9c33a01 rename to sql/hive/src/test/resources/golden/date_1-1-23edf29bf7376c70d5ecf12720f4b1eb diff --git a/sql/hive/src/test/resources/golden/date_1-12-c43300026ebf9d0f79b58d2bb7255599 b/sql/hive/src/test/resources/golden/date_1-12-480c5f024a28232b7857be327c992509 similarity index 100% rename from sql/hive/src/test/resources/golden/date_1-12-c43300026ebf9d0f79b58d2bb7255599 rename to sql/hive/src/test/resources/golden/date_1-12-480c5f024a28232b7857be327c992509 diff --git a/sql/hive/src/test/resources/golden/date_1-13-63ae717a423cb85b060ea837431efc23 b/sql/hive/src/test/resources/golden/date_1-13-4c0ed7fcb75770d8790575b586bf14f4 similarity index 100% rename from sql/hive/src/test/resources/golden/date_1-13-63ae717a423cb85b060ea837431efc23 rename to sql/hive/src/test/resources/golden/date_1-13-4c0ed7fcb75770d8790575b586bf14f4 diff --git a/sql/hive/src/test/resources/golden/date_1-14-67c78afa310d373a6197ba0a3cc6e70d b/sql/hive/src/test/resources/golden/date_1-14-44fc74c1993062c0a9522199ff27fea similarity index 100% rename from sql/hive/src/test/resources/golden/date_1-14-67c78afa310d373a6197ba0a3cc6e70d rename to sql/hive/src/test/resources/golden/date_1-14-44fc74c1993062c0a9522199ff27fea diff --git a/sql/hive/src/test/resources/golden/date_1-15-ca5bdcb066b100b8c676bce6535a71b1 b/sql/hive/src/test/resources/golden/date_1-15-4855a66124b16d1d0d003235995ac06b similarity index 100% rename from sql/hive/src/test/resources/golden/date_1-15-ca5bdcb066b100b8c676bce6535a71b1 rename to sql/hive/src/test/resources/golden/date_1-15-4855a66124b16d1d0d003235995ac06b diff --git a/sql/hive/src/test/resources/golden/date_1-16-79d09da7ad2beb7ee1da1f5889eae0a4 b/sql/hive/src/test/resources/golden/date_1-16-8bc190dba0f641840b5e1e198a14c55b similarity index 100% rename from sql/hive/src/test/resources/golden/date_1-16-79d09da7ad2beb7ee1da1f5889eae0a4 rename to sql/hive/src/test/resources/golden/date_1-16-8bc190dba0f641840b5e1e198a14c55b diff --git a/sql/hive/src/test/resources/golden/date_1-17-d4c25b848047acb9546eb2def9c33a01 b/sql/hive/src/test/resources/golden/date_1-17-23edf29bf7376c70d5ecf12720f4b1eb similarity index 100% rename from sql/hive/src/test/resources/golden/date_1-17-d4c25b848047acb9546eb2def9c33a01 rename to sql/hive/src/test/resources/golden/date_1-17-23edf29bf7376c70d5ecf12720f4b1eb diff --git a/sql/hive/src/test/resources/golden/date_1-3-97f403b3d42b6391934f6a9da6fef792 b/sql/hive/src/test/resources/golden/date_1-3-26b5c291400dfde455b3c1b878b71d0 similarity index 100% rename from sql/hive/src/test/resources/golden/date_1-3-97f403b3d42b6391934f6a9da6fef792 rename to sql/hive/src/test/resources/golden/date_1-3-26b5c291400dfde455b3c1b878b71d0 diff --git a/sql/hive/src/test/resources/golden/date_1-6-6724d5d81b68e07f0420a28b9232d464 b/sql/hive/src/test/resources/golden/date_1-6-559d01fb0b42c42f0c4927fa0f9deac4 similarity index 100% rename from sql/hive/src/test/resources/golden/date_1-6-6724d5d81b68e07f0420a28b9232d464 rename to sql/hive/src/test/resources/golden/date_1-6-559d01fb0b42c42f0c4927fa0f9deac4 diff --git a/sql/hive/src/test/resources/golden/date_1-9-241dff52c41c266c0e36f7814913326c b/sql/hive/src/test/resources/golden/date_1-9-8306558e0eabe936ac33dabaaa17fea4 similarity index 100% rename from sql/hive/src/test/resources/golden/date_1-9-241dff52c41c266c0e36f7814913326c rename to sql/hive/src/test/resources/golden/date_1-9-8306558e0eabe936ac33dabaaa17fea4 diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index 9300464b90e49..cda3d1f195290 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -1259,99 +1259,6 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { |SELECT * FROM alter_rename_partition WHERE pcol1='new_part1:' and pcol2='new_part2:'; """.stripMargin) - createQueryTest("date_1", - """ - |set hive.fetch.task.conversion=more; - | - |drop table if exists date_1; - | - |create table date_1 (d date); - | - |insert overwrite table date_1 - | select cast('2011-01-01' as date) from src tablesample (1 rows); - | - |select * from date_1 limit 1; - |select d, count(d) from date_1 group by d; - | - |insert overwrite table date_1 - | select date '2011-01-01' from src tablesample (1 rows); - | - |select * from date_1 limit 1; - |select d, count(d) from date_1 group by d; - | - |insert overwrite table date_1 - | select cast(cast('2011-01-01 00:00:00' as timestamp) as date) from src tablesample (1 rows); - | - |select * from date_1 limit 1; - |select d, count(d) from date_1 group by d; - | - |-- Valid casts - |select - | cast('2012-01-01' as string), - | cast(d as string), - | cast(d as timestamp), - | cast(cast(d as timestamp) as date), - | cast(d as date) - |from date_1 limit 1; - | - |-- Invalid casts. - |select - | cast(d as boolean), - | cast(d as tinyint), - | cast(d as smallint), - | cast(d as int), - | cast(d as bigint), - | cast(d as float), - | cast(d as double) - |from date_1 limit 1; - | - |-- These comparisons should all be true - |select - | date '2011-01-01' = date '2011-01-01', - | unix_timestamp(date '2011-01-01') = unix_timestamp(date '2011-01-01'), - | unix_timestamp(date '2011-01-01') = unix_timestamp(cast(date '2011-01-01' as timestamp)), - | unix_timestamp(date '2011-01-01') = unix_timestamp(cast(cast('2011-01-01 12:13:14' as timestamp) as date)), - | unix_timestamp(date '2011-01-01') < unix_timestamp(cast('2011-01-01 00:00:01' as timestamp)), - | unix_timestamp(date '2011-01-01') = unix_timestamp(cast('2011-01-01 00:00:00' as timestamp)), - | unix_timestamp(date '2011-01-01') > unix_timestamp(cast('2010-12-31 23:59:59' as timestamp)), - | date '2011-01-01' = cast(timestamp('2011-01-01 23:24:25') as date), - | '2011-01-01' = cast(d as string), - | '2011-01-01' = cast(date '2011-01-01' as string) - |from date_1 limit 1; - | - |select - | date('2001-01-28'), - | date('2001-02-28'), - | date('2001-03-28'), - | date('2001-04-28'), - | date('2001-05-28'), - | date('2001-06-28'), - | date('2001-07-28'), - | date('2001-08-28'), - | date('2001-09-28'), - | date('2001-10-28'), - | date('2001-11-28'), - | date('2001-12-28') - |from date_1 limit 1; - | - |select - | unix_timestamp(date('2001-01-28')) = unix_timestamp(cast('2001-01-28 0:0:0' as timestamp)), - | unix_timestamp(date('2001-02-28')) = unix_timestamp(cast('2001-02-28 0:0:0' as timestamp)), - | unix_timestamp(date('2001-03-28')) = unix_timestamp(cast('2001-03-28 0:0:0' as timestamp)), - | unix_timestamp(date('2001-04-28')) = unix_timestamp(cast('2001-04-28 0:0:0' as timestamp)), - | unix_timestamp(date('2001-05-28')) = unix_timestamp(cast('2001-05-28 0:0:0' as timestamp)), - | unix_timestamp(date('2001-06-28')) = unix_timestamp(cast('2001-06-28 0:0:0' as timestamp)), - | unix_timestamp(date('2001-07-28')) = unix_timestamp(cast('2001-07-28 0:0:0' as timestamp)), - | unix_timestamp(date('2001-08-28')) = unix_timestamp(cast('2001-08-28 0:0:0' as timestamp)), - | unix_timestamp(date('2001-09-28')) = unix_timestamp(cast('2001-09-28 0:0:0' as timestamp)), - | unix_timestamp(date('2001-10-28')) = unix_timestamp(cast('2001-10-28 0:0:0' as timestamp)), - | unix_timestamp(date('2001-11-28')) = unix_timestamp(cast('2001-11-28 0:0:0' as timestamp)), - | unix_timestamp(date('2001-12-28')) = unix_timestamp(cast('2001-12-28 0:0:0' as timestamp)) - |from date_1 limit 1; - | - |drop table if exists date_1; - """.stripMargin) - createQueryTest("date_4", """ |set hive.fetch.task.conversion=more;