From d9bcf920f3b17f9d4e6542e87e5d71b1a36817e2 Mon Sep 17 00:00:00 2001 From: Cheng Hao Date: Thu, 12 Feb 2015 23:10:34 -0800 Subject: [PATCH 1/3] Update the HiveContext Unittest --- .../spark/sql/hive/execution/HiveCompatibilitySuite.scala | 2 ++ .../main/scala/org/apache/spark/sql/hive/test/TestHive.scala | 5 +++-- .../golden/udf_reflect-0-50131c0ba7b7a6b65c789a5a8497bada | 1 + .../golden/udf_reflect-1-904138e2a1f831c308b7f0aacc859ae1 | 1 + .../golden/udf_reflect-2-21ec7caa253c7f95b7cf60191140e2ee | 4 ++++ .../golden/udf_reflect-3-f78a6582e4818a384997eefada54c77 | 0 .../golden/udf_reflect-4-2a9bfe456332e7ad776662e2449bd2e0 | 1 + .../golden/udf_reflect2-0-50131c0ba7b7a6b65c789a5a8497bada | 1 + .../golden/udf_reflect2-1-7bec330c7bc6f71cbaf9bf1883d1b184 | 1 + .../golden/udf_reflect2-2-c5a05379f482215a5a484bed0299bf19 | 3 +++ .../golden/udf_reflect2-3-effc057c78c00b0af26a4ac0f5f116ca | 0 .../golden/udf_reflect2-4-73d466e70e96e9e5f0cd373b37d4e1f4 | 5 +++++ 12 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 sql/hive/src/test/resources/golden/udf_reflect-0-50131c0ba7b7a6b65c789a5a8497bada create mode 100644 sql/hive/src/test/resources/golden/udf_reflect-1-904138e2a1f831c308b7f0aacc859ae1 create mode 100644 sql/hive/src/test/resources/golden/udf_reflect-2-21ec7caa253c7f95b7cf60191140e2ee create mode 100644 sql/hive/src/test/resources/golden/udf_reflect-3-f78a6582e4818a384997eefada54c77 create mode 100644 sql/hive/src/test/resources/golden/udf_reflect-4-2a9bfe456332e7ad776662e2449bd2e0 create mode 100644 sql/hive/src/test/resources/golden/udf_reflect2-0-50131c0ba7b7a6b65c789a5a8497bada create mode 100644 sql/hive/src/test/resources/golden/udf_reflect2-1-7bec330c7bc6f71cbaf9bf1883d1b184 create mode 100644 sql/hive/src/test/resources/golden/udf_reflect2-2-c5a05379f482215a5a484bed0299bf19 create mode 100644 sql/hive/src/test/resources/golden/udf_reflect2-3-effc057c78c00b0af26a4ac0f5f116ca create mode 100644 sql/hive/src/test/resources/golden/udf_reflect2-4-73d466e70e96e9e5f0cd373b37d4e1f4 diff --git a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala index e443e5bd5f54d..5497e207c759d 100644 --- a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala +++ b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala @@ -885,6 +885,8 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter { "udf_power", "udf_radians", "udf_rand", + "udf_reflect", + "udf_reflect2", "udf_regexp", "udf_regexp_extract", "udf_regexp_replace", diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala index 840fbc197259a..d49b60e0fadab 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala @@ -196,12 +196,13 @@ class TestHiveContext(sc: SparkContext) extends HiveContext(sc) { // The test tables that are defined in the Hive QTestUtil. // /itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java + // https://github.com/apache/hive/blob/branch-0.13/data/scripts/q_test_init.sql val hiveQTestUtilTables = Seq( TestTable("src", - "CREATE TABLE src (key INT, value STRING)".cmd, + "CREATE TABLE src (key STRING, value STRING)".cmd, s"LOAD DATA LOCAL INPATH '${getHiveFile("data/files/kv1.txt")}' INTO TABLE src".cmd), TestTable("src1", - "CREATE TABLE src1 (key INT, value STRING)".cmd, + "CREATE TABLE src1 (key STRING, value STRING)".cmd, s"LOAD DATA LOCAL INPATH '${getHiveFile("data/files/kv3.txt")}' INTO TABLE src1".cmd), TestTable("srcpart", () => { runSqlHive( diff --git a/sql/hive/src/test/resources/golden/udf_reflect-0-50131c0ba7b7a6b65c789a5a8497bada b/sql/hive/src/test/resources/golden/udf_reflect-0-50131c0ba7b7a6b65c789a5a8497bada new file mode 100644 index 0000000000000..573541ac9702d --- /dev/null +++ b/sql/hive/src/test/resources/golden/udf_reflect-0-50131c0ba7b7a6b65c789a5a8497bada @@ -0,0 +1 @@ +0 diff --git a/sql/hive/src/test/resources/golden/udf_reflect-1-904138e2a1f831c308b7f0aacc859ae1 b/sql/hive/src/test/resources/golden/udf_reflect-1-904138e2a1f831c308b7f0aacc859ae1 new file mode 100644 index 0000000000000..1d7658151cd62 --- /dev/null +++ b/sql/hive/src/test/resources/golden/udf_reflect-1-904138e2a1f831c308b7f0aacc859ae1 @@ -0,0 +1 @@ +reflect(class,method[,arg1[,arg2..]]) calls method with reflection diff --git a/sql/hive/src/test/resources/golden/udf_reflect-2-21ec7caa253c7f95b7cf60191140e2ee b/sql/hive/src/test/resources/golden/udf_reflect-2-21ec7caa253c7f95b7cf60191140e2ee new file mode 100644 index 0000000000000..ddf986c01e5b1 --- /dev/null +++ b/sql/hive/src/test/resources/golden/udf_reflect-2-21ec7caa253c7f95b7cf60191140e2ee @@ -0,0 +1,4 @@ +reflect(class,method[,arg1[,arg2..]]) calls method with reflection +Synonyms: java_method +Use this UDF to call Java methods by matching the argument signature + diff --git a/sql/hive/src/test/resources/golden/udf_reflect-3-f78a6582e4818a384997eefada54c77 b/sql/hive/src/test/resources/golden/udf_reflect-3-f78a6582e4818a384997eefada54c77 new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/sql/hive/src/test/resources/golden/udf_reflect-4-2a9bfe456332e7ad776662e2449bd2e0 b/sql/hive/src/test/resources/golden/udf_reflect-4-2a9bfe456332e7ad776662e2449bd2e0 new file mode 100644 index 0000000000000..a92d8dca89b3e --- /dev/null +++ b/sql/hive/src/test/resources/golden/udf_reflect-4-2a9bfe456332e7ad776662e2449bd2e0 @@ -0,0 +1 @@ +1 true 3 2 3 2.718281828459045 1.0 568 diff --git a/sql/hive/src/test/resources/golden/udf_reflect2-0-50131c0ba7b7a6b65c789a5a8497bada b/sql/hive/src/test/resources/golden/udf_reflect2-0-50131c0ba7b7a6b65c789a5a8497bada new file mode 100644 index 0000000000000..573541ac9702d --- /dev/null +++ b/sql/hive/src/test/resources/golden/udf_reflect2-0-50131c0ba7b7a6b65c789a5a8497bada @@ -0,0 +1 @@ +0 diff --git a/sql/hive/src/test/resources/golden/udf_reflect2-1-7bec330c7bc6f71cbaf9bf1883d1b184 b/sql/hive/src/test/resources/golden/udf_reflect2-1-7bec330c7bc6f71cbaf9bf1883d1b184 new file mode 100644 index 0000000000000..cd35e5b290db5 --- /dev/null +++ b/sql/hive/src/test/resources/golden/udf_reflect2-1-7bec330c7bc6f71cbaf9bf1883d1b184 @@ -0,0 +1 @@ +reflect2(arg0,method[,arg1[,arg2..]]) calls method of arg0 with reflection diff --git a/sql/hive/src/test/resources/golden/udf_reflect2-2-c5a05379f482215a5a484bed0299bf19 b/sql/hive/src/test/resources/golden/udf_reflect2-2-c5a05379f482215a5a484bed0299bf19 new file mode 100644 index 0000000000000..48ef97292ab62 --- /dev/null +++ b/sql/hive/src/test/resources/golden/udf_reflect2-2-c5a05379f482215a5a484bed0299bf19 @@ -0,0 +1,3 @@ +reflect2(arg0,method[,arg1[,arg2..]]) calls method of arg0 with reflection +Use this UDF to call Java methods by matching the argument signature + diff --git a/sql/hive/src/test/resources/golden/udf_reflect2-3-effc057c78c00b0af26a4ac0f5f116ca b/sql/hive/src/test/resources/golden/udf_reflect2-3-effc057c78c00b0af26a4ac0f5f116ca new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/sql/hive/src/test/resources/golden/udf_reflect2-4-73d466e70e96e9e5f0cd373b37d4e1f4 b/sql/hive/src/test/resources/golden/udf_reflect2-4-73d466e70e96e9e5f0cd373b37d4e1f4 new file mode 100644 index 0000000000000..176ea0358d7ea --- /dev/null +++ b/sql/hive/src/test/resources/golden/udf_reflect2-4-73d466e70e96e9e5f0cd373b37d4e1f4 @@ -0,0 +1,5 @@ +238 -18 238 238 238 238.0 238.0 238 val_238 val_238_concat false true false false false val_238 -1 -1 VALUE_238 al_238 al_2 VAL_238 val_238 2013-02-15 19:41:20 113 1 5 19 41 20 1360986080000 +86 86 86 86 86 86.0 86.0 86 val_86 val_86_concat true true true true true val_86 -1 -1 VALUE_86 al_86 al_8 VAL_86 val_86 2013-02-15 19:41:20 113 1 5 19 41 20 1360986080000 +311 55 311 311 311 311.0 311.0 311 val_311 val_311_concat false true false false false val_311 5 6 VALUE_311 al_311 al_3 VAL_311 val_311 2013-02-15 19:41:20 113 1 5 19 41 20 1360986080000 +27 27 27 27 27 27.0 27.0 27 val_27 val_27_concat false true false false false val_27 -1 -1 VALUE_27 al_27 al_2 VAL_27 val_27 2013-02-15 19:41:20 113 1 5 19 41 20 1360986080000 +165 -91 165 165 165 165.0 165.0 165 val_165 val_165_concat false true false false false val_165 4 4 VALUE_165 al_165 al_1 VAL_165 val_165 2013-02-15 19:41:20 113 1 5 19 41 20 1360986080000 From 184abfd1eb09066ec96b8ed7209d8741aaac947e Mon Sep 17 00:00:00 2001 From: Cheng Hao Date: Sat, 14 Feb 2015 16:29:10 -0800 Subject: [PATCH 2/3] revert the change to table src1 --- .../main/scala/org/apache/spark/sql/hive/test/TestHive.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala index d49b60e0fadab..d3a36135a8a9e 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala @@ -202,7 +202,7 @@ class TestHiveContext(sc: SparkContext) extends HiveContext(sc) { "CREATE TABLE src (key STRING, value STRING)".cmd, s"LOAD DATA LOCAL INPATH '${getHiveFile("data/files/kv1.txt")}' INTO TABLE src".cmd), TestTable("src1", - "CREATE TABLE src1 (key STRING, value STRING)".cmd, + "CREATE TABLE src1 (key INT, value STRING)".cmd, s"LOAD DATA LOCAL INPATH '${getHiveFile("data/files/kv3.txt")}' INTO TABLE src1".cmd), TestTable("srcpart", () => { runSqlHive( From e5bdc3a2f1847098f3f663d6e3a336cbdaf50bce Mon Sep 17 00:00:00 2001 From: Cheng Hao Date: Mon, 16 Feb 2015 23:42:27 -0800 Subject: [PATCH 3/3] Move the test case reflect into blacklist --- .../spark/sql/hive/execution/HiveCompatibilitySuite.scala | 6 +++++- .../scala/org/apache/spark/sql/hive/test/TestHive.scala | 2 +- .../golden/udf_reflect-0-50131c0ba7b7a6b65c789a5a8497bada | 1 - .../golden/udf_reflect-1-904138e2a1f831c308b7f0aacc859ae1 | 1 - .../golden/udf_reflect-2-21ec7caa253c7f95b7cf60191140e2ee | 4 ---- .../golden/udf_reflect-3-f78a6582e4818a384997eefada54c77 | 0 .../golden/udf_reflect-4-2a9bfe456332e7ad776662e2449bd2e0 | 1 - 7 files changed, 6 insertions(+), 9 deletions(-) delete mode 100644 sql/hive/src/test/resources/golden/udf_reflect-0-50131c0ba7b7a6b65c789a5a8497bada delete mode 100644 sql/hive/src/test/resources/golden/udf_reflect-1-904138e2a1f831c308b7f0aacc859ae1 delete mode 100644 sql/hive/src/test/resources/golden/udf_reflect-2-21ec7caa253c7f95b7cf60191140e2ee delete mode 100644 sql/hive/src/test/resources/golden/udf_reflect-3-f78a6582e4818a384997eefada54c77 delete mode 100644 sql/hive/src/test/resources/golden/udf_reflect-4-2a9bfe456332e7ad776662e2449bd2e0 diff --git a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala index 5497e207c759d..51664612ceeaa 100644 --- a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala +++ b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala @@ -225,6 +225,11 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter { // Needs constant object inspectors "udf_round", + // the table src(key INT, value STRING) is not the same as HIVE unittest. In Hive + // is src(key STRING, value STRING), and in the reflect.q, it failed in + // Integer.valueOf, which expect the first argument passed as STRING type not INT. + "udf_reflect", + // Sort with Limit clause causes failure. "ctas", "ctas_hadoop20", @@ -885,7 +890,6 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter { "udf_power", "udf_radians", "udf_rand", - "udf_reflect", "udf_reflect2", "udf_regexp", "udf_regexp_extract", diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala index d3a36135a8a9e..a2d99f1f4b28d 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala @@ -199,7 +199,7 @@ class TestHiveContext(sc: SparkContext) extends HiveContext(sc) { // https://github.com/apache/hive/blob/branch-0.13/data/scripts/q_test_init.sql val hiveQTestUtilTables = Seq( TestTable("src", - "CREATE TABLE src (key STRING, value STRING)".cmd, + "CREATE TABLE src (key INT, value STRING)".cmd, s"LOAD DATA LOCAL INPATH '${getHiveFile("data/files/kv1.txt")}' INTO TABLE src".cmd), TestTable("src1", "CREATE TABLE src1 (key INT, value STRING)".cmd, diff --git a/sql/hive/src/test/resources/golden/udf_reflect-0-50131c0ba7b7a6b65c789a5a8497bada b/sql/hive/src/test/resources/golden/udf_reflect-0-50131c0ba7b7a6b65c789a5a8497bada deleted file mode 100644 index 573541ac9702d..0000000000000 --- a/sql/hive/src/test/resources/golden/udf_reflect-0-50131c0ba7b7a6b65c789a5a8497bada +++ /dev/null @@ -1 +0,0 @@ -0 diff --git a/sql/hive/src/test/resources/golden/udf_reflect-1-904138e2a1f831c308b7f0aacc859ae1 b/sql/hive/src/test/resources/golden/udf_reflect-1-904138e2a1f831c308b7f0aacc859ae1 deleted file mode 100644 index 1d7658151cd62..0000000000000 --- a/sql/hive/src/test/resources/golden/udf_reflect-1-904138e2a1f831c308b7f0aacc859ae1 +++ /dev/null @@ -1 +0,0 @@ -reflect(class,method[,arg1[,arg2..]]) calls method with reflection diff --git a/sql/hive/src/test/resources/golden/udf_reflect-2-21ec7caa253c7f95b7cf60191140e2ee b/sql/hive/src/test/resources/golden/udf_reflect-2-21ec7caa253c7f95b7cf60191140e2ee deleted file mode 100644 index ddf986c01e5b1..0000000000000 --- a/sql/hive/src/test/resources/golden/udf_reflect-2-21ec7caa253c7f95b7cf60191140e2ee +++ /dev/null @@ -1,4 +0,0 @@ -reflect(class,method[,arg1[,arg2..]]) calls method with reflection -Synonyms: java_method -Use this UDF to call Java methods by matching the argument signature - diff --git a/sql/hive/src/test/resources/golden/udf_reflect-3-f78a6582e4818a384997eefada54c77 b/sql/hive/src/test/resources/golden/udf_reflect-3-f78a6582e4818a384997eefada54c77 deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/sql/hive/src/test/resources/golden/udf_reflect-4-2a9bfe456332e7ad776662e2449bd2e0 b/sql/hive/src/test/resources/golden/udf_reflect-4-2a9bfe456332e7ad776662e2449bd2e0 deleted file mode 100644 index a92d8dca89b3e..0000000000000 --- a/sql/hive/src/test/resources/golden/udf_reflect-4-2a9bfe456332e7ad776662e2449bd2e0 +++ /dev/null @@ -1 +0,0 @@ -1 true 3 2 3 2.718281828459045 1.0 568