diff --git a/hbase-handler/src/test/results/positive/hbase_viewjoins.q.out b/hbase-handler/src/test/results/positive/hbase_viewjoins.q.out index 9eae99d51a99..2686ea83d65a 100644 --- a/hbase-handler/src/test/results/positive/hbase_viewjoins.q.out +++ b/hbase-handler/src/test/results/positive/hbase_viewjoins.q.out @@ -64,9 +64,9 @@ POSTHOOK: type: CREATEVIEW POSTHOOK: Input: default@hbase_table_test_1 POSTHOOK: Output: database:default POSTHOOK: Output: default@VIEW_HBASE_TABLE_TEST_1 -POSTHOOK: Lineage: VIEW_HBASE_TABLE_TEST_1.ccount SIMPLE [(hbase_table_test_1)hbase_table_test_1.FieldSchema(name:ccount, type:int, comment:), ] -POSTHOOK: Lineage: VIEW_HBASE_TABLE_TEST_1.cvalue SIMPLE [(hbase_table_test_1)hbase_table_test_1.FieldSchema(name:cvalue, type:string, comment:), ] -POSTHOOK: Lineage: VIEW_HBASE_TABLE_TEST_1.pk SIMPLE [(hbase_table_test_1)hbase_table_test_1.FieldSchema(name:pk, type:string, comment:), ] +POSTHOOK: Lineage: view_hbase_table_test_1.ccount SIMPLE [(hbase_table_test_1)hbase_table_test_1.FieldSchema(name:ccount, type:int, comment:), ] +POSTHOOK: Lineage: view_hbase_table_test_1.cvalue SIMPLE [(hbase_table_test_1)hbase_table_test_1.FieldSchema(name:cvalue, type:string, comment:), ] +POSTHOOK: Lineage: view_hbase_table_test_1.pk SIMPLE [(hbase_table_test_1)hbase_table_test_1.FieldSchema(name:pk, type:string, comment:), ] PREHOOK: query: CREATE EXTERNAL TABLE HBASE_TABLE_TEST_2( cvalue string , pk string , @@ -121,9 +121,9 @@ POSTHOOK: type: CREATEVIEW POSTHOOK: Input: default@hbase_table_test_2 POSTHOOK: Output: database:default POSTHOOK: Output: default@VIEW_HBASE_TABLE_TEST_2 -POSTHOOK: Lineage: VIEW_HBASE_TABLE_TEST_2.ccount SIMPLE [(hbase_table_test_2)hbase_table_test_2.FieldSchema(name:ccount, type:int, comment:), ] -POSTHOOK: Lineage: VIEW_HBASE_TABLE_TEST_2.cvalue SIMPLE [(hbase_table_test_2)hbase_table_test_2.FieldSchema(name:cvalue, type:string, comment:), ] -POSTHOOK: Lineage: VIEW_HBASE_TABLE_TEST_2.pk SIMPLE [(hbase_table_test_2)hbase_table_test_2.FieldSchema(name:pk, type:string, comment:), ] +POSTHOOK: Lineage: view_hbase_table_test_2.ccount SIMPLE [(hbase_table_test_2)hbase_table_test_2.FieldSchema(name:ccount, type:int, comment:), ] +POSTHOOK: Lineage: view_hbase_table_test_2.cvalue SIMPLE [(hbase_table_test_2)hbase_table_test_2.FieldSchema(name:cvalue, type:string, comment:), ] +POSTHOOK: Lineage: view_hbase_table_test_2.pk SIMPLE [(hbase_table_test_2)hbase_table_test_2.FieldSchema(name:pk, type:string, comment:), ] PREHOOK: query: SELECT p.cvalue cvalue FROM `VIEW_HBASE_TABLE_TEST_1` `p` LEFT OUTER JOIN `VIEW_HBASE_TABLE_TEST_2` `A1` diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestAcidTableSetup.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestAcidTableSetup.java index 0f004a3372e1..5317a7ce3cae 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestAcidTableSetup.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestAcidTableSetup.java @@ -91,7 +91,7 @@ public void testTransactionalValidation() throws Throwable { client.createTable(t); fail("Expected exception"); } catch (MetaException e) { - assertEquals("'transactional' property of TBLPROPERTIES may only have value 'true': acidDb.acidTable", + assertEquals("'transactional' property of TBLPROPERTIES may only have value 'true': aciddb.acidtable", e.getMessage()); } @@ -108,7 +108,7 @@ public void testTransactionalValidation() throws Throwable { client.createTable(t); fail("Expected exception"); } catch (MetaException e) { - assertEquals("'transactional' property of TBLPROPERTIES may only have value 'true': acidDb.acidTable", + assertEquals("'transactional' property of TBLPROPERTIES may only have value 'true': aciddb.acidtable", e.getMessage()); } @@ -125,7 +125,7 @@ public void testTransactionalValidation() throws Throwable { client.createTable(t); fail("Expected exception"); } catch (MetaException e) { - assertEquals("The table must be stored using an ACID compliant format (such as ORC): acidDb.acidTable", + assertEquals("The table must be stored using an ACID compliant format (such as ORC): aciddb.acidtable", e.getMessage()); } @@ -145,7 +145,7 @@ public void testTransactionalValidation() throws Throwable { client.createTable(t); fail("Expected exception"); } catch (MetaException e) { - assertEquals("The table must be stored using an ACID compliant format (such as ORC): acidDb.acidTable", + assertEquals("The table must be stored using an ACID compliant format (such as ORC): aciddb.acidtable", e.getMessage()); } @@ -197,7 +197,7 @@ public void testTransactionalValidation() throws Throwable { client.alter_table(dbName, tblName, t); fail("Expected exception"); } catch (MetaException e) { - assertEquals("The table must be stored using an ACID compliant format (such as ORC): acidDb.acidTable1", + assertEquals("The table must be stored using an ACID compliant format (such as ORC): aciddb.acidtable1", e.getMessage()); } diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java index 03d2103a84b8..d0bee699a195 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java @@ -1865,7 +1865,7 @@ public void testOpenTxnEvent() throws Throwable { primary.dump(primaryDbName); long lastReplId = Long.parseLong(bootStrapDump.lastReplicationId); - primary.testEventCounts(primaryDbName, lastReplId, null, null, 10); + primary.testEventCounts(primaryDbName, lastReplId, null, null, 11); // Test load replica.load(replicatedDbName, primaryDbName) diff --git a/ql/src/test/queries/clientpositive/create_table.q b/ql/src/test/queries/clientpositive/create_table.q index f6262c726de5..8d182d0f67ea 100644 --- a/ql/src/test/queries/clientpositive/create_table.q +++ b/ql/src/test/queries/clientpositive/create_table.q @@ -37,3 +37,8 @@ describe formatted test31; create table test32 as select * from test31; show create table test32; describe formatted test32; + +-- create table with empty characters within quotes +create table ` default`.` table41`(i int); +show create table ` default`.` table41`; +describe formatted ` default`.` table41`; diff --git a/ql/src/test/queries/clientpositive/translated_external_alter.q b/ql/src/test/queries/clientpositive/translated_external_alter.q index 701078278401..aa94c414ce44 100644 --- a/ql/src/test/queries/clientpositive/translated_external_alter.q +++ b/ql/src/test/queries/clientpositive/translated_external_alter.q @@ -6,3 +6,7 @@ set hive.compute.query.using.stats=false; create table caseSensitive (a integer); alter table casesEnsitivE set tblproperties('some'='one'); + +-- alter table with empty characters in quotes shouldn't throw error +alter table ` default`.` caseSensitive ` add columns(i int); +describe formatted caseSensitive; \ No newline at end of file diff --git a/ql/src/test/results/clientnegative/alter_view_as_select_with_partition.q.out b/ql/src/test/results/clientnegative/alter_view_as_select_with_partition.q.out index 5261a3bd153e..7da6c8125bbd 100644 --- a/ql/src/test/results/clientnegative/alter_view_as_select_with_partition.q.out +++ b/ql/src/test/results/clientnegative/alter_view_as_select_with_partition.q.out @@ -16,7 +16,7 @@ POSTHOOK: type: CREATEVIEW POSTHOOK: Input: default@src POSTHOOK: Output: database:default POSTHOOK: Output: default@testViewPart -POSTHOOK: Lineage: testViewPart.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: testviewpart.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] PREHOOK: query: ALTER VIEW testViewPart ADD PARTITION (value='val_86') PARTITION (value='val_xyz') PREHOOK: type: ALTERTABLE_ADDPARTS diff --git a/ql/src/test/results/clientnegative/unset_view_property.q.out b/ql/src/test/results/clientnegative/unset_view_property.q.out index b5038d971bfd..7f1ab0216945 100644 --- a/ql/src/test/results/clientnegative/unset_view_property.q.out +++ b/ql/src/test/results/clientnegative/unset_view_property.q.out @@ -8,7 +8,7 @@ POSTHOOK: type: CREATEVIEW POSTHOOK: Input: default@src POSTHOOK: Output: database:default POSTHOOK: Output: default@testView -POSTHOOK: Lineage: testView.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: testview.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: ALTER VIEW testView SET TBLPROPERTIES ('propA'='100', 'propB'='200') PREHOOK: type: ALTERVIEW_PROPERTIES PREHOOK: Input: default@testview diff --git a/ql/src/test/results/clientpositive/create_table.q.out b/ql/src/test/results/clientpositive/create_table.q.out index dc7dbc63e7b1..f408986f8f5b 100644 --- a/ql/src/test/results/clientpositive/create_table.q.out +++ b/ql/src/test/results/clientpositive/create_table.q.out @@ -388,6 +388,67 @@ Table Parameters: totalSize 2 #### A masked pattern was here #### +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: create table ` default`.` table41`(i int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: default@ table41 +PREHOOK: Output: database:default +POSTHOOK: query: create table ` default`.` table41`(i int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@ table41 +POSTHOOK: Output: database:default +PREHOOK: query: show create table ` default`.` table41` +PREHOOK: type: SHOW_CREATETABLE +PREHOOK: Input: default@table41 +POSTHOOK: query: show create table ` default`.` table41` +POSTHOOK: type: SHOW_CREATETABLE +POSTHOOK: Input: default@table41 +CREATE TABLE `default`.`table41`( + `i` int) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.mapred.TextInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' +LOCATION +#### A masked pattern was here #### +TBLPROPERTIES ( + 'bucketing_version'='2', +#### A masked pattern was here #### +PREHOOK: query: describe formatted ` default`.` table41` +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@table41 +POSTHOOK: query: describe formatted ` default`.` table41` +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@table41 +# col_name data_type comment +i int + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"i\":\"true\"}} + bucketing_version 2 + numFiles 0 + numRows 0 + rawDataSize 0 + totalSize 0 +#### A masked pattern was here #### + # Storage Information SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe InputFormat: org.apache.hadoop.mapred.TextInputFormat diff --git a/ql/src/test/results/clientpositive/llap/alter_view_as_select.q.out b/ql/src/test/results/clientpositive/llap/alter_view_as_select.q.out index 9fc906e5be10..f45969efa1a8 100644 --- a/ql/src/test/results/clientpositive/llap/alter_view_as_select.q.out +++ b/ql/src/test/results/clientpositive/llap/alter_view_as_select.q.out @@ -14,10 +14,10 @@ POSTHOOK: type: CREATEVIEW POSTHOOK: Input: default@srcpart POSTHOOK: Output: database:tv POSTHOOK: Output: tv@testView -POSTHOOK: Lineage: testView.ds SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] -POSTHOOK: Lineage: testView.hr SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] -POSTHOOK: Lineage: testView.key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: testView.value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: testview.ds SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: testview.hr SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: testview.key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: testview.value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: DESCRIBE FORMATTED tv.testView PREHOOK: type: DESCTABLE PREHOOK: Input: tv@testview diff --git a/ql/src/test/results/clientpositive/llap/create_table.q.out b/ql/src/test/results/clientpositive/llap/create_table.q.out index 58a2327f59c0..2cecd51cd100 100644 --- a/ql/src/test/results/clientpositive/llap/create_table.q.out +++ b/ql/src/test/results/clientpositive/llap/create_table.q.out @@ -388,6 +388,67 @@ Table Parameters: totalSize 2 #### A masked pattern was here #### +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: create table ` default`.` table41`(i int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: default@ table41 +PREHOOK: Output: database:default +POSTHOOK: query: create table ` default`.` table41`(i int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@ table41 +POSTHOOK: Output: database:default +PREHOOK: query: show create table ` default`.` table41` +PREHOOK: type: SHOW_CREATETABLE +PREHOOK: Input: default@table41 +POSTHOOK: query: show create table ` default`.` table41` +POSTHOOK: type: SHOW_CREATETABLE +POSTHOOK: Input: default@table41 +CREATE TABLE `default`.`table41`( + `i` int) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.mapred.TextInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' +LOCATION + 'hdfs://### HDFS PATH ###' +TBLPROPERTIES ( + 'bucketing_version'='2', +#### A masked pattern was here #### +PREHOOK: query: describe formatted ` default`.` table41` +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@table41 +POSTHOOK: query: describe formatted ` default`.` table41` +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@table41 +# col_name data_type comment +i int + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"i\":\"true\"}} + bucketing_version 2 + numFiles 0 + numRows 0 + rawDataSize 0 + totalSize 0 +#### A masked pattern was here #### + # Storage Information SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe InputFormat: org.apache.hadoop.mapred.TextInputFormat diff --git a/ql/src/test/results/clientpositive/llap/dbtxnmgr_ddl1.q.out b/ql/src/test/results/clientpositive/llap/dbtxnmgr_ddl1.q.out index 4bcedc0abdb9..ea54b6220150 100644 --- a/ql/src/test/results/clientpositive/llap/dbtxnmgr_ddl1.q.out +++ b/ql/src/test/results/clientpositive/llap/dbtxnmgr_ddl1.q.out @@ -227,7 +227,7 @@ POSTHOOK: type: CREATEVIEW POSTHOOK: Input: default@t1_n50 POSTHOOK: Output: database:default POSTHOOK: Output: default@V1_n5 -POSTHOOK: Lineage: V1_n5.key SIMPLE [(t1_n50)t1_n50.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: v1_n5.key SIMPLE [(t1_n50)t1_n50.FieldSchema(name:key, type:string, comment:null), ] PREHOOK: query: alter view V1_n5 set tblproperties ('test'='thisisatest') PREHOOK: type: ALTERVIEW_PROPERTIES PREHOOK: Input: default@v1_n5 diff --git a/ql/src/test/results/clientpositive/llap/dbtxnmgr_query5.q.out b/ql/src/test/results/clientpositive/llap/dbtxnmgr_query5.q.out index ea76a37512d3..b919a62d2480 100644 --- a/ql/src/test/results/clientpositive/llap/dbtxnmgr_query5.q.out +++ b/ql/src/test/results/clientpositive/llap/dbtxnmgr_query5.q.out @@ -35,7 +35,7 @@ POSTHOOK: type: CREATEVIEW POSTHOOK: Input: foo@t1_n40 POSTHOOK: Output: database:foo POSTHOOK: Output: foo@V1_n3 -POSTHOOK: Lineage: V1_n3.key SIMPLE [(t1_n40)t1_n40.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: v1_n3.key SIMPLE [(t1_n40)t1_n40.FieldSchema(name:key, type:string, comment:null), ] PREHOOK: query: show tables PREHOOK: type: SHOWTABLES PREHOOK: Input: database:foo diff --git a/ql/src/test/results/clientpositive/llap/explain_ddl.q.out b/ql/src/test/results/clientpositive/llap/explain_ddl.q.out index 713d2f15445f..04d9fbcba1e8 100644 --- a/ql/src/test/results/clientpositive/llap/explain_ddl.q.out +++ b/ql/src/test/results/clientpositive/llap/explain_ddl.q.out @@ -8,8 +8,8 @@ POSTHOOK: type: CREATEVIEW POSTHOOK: Input: default@src POSTHOOK: Output: database:default POSTHOOK: Output: default@V1_n0 -POSTHOOK: Lineage: V1_n0.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: V1_n0.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: v1_n0.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: v1_n0.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: select count(*) from V1_n0 where key > 0 PREHOOK: type: QUERY PREHOOK: Input: default@src diff --git a/ql/src/test/results/clientpositive/llap/explain_dependency.q.out b/ql/src/test/results/clientpositive/llap/explain_dependency.q.out index da1036b3c7b9..5234ca009f07 100644 --- a/ql/src/test/results/clientpositive/llap/explain_dependency.q.out +++ b/ql/src/test/results/clientpositive/llap/explain_dependency.q.out @@ -8,8 +8,8 @@ POSTHOOK: type: CREATEVIEW POSTHOOK: Input: default@src POSTHOOK: Output: database:default POSTHOOK: Output: default@V1_n6 -POSTHOOK: Lineage: V1_n6.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: V1_n6.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: v1_n6.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: v1_n6.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: CREATE VIEW V2_n1 AS SELECT ds, key, value FROM srcpart WHERE ds IS NOT NULL PREHOOK: type: CREATEVIEW PREHOOK: Input: default@srcpart @@ -20,9 +20,9 @@ POSTHOOK: type: CREATEVIEW POSTHOOK: Input: default@srcpart POSTHOOK: Output: database:default POSTHOOK: Output: default@V2_n1 -POSTHOOK: Lineage: V2_n1.ds SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] -POSTHOOK: Lineage: V2_n1.key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: V2_n1.value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: v2_n1.ds SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: v2_n1.key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: v2_n1.value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: CREATE VIEW V3 AS SELECT src1.key, src2.value FROM V2_n1 src1 JOIN src src2 ON src1.key = src2.key WHERE src1.ds IS NOT NULL @@ -41,8 +41,8 @@ POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@v2_n1 POSTHOOK: Output: database:default POSTHOOK: Output: default@V3 -POSTHOOK: Lineage: V3.key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: V3.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: v3.key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: v3.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: CREATE VIEW V4 AS SELECT src1.key, src2.value as value1, src3.value as value2 FROM V1_n6 src1 JOIN V2_n1 src2 on src1.key = src2.key JOIN src src3 ON src2.key = src3.key @@ -63,9 +63,9 @@ POSTHOOK: Input: default@v1_n6 POSTHOOK: Input: default@v2_n1 POSTHOOK: Output: database:default POSTHOOK: Output: default@V4 -POSTHOOK: Lineage: V4.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: V4.value1 SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: V4.value2 SIMPLE [(src)src3.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: v4.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: v4.value1 SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: v4.value2 SIMPLE [(src)src3.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: EXPLAIN DEPENDENCY SELECT key, count(1) FROM srcpart WHERE ds IS NOT NULL GROUP BY key PREHOOK: type: QUERY @@ -230,10 +230,10 @@ POSTHOOK: type: CREATEVIEW POSTHOOK: Input: default@srcpart POSTHOOK: Output: database:default POSTHOOK: Output: default@V5 -POSTHOOK: Lineage: V5.ds SIMPLE [] -POSTHOOK: Lineage: V5.hr SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] -POSTHOOK: Lineage: V5.key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: V5.value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: v5.ds SIMPLE [] +POSTHOOK: Lineage: v5.hr SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: v5.key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: v5.value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: EXPLAIN DEPENDENCY SELECT * FROM V5 PREHOOK: type: QUERY PREHOOK: Input: default@srcpart diff --git a/ql/src/test/results/clientpositive/llap/explain_logical.q.out b/ql/src/test/results/clientpositive/llap/explain_logical.q.out index 508a72354c72..93318dbba0da 100644 --- a/ql/src/test/results/clientpositive/llap/explain_logical.q.out +++ b/ql/src/test/results/clientpositive/llap/explain_logical.q.out @@ -8,8 +8,8 @@ POSTHOOK: type: CREATEVIEW POSTHOOK: Input: default@src POSTHOOK: Output: database:default POSTHOOK: Output: default@V1_n8 -POSTHOOK: Lineage: V1_n8.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: V1_n8.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: v1_n8.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: v1_n8.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: CREATE VIEW V2_n3 AS SELECT ds, key, value FROM srcpart WHERE ds IS NOT NULL PREHOOK: type: CREATEVIEW PREHOOK: Input: default@srcpart @@ -20,9 +20,9 @@ POSTHOOK: type: CREATEVIEW POSTHOOK: Input: default@srcpart POSTHOOK: Output: database:default POSTHOOK: Output: default@V2_n3 -POSTHOOK: Lineage: V2_n3.ds SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] -POSTHOOK: Lineage: V2_n3.key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: V2_n3.value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: v2_n3.ds SIMPLE [(srcpart)srcpart.FieldSchema(name:ds, type:string, comment:null), ] +POSTHOOK: Lineage: v2_n3.key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: v2_n3.value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: CREATE VIEW V3_n1 AS SELECT src1.key, src2.value FROM V2_n3 src1 JOIN src src2 ON src1.key = src2.key WHERE src1.ds IS NOT NULL @@ -41,8 +41,8 @@ POSTHOOK: Input: default@srcpart POSTHOOK: Input: default@v2_n3 POSTHOOK: Output: database:default POSTHOOK: Output: default@V3_n1 -POSTHOOK: Lineage: V3_n1.key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: V3_n1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: v3_n1.key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: v3_n1.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: CREATE VIEW V4_n1 AS SELECT src1.key, src2.value as value1, src3.value as value2 FROM V1_n8 src1 JOIN V2_n3 src2 on src1.key = src2.key JOIN src src3 ON src2.key = src3.key @@ -63,9 +63,9 @@ POSTHOOK: Input: default@v1_n8 POSTHOOK: Input: default@v2_n3 POSTHOOK: Output: database:default POSTHOOK: Output: default@V4_n1 -POSTHOOK: Lineage: V4_n1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: V4_n1.value1 SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] -POSTHOOK: Lineage: V4_n1.value2 SIMPLE [(src)src3.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: v4_n1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: v4_n1.value1 SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: v4_n1.value2 SIMPLE [(src)src3.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: EXPLAIN LOGICAL SELECT key, count(1) FROM srcpart WHERE ds IS NOT NULL GROUP BY key PREHOOK: type: QUERY @@ -498,10 +498,10 @@ POSTHOOK: type: CREATEVIEW POSTHOOK: Input: default@srcpart POSTHOOK: Output: database:default POSTHOOK: Output: default@V5_n0 -POSTHOOK: Lineage: V5_n0.ds SIMPLE [] -POSTHOOK: Lineage: V5_n0.hr SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] -POSTHOOK: Lineage: V5_n0.key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: V5_n0.value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: v5_n0.ds SIMPLE [] +POSTHOOK: Lineage: v5_n0.hr SIMPLE [(srcpart)srcpart.FieldSchema(name:hr, type:string, comment:null), ] +POSTHOOK: Lineage: v5_n0.key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: v5_n0.value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: EXPLAIN LOGICAL SELECT * FROM V5_n0 PREHOOK: type: QUERY PREHOOK: Input: default@srcpart diff --git a/ql/src/test/results/clientpositive/llap/ppd_deterministic_expr.q.out b/ql/src/test/results/clientpositive/llap/ppd_deterministic_expr.q.out index 73e3b7d06e21..70b72eb0a67f 100644 --- a/ql/src/test/results/clientpositive/llap/ppd_deterministic_expr.q.out +++ b/ql/src/test/results/clientpositive/llap/ppd_deterministic_expr.q.out @@ -221,11 +221,11 @@ POSTHOOK: Input: default@testa@part1=US/part2=ABC/part3=123 POSTHOOK: Input: default@testa@part1=US/part2=DEF/part3=200 POSTHOOK: Output: database:default POSTHOOK: Output: default@viewDeterministicUDFA -POSTHOOK: Lineage: viewDeterministicUDFA.vcol1 EXPRESSION [(testa)testa.FieldSchema(name:col1, type:string, comment:), ] -POSTHOOK: Lineage: viewDeterministicUDFA.vcol2 EXPRESSION [(testa)testa.FieldSchema(name:col2, type:string, comment:), ] -POSTHOOK: Lineage: viewDeterministicUDFA.vcol3 EXPRESSION [(testa)testa.FieldSchema(name:col3, type:string, comment:), ] -POSTHOOK: Lineage: viewDeterministicUDFA.vcol4 EXPRESSION [(testa)testa.FieldSchema(name:col4, type:string, comment:), ] -POSTHOOK: Lineage: viewDeterministicUDFA.vcol5 EXPRESSION [(testa)testa.FieldSchema(name:col5, type:string, comment:), ] +POSTHOOK: Lineage: viewdeterministicudfa.vcol1 EXPRESSION [(testa)testa.FieldSchema(name:col1, type:string, comment:), ] +POSTHOOK: Lineage: viewdeterministicudfa.vcol2 EXPRESSION [(testa)testa.FieldSchema(name:col2, type:string, comment:), ] +POSTHOOK: Lineage: viewdeterministicudfa.vcol3 EXPRESSION [(testa)testa.FieldSchema(name:col3, type:string, comment:), ] +POSTHOOK: Lineage: viewdeterministicudfa.vcol4 EXPRESSION [(testa)testa.FieldSchema(name:col4, type:string, comment:), ] +POSTHOOK: Lineage: viewdeterministicudfa.vcol5 EXPRESSION [(testa)testa.FieldSchema(name:col5, type:string, comment:), ] PREHOOK: query: create view viewDeterministicUDFB partitioned on (vpart1, vpart2, vpart3) as select cast(cola as decimal(38,18)) as vcolA, cast(colb as decimal(38,18)) as vcolB, @@ -260,9 +260,9 @@ POSTHOOK: Input: default@testb@part1=US/part2=DEF/part3=200 POSTHOOK: Input: default@testb@part1=US/part2=PQR/part3=123 POSTHOOK: Output: database:default POSTHOOK: Output: default@viewDeterministicUDFB -POSTHOOK: Lineage: viewDeterministicUDFB.vcola EXPRESSION [(testb)testb.FieldSchema(name:cola, type:string, comment:), ] -POSTHOOK: Lineage: viewDeterministicUDFB.vcolb EXPRESSION [(testb)testb.FieldSchema(name:colb, type:string, comment:), ] -POSTHOOK: Lineage: viewDeterministicUDFB.vcolc EXPRESSION [(testb)testb.FieldSchema(name:colc, type:string, comment:), ] +POSTHOOK: Lineage: viewdeterministicudfb.vcola EXPRESSION [(testb)testb.FieldSchema(name:cola, type:string, comment:), ] +POSTHOOK: Lineage: viewdeterministicudfb.vcolb EXPRESSION [(testb)testb.FieldSchema(name:colb, type:string, comment:), ] +POSTHOOK: Lineage: viewdeterministicudfb.vcolc EXPRESSION [(testb)testb.FieldSchema(name:colc, type:string, comment:), ] PREHOOK: query: create view viewNoUDFA partitioned on (part1, part2, part3) as select cast(col1 as decimal(38,18)) as vcol1, cast(col2 as decimal(38,18)) as vcol2, @@ -299,11 +299,11 @@ POSTHOOK: Input: default@testa@part1=US/part2=ABC/part3=123 POSTHOOK: Input: default@testa@part1=US/part2=DEF/part3=200 POSTHOOK: Output: database:default POSTHOOK: Output: default@viewNoUDFA -POSTHOOK: Lineage: viewNoUDFA.vcol1 EXPRESSION [(testa)testa.FieldSchema(name:col1, type:string, comment:), ] -POSTHOOK: Lineage: viewNoUDFA.vcol2 EXPRESSION [(testa)testa.FieldSchema(name:col2, type:string, comment:), ] -POSTHOOK: Lineage: viewNoUDFA.vcol3 EXPRESSION [(testa)testa.FieldSchema(name:col3, type:string, comment:), ] -POSTHOOK: Lineage: viewNoUDFA.vcol4 EXPRESSION [(testa)testa.FieldSchema(name:col4, type:string, comment:), ] -POSTHOOK: Lineage: viewNoUDFA.vcol5 EXPRESSION [(testa)testa.FieldSchema(name:col5, type:string, comment:), ] +POSTHOOK: Lineage: viewnoudfa.vcol1 EXPRESSION [(testa)testa.FieldSchema(name:col1, type:string, comment:), ] +POSTHOOK: Lineage: viewnoudfa.vcol2 EXPRESSION [(testa)testa.FieldSchema(name:col2, type:string, comment:), ] +POSTHOOK: Lineage: viewnoudfa.vcol3 EXPRESSION [(testa)testa.FieldSchema(name:col3, type:string, comment:), ] +POSTHOOK: Lineage: viewnoudfa.vcol4 EXPRESSION [(testa)testa.FieldSchema(name:col4, type:string, comment:), ] +POSTHOOK: Lineage: viewnoudfa.vcol5 EXPRESSION [(testa)testa.FieldSchema(name:col5, type:string, comment:), ] PREHOOK: query: create view viewNoUDFB partitioned on (part1, part2, part3) as select cast(cola as decimal(38,18)) as vcolA, cast(colb as decimal(38,18)) as vcolB, @@ -338,9 +338,9 @@ POSTHOOK: Input: default@testb@part1=US/part2=DEF/part3=200 POSTHOOK: Input: default@testb@part1=US/part2=PQR/part3=123 POSTHOOK: Output: database:default POSTHOOK: Output: default@viewNoUDFB -POSTHOOK: Lineage: viewNoUDFB.vcola EXPRESSION [(testb)testb.FieldSchema(name:cola, type:string, comment:), ] -POSTHOOK: Lineage: viewNoUDFB.vcolb EXPRESSION [(testb)testb.FieldSchema(name:colb, type:string, comment:), ] -POSTHOOK: Lineage: viewNoUDFB.vcolc EXPRESSION [(testb)testb.FieldSchema(name:colc, type:string, comment:), ] +POSTHOOK: Lineage: viewnoudfb.vcola EXPRESSION [(testb)testb.FieldSchema(name:cola, type:string, comment:), ] +POSTHOOK: Lineage: viewnoudfb.vcolb EXPRESSION [(testb)testb.FieldSchema(name:colb, type:string, comment:), ] +POSTHOOK: Lineage: viewnoudfb.vcolc EXPRESSION [(testb)testb.FieldSchema(name:colc, type:string, comment:), ] PREHOOK: query: explain select vcol1, vcol2, vcol3, vcola, vcolb from viewDeterministicUDFA a inner join viewDeterministicUDFB b diff --git a/ql/src/test/results/clientpositive/llap/subquery_notin.q.out b/ql/src/test/results/clientpositive/llap/subquery_notin.q.out index 31350595c655..e26868dbe62b 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_notin.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_notin.q.out @@ -1440,7 +1440,7 @@ POSTHOOK: type: CREATEVIEW POSTHOOK: Input: default@src POSTHOOK: Output: database:default POSTHOOK: Output: default@T1_v -POSTHOOK: Lineage: T1_v.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: t1_v.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] PREHOOK: query: create view T2_v as select case when key > '104' then null else key end as key from T1_v PREHOOK: type: CREATEVIEW @@ -1455,7 +1455,7 @@ POSTHOOK: Input: default@src POSTHOOK: Input: default@t1_v POSTHOOK: Output: database:default POSTHOOK: Output: default@T2_v -POSTHOOK: Lineage: T2_v.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: t2_v.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] Warning: Shuffle Join MERGEJOIN[34][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product PREHOOK: query: explain select * diff --git a/ql/src/test/results/clientpositive/llap/translated_external_alter.q.out b/ql/src/test/results/clientpositive/llap/translated_external_alter.q.out index 04b13b21bb9f..b1dec0b7f0ae 100644 --- a/ql/src/test/results/clientpositive/llap/translated_external_alter.q.out +++ b/ql/src/test/results/clientpositive/llap/translated_external_alter.q.out @@ -14,3 +14,51 @@ POSTHOOK: query: alter table casesEnsitivE set tblproperties('some'='one') POSTHOOK: type: ALTERTABLE_PROPERTIES POSTHOOK: Input: default@casesensitive POSTHOOK: Output: default@casesensitive +PREHOOK: query: alter table ` default`.` caseSensitive ` add columns(i int) +PREHOOK: type: ALTERTABLE_ADDCOLS +PREHOOK: Input: default@casesensitive +PREHOOK: Output: default@casesensitive +POSTHOOK: query: alter table ` default`.` caseSensitive ` add columns(i int) +POSTHOOK: type: ALTERTABLE_ADDCOLS +POSTHOOK: Input: default@casesensitive +POSTHOOK: Output: default@casesensitive +PREHOOK: query: describe formatted caseSensitive +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@casesensitive +POSTHOOK: query: describe formatted caseSensitive +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@casesensitive +# col_name data_type comment +a int +i int + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: EXTERNAL_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\"}} + EXTERNAL TRUE + TRANSLATED_TO_EXTERNAL TRUE + bucketing_version 2 + external.table.purge TRUE +#### A masked pattern was here #### + numFiles 0 + numRows 0 + rawDataSize 0 + some one + totalSize 0 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 diff --git a/ql/src/test/results/clientpositive/llap/unset_table_view_property.q.out b/ql/src/test/results/clientpositive/llap/unset_table_view_property.q.out index 5d140d68f946..ecc6601788e7 100644 --- a/ql/src/test/results/clientpositive/llap/unset_table_view_property.q.out +++ b/ql/src/test/results/clientpositive/llap/unset_table_view_property.q.out @@ -223,7 +223,7 @@ POSTHOOK: type: CREATEVIEW POSTHOOK: Input: default@src POSTHOOK: Output: database:vt POSTHOOK: Output: vt@testView -POSTHOOK: Lineage: testView.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: testview.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: ALTER VIEW vt.testView SET TBLPROPERTIES ('propA'='100', 'propB'='200') PREHOOK: type: ALTERVIEW_PROPERTIES PREHOOK: Input: vt@testview diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java index d544bf30ed80..f272b42c0ce4 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java @@ -128,6 +128,7 @@ import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.parseDbName; import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.prependCatalogToDbName; import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.prependNotNullCatToDbName; +import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier; /** * Default handler for all Hive Metastore methods. Implements methods defined in hive_metastore.thrift. @@ -2276,6 +2277,9 @@ private void create_table_core(final RawStore ms, final CreateTableRequest req) + " already exists"); } + tbl.setDbName(normalizeIdentifier(tbl.getDbName())); + tbl.setTableName(normalizeIdentifier(tbl.getTableName())); + if (transformer != null) { tbl = transformer.transformCreateTable(tbl, processorCapabilities, processorId); } @@ -3789,7 +3793,7 @@ private List getTableObjectsInternal(String catName, String dbName, if (distinctTableNames.size() > tableBatchSize) { List lowercaseTableNames = new ArrayList<>(); for (String tableName : tableNames) { - lowercaseTableNames.add(org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier(tableName)); + lowercaseTableNames.add(normalizeIdentifier(tableName)); } distinctTableNames = new ArrayList<>(new HashSet<>(lowercaseTableNames)); } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java index fc413504ec1e..5cb895c80fbd 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java @@ -106,8 +106,8 @@ public void alterTable(RawStore msdb, Warehouse wh, String catName, String dbnam IHMSHandler handler, String writeIdList) throws InvalidOperationException, MetaException { catName = normalizeIdentifier(catName); - name = name.toLowerCase(); - dbname = dbname.toLowerCase(); + name = normalizeIdentifier(name); + dbname = normalizeIdentifier(dbname); final boolean cascade; final boolean replDataLocationChanged;