HIVE-2086. Add test coverage for external table data loss issue (Jona…

…than Natkins via cws) git-svn-id: https://svn.apache.org/repos/asf/hive/trunk@1149331 13f79535-47bb-0310-9956-ffa450edef68
zshao · Jul 21, 2011 · 27de599 · 27de599
1 parent 51c1a12
commit 27de599
Show file tree

Hide file tree

Showing 5 changed files with 159 additions and 16 deletions.
diff --git a/build-common.xml b/build-common.xml
@@ -458,6 +458,7 @@
       <sysproperty key="hadoop.log.dir" value="${test.log.dir}"/>
       <sysproperty key="test.silent" value="${test.silent}"/>
       <sysproperty key="test.tmp.dir" value="${build.dir}/tmp"/>
+      <sysproperty key="test.src.data.dir" value="${test.src.data.dir}"/>
       <sysproperty key="test.warehouse.dir" value="${test.warehouse.dir}"/>
       <sysproperty key="mapred.job.tracker" value="${mapred.job.tracker}"/>
       <sysproperty key="fs.default.name" value="${fs.default.name}"/>

diff --git a/data/files/ext_test/test.dat b/data/files/ext_test/test.dat
@@ -0,0 +1,6 @@
+1
+2
+3
+4
+5
+6
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java b/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -938,6 +938,7 @@ public int checkCliDriverResults(String tname) throws Exception {
         "-I", "CreateTime",
         "-I", "LastAccessTime",
         "-I", "Location",
+        "-I", "LOCATION '",
         "-I", "transient_lastDdlTime",
         "-I", "last_modified_",
         "-I", "java.lang.RuntimeException",

diff --git a/ql/src/test/queries/clientpositive/create_like.q b/ql/src/test/queries/clientpositive/create_like.q
@@ -21,3 +21,15 @@ INSERT OVERWRITE TABLE table2 SELECT key, value FROM src WHERE key = 100;
 SELECT * FROM table1;
 SELECT * FROM table2;
 
+CREATE EXTERNAL TABLE table4 (a INT) LOCATION '${system:test.src.data.dir}/files/ext_test';
+CREATE EXTERNAL TABLE table5 LIKE table4 LOCATION '${system:test.src.data.dir}/files/ext_test';
+
+SELECT * FROM table4;
+SELECT * FROM table5;
+
+DROP TABLE table5;
+SELECT * FROM table4;
+DROP TABLE table4;
+
+CREATE EXTERNAL TABLE table4 (a INT) LOCATION '${system:test.src.data.dir}/files/ext_test';
+SELECT * FROM table4;
diff --git a/ql/src/test/results/clientpositive/create_like.q.out b/ql/src/test/results/clientpositive/create_like.q.out
@@ -14,15 +14,15 @@ b                   	string              	None
 
 # Detailed Table Information	 	 
 Database:           	default             	 
-Owner:              	charleschen         	 
-CreateTime:         	Sat Jul 16 15:44:24 PDT 2011	 
+Owner:              	natty
+CreateTime:         	Mon Jul 18 21:36:36 PDT 2011
 LastAccessTime:     	UNKNOWN             	 
 Protect Mode:       	None                	 
 Retention:          	0                   	 
-Location:           	pfile:/Users/charleschen/hive-trunk2/build/ql/test/data/warehouse/table1	 
+Location:           	pfile:/home/natty/apache/hive/build/ql/test/data/warehouse/table1
 Table Type:         	MANAGED_TABLE       	 
 Table Parameters:	 	 
-	transient_lastDdlTime	1310856264          
+	transient_lastDdlTime	1311050196
 
 # Storage Information	 	 
 SerDe Library:      	org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe	 
@@ -50,15 +50,15 @@ b                   	string              	None
 
 # Detailed Table Information	 	 
 Database:           	default             	 
-Owner:              	charleschen         	 
-CreateTime:         	Sat Jul 16 15:44:24 PDT 2011	 
+Owner:              	natty
+CreateTime:         	Mon Jul 18 21:36:36 PDT 2011
 LastAccessTime:     	UNKNOWN             	 
 Protect Mode:       	None                	 
 Retention:          	0                   	 
-Location:           	pfile:/Users/charleschen/hive-trunk2/build/ql/test/data/warehouse/table2	 
+Location:           	pfile:/home/natty/apache/hive/build/ql/test/data/warehouse/table2
 Table Type:         	MANAGED_TABLE       	 
 Table Parameters:	 	 
-	transient_lastDdlTime	1310856264          
+	transient_lastDdlTime	1311050196
 
 # Storage Information	 	 
 SerDe Library:      	org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe	 
@@ -94,16 +94,16 @@ b                   	string              	None
 
 # Detailed Table Information	 	 
 Database:           	default             	 
-Owner:              	charleschen         	 
-CreateTime:         	Sat Jul 16 15:44:24 PDT 2011	 
+Owner:              	natty
+CreateTime:         	Mon Jul 18 21:36:37 PDT 2011
 LastAccessTime:     	UNKNOWN             	 
 Protect Mode:       	None                	 
 Retention:          	0                   	 
-Location:           	pfile:/Users/charleschen/hive-trunk2/build/ql/test/data/warehouse/table3	 
+Location:           	pfile:/home/natty/apache/hive/build/ql/test/data/warehouse/table3
 Table Type:         	EXTERNAL_TABLE      	 
 Table Parameters:	 	 
 	EXTERNAL            	TRUE                
-	transient_lastDdlTime	1310856264          
+	transient_lastDdlTime	1311050197
 
 # Storage Information	 	 
 SerDe Library:      	org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe	 
@@ -140,11 +140,11 @@ POSTHOOK: Lineage: table2.b SIMPLE [(src)src.FieldSchema(name:value, type:string
 PREHOOK: query: SELECT * FROM table1
 PREHOOK: type: QUERY
 PREHOOK: Input: default@table1
-PREHOOK: Output: file:/var/folders/fe/fefl-ow3ElaEd1ns0G7jB0uewEQ/-Tmp-/charleschen/hive_2011-07-16_15-44-45_490_2335283351138014900/-mr-10000
+PREHOOK: Output: file:/tmp/natty/hive_2011-07-18_21-36-57_232_3888885319138532825/-mr-10000
 POSTHOOK: query: SELECT * FROM table1
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@table1
-POSTHOOK: Output: file:/var/folders/fe/fefl-ow3ElaEd1ns0G7jB0uewEQ/-Tmp-/charleschen/hive_2011-07-16_15-44-45_490_2335283351138014900/-mr-10000
+POSTHOOK: Output: file:/tmp/natty/hive_2011-07-18_21-36-57_232_3888885319138532825/-mr-10000
 POSTHOOK: Lineage: table1.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
 POSTHOOK: Lineage: table1.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
 POSTHOOK: Lineage: table2.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
@@ -153,14 +153,137 @@ POSTHOOK: Lineage: table2.b SIMPLE [(src)src.FieldSchema(name:value, type:string
 PREHOOK: query: SELECT * FROM table2
 PREHOOK: type: QUERY
 PREHOOK: Input: default@table2
-PREHOOK: Output: file:/var/folders/fe/fefl-ow3ElaEd1ns0G7jB0uewEQ/-Tmp-/charleschen/hive_2011-07-16_15-44-45_655_7605576315633944704/-mr-10000
+PREHOOK: Output: file:/tmp/natty/hive_2011-07-18_21-36-57_724_8379773793855049431/-mr-10000
 POSTHOOK: query: SELECT * FROM table2
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@table2
-POSTHOOK: Output: file:/var/folders/fe/fefl-ow3ElaEd1ns0G7jB0uewEQ/-Tmp-/charleschen/hive_2011-07-16_15-44-45_655_7605576315633944704/-mr-10000
+POSTHOOK: Output: file:/tmp/natty/hive_2011-07-18_21-36-57_724_8379773793855049431/-mr-10000
 POSTHOOK: Lineage: table1.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
 POSTHOOK: Lineage: table1.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
 POSTHOOK: Lineage: table2.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
 POSTHOOK: Lineage: table2.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
 100	val_100
 100	val_100
+PREHOOK: query: CREATE EXTERNAL TABLE table4 (a INT) LOCATION '/home/natty/apache/hive/data/files/ext_test'
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE EXTERNAL TABLE table4 (a INT) LOCATION '/home/natty/apache/hive/data/files/ext_test'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@table4
+POSTHOOK: Lineage: table1.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table1.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: table2.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table2.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: CREATE EXTERNAL TABLE table5 LIKE table4 LOCATION '/home/natty/apache/hive/data/files/ext_test'
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE EXTERNAL TABLE table5 LIKE table4 LOCATION '/home/natty/apache/hive/data/files/ext_test'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@table5
+POSTHOOK: Lineage: table1.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table1.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: table2.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table2.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: SELECT * FROM table4
+PREHOOK: type: QUERY
+PREHOOK: Input: default@table4
+PREHOOK: Output: file:/tmp/natty/hive_2011-07-18_21-36-58_220_5774951150754164269/-mr-10000
+POSTHOOK: query: SELECT * FROM table4
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@table4
+POSTHOOK: Output: file:/tmp/natty/hive_2011-07-18_21-36-58_220_5774951150754164269/-mr-10000
+POSTHOOK: Lineage: table1.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table1.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: table2.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table2.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+1
+2
+3
+4
+5
+6
+PREHOOK: query: SELECT * FROM table5
+PREHOOK: type: QUERY
+PREHOOK: Input: default@table5
+PREHOOK: Output: file:/tmp/natty/hive_2011-07-18_21-36-58_497_977305450352332337/-mr-10000
+POSTHOOK: query: SELECT * FROM table5
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@table5
+POSTHOOK: Output: file:/tmp/natty/hive_2011-07-18_21-36-58_497_977305450352332337/-mr-10000
+POSTHOOK: Lineage: table1.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table1.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: table2.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table2.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+1
+2
+3
+4
+5
+6
+PREHOOK: query: DROP TABLE table5
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@table5
+PREHOOK: Output: default@table5
+POSTHOOK: query: DROP TABLE table5
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@table5
+POSTHOOK: Output: default@table5
+POSTHOOK: Lineage: table1.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table1.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: table2.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table2.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: SELECT * FROM table4
+PREHOOK: type: QUERY
+PREHOOK: Input: default@table4
+PREHOOK: Output: file:/tmp/natty/hive_2011-07-18_21-37-01_834_8386395890966387972/-mr-10000
+POSTHOOK: query: SELECT * FROM table4
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@table4
+POSTHOOK: Output: file:/tmp/natty/hive_2011-07-18_21-37-01_834_8386395890966387972/-mr-10000
+POSTHOOK: Lineage: table1.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table1.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: table2.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table2.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+1
+2
+3
+4
+5
+6
+PREHOOK: query: DROP TABLE table4
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@table4
+PREHOOK: Output: default@table4
+POSTHOOK: query: DROP TABLE table4
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@table4
+POSTHOOK: Output: default@table4
+POSTHOOK: Lineage: table1.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table1.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: table2.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table2.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: CREATE EXTERNAL TABLE table4 (a INT) LOCATION '/home/natty/apache/hive/data/files/ext_test'
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE EXTERNAL TABLE table4 (a INT) LOCATION '/home/natty/apache/hive/data/files/ext_test'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@table4
+POSTHOOK: Lineage: table1.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table1.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: table2.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table2.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: SELECT * FROM table4
+PREHOOK: type: QUERY
+PREHOOK: Input: default@table4
+PREHOOK: Output: file:/tmp/natty/hive_2011-07-18_21-37-02_667_3267843075880450853/-mr-10000
+POSTHOOK: query: SELECT * FROM table4
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@table4
+POSTHOOK: Output: file:/tmp/natty/hive_2011-07-18_21-37-02_667_3267843075880450853/-mr-10000
+POSTHOOK: Lineage: table1.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table1.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: table2.a SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: table2.b SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+1
+2
+3
+4
+5
+6