15 ql/src/test/results/clientpositive/spark/union28.q.out
View file
@@ -37,9 +37,6 @@ STAGE DEPENDENCIES:
Stage-2 depends on stages: Stage-1
Stage-0 depends on stages: Stage-2
Stage-3 depends on stages: Stage-0
Stage-4 depends on stages: Stage-0
Stage-5 depends on stages: Stage-0
Stage-6 depends on stages: Stage-0

STAGE PLANS:
Stage: Stage-1
@@ -174,15 +171,6 @@ STAGE PLANS:
Stage: Stage-3
Stats-Aggr Operator

Stage: Stage-4
Stats-Aggr Operator

Stage: Stage-5
Stats-Aggr Operator

Stage: Stage-6
Stats-Aggr Operator

PREHOOK: query: insert overwrite table union_subq_union
select * from (
select key, value from src
@@ -198,9 +186,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@union_subq_union
[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
POSTHOOK: query: insert overwrite table union_subq_union
select * from (
select key, value from src
15 ql/src/test/results/clientpositive/spark/union29.q.out
View file
@@ -37,9 +37,6 @@ STAGE DEPENDENCIES:
Stage-2 depends on stages: Stage-1
Stage-0 depends on stages: Stage-2
Stage-3 depends on stages: Stage-0
Stage-4 depends on stages: Stage-0
Stage-5 depends on stages: Stage-0
Stage-6 depends on stages: Stage-0

STAGE PLANS:
Stage: Stage-1
@@ -124,15 +121,6 @@ STAGE PLANS:
Stage: Stage-3
Stats-Aggr Operator

Stage: Stage-4
Stats-Aggr Operator

Stage: Stage-5
Stats-Aggr Operator

Stage: Stage-6
Stats-Aggr Operator

PREHOOK: query: insert overwrite table union_subq_union
select * from (
select key, value from src
@@ -148,9 +136,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@union_subq_union
[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
POSTHOOK: query: insert overwrite table union_subq_union
select * from (
select key, value from src
4 ql/src/test/results/clientpositive/spark/union3.q.out
View file
@@ -239,10 +239,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@union_out
[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
POSTHOOK: query: insert overwrite table union_out
SELECT *
FROM (
20 ql/src/test/results/clientpositive/spark/union30.q.out
View file
@@ -51,10 +51,6 @@ STAGE DEPENDENCIES:
Stage-2 depends on stages: Stage-1
Stage-0 depends on stages: Stage-2
Stage-3 depends on stages: Stage-0
Stage-4 depends on stages: Stage-0
Stage-5 depends on stages: Stage-0
Stage-6 depends on stages: Stage-0
Stage-7 depends on stages: Stage-0

STAGE PLANS:
Stage: Stage-1
@@ -215,18 +211,6 @@ STAGE PLANS:
Stage: Stage-3
Stats-Aggr Operator

Stage: Stage-4
Stats-Aggr Operator

Stage: Stage-5
Stats-Aggr Operator

Stage: Stage-6
Stats-Aggr Operator

Stage: Stage-7
Stats-Aggr Operator

PREHOOK: query: insert overwrite table union_subq_union
select * from (

@@ -249,10 +233,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@union_subq_union
[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
POSTHOOK: query: insert overwrite table union_subq_union
select * from (

20 ql/src/test/results/clientpositive/spark/union33.q.out
View file
@@ -37,8 +37,6 @@ STAGE DEPENDENCIES:
Stage-2 depends on stages: Stage-1
Stage-0 depends on stages: Stage-2
Stage-3 depends on stages: Stage-0
Stage-4 depends on stages: Stage-0
Stage-5 depends on stages: Stage-0

STAGE PLANS:
Stage: Stage-1
@@ -142,12 +140,6 @@ STAGE PLANS:
Stage: Stage-3
Stats-Aggr Operator

Stage: Stage-4
Stats-Aggr Operator

Stage: Stage-5
Stats-Aggr Operator

PREHOOK: query: INSERT OVERWRITE TABLE test_src
SELECT key, value FROM (
SELECT key, value FROM src
@@ -160,8 +152,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@test_src
[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
POSTHOOK: query: INSERT OVERWRITE TABLE test_src
SELECT key, value FROM (
SELECT key, value FROM src
@@ -207,8 +197,6 @@ STAGE DEPENDENCIES:
Stage-2 depends on stages: Stage-1
Stage-0 depends on stages: Stage-2
Stage-3 depends on stages: Stage-0
Stage-4 depends on stages: Stage-0
Stage-5 depends on stages: Stage-0

STAGE PLANS:
Stage: Stage-1
@@ -312,12 +300,6 @@ STAGE PLANS:
Stage: Stage-3
Stats-Aggr Operator

Stage: Stage-4
Stats-Aggr Operator

Stage: Stage-5
Stats-Aggr Operator

PREHOOK: query: INSERT OVERWRITE TABLE test_src
SELECT key, value FROM (
SELECT key, COUNT(*) AS value FROM src
@@ -330,8 +312,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@test_src
[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
POSTHOOK: query: INSERT OVERWRITE TABLE test_src
SELECT key, value FROM (
SELECT key, COUNT(*) AS value FROM src
10 ql/src/test/results/clientpositive/spark/union4.q.out
View file
@@ -29,8 +29,6 @@ STAGE DEPENDENCIES:
Stage-2 depends on stages: Stage-1
Stage-0 depends on stages: Stage-2
Stage-3 depends on stages: Stage-0
Stage-4 depends on stages: Stage-0
Stage-5 depends on stages: Stage-0

STAGE PLANS:
Stage: Stage-1
@@ -130,12 +128,6 @@ STAGE PLANS:
Stage: Stage-3
Stats-Aggr Operator

Stage: Stage-4
Stats-Aggr Operator

Stage: Stage-5
Stats-Aggr Operator

PREHOOK: query: insert overwrite table tmptable
select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1
UNION ALL
@@ -144,8 +136,6 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@tmptable
[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
POSTHOOK: query: insert overwrite table tmptable
select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1
UNION ALL
10 ql/src/test/results/clientpositive/spark/union6.q.out
View file
@@ -29,8 +29,6 @@ STAGE DEPENDENCIES:
Stage-2 depends on stages: Stage-1
Stage-0 depends on stages: Stage-2
Stage-3 depends on stages: Stage-0
Stage-4 depends on stages: Stage-0
Stage-5 depends on stages: Stage-0

STAGE PLANS:
Stage: Stage-1
@@ -111,12 +109,6 @@ STAGE PLANS:
Stage: Stage-3
Stats-Aggr Operator

Stage: Stage-4
Stats-Aggr Operator

Stage: Stage-5
Stats-Aggr Operator

PREHOOK: query: insert overwrite table tmptable
select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1
UNION ALL
@@ -126,8 +118,6 @@ PREHOOK: Input: default@src
PREHOOK: Input: default@src1
PREHOOK: Output: default@tmptable
[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to
POSTHOOK: query: insert overwrite table tmptable
select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1
UNION ALL
49 ql/src/test/results/clientpositive/spark/union_remove_1.q.out
View file
@@ -70,7 +70,8 @@ STAGE PLANS:
Spark
Edges:
Reducer 2 <- Map 1 (GROUP)
Reducer 4 <- Map 3 (GROUP)
Reducer 5 <- Map 4 (GROUP)
Union 3 <- Reducer 2 (NONE), Reducer 5 (NONE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -94,7 +95,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
value expressions: _col1 (type: bigint)
Map 3
Map 4
Map Operator Tree:
TableScan
alias: inputtbl1
@@ -122,39 +123,41 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Reducer 4
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
File Output Operator
compressed: false
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Reducer 5
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
File Output Operator
compressed: false
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Union 3
Vertex: Union 3

Stage: Stage-2
Dependency Collection
88 ql/src/test/results/clientpositive/spark/union_remove_10.q.out
View file
@@ -78,15 +78,20 @@ select * FROM (
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
Stage-2 depends on stages: Stage-1
Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5
Stage-4
Stage-2 depends on stages: Stage-4, Stage-3, Stage-6
Stage-0 depends on stages: Stage-2
Stage-3
Stage-5
Stage-6 depends on stages: Stage-5

STAGE PLANS:
Stage: Stage-1
Spark
Edges:
Reducer 2 <- Map 1 (GROUP)
Union 3 <- Map 5 (NONE), Reducer 2 (NONE)
Union 3 <- Map 4 (NONE), Map 5 (NONE), Reducer 2 (NONE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -114,26 +119,9 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: inputtbl1
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: string), UDFToLong(1) (type: bigint)
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
table:
input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
name: default.outputtbl1
Map 5
Map Operator Tree:
TableScan
alias: inputtbl1
Select Operator
expressions: key (type: string), UDFToLong(2) (type: bigint)
outputColumnNames: _col0, _col1
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
@@ -144,6 +132,26 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
name: default.outputtbl1
Map 5
Map Operator Tree:
TableScan
alias: inputtbl1
Select Operator
expressions: key (type: string), UDFToLong(2) (type: bigint)
outputColumnNames: _col0, _col1
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
File Output Operator
compressed: false
table:
input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
name: default.outputtbl1
Reducer 2
Reduce Operator Tree:
Group By Operator
@@ -157,16 +165,28 @@ STAGE PLANS:
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
File Output Operator
compressed: false
table:
input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
name: default.outputtbl1
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
File Output Operator
compressed: false
table:
input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
name: default.outputtbl1
Union 3
Vertex: Union 3

Stage: Stage-7
Conditional Operator

Stage: Stage-4
Move Operator
files:
hdfs directory: true
#### A masked pattern was here ####

Stage: Stage-2
Dependency Collection

@@ -180,6 +200,22 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
name: default.outputtbl1

Stage: Stage-3
Merge Work
merge level: block
input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat

Stage: Stage-5
Merge Work
merge level: block
input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat

Stage: Stage-6
Move Operator
files:
hdfs directory: true
#### A masked pattern was here ####

PREHOOK: query: insert overwrite table outputTbl1
SELECT * FROM
(
32 ql/src/test/results/clientpositive/spark/union_remove_11.q.out
View file
@@ -78,8 +78,13 @@ select * FROM (
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
Stage-2 depends on stages: Stage-1
Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5
Stage-4
Stage-2 depends on stages: Stage-4, Stage-3, Stage-6
Stage-0 depends on stages: Stage-2
Stage-3
Stage-5
Stage-6 depends on stages: Stage-5

STAGE PLANS:
Stage: Stage-1
@@ -148,6 +153,15 @@ STAGE PLANS:
Union 2
Vertex: Union 2

Stage: Stage-7
Conditional Operator

Stage: Stage-4
Move Operator
files:
hdfs directory: true
#### A masked pattern was here ####

Stage: Stage-2
Dependency Collection

@@ -161,6 +175,22 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
name: default.outputtbl1

Stage: Stage-3
Merge Work
merge level: block
input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat

Stage: Stage-5
Merge Work
merge level: block
input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat

Stage: Stage-6
Move Operator
files:
hdfs directory: true
#### A masked pattern was here ####

PREHOOK: query: insert overwrite table outputTbl1
SELECT * FROM
(
56 ql/src/test/results/clientpositive/spark/union_remove_15.q.out
View file
@@ -76,7 +76,9 @@ STAGE PLANS:
Spark
Edges:
Reducer 2 <- Map 1 (GROUP)
Reducer 4 <- Map 3 (GROUP)
Reducer 4 <- Union 3 (GROUP SORT)
Reducer 6 <- Map 5 (GROUP)
Union 3 <- Reducer 2 (NONE), Reducer 6 (NONE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -100,7 +102,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
value expressions: _col1 (type: bigint)
Map 3
Map 5
Map Operator Tree:
TableScan
alias: inputtbl1
@@ -128,39 +130,49 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint), '2' (type: string)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
table:
input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
name: default.outputtbl1
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string)
outputColumnNames: _col0, _col1, _col2
Reduce Output Operator
key expressions: _col2 (type: string)
sort order: +
Map-reduce partition columns: _col2 (type: string)
value expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string)
Reducer 4
Reduce Operator Tree:
Extract
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
table:
input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
name: default.outputtbl1
Reducer 6
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint), '1' (type: string)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
table:
input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
name: default.outputtbl1
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string)
outputColumnNames: _col0, _col1, _col2
Reduce Output Operator
key expressions: _col2 (type: string)
sort order: +
Map-reduce partition columns: _col2 (type: string)
value expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string)
Union 3
Vertex: Union 3

Stage: Stage-2
Dependency Collection
88 ql/src/test/results/clientpositive/spark/union_remove_16.q.out
View file
@@ -66,15 +66,22 @@ FROM (
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
Stage-2 depends on stages: Stage-1
Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5
Stage-4
Stage-2 depends on stages: Stage-4, Stage-3, Stage-6
Stage-0 depends on stages: Stage-2
Stage-3
Stage-5
Stage-6 depends on stages: Stage-5

STAGE PLANS:
Stage: Stage-1
Spark
Edges:
Reducer 2 <- Map 1 (GROUP)
Reducer 4 <- Map 3 (GROUP)
Reducer 4 <- Union 3 (GROUP SORT)
Reducer 6 <- Map 5 (GROUP)
Union 3 <- Reducer 2 (NONE), Reducer 6 (NONE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -98,7 +105,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
value expressions: _col1 (type: bigint)
Map 3
Map 5
Map Operator Tree:
TableScan
alias: inputtbl1
@@ -126,39 +133,58 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint), '2' (type: string)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
table:
input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
name: default.outputtbl1
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string)
outputColumnNames: _col0, _col1, _col2
Reduce Output Operator
key expressions: _col2 (type: string)
sort order: +
Map-reduce partition columns: _col2 (type: string)
value expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string)
Reducer 4
Reduce Operator Tree:
Extract
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
table:
input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
name: default.outputtbl1
Reducer 6
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint), '1' (type: string)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
table:
input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
name: default.outputtbl1
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string)
outputColumnNames: _col0, _col1, _col2
Reduce Output Operator
key expressions: _col2 (type: string)
sort order: +
Map-reduce partition columns: _col2 (type: string)
value expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string)
Union 3
Vertex: Union 3

Stage: Stage-7
Conditional Operator

Stage: Stage-4
Move Operator
files:
hdfs directory: true
#### A masked pattern was here ####

Stage: Stage-2
Dependency Collection
@@ -175,6 +201,22 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
name: default.outputtbl1

Stage: Stage-3
Merge Work
merge level: block
input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat

Stage: Stage-5
Merge Work
merge level: block
input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat

Stage: Stage-6
Move Operator
files:
hdfs directory: true
#### A masked pattern was here ####

PREHOOK: query: insert overwrite table outputTbl1 partition (ds)
SELECT *
FROM (
56 ql/src/test/results/clientpositive/spark/union_remove_18.q.out
View file
@@ -74,7 +74,9 @@ STAGE PLANS:
Spark
Edges:
Reducer 2 <- Map 1 (GROUP)
Reducer 4 <- Map 3 (GROUP)
Reducer 4 <- Union 3 (GROUP SORT)
Reducer 6 <- Map 5 (GROUP)
Union 3 <- Reducer 2 (NONE), Reducer 6 (NONE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -98,7 +100,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
value expressions: _col2 (type: bigint)
Map 3
Map 5
Map Operator Tree:
TableScan
alias: inputtbl1
@@ -126,39 +128,49 @@ STAGE PLANS:
keys: KEY._col0 (type: string), KEY._col1 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col2 (type: bigint), _col1 (type: string)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string)
outputColumnNames: _col0, _col1, _col2
Reduce Output Operator
key expressions: _col2 (type: string)
sort order: +
Map-reduce partition columns: _col2 (type: string)
value expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string)
Reducer 4
Reduce Operator Tree:
Extract
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Reducer 6
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
keys: KEY._col0 (type: string), KEY._col1 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col2 (type: bigint), _col1 (type: string)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string)
outputColumnNames: _col0, _col1, _col2
Reduce Output Operator
key expressions: _col2 (type: string)
sort order: +
Map-reduce partition columns: _col2 (type: string)
value expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string)
Union 3
Vertex: Union 3

Stage: Stage-2
Dependency Collection
175 ql/src/test/results/clientpositive/spark/union_remove_19.q.out
View file
@@ -74,7 +74,8 @@ STAGE PLANS:
Spark
Edges:
Reducer 2 <- Map 1 (GROUP)
Reducer 4 <- Map 3 (GROUP)
Reducer 5 <- Map 4 (GROUP)
Union 3 <- Reducer 2 (NONE), Reducer 5 (NONE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -98,7 +99,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
value expressions: _col1 (type: bigint)
Map 3
Map 4
Map Operator Tree:
TableScan
alias: inputtbl1
@@ -126,39 +127,41 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Reducer 4
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
File Output Operator
compressed: false
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Reducer 5
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
File Output Operator
compressed: false
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Union 3
Vertex: Union 3

Stage: Stage-2
Dependency Collection
@@ -279,7 +282,8 @@ STAGE PLANS:
Spark
Edges:
Reducer 2 <- Map 1 (GROUP)
Reducer 4 <- Map 3 (GROUP)
Reducer 5 <- Map 4 (GROUP)
Union 3 <- Reducer 2 (NONE), Reducer 5 (NONE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -306,7 +310,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 15 Data size: 15 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint)
Map 3
Map 4
Map Operator Tree:
TableScan
alias: inputtbl1
@@ -337,39 +341,41 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
Statistics: Num rows: 7 Data size: 7 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
Statistics: Num rows: 7 Data size: 7 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 7 Data size: 7 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Reducer 4
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
File Output Operator
compressed: false
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Reducer 5
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
Statistics: Num rows: 7 Data size: 7 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
Statistics: Num rows: 7 Data size: 7 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 7 Data size: 7 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
File Output Operator
compressed: false
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Union 3
Vertex: Union 3

Stage: Stage-2
Dependency Collection
@@ -452,7 +458,8 @@ STAGE PLANS:
Spark
Edges:
Reducer 2 <- Map 1 (GROUP)
Reducer 4 <- Map 3 (GROUP)
Reducer 5 <- Map 4 (GROUP)
Union 3 <- Reducer 2 (NONE), Reducer 5 (NONE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -476,7 +483,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
value expressions: _col1 (type: bigint)
Map 3
Map 4
Map Operator Tree:
TableScan
alias: inputtbl1
@@ -504,53 +511,51 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: (_col0 + _col0) (type: double), _col1 (type: bigint)
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Filter Operator
predicate: (_col0 >= 7.0) (type: boolean)
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: _col0 (type: double), _col1 (type: bigint)
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Reducer 4
Select Operator
expressions: (_col0 + _col0) (type: double), _col1 (type: bigint)
outputColumnNames: _col0, _col1
Filter Operator
predicate: (_col0 >= 7.0) (type: boolean)
Select Operator
expressions: _col0 (type: double), _col1 (type: bigint)
outputColumnNames: _col0, _col1
File Output Operator
compressed: false
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Reducer 5
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: (_col0 + _col0) (type: double), _col1 (type: bigint)
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Filter Operator
predicate: (_col0 >= 7.0) (type: boolean)
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: _col0 (type: double), _col1 (type: bigint)
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Select Operator
expressions: (_col0 + _col0) (type: double), _col1 (type: bigint)
outputColumnNames: _col0, _col1
Filter Operator
predicate: (_col0 >= 7.0) (type: boolean)
Select Operator
expressions: _col0 (type: double), _col1 (type: bigint)
outputColumnNames: _col0, _col1
File Output Operator
compressed: false
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Union 3
Vertex: Union 3

Stage: Stage-2
Dependency Collection
71 ql/src/test/results/clientpositive/spark/union_remove_2.q.out
View file
@@ -75,27 +75,28 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
Reducer 3 <- Map 2 (GROUP)
Reducer 4 <- Map 3 (GROUP)
Union 2 <- Map 1 (NONE), Map 5 (NONE), Reducer 4 (NONE)
#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
TableScan
alias: inputtbl1
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: string), UDFToLong(2) (type: bigint)
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Map 2
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
File Output Operator
compressed: false
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Map 3
Map Operator Tree:
TableScan
alias: inputtbl1
@@ -116,43 +117,45 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
value expressions: _col1 (type: bigint)
Map 4
Map 5
Map Operator Tree:
TableScan
alias: inputtbl1
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: string), UDFToLong(1) (type: bigint)
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Reducer 3
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
File Output Operator
compressed: false
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Reducer 4
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
File Output Operator
compressed: false
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Union 2
Vertex: Union 2

Stage: Stage-2
Dependency Collection
53 ql/src/test/results/clientpositive/spark/union_remove_20.q.out
View file
@@ -72,7 +72,8 @@ STAGE PLANS:
Spark
Edges:
Reducer 2 <- Map 1 (GROUP)
Reducer 4 <- Map 3 (GROUP)
Reducer 5 <- Map 4 (GROUP)
Union 3 <- Reducer 2 (NONE), Reducer 5 (NONE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -96,7 +97,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
value expressions: _col1 (type: bigint)
Map 3
Map 4
Map Operator Tree:
TableScan
alias: inputtbl1
@@ -124,39 +125,41 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: _col1 (type: bigint), _col0 (type: string)
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Reducer 4
Select Operator
expressions: _col1 (type: bigint), _col0 (type: string)
outputColumnNames: _col0, _col1
File Output Operator
compressed: false
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Reducer 5
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: _col1 (type: bigint), _col0 (type: string)
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Select Operator
expressions: _col1 (type: bigint), _col0 (type: string)
outputColumnNames: _col0, _col1
File Output Operator
compressed: false
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Union 3
Vertex: Union 3

Stage: Stage-2
Dependency Collection
49 ql/src/test/results/clientpositive/spark/union_remove_21.q.out
View file
@@ -72,7 +72,8 @@ STAGE PLANS:
Spark
Edges:
Reducer 2 <- Map 1 (GROUP)
Reducer 4 <- Map 3 (GROUP)
Reducer 5 <- Map 4 (GROUP)
Union 3 <- Reducer 2 (NONE), Reducer 5 (NONE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -96,7 +97,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
value expressions: _col1 (type: bigint)
Map 3
Map 4
Map Operator Tree:
TableScan
alias: inputtbl1
@@ -124,39 +125,41 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: _col0 (type: string)
outputColumnNames: _col0
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Reducer 4
Select Operator
expressions: _col0 (type: string)
outputColumnNames: _col0
File Output Operator
compressed: false
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Reducer 5
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: _col0 (type: string)
outputColumnNames: _col0
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Select Operator
expressions: _col0 (type: string)
outputColumnNames: _col0
File Output Operator
compressed: false
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Union 3
Vertex: Union 3

Stage: Stage-2
Dependency Collection
49 ql/src/test/results/clientpositive/spark/union_remove_24.q.out
View file
@@ -68,7 +68,8 @@ STAGE PLANS:
Spark
Edges:
Reducer 2 <- Map 1 (GROUP)
Reducer 4 <- Map 3 (GROUP)
Reducer 5 <- Map 4 (GROUP)
Union 3 <- Reducer 2 (NONE), Reducer 5 (NONE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -92,7 +93,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
value expressions: _col1 (type: bigint)
Map 3
Map 4
Map Operator Tree:
TableScan
alias: inputtbl1
@@ -120,39 +121,41 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: UDFToDouble(UDFToLong(_col0)) (type: double), _col1 (type: bigint)
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Reducer 4
Select Operator
expressions: _col0 (type: double), _col1 (type: bigint)
outputColumnNames: _col0, _col1
File Output Operator
compressed: false
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Reducer 5
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: UDFToDouble(_col0) (type: double), _col1 (type: bigint)
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Select Operator
expressions: _col0 (type: double), _col1 (type: bigint)
outputColumnNames: _col0, _col1
File Output Operator
compressed: false
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Union 3
Vertex: Union 3

Stage: Stage-2
Dependency Collection
157 ql/src/test/results/clientpositive/spark/union_remove_25.q.out
View file
@@ -86,7 +86,8 @@ STAGE PLANS:
Spark
Edges:
Reducer 2 <- Map 1 (GROUP)
Reducer 4 <- Map 3 (GROUP)
Reducer 5 <- Map 4 (GROUP)
Union 3 <- Reducer 2 (NONE), Reducer 5 (NONE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -110,7 +111,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
value expressions: _col1 (type: bigint)
Map 3
Map 4
Map Operator Tree:
TableScan
alias: inputtbl1
@@ -138,39 +139,41 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Reducer 4
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
File Output Operator
compressed: false
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Reducer 5
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
File Output Operator
compressed: false
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Union 3
Vertex: Union 3

Stage: Stage-2
Dependency Collection
@@ -298,7 +301,9 @@ STAGE PLANS:
Spark
Edges:
Reducer 2 <- Map 1 (GROUP)
Reducer 4 <- Map 3 (GROUP)
Reducer 4 <- Union 3 (GROUP SORT)
Reducer 6 <- Map 5 (GROUP)
Union 3 <- Reducer 2 (NONE), Reducer 6 (NONE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -317,7 +322,7 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string)
Map 3
Map 5
Map Operator Tree:
TableScan
alias: srcpart
@@ -338,43 +343,45 @@ STAGE PLANS:
Select Operator
expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 500
Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), _col2 (type: string)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl2
Reduce Output Operator
key expressions: _col2 (type: string)
sort order: +
Map-reduce partition columns: _col2 (type: string)
value expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string)
Reducer 4
Reduce Operator Tree:
Extract
Statistics: Num rows: 116 Data size: 23248 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 116 Data size: 23248 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl2
Reducer 6
Reduce Operator Tree:
Select Operator
expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 500
Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), _col2 (type: string)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl2
Reduce Output Operator
key expressions: _col2 (type: string)
sort order: +
Map-reduce partition columns: _col2 (type: string)
value expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string)
Union 3
Vertex: Union 3

Stage: Stage-2
Dependency Collection
@@ -449,7 +456,7 @@ Protect Mode: None
#### A masked pattern was here ####
Partition Parameters:
COLUMN_STATS_ACCURATE false
numFiles 2
numFiles 1
numRows -1
rawDataSize -1
totalSize 6826
@@ -491,7 +498,9 @@ STAGE PLANS:
Spark
Edges:
Reducer 2 <- Map 1 (GROUP)
Reducer 4 <- Map 3 (GROUP)
Reducer 4 <- Union 3 (GROUP SORT)
Reducer 6 <- Map 5 (GROUP)
Union 3 <- Reducer 2 (NONE), Reducer 6 (NONE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -510,7 +519,7 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string)
Map 3
Map 5
Map Operator Tree:
TableScan
alias: srcpart
@@ -531,43 +540,45 @@ STAGE PLANS:
Select Operator
expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 1000
Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), _col2 (type: string), _col3 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl3
Reduce Output Operator
key expressions: _col2 (type: string), _col3 (type: string)
sort order: ++
Map-reduce partition columns: _col2 (type: string), _col3 (type: string)
value expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string), _col3 (type: string)
Reducer 4
Reduce Operator Tree:
Extract
Statistics: Num rows: 116 Data size: 23248 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 116 Data size: 23248 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl3
Reducer 6
Reduce Operator Tree:
Select Operator
expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 1000
Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), _col2 (type: string), _col3 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl3
Reduce Output Operator
key expressions: _col2 (type: string), _col3 (type: string)
sort order: ++
Map-reduce partition columns: _col2 (type: string), _col3 (type: string)
value expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string), _col3 (type: string)
Union 3
Vertex: Union 3

Stage: Stage-2
Dependency Collection
@@ -648,7 +659,7 @@ Protect Mode: None
#### A masked pattern was here ####
Partition Parameters:
COLUMN_STATS_ACCURATE false
numFiles 2
numFiles 1
numRows -1
rawDataSize -1
totalSize 6812
101 ql/src/test/results/clientpositive/spark/union_remove_4.q.out
View file
@@ -62,15 +62,21 @@ FROM (
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
Stage-2 depends on stages: Stage-1
Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5
Stage-4
Stage-2 depends on stages: Stage-4, Stage-3, Stage-6
Stage-0 depends on stages: Stage-2
Stage-3
Stage-5
Stage-6 depends on stages: Stage-5

STAGE PLANS:
Stage: Stage-1
Spark
Edges:
Reducer 2 <- Map 1 (GROUP)
Reducer 4 <- Map 3 (GROUP)
Reducer 5 <- Map 4 (GROUP)
Union 3 <- Reducer 2 (NONE), Reducer 5 (NONE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -94,7 +100,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
value expressions: _col1 (type: bigint)
Map 3
Map 4
Map Operator Tree:
TableScan
alias: inputtbl1
@@ -122,39 +128,50 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Reducer 4
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
File Output Operator
compressed: false
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Reducer 5
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
File Output Operator
compressed: false
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Union 3
Vertex: Union 3

Stage: Stage-7
Conditional Operator

Stage: Stage-4
Move Operator
files:
hdfs directory: true
#### A masked pattern was here ####

Stage: Stage-2
Dependency Collection
@@ -169,6 +186,42 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1

Stage: Stage-3
Spark
#### A masked pattern was here ####
Vertices:
Merge
Map Operator Tree:
TableScan
File Output Operator
compressed: false
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1

Stage: Stage-5
Spark
#### A masked pattern was here ####
Vertices:
Merge
Map Operator Tree:
TableScan
File Output Operator
compressed: false
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1

Stage: Stage-6
Move Operator
files:
hdfs directory: true
#### A masked pattern was here ####

PREHOOK: query: insert overwrite table outputTbl1
SELECT *
FROM (
123 ql/src/test/results/clientpositive/spark/union_remove_5.q.out
View file
@@ -70,34 +70,40 @@ FROM (
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
Stage-2 depends on stages: Stage-1
Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5
Stage-4
Stage-2 depends on stages: Stage-4, Stage-3, Stage-6
Stage-0 depends on stages: Stage-2
Stage-3
Stage-5
Stage-6 depends on stages: Stage-5

STAGE PLANS:
Stage: Stage-1
Spark
Edges:
Reducer 3 <- Map 2 (GROUP)
Reducer 4 <- Map 3 (GROUP)
Union 2 <- Map 1 (NONE), Map 5 (NONE), Reducer 4 (NONE)
#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
TableScan
alias: inputtbl1
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: string), UDFToLong(2) (type: bigint)
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Map 2
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
File Output Operator
compressed: false
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Map 3
Map Operator Tree:
TableScan
alias: inputtbl1
@@ -118,43 +124,54 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
value expressions: _col1 (type: bigint)
Map 4
Map 5
Map Operator Tree:
TableScan
alias: inputtbl1
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: string), UDFToLong(1) (type: bigint)
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Reducer 3
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
File Output Operator
compressed: false
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Reducer 4
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
File Output Operator
compressed: false
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
File Output Operator
compressed: false
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
Union 2
Vertex: Union 2

Stage: Stage-7
Conditional Operator

Stage: Stage-4
Move Operator
files:
hdfs directory: true
#### A masked pattern was here ####

Stage: Stage-2
Dependency Collection
@@ -169,6 +186,42 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1

Stage: Stage-3
Spark
#### A masked pattern was here ####
Vertices:
Merge
Map Operator Tree:
TableScan
File Output Operator
compressed: false
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1

Stage: Stage-5
Spark
#### A masked pattern was here ####
Vertices:
Merge
Map Operator Tree:
TableScan
File Output Operator
compressed: false
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1

Stage: Stage-6
Move Operator
files:
hdfs directory: true
#### A masked pattern was here ####

PREHOOK: query: insert overwrite table outputTbl1
SELECT *
FROM (