Skip to content

Commit

Permalink
HIVE-2451: TABLESAMBLE(BUCKET xxx) sometimes doesn't trigger input pr…
Browse files Browse the repository at this point in the history 
…uning (Siying Dong via He Yongqiang)

git-svn-id: https://svn.apache.org/repos/asf/hive/trunk@1172891 13f79535-47bb-0310-9956-ffa450edef68
  • Loading branch information
Yongqiang He committed Sep 19, 2011
1 parent 2558401 commit 7bc466c
Show file tree
Hide file tree
Showing 9 changed files with 335 additions and 560 deletions.
6 changes: 4 additions & 2 deletions ql/src/java/org/apache/hadoop/hive/ql/optimizer/SamplePruner.java
View file
Expand Up @@ -104,7 +104,7 @@ public ParseContext transform(ParseContext pctx) throws SemanticException {
.getOpToSamplePruner());

Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
opRules.put(new RuleRegExp("R1", "(TS%FIL%FIL%)"), getFilterProc());
opRules.put(new RuleRegExp("R1", "(TS%FIL%FIL%|TS%FIL%)"), getFilterProc());

// The dispatcher fires the processor corresponding to the closest matching
// rule and passes the context along
Expand Down Expand Up @@ -136,7 +136,9 @@ public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
return null;
}

assert stack.size() == 3;
assert (stack.size() == 3 && stack.get(1) instanceof FilterOperator) ||
stack.size() == 2;

TableScanOperator tsOp = (TableScanOperator) stack.get(0);
((SamplePrunerCtx) procCtx).getOpToSamplePruner().put(tsOp, sampleDescr);
return null;
Expand Down
72 changes: 36 additions & 36 deletions ql/src/test/results/clientpositive/sample10.q.out
View file

Large diffs are not rendered by default.

60 changes: 30 additions & 30 deletions ql/src/test/results/clientpositive/sample2.q.out
View file
Expand Up @@ -49,9 +49,9 @@ STAGE PLANS:
File Output Operator
compressed: false
GlobalTableId: 1
directory: pfile:/home/amarsri/workspace/hive/build/ql/scratchdir/hive_2011-07-11_02-57-42_556_3775563263647495718/-ext-10002
directory: pfile:/data/users/sdong/www/open-source-hive-fix/build/ql/scratchdir/hive_2011-09-16_11-00-41_794_7183228739920603288/-ext-10002
NumFilesPerFileSink: 1
Stats Publishing Key Prefix: pfile:/home/amarsri/workspace/hive/build/ql/scratchdir/hive_2011-07-11_02-57-42_556_3775563263647495718/-ext-10000/
Stats Publishing Key Prefix: pfile:/data/users/sdong/www/open-source-hive-fix/build/ql/scratchdir/hive_2011-09-16_11-00-41_794_7183228739920603288/-ext-10000/
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
Expand All @@ -61,24 +61,24 @@ STAGE PLANS:
columns.types int:string
file.inputformat org.apache.hadoop.mapred.TextInputFormat
file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
location pfile:/home/amarsri/workspace/hive/build/ql/test/data/warehouse/dest1
location pfile:/data/users/sdong/www/open-source-hive-fix/build/ql/test/data/warehouse/dest1
name default.dest1
serialization.ddl struct dest1 { i32 key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
transient_lastDdlTime 1310378262
transient_lastDdlTime 1316196041
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.dest1
TotalFiles: 1
GatherStats: true
MultiFileSpray: false
Needs Tagging: false
Path -> Alias:
pfile:/home/amarsri/workspace/hive/build/ql/test/data/warehouse/srcbucket [s]
pfile:/data/users/sdong/www/open-source-hive-fix/build/ql/test/data/warehouse/srcbucket/srcbucket0.txt [s]
Path -> Partition:
pfile:/home/amarsri/workspace/hive/build/ql/test/data/warehouse/srcbucket
pfile:/data/users/sdong/www/open-source-hive-fix/build/ql/test/data/warehouse/srcbucket/srcbucket0.txt
Partition
base file name: srcbucket
base file name: srcbucket0.txt
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
Expand All @@ -88,12 +88,12 @@ STAGE PLANS:
columns.types int:string
file.inputformat org.apache.hadoop.mapred.TextInputFormat
file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
location pfile:/home/amarsri/workspace/hive/build/ql/test/data/warehouse/srcbucket
location pfile:/data/users/sdong/www/open-source-hive-fix/build/ql/test/data/warehouse/srcbucket
name default.srcbucket
serialization.ddl struct srcbucket { i32 key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
transient_lastDdlTime 1310371847
transient_lastDdlTime 1316195825
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe

input format: org.apache.hadoop.mapred.TextInputFormat
Expand All @@ -105,12 +105,12 @@ STAGE PLANS:
columns.types int:string
file.inputformat org.apache.hadoop.mapred.TextInputFormat
file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
location pfile:/home/amarsri/workspace/hive/build/ql/test/data/warehouse/srcbucket
location pfile:/data/users/sdong/www/open-source-hive-fix/build/ql/test/data/warehouse/srcbucket
name default.srcbucket
serialization.ddl struct srcbucket { i32 key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
transient_lastDdlTime 1310371847
transient_lastDdlTime 1316195825
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.srcbucket
name: default.srcbucket
Expand All @@ -122,14 +122,14 @@ STAGE PLANS:
Move Operator
files:
hdfs directory: true
source: pfile:/home/amarsri/workspace/hive/build/ql/scratchdir/hive_2011-07-11_02-57-42_556_3775563263647495718/-ext-10002
destination: pfile:/home/amarsri/workspace/hive/build/ql/scratchdir/hive_2011-07-11_02-57-42_556_3775563263647495718/-ext-10000
source: pfile:/data/users/sdong/www/open-source-hive-fix/build/ql/scratchdir/hive_2011-09-16_11-00-41_794_7183228739920603288/-ext-10002
destination: pfile:/data/users/sdong/www/open-source-hive-fix/build/ql/scratchdir/hive_2011-09-16_11-00-41_794_7183228739920603288/-ext-10000

Stage: Stage-0
Move Operator
tables:
replace: true
source: pfile:/home/amarsri/workspace/hive/build/ql/scratchdir/hive_2011-07-11_02-57-42_556_3775563263647495718/-ext-10000
source: pfile:/data/users/sdong/www/open-source-hive-fix/build/ql/scratchdir/hive_2011-09-16_11-00-41_794_7183228739920603288/-ext-10000
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
Expand All @@ -139,28 +139,28 @@ STAGE PLANS:
columns.types int:string
file.inputformat org.apache.hadoop.mapred.TextInputFormat
file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
location pfile:/home/amarsri/workspace/hive/build/ql/test/data/warehouse/dest1
location pfile:/data/users/sdong/www/open-source-hive-fix/build/ql/test/data/warehouse/dest1
name default.dest1
serialization.ddl struct dest1 { i32 key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
transient_lastDdlTime 1310378262
transient_lastDdlTime 1316196041
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.dest1
tmp directory: pfile:/home/amarsri/workspace/hive/build/ql/scratchdir/hive_2011-07-11_02-57-42_556_3775563263647495718/-ext-10001
tmp directory: pfile:/data/users/sdong/www/open-source-hive-fix/build/ql/scratchdir/hive_2011-09-16_11-00-41_794_7183228739920603288/-ext-10001

Stage: Stage-2
Stats-Aggr Operator
Stats Aggregation Key Prefix: pfile:/home/amarsri/workspace/hive/build/ql/scratchdir/hive_2011-07-11_02-57-42_556_3775563263647495718/-ext-10000/
Stats Aggregation Key Prefix: pfile:/data/users/sdong/www/open-source-hive-fix/build/ql/scratchdir/hive_2011-09-16_11-00-41_794_7183228739920603288/-ext-10000/

Stage: Stage-3
Map Reduce
Alias -> Map Operator Tree:
pfile:/home/amarsri/workspace/hive/build/ql/scratchdir/hive_2011-07-11_02-57-42_556_3775563263647495718/-ext-10002
pfile:/data/users/sdong/www/open-source-hive-fix/build/ql/scratchdir/hive_2011-09-16_11-00-41_794_7183228739920603288/-ext-10002
File Output Operator
compressed: false
GlobalTableId: 0
directory: pfile:/home/amarsri/workspace/hive/build/ql/scratchdir/hive_2011-07-11_02-57-42_556_3775563263647495718/-ext-10000
directory: pfile:/data/users/sdong/www/open-source-hive-fix/build/ql/scratchdir/hive_2011-09-16_11-00-41_794_7183228739920603288/-ext-10000
NumFilesPerFileSink: 1
table:
input format: org.apache.hadoop.mapred.TextInputFormat
Expand All @@ -171,22 +171,22 @@ STAGE PLANS:
columns.types int:string
file.inputformat org.apache.hadoop.mapred.TextInputFormat
file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
location pfile:/home/amarsri/workspace/hive/build/ql/test/data/warehouse/dest1
location pfile:/data/users/sdong/www/open-source-hive-fix/build/ql/test/data/warehouse/dest1
name default.dest1
serialization.ddl struct dest1 { i32 key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
transient_lastDdlTime 1310378262
transient_lastDdlTime 1316196041
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.dest1
TotalFiles: 1
GatherStats: false
MultiFileSpray: false
Needs Tagging: false
Path -> Alias:
pfile:/home/amarsri/workspace/hive/build/ql/scratchdir/hive_2011-07-11_02-57-42_556_3775563263647495718/-ext-10002 [pfile:/home/amarsri/workspace/hive/build/ql/scratchdir/hive_2011-07-11_02-57-42_556_3775563263647495718/-ext-10002]
pfile:/data/users/sdong/www/open-source-hive-fix/build/ql/scratchdir/hive_2011-09-16_11-00-41_794_7183228739920603288/-ext-10002 [pfile:/data/users/sdong/www/open-source-hive-fix/build/ql/scratchdir/hive_2011-09-16_11-00-41_794_7183228739920603288/-ext-10002]
Path -> Partition:
pfile:/home/amarsri/workspace/hive/build/ql/scratchdir/hive_2011-07-11_02-57-42_556_3775563263647495718/-ext-10002
pfile:/data/users/sdong/www/open-source-hive-fix/build/ql/scratchdir/hive_2011-09-16_11-00-41_794_7183228739920603288/-ext-10002
Partition
base file name: -ext-10002
input format: org.apache.hadoop.mapred.TextInputFormat
Expand All @@ -197,12 +197,12 @@ STAGE PLANS:
columns.types int:string
file.inputformat org.apache.hadoop.mapred.TextInputFormat
file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
location pfile:/home/amarsri/workspace/hive/build/ql/test/data/warehouse/dest1
location pfile:/data/users/sdong/www/open-source-hive-fix/build/ql/test/data/warehouse/dest1
name default.dest1
serialization.ddl struct dest1 { i32 key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
transient_lastDdlTime 1310378262
transient_lastDdlTime 1316196041
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe

input format: org.apache.hadoop.mapred.TextInputFormat
Expand All @@ -213,12 +213,12 @@ STAGE PLANS:
columns.types int:string
file.inputformat org.apache.hadoop.mapred.TextInputFormat
file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
location pfile:/home/amarsri/workspace/hive/build/ql/test/data/warehouse/dest1
location pfile:/data/users/sdong/www/open-source-hive-fix/build/ql/test/data/warehouse/dest1
name default.dest1
serialization.ddl struct dest1 { i32 key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
transient_lastDdlTime 1310378262
transient_lastDdlTime 1316196041
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.dest1
name: default.dest1
Expand All @@ -239,11 +239,11 @@ POSTHOOK: Lineage: dest1.value SIMPLE [(srcbucket)s.FieldSchema(name:value, type
PREHOOK: query: SELECT dest1.* FROM dest1
PREHOOK: type: QUERY
PREHOOK: Input: default@dest1
PREHOOK: Output: file:/tmp/amarsri/hive_2011-07-11_02-57-46_140_4116462247752325624/-mr-10000
PREHOOK: Output: file:/tmp/sdong/hive_2011-09-16_11-00-47_297_3168534146082599541/-mr-10000
POSTHOOK: query: SELECT dest1.* FROM dest1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@dest1
POSTHOOK: Output: file:/tmp/amarsri/hive_2011-07-11_02-57-46_140_4116462247752325624/-mr-10000
POSTHOOK: Output: file:/tmp/sdong/hive_2011-09-16_11-00-47_297_3168534146082599541/-mr-10000
POSTHOOK: Lineage: dest1.key SIMPLE [(srcbucket)s.FieldSchema(name:key, type:int, comment:null), ]
POSTHOOK: Lineage: dest1.value SIMPLE [(srcbucket)s.FieldSchema(name:value, type:string, comment:null), ]
474 val_475
Expand Down

0 comments on commit 7bc466c

Please sign in to comment.