diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 5c4f049f0350..c28782aff44a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -117,6 +117,7 @@ import org.apache.hadoop.hive.ql.ddl.DDLDescWithTableProperties; import org.apache.hadoop.hive.ql.ddl.DDLWork; import org.apache.hadoop.hive.ql.ddl.misc.hooks.InsertCommitHookDesc; +import org.apache.hadoop.hive.ql.ddl.table.create.CreateTableAnalyzer; import org.apache.hadoop.hive.ql.ddl.table.create.CreateTableDesc; import org.apache.hadoop.hive.ql.ddl.table.misc.preinsert.PreInsertTableDesc; import org.apache.hadoop.hive.ql.ddl.table.misc.properties.AlterTableUnsetPropertiesDesc; @@ -1568,7 +1569,7 @@ Table materializeCTE(String cteName, CTEClause cte) throws HiveException { createTable.addChild(temporary); createTable.addChild(cte.cteNode); - SemanticAnalyzer analyzer = new SemanticAnalyzer(queryState); + CreateTableAnalyzer analyzer = new CreateTableAnalyzer(queryState); analyzer.initCtx(ctx); analyzer.init(false); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSemanticAnalyzer.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSemanticAnalyzer.java index dbdc79769dc8..1b07a19d9180 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSemanticAnalyzer.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSemanticAnalyzer.java @@ -493,4 +493,25 @@ private void checkTablesUsed(String query, Set tables) throws Exception Assert.assertEquals(new TreeSet<>(tables), new TreeSet<>(result)); } + + @Test + public void testMaterializeCTEWithCBODisabled() throws Exception { + HiveConf testConf = new HiveConf(conf); + testConf.setBoolVar(HiveConf.ConfVars.HIVE_CBO_ENABLED, false); + testConf.setIntVar(HiveConf.ConfVars.HIVE_CTE_MATERIALIZE_THRESHOLD, 2); + + SessionState.start(testConf); + Context ctx = new Context(testConf); + + String query = "WITH cte AS (SELECT COUNT(*) as cnt FROM table1) " + + "SELECT * FROM cte UNION ALL SELECT * FROM cte"; + + ASTNode astNode = ParseUtils.parse(query, ctx); + QueryState queryState = new QueryState.Builder().withHiveConf(testConf).build(); + BaseSemanticAnalyzer analyzer = SemanticAnalyzerFactory.get(queryState, astNode); + analyzer.initCtx(ctx); + + // This should not throw NPE after the fix + analyzer.analyze(astNode, ctx); + } } diff --git a/ql/src/test/queries/clientpositive/cte_materialize_no_cbo.q b/ql/src/test/queries/clientpositive/cte_materialize_no_cbo.q new file mode 100644 index 000000000000..04b9a153aeca --- /dev/null +++ b/ql/src/test/queries/clientpositive/cte_materialize_no_cbo.q @@ -0,0 +1,16 @@ +-- HIVE-28724 regression: SemanticAnalyzer.materializeCTE uses wrong analyzer class +-- CalcitePlanner.materializeCTE was fixed to use CreateTableAnalyzer, +-- but SemanticAnalyzer.materializeCTE still uses SemanticAnalyzer directly. +-- Bug triggers when CBO is disabled + +set hive.cbo.enable=false; + +explain +WITH cte AS ( + SELECT COUNT(*) as cnt FROM (SELECT 1 as id) t +) +SELECT * FROM cte +UNION ALL +SELECT * FROM cte +UNION ALL +SELECT * FROM cte; diff --git a/ql/src/test/results/clientpositive/llap/cte_materialize_no_cbo.q.out b/ql/src/test/results/clientpositive/llap/cte_materialize_no_cbo.q.out new file mode 100644 index 000000000000..37210c0049d9 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/cte_materialize_no_cbo.q.out @@ -0,0 +1,158 @@ +PREHOOK: query: explain +WITH cte AS ( + SELECT COUNT(*) as cnt FROM (SELECT 1 as id) t +) +SELECT * FROM cte +UNION ALL +SELECT * FROM cte +UNION ALL +SELECT * FROM cte +PREHOOK: type: QUERY +PREHOOK: Input: default@cte +#### A masked pattern was here #### +POSTHOOK: query: explain +WITH cte AS ( + SELECT COUNT(*) as cnt FROM (SELECT 1 as id) t +) +SELECT * FROM cte +UNION ALL +SELECT * FROM cte +UNION ALL +SELECT * FROM cte +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cte +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-4 depends on stages: Stage-2, Stage-0 + Stage-0 depends on stages: Stage-1 + Stage-3 depends on stages: Stage-4 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.4 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.cte + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-4 + Tez +#### A masked pattern was here #### + Edges: + Map 3 <- Union 4 (CONTAINS) + Map 5 <- Union 4 (CONTAINS) + Map 6 <- Union 4 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 3 + Map Operator Tree: + TableScan + alias: cte + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cnt (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: all inputs + Map 5 + Map Operator Tree: + TableScan + alias: cte + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cnt (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: all inputs + Map 6 + Map Operator Tree: + TableScan + alias: cte + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cnt (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: all inputs + Union 4 + Vertex: Union 4 + + Stage: Stage-0 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-3 + Fetch Operator + limit: -1 + Processor Tree: + ListSink +