From e906ef0f7b27998aa944332d25d4ce9cf4387cd3 Mon Sep 17 00:00:00 2001
From: Slim Bouguerra <slim.bouguerra@gmail.com>
Date: Fri, 20 Jan 2017 16:01:43 -0800
Subject: [PATCH 1/4] Adding pre insert work and block insert intot statement

---
 .../hive/druid/DruidStorageHandler.java       |  5 ++-
 .../hive/metastore/HiveMetaStoreClient.java   | 15 ++++++-
 .../hive/metastore/IMetaStoreClient.java      | 11 ++++-
 .../apache/hadoop/hive/ql/exec/DDLTask.java   | 16 ++++++-
 .../hive/ql/parse/PreInsertTableDesc.java     | 42 +++++++++++++++++++
 .../hive/ql/parse/SemanticAnalyzer.java       |  8 +++-
 .../apache/hadoop/hive/ql/plan/DDLWork.java   | 17 ++++++++
 7 files changed, 108 insertions(+), 6 deletions(-)
 create mode 100644 ql/src/java/org/apache/hadoop/hive/ql/parse/PreInsertTableDesc.java

diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java
index 904ac800566d..d40468b1803a 100644
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java
+++ b/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java
@@ -55,6 +55,7 @@
 import org.apache.hadoop.hive.metastore.api.MetaException;
 import org.apache.hadoop.hive.metastore.api.Table;
 import org.apache.hadoop.hive.ql.metadata.DefaultStorageHandler;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
 import org.apache.hadoop.hive.ql.plan.TableDesc;
 import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hadoop.hive.serde2.AbstractSerDe;
@@ -476,7 +477,9 @@ public void commitInsertTable(Table table, boolean overwrite) throws MetaExcepti
 
   @Override
   public void preInsertTable(Table table, boolean overwrite) throws MetaException {
-    //do nothing
+    if (!overwrite) {
+      throw new MetaException("INSERT INTO statement is not allowed by druid storage handler");
+    }
   }
 
   @Override
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index b5d007d3e41d..e686e20465a1 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -2215,6 +2215,16 @@ public void addDynamicPartitions(long txnId, String dbName, String tableName,
     client.add_dynamic_partitions(adp);
   }
 
+  @Override
+  public void preInsertTable(Table table, boolean overwrite) throws MetaException {
+    HiveMetaHook hook = getHook(table);
+    if (hook == null || !(hook instanceof HiveMetaHookV2)) {
+      return;
+    }
+    HiveMetaHookV2 hiveMetaHook = (HiveMetaHookV2) hook;
+    hiveMetaHook.preInsertTable(table, overwrite);
+  }
+
   @Override
   public void insertTable(Table table, boolean overwrite) throws MetaException {
     boolean failed = true;
@@ -2224,9 +2234,10 @@ public void insertTable(Table table, boolean overwrite) throws MetaException {
     }
     HiveMetaHookV2 hiveMetaHook = (HiveMetaHookV2) hook;
     try {
-      hiveMetaHook.preInsertTable(table, overwrite);
       hiveMetaHook.commitInsertTable(table, overwrite);
-    } finally {
+      failed = false;
+    }
+    finally {
       if (failed) {
         hiveMetaHook.rollbackInsertTable(table, overwrite);
       }
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
index 84ec3324592a..6521ae21cadf 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
@@ -1509,7 +1509,7 @@ void addDynamicPartitions(long txnId, String dbName, String tableName, List<Stri
     throws TException;
 
   /**
-   * Performs the pre/commit/rollback to the metadata storage for insert operator from external storage handler.
+   * Performs the commit/rollback to the metadata storage for insert operator from external storage handler.
    * @param table table name
    * @param overwrite true if the insert is overwrite
    *
@@ -1517,6 +1517,15 @@ void addDynamicPartitions(long txnId, String dbName, String tableName, List<Stri
    */
   void insertTable(Table table, boolean overwrite) throws MetaException;
 
+  /**
+   * Performs the pre insert operation before the job starts
+   * @param table table name
+   * @param overwrite true if the insert is overwrite
+   *
+   * @throws MetaException
+   */
+  void preInsertTable(Table table, boolean overwrite) throws MetaException;
+
   /**
    * A filter provided by the client that determines if a given notification event should be
    * returned.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index 9511b4663879..c5436f9eea86 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -138,6 +138,7 @@
 import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer;
 import org.apache.hadoop.hive.ql.parse.DDLSemanticAnalyzer;
 import org.apache.hadoop.hive.ql.parse.ExplainConfiguration.AnalyzeState;
+import org.apache.hadoop.hive.ql.parse.PreInsertTableDesc;
 import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
 import org.apache.hadoop.hive.ql.parse.SemanticException;
 import org.apache.hadoop.hive.ql.plan.AbortTxnsDesc;
@@ -567,6 +568,10 @@ public int execute(DriverContext driverContext) {
       if (insertTableDesc != null) {
         return insertCommitWork(db, insertTableDesc);
       }
+      PreInsertTableDesc preInsertTableDesc = work.getPreInsertTableDesc();
+      if (preInsertTableDesc != null) {
+        return preInsertWork(db, preInsertTableDesc);
+      }
     } catch (Throwable e) {
       failed(e);
       return 1;
@@ -575,6 +580,15 @@ public int execute(DriverContext driverContext) {
     return 0;
   }
 
+  private int preInsertWork(Hive db, PreInsertTableDesc preInsertTableDesc) throws HiveException {
+    try{
+      db.getMSC().preInsertTable(preInsertTableDesc.getTable(), preInsertTableDesc.isOverwrite());
+    } catch (MetaException e) {
+      throw new HiveException(e);
+    }
+    return 0;
+  }
+
   private int insertCommitWork(Hive db, InsertTableDesc insertTableDesc) throws HiveException {
     try {
       db.getMSC().insertTable(insertTableDesc.getTable(), insertTableDesc.isOverwrite());
@@ -4434,7 +4448,7 @@ private int createView(Hive db, CreateViewDesc crtView) throws HiveException {
         HiveMaterializedViewsRegistry.get().addMaterializedView(tbl);
       }
       addIfAbsentByName(new WriteEntity(tbl, WriteEntity.WriteType.DDL_NO_LOCK));
-      
+
       //set lineage info
       DataContainer dc = new DataContainer(tbl.getTTable());
       SessionState.get().getLineageState().setLineage(new Path(crtView.getViewName()), dc, tbl.getCols());
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/PreInsertTableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/PreInsertTableDesc.java
new file mode 100644
index 000000000000..47b98ff69de4
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/PreInsertTableDesc.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.parse;
+
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.ql.plan.DDLDesc;
+import org.apache.hadoop.hive.ql.plan.Explain;
+
+@Explain(displayName = "Pre-Insert task", explainLevels = { Explain.Level.USER, Explain.Level.DEFAULT, Explain.Level.EXTENDED })
+public class PreInsertTableDesc extends DDLDesc {
+  private final boolean isOverwrite;
+  private final Table table;
+
+  public PreInsertTableDesc(Table table, boolean isOverwrite) {
+    this.table = table;
+    this.isOverwrite = isOverwrite;
+  }
+
+  public Table getTable() {
+    return table;
+  }
+
+  public boolean isOverwrite() {
+    return isOverwrite;
+  }
+}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 6249475beb94..b238987baba5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -6771,7 +6771,10 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input)
         // This is a non-native table.
         // We need to set stats as inaccurate.
         setStatsForNonNativeTable(dest_tab);
-        createInsertDesc(dest_tab, !qb.getParseInfo().isInsertIntoTable(dest_tab.getTableName()));
+        // true if it is insert overwrite.
+        boolean overwrite = !qb.getParseInfo().isInsertIntoTable(
+                String.format("%s.%s", dest_tab.getDbName(), dest_tab.getTableName()));
+        createInsertDesc(dest_tab, overwrite);
       }
 
       WriteEntity output = null;
@@ -7193,7 +7196,10 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input)
   private void createInsertDesc(Table table, boolean overwrite) {
     Task<? extends Serializable>[] tasks = new Task[this.rootTasks.size()];
     tasks = this.rootTasks.toArray(tasks);
+    PreInsertTableDesc preInsertTableDesc = new PreInsertTableDesc(table.getTTable(), overwrite);
     InsertTableDesc insertTableDesc = new InsertTableDesc(table.getTTable(), overwrite);
+    this.rootTasks
+            .add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), preInsertTableDesc), conf));
     TaskFactory
             .getAndMakeChild(new DDLWork(getInputs(), getOutputs(), insertTableDesc), conf, tasks);
   }
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java
index c4efb3fb54ac..2b9e897a54ed 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java
@@ -23,6 +23,7 @@
 import org.apache.hadoop.hive.ql.hooks.ReadEntity;
 import org.apache.hadoop.hive.ql.hooks.WriteEntity;
 import org.apache.hadoop.hive.ql.parse.AlterTablePartMergeFilesDesc;
+import org.apache.hadoop.hive.ql.parse.PreInsertTableDesc;
 import org.apache.hadoop.hive.ql.plan.Explain.Level;
 
 /**
@@ -32,6 +33,7 @@
 public class DDLWork implements Serializable {
   private static final long serialVersionUID = 1L;
 
+  private PreInsertTableDesc preInsertTableDesc;
   private InsertTableDesc insertTableDesc;
   private CreateIndexDesc createIndexDesc;
   private AlterIndexDesc alterIndexDesc;
@@ -532,6 +534,12 @@ public DDLWork(HashSet<ReadEntity> inputs, HashSet<WriteEntity> outputs,
     this.insertTableDesc = insertTableDesc;
   }
 
+  public DDLWork(HashSet<ReadEntity> inputs, HashSet<WriteEntity> outputs,
+          PreInsertTableDesc preInsertTableDesc) {
+    this(inputs, outputs);
+    this.preInsertTableDesc = preInsertTableDesc;
+  }
+
   /**
    * @return Create Database descriptor
    */
@@ -1202,4 +1210,13 @@ public InsertTableDesc getInsertTableDesc() {
   public void setInsertTableDesc(InsertTableDesc insertTableDesc) {
     this.insertTableDesc = insertTableDesc;
   }
+
+  @Explain(displayName = "Pre Insert operator", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
+  public PreInsertTableDesc getPreInsertTableDesc() {
+    return preInsertTableDesc;
+  }
+
+  public void setPreInsertTableDesc(PreInsertTableDesc preInsertTableDesc) {
+    this.preInsertTableDesc = preInsertTableDesc;
+  }
 }

From 88d0ff565a716b6fce2d718c4d0f05afefd7f2b9 Mon Sep 17 00:00:00 2001
From: Slim Bouguerra <slim.bouguerra@gmail.com>
Date: Thu, 26 Jan 2017 17:23:12 -0800
Subject: [PATCH 2/4] fix UTs

---
 .../results/positive/accumulo_queries.q.out   | 30 +++++++++-----
 ...accumulo_single_sourced_multi_insert.q.out | 39 +++++++++++--------
 .../test/results/positive/hbase_queries.q.out | 30 +++++++++-----
 .../hbase_single_sourced_multi_insert.q.out   | 39 +++++++++++--------
 .../test/results/positive/hbasestats.q.out    | 13 +++++--
 5 files changed, 93 insertions(+), 58 deletions(-)

diff --git a/accumulo-handler/src/test/results/positive/accumulo_queries.q.out b/accumulo-handler/src/test/results/positive/accumulo_queries.q.out
index d7cceec2e0c4..de82857c253a 100644
--- a/accumulo-handler/src/test/results/positive/accumulo_queries.q.out
+++ b/accumulo-handler/src/test/results/positive/accumulo_queries.q.out
@@ -40,8 +40,9 @@ POSTHOOK: query: EXPLAIN FROM src INSERT OVERWRITE TABLE accumulo_table_1 SELECT
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
   Stage-0 is a root stage
-  Stage-1
-  Stage-2 is a root stage
+  Stage-2
+  Stage-1 is a root stage
+  Stage-3 is a root stage
 
 STAGE PLANS:
   Stage: Stage-0
@@ -52,11 +53,15 @@ STAGE PLANS:
           properties:
             COLUMN_STATS_ACCURATE 
 
-  Stage: Stage-1
+  Stage: Stage-2
       Insert operator:
         Insert
 
-  Stage: Stage-2
+  Stage: Stage-1
+      Pre Insert operator:
+        Pre-Insert task
+
+  Stage: Stage-3
     Map Reduce
       Map Operator Tree:
           TableScan
@@ -495,9 +500,10 @@ ON (x.key = Y.key)
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
   Stage-0 is a root stage
-  Stage-1
-  Stage-3 is a root stage
-  Stage-2 depends on stages: Stage-3
+  Stage-2
+  Stage-1 is a root stage
+  Stage-4 is a root stage
+  Stage-3 depends on stages: Stage-4
 
 STAGE PLANS:
   Stage: Stage-0
@@ -508,11 +514,15 @@ STAGE PLANS:
           properties:
             COLUMN_STATS_ACCURATE 
 
-  Stage: Stage-1
+  Stage: Stage-2
       Insert operator:
         Insert
 
-  Stage: Stage-3
+  Stage: Stage-1
+      Pre Insert operator:
+        Pre-Insert task
+
+  Stage: Stage-4
     Map Reduce
       Map Operator Tree:
           TableScan
@@ -547,7 +557,7 @@ STAGE PLANS:
                 output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
                 serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
 
-  Stage: Stage-2
+  Stage: Stage-3
     Map Reduce
       Map Operator Tree:
           TableScan
diff --git a/accumulo-handler/src/test/results/positive/accumulo_single_sourced_multi_insert.q.out b/accumulo-handler/src/test/results/positive/accumulo_single_sourced_multi_insert.q.out
index 7330746d5770..6621a4e2045a 100644
--- a/accumulo-handler/src/test/results/positive/accumulo_single_sourced_multi_insert.q.out
+++ b/accumulo-handler/src/test/results/positive/accumulo_single_sourced_multi_insert.q.out
@@ -34,15 +34,16 @@ select value,"" where a.key > 50 AND a.key < 100
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
   Stage-0 is a root stage
-  Stage-1
-  Stage-3 is a root stage
-  Stage-9 depends on stages: Stage-3 , consists of Stage-6, Stage-5, Stage-7
-  Stage-6
-  Stage-2 depends on stages: Stage-6, Stage-5, Stage-8
-  Stage-4 depends on stages: Stage-2
-  Stage-5
+  Stage-2
+  Stage-1 is a root stage
+  Stage-4 is a root stage
+  Stage-10 depends on stages: Stage-4 , consists of Stage-7, Stage-6, Stage-8
   Stage-7
-  Stage-8 depends on stages: Stage-7
+  Stage-3 depends on stages: Stage-7, Stage-6, Stage-9
+  Stage-5 depends on stages: Stage-3
+  Stage-6
+  Stage-8
+  Stage-9 depends on stages: Stage-8
 
 STAGE PLANS:
   Stage: Stage-0
@@ -53,11 +54,15 @@ STAGE PLANS:
           properties:
             COLUMN_STATS_ACCURATE 
 
-  Stage: Stage-1
+  Stage: Stage-2
       Insert operator:
         Insert
 
-  Stage: Stage-3
+  Stage: Stage-1
+      Pre Insert operator:
+        Pre-Insert task
+
+  Stage: Stage-4
     Map Reduce
       Map Operator Tree:
           TableScan
@@ -94,16 +99,16 @@ STAGE PLANS:
                       serde: org.apache.hadoop.hive.accumulo.serde.AccumuloSerDe
                       name: default.src_x2
 
-  Stage: Stage-9
+  Stage: Stage-10
     Conditional Operator
 
-  Stage: Stage-6
+  Stage: Stage-7
     Move Operator
       files:
           hdfs directory: true
 #### A masked pattern was here ####
 
-  Stage: Stage-2
+  Stage: Stage-3
     Move Operator
       tables:
           replace: true
@@ -113,10 +118,10 @@ STAGE PLANS:
               serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
               name: default.src_x1
 
-  Stage: Stage-4
+  Stage: Stage-5
     Stats-Aggr Operator
 
-  Stage: Stage-5
+  Stage: Stage-6
     Map Reduce
       Map Operator Tree:
           TableScan
@@ -128,7 +133,7 @@ STAGE PLANS:
                   serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
                   name: default.src_x1
 
-  Stage: Stage-7
+  Stage: Stage-8
     Map Reduce
       Map Operator Tree:
           TableScan
@@ -140,7 +145,7 @@ STAGE PLANS:
                   serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
                   name: default.src_x1
 
-  Stage: Stage-8
+  Stage: Stage-9
     Move Operator
       files:
           hdfs directory: true
diff --git a/hbase-handler/src/test/results/positive/hbase_queries.q.out b/hbase-handler/src/test/results/positive/hbase_queries.q.out
index 1eeaf8018231..276b6b8328d3 100644
--- a/hbase-handler/src/test/results/positive/hbase_queries.q.out
+++ b/hbase-handler/src/test/results/positive/hbase_queries.q.out
@@ -40,8 +40,9 @@ POSTHOOK: query: EXPLAIN FROM src INSERT OVERWRITE TABLE hbase_table_1 SELECT *
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
   Stage-0 is a root stage
-  Stage-1
-  Stage-2 is a root stage
+  Stage-2
+  Stage-1 is a root stage
+  Stage-3 is a root stage
 
 STAGE PLANS:
   Stage: Stage-0
@@ -52,11 +53,15 @@ STAGE PLANS:
           properties:
             COLUMN_STATS_ACCURATE 
 
-  Stage: Stage-1
+  Stage: Stage-2
       Insert operator:
         Insert
 
-  Stage: Stage-2
+  Stage: Stage-1
+      Pre Insert operator:
+        Pre-Insert task
+
+  Stage: Stage-3
     Map Reduce
       Map Operator Tree:
           TableScan
@@ -498,9 +503,10 @@ ON (x.key = Y.key)
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
   Stage-0 is a root stage
-  Stage-1
-  Stage-3 is a root stage
-  Stage-2 depends on stages: Stage-3
+  Stage-2
+  Stage-1 is a root stage
+  Stage-4 is a root stage
+  Stage-3 depends on stages: Stage-4
 
 STAGE PLANS:
   Stage: Stage-0
@@ -511,11 +517,15 @@ STAGE PLANS:
           properties:
             COLUMN_STATS_ACCURATE 
 
-  Stage: Stage-1
+  Stage: Stage-2
       Insert operator:
         Insert
 
-  Stage: Stage-3
+  Stage: Stage-1
+      Pre Insert operator:
+        Pre-Insert task
+
+  Stage: Stage-4
     Map Reduce
       Map Operator Tree:
           TableScan
@@ -550,7 +560,7 @@ STAGE PLANS:
                 output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
                 serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
 
-  Stage: Stage-2
+  Stage: Stage-3
     Map Reduce
       Map Operator Tree:
           TableScan
diff --git a/hbase-handler/src/test/results/positive/hbase_single_sourced_multi_insert.q.out b/hbase-handler/src/test/results/positive/hbase_single_sourced_multi_insert.q.out
index 079fb0e828ab..68a417d0c15b 100644
--- a/hbase-handler/src/test/results/positive/hbase_single_sourced_multi_insert.q.out
+++ b/hbase-handler/src/test/results/positive/hbase_single_sourced_multi_insert.q.out
@@ -34,15 +34,16 @@ select value,"" where a.key > 50 AND a.key < 100
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
   Stage-0 is a root stage
-  Stage-1
-  Stage-3 is a root stage
-  Stage-9 depends on stages: Stage-3 , consists of Stage-6, Stage-5, Stage-7
-  Stage-6
-  Stage-2 depends on stages: Stage-6, Stage-5, Stage-8
-  Stage-4 depends on stages: Stage-2
-  Stage-5
+  Stage-2
+  Stage-1 is a root stage
+  Stage-4 is a root stage
+  Stage-10 depends on stages: Stage-4 , consists of Stage-7, Stage-6, Stage-8
   Stage-7
-  Stage-8 depends on stages: Stage-7
+  Stage-3 depends on stages: Stage-7, Stage-6, Stage-9
+  Stage-5 depends on stages: Stage-3
+  Stage-6
+  Stage-8
+  Stage-9 depends on stages: Stage-8
 
 STAGE PLANS:
   Stage: Stage-0
@@ -53,11 +54,15 @@ STAGE PLANS:
           properties:
             COLUMN_STATS_ACCURATE 
 
-  Stage: Stage-1
+  Stage: Stage-2
       Insert operator:
         Insert
 
-  Stage: Stage-3
+  Stage: Stage-1
+      Pre Insert operator:
+        Pre-Insert task
+
+  Stage: Stage-4
     Map Reduce
       Map Operator Tree:
           TableScan
@@ -94,16 +99,16 @@ STAGE PLANS:
                       serde: org.apache.hadoop.hive.hbase.HBaseSerDe
                       name: default.src_x2
 
-  Stage: Stage-9
+  Stage: Stage-10
     Conditional Operator
 
-  Stage: Stage-6
+  Stage: Stage-7
     Move Operator
       files:
           hdfs directory: true
 #### A masked pattern was here ####
 
-  Stage: Stage-2
+  Stage: Stage-3
     Move Operator
       tables:
           replace: true
@@ -113,10 +118,10 @@ STAGE PLANS:
               serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
               name: default.src_x1
 
-  Stage: Stage-4
+  Stage: Stage-5
     Stats-Aggr Operator
 
-  Stage: Stage-5
+  Stage: Stage-6
     Map Reduce
       Map Operator Tree:
           TableScan
@@ -128,7 +133,7 @@ STAGE PLANS:
                   serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
                   name: default.src_x1
 
-  Stage: Stage-7
+  Stage: Stage-8
     Map Reduce
       Map Operator Tree:
           TableScan
@@ -140,7 +145,7 @@ STAGE PLANS:
                   serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
                   name: default.src_x1
 
-  Stage: Stage-8
+  Stage: Stage-9
     Move Operator
       files:
           hdfs directory: true
diff --git a/hbase-handler/src/test/results/positive/hbasestats.q.out b/hbase-handler/src/test/results/positive/hbasestats.q.out
index 4e47bf5b228b..bf902e49b57f 100644
--- a/hbase-handler/src/test/results/positive/hbasestats.q.out
+++ b/hbase-handler/src/test/results/positive/hbasestats.q.out
@@ -63,8 +63,9 @@ POSTHOOK: query: explain INSERT OVERWRITE TABLE users SELECT 'user1', 'IA', 'USA
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
   Stage-0 is a root stage
-  Stage-1
-  Stage-2 is a root stage
+  Stage-2
+  Stage-1 is a root stage
+  Stage-3 is a root stage
 
 STAGE PLANS:
   Stage: Stage-0
@@ -75,11 +76,15 @@ STAGE PLANS:
           properties:
             COLUMN_STATS_ACCURATE 
 
-  Stage: Stage-1
+  Stage: Stage-2
       Insert operator:
         Insert
 
-  Stage: Stage-2
+  Stage: Stage-1
+      Pre Insert operator:
+        Pre-Insert task
+
+  Stage: Stage-3
     Map Reduce
       Map Operator Tree:
           TableScan

From c8331159beeecd9f90994d7403aa9ec77a50ad6f Mon Sep 17 00:00:00 2001
From: Slim Bouguerra <slim.bouguerra@gmail.com>
Date: Tue, 31 Jan 2017 16:34:49 -0800
Subject: [PATCH 3/4] move preinsert method out of the metastore interface

---
 .../hadoop/hive/metastore/HiveMetaStoreClient.java     | 10 ----------
 .../apache/hadoop/hive/metastore/IMetaStoreClient.java |  9 ---------
 .../java/org/apache/hadoop/hive/ql/exec/DDLTask.java   |  9 ++++++++-
 .../hadoop/hive/ql/parse/PreInsertTableDesc.java       |  6 +++---
 .../apache/hadoop/hive/ql/parse/SemanticAnalyzer.java  |  2 +-
 5 files changed, 12 insertions(+), 24 deletions(-)

diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index e686e20465a1..2b8d2e92d3c8 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -2215,16 +2215,6 @@ public void addDynamicPartitions(long txnId, String dbName, String tableName,
     client.add_dynamic_partitions(adp);
   }
 
-  @Override
-  public void preInsertTable(Table table, boolean overwrite) throws MetaException {
-    HiveMetaHook hook = getHook(table);
-    if (hook == null || !(hook instanceof HiveMetaHookV2)) {
-      return;
-    }
-    HiveMetaHookV2 hiveMetaHook = (HiveMetaHookV2) hook;
-    hiveMetaHook.preInsertTable(table, overwrite);
-  }
-
   @Override
   public void insertTable(Table table, boolean overwrite) throws MetaException {
     boolean failed = true;
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
index 6521ae21cadf..a07c6950cbcf 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
@@ -1517,15 +1517,6 @@ void addDynamicPartitions(long txnId, String dbName, String tableName, List<Stri
    */
   void insertTable(Table table, boolean overwrite) throws MetaException;
 
-  /**
-   * Performs the pre insert operation before the job starts
-   * @param table table name
-   * @param overwrite true if the insert is overwrite
-   *
-   * @throws MetaException
-   */
-  void preInsertTable(Table table, boolean overwrite) throws MetaException;
-
   /**
    * A filter provided by the client that determines if a given notification event should be
    * returned.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index c5436f9eea86..ef4e4cba3d96 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -63,6 +63,8 @@
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.io.HdfsUtils;
+import org.apache.hadoop.hive.metastore.HiveMetaHook;
+import org.apache.hadoop.hive.metastore.HiveMetaHookV2;
 import org.apache.hadoop.hive.metastore.MetaStoreUtils;
 import org.apache.hadoop.hive.metastore.PartitionDropOptions;
 import org.apache.hadoop.hive.metastore.TableType;
@@ -582,7 +584,12 @@ public int execute(DriverContext driverContext) {
 
   private int preInsertWork(Hive db, PreInsertTableDesc preInsertTableDesc) throws HiveException {
     try{
-      db.getMSC().preInsertTable(preInsertTableDesc.getTable(), preInsertTableDesc.isOverwrite());
+      HiveMetaHook hook = preInsertTableDesc.getTable().getStorageHandler().getMetaHook();
+      if (hook == null || !(hook instanceof HiveMetaHookV2)) {
+        return 0;
+      }
+      HiveMetaHookV2 hiveMetaHook = (HiveMetaHookV2) hook;
+      hiveMetaHook.preInsertTable(preInsertTableDesc.getTable().getTTable(), preInsertTableDesc.isOverwrite());
     } catch (MetaException e) {
       throw new HiveException(e);
     }
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/PreInsertTableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/PreInsertTableDesc.java
index 47b98ff69de4..84f6de4ddbc6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/PreInsertTableDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/PreInsertTableDesc.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.hive.ql.parse;
 
-import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.hive.ql.plan.DDLDesc;
 import org.apache.hadoop.hive.ql.plan.Explain;
 
@@ -27,9 +27,9 @@ public class PreInsertTableDesc extends DDLDesc {
   private final boolean isOverwrite;
   private final Table table;
 
-  public PreInsertTableDesc(Table table, boolean isOverwrite) {
+  public PreInsertTableDesc(org.apache.hadoop.hive.ql.metadata.Table table, boolean overwrite) {
     this.table = table;
-    this.isOverwrite = isOverwrite;
+    this.isOverwrite = overwrite;
   }
 
   public Table getTable() {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index b238987baba5..7500746ea94c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -7196,7 +7196,7 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input)
   private void createInsertDesc(Table table, boolean overwrite) {
     Task<? extends Serializable>[] tasks = new Task[this.rootTasks.size()];
     tasks = this.rootTasks.toArray(tasks);
-    PreInsertTableDesc preInsertTableDesc = new PreInsertTableDesc(table.getTTable(), overwrite);
+    PreInsertTableDesc preInsertTableDesc = new PreInsertTableDesc(table, overwrite);
     InsertTableDesc insertTableDesc = new InsertTableDesc(table.getTTable(), overwrite);
     this.rootTasks
             .add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), preInsertTableDesc), conf));

From 5ad695caa559675541b67b3fb48505765c335c3e Mon Sep 17 00:00:00 2001
From: Slim Bouguerra <slim.bouguerra@gmail.com>
Date: Tue, 31 Jan 2017 18:25:19 -0800
Subject: [PATCH 4/4] move out the insertTable hook from metastore client

---
 .../apache/hadoop/hive/ql/exec/DDLTask.java   | 23 +++++++++++++++----
 .../hive/ql/parse/PreInsertTableDesc.java     |  2 +-
 .../hive/ql/parse/SemanticAnalyzer.java       |  2 +-
 .../hadoop/hive/ql/plan/InsertTableDesc.java  |  2 +-
 4 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index ef4e4cba3d96..8dbe85a4c333 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -596,13 +596,26 @@ private int preInsertWork(Hive db, PreInsertTableDesc preInsertTableDesc) throws
     return 0;
   }
 
-  private int insertCommitWork(Hive db, InsertTableDesc insertTableDesc) throws HiveException {
-    try {
-      db.getMSC().insertTable(insertTableDesc.getTable(), insertTableDesc.isOverwrite());
+  private int insertCommitWork(Hive db, InsertTableDesc insertTableDesc) throws MetaException {
+    boolean failed = true;
+    HiveMetaHook hook = insertTableDesc.getTable().getStorageHandler().getMetaHook();
+    if (hook == null || !(hook instanceof HiveMetaHookV2)) {
       return 0;
-    } catch (MetaException e) {
-      throw new HiveException(e);
     }
+    HiveMetaHookV2 hiveMetaHook = (HiveMetaHookV2) hook;
+    try {
+      hiveMetaHook.commitInsertTable(insertTableDesc.getTable().getTTable(),
+              insertTableDesc.isOverwrite()
+      );
+      failed = false;
+    } finally {
+      if (failed) {
+        hiveMetaHook.rollbackInsertTable(insertTableDesc.getTable().getTTable(),
+                insertTableDesc.isOverwrite()
+        );
+      }
+    }
+    return 0;
   }
 
   private int cacheMetadata(Hive db, CacheMetadataDesc desc) throws HiveException {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/PreInsertTableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/PreInsertTableDesc.java
index 84f6de4ddbc6..2c8e1e18d0a8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/PreInsertTableDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/PreInsertTableDesc.java
@@ -27,7 +27,7 @@ public class PreInsertTableDesc extends DDLDesc {
   private final boolean isOverwrite;
   private final Table table;
 
-  public PreInsertTableDesc(org.apache.hadoop.hive.ql.metadata.Table table, boolean overwrite) {
+  public PreInsertTableDesc(Table table, boolean overwrite) {
     this.table = table;
     this.isOverwrite = overwrite;
   }
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 7500746ea94c..24461624e51e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -7197,7 +7197,7 @@ private void createInsertDesc(Table table, boolean overwrite) {
     Task<? extends Serializable>[] tasks = new Task[this.rootTasks.size()];
     tasks = this.rootTasks.toArray(tasks);
     PreInsertTableDesc preInsertTableDesc = new PreInsertTableDesc(table, overwrite);
-    InsertTableDesc insertTableDesc = new InsertTableDesc(table.getTTable(), overwrite);
+    InsertTableDesc insertTableDesc = new InsertTableDesc(table, overwrite);
     this.rootTasks
             .add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), preInsertTableDesc), conf));
     TaskFactory
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/InsertTableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/InsertTableDesc.java
index 1397b8ab2774..212bc7ae74b1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/InsertTableDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/InsertTableDesc.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.hive.ql.plan;
 
-import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.ql.metadata.Table;
 
 @Explain(displayName = "Insert", explainLevels = { Explain.Level.USER, Explain.Level.DEFAULT, Explain.Level.EXTENDED })
 public class InsertTableDesc extends DDLDesc {