HIVE-26231: Generate insert notification events when dynamic partitio…

…n insert is done on existing partitions ( Sourabh Badhya, reviewed by Denys Kuzmenko)

Closes #3294

itests/src/test/resources/testconfiguration.properties

@@ -83,6 +83,7 @@ minillap.query.files=\
   input36.q,\
   input38.q,\
   input5.q,\
+  insert_dynamic_partitions_notification_log.q,\
   insert_into1.q,\
   insert_into2.q,\
   insert_into3.q,\

ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java

@@ -3156,7 +3156,7 @@ public Map<Map<String, String>, Partition> loadDynamicPartitions(final LoadTable
           if (partitionDetails.newFiles != null) {
             // If we already know the files from the direct insert manifest, use them
             newFiles = partitionDetails.newFiles;
-          } else if (conf.getBoolVar(ConfVars.FIRE_EVENTS_FOR_DML) && !tbl.isTemporary() && oldPartition == null) {
+          } else if (conf.getBoolVar(ConfVars.FIRE_EVENTS_FOR_DML) && !tbl.isTemporary()) {
             // Otherwise only collect them, if we are going to fire write notifications
             newFiles = Collections.synchronizedList(new ArrayList<>());
           }

ql/src/test/queries/clientpositive/insert_dynamic_partitions_notification_log.q

@@ -0,0 +1,15 @@
+--! qt:sysdb
+set hive.metastore.event.listeners=org.apache.hive.hcatalog.listener.DbNotificationListener;
+set hive.metastore.dml.events=true;
+
+CREATE EXTERNAL TABLE exttable (b INT) PARTITIONED BY (a INT) STORED AS ORC;
+
+INSERT INTO TABLE exttable PARTITION (a) VALUES (1,2), (2,3), (3,4), (4,5), (5,6);
+
+SELECT COUNT(*) FROM sys.notification_log WHERE tbl_name='exttable' AND event_type='INSERT';
+
+INSERT INTO TABLE exttable PARTITION (a) VALUES (1,2), (2,3), (3,4), (4,5), (5,6);
+
+SELECT COUNT(*) FROM sys.notification_log WHERE tbl_name='exttable' AND event_type='INSERT';
+
+DROP TABLE exttable;

ql/src/test/results/clientpositive/llap/insert_dynamic_partitions_notification_log.q.out

@@ -0,0 +1,70 @@
+PREHOOK: query: CREATE EXTERNAL TABLE exttable (b INT) PARTITIONED BY (a INT) STORED AS ORC
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@exttable
+POSTHOOK: query: CREATE EXTERNAL TABLE exttable (b INT) PARTITIONED BY (a INT) STORED AS ORC
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@exttable
+PREHOOK: query: INSERT INTO TABLE exttable PARTITION (a) VALUES (1,2), (2,3), (3,4), (4,5), (5,6)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@exttable
+POSTHOOK: query: INSERT INTO TABLE exttable PARTITION (a) VALUES (1,2), (2,3), (3,4), (4,5), (5,6)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@exttable
+POSTHOOK: Output: default@exttable@a=2
+POSTHOOK: Output: default@exttable@a=3
+POSTHOOK: Output: default@exttable@a=4
+POSTHOOK: Output: default@exttable@a=5
+POSTHOOK: Output: default@exttable@a=6
+POSTHOOK: Lineage: exttable PARTITION(a=2).b SCRIPT []
+POSTHOOK: Lineage: exttable PARTITION(a=3).b SCRIPT []
+POSTHOOK: Lineage: exttable PARTITION(a=4).b SCRIPT []
+POSTHOOK: Lineage: exttable PARTITION(a=5).b SCRIPT []
+POSTHOOK: Lineage: exttable PARTITION(a=6).b SCRIPT []
+PREHOOK: query: SELECT COUNT(*) FROM sys.notification_log WHERE tbl_name='exttable' AND event_type='INSERT'
+PREHOOK: type: QUERY
+PREHOOK: Input: sys@notification_log
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT COUNT(*) FROM sys.notification_log WHERE tbl_name='exttable' AND event_type='INSERT'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: sys@notification_log
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+0
+PREHOOK: query: INSERT INTO TABLE exttable PARTITION (a) VALUES (1,2), (2,3), (3,4), (4,5), (5,6)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@exttable
+POSTHOOK: query: INSERT INTO TABLE exttable PARTITION (a) VALUES (1,2), (2,3), (3,4), (4,5), (5,6)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@exttable
+POSTHOOK: Output: default@exttable@a=2
+POSTHOOK: Output: default@exttable@a=3
+POSTHOOK: Output: default@exttable@a=4
+POSTHOOK: Output: default@exttable@a=5
+POSTHOOK: Output: default@exttable@a=6
+POSTHOOK: Lineage: exttable PARTITION(a=2).b SCRIPT []
+POSTHOOK: Lineage: exttable PARTITION(a=3).b SCRIPT []
+POSTHOOK: Lineage: exttable PARTITION(a=4).b SCRIPT []
+POSTHOOK: Lineage: exttable PARTITION(a=5).b SCRIPT []
+POSTHOOK: Lineage: exttable PARTITION(a=6).b SCRIPT []
+PREHOOK: query: SELECT COUNT(*) FROM sys.notification_log WHERE tbl_name='exttable' AND event_type='INSERT'
+PREHOOK: type: QUERY
+PREHOOK: Input: sys@notification_log
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT COUNT(*) FROM sys.notification_log WHERE tbl_name='exttable' AND event_type='INSERT'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: sys@notification_log
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+5
+PREHOOK: query: DROP TABLE exttable
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@exttable
+PREHOOK: Output: default@exttable
+POSTHOOK: query: DROP TABLE exttable
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@exttable
+POSTHOOK: Output: default@exttable