Skip to content
Permalink
Browse files
HIVE-26231: Generate insert notification events when dynamic partitio…
…n insert is done on existing partitions ( Sourabh Badhya, reviewed by Denys Kuzmenko)

Closes #3294
  • Loading branch information
SourabhBadhya committed May 18, 2022
1 parent 54a0ab9 commit a0cdd365e6cac90fde66483df834ca88578b30cf
Showing 4 changed files with 87 additions and 1 deletion.
@@ -83,6 +83,7 @@ minillap.query.files=\
input36.q,\
input38.q,\
input5.q,\
insert_dynamic_partitions_notification_log.q,\
insert_into1.q,\
insert_into2.q,\
insert_into3.q,\
@@ -3156,7 +3156,7 @@ public Map<Map<String, String>, Partition> loadDynamicPartitions(final LoadTable
if (partitionDetails.newFiles != null) {
// If we already know the files from the direct insert manifest, use them
newFiles = partitionDetails.newFiles;
} else if (conf.getBoolVar(ConfVars.FIRE_EVENTS_FOR_DML) && !tbl.isTemporary() && oldPartition == null) {
} else if (conf.getBoolVar(ConfVars.FIRE_EVENTS_FOR_DML) && !tbl.isTemporary()) {
// Otherwise only collect them, if we are going to fire write notifications
newFiles = Collections.synchronizedList(new ArrayList<>());
}
@@ -0,0 +1,15 @@
--! qt:sysdb
set hive.metastore.event.listeners=org.apache.hive.hcatalog.listener.DbNotificationListener;
set hive.metastore.dml.events=true;

CREATE EXTERNAL TABLE exttable (b INT) PARTITIONED BY (a INT) STORED AS ORC;

INSERT INTO TABLE exttable PARTITION (a) VALUES (1,2), (2,3), (3,4), (4,5), (5,6);

SELECT COUNT(*) FROM sys.notification_log WHERE tbl_name='exttable' AND event_type='INSERT';

INSERT INTO TABLE exttable PARTITION (a) VALUES (1,2), (2,3), (3,4), (4,5), (5,6);

SELECT COUNT(*) FROM sys.notification_log WHERE tbl_name='exttable' AND event_type='INSERT';

DROP TABLE exttable;
@@ -0,0 +1,70 @@
PREHOOK: query: CREATE EXTERNAL TABLE exttable (b INT) PARTITIONED BY (a INT) STORED AS ORC
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@exttable
POSTHOOK: query: CREATE EXTERNAL TABLE exttable (b INT) PARTITIONED BY (a INT) STORED AS ORC
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@exttable
PREHOOK: query: INSERT INTO TABLE exttable PARTITION (a) VALUES (1,2), (2,3), (3,4), (4,5), (5,6)
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@exttable
POSTHOOK: query: INSERT INTO TABLE exttable PARTITION (a) VALUES (1,2), (2,3), (3,4), (4,5), (5,6)
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@exttable
POSTHOOK: Output: default@exttable@a=2
POSTHOOK: Output: default@exttable@a=3
POSTHOOK: Output: default@exttable@a=4
POSTHOOK: Output: default@exttable@a=5
POSTHOOK: Output: default@exttable@a=6
POSTHOOK: Lineage: exttable PARTITION(a=2).b SCRIPT []
POSTHOOK: Lineage: exttable PARTITION(a=3).b SCRIPT []
POSTHOOK: Lineage: exttable PARTITION(a=4).b SCRIPT []
POSTHOOK: Lineage: exttable PARTITION(a=5).b SCRIPT []
POSTHOOK: Lineage: exttable PARTITION(a=6).b SCRIPT []
PREHOOK: query: SELECT COUNT(*) FROM sys.notification_log WHERE tbl_name='exttable' AND event_type='INSERT'
PREHOOK: type: QUERY
PREHOOK: Input: sys@notification_log
PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: SELECT COUNT(*) FROM sys.notification_log WHERE tbl_name='exttable' AND event_type='INSERT'
POSTHOOK: type: QUERY
POSTHOOK: Input: sys@notification_log
POSTHOOK: Output: hdfs://### HDFS PATH ###
0
PREHOOK: query: INSERT INTO TABLE exttable PARTITION (a) VALUES (1,2), (2,3), (3,4), (4,5), (5,6)
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@exttable
POSTHOOK: query: INSERT INTO TABLE exttable PARTITION (a) VALUES (1,2), (2,3), (3,4), (4,5), (5,6)
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@exttable
POSTHOOK: Output: default@exttable@a=2
POSTHOOK: Output: default@exttable@a=3
POSTHOOK: Output: default@exttable@a=4
POSTHOOK: Output: default@exttable@a=5
POSTHOOK: Output: default@exttable@a=6
POSTHOOK: Lineage: exttable PARTITION(a=2).b SCRIPT []
POSTHOOK: Lineage: exttable PARTITION(a=3).b SCRIPT []
POSTHOOK: Lineage: exttable PARTITION(a=4).b SCRIPT []
POSTHOOK: Lineage: exttable PARTITION(a=5).b SCRIPT []
POSTHOOK: Lineage: exttable PARTITION(a=6).b SCRIPT []
PREHOOK: query: SELECT COUNT(*) FROM sys.notification_log WHERE tbl_name='exttable' AND event_type='INSERT'
PREHOOK: type: QUERY
PREHOOK: Input: sys@notification_log
PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: SELECT COUNT(*) FROM sys.notification_log WHERE tbl_name='exttable' AND event_type='INSERT'
POSTHOOK: type: QUERY
POSTHOOK: Input: sys@notification_log
POSTHOOK: Output: hdfs://### HDFS PATH ###
5
PREHOOK: query: DROP TABLE exttable
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@exttable
PREHOOK: Output: default@exttable
POSTHOOK: query: DROP TABLE exttable
POSTHOOK: type: DROPTABLE
POSTHOOK: Input: default@exttable
POSTHOOK: Output: default@exttable

0 comments on commit a0cdd36

Please sign in to comment.