bug 1548104. Generate record uuids upon ingestion to minimize duplicates

openshift · Mar 1, 2018 · 5e1e372 · 5e1e372
1 parent 253f856
commit 5e1e372
Show file tree

Hide file tree

Showing 5 changed files with 12 additions and 1 deletion.
diff --git a/fluentd/configs.d/openshift/filter-post-genid.conf b/fluentd/configs.d/openshift/filter-post-genid.conf
@@ -0,0 +1,6 @@
+# generate ids at source mitigate creating duplicate records
+# requires fluent-plugin-elasticsearch
+<filter **>
+  @type elasticsearch_genid
+  hash_id_key viaq_msg_id
+</filter>
diff --git a/fluentd/configs.d/openshift/output-es-config.conf b/fluentd/configs.d/openshift/output-es-config.conf
@@ -4,6 +4,7 @@
       port "#{ENV['ES_PORT']}"
       scheme https
       target_index_key viaq_index_name
+      id_key viaq_msg_id
       remove_keys viaq_index_name
       user fluentd
       password changeme

diff --git a/fluentd/configs.d/openshift/output-es-ops-config.conf b/fluentd/configs.d/openshift/output-es-ops-config.conf
@@ -4,6 +4,7 @@
       port "#{ENV['OPS_PORT']}"
       scheme https
       target_index_key viaq_index_name
+      id_key viaq_msg_id
       remove_keys viaq_index_name
       user fluentd
       password changeme

diff --git a/fluentd/configs.d/user/fluent.conf b/fluentd/configs.d/user/fluent.conf
@@ -33,4 +33,3 @@
   # no post - applications.conf matches everything left
 ##
 </label>
-
diff --git a/test/fluentd-forward.sh b/test/fluentd-forward.sh
@@ -55,6 +55,10 @@ update_current_fluentd() {
       else
     # edit so we don't send to ES
     oc get configmap/logging-fluentd -o yaml | sed '/## matches/ a\
+      <filter **>\
+        @type record_transformer\
+        remove_keys _id, viaq_msg_id\
+      </filter>\
       <match **>\
         @type copy\
         @include configs.d/user/secure-forward1.conf\