fix: fix fd record transformer parameters (#960)

* fix autoTypeCast & renewRecord Signed-off-by: juicer <cwguoz@gmail.com> * add tests Signed-off-by: juicer <cwguoz@gmail.com> --------- Signed-off-by: juicer <cwguoz@gmail.com>
fluent · Oct 23, 2023 · 3bc3c4c · 3bc3c4c
1 parent c62d73a
commit 3bc3c4c
Show file tree

Hide file tree

Showing 4 changed files with 86 additions and 1 deletion.
diff --git a/apis/fluentd/v1alpha1/plugins/filter/types.go b/apis/fluentd/v1alpha1/plugins/filter/types.go
@@ -164,7 +164,10 @@ func (f *Filter) recordTransformerPlugin(parent *params.PluginStore, loader plug
 			parent.InsertPairs("enable_ruby", fmt.Sprint(*f.RecordTransformer.EnableRuby))
 		}
 		if f.RecordTransformer.AutoTypeCast != nil {
-			parent.InsertPairs("renew_record", fmt.Sprint(*f.RecordTransformer.AutoTypeCast))
+			parent.InsertPairs("auto_typecast", fmt.Sprint(*f.RecordTransformer.AutoTypeCast))
+		}
+		if f.RecordTransformer.RenewRecord != nil {
+			parent.InsertPairs("renew_record", fmt.Sprint(*f.RecordTransformer.RenewRecord))
 		}
 		if f.RecordTransformer.RenewTimeKey != nil {
 			parent.InsertPairs("renew_time_key", fmt.Sprint(*f.RecordTransformer.RenewTimeKey))

diff --git a/apis/fluentd/v1alpha1/tests/expected/fluentd-cluster-cfg-filter-recordTransformer.cfg b/apis/fluentd/v1alpha1/tests/expected/fluentd-cluster-cfg-filter-recordTransformer.cfg
@@ -0,0 +1,35 @@
+<source>
+  @type  forward
+  bind  0.0.0.0
+  port  24224
+</source>
+<match **>
+  @id  main
+  @type  label_router
+  <route>
+    @label  @a2170d34e9940ec56d328100e375c43e
+    <match>
+      namespaces  default,kube-system
+    </match>
+  </route>
+</match>
+<label @a2170d34e9940ec56d328100e375c43e>
+  <filter **>
+    @id  ClusterFluentdConfig-cluster-fluentd-config::cluster::clusterfilter::fluentd-filter-0
+    @type  record_transformer
+    auto_typecast  true
+    enable_ruby  true
+    renew_record  true
+    <record>
+      kubernetes_ns  ${record["kubernetes"]["namespace_name"]}
+    </record>
+  </filter>
+  <match **>
+    @id  ClusterFluentdConfig-cluster-fluentd-config::cluster::clusteroutput::fluentd-output-cluster-0
+    @type  elasticsearch
+    host  elasticsearch-logging-data.kubesphere-logging-system.svc
+    logstash_format  true
+    logstash_prefix  ks-logstash-log
+    port  9200
+  </match>
+</label>
diff --git a/apis/fluentd/v1alpha1/tests/helper_test.go b/apis/fluentd/v1alpha1/tests/helper_test.go
@@ -607,3 +607,30 @@ func Test_DuplicateRemovalCRSpecs(t *testing.T) {
 		i++
 	}
 }
+
+func Test_RecordTransformer(t *testing.T) {
+	g := NewGomegaWithT(t)
+	sl := plugins.NewSecretLoader(nil, Fluentd.Namespace, logr.Logger{})
+
+	psr := fluentdv1alpha1.NewGlobalPluginResources("main")
+	psr.CombineGlobalInputsPlugins(sl, Fluentd.Spec.GlobalInputs)
+
+	clustercfgRouter, err := psr.BuildCfgRouter(&FluentdClusterFluentdConfig1)
+	g.Expect(err).NotTo(HaveOccurred())
+	clusterFilters := []fluentdv1alpha1.ClusterFilter{FluentdClusterRecordTransformerFilter}
+	clusterOutputs := []fluentdv1alpha1.ClusterOutput{FluentdClusterOutputCluster}
+	clustercfgResources, _ := psr.PatchAndFilterClusterLevelResources(sl, FluentdClusterFluentdConfig1.GetCfgId(), clusterFilters, clusterOutputs)
+	err = psr.WithCfgResources(*clustercfgRouter.Label, clustercfgResources)
+	g.Expect(err).NotTo(HaveOccurred())
+
+	// we should not see any permutations in serialized config
+	i := 0
+	for i < maxRuntimes {
+		config, errs := psr.RenderMainConfig(false)
+		// fmt.Println(config)
+		g.Expect(errs).NotTo(HaveOccurred())
+		g.Expect(string(getExpectedCfg("./expected/fluentd-cluster-cfg-filter-recordTransformer.cfg"))).To(Equal(config))
+
+		i++
+	}
+}
diff --git a/apis/fluentd/v1alpha1/tests/tools.go b/apis/fluentd/v1alpha1/tests/tools.go
@@ -199,6 +199,25 @@ spec:
         value: ${record["kubernetes"]["namespace_name"]}
 `
 
+	FluentdClusterRecordTransformerFilter fluentdv1alpha1.ClusterFilter
+	FluentdClusterRecordTransformerRaw    = `
+apiVersion: fluentd.fluent.io/v1alpha1
+kind: ClusterFilter
+metadata:
+  name: fluentd-filter
+  labels:
+    filter.fluentd.fluent.io/enabled: "true"
+spec:
+  filters:
+  - recordTransformer:
+      enableRuby: true
+      autoTypeCast: true
+      renewRecord: true
+      records:
+      - key: kubernetes_ns
+        value: ${record["kubernetes"]["namespace_name"]}
+`
+
 	FluentdClusterOutputBuffer    fluentdv1alpha1.ClusterOutput
 	FluentdClusterOutputBufferRaw = `
 apiVersion: fluentd.fluent.io/v1alpha1
@@ -562,6 +581,7 @@ func init() {
 			ParseIntoObject(FluentdConfigUser1Raw, &FluentdConfigUser1)
 			ParseIntoObject(FluentdConfig1Raw, &FluentdConfig1)
 			ParseIntoObject(FluentdClusterFilter1Raw, &FluentdClusterFilter1)
+			ParseIntoObject(FluentdClusterRecordTransformerRaw, &FluentdClusterRecordTransformerFilter)
 			ParseIntoObject(FluentdClusterOutputClusterRaw, &FluentdClusterOutputCluster)
 			ParseIntoObject(FluentdClusterOutputLogOperatorRaw, &FluentdClusterOutputLogOperator)
 			ParseIntoObject(FluentdClusterOutputBufferRaw, &FluentdClusterOutputBuffer)