Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: ordered fluentd-config #1104 #1106

Merged
merged 1 commit into from
Apr 4, 2024
Merged

fix: ordered fluentd-config #1104 #1106

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
8 changes: 7 additions & 1 deletion apis/fluentd/v1alpha1/helper.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -380,14 +380,16 @@ func (pgr *PluginResources) RenderMainConfig(enableMultiWorkers bool) (string, e

// sort main routers
childRouters := ByRouteLabelsPointers(pgr.MainRouterPlugins.Childs)
sort.SliceStable(childRouters[:], childRouters.Less)
pgr.MainRouterPlugins.Childs = childRouters
if enableMultiWorkers {
pgr.MainRouterPlugins.SetIgnorePath()
}
buf.WriteString(pgr.MainRouterPlugins.String())

// sort label plugins
labelPlugins := ByRouteLabels(pgr.LabelPluginResources)
labelPlugins := ByTags(pgr.LabelPluginResources)
sort.SliceStable(labelPlugins[:], labelPlugins.Less)
for _, labelPlugin := range labelPlugins {
if enableMultiWorkers {
labelPlugin.SetIgnorePath()
Expand All @@ -407,9 +409,13 @@ type ByRouteLabelsPointers []*params.PluginStore
// +kubebuilder:object:generate:=false
type ByRouteLabels []params.PluginStore

// +kubebuilder:object:generate:=false
type ByTags []params.PluginStore

func (a ByHashcode) Less(i, j int) bool { return a[i].Hash() < a[j].Hash() }
func (a ByRouteLabelsPointers) Less(i, j int) bool { return a[i].RouteLabel() < a[j].RouteLabel() }
func (a ByRouteLabels) Less(i, j int) bool { return a[i].RouteLabel() < a[j].RouteLabel() }
func (a ByTags) Less(i, j int) bool { return a[i].GetTag() < a[j].GetTag() }

var _ Renderer = &FluentdConfig{}
var _ Renderer = &ClusterFluentdConfig{}
30 changes: 29 additions & 1 deletion apis/fluentd/v1alpha1/plugins/params/model.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,15 @@ func (ps *PluginStore) GetTag() string {
return ps.Store["tag"]
}

// Returns @id value
func (ps *PluginStore) GetId() string {
if value, ok := ps.Store["@id"]; !ok {
return ""
} else {
return value
}
}

// Returns the @label value string of this plugin store
func (ps *PluginStore) RouteLabel() string {
if ps.Name != "route" {
Expand Down Expand Up @@ -118,7 +127,7 @@ func (ps *PluginStore) String() string {
ps.setWhitespaces(parentPrefixWhitespaces + IntervalWhitespaces)
ps.processBody(&buf)
if len(ps.Childs) > 0 {
sort.Sort(PluginStoreByName(ps.Childs))
sort.Sort(PluginStoreByNameById(ps.Childs))
for _, child := range ps.Childs {
child.setWhitespaces(ps.PrefixWhitespaces)
buf.WriteString(child.String())
Expand Down Expand Up @@ -202,3 +211,22 @@ type PluginStoreByName []*PluginStore
func (a PluginStoreByName) Len() int { return len(a) }
func (a PluginStoreByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a PluginStoreByName) Less(i, j int) bool { return a[i].Name < a[j].Name }

// +kubebuilder:object:generate=false
type PluginStoreByNameById []*PluginStore

func (a PluginStoreByNameById) Len() int { return len(a) }
func (a PluginStoreByNameById) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a PluginStoreByNameById) Less(i, j int) bool {
if a[i].Name == a[j].Name {
if a[i].GetTag() == "**" && a[j].GetTag() != "**" {
return false
}
if a[i].GetTag() != "**" && a[j].GetTag() == "**" {
return true
}
return a[i].GetId() < a[j].GetId()
} else {
return a[i].Name < a[j].Name
}
}
22 changes: 11 additions & 11 deletions apis/fluentd/v1alpha1/tests/expected/fluentd-mixed-cfgs-multi-tenant-output.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,29 +7,27 @@
@id main
@type label_router
<route>
@label @b03d7a71344b3738580b3c8ef54d7a75
@label @a504500266de68fa800121c2f3c9088e
<match>
namespaces default,kube-system
namespaces fluent
</match>
</route>
<route>
@label @a504500266de68fa800121c2f3c9088e
@label @b03d7a71344b3738580b3c8ef54d7a75
<match>
namespaces fluent
namespaces default,kube-system
</match>
</route>
</match>
<label @b03d7a71344b3738580b3c8ef54d7a75>
<label @a504500266de68fa800121c2f3c9088e>
<match **>
@id ClusterFluentdConfig-cluster-fluentd-config-cluster::cluster::clusteroutput::fluentd-output-cluster-0
@id FluentdConfig-fluent-fluentd-config-user1::cluster::clusteroutput::fluentd-output-log-operator-0
@type elasticsearch
host elasticsearch-logging-data.kubesphere-logging-system.svc
logstash_format true
logstash_prefix ks-logstash-log
logstash_prefix ks-logstash-log-operator
port 9200
</match>
</label>
<label @a504500266de68fa800121c2f3c9088e>
<match **>
@id FluentdConfig-fluent-fluentd-config-user1::fluent::output::fluentd-output-user1-0
@type elasticsearch
Expand All @@ -38,12 +36,14 @@
logstash_prefix ks-logstash-log-user1
port 9200
</match>
</label>
<label @b03d7a71344b3738580b3c8ef54d7a75>
<match **>
@id FluentdConfig-fluent-fluentd-config-user1::cluster::clusteroutput::fluentd-output-log-operator-0
@id ClusterFluentdConfig-cluster-fluentd-config-cluster::cluster::clusteroutput::fluentd-output-cluster-0
@type elasticsearch
host elasticsearch-logging-data.kubesphere-logging-system.svc
logstash_format true
logstash_prefix ks-logstash-log-operator
logstash_prefix ks-logstash-log
port 9200
</match>
</label>
30 changes: 15 additions & 15 deletions apis/fluentd/v1alpha1/tests/expected/fluentd-mixed-cfgs-output-copy-1.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,21 @@
@id FluentdConfig-fluent-fluentd-config::fluent::output::fluentd-mixed-copy-es-1-0
@type copy
copy_mode no_copy
<store>
@id FluentdConfig-fluent-fluentd-config::cluster::clusteroutput::fluentd-output-loki-0
@type loki
drop_single_key true
extra_labels {"key11":"value11","key12":"value12"}
extract_kubernetes_labels true
include_thread_label true
insecure_tls true
remove_keys key31,key32
url http://loki-logging-data.kubesphere-logging-system.svc:3100
<label>
key21 key21
key22 key22
</label>
</store>
<store>
@id FluentdConfig-fluent-fluentd-config::fluent::output::fluentd-mixed-copy-es-1-1
@type elasticsearch
Expand All @@ -34,20 +49,5 @@
port 9243
scheme https
</store>
<store>
@id FluentdConfig-fluent-fluentd-config::cluster::clusteroutput::fluentd-output-loki-0
@type loki
drop_single_key true
extra_labels {"key11":"value11","key12":"value12"}
extract_kubernetes_labels true
include_thread_label true
insecure_tls true
remove_keys key31,key32
url http://loki-logging-data.kubesphere-logging-system.svc:3100
<label>
key21 key21
key22 key22
</label>
</store>
</match>
</label>
114 changes: 114 additions & 0 deletions apis/fluentd/v1alpha1/tests/expected/fluentd-mixed-cfgs-output-copy-4.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
<source>
@type forward
bind 0.0.0.0
port 24224
</source>
<match **>
@id main
@type label_router
<route>
@label @2d9e59757d3bfc66d93c3bc44b408922
<match>
namespaces fluent
</match>
</route>
</match>
<label @2d9e59757d3bfc66d93c3bc44b408922>
<filter **>
@id FluentdConfig-fluent-fluentd-config::fluent::filter::fluentd-filter-0
@type record_transformer
<record>
application my-application
environment uat
loki-tenant 9705b9de-d60e-4492-b322-eac870
</record>
</filter>
<match specific_tag_1>
@id FluentdConfig-fluent-fluentd-config::fluent::output::fluentd-output-es2-0
@type copy
copy_mode no_copy
<store>
@id FluentdConfig-fluent-fluentd-config::fluent::output::fluentd-output-es2-1
@type elasticsearch
host elasticsearch-logging-data.kubesphere-logging-system.svc
index_name es2-specific-tag-1-1
port 9243
scheme https
ssl_verify false
</store>
<store>
@id FluentdConfig-fluent-fluentd-config::fluent::output::fluentd-output-es2-2
@type elasticsearch
host elasticsearch-logging-data.kubesphere-logging-system.svc
index_name es2-specific-tag-1-2
port 9243
scheme https
ssl_verify false
</store>
</match>
<match specific_tag_2>
@id FluentdConfig-fluent-fluentd-config::fluent::output::fluentd-output-es3-0
@type elasticsearch
host elasticsearch-logging-data.kubesphere-logging-system.svc
index_name es3-specific-tag-2-1
port 9243
scheme https
ssl_verify false
</match>
<match specific_tag_2>
@id FluentdConfig-fluent-fluentd-config::fluent::output::fluentd-output-es3-1
@type elasticsearch
host elasticsearch-logging-data.kubesphere-logging-system.svc
index_name es3-specific-tag-2-2
port 9243
scheme https
ssl_verify false
</match>
<match **>
@id FluentdConfig-fluent-fluentd-config::fluent::output::fluentd-output-es-0
@type copy
copy_mode no_copy
<store>
@id FluentdConfig-fluent-fluentd-config::cluster::clusteroutput::fluentd-output-loki-0
@type loki
drop_single_key true
extra_labels {"key11":"value11","key12":"value12"}
extract_kubernetes_labels true
include_thread_label true
insecure_tls true
remove_keys key31,key32
url http://loki-logging-data.kubesphere-logging-system.svc:3100
<label>
key21 key21
key22 key22
</label>
</store>
<store>
@id FluentdConfig-fluent-fluentd-config::fluent::output::fluentd-output-es-1
@type elasticsearch
host elasticsearch-logging-data.kubesphere-logging-system.svc
index_name es1-notag-1
port 9243
scheme https
ssl_verify false
</store>
<store>
@id FluentdConfig-fluent-fluentd-config::fluent::output::fluentd-output-es-2
@type elasticsearch
host elasticsearch-logging-data.kubesphere-logging-system.svc
index_name es1-notag-2
port 9243
scheme https
ssl_verify false
</store>
<store>
@id FluentdConfig-fluent-fluentd-config::fluent::output::fluentd-output-es4-0
@type elasticsearch
host elasticsearch-logging-data.kubesphere-logging-system.svc
index_name es4-notag-1
port 9243
scheme https
ssl_verify false
</store>
</match>
</label>
16 changes: 8 additions & 8 deletions apis/fluentd/v1alpha1/tests/expected/fluentd-mixed-cfgs-output-es.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,31 +7,31 @@
@id main
@type label_router
<route>
@label @a2170d34e9940ec56d328100e375c43e
@label @2d9e59757d3bfc66d93c3bc44b408922
<match>
namespaces default,kube-system
namespaces fluent
</match>
</route>
<route>
@label @2d9e59757d3bfc66d93c3bc44b408922
@label @a2170d34e9940ec56d328100e375c43e
<match>
namespaces fluent
namespaces default,kube-system
</match>
</route>
</match>
<label @a2170d34e9940ec56d328100e375c43e>
<label @2d9e59757d3bfc66d93c3bc44b408922>
<match **>
@id ClusterFluentdConfig-cluster-fluentd-config::cluster::clusteroutput::fluentd-output-es-0
@id FluentdConfig-fluent-fluentd-config::cluster::clusteroutput::fluentd-output-es-0
@type elasticsearch
host elasticsearch-logging-data.kubesphere-logging-system.svc
logstash_format true
logstash_prefix ks-logstash-log
port 9200
</match>
</label>
<label @2d9e59757d3bfc66d93c3bc44b408922>
<label @a2170d34e9940ec56d328100e375c43e>
<match **>
@id FluentdConfig-fluent-fluentd-config::cluster::clusteroutput::fluentd-output-es-0
@id ClusterFluentdConfig-cluster-fluentd-config::cluster::clusteroutput::fluentd-output-es-0
@type elasticsearch
host elasticsearch-logging-data.kubesphere-logging-system.svc
logstash_format true
Expand Down
16 changes: 8 additions & 8 deletions apis/fluentd/v1alpha1/tests/expected/fluentd-mixed-cfgs-output-opensearch.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,31 +7,31 @@
@id main
@type label_router
<route>
@label @a2170d34e9940ec56d328100e375c43e
@label @2d9e59757d3bfc66d93c3bc44b408922
<match>
namespaces default,kube-system
namespaces fluent
</match>
</route>
<route>
@label @2d9e59757d3bfc66d93c3bc44b408922
@label @a2170d34e9940ec56d328100e375c43e
<match>
namespaces fluent
namespaces default,kube-system
</match>
</route>
</match>
<label @a2170d34e9940ec56d328100e375c43e>
<label @2d9e59757d3bfc66d93c3bc44b408922>
<match **>
@id ClusterFluentdConfig-cluster-fluentd-config::cluster::clusteroutput::fluentd-output-opensearch-0
@id FluentdConfig-fluent-fluentd-config::cluster::clusteroutput::fluentd-output-opensearch-0
@type opensearch
host opensearch-logging-data.kubesphere-logging-system.svc
logstash_format true
logstash_prefix ks-logstash-log
port 9200
</match>
</label>
<label @2d9e59757d3bfc66d93c3bc44b408922>
<label @a2170d34e9940ec56d328100e375c43e>
<match **>
@id FluentdConfig-fluent-fluentd-config::cluster::clusteroutput::fluentd-output-opensearch-0
@id ClusterFluentdConfig-cluster-fluentd-config::cluster::clusteroutput::fluentd-output-opensearch-0
@type opensearch
host opensearch-logging-data.kubesphere-logging-system.svc
logstash_format true
Expand Down