From 3584b386072f679952f7f22185961632ba4c06c8 Mon Sep 17 00:00:00 2001 From: Calvin Lee Date: Mon, 6 Jan 2025 12:01:16 -0500 Subject: [PATCH] LOG-6373: Port LOG-6354 to release-6.1 --- internal/generator/vector/conf/complex.toml | 14 ++++++++++++-- .../vector/conf/complex_http_receiver.toml | 14 ++++++++++++-- internal/generator/vector/conf/container.toml | 14 ++++++++++++-- .../generator/vector/input/application.toml | 7 ++++++- ...lication_exclude_container_from_infra.toml | 7 ++++++- .../input/application_excludes_container.toml | 7 ++++++- .../input/application_includes_container.toml | 7 ++++++- .../application_with_includes_excludes.toml | 7 ++++++- ...lication_with_infra_includes_excludes.toml | 7 ++++++- ...on_with_infra_includes_infra_excludes.toml | 7 ++++++- .../input/application_with_matchLabels.toml | 7 ++++++- ...pecific_infra_includes_infra_excludes.toml | 7 ++++++- .../input/application_with_throttle.toml | 7 ++++++- internal/generator/vector/input/audit.go | 8 ++++---- .../vector/input/infrastructure.toml | 7 ++++++- .../input/infrastructure_container.toml | 7 ++++++- internal/generator/vector/input/journal.go | 2 +- internal/generator/vector/input/meta.go | 10 ++++++++-- internal/generator/vector/input/receiver.go | 4 ++-- internal/generator/vector/input/source.go | 19 +++++++++++++++++-- 20 files changed, 140 insertions(+), 29 deletions(-) diff --git a/internal/generator/vector/conf/complex.toml b/internal/generator/vector/conf/complex.toml index 27105b608e..f0b255b203 100644 --- a/internal/generator/vector/conf/complex.toml +++ b/internal/generator/vector/conf/complex.toml @@ -109,7 +109,12 @@ type = "remap" inputs = ["input_infrastructure_container"] source = ''' .log_source = "container" - .log_type = "infrastructure" + # If namespace is infra, label log_type as infra + if match_any(string!(.kubernetes.namespace_name), [r'^default$', r'^openshift(-.+)?$', r'^kube(-.+)?$']) { + .log_type = "infrastructure" + } else { + .log_type = "application" + } ''' [sources.input_infrastructure_journal] @@ -145,7 +150,12 @@ type = "remap" inputs = ["input_mytestapp_container"] source = ''' .log_source = "container" - .log_type = "application" + # If namespace is infra, label log_type as infra + if match_any(string!(.kubernetes.namespace_name), [r'^default$', r'^openshift(-.+)?$', r'^kube(-.+)?$']) { + .log_type = "infrastructure" + } else { + .log_type = "application" + } ''' [transforms.pipeline_pipeline_viaqjournal_0] diff --git a/internal/generator/vector/conf/complex_http_receiver.toml b/internal/generator/vector/conf/complex_http_receiver.toml index c763ab9c35..258c0dca0a 100644 --- a/internal/generator/vector/conf/complex_http_receiver.toml +++ b/internal/generator/vector/conf/complex_http_receiver.toml @@ -109,7 +109,12 @@ type = "remap" inputs = ["input_infrastructure_container"] source = ''' .log_source = "container" - .log_type = "infrastructure" + # If namespace is infra, label log_type as infra + if match_any(string!(.kubernetes.namespace_name), [r'^default$', r'^openshift(-.+)?$', r'^kube(-.+)?$']) { + .log_type = "infrastructure" + } else { + .log_type = "application" + } ''' [sources.input_infrastructure_journal] @@ -179,7 +184,12 @@ type = "remap" inputs = ["input_mytestapp_container"] source = ''' .log_source = "container" - .log_type = "application" + # If namespace is infra, label log_type as infra + if match_any(string!(.kubernetes.namespace_name), [r'^default$', r'^openshift(-.+)?$', r'^kube(-.+)?$']) { + .log_type = "infrastructure" + } else { + .log_type = "application" + } ''' [transforms.pipeline_pipeline_viaqjournal_0] diff --git a/internal/generator/vector/conf/container.toml b/internal/generator/vector/conf/container.toml index 2d3aad0681..c36cb52a87 100644 --- a/internal/generator/vector/conf/container.toml +++ b/internal/generator/vector/conf/container.toml @@ -33,7 +33,12 @@ type = "remap" inputs = ["input_myinfra_container"] source = ''' .log_source = "container" - .log_type = "infrastructure" + # If namespace is infra, label log_type as infra + if match_any(string!(.kubernetes.namespace_name), [r'^default$', r'^openshift(-.+)?$', r'^kube(-.+)?$']) { + .log_type = "infrastructure" + } else { + .log_type = "application" + } ''' # Logs from containers (including openshift containers) @@ -57,7 +62,12 @@ type = "remap" inputs = ["input_mytestapp_container"] source = ''' .log_source = "container" - .log_type = "application" + # If namespace is infra, label log_type as infra + if match_any(string!(.kubernetes.namespace_name), [r'^default$', r'^openshift(-.+)?$', r'^kube(-.+)?$']) { + .log_type = "infrastructure" + } else { + .log_type = "application" + } ''' [transforms.pipeline_mypipeline_viaq_0] diff --git a/internal/generator/vector/input/application.toml b/internal/generator/vector/input/application.toml index bc837c0642..a667faa668 100644 --- a/internal/generator/vector/input/application.toml +++ b/internal/generator/vector/input/application.toml @@ -18,5 +18,10 @@ type = "remap" inputs = ["input_application_container"] source = ''' .log_source = "container" - .log_type = "application" + # If namespace is infra, label log_type as infra + if match_any(string!(.kubernetes.namespace_name), [r'^default$', r'^openshift(-.+)?$', r'^kube(-.+)?$']) { + .log_type = "infrastructure" + } else { + .log_type = "application" + } ''' \ No newline at end of file diff --git a/internal/generator/vector/input/application_exclude_container_from_infra.toml b/internal/generator/vector/input/application_exclude_container_from_infra.toml index bfdeada0cb..c652ca8f49 100644 --- a/internal/generator/vector/input/application_exclude_container_from_infra.toml +++ b/internal/generator/vector/input/application_exclude_container_from_infra.toml @@ -19,5 +19,10 @@ type = "remap" inputs = ["input_my_app_container"] source = ''' .log_source = "container" - .log_type = "application" + # If namespace is infra, label log_type as infra + if match_any(string!(.kubernetes.namespace_name), [r'^default$', r'^openshift(-.+)?$', r'^kube(-.+)?$']) { + .log_type = "infrastructure" + } else { + .log_type = "application" + } ''' \ No newline at end of file diff --git a/internal/generator/vector/input/application_excludes_container.toml b/internal/generator/vector/input/application_excludes_container.toml index 9ce63786da..c20697fe20 100644 --- a/internal/generator/vector/input/application_excludes_container.toml +++ b/internal/generator/vector/input/application_excludes_container.toml @@ -18,5 +18,10 @@ type = "remap" inputs = ["input_my_app_container"] source = ''' .log_source = "container" - .log_type = "application" + # If namespace is infra, label log_type as infra + if match_any(string!(.kubernetes.namespace_name), [r'^default$', r'^openshift(-.+)?$', r'^kube(-.+)?$']) { + .log_type = "infrastructure" + } else { + .log_type = "application" + } ''' \ No newline at end of file diff --git a/internal/generator/vector/input/application_includes_container.toml b/internal/generator/vector/input/application_includes_container.toml index 6c4a311bb1..fe7af53f1d 100644 --- a/internal/generator/vector/input/application_includes_container.toml +++ b/internal/generator/vector/input/application_includes_container.toml @@ -19,5 +19,10 @@ type = "remap" inputs = ["input_my_app_container"] source = ''' .log_source = "container" - .log_type = "application" + # If namespace is infra, label log_type as infra + if match_any(string!(.kubernetes.namespace_name), [r'^default$', r'^openshift(-.+)?$', r'^kube(-.+)?$']) { + .log_type = "infrastructure" + } else { + .log_type = "application" + } ''' \ No newline at end of file diff --git a/internal/generator/vector/input/application_with_includes_excludes.toml b/internal/generator/vector/input/application_with_includes_excludes.toml index 67f1cee0f2..fc6dfdd208 100644 --- a/internal/generator/vector/input/application_with_includes_excludes.toml +++ b/internal/generator/vector/input/application_with_includes_excludes.toml @@ -19,5 +19,10 @@ type = "remap" inputs = ["input_my_app_container"] source = ''' .log_source = "container" - .log_type = "application" + # If namespace is infra, label log_type as infra + if match_any(string!(.kubernetes.namespace_name), [r'^default$', r'^openshift(-.+)?$', r'^kube(-.+)?$']) { + .log_type = "infrastructure" + } else { + .log_type = "application" + } ''' \ No newline at end of file diff --git a/internal/generator/vector/input/application_with_infra_includes_excludes.toml b/internal/generator/vector/input/application_with_infra_includes_excludes.toml index 6ff56847ca..8f17f8a050 100644 --- a/internal/generator/vector/input/application_with_infra_includes_excludes.toml +++ b/internal/generator/vector/input/application_with_infra_includes_excludes.toml @@ -19,5 +19,10 @@ type = "remap" inputs = ["input_my_app_container"] source = ''' .log_source = "container" - .log_type = "application" + # If namespace is infra, label log_type as infra + if match_any(string!(.kubernetes.namespace_name), [r'^default$', r'^openshift(-.+)?$', r'^kube(-.+)?$']) { + .log_type = "infrastructure" + } else { + .log_type = "application" + } ''' \ No newline at end of file diff --git a/internal/generator/vector/input/application_with_infra_includes_infra_excludes.toml b/internal/generator/vector/input/application_with_infra_includes_infra_excludes.toml index 23773bac4a..d267d815e4 100644 --- a/internal/generator/vector/input/application_with_infra_includes_infra_excludes.toml +++ b/internal/generator/vector/input/application_with_infra_includes_infra_excludes.toml @@ -19,5 +19,10 @@ type = "remap" inputs = ["input_my_app_container"] source = ''' .log_source = "container" - .log_type = "application" + # If namespace is infra, label log_type as infra + if match_any(string!(.kubernetes.namespace_name), [r'^default$', r'^openshift(-.+)?$', r'^kube(-.+)?$']) { + .log_type = "infrastructure" + } else { + .log_type = "application" + } ''' \ No newline at end of file diff --git a/internal/generator/vector/input/application_with_matchLabels.toml b/internal/generator/vector/input/application_with_matchLabels.toml index a57537f616..7e0a2e227d 100644 --- a/internal/generator/vector/input/application_with_matchLabels.toml +++ b/internal/generator/vector/input/application_with_matchLabels.toml @@ -19,5 +19,10 @@ type = "remap" inputs = ["input_my_app_container"] source = ''' .log_source = "container" - .log_type = "application" + # If namespace is infra, label log_type as infra + if match_any(string!(.kubernetes.namespace_name), [r'^default$', r'^openshift(-.+)?$', r'^kube(-.+)?$']) { + .log_type = "infrastructure" + } else { + .log_type = "application" + } ''' \ No newline at end of file diff --git a/internal/generator/vector/input/application_with_specific_infra_includes_infra_excludes.toml b/internal/generator/vector/input/application_with_specific_infra_includes_infra_excludes.toml index 0e818329a2..006c29363f 100644 --- a/internal/generator/vector/input/application_with_specific_infra_includes_infra_excludes.toml +++ b/internal/generator/vector/input/application_with_specific_infra_includes_infra_excludes.toml @@ -19,5 +19,10 @@ type = "remap" inputs = ["input_my_app_container"] source = ''' .log_source = "container" - .log_type = "application" + # If namespace is infra, label log_type as infra + if match_any(string!(.kubernetes.namespace_name), [r'^default$', r'^openshift(-.+)?$', r'^kube(-.+)?$']) { + .log_type = "infrastructure" + } else { + .log_type = "application" + } ''' \ No newline at end of file diff --git a/internal/generator/vector/input/application_with_throttle.toml b/internal/generator/vector/input/application_with_throttle.toml index 33bc3f9ee4..33eab3e269 100644 --- a/internal/generator/vector/input/application_with_throttle.toml +++ b/internal/generator/vector/input/application_with_throttle.toml @@ -18,7 +18,12 @@ type = "remap" inputs = ["input_application_container"] source = ''' .log_source = "container" - .log_type = "application" + # If namespace is infra, label log_type as infra + if match_any(string!(.kubernetes.namespace_name), [r'^default$', r'^openshift(-.+)?$', r'^kube(-.+)?$']) { + .log_type = "infrastructure" + } else { + .log_type = "application" + } ''' [transforms.input_application_container_throttle] diff --git a/internal/generator/vector/input/audit.go b/internal/generator/vector/input/audit.go index 0aaddd9dfc..2aa43be13e 100644 --- a/internal/generator/vector/input/audit.go +++ b/internal/generator/vector/input/audit.go @@ -23,7 +23,7 @@ func NewAuditAuditdSource(input obs.InputSpec, op generator.Options) ([]generato metaID := helpers.MakeID(hostID, "meta") el := []generator.Element{ sources.NewHostAuditLog(hostID), - NewLogSourceAndType(metaID, obs.AuditSourceAuditd, obs.InputTypeAudit, hostID), + NewLogSourceAndType(metaID, obs.AuditSourceAuditd, obs.InputTypeAudit, hostID, nil), } return el, []string{metaID} } @@ -33,7 +33,7 @@ func NewK8sAuditSource(input obs.InputSpec, op generator.Options) ([]generator.E metaID := helpers.MakeID(id, "meta") el := []generator.Element{ sources.NewK8sAuditLog(id), - NewLogSourceAndType(metaID, obs.AuditSourceKube, obs.InputTypeAudit, id), + NewLogSourceAndType(metaID, obs.AuditSourceKube, obs.InputTypeAudit, id, nil), } return el, []string{metaID} } @@ -43,7 +43,7 @@ func NewOpenshiftAuditSource(input obs.InputSpec, op generator.Options) ([]gener metaID := helpers.MakeID(id, "meta") el := []generator.Element{ sources.NewOpenshiftAuditLog(id), - NewLogSourceAndType(metaID, obs.AuditSourceOpenShift, obs.InputTypeAudit, id), + NewLogSourceAndType(metaID, obs.AuditSourceOpenShift, obs.InputTypeAudit, id, nil), } return el, []string{metaID} } @@ -53,7 +53,7 @@ func NewOVNAuditSource(input obs.InputSpec, op generator.Options) ([]generator.E metaID := helpers.MakeID(id, "meta") el := []generator.Element{ sources.NewOVNAuditLog(id), - NewLogSourceAndType(metaID, obs.AuditSourceOVN, obs.InputTypeAudit, id), + NewLogSourceAndType(metaID, obs.AuditSourceOVN, obs.InputTypeAudit, id, nil), } return el, []string{metaID} } diff --git a/internal/generator/vector/input/infrastructure.toml b/internal/generator/vector/input/infrastructure.toml index 44119288a1..19ae940821 100644 --- a/internal/generator/vector/input/infrastructure.toml +++ b/internal/generator/vector/input/infrastructure.toml @@ -19,7 +19,12 @@ type = "remap" inputs = ["input_infrastructure_container"] source = ''' .log_source = "container" - .log_type = "infrastructure" + # If namespace is infra, label log_type as infra + if match_any(string!(.kubernetes.namespace_name), [r'^default$', r'^openshift(-.+)?$', r'^kube(-.+)?$']) { + .log_type = "infrastructure" + } else { + .log_type = "application" + } ''' [sources.input_infrastructure_journal] diff --git a/internal/generator/vector/input/infrastructure_container.toml b/internal/generator/vector/input/infrastructure_container.toml index 5df486143a..4edc6ea404 100644 --- a/internal/generator/vector/input/infrastructure_container.toml +++ b/internal/generator/vector/input/infrastructure_container.toml @@ -19,5 +19,10 @@ type = "remap" inputs = ["input_myinfra_container"] source = ''' .log_source = "container" - .log_type = "infrastructure" + # If namespace is infra, label log_type as infra + if match_any(string!(.kubernetes.namespace_name), [r'^default$', r'^openshift(-.+)?$', r'^kube(-.+)?$']) { + .log_type = "infrastructure" + } else { + .log_type = "application" + } ''' \ No newline at end of file diff --git a/internal/generator/vector/input/journal.go b/internal/generator/vector/input/journal.go index 3cbd398402..332b41eeaa 100644 --- a/internal/generator/vector/input/journal.go +++ b/internal/generator/vector/input/journal.go @@ -12,7 +12,7 @@ func NewJournalSource(input obs.InputSpec) ([]Element, []string) { metaID := helpers.MakeID(id, "meta") el := []Element{ source.NewJournalLog(id), - NewLogSourceAndType(metaID, string(obs.InfrastructureSourceNode), string(obs.InputTypeInfrastructure), id), + NewLogSourceAndType(metaID, string(obs.InfrastructureSourceNode), string(obs.InputTypeInfrastructure), id, nil), } return el, []string{metaID} } diff --git a/internal/generator/vector/input/meta.go b/internal/generator/vector/input/meta.go index 9d85fc6033..2dbbc7efaa 100644 --- a/internal/generator/vector/input/meta.go +++ b/internal/generator/vector/input/meta.go @@ -2,15 +2,21 @@ package input import ( "fmt" + "github.com/openshift/cluster-logging-operator/internal/generator/framework" "github.com/openshift/cluster-logging-operator/internal/generator/vector/elements" "github.com/openshift/cluster-logging-operator/internal/generator/vector/helpers" ) -func NewLogSourceAndType(id string, logSource, logType interface{}, inputs string) framework.Element { - return elements.Remap{ +func NewLogSourceAndType(id string, logSource, logType interface{}, inputs string, visit func(remap *elements.Remap)) framework.Element { + ele := elements.Remap{ ComponentID: id, Inputs: helpers.MakeInputs(inputs), VRL: fmt.Sprintf(".log_source = %q\n.log_type = %q", logSource, logType), } + + if visit != nil { + visit(&ele) + } + return ele } diff --git a/internal/generator/vector/input/receiver.go b/internal/generator/vector/input/receiver.go index f3d07bfdfa..da68f2300f 100644 --- a/internal/generator/vector/input/receiver.go +++ b/internal/generator/vector/input/receiver.go @@ -22,7 +22,7 @@ func NewViaqReceiverSource(spec obs.InputSpec, resNames factory.ForwarderResourc els = append(els, source.NewSyslogSource(base, resNames.GenerateInputServiceName(spec.Name), spec), tlsConfig, - NewLogSourceAndType(metaID, obs.InfrastructureSourceNode, obs.InputTypeInfrastructure, base), + NewLogSourceAndType(metaID, obs.InfrastructureSourceNode, obs.InputTypeInfrastructure, base, nil), ) case obs.ReceiverTypeHTTP: el, id := source.NewHttpSource(base, resNames.GenerateInputServiceName(spec.Name), spec) @@ -33,7 +33,7 @@ func NewViaqReceiverSource(spec obs.InputSpec, resNames factory.ForwarderResourc tlsConfig, split, items, - NewLogSourceAndType(metaID, obs.AuditSourceKube, obs.InputTypeAudit, itemsID), + NewLogSourceAndType(metaID, obs.AuditSourceKube, obs.InputTypeAudit, itemsID, nil), ) } return els, []string{metaID} diff --git a/internal/generator/vector/input/source.go b/internal/generator/vector/input/source.go index 729a678f5d..3cca2a046e 100644 --- a/internal/generator/vector/input/source.go +++ b/internal/generator/vector/input/source.go @@ -2,17 +2,19 @@ package input import ( "fmt" + "regexp" + obs "github.com/openshift/cluster-logging-operator/api/observability/v1" internalobs "github.com/openshift/cluster-logging-operator/internal/api/observability" "github.com/openshift/cluster-logging-operator/internal/constants" "github.com/openshift/cluster-logging-operator/internal/factory" "github.com/openshift/cluster-logging-operator/internal/generator/framework" + "github.com/openshift/cluster-logging-operator/internal/generator/vector/elements" "github.com/openshift/cluster-logging-operator/internal/generator/vector/helpers" "github.com/openshift/cluster-logging-operator/internal/generator/vector/source" "github.com/openshift/cluster-logging-operator/internal/utils/sets" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/set" - "regexp" ) const ( @@ -166,7 +168,20 @@ func NewContainerSource(spec obs.InputSpec, namespace, includes, excludes string ExcludePaths: excludes, ExtraLabelSelector: source.LabelSelectorFrom(selector), }, - NewLogSourceAndType(metaID, logSource, logType, base), + NewLogSourceAndType(metaID, logSource, logType, base, func(remap *elements.Remap) { + remap.VRL = fmt.Sprintf( + ` +.log_source = %q +# If namespace is infra, label log_type as infra +if match_any(string!(.kubernetes.namespace_name), [r'^default$', r'^openshift(-.+)?$', r'^kube(-.+)?$']) { + .log_type = %q +} else { + .log_type = %q +}`, + logSource, + obs.InputTypeInfrastructure, + obs.InputTypeApplication) + }), } inputID := metaID //TODO: DETERMINE IF key field is correct and actually works