LOG-4943: Enhance Infrastructure permissions to support application l…

…og collection

internal/generator/vector/input/viaq.go

@@ -2,6 +2,8 @@ package input
 
 import (
 	"fmt"
+	"regexp"
+	"sort"
 	"strings"
 
 	logging "github.com/openshift/cluster-logging-operator/apis/logging/v1"
@@ -34,9 +36,11 @@ var (
 		Build()
 	excludeExtensions = []string{"gz", "tmp"}
 	infraNamespaces   = []string{"default", "openshift*", "kube*"}
-	infraExcludes     = source.NewContainerPathGlobBuilder().
-				AddNamespaces(infraNamespaces...).AddExtensions(excludeExtensions...).
-				Build()
+	infraNSRegex      = regexp.MustCompile(`^(?P<default>default)|(?P<openshift>openshift.*)|(?P<kube>kube.*)$`)
+
+	infraExcludes = source.NewContainerPathGlobBuilder().
+			AddNamespaces(infraNamespaces...).AddExtensions(excludeExtensions...).
+			Build()
 )
 
 // NewViaQ creates an input adapter to generate config for ViaQ sources to collect logs excluding the
@@ -67,8 +71,15 @@ func NewViaQ(input logging.InputSpec, collectorNS string, resNames *factory.Forw
 				ib.AddNamespaces(input.Application.Namespaces...)
 			} else {
 				ib.AddNamespaces(input.Application.Namespaces...)
-				eb.AddNamespaces(input.Application.ExcludeNamespaces...).
-					AddNamespaces(infraNamespaces...).
+
+				// Prune excluded infra namespaces if includes has any infra namespaces
+				finalExcludeList := append(input.Application.ExcludeNamespaces,
+					pruneInfraNS(input.Application.Namespaces)...)
+
+				// Sort outputs, because we have tests depending on the exact generated configuration
+				sort.Strings(finalExcludeList)
+
+				eb.AddNamespaces(finalExcludeList...).
 					AddExtensions(excludeExtensions...)
 			}
 			if input.Application.Containers != nil {
@@ -182,3 +193,37 @@ func NewViaqContainerSource(spec logging.InputSpec, namespace, includes, exclude
 
 	return el, []string{id}
 }
+
+// pruneInfraNS returns a pruned infra namespace list depending on which infra namespaces were included
+// since the exclusion list includes all infra namespaces by default
+// Example:
+// Include: ["openshift-logging"]
+// Default Exclude: ["default", "openshift*", "kube*"]
+// Final infra namespaces in Exclude: ["default", "kube*"]
+func pruneInfraNS(includes []string) []string {
+	foundInfraNamespaces := make(map[string]string)
+	for _, ns := range includes {
+		matches := infraNSRegex.FindStringSubmatch(ns)
+		if matches != nil {
+			for i, name := range infraNSRegex.SubexpNames() {
+				if i != 0 && matches[i] != "" {
+					foundInfraNamespaces[name] = matches[i]
+				}
+			}
+		}
+	}
+
+	infraNSSet := sets.NewString(infraNamespaces...)
+	// Remove infra namespace depending on the named capture group
+	for k := range foundInfraNamespaces {
+		switch k {
+		case "default":
+			infraNSSet.Remove("default")
+		case "openshift":
+			infraNSSet.Remove("openshift*")
+		case "kube":
+			infraNSSet.Remove("kube*")
+		}
+	}
+	return infraNSSet.List()
+}

internal/generator/vector/input/viaq_application_with_excludes.toml

@@ -4,7 +4,7 @@ type = "kubernetes_logs"
 max_read_bytes = 3145728
 glob_minimum_cooldown_ms = 15000
 auto_partial_merge = true
-exclude_paths_glob_patterns = ["/var/log/pods/test-ns1_*/*/*.log", "/var/log/pods/test-ns2_*/*/*.log", "/var/log/pods/default_*/*/*.log", "/var/log/pods/openshift*_*/*/*.log", "/var/log/pods/kube*_*/*/*.log", "/var/log/pods/*/*/*.gz", "/var/log/pods/*/*/*.tmp", "/var/log/pods/*/mesh*/*.log"]
+exclude_paths_glob_patterns = ["/var/log/pods/default_*/*/*.log", "/var/log/pods/kube*_*/*/*.log", "/var/log/pods/openshift*_*/*/*.log", "/var/log/pods/test-ns1_*/*/*.log", "/var/log/pods/test-ns2_*/*/*.log", "/var/log/pods/*/*/*.gz", "/var/log/pods/*/*/*.tmp", "/var/log/pods/*/mesh*/*.log"]
 pod_annotation_fields.pod_labels = "kubernetes.labels"
 pod_annotation_fields.pod_namespace = "kubernetes.namespace_name"
 pod_annotation_fields.pod_annotations = "kubernetes.annotations"

internal/generator/vector/input/viaq_application_with_includes_excludes.toml

@@ -5,7 +5,7 @@ max_read_bytes = 3145728
 glob_minimum_cooldown_ms = 15000
 auto_partial_merge = true
 include_paths_glob_patterns = ["/var/log/pods/test-ns-foo_*/*/*.log", "/var/log/pods/test-ns-bar_*/*/*.log"]
-exclude_paths_glob_patterns = ["/var/log/pods/test-ns1_*/*/*.log", "/var/log/pods/test-ns2_*/*/*.log", "/var/log/pods/default_*/*/*.log", "/var/log/pods/openshift*_*/*/*.log", "/var/log/pods/kube*_*/*/*.log", "/var/log/pods/*/*/*.gz", "/var/log/pods/*/*/*.tmp", "/var/log/pods/*/mesh*/*.log"]
+exclude_paths_glob_patterns = ["/var/log/pods/default_*/*/*.log", "/var/log/pods/kube*_*/*/*.log", "/var/log/pods/openshift*_*/*/*.log", "/var/log/pods/test-ns1_*/*/*.log", "/var/log/pods/test-ns2_*/*/*.log", "/var/log/pods/*/*/*.gz", "/var/log/pods/*/*/*.tmp", "/var/log/pods/*/mesh*/*.log"]
 pod_annotation_fields.pod_labels = "kubernetes.labels"
 pod_annotation_fields.pod_namespace = "kubernetes.namespace_name"
 pod_annotation_fields.pod_annotations = "kubernetes.annotations"

internal/generator/vector/input/viaq_application_with_infra_includes_excludes.toml

@@ -0,0 +1,78 @@
+# Logs from containers (including openshift containers)
+[sources.input_my_app_container]
+type = "kubernetes_logs"
+max_read_bytes = 3145728
+glob_minimum_cooldown_ms = 15000
+auto_partial_merge = true
+include_paths_glob_patterns = ["/var/log/pods/test-ns-foo_*/*/*.log", "/var/log/pods/openshift-logging_*/*/*.log", "/var/log/pods/kube-apiserver_*/*/*.log"]
+exclude_paths_glob_patterns = ["/var/log/pods/default_*/*/*.log", "/var/log/pods/test-ns1_*/*/*.log", "/var/log/pods/*/*/*.gz", "/var/log/pods/*/*/*.tmp"]
+pod_annotation_fields.pod_labels = "kubernetes.labels"
+pod_annotation_fields.pod_namespace = "kubernetes.namespace_name"
+pod_annotation_fields.pod_annotations = "kubernetes.annotations"
+pod_annotation_fields.pod_uid = "kubernetes.pod_id"
+pod_annotation_fields.pod_node_name = "hostname"
+namespace_annotation_fields.namespace_uid = "kubernetes.namespace_id"
+rotate_wait_ms = 5000
+
+[transforms.input_my_app_container_viaq]
+type = "remap"
+inputs = ["input_my_app_container"]
+source = '''
+  .openshift.cluster_id = "${OPENSHIFT_CLUSTER_ID:-}"
+   if !exists(.level) {
+    .level = "default"
+    if match!(.message, r'Warning|WARN|^W[0-9]+|level=warn|Value:warn|"level":"warn"|<warn>') {
+      .level = "warn"
+    } else if match!(.message, r'Error|ERROR|^E[0-9]+|level=error|Value:error|"level":"error"|<error>') {
+      .level = "error"
+    } else if match!(.message, r'Critical|CRITICAL|^C[0-9]+|level=critical|Value:critical|"level":"critical"|<critical>') {
+      .level = "critical"
+    } else if match!(.message, r'Debug|DEBUG|^D[0-9]+|level=debug|Value:debug|"level":"debug"|<debug>') {
+      .level = "debug"
+    } else if match!(.message, r'Notice|NOTICE|^N[0-9]+|level=notice|Value:notice|"level":"notice"|<notice>') {
+      .level = "notice"
+    } else if match!(.message, r'Alert|ALERT|^A[0-9]+|level=alert|Value:alert|"level":"alert"|<alert>') {
+      .level = "alert"
+    } else if match!(.message, r'Emergency|EMERGENCY|^EM[0-9]+|level=emergency|Value:emergency|"level":"emergency"|<emergency>') {
+      .level = "emergency"
+    } else if match!(.message, r'(?i)\b(?:info)\b|^I[0-9]+|level=info|Value:info|"level":"info"|<info>') {
+      .level = "info"
+    }
+  }
+  pod_name = string!(.kubernetes.pod_name)
+  if starts_with(pod_name, "eventrouter-") {
+    parsed, err = parse_json(.message)
+    if err != null {
+      log("Unable to process EventRouter log: " + err, level: "info")
+    } else {
+      ., err = merge(.,parsed)
+      if err == null && exists(.event) && is_object(.event) {
+          if exists(.verb) {
+            .event.verb = .verb
+            del(.verb)
+          }
+          .kubernetes.event = del(.event)
+          .message = del(.kubernetes.event.message)
+          set!(., ["@timestamp"], .kubernetes.event.metadata.creationTimestamp)
+          del(.kubernetes.event.metadata.creationTimestamp)
+		  . = compact(., nullish: true)
+      } else {
+        log("Unable to merge EventRouter log message into record: " + err, level: "info")
+      }
+    }
+  }
+  del(.source_type)
+  del(.stream)
+  del(.kubernetes.pod_ips)
+  del(.kubernetes.node_labels)
+  del(.timestamp_end)
+  ts = del(.timestamp); if !exists(."@timestamp") {."@timestamp" = ts}
+'''
+
+# Set log_type
+[transforms.input_my_app_viaq_logtype]
+type = "remap"
+inputs = ["input_my_app_container_viaq"]
+source = '''
+  .log_type = "application"
+'''

internal/generator/vector/input/viaq_application_with_infra_includes_infra_excludes.toml

@@ -0,0 +1,78 @@
+# Logs from containers (including openshift containers)
+[sources.input_my_app_container]
+type = "kubernetes_logs"
+max_read_bytes = 3145728
+glob_minimum_cooldown_ms = 15000
+auto_partial_merge = true
+include_paths_glob_patterns = ["/var/log/pods/test-ns-foo_*/*/*.log", "/var/log/pods/openshift*_*/*/*.log", "/var/log/pods/kube-apiserver_*/*/*.log"]
+exclude_paths_glob_patterns = ["/var/log/pods/default_*/*/*.log", "/var/log/pods/openshift-logging_*/*/*.log", "/var/log/pods/test-ns1_*/*/*.log", "/var/log/pods/*/*/*.gz", "/var/log/pods/*/*/*.tmp"]
+pod_annotation_fields.pod_labels = "kubernetes.labels"
+pod_annotation_fields.pod_namespace = "kubernetes.namespace_name"
+pod_annotation_fields.pod_annotations = "kubernetes.annotations"
+pod_annotation_fields.pod_uid = "kubernetes.pod_id"
+pod_annotation_fields.pod_node_name = "hostname"
+namespace_annotation_fields.namespace_uid = "kubernetes.namespace_id"
+rotate_wait_ms = 5000
+
+[transforms.input_my_app_container_viaq]
+type = "remap"
+inputs = ["input_my_app_container"]
+source = '''
+  .openshift.cluster_id = "${OPENSHIFT_CLUSTER_ID:-}"
+   if !exists(.level) {
+    .level = "default"
+    if match!(.message, r'Warning|WARN|^W[0-9]+|level=warn|Value:warn|"level":"warn"|<warn>') {
+      .level = "warn"
+    } else if match!(.message, r'Error|ERROR|^E[0-9]+|level=error|Value:error|"level":"error"|<error>') {
+      .level = "error"
+    } else if match!(.message, r'Critical|CRITICAL|^C[0-9]+|level=critical|Value:critical|"level":"critical"|<critical>') {
+      .level = "critical"
+    } else if match!(.message, r'Debug|DEBUG|^D[0-9]+|level=debug|Value:debug|"level":"debug"|<debug>') {
+      .level = "debug"
+    } else if match!(.message, r'Notice|NOTICE|^N[0-9]+|level=notice|Value:notice|"level":"notice"|<notice>') {
+      .level = "notice"
+    } else if match!(.message, r'Alert|ALERT|^A[0-9]+|level=alert|Value:alert|"level":"alert"|<alert>') {
+      .level = "alert"
+    } else if match!(.message, r'Emergency|EMERGENCY|^EM[0-9]+|level=emergency|Value:emergency|"level":"emergency"|<emergency>') {
+      .level = "emergency"
+    } else if match!(.message, r'(?i)\b(?:info)\b|^I[0-9]+|level=info|Value:info|"level":"info"|<info>') {
+      .level = "info"
+    }
+  }
+  pod_name = string!(.kubernetes.pod_name)
+  if starts_with(pod_name, "eventrouter-") {
+    parsed, err = parse_json(.message)
+    if err != null {
+      log("Unable to process EventRouter log: " + err, level: "info")
+    } else {
+      ., err = merge(.,parsed)
+      if err == null && exists(.event) && is_object(.event) {
+          if exists(.verb) {
+            .event.verb = .verb
+            del(.verb)
+          }
+          .kubernetes.event = del(.event)
+          .message = del(.kubernetes.event.message)
+          set!(., ["@timestamp"], .kubernetes.event.metadata.creationTimestamp)
+          del(.kubernetes.event.metadata.creationTimestamp)
+		  . = compact(., nullish: true)
+      } else {
+        log("Unable to merge EventRouter log message into record: " + err, level: "info")
+      }
+    }
+  }
+  del(.source_type)
+  del(.stream)
+  del(.kubernetes.pod_ips)
+  del(.kubernetes.node_labels)
+  del(.timestamp_end)
+  ts = del(.timestamp); if !exists(."@timestamp") {."@timestamp" = ts}
+'''
+
+# Set log_type
+[transforms.input_my_app_viaq_logtype]
+type = "remap"
+inputs = ["input_my_app_container_viaq"]
+source = '''
+  .log_type = "application"
+'''

internal/generator/vector/input/viaq_application_with_matchLabels.toml

@@ -4,7 +4,7 @@ type = "kubernetes_logs"
 max_read_bytes = 3145728
 glob_minimum_cooldown_ms = 15000
 auto_partial_merge = true
-exclude_paths_glob_patterns = ["/var/log/pods/default_*/*/*.log", "/var/log/pods/openshift*_*/*/*.log", "/var/log/pods/kube*_*/*/*.log", "/var/log/pods/*/*/*.gz", "/var/log/pods/*/*/*.tmp"]
+exclude_paths_glob_patterns = ["/var/log/pods/default_*/*/*.log", "/var/log/pods/kube*_*/*/*.log", "/var/log/pods/openshift*_*/*/*.log", "/var/log/pods/*/*/*.gz", "/var/log/pods/*/*/*.tmp"]
 extra_label_selector = "key1=value1,key2=value2"
 pod_annotation_fields.pod_labels = "kubernetes.labels"
 pod_annotation_fields.pod_namespace = "kubernetes.namespace_name"

internal/generator/vector/input/viaq_application_with_specific_infra_includes_infra_excludes.toml

@@ -0,0 +1,78 @@
+# Logs from containers (including openshift containers)
+[sources.input_my_app_container]
+type = "kubernetes_logs"
+max_read_bytes = 3145728
+glob_minimum_cooldown_ms = 15000
+auto_partial_merge = true
+include_paths_glob_patterns = ["/var/log/pods/test-ns-foo_*/*/*.log", "/var/log/pods/openshift-logging_*/*/*.log", "/var/log/pods/kube-apiserver_*/*/*.log"]
+exclude_paths_glob_patterns = ["/var/log/pods/default_*/*/*.log", "/var/log/pods/openshift*_*/*/*.log", "/var/log/pods/test-ns1_*/*/*.log", "/var/log/pods/*/*/*.gz", "/var/log/pods/*/*/*.tmp"]
+pod_annotation_fields.pod_labels = "kubernetes.labels"
+pod_annotation_fields.pod_namespace = "kubernetes.namespace_name"
+pod_annotation_fields.pod_annotations = "kubernetes.annotations"
+pod_annotation_fields.pod_uid = "kubernetes.pod_id"
+pod_annotation_fields.pod_node_name = "hostname"
+namespace_annotation_fields.namespace_uid = "kubernetes.namespace_id"
+rotate_wait_ms = 5000
+
+[transforms.input_my_app_container_viaq]
+type = "remap"
+inputs = ["input_my_app_container"]
+source = '''
+  .openshift.cluster_id = "${OPENSHIFT_CLUSTER_ID:-}"
+   if !exists(.level) {
+    .level = "default"
+    if match!(.message, r'Warning|WARN|^W[0-9]+|level=warn|Value:warn|"level":"warn"|<warn>') {
+      .level = "warn"
+    } else if match!(.message, r'Error|ERROR|^E[0-9]+|level=error|Value:error|"level":"error"|<error>') {
+      .level = "error"
+    } else if match!(.message, r'Critical|CRITICAL|^C[0-9]+|level=critical|Value:critical|"level":"critical"|<critical>') {
+      .level = "critical"
+    } else if match!(.message, r'Debug|DEBUG|^D[0-9]+|level=debug|Value:debug|"level":"debug"|<debug>') {
+      .level = "debug"
+    } else if match!(.message, r'Notice|NOTICE|^N[0-9]+|level=notice|Value:notice|"level":"notice"|<notice>') {
+      .level = "notice"
+    } else if match!(.message, r'Alert|ALERT|^A[0-9]+|level=alert|Value:alert|"level":"alert"|<alert>') {
+      .level = "alert"
+    } else if match!(.message, r'Emergency|EMERGENCY|^EM[0-9]+|level=emergency|Value:emergency|"level":"emergency"|<emergency>') {
+      .level = "emergency"
+    } else if match!(.message, r'(?i)\b(?:info)\b|^I[0-9]+|level=info|Value:info|"level":"info"|<info>') {
+      .level = "info"
+    }
+  }
+  pod_name = string!(.kubernetes.pod_name)
+  if starts_with(pod_name, "eventrouter-") {
+    parsed, err = parse_json(.message)
+    if err != null {
+      log("Unable to process EventRouter log: " + err, level: "info")
+    } else {
+      ., err = merge(.,parsed)
+      if err == null && exists(.event) && is_object(.event) {
+          if exists(.verb) {
+            .event.verb = .verb
+            del(.verb)
+          }
+          .kubernetes.event = del(.event)
+          .message = del(.kubernetes.event.message)
+          set!(., ["@timestamp"], .kubernetes.event.metadata.creationTimestamp)
+          del(.kubernetes.event.metadata.creationTimestamp)
+		  . = compact(., nullish: true)
+      } else {
+        log("Unable to merge EventRouter log message into record: " + err, level: "info")
+      }
+    }
+  }
+  del(.source_type)
+  del(.stream)
+  del(.kubernetes.pod_ips)
+  del(.kubernetes.node_labels)
+  del(.timestamp_end)
+  ts = del(.timestamp); if !exists(."@timestamp") {."@timestamp" = ts}
+'''
+
+# Set log_type
+[transforms.input_my_app_viaq_logtype]
+type = "remap"
+inputs = ["input_my_app_container_viaq"]
+source = '''
+  .log_type = "application"
+'''

internal/generator/vector/input/viaq_test.go

@@ -79,6 +79,33 @@ var _ = Describe("inputs", func() {
 		},
 			"viaq_application_with_includes_excludes.toml",
 		),
+		Entry("with an application that specs infra namespaces and exclude namespaces", logging.InputSpec{
+			Name: "my-app",
+			Application: &logging.Application{
+				Namespaces:        []string{"test-ns-foo", "openshift-logging", "kube-apiserver"},
+				ExcludeNamespaces: []string{"test-ns1"},
+			},
+		},
+			"viaq_application_with_infra_includes_excludes.toml",
+		),
+		Entry("with an application that specs infra namespaces and excludes infra namespaces", logging.InputSpec{
+			Name: "my-app",
+			Application: &logging.Application{
+				Namespaces:        []string{"test-ns-foo", "openshift*", "kube-apiserver"},
+				ExcludeNamespaces: []string{"test-ns1", "openshift-logging"},
+			},
+		},
+			"viaq_application_with_infra_includes_infra_excludes.toml",
+		),
+		Entry("with an application that specs specific infra namespace and excludes infra namespaces", logging.InputSpec{
+			Name: "my-app",
+			Application: &logging.Application{
+				Namespaces:        []string{"test-ns-foo", "openshift-logging", "kube-apiserver"},
+				ExcludeNamespaces: []string{"test-ns1", "openshift*"},
+			},
+		},
+			"viaq_application_with_specific_infra_includes_infra_excludes.toml",
+		),
 		Entry("with an application that specs specific match labels", logging.InputSpec{
 			Name: "my-app",
 			Application: &logging.Application{