diff --git a/internal/generator/vector/output/factory_test_loki_with_throttle.toml b/internal/generator/vector/output/factory_test_loki_with_throttle.toml index 7723a3b035..016f1e5e01 100644 --- a/internal/generator/vector/output/factory_test_loki_with_throttle.toml +++ b/internal/generator/vector/output/factory_test_loki_with_throttle.toml @@ -38,11 +38,16 @@ codec = "json" except_fields = ["_internal"] [sinks.output_default_loki_apps.labels] +k8s_container_name = "{{kubernetes.container_name}}" +k8s_namespace_name = "{{kubernetes.namespace_name}}" +k8s_node_name = "${VECTOR_SELF_NODE_NAME}" +k8s_pod_name = "{{kubernetes.pod_name}}" kubernetes_container_name = "{{kubernetes.container_name}}" kubernetes_host = "${VECTOR_SELF_NODE_NAME}" kubernetes_namespace_name = "{{kubernetes.namespace_name}}" kubernetes_pod_name = "{{kubernetes.pod_name}}" log_type = "{{log_type}}" +openshift_log_type = "{{log_type}}" [sinks.output_default_loki_apps.tls] min_tls_version = "VersionTLS12" diff --git a/internal/generator/vector/output/loki/loki.go b/internal/generator/vector/output/loki/loki.go index 5b8ce8b612..c01ce3f0dc 100644 --- a/internal/generator/vector/output/loki/loki.go +++ b/internal/generator/vector/output/loki/loki.go @@ -2,9 +2,10 @@ package loki import ( "fmt" - "github.com/openshift/cluster-logging-operator/internal/api/observability" "strings" + "github.com/openshift/cluster-logging-operator/internal/api/observability" + "github.com/openshift/cluster-logging-operator/internal/generator/vector/output/common/tls" . "github.com/openshift/cluster-logging-operator/internal/generator/framework" @@ -14,7 +15,6 @@ import ( obs "github.com/openshift/cluster-logging-operator/api/observability/v1" genhelper "github.com/openshift/cluster-logging-operator/internal/generator/helpers" . "github.com/openshift/cluster-logging-operator/internal/generator/vector/elements" - "github.com/openshift/cluster-logging-operator/internal/generator/vector/helpers" vectorhelpers "github.com/openshift/cluster-logging-operator/internal/generator/vector/helpers" commontemplate "github.com/openshift/cluster-logging-operator/internal/generator/vector/output/common/template" "github.com/openshift/cluster-logging-operator/internal/utils/sets" @@ -27,6 +27,13 @@ const ( lokiLabelKubernetesHost = "kubernetes.host" lokiLabelKubernetesContainerName = "kubernetes.container_name" podNamespace = "kubernetes.namespace_name" + + // OTel + otellogType = "openshift.log_type" + otellokiLabelKubernetesNamespaceName = "k8s.namespace_name" + otellokiLabelKubernetesPodName = "k8s.pod_name" + otellokiLabelKubernetesContainerName = "k8s.container_name" + otellokiLabelKubernetesNodeName = "k8s.node_name" ) var ( @@ -38,6 +45,12 @@ var ( lokiLabelKubernetesNamespaceName, lokiLabelKubernetesPodName, lokiLabelKubernetesContainerName, + + // OTel labels + otellogType, + otellokiLabelKubernetesNamespaceName, + otellokiLabelKubernetesPodName, + otellokiLabelKubernetesContainerName, } containerLabels = []string{ @@ -47,8 +60,16 @@ var ( } requiredLabelKeys = []string{ + otellokiLabelKubernetesNodeName, lokiLabelKubernetesHost, } + + viaqOtelLabelMap = map[string]string{ + logType: otellogType, + lokiLabelKubernetesNamespaceName: otellokiLabelKubernetesNamespaceName, + lokiLabelKubernetesPodName: otellokiLabelKubernetesPodName, + lokiLabelKubernetesContainerName: otellokiLabelKubernetesContainerName, + } ) type Loki struct { @@ -173,6 +194,8 @@ func lokiLabelKeys(l *obs.Loki) []string { var keys sets.String if l != nil && len(l.LabelKeys) != 0 { keys = *sets.NewString(l.LabelKeys...) + // Determine which of the OTel labels need to also be added based on spec'd custom labels + keys.Insert(addOtelEquivalentLabels(l.LabelKeys)...) } else { keys = *sets.NewString(DefaultLabelKeys...) } @@ -190,17 +213,49 @@ func lokiLabels(lo *obs.Loki) []Label { Name: name, Value: formatLokiLabelValue(k), } - if k == lokiLabelKubernetesHost { - l.Value = "${VECTOR_SELF_NODE_NAME}" - } - if k == lokiLabelKubernetesNamespaceName { - l.Value = fmt.Sprintf("{{%s}}", podNamespace) + if val := generateCustomLabelValues(k); val != "" { + l.Value = val } ls = append(ls, l) } return ls } +// addOtelEquivalentLabels checks spec'd custom label keys to add matching otel labels +// e.g kubernetes.namespace_name = k8s.namespace_name +func addOtelEquivalentLabels(customLabelKeys []string) []string { + matchingLabels := []string{} + + for _, label := range customLabelKeys { + if val, ok := viaqOtelLabelMap[label]; ok { + matchingLabels = append(matchingLabels, val) + } + } + return matchingLabels +} + +// generateCustomLabelValues generates custom values for specific labels like kubernetes.host, k8s_* labels +func generateCustomLabelValues(value string) string { + var labelVal string + + switch value { + case otellogType: + labelVal = logType + case otellokiLabelKubernetesContainerName: + labelVal = lokiLabelKubernetesContainerName + case lokiLabelKubernetesNamespaceName, otellokiLabelKubernetesNamespaceName: + labelVal = podNamespace + case otellokiLabelKubernetesPodName: + labelVal = lokiLabelKubernetesPodName + // Special case for the kubernetes node name (same as kubernetes.host) + case lokiLabelKubernetesHost, otellokiLabelKubernetesNodeName: + return "${VECTOR_SELF_NODE_NAME}" + default: + return "" + } + return fmt.Sprintf("{{%s}}", labelVal) +} + func remapLabelsVrl(labels []string) string { k8sEventLabel := ` if !exists(.%s) { @@ -227,7 +282,7 @@ func formatLokiLabelValue(value string) string { func RemapLabels(id string, o obs.OutputSpec, inputs []string) Element { return Remap{ ComponentID: id, - Inputs: helpers.MakeInputs(inputs...), + Inputs: vectorhelpers.MakeInputs(inputs...), VRL: remapLabelsVrl(containerLabels), } } @@ -253,7 +308,7 @@ func Tenant(l *obs.Loki, tenant string) Element { func CleanupFields(id string, inputs []string) Element { return Remap{ ComponentID: id, - Inputs: helpers.MakeInputs(inputs...), + Inputs: vectorhelpers.MakeInputs(inputs...), VRL: "del(.tag)", } } diff --git a/internal/generator/vector/output/loki/with_custom_bearer_token.toml b/internal/generator/vector/output/loki/with_custom_bearer_token.toml index fb33652fe2..68a0ebe1ff 100644 --- a/internal/generator/vector/output/loki/with_custom_bearer_token.toml +++ b/internal/generator/vector/output/loki/with_custom_bearer_token.toml @@ -32,11 +32,16 @@ codec = "json" except_fields = ["_internal"] [sinks.loki_receiver.labels] +k8s_container_name = "{{kubernetes.container_name}}" +k8s_namespace_name = "{{kubernetes.namespace_name}}" +k8s_node_name = "${VECTOR_SELF_NODE_NAME}" +k8s_pod_name = "{{kubernetes.pod_name}}" kubernetes_container_name = "{{kubernetes.container_name}}" kubernetes_host = "${VECTOR_SELF_NODE_NAME}" kubernetes_namespace_name = "{{kubernetes.namespace_name}}" kubernetes_pod_name = "{{kubernetes.pod_name}}" log_type = "{{log_type}}" +openshift_log_type = "{{log_type}}" [sinks.loki_receiver.auth] strategy = "bearer" diff --git a/internal/generator/vector/output/loki/with_custom_labels.toml b/internal/generator/vector/output/loki/with_custom_labels.toml index 8985c58516..e3ed00089c 100644 --- a/internal/generator/vector/output/loki/with_custom_labels.toml +++ b/internal/generator/vector/output/loki/with_custom_labels.toml @@ -32,6 +32,8 @@ codec = "json" except_fields = ["_internal"] [sinks.loki_receiver.labels] +k8s_container_name = "{{kubernetes.container_name}}" +k8s_node_name = "${VECTOR_SELF_NODE_NAME}" kubernetes_container_name = "{{kubernetes.container_name}}" kubernetes_host = "${VECTOR_SELF_NODE_NAME}" kubernetes_labels_app = "{{kubernetes.labels.\"app\"}}" diff --git a/internal/generator/vector/output/loki/with_default_labels.toml b/internal/generator/vector/output/loki/with_default_labels.toml index a22cd8f6f7..309c5130e1 100644 --- a/internal/generator/vector/output/loki/with_default_labels.toml +++ b/internal/generator/vector/output/loki/with_default_labels.toml @@ -32,8 +32,13 @@ codec = "json" except_fields = ["_internal"] [sinks.loki_receiver.labels] +k8s_container_name = "{{kubernetes.container_name}}" +k8s_namespace_name = "{{kubernetes.namespace_name}}" +k8s_node_name = "${VECTOR_SELF_NODE_NAME}" +k8s_pod_name = "{{kubernetes.pod_name}}" kubernetes_container_name = "{{kubernetes.container_name}}" kubernetes_host = "${VECTOR_SELF_NODE_NAME}" kubernetes_namespace_name = "{{kubernetes.namespace_name}}" kubernetes_pod_name = "{{kubernetes.pod_name}}" log_type = "{{log_type}}" +openshift_log_type = "{{log_type}}" diff --git a/internal/generator/vector/output/loki/with_default_tls.toml b/internal/generator/vector/output/loki/with_default_tls.toml index 0745ba6957..44d0194953 100644 --- a/internal/generator/vector/output/loki/with_default_tls.toml +++ b/internal/generator/vector/output/loki/with_default_tls.toml @@ -32,11 +32,16 @@ codec = "json" except_fields = ["_internal"] [sinks.loki_receiver.labels] +k8s_container_name = "{{kubernetes.container_name}}" +k8s_namespace_name = "{{kubernetes.namespace_name}}" +k8s_node_name = "${VECTOR_SELF_NODE_NAME}" +k8s_pod_name = "{{kubernetes.pod_name}}" kubernetes_container_name = "{{kubernetes.container_name}}" kubernetes_host = "${VECTOR_SELF_NODE_NAME}" kubernetes_namespace_name = "{{kubernetes.namespace_name}}" kubernetes_pod_name = "{{kubernetes.pod_name}}" log_type = "{{log_type}}" +openshift_log_type = "{{log_type}}" [sinks.loki_receiver.tls] min_tls_version = "VersionTLS12" diff --git a/internal/generator/vector/output/loki/with_insecure.toml b/internal/generator/vector/output/loki/with_insecure.toml index 5382e09fb8..01e7628a47 100644 --- a/internal/generator/vector/output/loki/with_insecure.toml +++ b/internal/generator/vector/output/loki/with_insecure.toml @@ -33,11 +33,16 @@ codec = "json" except_fields = ["_internal"] [sinks.loki_receiver.labels] +k8s_container_name = "{{kubernetes.container_name}}" +k8s_namespace_name = "{{kubernetes.namespace_name}}" +k8s_node_name = "${VECTOR_SELF_NODE_NAME}" +k8s_pod_name = "{{kubernetes.pod_name}}" kubernetes_container_name = "{{kubernetes.container_name}}" kubernetes_host = "${VECTOR_SELF_NODE_NAME}" kubernetes_namespace_name = "{{kubernetes.namespace_name}}" kubernetes_pod_name = "{{kubernetes.pod_name}}" log_type = "{{log_type}}" +openshift_log_type = "{{log_type}}" [sinks.loki_receiver.tls] diff --git a/internal/generator/vector/output/loki/with_insecure_nocert.toml b/internal/generator/vector/output/loki/with_insecure_nocert.toml index e3a4259365..46bac4cd35 100644 --- a/internal/generator/vector/output/loki/with_insecure_nocert.toml +++ b/internal/generator/vector/output/loki/with_insecure_nocert.toml @@ -32,11 +32,16 @@ codec = "json" except_fields = ["_internal"] [sinks.loki_receiver.labels] +k8s_container_name = "{{kubernetes.container_name}}" +k8s_namespace_name = "{{kubernetes.namespace_name}}" +k8s_node_name = "${VECTOR_SELF_NODE_NAME}" +k8s_pod_name = "{{kubernetes.pod_name}}" kubernetes_container_name = "{{kubernetes.container_name}}" kubernetes_host = "${VECTOR_SELF_NODE_NAME}" kubernetes_namespace_name = "{{kubernetes.namespace_name}}" kubernetes_pod_name = "{{kubernetes.pod_name}}" log_type = "{{log_type}}" +openshift_log_type = "{{log_type}}" [sinks.loki_receiver.tls] diff --git a/internal/generator/vector/output/loki/with_sa_token.toml b/internal/generator/vector/output/loki/with_sa_token.toml index 5a92293473..46b70c8419 100644 --- a/internal/generator/vector/output/loki/with_sa_token.toml +++ b/internal/generator/vector/output/loki/with_sa_token.toml @@ -32,11 +32,16 @@ codec = "json" except_fields = ["_internal"] [sinks.loki_receiver.labels] +k8s_container_name = "{{kubernetes.container_name}}" +k8s_namespace_name = "{{kubernetes.namespace_name}}" +k8s_node_name = "${VECTOR_SELF_NODE_NAME}" +k8s_pod_name = "{{kubernetes.pod_name}}" kubernetes_container_name = "{{kubernetes.container_name}}" kubernetes_host = "${VECTOR_SELF_NODE_NAME}" kubernetes_namespace_name = "{{kubernetes.namespace_name}}" kubernetes_pod_name = "{{kubernetes.pod_name}}" log_type = "{{log_type}}" +openshift_log_type = "{{log_type}}" [sinks.loki_receiver.auth] strategy = "bearer" diff --git a/internal/generator/vector/output/loki/with_tenant_id.toml b/internal/generator/vector/output/loki/with_tenant_id.toml index a67c920718..b49fd55132 100644 --- a/internal/generator/vector/output/loki/with_tenant_id.toml +++ b/internal/generator/vector/output/loki/with_tenant_id.toml @@ -41,8 +41,13 @@ codec = "json" except_fields = ["_internal"] [sinks.loki_receiver.labels] +k8s_container_name = "{{kubernetes.container_name}}" +k8s_namespace_name = "{{kubernetes.namespace_name}}" +k8s_node_name = "${VECTOR_SELF_NODE_NAME}" +k8s_pod_name = "{{kubernetes.pod_name}}" kubernetes_container_name = "{{kubernetes.container_name}}" kubernetes_host = "${VECTOR_SELF_NODE_NAME}" kubernetes_namespace_name = "{{kubernetes.namespace_name}}" kubernetes_pod_name = "{{kubernetes.pod_name}}" log_type = "{{log_type}}" +openshift_log_type = "{{log_type}}" diff --git a/internal/generator/vector/output/loki/with_tls.toml b/internal/generator/vector/output/loki/with_tls.toml index ac4294cad7..f612d3002c 100644 --- a/internal/generator/vector/output/loki/with_tls.toml +++ b/internal/generator/vector/output/loki/with_tls.toml @@ -33,11 +33,16 @@ codec = "json" except_fields = ["_internal"] [sinks.loki_receiver.labels] +k8s_container_name = "{{kubernetes.container_name}}" +k8s_namespace_name = "{{kubernetes.namespace_name}}" +k8s_node_name = "${VECTOR_SELF_NODE_NAME}" +k8s_pod_name = "{{kubernetes.pod_name}}" kubernetes_container_name = "{{kubernetes.container_name}}" kubernetes_host = "${VECTOR_SELF_NODE_NAME}" kubernetes_namespace_name = "{{kubernetes.namespace_name}}" kubernetes_pod_name = "{{kubernetes.pod_name}}" log_type = "{{log_type}}" +openshift_log_type = "{{log_type}}" [sinks.loki_receiver.tls] diff --git a/internal/generator/vector/output/loki/with_tuning.toml b/internal/generator/vector/output/loki/with_tuning.toml index 4de60a2a9b..ea4c08b126 100644 --- a/internal/generator/vector/output/loki/with_tuning.toml +++ b/internal/generator/vector/output/loki/with_tuning.toml @@ -44,9 +44,14 @@ retry_initial_backoff_secs = 20 retry_max_duration_secs = 35 [sinks.loki_receiver.labels] +k8s_container_name = "{{kubernetes.container_name}}" +k8s_namespace_name = "{{kubernetes.namespace_name}}" +k8s_node_name = "${VECTOR_SELF_NODE_NAME}" +k8s_pod_name = "{{kubernetes.pod_name}}" kubernetes_container_name = "{{kubernetes.container_name}}" kubernetes_host = "${VECTOR_SELF_NODE_NAME}" kubernetes_namespace_name = "{{kubernetes.namespace_name}}" kubernetes_pod_name = "{{kubernetes.pod_name}}" log_type = "{{log_type}}" +openshift_log_type = "{{log_type}}" diff --git a/internal/generator/vector/output/loki/with_username_password.toml b/internal/generator/vector/output/loki/with_username_password.toml index f77a3049db..f233676094 100644 --- a/internal/generator/vector/output/loki/with_username_password.toml +++ b/internal/generator/vector/output/loki/with_username_password.toml @@ -32,11 +32,16 @@ codec = "json" except_fields = ["_internal"] [sinks.loki_receiver.labels] +k8s_container_name = "{{kubernetes.container_name}}" +k8s_namespace_name = "{{kubernetes.namespace_name}}" +k8s_node_name = "${VECTOR_SELF_NODE_NAME}" +k8s_pod_name = "{{kubernetes.pod_name}}" kubernetes_container_name = "{{kubernetes.container_name}}" kubernetes_host = "${VECTOR_SELF_NODE_NAME}" kubernetes_namespace_name = "{{kubernetes.namespace_name}}" kubernetes_pod_name = "{{kubernetes.pod_name}}" log_type = "{{log_type}}" +openshift_log_type = "{{log_type}}" [sinks.loki_receiver.auth] strategy = "basic" diff --git a/test/functional/outputs/loki/application_logs_vector_test.go b/test/functional/outputs/loki/application_logs_vector_test.go index e42120b6a4..d3afa4899b 100644 --- a/test/functional/outputs/loki/application_logs_vector_test.go +++ b/test/functional/outputs/loki/application_logs_vector_test.go @@ -67,9 +67,10 @@ var _ = Describe("[Functional][Outputs][Loki] Forwarding to Loki", func() { Expect(strings.Contains(lines[2], "Present days")).To(BeTrue()) }) }) - Context("when label keys are defined that include slashes and dashes. Ref(LOG-4095, LOG-4460)", func() { + + Context("labelKeys", func() { const myValue = "foobarvalue" - BeforeEach(func() { + It("should handle the configuration so the collector starts when label keys are defined that include slashes and dashes. Ref(LOG-4095, LOG-4460)", func() { f.Labels["app.kubernetes.io/name"] = myValue f.Labels["prefix-cloud_com_platform-stage"] = "dev" f.Forwarder.Spec.Outputs[0].Loki.LabelKeys = []string{ @@ -79,8 +80,6 @@ var _ = Describe("[Functional][Outputs][Loki] Forwarding to Loki", func() { "kubernetes.labels.prefix-cloud_com_platform-stage", } Expect(f.Deploy()).To(BeNil()) - }) - It("should handle the configuration so the collector starts", func() { now := time.Now() tsNow := functional.CRIOTime(now) msg := functional.NewFullCRIOLogMessage(tsNow, "Present days") @@ -95,6 +94,9 @@ var _ = Describe("[Functional][Outputs][Loki] Forwarding to Loki", func() { Expect(len(lines)).To(Equal(1)) want := map[string]string{ + "k8s_namespace_name": f.Namespace, + "k8s_pod_name": f.Pod.Name, + "k8s_node_name": f.Pod.Spec.NodeName, "kubernetes_namespace_name": f.Namespace, "kubernetes_pod_name": f.Pod.Name, "kubernetes_labels_app_kubernetes_io_name": myValue, @@ -102,6 +104,39 @@ var _ = Describe("[Functional][Outputs][Loki] Forwarding to Loki", func() { "kubernetes_host": f.Pod.Spec.NodeName, } labels := result[0].Stream + Expect(len(labels)).To(Equal(8)) + Expect(labels).To(BeEquivalentTo(want)) + }) + + It("should add all otel equivalent default labels when loki.LabelKeys are not defined", func() { + Expect(f.Deploy()).To(BeNil()) + now := time.Now() + tsNow := functional.CRIOTime(now) + msg := functional.NewFullCRIOLogMessage(tsNow, "Present days") + Expect(f.WriteMessagesToApplicationLog(msg, 1)).To(Succeed()) + + query := fmt.Sprintf(`{openshift_log_type=%q}`, obs.InputTypeApplication) + result, err := l.QueryUntil(query, "", 1) + Expect(err).To(BeNil()) + Expect(result).NotTo(BeNil()) + Expect(len(result)).To(Equal(1)) + lines := result[0].Lines() + Expect(len(lines)).To(Equal(1)) + + want := map[string]string{ + "k8s_container_name": f.Pod.Spec.Containers[0].Name, + "k8s_namespace_name": f.Namespace, + "k8s_pod_name": f.Pod.Name, + "k8s_node_name": f.Pod.Spec.NodeName, + "kubernetes_container_name": f.Pod.Spec.Containers[0].Name, + "kubernetes_namespace_name": f.Namespace, + "kubernetes_pod_name": f.Pod.Name, + "kubernetes_host": f.Pod.Spec.NodeName, + "log_type": string(obs.InputTypeApplication), + "openshift_log_type": string(obs.InputTypeApplication), + } + labels := result[0].Stream + Expect(len(labels)).To(Equal(10)) Expect(labels).To(BeEquivalentTo(want)) }) }) diff --git a/test/functional/outputs/loki/audit_logs_vector_test.go b/test/functional/outputs/loki/audit_logs_vector_test.go index 283e963cda..7f37a29c16 100644 --- a/test/functional/outputs/loki/audit_logs_vector_test.go +++ b/test/functional/outputs/loki/audit_logs_vector_test.go @@ -49,8 +49,10 @@ var _ = Describe("[Functional][Outputs][Loki] Forwarding to Loki", func() { Expect(records).To(HaveCap(1), "Exp. the record to be ingested") expLabels := map[string]string{ - "kubernetes_host": f.Pod.Spec.NodeName, - "log_type": "audit", + "kubernetes_host": f.Pod.Spec.NodeName, + "log_type": string(obs.InputTypeAudit), + "openshift_log_type": string(obs.InputTypeAudit), + "k8s_node_name": f.Pod.Spec.NodeName, } actualLabels := r[0].Stream Expect(actualLabels).To(BeEquivalentTo(expLabels), "Exp. labels to be added to the log record")