Skip to content

Commit

Permalink
Merge pull request #2031 from jcantrill/log4095_on_master
Browse files Browse the repository at this point in the history 
LOG-4095: fix loki labelKeys that contain slash
  • Loading branch information
@openshift-merge-robot
openshift-merge-robot committed Jun 1, 2023
2 parents 6eaed5d + 586cabc commit 9d278f4
Show file tree
Hide file tree
Showing 6 changed files with 114 additions and 49 deletions.
15 changes: 7 additions & 8 deletions apis/logging/v1/output_types.go
View file
Expand Up @@ -233,19 +233,18 @@ type Loki struct {
// +optional
TenantKey string `json:"tenantKey,omitempty"`

// LabelKeys is a list of meta-data field keys to replace the default Loki labels.
//
// Loki label names must match the regular expression "[a-zA-Z_:][a-zA-Z0-9_:]*".
// Illegal characters in meta-data keys are replaced with "_" to form the label name.
// For example meta-data key "kubernetes.labels.foo" becomes Loki label "kubernetes_labels_foo".
// LabelKeys is a list of log record keys that will be used as Loki labels with the corresponding log record value.
//
// If LabelKeys is not set, the default keys are `[log_type, kubernetes.namespace_name, kubernetes.pod_name, kubernetes_host]`
// These keys are translated to Loki labels by replacing '.' with '_' as: `log_type`, `kubernetes_namespace_name`, `kubernetes_pod_name`, `kubernetes_host`
// Note that not all logs will include all of these keys: audit logs and infrastructure journal logs do not have namespace or pod name.
//
// Note: Loki label names must match the regular expression "[a-zA-Z_:][a-zA-Z0-9_:]*"
// Log record keys may contain characters like "." and "/" that are not allowed in Loki labels.
// Log record keys are translated to Loki labels by replacing any illegal characters with '_'.
// For example the default log record keys translate to these Loki labels: `log_type`, `kubernetes_namespace_name`, `kubernetes_pod_name`, `kubernetes_host`
//
// Note: the set of labels should be small, Loki imposes limits on the size and number of labels allowed.
// See https://grafana.com/docs/loki/latest/configuration/#limits_config for more.
// You can still query based on any log record field using query filters.
// Loki queries can also query based on any log record field (not just labels) using query filters.
//
// +optional
LabelKeys []string `json:"labelKeys,omitempty"`
Expand Down
34 changes: 16 additions & 18 deletions bundle/manifests/logging.openshift.io_clusterlogforwarders.yaml
View file
Expand Up @@ -269,24 +269,22 @@ spec:
loki`'
properties:
labelKeys:
description: "LabelKeys is a list of meta-data field keys
to replace the default Loki labels. \n Loki label names
must match the regular expression \"[a-zA-Z_:][a-zA-Z0-9_:]*\".
Illegal characters in meta-data keys are replaced with
\"_\" to form the label name. For example meta-data key
\"kubernetes.labels.foo\" becomes Loki label \"kubernetes_labels_foo\".
\n If LabelKeys is not set, the default keys are `[log_type,
kubernetes.namespace_name, kubernetes.pod_name, kubernetes_host]`
These keys are translated to Loki labels by replacing
'.' with '_' as: `log_type`, `kubernetes_namespace_name`,
`kubernetes_pod_name`, `kubernetes_host` Note that not
all logs will include all of these keys: audit logs and
infrastructure journal logs do not have namespace or pod
name. \n Note: the set of labels should be small, Loki
imposes limits on the size and number of labels allowed.
See https://grafana.com/docs/loki/latest/configuration/#limits_config
for more. You can still query based on any log record
field using query filters."
description: "LabelKeys is a list of log record keys that
will be used as Loki labels with the corresponding log
record value. \n If LabelKeys is not set, the default
keys are `[log_type, kubernetes.namespace_name, kubernetes.pod_name,
kubernetes_host]` \n Note: Loki label names must match
the regular expression \"[a-zA-Z_:][a-zA-Z0-9_:]*\" Log
record keys may contain characters like \".\" and \"/\"
that are not allowed in Loki labels. Log record keys are
translated to Loki labels by replacing any illegal characters
with '_'. For example the default log record keys translate
to these Loki labels: `log_type`, `kubernetes_namespace_name`,
`kubernetes_pod_name`, `kubernetes_host` \n Note: the
set of labels should be small, Loki imposes limits on
the size and number of labels allowed. See https://grafana.com/docs/loki/latest/configuration/#limits_config
for more. Loki queries can also query based on any log
record field (not just labels) using query filters."
items:
type: string
type: array
Expand Down
34 changes: 16 additions & 18 deletions config/crd/bases/logging.openshift.io_clusterlogforwarders.yaml
View file
Expand Up @@ -265,24 +265,22 @@ spec:
loki`'
properties:
labelKeys:
description: "LabelKeys is a list of meta-data field keys
to replace the default Loki labels. \n Loki label names
must match the regular expression \"[a-zA-Z_:][a-zA-Z0-9_:]*\".
Illegal characters in meta-data keys are replaced with
\"_\" to form the label name. For example meta-data key
\"kubernetes.labels.foo\" becomes Loki label \"kubernetes_labels_foo\".
\n If LabelKeys is not set, the default keys are `[log_type,
kubernetes.namespace_name, kubernetes.pod_name, kubernetes_host]`
These keys are translated to Loki labels by replacing
'.' with '_' as: `log_type`, `kubernetes_namespace_name`,
`kubernetes_pod_name`, `kubernetes_host` Note that not
all logs will include all of these keys: audit logs and
infrastructure journal logs do not have namespace or pod
name. \n Note: the set of labels should be small, Loki
imposes limits on the size and number of labels allowed.
See https://grafana.com/docs/loki/latest/configuration/#limits_config
for more. You can still query based on any log record
field using query filters."
description: "LabelKeys is a list of log record keys that
will be used as Loki labels with the corresponding log
record value. \n If LabelKeys is not set, the default
keys are `[log_type, kubernetes.namespace_name, kubernetes.pod_name,
kubernetes_host]` \n Note: Loki label names must match
the regular expression \"[a-zA-Z_:][a-zA-Z0-9_:]*\" Log
record keys may contain characters like \".\" and \"/\"
that are not allowed in Loki labels. Log record keys are
translated to Loki labels by replacing any illegal characters
with '_'. For example the default log record keys translate
to these Loki labels: `log_type`, `kubernetes_namespace_name`,
`kubernetes_pod_name`, `kubernetes_host` \n Note: the
set of labels should be small, Loki imposes limits on
the size and number of labels allowed. See https://grafana.com/docs/loki/latest/configuration/#limits_config
for more. Loki queries can also query based on any log
record field (not just labels) using query filters."
items:
type: string
type: array
Expand Down
35 changes: 33 additions & 2 deletions docs/reference/operator/api.adoc
View file
Expand Up @@ -1036,7 +1036,9 @@ This is the struct that will contain information pertinent to Log visualization
|Property|Type|Description

|kibana|object| **(DEPRECATED)** *(optional)* Specification of the Kibana Visualization component
|nodeSelector|object| Define which Nodes the Pods are scheduled on.
|ocpConsole|object| *(optional)* OCPConsole is the specification for the OCP console plugin
|tolerations|array| *(optional)* Define the tolerations the Pods will accept
|type|string| The type of Visualization to configure
|======================

Expand All @@ -1050,11 +1052,11 @@ This is the struct that will contain information pertinent to Log visualization
|======================
|Property|Type|Description

|nodeSelector|object| Define which Nodes the Pods are scheduled on.
|nodeSelector|object| **(DEPRECATED)** Define which Nodes the Pods are scheduled on.
|proxy|object| Specification of the Kibana Proxy component
|replicas|int| *(optional)* Number of instances to deploy for a Kibana deployment
|resources|object| *(optional)* The resource requirements for Kibana
|tolerations|array|
|tolerations|array| **(DEPRECATED)** Define the tolerations the Pods will accept
|======================

=== .spec.visualization.kibana.nodeSelector
Expand Down Expand Up @@ -1157,6 +1159,12 @@ This is the struct that will contain information pertinent to Log visualization
===== Type
* int

=== .spec.visualization.nodeSelector
===== Description

===== Type
* object

=== .spec.visualization.ocpConsole
===== Description

Expand All @@ -1171,6 +1179,29 @@ This is the struct that will contain information pertinent to Log visualization
|timeout|string| *(optional)* Timeout is the max duration before a query timeout
|======================

=== .spec.visualization.tolerations[]
===== Description

===== Type
* array

[options="header"]
|======================
|Property|Type|Description

|effect|string| *(optional)* Effect indicates the taint effect to match. Empty means match all taint effects.
|key|string| *(optional)* Key is the taint key that the toleration applies to. Empty means match all taint keys.
|operator|string| *(optional)* Operator represents a key's relationship to the value.
|tolerationSeconds|int| *(optional)* TolerationSeconds represents the period of time the toleration (which must be
|value|string| *(optional)* Value is the taint value the toleration matches to.
|======================

=== .spec.visualization.tolerations[].tolerationSeconds
===== Description

===== Type
* int

=== .status
===== Description
ClusterLoggingStatus defines the observed state of ClusterLogging
Expand Down
15 changes: 13 additions & 2 deletions internal/generator/vector/output/loki/loki.go
View file
Expand Up @@ -160,9 +160,11 @@ func lokiLabelKeys(l *logging.Loki) []string {
func lokiLabels(lo *logging.Loki) []Label {
ls := []Label{}
for _, k := range lokiLabelKeys(lo) {
name := strings.ReplaceAll(k, ".", "_")
name = strings.ReplaceAll(name, "/", "_")
l := Label{
Name: strings.ReplaceAll(k, ".", "_"),
Value: fmt.Sprintf("{{%s}}", k),
Name: name,
Value: formatLokiLabelValue(k),
}
if k == lokiLabelKubernetesHost {
l.Value = "${VECTOR_SELF_NODE_NAME}"
Expand All @@ -175,6 +177,15 @@ func lokiLabels(lo *logging.Loki) []Label {
return ls
}

func formatLokiLabelValue(value string) string {
if strings.HasPrefix(value, "kubernetes.labels.") || strings.HasPrefix(value, "kubernetes.namespace_labels.") {
parts := strings.SplitAfterN(value, ".", 2)
key := strings.ReplaceAll(parts[1], "/", "_")
value = fmt.Sprintf("%s%s", parts[0], key)
}
return fmt.Sprintf("{{%s}}", value)
}

func Labels(o logging.OutputSpec) Element {
return LokiLabels{
ComponentID: strings.ToLower(vectorhelpers.Replacer.Replace(o.Name)),
Expand Down
30 changes: 29 additions & 1 deletion test/functional/outputs/loki/application_logs_vector_test.go
View file
Expand Up @@ -46,14 +46,16 @@ var _ = Describe("[Functional][Outputs][Loki] Forwarding to Loki", func() {
Labels: map[string]string{"logging": "logging-value"},
})

Expect(f.Deploy()).To(BeNil())
})

AfterEach(func() {
f.Cleanup()
})

Context("with vector not ordered events", func() {
BeforeEach(func() {
Expect(f.Deploy()).To(BeNil())
})
It("should accept not ordered event", func() {
now := time.Now()
tsNow := functional.CRIOTime(now)
Expand All @@ -79,5 +81,31 @@ var _ = Describe("[Functional][Outputs][Loki] Forwarding to Loki", func() {
Expect(strings.Contains(lines[2], "Present days")).To(BeTrue())
})
})
Context("when label keys are defined that include slashes. Ref LOG-4095", func() {
const myValue = "foobarvalue"
BeforeEach(func() {
f.Labels["foo/bar"] = myValue
f.Forwarder.Spec.Outputs[0].Loki.LabelKeys = []string{
"kubernetes.namespace_name",
"kubernetes.pod_name",
"kubernetes.labels.foo/bar",
}
Expect(f.Deploy()).To(BeNil())
})
It("should handle the configuration so the collector starts", func() {
now := time.Now()
tsNow := functional.CRIOTime(now)
msg := functional.NewFullCRIOLogMessage(tsNow, "Present days")
Expect(f.WriteMessagesToApplicationLog(msg, 1)).To(Succeed())

query := fmt.Sprintf(`{kubernetes_labels_foo_bar=%q}`, myValue)
result, err := l.QueryUntil(query, "", 1)
Expect(err).To(BeNil())
Expect(result).NotTo(BeNil())
Expect(len(result)).To(Equal(1))
lines := result[0].Lines()
Expect(len(lines)).To(Equal(1))
})
})

})

0 comments on commit 9d278f4

Please sign in to comment.