Skip to content

Commit

Permalink
Merge pull request #2023 from jcantrill/log4095
Browse files Browse the repository at this point in the history 
LOG-4095: fix loki labelKeys that contain slash
  • Loading branch information
@openshift-merge-robot
openshift-merge-robot committed May 31, 2023
2 parents 91403ff + d767496 commit 0c7f79d
Show file tree
Hide file tree
Showing 5 changed files with 81 additions and 47 deletions.
15 changes: 7 additions & 8 deletions apis/logging/v1/output_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -233,19 +233,18 @@ type Loki struct {
// +optional
TenantKey string `json:"tenantKey,omitempty"`

// LabelKeys is a list of meta-data field keys to replace the default Loki labels.
//
// Loki label names must match the regular expression "[a-zA-Z_:][a-zA-Z0-9_:]*".
// Illegal characters in meta-data keys are replaced with "_" to form the label name.
// For example meta-data key "kubernetes.labels.foo" becomes Loki label "kubernetes_labels_foo".
// LabelKeys is a list of log record keys that will be used as Loki labels with the corresponding log record value.
//
// If LabelKeys is not set, the default keys are `[log_type, kubernetes.namespace_name, kubernetes.pod_name, kubernetes_host]`
// These keys are translated to Loki labels by replacing '.' with '_' as: `log_type`, `kubernetes_namespace_name`, `kubernetes_pod_name`, `kubernetes_host`
// Note that not all logs will include all of these keys: audit logs and infrastructure journal logs do not have namespace or pod name.
//
// Note: Loki label names must match the regular expression "[a-zA-Z_:][a-zA-Z0-9_:]*"
// Log record keys may contain characters like "." and "/" that are not allowed in Loki labels.
// Log record keys are translated to Loki labels by replacing any illegal characters with '_'.
// For example the default log record keys translate to these Loki labels: `log_type`, `kubernetes_namespace_name`, `kubernetes_pod_name`, `kubernetes_host`
//
// Note: the set of labels should be small, Loki imposes limits on the size and number of labels allowed.
// See https://grafana.com/docs/loki/latest/configuration/#limits_config for more.
// You can still query based on any log record field using query filters.
// Loki queries can also query based on any log record field (not just labels) using query filters.
//
// +optional
LabelKeys []string `json:"labelKeys,omitempty"`
Expand Down
34 changes: 16 additions & 18 deletions bundle/manifests/logging.openshift.io_clusterlogforwarders.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -269,24 +269,22 @@ spec:
loki`'
properties:
labelKeys:
description: "LabelKeys is a list of meta-data field keys
to replace the default Loki labels. \n Loki label names
must match the regular expression \"[a-zA-Z_:][a-zA-Z0-9_:]*\".
Illegal characters in meta-data keys are replaced with
\"_\" to form the label name. For example meta-data key
\"kubernetes.labels.foo\" becomes Loki label \"kubernetes_labels_foo\".
\n If LabelKeys is not set, the default keys are `[log_type,
kubernetes.namespace_name, kubernetes.pod_name, kubernetes_host]`
These keys are translated to Loki labels by replacing
'.' with '_' as: `log_type`, `kubernetes_namespace_name`,
`kubernetes_pod_name`, `kubernetes_host` Note that not
all logs will include all of these keys: audit logs and
infrastructure journal logs do not have namespace or pod
name. \n Note: the set of labels should be small, Loki
imposes limits on the size and number of labels allowed.
See https://grafana.com/docs/loki/latest/configuration/#limits_config
for more. You can still query based on any log record
field using query filters."
description: "LabelKeys is a list of log record keys that
will be used as Loki labels with the corresponding log
record value. \n If LabelKeys is not set, the default
keys are `[log_type, kubernetes.namespace_name, kubernetes.pod_name,
kubernetes_host]` \n Note: Loki label names must match
the regular expression \"[a-zA-Z_:][a-zA-Z0-9_:]*\" Log
record keys may contain characters like \".\" and \"/\"
that are not allowed in Loki labels. Log record keys are
translated to Loki labels by replacing any illegal characters
with '_'. For example the default log record keys translate
to these Loki labels: `log_type`, `kubernetes_namespace_name`,
`kubernetes_pod_name`, `kubernetes_host` \n Note: the
set of labels should be small, Loki imposes limits on
the size and number of labels allowed. See https://grafana.com/docs/loki/latest/configuration/#limits_config
for more. Loki queries can also query based on any log
record field (not just labels) using query filters."
items:
type: string
type: array
Expand Down
34 changes: 16 additions & 18 deletions config/crd/bases/logging.openshift.io_clusterlogforwarders.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -265,24 +265,22 @@ spec:
loki`'
properties:
labelKeys:
description: "LabelKeys is a list of meta-data field keys
to replace the default Loki labels. \n Loki label names
must match the regular expression \"[a-zA-Z_:][a-zA-Z0-9_:]*\".
Illegal characters in meta-data keys are replaced with
\"_\" to form the label name. For example meta-data key
\"kubernetes.labels.foo\" becomes Loki label \"kubernetes_labels_foo\".
\n If LabelKeys is not set, the default keys are `[log_type,
kubernetes.namespace_name, kubernetes.pod_name, kubernetes_host]`
These keys are translated to Loki labels by replacing
'.' with '_' as: `log_type`, `kubernetes_namespace_name`,
`kubernetes_pod_name`, `kubernetes_host` Note that not
all logs will include all of these keys: audit logs and
infrastructure journal logs do not have namespace or pod
name. \n Note: the set of labels should be small, Loki
imposes limits on the size and number of labels allowed.
See https://grafana.com/docs/loki/latest/configuration/#limits_config
for more. You can still query based on any log record
field using query filters."
description: "LabelKeys is a list of log record keys that
will be used as Loki labels with the corresponding log
record value. \n If LabelKeys is not set, the default
keys are `[log_type, kubernetes.namespace_name, kubernetes.pod_name,
kubernetes_host]` \n Note: Loki label names must match
the regular expression \"[a-zA-Z_:][a-zA-Z0-9_:]*\" Log
record keys may contain characters like \".\" and \"/\"
that are not allowed in Loki labels. Log record keys are
translated to Loki labels by replacing any illegal characters
with '_'. For example the default log record keys translate
to these Loki labels: `log_type`, `kubernetes_namespace_name`,
`kubernetes_pod_name`, `kubernetes_host` \n Note: the
set of labels should be small, Loki imposes limits on
the size and number of labels allowed. See https://grafana.com/docs/loki/latest/configuration/#limits_config
for more. Loki queries can also query based on any log
record field (not just labels) using query filters."
items:
type: string
type: array
Expand Down
15 changes: 13 additions & 2 deletions internal/generator/vector/output/loki/loki.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,9 +160,11 @@ func lokiLabelKeys(l *logging.Loki) []string {
func lokiLabels(lo *logging.Loki) []Label {
ls := []Label{}
for _, k := range lokiLabelKeys(lo) {
name := strings.ReplaceAll(k, ".", "_")
name = strings.ReplaceAll(name, "/", "_")
l := Label{
Name: strings.ReplaceAll(k, ".", "_"),
Value: fmt.Sprintf("{{%s}}", k),
Name: name,
Value: formatLokiLabelValue(k),
}
if k == lokiLabelKubernetesHost {
l.Value = "${VECTOR_SELF_NODE_NAME}"
Expand All @@ -175,6 +177,15 @@ func lokiLabels(lo *logging.Loki) []Label {
return ls
}

func formatLokiLabelValue(value string) string {
if strings.HasPrefix(value, "kubernetes.labels.") || strings.HasPrefix(value, "kubernetes.namespace_labels.") {
parts := strings.SplitAfterN(value, ".", 2)
key := strings.ReplaceAll(parts[1], "/", "_")
value = fmt.Sprintf("%s%s", parts[0], key)
}
return fmt.Sprintf("{{%s}}", value)
}

func Labels(o logging.OutputSpec) Element {
return LokiLabels{
ComponentID: vectorhelpers.FormatComponentID(o.Name),
Expand Down
30 changes: 29 additions & 1 deletion test/functional/outputs/loki/application_logs_vector_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,14 +46,16 @@ var _ = Describe("[Functional][Outputs][Loki] Forwarding to Loki", func() {
Labels: map[string]string{"logging": "logging-value"},
})

Expect(f.Deploy()).To(BeNil())
})

AfterEach(func() {
f.Cleanup()
})

Context("with vector not ordered events", func() {
BeforeEach(func() {
Expect(f.Deploy()).To(BeNil())
})
It("should accept not ordered event", func() {
now := time.Now()
tsNow := functional.CRIOTime(now)
Expand All @@ -79,5 +81,31 @@ var _ = Describe("[Functional][Outputs][Loki] Forwarding to Loki", func() {
Expect(strings.Contains(lines[2], "Present days")).To(BeTrue())
})
})
Context("when label keys are defined that include slashes. Ref LOG-4095", func() {
const myValue = "foobarvalue"
BeforeEach(func() {
f.Labels["foo/bar"] = myValue
f.Forwarder.Spec.Outputs[0].Loki.LabelKeys = []string{
"kubernetes.namespace_name",
"kubernetes.pod_name",
"kubernetes.labels.foo/bar",
}
Expect(f.Deploy()).To(BeNil())
})
It("should handle the configuration so the collector starts", func() {
now := time.Now()
tsNow := functional.CRIOTime(now)
msg := functional.NewFullCRIOLogMessage(tsNow, "Present days")
Expect(f.WriteMessagesToApplicationLog(msg, 1)).To(Succeed())

query := fmt.Sprintf(`{kubernetes_labels_foo_bar=%q}`, myValue)
result, err := l.QueryUntil(query, "", 1)
Expect(err).To(BeNil())
Expect(result).NotTo(BeNil())
Expect(len(result)).To(Equal(1))
lines := result[0].Lines()
Expect(len(lines)).To(Equal(1))
})
})

})

0 comments on commit 0c7f79d

Please sign in to comment.