Skip to content

LOG-5519: Refactor outputs to support templating #2562

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jun 22, 2024
Merged

LOG-5519: Refactor outputs to support templating #2562

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 13 additions & 29 deletions api/observability/v1/output_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -283,18 +283,12 @@ type Cloudwatch struct {
// +kubebuilder:validation:Required
Region string `json:"region"`

// GroupBy defines the strategy for grouping logstreams
// GroupName defines the strategy for grouping logstreams
//
// +kubebuilder:validation:Pattern:=`^([a-zA-Z0-9-_.\/])*(\{\{[ ]?\.[a-zA-Z0-9_.]+?[ ]?\}\}([a-zA-Z0-9-_.\/])*)*([a-zA-Z0-9-_.\/])*$`
// +kubebuilder:default:="{{.log_type}}"
// +kubebuilder:validation:Required
GroupBy LogGroupByType `json:"groupBy"`

// GroupPrefix Add this prefix to all group names.
//
// Useful to avoid group name clashes if an AWS account is used for multiple clusters and
// used verbatim (e.g. "" means no prefix). The default prefix is cluster-name/log-type
//
// +kubebuilder:validation:Optional
GroupPrefix string `json:"groupPrefix,omitempty"`
GroupName string `json:"groupName"`
}

type CloudwatchAuthType string
Expand Down Expand Up @@ -359,28 +353,11 @@ type CloudwatchAWSAccessKey struct {
KeySecret *SecretKey `json:"keySecret,omitempty"`
}

// LogGroupByType defines a fixed strategy type
//
// +kubebuilder:validation:Enum:=logType;namespaceName;namespaceUUID
type LogGroupByType string

const (
// LogGroupByLogType is the strategy to group logs by source(e.g. app, infra)
LogGroupByLogType LogGroupByType = "logType"

// LogGroupByNamespaceName is the strategy to use for grouping logs by namespace. Infrastructure and
// audit logs are always grouped by "logType"
LogGroupByNamespaceName LogGroupByType = "namespaceName"

// LogGroupByNamespaceUUID is the strategy to use for grouping logs by namespace UUID. Infrastructure and
// audit logs are always grouped by "logType"
LogGroupByNamespaceUUID LogGroupByType = "namespaceUUID"
)

type IndexSpec struct {
// Index is the tenant for the logs. This supports template syntax
// to allow dynamic per-event values. Defaults to the log type (i.e. application, audit, infrastructure)
//
// +kubebuilder:validation:Pattern:=`^([a-zA-Z0-9-_.\/])*(\{\{[ ]?\.[a-zA-Z0-9_.]+?[ ]?\}\}([a-zA-Z0-9-_.\/])*)*([a-zA-Z0-9-_.\/])*$`
// +kubebuilder:validation:Required
// +kubebuilder:default:="{{.log_type}}"
Index string `json:"index,omitempty"`
Expand Down Expand Up @@ -447,6 +424,10 @@ type GoogleCloudLogging struct {
ID GoogleGloudLoggingID `json:"id,omitempty"`

// LogID is the log ID to which to publish logs. This identifies log stream.
//
// +kubebuilder:validation:Pattern:=`^([a-zA-Z0-9-_.\/])*(\{\{[ ]?\.[a-zA-Z0-9_.]+?[ ]?\}\}([a-zA-Z0-9-_.\/])*)*([a-zA-Z0-9-_.\/])*$`
// +kubebuilder:validation:Required
// +kubebuilder:default:="{{.log_type}}"
LogID string `json:"logId,omitempty"`

// Tuning specs tuning for the output
Expand Down Expand Up @@ -575,7 +556,9 @@ type Kafka struct {

// Topic specifies the target topic to send logs to.
//
// +kubebuilder:validation:Optional
// +kubebuilder:validation:Pattern:=`^([a-zA-Z0-9-_.\/])*(\{\{[ ]?\.[a-zA-Z0-9_.]+?[ ]?\}\}([a-zA-Z0-9-_.\/])*)*([a-zA-Z0-9-_.\/])*$`
// +kubebuilder:validation:Required
// +kubebuilder:default:="{{.log_type}}"
Topic string `json:"topic,omitempty"`

// Brokers specifies the list of broker endpoints of a Kafka cluster.
Expand Down Expand Up @@ -679,6 +662,7 @@ type Loki struct {
// TenantKey is the tenant for the logs. This supports vector's template syntax
// to allow dynamic per-event values. Defaults to the log type (i.e. application, audit, infrastructure)
//
// +kubebuilder:validation:Pattern:=`^([a-zA-Z0-9-_.\/])*(\{\{[ ]?\.[a-zA-Z0-9_.]+?[ ]?\}\}([a-zA-Z0-9-_.\/])*)*([a-zA-Z0-9-_.\/])*$`
// +kubebuilder:validation:Required
// +kubebuilder:default:="{{.log_type}}"
TenantKey string `json:"tenantKey,omitempty"`
Expand Down
32 changes: 19 additions & 13 deletions bundle/manifests/observability.openshift.io_clusterlogforwarders.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -994,18 +994,12 @@ spec:
- iamRole
type: string
type: object
groupBy:
description: GroupBy defines the strategy for grouping logstreams
enum:
- logType
- namespaceName
- namespaceUUID
type: string
groupPrefix:
description: "GroupPrefix Add this prefix to all group names.
\n Useful to avoid group name clashes if an AWS account
is used for multiple clusters and used verbatim (e.g.
\"\" means no prefix). The default prefix is cluster-name/log-type"
groupName:
default: '{{.log_type}}'
description: GroupName defines the strategy for grouping
logstreams
pattern: ^([a-zA-Z0-9-_.\/])*(\{\{[ ]?\.[a-zA-Z0-9_.]+?[
]?\}\}([a-zA-Z0-9-_.\/])*)*([a-zA-Z0-9-_.\/])*$
type: string
region:
type: string
Expand Down Expand Up @@ -1057,7 +1051,7 @@ spec:
part of `url` is ignored."
type: string
required:
- groupBy
- groupName
- region
type: object
elasticsearch:
Expand Down Expand Up @@ -1149,6 +1143,8 @@ spec:
description: Index is the tenant for the logs. This supports
template syntax to allow dynamic per-event values. Defaults
to the log type (i.e. application, audit, infrastructure)
pattern: ^([a-zA-Z0-9-_.\/])*(\{\{[ ]?\.[a-zA-Z0-9_.]+?[
]?\}\}([a-zA-Z0-9-_.\/])*)*([a-zA-Z0-9-_.\/])*$
type: string
tuning:
description: Tuning specs tuning for the output
Expand Down Expand Up @@ -1262,8 +1258,11 @@ spec:
type: string
type: object
logId:
default: '{{.log_type}}'
description: LogID is the log ID to which to publish logs.
This identifies log stream.
pattern: ^([a-zA-Z0-9-_.\/])*(\{\{[ ]?\.[a-zA-Z0-9_.]+?[
]?\}\}([a-zA-Z0-9-_.\/])*)*([a-zA-Z0-9-_.\/])*$
type: string
tuning:
description: Tuning specs tuning for the output
Expand Down Expand Up @@ -1536,8 +1535,11 @@ spec:
type: string
type: array
topic:
default: '{{.log_type}}'
description: Topic specifies the target topic to send logs
to.
pattern: ^([a-zA-Z0-9-_.\/])*(\{\{[ ]?\.[a-zA-Z0-9_.]+?[
]?\}\}([a-zA-Z0-9-_.\/])*)*([a-zA-Z0-9-_.\/])*$
type: string
tuning:
description: Tuning specs tuning for the output
Expand Down Expand Up @@ -1694,6 +1696,8 @@ spec:
supports vector's template syntax to allow dynamic per-event
values. Defaults to the log type (i.e. application, audit,
infrastructure)
pattern: ^([a-zA-Z0-9-_.\/])*(\{\{[ ]?\.[a-zA-Z0-9_.]+?[
]?\}\}([a-zA-Z0-9-_.\/])*)*([a-zA-Z0-9-_.\/])*$
type: string
tuning:
description: Tuning specs tuning for the output
Expand Down Expand Up @@ -1968,6 +1972,8 @@ spec:
description: Index is the tenant for the logs. This supports
template syntax to allow dynamic per-event values. Defaults
to the log type (i.e. application, audit, infrastructure)
pattern: ^([a-zA-Z0-9-_.\/])*(\{\{[ ]?\.[a-zA-Z0-9_.]+?[
]?\}\}([a-zA-Z0-9-_.\/])*)*([a-zA-Z0-9-_.\/])*$
type: string
tuning:
description: Tuning specs tuning for the output
Expand Down
32 changes: 19 additions & 13 deletions config/crd/bases/observability.openshift.io_clusterlogforwarders.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -995,18 +995,12 @@ spec:
- iamRole
type: string
type: object
groupBy:
description: GroupBy defines the strategy for grouping logstreams
enum:
- logType
- namespaceName
- namespaceUUID
type: string
groupPrefix:
description: "GroupPrefix Add this prefix to all group names.
\n Useful to avoid group name clashes if an AWS account
is used for multiple clusters and used verbatim (e.g.
\"\" means no prefix). The default prefix is cluster-name/log-type"
groupName:
default: '{{.log_type}}'
description: GroupName defines the strategy for grouping
logstreams
pattern: ^([a-zA-Z0-9-_.\/])*(\{\{[ ]?\.[a-zA-Z0-9_.]+?[
]?\}\}([a-zA-Z0-9-_.\/])*)*([a-zA-Z0-9-_.\/])*$
type: string
region:
type: string
Expand Down Expand Up @@ -1058,7 +1052,7 @@ spec:
part of `url` is ignored."
type: string
required:
- groupBy
- groupName
- region
type: object
elasticsearch:
Expand Down Expand Up @@ -1150,6 +1144,8 @@ spec:
description: Index is the tenant for the logs. This supports
template syntax to allow dynamic per-event values. Defaults
to the log type (i.e. application, audit, infrastructure)
pattern: ^([a-zA-Z0-9-_.\/])*(\{\{[ ]?\.[a-zA-Z0-9_.]+?[
]?\}\}([a-zA-Z0-9-_.\/])*)*([a-zA-Z0-9-_.\/])*$
type: string
tuning:
description: Tuning specs tuning for the output
Expand Down Expand Up @@ -1263,8 +1259,11 @@ spec:
type: string
type: object
logId:
default: '{{.log_type}}'
description: LogID is the log ID to which to publish logs.
This identifies log stream.
pattern: ^([a-zA-Z0-9-_.\/])*(\{\{[ ]?\.[a-zA-Z0-9_.]+?[
]?\}\}([a-zA-Z0-9-_.\/])*)*([a-zA-Z0-9-_.\/])*$
type: string
tuning:
description: Tuning specs tuning for the output
Expand Down Expand Up @@ -1537,8 +1536,11 @@ spec:
type: string
type: array
topic:
default: '{{.log_type}}'
description: Topic specifies the target topic to send logs
to.
pattern: ^([a-zA-Z0-9-_.\/])*(\{\{[ ]?\.[a-zA-Z0-9_.]+?[
]?\}\}([a-zA-Z0-9-_.\/])*)*([a-zA-Z0-9-_.\/])*$
type: string
tuning:
description: Tuning specs tuning for the output
Expand Down Expand Up @@ -1695,6 +1697,8 @@ spec:
supports vector's template syntax to allow dynamic per-event
values. Defaults to the log type (i.e. application, audit,
infrastructure)
pattern: ^([a-zA-Z0-9-_.\/])*(\{\{[ ]?\.[a-zA-Z0-9_.]+?[
]?\}\}([a-zA-Z0-9-_.\/])*)*([a-zA-Z0-9-_.\/])*$
type: string
tuning:
description: Tuning specs tuning for the output
Expand Down Expand Up @@ -1969,6 +1973,8 @@ spec:
description: Index is the tenant for the logs. This supports
template syntax to allow dynamic per-event values. Defaults
to the log type (i.e. application, audit, infrastructure)
pattern: ^([a-zA-Z0-9-_.\/])*(\{\{[ ]?\.[a-zA-Z0-9_.]+?[
]?\}\}([a-zA-Z0-9-_.\/])*)*([a-zA-Z0-9-_.\/])*$
type: string
tuning:
description: Tuning specs tuning for the output
Expand Down
7 changes: 4 additions & 3 deletions internal/collector/collector_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package collector

import (
"os"

. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
obs "github.com/openshift/cluster-logging-operator/api/observability/v1"
Expand All @@ -14,7 +16,6 @@ import (
"github.com/openshift/cluster-logging-operator/internal/tls"
"github.com/openshift/cluster-logging-operator/internal/utils"
. "github.com/openshift/cluster-logging-operator/test/matchers"
"os"

v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
Expand Down Expand Up @@ -614,8 +615,8 @@ var _ = Describe("Factory#NewPodSpec Add Cloudwatch STS Resources", func() {
Type: obs.OutputTypeCloudwatch,
Name: "cw",
Cloudwatch: &obs.Cloudwatch{
Region: "us-east-77",
GroupBy: obs.LogGroupByNamespaceName,
Region: "us-east-77",
GroupName: "{{.namespace_name}}",
Authentication: &obs.CloudwatchAuthentication{
Type: obs.CloudwatchAuthTypeIAMRole,
IAMRole: &obs.CloudwatchIAMRole{
Expand Down
Loading