From 7d9afc2d52e401166c58ec13c07c98c958406ac1 Mon Sep 17 00:00:00 2001 From: Jeff Cantrill Date: Fri, 29 Sep 2023 16:10:34 -0400 Subject: [PATCH] LOG-4568: change limitspec field name to PodLimit --- apis/logging/v1/cluster_log_forwarder.go | 6 +++--- .../logging/v1/cluster_log_forwarder_types.go | 6 +++--- apis/logging/v1/zz_generated.deepcopy.go | 4 ++-- .../clusterlogging.clusterserviceversion.yaml | 2 +- ...ing.openshift.io_clusterlogforwarders.yaml | 21 +++++++++---------- ...ing.openshift.io_clusterlogforwarders.yaml | 21 +++++++++---------- docs/features/collection.adoc | 11 ++++++++++ docs/reference/operator/api.adoc | 16 +++++++------- internal/generator/vector/inputs.go | 4 ++-- .../vector/sources_to_pipelines_test.go | 2 +- .../validate_clusterlogforwarderspec.go | 2 +- .../validate_clusterlogforwarderspec_test.go | 6 +++--- test/e2e/flowcontrol/flowcontrol_test.go | 6 +++--- .../flowcontrol/application_test.go | 2 +- 14 files changed, 59 insertions(+), 50 deletions(-) diff --git a/apis/logging/v1/cluster_log_forwarder.go b/apis/logging/v1/cluster_log_forwarder.go index 692e93a99..0e3ca61c8 100644 --- a/apis/logging/v1/cluster_log_forwarder.go +++ b/apis/logging/v1/cluster_log_forwarder.go @@ -171,7 +171,7 @@ func (input *InputSpec) Types() sets.String { // HasPolicy returns whether the input spec has flow control policies defined in it. func (input *InputSpec) HasPolicy() bool { if input.Application != nil && - (input.Application.ContainerLimit != nil || + (input.Application.PodLimit != nil || input.Application.GroupLimit != nil) { return true } @@ -179,8 +179,8 @@ func (input *InputSpec) HasPolicy() bool { } func (input *InputSpec) GetMaxRecordsPerSecond() int64 { - if input.Application.ContainerLimit != nil { - return input.Application.ContainerLimit.MaxRecordsPerSecond + if input.Application.PodLimit != nil { + return input.Application.PodLimit.MaxRecordsPerSecond } else { return input.Application.GroupLimit.MaxRecordsPerSecond } diff --git a/apis/logging/v1/cluster_log_forwarder_types.go b/apis/logging/v1/cluster_log_forwarder_types.go index 2f151e1cc..7b4fb06e6 100644 --- a/apis/logging/v1/cluster_log_forwarder_types.go +++ b/apis/logging/v1/cluster_log_forwarder_types.go @@ -171,12 +171,12 @@ type Application struct { //+operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:hidden"} GroupLimit *LimitSpec `json:"-"` //`json:"groupLimit,omitempty"` - // Container limit applied to each container selected - // by this input. No container selected by this input can + // PodLimit applied to each pod selected + // by this input. No pod selected by this input can // exceed this limit. // // +optional - ContainerLimit *LimitSpec `json:"containerLimit,omitempty"` + PodLimit *LimitSpec `json:"podLimit,omitempty"` } // Infrastructure enables infrastructure logs. Filtering may be added in future. diff --git a/apis/logging/v1/zz_generated.deepcopy.go b/apis/logging/v1/zz_generated.deepcopy.go index b62521758..df2fa0ea2 100644 --- a/apis/logging/v1/zz_generated.deepcopy.go +++ b/apis/logging/v1/zz_generated.deepcopy.go @@ -32,8 +32,8 @@ func (in *Application) DeepCopyInto(out *Application) { *out = new(LimitSpec) **out = **in } - if in.ContainerLimit != nil { - in, out := &in.ContainerLimit, &out.ContainerLimit + if in.PodLimit != nil { + in, out := &in.PodLimit, &out.PodLimit *out = new(LimitSpec) **out = **in } diff --git a/bundle/manifests/clusterlogging.clusterserviceversion.yaml b/bundle/manifests/clusterlogging.clusterserviceversion.yaml index 31d5bb3e8..0066e322e 100644 --- a/bundle/manifests/clusterlogging.clusterserviceversion.yaml +++ b/bundle/manifests/clusterlogging.clusterserviceversion.yaml @@ -118,7 +118,7 @@ metadata: certified: "false" console.openshift.io/plugins: '["logging-view-plugin"]' containerImage: quay.io/openshift-logging/cluster-logging-operator:latest - createdAt: "2023-09-26T17:11:39Z" + createdAt: "2023-10-02T18:19:30Z" description: The Red Hat OpenShift Logging Operator for OCP provides a means for configuring and managing your aggregated logging stack. olm.skipRange: '>=5.6.0-0 <5.8.0' diff --git a/bundle/manifests/logging.openshift.io_clusterlogforwarders.yaml b/bundle/manifests/logging.openshift.io_clusterlogforwarders.yaml index 7a7cf209f..7193b636d 100644 --- a/bundle/manifests/logging.openshift.io_clusterlogforwarders.yaml +++ b/bundle/manifests/logging.openshift.io_clusterlogforwarders.yaml @@ -258,17 +258,6 @@ spec: description: Application, if present, enables named set of `application` logs that can specify a set of match criteria properties: - containerLimit: - description: Container limit applied to each container selected - by this input. No container selected by this input can - exceed this limit. - properties: - maxRecordsPerSecond: - description: MaxRecordsPerSecond is the maximum number - of log records allowed per input/output in a pipeline - format: int64 - type: integer - type: object namespaces: description: Namespaces from which to collect application logs. Only messages from these namespaces are collected. @@ -276,6 +265,16 @@ spec: items: type: string type: array + podLimit: + description: PodLimit applied to each pod selected by this + input. No pod selected by this input can exceed this limit. + properties: + maxRecordsPerSecond: + description: MaxRecordsPerSecond is the maximum number + of log records allowed per input/output in a pipeline + format: int64 + type: integer + type: object selector: description: Selector for logs from pods with matching labels. Only messages from pods with these labels are collected. diff --git a/config/crd/bases/logging.openshift.io_clusterlogforwarders.yaml b/config/crd/bases/logging.openshift.io_clusterlogforwarders.yaml index ef4237bac..aa794bab0 100644 --- a/config/crd/bases/logging.openshift.io_clusterlogforwarders.yaml +++ b/config/crd/bases/logging.openshift.io_clusterlogforwarders.yaml @@ -259,17 +259,6 @@ spec: description: Application, if present, enables named set of `application` logs that can specify a set of match criteria properties: - containerLimit: - description: Container limit applied to each container selected - by this input. No container selected by this input can - exceed this limit. - properties: - maxRecordsPerSecond: - description: MaxRecordsPerSecond is the maximum number - of log records allowed per input/output in a pipeline - format: int64 - type: integer - type: object namespaces: description: Namespaces from which to collect application logs. Only messages from these namespaces are collected. @@ -277,6 +266,16 @@ spec: items: type: string type: array + podLimit: + description: PodLimit applied to each pod selected by this + input. No pod selected by this input can exceed this limit. + properties: + maxRecordsPerSecond: + description: MaxRecordsPerSecond is the maximum number + of log records allowed per input/output in a pipeline + format: int64 + type: integer + type: object selector: description: Selector for logs from pods with matching labels. Only messages from pods with these labels are collected. diff --git a/docs/features/collection.adoc b/docs/features/collection.adoc index b18b449c1..7dbdffb33 100644 --- a/docs/features/collection.adoc +++ b/docs/features/collection.adoc @@ -111,6 +111,17 @@ logstash 7.10.1|✓| - retrymaxinterval - retrytimeout +|====== +.Vector Tuning +[options="header"] +|====== +|Feature|Desc. +|Application Input Flow Control +| Specify the max rate of incoming logs for a defined application input +|Output Flow Control +| Specify the max rate of logs sent to a given output. Excess logs are dropped + + |====== === Metrics and Alerting diff --git a/docs/reference/operator/api.adoc b/docs/reference/operator/api.adoc index bc7515db4..b64d96a9e 100644 --- a/docs/reference/operator/api.adoc +++ b/docs/reference/operator/api.adoc @@ -94,12 +94,18 @@ All conditions in the selector must be satisfied (logical AND) to select logs. |====================== |Property|Type|Description -|containerLimit|object| *(optional)* Container limit applied to each container selected |namespaces|array| *(optional)* Namespaces from which to collect application logs. +|podLimit|object| *(optional)* PodLimit applied to each pod selected |selector|object| *(optional)* Selector for logs from pods with matching labels. |====================== -=== .spec.inputs[].application.containerLimit +=== .spec.inputs[].application.namespaces[] +===== Description + +===== Type +* array + +=== .spec.inputs[].application.podLimit ===== Description ===== Type @@ -112,12 +118,6 @@ All conditions in the selector must be satisfied (logical AND) to select logs. |maxRecordsPerSecond|int| MaxRecordsPerSecond is the maximum number of log records |====================== -=== .spec.inputs[].application.namespaces[] -===== Description - -===== Type -* array - === .spec.inputs[].application.selector ===== Description diff --git a/internal/generator/vector/inputs.go b/internal/generator/vector/inputs.go index 8b6b5af8d..f4e55df19 100644 --- a/internal/generator/vector/inputs.go +++ b/internal/generator/vector/inputs.go @@ -68,8 +68,8 @@ func AddThrottle(spec *logging.InputSpec) []generator.Element { el := []generator.Element{} input := fmt.Sprintf(UserDefinedInput, spec.Name) - if spec.Application.ContainerLimit != nil { - threshold = spec.Application.ContainerLimit.MaxRecordsPerSecond + if spec.Application.PodLimit != nil { + threshold = spec.Application.PodLimit.MaxRecordsPerSecond throttle_key = perContainerLimitKeyField } else { threshold = spec.Application.GroupLimit.MaxRecordsPerSecond diff --git a/internal/generator/vector/sources_to_pipelines_test.go b/internal/generator/vector/sources_to_pipelines_test.go index 680f13f0b..c6ecc5308 100644 --- a/internal/generator/vector/sources_to_pipelines_test.go +++ b/internal/generator/vector/sources_to_pipelines_test.go @@ -447,7 +447,7 @@ source = ''' "podname": "very-important", }, }, - ContainerLimit: &logging.LimitSpec{ + PodLimit: &logging.LimitSpec{ MaxRecordsPerSecond: 100, }, }, diff --git a/internal/validations/clusterlogforwarder/validate_clusterlogforwarderspec.go b/internal/validations/clusterlogforwarder/validate_clusterlogforwarderspec.go index 2e82c45fc..ab76e7ada 100644 --- a/internal/validations/clusterlogforwarder/validate_clusterlogforwarderspec.go +++ b/internal/validations/clusterlogforwarder/validate_clusterlogforwarderspec.go @@ -191,7 +191,7 @@ func verifyInputs(spec *loggingv1.ClusterLogForwarderSpec, status *loggingv1.Clu // Check if inputspec has application, infrastructure, audit or receiver specs case input.Application == nil && input.Infrastructure == nil && input.Audit == nil && input.Receiver == nil: badInput("inputspec must define one or more of application, infrastructure, audit or receiver") - case input.HasPolicy() && input.Application.ContainerLimit != nil && input.Application.GroupLimit != nil: + case input.HasPolicy() && input.Application.PodLimit != nil && input.Application.GroupLimit != nil: badInput("inputspec must define only one of container or group limit") case input.HasPolicy() && input.GetMaxRecordsPerSecond() < 0: badInput("inputspec cannot have a negative limit threshold") diff --git a/internal/validations/clusterlogforwarder/validate_clusterlogforwarderspec_test.go b/internal/validations/clusterlogforwarder/validate_clusterlogforwarderspec_test.go index 0275e1544..f816434fe 100644 --- a/internal/validations/clusterlogforwarder/validate_clusterlogforwarderspec_test.go +++ b/internal/validations/clusterlogforwarder/validate_clusterlogforwarderspec_test.go @@ -204,7 +204,7 @@ var _ = Describe("Validate clusterlogforwarderspec", func() { { Name: "custom-app", Application: &loggingv1.Application{ - ContainerLimit: &loggingv1.LimitSpec{ + PodLimit: &loggingv1.LimitSpec{ MaxRecordsPerSecond: 100, }, GroupLimit: &loggingv1.LimitSpec{ @@ -223,7 +223,7 @@ var _ = Describe("Validate clusterlogforwarderspec", func() { { Name: "custom-app-container-limit", Application: &loggingv1.Application{ - ContainerLimit: &loggingv1.LimitSpec{ + PodLimit: &loggingv1.LimitSpec{ MaxRecordsPerSecond: 100, }, }, @@ -248,7 +248,7 @@ var _ = Describe("Validate clusterlogforwarderspec", func() { { Name: "custom-app-container-limit", Application: &loggingv1.Application{ - ContainerLimit: &loggingv1.LimitSpec{ + PodLimit: &loggingv1.LimitSpec{ MaxRecordsPerSecond: -100, }, }, diff --git a/test/e2e/flowcontrol/flowcontrol_test.go b/test/e2e/flowcontrol/flowcontrol_test.go index 27624536f..cdc8f6150 100644 --- a/test/e2e/flowcontrol/flowcontrol_test.go +++ b/test/e2e/flowcontrol/flowcontrol_test.go @@ -93,7 +93,7 @@ var _ = Describe("[E2E] FlowControl", func() { Name: "custom-app-0", Application: &loggingv1.Application{ Namespaces: []string{stressorNS}, - ContainerLimit: &loggingv1.LimitSpec{ + PodLimit: &loggingv1.LimitSpec{ MaxRecordsPerSecond: 50, }, // 10 files and 100 group limit, so 10 lines per file, }, @@ -102,7 +102,7 @@ var _ = Describe("[E2E] FlowControl", func() { Name: "custom-app-1", Application: &loggingv1.Application{ Namespaces: []string{stressorNS}, - ContainerLimit: &loggingv1.LimitSpec{ + PodLimit: &loggingv1.LimitSpec{ MaxRecordsPerSecond: 50, }, // 10 files and 100 group limit, so 10 lines per file, }, @@ -175,7 +175,7 @@ var _ = Describe("[E2E] FlowControl", func() { //GroupLimit: &loggingv1.LimitSpec{ // MaxRecordsPerSecond: 200, //}, - ContainerLimit: &loggingv1.LimitSpec{ + PodLimit: &loggingv1.LimitSpec{ MaxRecordsPerSecond: 200, }, }, diff --git a/test/functional/flowcontrol/application_test.go b/test/functional/flowcontrol/application_test.go index 0320af94c..996e598a8 100644 --- a/test/functional/flowcontrol/application_test.go +++ b/test/functional/flowcontrol/application_test.go @@ -69,7 +69,7 @@ var _ = Describe("[Functional][FlowControl] Policies at Input", func() { Skip("Skipping test since flow-control is not supported with fluentd") } - f.Forwarder.Spec.Inputs[0].Application.ContainerLimit = &logging.LimitSpec{ + f.Forwarder.Spec.Inputs[0].Application.PodLimit = &logging.LimitSpec{ MaxRecordsPerSecond: 10, }