From 7d31a59a2fc7747a7a159c80747a9d2f8a485eff Mon Sep 17 00:00:00 2001 From: Joao Henri Date: Wed, 4 Oct 2023 13:12:39 -0300 Subject: [PATCH 01/21] Create OTELConfigSpec for OTEL Collector fine tuning Signed-off-by: Joao Henri --- apis/core/v1beta1/collector_types.go | 2 ++ apis/core/v1beta1/zz_generated.deepcopy.go | 6 ++++++ config/crd/bases/core.opni.io_collectors.yaml | 5 +++++ go.mod | 12 +++++++++++ go.sum | 21 +++++++++++++++++++ pkg/otel/types.go | 11 ++++++++++ pkg/resources/collector/workloads.go | 1 + .../kubernetes_manager/kubernetes_manager.go | 18 ++++++++++++++++ 8 files changed, 76 insertions(+) diff --git a/apis/core/v1beta1/collector_types.go b/apis/core/v1beta1/collector_types.go index a1301f91d4..9c8798a347 100644 --- a/apis/core/v1beta1/collector_types.go +++ b/apis/core/v1beta1/collector_types.go @@ -1,6 +1,7 @@ package v1beta1 import ( + "github.com/rancher/opni/pkg/otel" opnimeta "github.com/rancher/opni/pkg/util/meta" "github.com/rancher/wrangler/pkg/crd" "github.com/rancher/wrangler/pkg/schemas/openapi" @@ -24,6 +25,7 @@ type CollectorSpec struct { MetricsConfig *corev1.LocalObjectReference `json:"metricsConfig,omitempty"` ConfigReloader *ConfigReloaderSpec `json:"configReloader,omitempty"` LogLevel string `json:"logLevel,omitempty"` + OTELSpec *otel.OTELConfigSpec `json:"otel,omitempty"` } type ConfigReloaderSpec struct { diff --git a/apis/core/v1beta1/zz_generated.deepcopy.go b/apis/core/v1beta1/zz_generated.deepcopy.go index a766fd422b..a2404bb9df 100644 --- a/apis/core/v1beta1/zz_generated.deepcopy.go +++ b/apis/core/v1beta1/zz_generated.deepcopy.go @@ -5,6 +5,7 @@ package v1beta1 import ( + "github.com/rancher/opni/pkg/otel" "github.com/rancher/opni/pkg/util/meta" appsv1 "k8s.io/api/apps/v1" "k8s.io/api/core/v1" @@ -401,6 +402,11 @@ func (in *CollectorSpec) DeepCopyInto(out *CollectorSpec) { *out = new(ConfigReloaderSpec) (*in).DeepCopyInto(*out) } + if in.OTELSpec != nil { + in, out := &in.OTELSpec, &out.OTELSpec + *out = new(otel.OTELConfigSpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CollectorSpec. diff --git a/config/crd/bases/core.opni.io_collectors.yaml b/config/crd/bases/core.opni.io_collectors.yaml index a96803ff3c..e560f5312f 100644 --- a/config/crd/bases/core.opni.io_collectors.yaml +++ b/config/crd/bases/core.opni.io_collectors.yaml @@ -69,6 +69,11 @@ spec: type: string type: object x-kubernetes-map-type: atomic + otel: + properties: + batch: {} + memoryLimiter: {} + type: object systemNamespace: type: string type: object diff --git a/go.mod b/go.mod index d863b19f05..53e617fea7 100644 --- a/go.mod +++ b/go.mod @@ -116,6 +116,8 @@ require ( go.etcd.io/etcd/client/v3 v3.5.9 go.etcd.io/etcd/etcdctl/v3 v3.5.9 go.opentelemetry.io/collector/pdata v1.0.0-rcv0014 + go.opentelemetry.io/collector/processor/batchprocessor v0.80.0 + go.opentelemetry.io/collector/processor/memorylimiterprocessor v0.80.0 go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.40.0 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0 @@ -163,6 +165,16 @@ require ( sigs.k8s.io/yaml v1.3.0 ) +require ( + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/shirou/gopsutil/v3 v3.23.5 // indirect + github.com/tklauser/go-sysconf v0.3.11 // indirect + github.com/tklauser/numcpus v0.6.0 // indirect + github.com/yusufpapurcu/wmi v1.2.3 // indirect +) + require ( cloud.google.com/go v0.110.7 // indirect cloud.google.com/go/compute v1.23.0 // indirect diff --git a/go.sum b/go.sum index 89d01d7309..9797db0646 100644 --- a/go.sum +++ b/go.sum @@ -1046,6 +1046,8 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= @@ -1703,6 +1705,8 @@ github.com/longhorn/upgrade-responder v0.1.5 h1:d1GksWjckahzFruqmeHyWwOEt8+P6H5g github.com/longhorn/upgrade-responder v0.1.5/go.mod h1:6n1YZwCwERleK5BdvKsGnqV6Z7HJanAd2Pp/rl4Nw4c= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= @@ -2048,6 +2052,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/poy/onpar v0.0.0-20200406201722-06f95a1c68e8/go.mod h1:nSbFQvMj97ZyhFRSJYtut+msi4sOY6zJDGCdSc+/rZU= github.com/poy/onpar v1.1.2 h1:QaNrNiZx0+Nar5dLgTVp5mXkyoVFIbepjyEoGSnhbAY= github.com/poy/onpar v1.1.2/go.mod h1:6X8FLNoxyr9kkmnlqpK6LSoiOtrO6MICtWwEuWkLjzg= @@ -2191,6 +2197,10 @@ github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAm github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= +github.com/shirou/gopsutil/v3 v3.23.5 h1:5SgDCeQ0KW0S4N0znjeM/eFHXXOKyv2dVNgRq/c9P6Y= +github.com/shirou/gopsutil/v3 v3.23.5/go.mod h1:Ng3Maa27Q2KARVJ0SPZF5NdrQSC3XHKP8IIWrHgMeLY= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= @@ -2326,6 +2336,10 @@ github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhso github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= +github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= +github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= +github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31 h1:OXcKh35JaYsGMRzpvFkLv/MEyPuL49CThT1pZ8aSml4= @@ -2403,6 +2417,8 @@ github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/gopher-lua v1.1.0 h1:BojcDhfyDWgU2f2TOzYK/g5p2gxMrku8oupLDqlnSqE= github.com/yuin/gopher-lua v1.1.0/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= +github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43 h1:+lm10QQTNSBd8DVTNGHx7o/IKu9HYDvLMffDhbyLccI= github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50 h1:hlE8//ciYMztlGpl/VA+Zm1AcTPHYkHJPbHqE6WJUXE= @@ -2487,6 +2503,10 @@ go.opentelemetry.io/collector/pdata v1.0.0-rcv0014 h1:iT5qH0NLmkGeIdDtnBogYDx7L5 go.opentelemetry.io/collector/pdata v1.0.0-rcv0014/go.mod h1:BRvDrx43kiSoUx3mr7SoA7h9B8+OY99mUK+CZSQFWW4= go.opentelemetry.io/collector/processor v0.80.0 h1:UUKPh2E/pIru2WChWiFVrugc48m2Al5kHQ0d66cxh6w= go.opentelemetry.io/collector/processor v0.80.0/go.mod h1:ZFtJUIgpJmgkSnBGd4F+q4zb1TrhrQsNHGdDmijYlU8= +go.opentelemetry.io/collector/processor/batchprocessor v0.80.0 h1:B3phgqGYRbAM1ZjR7OlAS4hK1XcM2/FDLvzOCycyE6A= +go.opentelemetry.io/collector/processor/batchprocessor v0.80.0/go.mod h1:Qx1iqTNROrUIzecHbbQenO3NacvkKknnZzikD5qUFOg= +go.opentelemetry.io/collector/processor/memorylimiterprocessor v0.80.0 h1:ZK9GRIjqHB74LIUwhCUv7tWO0HlVMdqpo0kTNs2TkF8= +go.opentelemetry.io/collector/processor/memorylimiterprocessor v0.80.0/go.mod h1:Saosjjmi1q1vkLJ8EH/jTvhp37/tTPDa2Sa28G8jCqk= go.opentelemetry.io/collector/receiver v0.80.0 h1:+dJn57mWaksgKDnR06mAu5aReOcqg1pdgC6zqqLmtZ4= go.opentelemetry.io/collector/receiver v0.80.0/go.mod h1:DsyjwQZjlcbwoY4Kq4yx1tNL8Z4d8E7e4tbQYzBUw/g= go.opentelemetry.io/collector/semconv v0.81.0 h1:lCYNNo3powDvFIaTPP2jDKIrBiV1T92NK4QgL/aHYXw= @@ -2857,6 +2877,7 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/pkg/otel/types.go b/pkg/otel/types.go index 5a676940da..6a35b0a1a4 100644 --- a/pkg/otel/types.go +++ b/pkg/otel/types.go @@ -6,6 +6,8 @@ import ( "github.com/rancher/opni/pkg/util" "github.com/samber/lo" + "go.opentelemetry.io/collector/processor/batchprocessor" + "go.opentelemetry.io/collector/processor/memorylimiterprocessor" "google.golang.org/protobuf/types/known/durationpb" "gopkg.in/yaml.v2" ) @@ -35,6 +37,7 @@ type AggregatorConfig struct { Metrics MetricsConfig Containerized bool LogLevel string + OTELConfig OTELConfigSpec } type LoggingConfig struct { @@ -59,6 +62,14 @@ type OTELSpec struct { HostMetrics *bool `json:"hostMetrics,omitempty"` } +type OTELConfigSpec struct { + // Batch Processor Configs + BatchProcessor batchprocessor.Config `json:"batch,omitempty"` + + // Memory Limiter Processor Configs + MemoryLimiterProcessor memorylimiterprocessor.Config `json:"memoryLimiter,omitempty"` +} + type ScrapeConfig struct { JobName string `json:"jobName,omitempty"` Targets []string `json:"targets,omitempty"` diff --git a/pkg/resources/collector/workloads.go b/pkg/resources/collector/workloads.go index 76539e096d..1850c4b352 100644 --- a/pkg/resources/collector/workloads.go +++ b/pkg/resources/collector/workloads.go @@ -73,6 +73,7 @@ func (r *Reconciler) getAggregatorConfig( AgentEndpoint: r.collector.Spec.AgentEndpoint, Containerized: true, LogLevel: r.collector.Spec.LogLevel, + OTELConfig: *r.collector.Spec.OTELSpec, } } diff --git a/plugins/logging/pkg/agent/drivers/kubernetes_manager/kubernetes_manager.go b/plugins/logging/pkg/agent/drivers/kubernetes_manager/kubernetes_manager.go index c3e7ede0da..09c0e96578 100644 --- a/plugins/logging/pkg/agent/drivers/kubernetes_manager/kubernetes_manager.go +++ b/plugins/logging/pkg/agent/drivers/kubernetes_manager/kubernetes_manager.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "sync" + "time" "github.com/cisco-open/k8s-objectmatcher/patch" "github.com/lestrrat-go/backoff/v2" @@ -21,6 +22,8 @@ import ( "github.com/rancher/opni/plugins/logging/apis/node" "github.com/rancher/opni/plugins/logging/pkg/agent/drivers" "github.com/samber/lo" + "go.opentelemetry.io/collector/processor/batchprocessor" + "go.opentelemetry.io/collector/processor/memorylimiterprocessor" "go.uber.org/zap" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -323,6 +326,21 @@ func (m *KubernetesManagerDriver) buildEmptyCollector() *opnicorev1beta1.Collect }, SystemNamespace: m.Namespace, AgentEndpoint: otel.AgentEndpoint(serviceName), + OTELSpec: &otel.OTELConfigSpec{ + MemoryLimiterProcessor: memorylimiterprocessor.Config{ + MemoryLimitMiB: 1000, + MemorySpikeLimitMiB: 350, + CheckInterval: 1 * time.Second, + }, + BatchProcessor: batchprocessor.Config{ + SendBatchSize: 1000, + Timeout: 15 * time.Second, + }, + // OTLPSendingQueue: exporterhelper.QueueSettings{ + // NumConsumers: 4, + // QueueSize: 100, + // }, + }, }, } } From 00ccaec48f49d3bf01eba49089a0fd6b07385131 Mon Sep 17 00:00:00 2001 From: Joao Henri Date: Wed, 4 Oct 2023 15:21:57 -0300 Subject: [PATCH 02/21] Embed OTEL Processor Configs into Collector Signed-off-by: Joao Henri --- apis/core/v1beta1/collector_types.go | 2 +- apis/core/v1beta1/zz_generated.deepcopy.go | 4 +- config/crd/bases/core.opni.io_collectors.yaml | 41 +++++++++++- pkg/otel/types.go | 65 +++++++++++++++++-- 4 files changed, 102 insertions(+), 10 deletions(-) diff --git a/apis/core/v1beta1/collector_types.go b/apis/core/v1beta1/collector_types.go index 9c8798a347..b6e9909d3c 100644 --- a/apis/core/v1beta1/collector_types.go +++ b/apis/core/v1beta1/collector_types.go @@ -25,7 +25,7 @@ type CollectorSpec struct { MetricsConfig *corev1.LocalObjectReference `json:"metricsConfig,omitempty"` ConfigReloader *ConfigReloaderSpec `json:"configReloader,omitempty"` LogLevel string `json:"logLevel,omitempty"` - OTELSpec *otel.OTELConfigSpec `json:"otel,omitempty"` + OTELConfigSpec *otel.OTELConfigSpec `json:"otelSpec,omitempty"` } type ConfigReloaderSpec struct { diff --git a/apis/core/v1beta1/zz_generated.deepcopy.go b/apis/core/v1beta1/zz_generated.deepcopy.go index a2404bb9df..9bef0cf6d1 100644 --- a/apis/core/v1beta1/zz_generated.deepcopy.go +++ b/apis/core/v1beta1/zz_generated.deepcopy.go @@ -402,8 +402,8 @@ func (in *CollectorSpec) DeepCopyInto(out *CollectorSpec) { *out = new(ConfigReloaderSpec) (*in).DeepCopyInto(*out) } - if in.OTELSpec != nil { - in, out := &in.OTELSpec, &out.OTELSpec + if in.OTELConfigSpec != nil { + in, out := &in.OTELConfigSpec, &out.OTELConfigSpec *out = new(otel.OTELConfigSpec) (*in).DeepCopyInto(*out) } diff --git a/config/crd/bases/core.opni.io_collectors.yaml b/config/crd/bases/core.opni.io_collectors.yaml index e560f5312f..eab71186cd 100644 --- a/config/crd/bases/core.opni.io_collectors.yaml +++ b/config/crd/bases/core.opni.io_collectors.yaml @@ -69,10 +69,45 @@ spec: type: string type: object x-kubernetes-map-type: atomic - otel: + otelSpec: properties: - batch: {} - memoryLimiter: {} + batch: + properties: + metadataCardinalityLimit: + format: int32 + type: integer + metadataKeys: + items: + type: string + type: array + sendBatchMaxSize: + format: int32 + type: integer + sendBatchSize: + format: int32 + type: integer + timeout: + format: int64 + type: integer + type: object + memoryLimiter: + properties: + checkInterval: + format: int64 + type: integer + limitMib: + format: int32 + type: integer + limitPercentage: + format: int32 + type: integer + spikeLimitMib: + format: int32 + type: integer + spikeLimitPercentage: + format: int32 + type: integer + type: object type: object systemNamespace: type: string diff --git a/pkg/otel/types.go b/pkg/otel/types.go index 6a35b0a1a4..1073ffaa9a 100644 --- a/pkg/otel/types.go +++ b/pkg/otel/types.go @@ -3,11 +3,10 @@ package otel import ( "bytes" "fmt" + "time" "github.com/rancher/opni/pkg/util" "github.com/samber/lo" - "go.opentelemetry.io/collector/processor/batchprocessor" - "go.opentelemetry.io/collector/processor/memorylimiterprocessor" "google.golang.org/protobuf/types/known/durationpb" "gopkg.in/yaml.v2" ) @@ -64,10 +63,68 @@ type OTELSpec struct { type OTELConfigSpec struct { // Batch Processor Configs - BatchProcessor batchprocessor.Config `json:"batch,omitempty"` + BatchProcessor BatchConfig `json:"batch,omitempty"` // Memory Limiter Processor Configs - MemoryLimiterProcessor memorylimiterprocessor.Config `json:"memoryLimiter,omitempty"` + MemoryLimiterProcessor MemoryLimiterConfig `json:"memoryLimiter,omitempty"` +} + +// BatchConfig defines configuration for batch processor. +type BatchConfig struct { + // Timeout sets the time after which a batch will be sent regardless of size. + // When this is set to zero, batched data will be sent immediately. + Timeout time.Duration `json:"timeout,omitempty"` + + // SendBatchSize is the size of a batch which after hit, will trigger it to be sent. + // When this is set to zero, the batch size is ignored and data will be sent immediately + // subject to only send_batch_max_size. + SendBatchSize uint32 `json:"sendBatchSize,omitempty"` + + // SendBatchMaxSize is the maximum size of a batch. It must be larger than SendBatchSize. + // Larger batches are split into smaller units. + // Default value is 0, that means no maximum size. + SendBatchMaxSize uint32 `json:"sendBatchMaxSize,omitempty"` + + // MetadataKeys is a list of client.Metadata keys that will be + // used to form distinct batchers. If this setting is empty, + // a single batcher instance will be used. When this setting + // is not empty, one batcher will be used per distinct + // combination of values for the listed metadata keys. + // + // Empty value and unset metadata are treated as distinct cases. + // + // Entries are case-insensitive. Duplicated entries will + // trigger a validation error. + MetadataKeys []string `json:"metadataKeys,omitempty"` + + // MetadataCardinalityLimit indicates the maximum number of + // batcher instances that will be created through a distinct + // combination of MetadataKeys. + MetadataCardinalityLimit uint32 `json:"metadataCardinalityLimit,omitempty"` +} + +// MemoryLimiterConfig defines configuration for memory memoryLimiter processor. +type MemoryLimiterConfig struct { + // CheckInterval is the time between measurements of memory usage for the + // purposes of avoiding going over the limits. Defaults to zero, so no + // checks will be performed. + CheckInterval time.Duration `json:"checkInterval,omitempty"` + + // MemoryLimitMiB is the maximum amount of memory, in MiB, targeted to be + // allocated by the process. + MemoryLimitMiB uint32 `json:"limitMib,omitempty"` + + // MemorySpikeLimitMiB is the maximum, in MiB, spike expected between the + // measurements of memory usage. + MemorySpikeLimitMiB uint32 `json:"spikeLimitMib,omitempty"` + + // MemoryLimitPercentage is the maximum amount of memory, in %, targeted to be + // allocated by the process. The fixed memory settings MemoryLimitMiB has a higher precedence. + MemoryLimitPercentage uint32 `json:"limitPercentage,omitempty"` + + // MemorySpikePercentage is the maximum, in percents against the total memory, + // spike expected between the measurements of memory usage. + MemorySpikePercentage uint32 `json:"spikeLimitPercentage,omitempty"` } type ScrapeConfig struct { From 3f62a8042b19de10d7d3f358d6679018caca7089 Mon Sep 17 00:00:00 2001 From: Joao Henri Date: Thu, 5 Oct 2023 10:52:32 -0300 Subject: [PATCH 03/21] Add OTELConfigSpec to empty collector Signed-off-by: Joao Henri --- .../drivers/kubernetes_manager/kubernetes_manager.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/plugins/logging/pkg/agent/drivers/kubernetes_manager/kubernetes_manager.go b/plugins/logging/pkg/agent/drivers/kubernetes_manager/kubernetes_manager.go index 09c0e96578..d1d88239ae 100644 --- a/plugins/logging/pkg/agent/drivers/kubernetes_manager/kubernetes_manager.go +++ b/plugins/logging/pkg/agent/drivers/kubernetes_manager/kubernetes_manager.go @@ -22,8 +22,6 @@ import ( "github.com/rancher/opni/plugins/logging/apis/node" "github.com/rancher/opni/plugins/logging/pkg/agent/drivers" "github.com/samber/lo" - "go.opentelemetry.io/collector/processor/batchprocessor" - "go.opentelemetry.io/collector/processor/memorylimiterprocessor" "go.uber.org/zap" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -326,13 +324,13 @@ func (m *KubernetesManagerDriver) buildEmptyCollector() *opnicorev1beta1.Collect }, SystemNamespace: m.Namespace, AgentEndpoint: otel.AgentEndpoint(serviceName), - OTELSpec: &otel.OTELConfigSpec{ - MemoryLimiterProcessor: memorylimiterprocessor.Config{ + OTELConfigSpec: &otel.OTELConfigSpec{ + MemoryLimiterProcessor: otel.MemoryLimiterConfig{ MemoryLimitMiB: 1000, MemorySpikeLimitMiB: 350, CheckInterval: 1 * time.Second, }, - BatchProcessor: batchprocessor.Config{ + BatchProcessor: otel.BatchConfig{ SendBatchSize: 1000, Timeout: 15 * time.Second, }, From 15eb17225ec75288535fb70bc280c2fe50170981 Mon Sep 17 00:00:00 2001 From: Joao Henri Date: Thu, 5 Oct 2023 14:20:44 -0300 Subject: [PATCH 04/21] Allow fine tuning for aggregator's memory limiter Signed-off-by: Joao Henri --- apis/core/v1beta1/collector_types.go | 37 +++++++- apis/core/v1beta1/zz_generated.deepcopy.go | 36 +++++++- config/crd/bases/core.opni.io_collectors.yaml | 22 +---- pkg/otel/types.go | 85 ++++--------------- pkg/resources/collector/templates.go | 6 +- pkg/resources/collector/workloads.go | 13 ++- .../kubernetes_manager/kubernetes_manager.go | 12 +-- 7 files changed, 105 insertions(+), 106 deletions(-) diff --git a/apis/core/v1beta1/collector_types.go b/apis/core/v1beta1/collector_types.go index b6e9909d3c..94afb9f81f 100644 --- a/apis/core/v1beta1/collector_types.go +++ b/apis/core/v1beta1/collector_types.go @@ -1,7 +1,8 @@ package v1beta1 import ( - "github.com/rancher/opni/pkg/otel" + "time" + opnimeta "github.com/rancher/opni/pkg/util/meta" "github.com/rancher/wrangler/pkg/crd" "github.com/rancher/wrangler/pkg/schemas/openapi" @@ -25,13 +26,45 @@ type CollectorSpec struct { MetricsConfig *corev1.LocalObjectReference `json:"metricsConfig,omitempty"` ConfigReloader *ConfigReloaderSpec `json:"configReloader,omitempty"` LogLevel string `json:"logLevel,omitempty"` - OTELConfigSpec *otel.OTELConfigSpec `json:"otelSpec,omitempty"` + OTELConfigSpec *OTELConfigSpec `json:"otelSpec,omitempty"` } type ConfigReloaderSpec struct { opnimeta.ImageSpec `json:",inline,omitempty"` } +type OTELConfigSpec struct { + // Memory Limiter Processor Configs + MemoryLimiterProcessor MemoryLimiterConfig `json:"memoryLimiter,omitempty"` +} + +// MemoryLimiterConfig defines configuration for the memoryLimiter processor. +type MemoryLimiterConfig struct { + // CheckInterval is the time between measurements of memory usage for the + // purposes of avoiding going over the limits. Defaults to zero, so no + // checks will be performed. + //+kubebuilder:default="1s" + CheckInterval time.Duration `json:"checkInterval,omitempty"` + + // MemoryLimitMiB is the maximum amount of memory, in MiB, targeted to be + // allocated by the process. + //+kubebuilder:default=1000 + MemoryLimitMiB uint32 `json:"limitMib,omitempty"` + + // MemorySpikeLimitMiB is the maximum, in MiB, spike expected between the + // measurements of memory usage. + //+kubebuilder:default=350 + MemorySpikeLimitMiB uint32 `json:"spikeLimitMib,omitempty"` + + // MemoryLimitPercentage is the maximum amount of memory, in %, targeted to be + // allocated by the process. The fixed memory settings MemoryLimitMiB has a higher precedence. + MemoryLimitPercentage uint32 `json:"limitPercentage,omitempty"` + + // MemorySpikePercentage is the maximum, in percents against the total memory, + // spike expected between the measurements of memory usage. + MemorySpikePercentage uint32 `json:"spikeLimitPercentage,omitempty"` +} + // CollectorStatus defines the observed state of Collector type CollectorStatus struct { Conditions []string `json:"conditions,omitempty"` diff --git a/apis/core/v1beta1/zz_generated.deepcopy.go b/apis/core/v1beta1/zz_generated.deepcopy.go index 9bef0cf6d1..f98a950656 100644 --- a/apis/core/v1beta1/zz_generated.deepcopy.go +++ b/apis/core/v1beta1/zz_generated.deepcopy.go @@ -5,7 +5,6 @@ package v1beta1 import ( - "github.com/rancher/opni/pkg/otel" "github.com/rancher/opni/pkg/util/meta" appsv1 "k8s.io/api/apps/v1" "k8s.io/api/core/v1" @@ -404,8 +403,8 @@ func (in *CollectorSpec) DeepCopyInto(out *CollectorSpec) { } if in.OTELConfigSpec != nil { in, out := &in.OTELConfigSpec, &out.OTELConfigSpec - *out = new(otel.OTELConfigSpec) - (*in).DeepCopyInto(*out) + *out = new(OTELConfigSpec) + **out = **in } } @@ -922,6 +921,21 @@ func (in *LoggingClusterStatus) DeepCopy() *LoggingClusterStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MemoryLimiterConfig) DeepCopyInto(out *MemoryLimiterConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MemoryLimiterConfig. +func (in *MemoryLimiterConfig) DeepCopy() *MemoryLimiterConfig { + if in == nil { + return nil + } + out := new(MemoryLimiterConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MonitoringCluster) DeepCopyInto(out *MonitoringCluster) { *out = *in @@ -1134,6 +1148,22 @@ func (in *NatsStatus) DeepCopy() *NatsStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OTELConfigSpec) DeepCopyInto(out *OTELConfigSpec) { + *out = *in + out.MemoryLimiterProcessor = in.MemoryLimiterProcessor +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OTELConfigSpec. +func (in *OTELConfigSpec) DeepCopy() *OTELConfigSpec { + if in == nil { + return nil + } + out := new(OTELConfigSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OpenIDConfigSpec) DeepCopyInto(out *OpenIDConfigSpec) { *out = *in diff --git a/config/crd/bases/core.opni.io_collectors.yaml b/config/crd/bases/core.opni.io_collectors.yaml index eab71186cd..97dcdf5306 100644 --- a/config/crd/bases/core.opni.io_collectors.yaml +++ b/config/crd/bases/core.opni.io_collectors.yaml @@ -71,37 +71,21 @@ spec: x-kubernetes-map-type: atomic otelSpec: properties: - batch: - properties: - metadataCardinalityLimit: - format: int32 - type: integer - metadataKeys: - items: - type: string - type: array - sendBatchMaxSize: - format: int32 - type: integer - sendBatchSize: - format: int32 - type: integer - timeout: - format: int64 - type: integer - type: object memoryLimiter: properties: checkInterval: + default: 1s format: int64 type: integer limitMib: + default: 1000 format: int32 type: integer limitPercentage: format: int32 type: integer spikeLimitMib: + default: 350 format: int32 type: integer spikeLimitPercentage: diff --git a/pkg/otel/types.go b/pkg/otel/types.go index 1073ffaa9a..59d1752310 100644 --- a/pkg/otel/types.go +++ b/pkg/otel/types.go @@ -3,10 +3,11 @@ package otel import ( "bytes" "fmt" - "time" "github.com/rancher/opni/pkg/util" "github.com/samber/lo" + "go.opentelemetry.io/collector/processor/batchprocessor" + "go.opentelemetry.io/collector/processor/memorylimiterprocessor" "google.golang.org/protobuf/types/known/durationpb" "gopkg.in/yaml.v2" ) @@ -36,7 +37,21 @@ type AggregatorConfig struct { Metrics MetricsConfig Containerized bool LogLevel string - OTELConfig OTELConfigSpec + OTELConfig AggregatorOTELConfig +} + +type AggregatorOTELConfig struct { + Processors *AggregatorOTELProcessors + Exporters *AggregatorOTELExporters +} + +type AggregatorOTELProcessors struct { + Batch batchprocessor.Config + MemoryLimiter memorylimiterprocessor.Config +} + +type AggregatorOTELExporters struct { + OTLPHTTP string // otlphttpexporter.Config } type LoggingConfig struct { @@ -61,72 +76,6 @@ type OTELSpec struct { HostMetrics *bool `json:"hostMetrics,omitempty"` } -type OTELConfigSpec struct { - // Batch Processor Configs - BatchProcessor BatchConfig `json:"batch,omitempty"` - - // Memory Limiter Processor Configs - MemoryLimiterProcessor MemoryLimiterConfig `json:"memoryLimiter,omitempty"` -} - -// BatchConfig defines configuration for batch processor. -type BatchConfig struct { - // Timeout sets the time after which a batch will be sent regardless of size. - // When this is set to zero, batched data will be sent immediately. - Timeout time.Duration `json:"timeout,omitempty"` - - // SendBatchSize is the size of a batch which after hit, will trigger it to be sent. - // When this is set to zero, the batch size is ignored and data will be sent immediately - // subject to only send_batch_max_size. - SendBatchSize uint32 `json:"sendBatchSize,omitempty"` - - // SendBatchMaxSize is the maximum size of a batch. It must be larger than SendBatchSize. - // Larger batches are split into smaller units. - // Default value is 0, that means no maximum size. - SendBatchMaxSize uint32 `json:"sendBatchMaxSize,omitempty"` - - // MetadataKeys is a list of client.Metadata keys that will be - // used to form distinct batchers. If this setting is empty, - // a single batcher instance will be used. When this setting - // is not empty, one batcher will be used per distinct - // combination of values for the listed metadata keys. - // - // Empty value and unset metadata are treated as distinct cases. - // - // Entries are case-insensitive. Duplicated entries will - // trigger a validation error. - MetadataKeys []string `json:"metadataKeys,omitempty"` - - // MetadataCardinalityLimit indicates the maximum number of - // batcher instances that will be created through a distinct - // combination of MetadataKeys. - MetadataCardinalityLimit uint32 `json:"metadataCardinalityLimit,omitempty"` -} - -// MemoryLimiterConfig defines configuration for memory memoryLimiter processor. -type MemoryLimiterConfig struct { - // CheckInterval is the time between measurements of memory usage for the - // purposes of avoiding going over the limits. Defaults to zero, so no - // checks will be performed. - CheckInterval time.Duration `json:"checkInterval,omitempty"` - - // MemoryLimitMiB is the maximum amount of memory, in MiB, targeted to be - // allocated by the process. - MemoryLimitMiB uint32 `json:"limitMib,omitempty"` - - // MemorySpikeLimitMiB is the maximum, in MiB, spike expected between the - // measurements of memory usage. - MemorySpikeLimitMiB uint32 `json:"spikeLimitMib,omitempty"` - - // MemoryLimitPercentage is the maximum amount of memory, in %, targeted to be - // allocated by the process. The fixed memory settings MemoryLimitMiB has a higher precedence. - MemoryLimitPercentage uint32 `json:"limitPercentage,omitempty"` - - // MemorySpikePercentage is the maximum, in percents against the total memory, - // spike expected between the measurements of memory usage. - MemorySpikePercentage uint32 `json:"spikeLimitPercentage,omitempty"` -} - type ScrapeConfig struct { JobName string `json:"jobName,omitempty"` Targets []string `json:"targets,omitempty"` diff --git a/pkg/resources/collector/templates.go b/pkg/resources/collector/templates.go index 6703685dfa..f2f1e59563 100644 --- a/pkg/resources/collector/templates.go +++ b/pkg/resources/collector/templates.go @@ -272,9 +272,9 @@ processors: send_batch_size: 1000 timeout: 15s memory_limiter: - limit_mib: 1000 - spike_limit_mib: 350 - check_interval: 1s + limit_mib: {{ .OTELConfig.Processors.MemoryLimiter.MemoryLimitMiB }} + spike_limit_mib: {{ .OTELConfig.Processors.MemoryLimiter.MemorySpikeLimitMiB }} + check_interval: {{ .OTELConfig.Processors.MemoryLimiter.CheckInterval }} transform: log_statements: - context: log diff --git a/pkg/resources/collector/workloads.go b/pkg/resources/collector/workloads.go index 1850c4b352..5a5e2703bc 100644 --- a/pkg/resources/collector/workloads.go +++ b/pkg/resources/collector/workloads.go @@ -12,6 +12,7 @@ import ( "github.com/rancher/opni/pkg/resources" opnimeta "github.com/rancher/opni/pkg/util/meta" "github.com/samber/lo" + "go.opentelemetry.io/collector/processor/memorylimiterprocessor" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -73,7 +74,17 @@ func (r *Reconciler) getAggregatorConfig( AgentEndpoint: r.collector.Spec.AgentEndpoint, Containerized: true, LogLevel: r.collector.Spec.LogLevel, - OTELConfig: *r.collector.Spec.OTELSpec, + OTELConfig: otel.AggregatorOTELConfig{ + Processors: &otel.AggregatorOTELProcessors{ + MemoryLimiter: memorylimiterprocessor.Config{ + CheckInterval: r.collector.Spec.OTELConfigSpec.MemoryLimiterProcessor.CheckInterval, + MemoryLimitMiB: r.collector.Spec.OTELConfigSpec.MemoryLimiterProcessor.MemoryLimitMiB, + MemorySpikeLimitMiB: r.collector.Spec.OTELConfigSpec.MemoryLimiterProcessor.MemorySpikeLimitMiB, + MemoryLimitPercentage: r.collector.Spec.OTELConfigSpec.MemoryLimiterProcessor.MemoryLimitPercentage, + MemorySpikePercentage: r.collector.Spec.OTELConfigSpec.MemoryLimiterProcessor.MemorySpikePercentage, + }, + }, + }, } } diff --git a/plugins/logging/pkg/agent/drivers/kubernetes_manager/kubernetes_manager.go b/plugins/logging/pkg/agent/drivers/kubernetes_manager/kubernetes_manager.go index d1d88239ae..61313d585f 100644 --- a/plugins/logging/pkg/agent/drivers/kubernetes_manager/kubernetes_manager.go +++ b/plugins/logging/pkg/agent/drivers/kubernetes_manager/kubernetes_manager.go @@ -324,20 +324,12 @@ func (m *KubernetesManagerDriver) buildEmptyCollector() *opnicorev1beta1.Collect }, SystemNamespace: m.Namespace, AgentEndpoint: otel.AgentEndpoint(serviceName), - OTELConfigSpec: &otel.OTELConfigSpec{ - MemoryLimiterProcessor: otel.MemoryLimiterConfig{ + OTELConfigSpec: &opnicorev1beta1.OTELConfigSpec{ + MemoryLimiterProcessor: opnicorev1beta1.MemoryLimiterConfig{ MemoryLimitMiB: 1000, MemorySpikeLimitMiB: 350, CheckInterval: 1 * time.Second, }, - BatchProcessor: otel.BatchConfig{ - SendBatchSize: 1000, - Timeout: 15 * time.Second, - }, - // OTLPSendingQueue: exporterhelper.QueueSettings{ - // NumConsumers: 4, - // QueueSize: 100, - // }, }, }, } From 784145260c0a65c1e6e06ff51c55745cd9e9b38a Mon Sep 17 00:00:00 2001 From: Joao Henri Date: Thu, 5 Oct 2023 16:44:15 -0300 Subject: [PATCH 05/21] Allow fine tuning for aggregator's batch processor Signed-off-by: Joao Henri --- apis/core/v1beta1/collector_types.go | 73 ++++++++++++-- apis/core/v1beta1/zz_generated.deepcopy.go | 97 +++++++++++++++++-- config/crd/bases/core.opni.io_collectors.yaml | 83 ++++++++++++---- pkg/otel/types.go | 8 +- pkg/resources/collector/templates.go | 5 +- pkg/resources/collector/workloads.go | 27 +++++- .../kubernetes_manager/kubernetes_manager.go | 14 ++- 7 files changed, 258 insertions(+), 49 deletions(-) diff --git a/apis/core/v1beta1/collector_types.go b/apis/core/v1beta1/collector_types.go index 94afb9f81f..f5c52adbf1 100644 --- a/apis/core/v1beta1/collector_types.go +++ b/apis/core/v1beta1/collector_types.go @@ -26,7 +26,7 @@ type CollectorSpec struct { MetricsConfig *corev1.LocalObjectReference `json:"metricsConfig,omitempty"` ConfigReloader *ConfigReloaderSpec `json:"configReloader,omitempty"` LogLevel string `json:"logLevel,omitempty"` - OTELConfigSpec *OTELConfigSpec `json:"otelSpec,omitempty"` + OTELConfigSpec *OTELConfigSpec `json:"otelCollectorSpec,omitempty"` } type ConfigReloaderSpec struct { @@ -34,26 +34,35 @@ type ConfigReloaderSpec struct { } type OTELConfigSpec struct { - // Memory Limiter Processor Configs - MemoryLimiterProcessor MemoryLimiterConfig `json:"memoryLimiter,omitempty"` + Processors OTELProcessors `json:"processors,omitempty"` + Exporters OTELExporters `json:"exporters,omitempty"` } -// MemoryLimiterConfig defines configuration for the memoryLimiter processor. -type MemoryLimiterConfig struct { +type OTELProcessors struct { + Batch BatchProcessorConfig `json:"batch,omitempty"` + MemoryLimiter MemoryLimiterProcessorConfig `json:"memoryLimiter,omitempty"` +} + +type OTELExporters struct { + OTLPHTTP OTLPHTTPExporterConfig `json:"otlphttp,omitempty"` +} + +type MemoryLimiterProcessorConfig struct { // CheckInterval is the time between measurements of memory usage for the // purposes of avoiding going over the limits. Defaults to zero, so no // checks will be performed. - //+kubebuilder:default="1s" + // +kubebuilder:default="1s" + // +kubebuilder:validation:Format=duration CheckInterval time.Duration `json:"checkInterval,omitempty"` // MemoryLimitMiB is the maximum amount of memory, in MiB, targeted to be // allocated by the process. - //+kubebuilder:default=1000 + // +kubebuilder:default=1000 MemoryLimitMiB uint32 `json:"limitMib,omitempty"` // MemorySpikeLimitMiB is the maximum, in MiB, spike expected between the // measurements of memory usage. - //+kubebuilder:default=350 + // +kubebuilder:default=350 MemorySpikeLimitMiB uint32 `json:"spikeLimitMib,omitempty"` // MemoryLimitPercentage is the maximum amount of memory, in %, targeted to be @@ -65,6 +74,54 @@ type MemoryLimiterConfig struct { MemorySpikePercentage uint32 `json:"spikeLimitPercentage,omitempty"` } +type BatchProcessorConfig struct { + // Timeout sets the time after which a batch will be sent regardless of size. + // When this is set to zero, batched data will be sent immediately. + // +kubebuilder:default="15s" + // +kubebuilder:validation:Format=duration + Timeout time.Duration `json:"timeout,omitempty"` + + // SendBatchSize is the size of a batch which after hit, will trigger it to be sent. + // When this is set to zero, the batch size is ignored and data will be sent immediately + // subject to only send_batch_max_size. + // +kubebuilder:default=1000 + SendBatchSize uint32 `json:"sendBatchSize,omitempty"` + + // SendBatchMaxSize is the maximum size of a batch. It must be larger than SendBatchSize. + // Larger batches are split into smaller units. + // Default value is 0, that means no maximum size. + SendBatchMaxSize uint32 `json:"sendBatchMaxSize,omitempty"` + + // MetadataKeys is a list of client.Metadata keys that will be + // used to form distinct batchers. If this setting is empty, + // a single batcher instance will be used. When this setting + // is not empty, one batcher will be used per distinct + // combination of values for the listed metadata keys. + // + // Empty value and unset metadata are treated as distinct cases. + // + // Entries are case-insensitive. Duplicated entries will + // trigger a validation error. + MetadataKeys []string `json:"metadataKeys,omitempty"` + + // MetadataCardinalityLimit indicates the maximum number of + // batcher instances that will be created through a distinct + // combination of MetadataKeys. + MetadataCardinalityLimit uint32 `json:"metadataCardinalityLimit,omitempty"` +} + +type OTLPHTTPSendingQueue struct { + // Enabled indicates whether to not enqueue batches before sending to the consumerSender. + Enabled bool `json:"enabled,omitempty"` + // NumConsumers is the number of consumers from the queue. + NumConsumers int `json:"numConsumers,omitempty"` + // QueueSize is the maximum number of batches allowed in queue at a given time. + QueueSize int `json:"queueSize,omitempty"` +} +type OTLPHTTPExporterConfig struct { + SendingQueue OTLPHTTPSendingQueue `json:"sendingQueue,omitempty"` +} + // CollectorStatus defines the observed state of Collector type CollectorStatus struct { Conditions []string `json:"conditions,omitempty"` diff --git a/apis/core/v1beta1/zz_generated.deepcopy.go b/apis/core/v1beta1/zz_generated.deepcopy.go index f98a950656..33a40c1484 100644 --- a/apis/core/v1beta1/zz_generated.deepcopy.go +++ b/apis/core/v1beta1/zz_generated.deepcopy.go @@ -262,6 +262,26 @@ func (in *AuthSpec) DeepCopy() *AuthSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BatchProcessorConfig) DeepCopyInto(out *BatchProcessorConfig) { + *out = *in + if in.MetadataKeys != nil { + in, out := &in.MetadataKeys, &out.MetadataKeys + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BatchProcessorConfig. +func (in *BatchProcessorConfig) DeepCopy() *BatchProcessorConfig { + if in == nil { + return nil + } + out := new(BatchProcessorConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BootstrapToken) DeepCopyInto(out *BootstrapToken) { *out = *in @@ -404,7 +424,7 @@ func (in *CollectorSpec) DeepCopyInto(out *CollectorSpec) { if in.OTELConfigSpec != nil { in, out := &in.OTELConfigSpec, &out.OTELConfigSpec *out = new(OTELConfigSpec) - **out = **in + (*in).DeepCopyInto(*out) } } @@ -922,16 +942,16 @@ func (in *LoggingClusterStatus) DeepCopy() *LoggingClusterStatus { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *MemoryLimiterConfig) DeepCopyInto(out *MemoryLimiterConfig) { +func (in *MemoryLimiterProcessorConfig) DeepCopyInto(out *MemoryLimiterProcessorConfig) { *out = *in } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MemoryLimiterConfig. -func (in *MemoryLimiterConfig) DeepCopy() *MemoryLimiterConfig { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MemoryLimiterProcessorConfig. +func (in *MemoryLimiterProcessorConfig) DeepCopy() *MemoryLimiterProcessorConfig { if in == nil { return nil } - out := new(MemoryLimiterConfig) + out := new(MemoryLimiterProcessorConfig) in.DeepCopyInto(out) return out } @@ -1151,7 +1171,8 @@ func (in *NatsStatus) DeepCopy() *NatsStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OTELConfigSpec) DeepCopyInto(out *OTELConfigSpec) { *out = *in - out.MemoryLimiterProcessor = in.MemoryLimiterProcessor + in.Processors.DeepCopyInto(&out.Processors) + out.Exporters = in.Exporters } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OTELConfigSpec. @@ -1164,6 +1185,70 @@ func (in *OTELConfigSpec) DeepCopy() *OTELConfigSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OTELExporters) DeepCopyInto(out *OTELExporters) { + *out = *in + out.OTLPHTTP = in.OTLPHTTP +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OTELExporters. +func (in *OTELExporters) DeepCopy() *OTELExporters { + if in == nil { + return nil + } + out := new(OTELExporters) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OTELProcessors) DeepCopyInto(out *OTELProcessors) { + *out = *in + in.Batch.DeepCopyInto(&out.Batch) + out.MemoryLimiter = in.MemoryLimiter +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OTELProcessors. +func (in *OTELProcessors) DeepCopy() *OTELProcessors { + if in == nil { + return nil + } + out := new(OTELProcessors) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OTLPHTTPExporterConfig) DeepCopyInto(out *OTLPHTTPExporterConfig) { + *out = *in + out.SendingQueue = in.SendingQueue +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OTLPHTTPExporterConfig. +func (in *OTLPHTTPExporterConfig) DeepCopy() *OTLPHTTPExporterConfig { + if in == nil { + return nil + } + out := new(OTLPHTTPExporterConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OTLPHTTPSendingQueue) DeepCopyInto(out *OTLPHTTPSendingQueue) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OTLPHTTPSendingQueue. +func (in *OTLPHTTPSendingQueue) DeepCopy() *OTLPHTTPSendingQueue { + if in == nil { + return nil + } + out := new(OTLPHTTPSendingQueue) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OpenIDConfigSpec) DeepCopyInto(out *OpenIDConfigSpec) { *out = *in diff --git a/config/crd/bases/core.opni.io_collectors.yaml b/config/crd/bases/core.opni.io_collectors.yaml index 97dcdf5306..05031e610e 100644 --- a/config/crd/bases/core.opni.io_collectors.yaml +++ b/config/crd/bases/core.opni.io_collectors.yaml @@ -69,28 +69,71 @@ spec: type: string type: object x-kubernetes-map-type: atomic - otelSpec: + otelCollectorSpec: properties: - memoryLimiter: + exporters: properties: - checkInterval: - default: 1s - format: int64 - type: integer - limitMib: - default: 1000 - format: int32 - type: integer - limitPercentage: - format: int32 - type: integer - spikeLimitMib: - default: 350 - format: int32 - type: integer - spikeLimitPercentage: - format: int32 - type: integer + otlphttp: + properties: + sendingQueue: + properties: + enabled: + type: boolean + numConsumers: + type: integer + queueSize: + type: integer + type: object + type: object + type: object + processors: + properties: + batch: + properties: + metadataCardinalityLimit: + format: int32 + type: integer + metadataKeys: + items: + type: string + type: array + sendBatchMaxSize: + format: int32 + type: integer + sendBatchSize: + default: 1000 + format: int32 + type: integer + timeout: + allOf: + - format: int64 + - format: duration + default: 15s + type: integer + type: object + memoryLimiter: + properties: + checkInterval: + allOf: + - format: int64 + - format: duration + default: 1s + type: integer + limitMib: + default: 1000 + format: int32 + type: integer + limitPercentage: + format: int32 + type: integer + spikeLimitMib: + default: 350 + format: int32 + type: integer + spikeLimitPercentage: + format: int32 + type: integer + type: object type: object type: object systemNamespace: diff --git a/pkg/otel/types.go b/pkg/otel/types.go index 59d1752310..1a7efb58a1 100644 --- a/pkg/otel/types.go +++ b/pkg/otel/types.go @@ -42,7 +42,7 @@ type AggregatorConfig struct { type AggregatorOTELConfig struct { Processors *AggregatorOTELProcessors - Exporters *AggregatorOTELExporters + // Exporters *AggregatorOTELExporters } type AggregatorOTELProcessors struct { @@ -50,9 +50,9 @@ type AggregatorOTELProcessors struct { MemoryLimiter memorylimiterprocessor.Config } -type AggregatorOTELExporters struct { - OTLPHTTP string // otlphttpexporter.Config -} +// type AggregatorOTELExporters struct { +// OTLPHTTP otlphttpexporter.Config +// } type LoggingConfig struct { Enabled bool diff --git a/pkg/resources/collector/templates.go b/pkg/resources/collector/templates.go index f2f1e59563..c84f519aee 100644 --- a/pkg/resources/collector/templates.go +++ b/pkg/resources/collector/templates.go @@ -269,8 +269,9 @@ receivers: processors: batch: - send_batch_size: 1000 - timeout: 15s + timeout: {{ .OTELConfig.Processors.Batch.Timeout }} + send_batch_size: {{ .OTELConfig.Processors.Batch.SendBatchSize }} + send_batch_max_size: timeout: {{ .OTELConfig.Processors.Batch.SendBatchMaxSize }} memory_limiter: limit_mib: {{ .OTELConfig.Processors.MemoryLimiter.MemoryLimitMiB }} spike_limit_mib: {{ .OTELConfig.Processors.MemoryLimiter.MemorySpikeLimitMiB }} diff --git a/pkg/resources/collector/workloads.go b/pkg/resources/collector/workloads.go index 5a5e2703bc..8781d74cfa 100644 --- a/pkg/resources/collector/workloads.go +++ b/pkg/resources/collector/workloads.go @@ -12,6 +12,7 @@ import ( "github.com/rancher/opni/pkg/resources" opnimeta "github.com/rancher/opni/pkg/util/meta" "github.com/samber/lo" + "go.opentelemetry.io/collector/processor/batchprocessor" "go.opentelemetry.io/collector/processor/memorylimiterprocessor" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -77,13 +78,29 @@ func (r *Reconciler) getAggregatorConfig( OTELConfig: otel.AggregatorOTELConfig{ Processors: &otel.AggregatorOTELProcessors{ MemoryLimiter: memorylimiterprocessor.Config{ - CheckInterval: r.collector.Spec.OTELConfigSpec.MemoryLimiterProcessor.CheckInterval, - MemoryLimitMiB: r.collector.Spec.OTELConfigSpec.MemoryLimiterProcessor.MemoryLimitMiB, - MemorySpikeLimitMiB: r.collector.Spec.OTELConfigSpec.MemoryLimiterProcessor.MemorySpikeLimitMiB, - MemoryLimitPercentage: r.collector.Spec.OTELConfigSpec.MemoryLimiterProcessor.MemoryLimitPercentage, - MemorySpikePercentage: r.collector.Spec.OTELConfigSpec.MemoryLimiterProcessor.MemorySpikePercentage, + CheckInterval: r.collector.Spec.OTELConfigSpec.Processors.MemoryLimiter.CheckInterval, + MemoryLimitMiB: r.collector.Spec.OTELConfigSpec.Processors.MemoryLimiter.MemoryLimitMiB, + MemorySpikeLimitMiB: r.collector.Spec.OTELConfigSpec.Processors.MemoryLimiter.MemorySpikeLimitMiB, + MemoryLimitPercentage: r.collector.Spec.OTELConfigSpec.Processors.MemoryLimiter.MemoryLimitPercentage, + MemorySpikePercentage: r.collector.Spec.OTELConfigSpec.Processors.MemoryLimiter.MemorySpikePercentage, + }, + Batch: batchprocessor.Config{ + Timeout: r.collector.Spec.OTELConfigSpec.Processors.Batch.Timeout, + SendBatchSize: r.collector.Spec.OTELConfigSpec.Processors.Batch.SendBatchSize, + SendBatchMaxSize: r.collector.Spec.OTELConfigSpec.Processors.Batch.SendBatchMaxSize, + MetadataKeys: r.collector.Spec.OTELConfigSpec.Processors.Batch.MetadataKeys, + MetadataCardinalityLimit: r.collector.Spec.OTELConfigSpec.Processors.Batch.MetadataCardinalityLimit, }, }, + // Exporters: &otel.AggregatorOTELExporters{ + // OTLPHTTP: otlphttpexporter.Config{ + // QueueSettings: exporterhelper.QueueSettings{ + // Enabled: r.collector.Spec.OTELConfigSpec.Exporters.OTLPHTTP.SendingQueue.Enabled, + // NumConsumers: r.collector.Spec.OTELConfigSpec.Exporters.OTLPHTTP.SendingQueue.NumConsumers, + // QueueSize: r.collector.Spec.OTELConfigSpec.Exporters.OTLPHTTP.SendingQueue.QueueSize, + // }, + // }, + // }, }, } } diff --git a/plugins/logging/pkg/agent/drivers/kubernetes_manager/kubernetes_manager.go b/plugins/logging/pkg/agent/drivers/kubernetes_manager/kubernetes_manager.go index 61313d585f..0d2abfd07e 100644 --- a/plugins/logging/pkg/agent/drivers/kubernetes_manager/kubernetes_manager.go +++ b/plugins/logging/pkg/agent/drivers/kubernetes_manager/kubernetes_manager.go @@ -325,10 +325,16 @@ func (m *KubernetesManagerDriver) buildEmptyCollector() *opnicorev1beta1.Collect SystemNamespace: m.Namespace, AgentEndpoint: otel.AgentEndpoint(serviceName), OTELConfigSpec: &opnicorev1beta1.OTELConfigSpec{ - MemoryLimiterProcessor: opnicorev1beta1.MemoryLimiterConfig{ - MemoryLimitMiB: 1000, - MemorySpikeLimitMiB: 350, - CheckInterval: 1 * time.Second, + Processors: opnicorev1beta1.OTELProcessors{ + MemoryLimiter: opnicorev1beta1.MemoryLimiterProcessorConfig{ + MemoryLimitMiB: 1000, + MemorySpikeLimitMiB: 350, + CheckInterval: 1 * time.Second, + }, + Batch: opnicorev1beta1.BatchProcessorConfig{ + SendBatchSize: 1000, + Timeout: 15 * time.Second, + }, }, }, }, From 72f9e8c773fa9751519008c776e28439e5e47493 Mon Sep 17 00:00:00 2001 From: Joao Henri Date: Thu, 5 Oct 2023 17:47:19 -0300 Subject: [PATCH 06/21] Add OTLPHTTP exporter for aggregator fine tuning Signed-off-by: Joao Henri --- go.mod | 12 ++++++-- go.sum | 28 +++++++++++++++++-- pkg/otel/types.go | 9 +++--- pkg/resources/collector/workloads.go | 20 +++++++------ .../kubernetes_manager/kubernetes_manager.go | 9 ++++++ 5 files changed, 61 insertions(+), 17 deletions(-) diff --git a/go.mod b/go.mod index 53e617fea7..3dae55be20 100644 --- a/go.mod +++ b/go.mod @@ -115,11 +115,12 @@ require ( go.etcd.io/etcd/api/v3 v3.5.9 go.etcd.io/etcd/client/v3 v3.5.9 go.etcd.io/etcd/etcdctl/v3 v3.5.9 + go.opentelemetry.io/collector/exporter/otlphttpexporter v0.80.0 go.opentelemetry.io/collector/pdata v1.0.0-rcv0014 go.opentelemetry.io/collector/processor/batchprocessor v0.80.0 go.opentelemetry.io/collector/processor/memorylimiterprocessor v0.80.0 go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.40.0 - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0 + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.1-0.20230612162650-64be7e574a17 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0 go.opentelemetry.io/contrib/propagators/autoprop v0.42.0 go.opentelemetry.io/otel v1.16.0 @@ -173,6 +174,13 @@ require ( github.com/tklauser/go-sysconf v0.3.11 // indirect github.com/tklauser/numcpus v0.6.0 // indirect github.com/yusufpapurcu/wmi v1.2.3 // indirect + go.opentelemetry.io/collector/config/configauth v0.80.0 // indirect + go.opentelemetry.io/collector/config/configcompression v0.80.0 // indirect + go.opentelemetry.io/collector/config/confighttp v0.80.0 // indirect + go.opentelemetry.io/collector/config/configopaque v0.80.0 // indirect + go.opentelemetry.io/collector/config/configtls v0.80.0 // indirect + go.opentelemetry.io/collector/config/internal v0.80.0 // indirect + go.opentelemetry.io/collector/extension/auth v0.80.0 // indirect ) require ( @@ -515,7 +523,7 @@ require ( go.opentelemetry.io/collector/config/configtelemetry v0.80.0 // indirect go.opentelemetry.io/collector/confmap v0.80.1-0.20230629234129-50c94c941969 // indirect go.opentelemetry.io/collector/consumer v0.80.0 // indirect - go.opentelemetry.io/collector/exporter v0.80.0 // indirect + go.opentelemetry.io/collector/exporter v0.80.0 go.opentelemetry.io/collector/extension v0.80.0 // indirect go.opentelemetry.io/collector/featuregate v1.0.0-rcv0013 // indirect go.opentelemetry.io/collector/processor v0.80.0 // indirect diff --git a/go.sum b/go.sum index 9797db0646..e5a32b630f 100644 --- a/go.sum +++ b/go.sum @@ -1861,6 +1861,8 @@ github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/mostynb/go-grpc-compression v1.1.19 h1:T57PAuic1tP+6PVgPpeZ15Nx6r6FqUqdnXeAslzxn0Y= +github.com/mostynb/go-grpc-compression v1.1.19/go.mod h1:oidYvYyefMmhcuvU8fLJ8FfZyTyVzJ6SkmD5fIKgRe8= github.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ= github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= @@ -2487,16 +2489,36 @@ go.opentelemetry.io/collector v0.80.0 h1:jAsJzGhH++WI9lCGSS0ykfo+XX4H3Fp0bs5UaZP go.opentelemetry.io/collector v0.80.0/go.mod h1:WuFR0SJJbUsBF7YF8w0vjvfoxodzIAkFt+HIeF9eQ5A= go.opentelemetry.io/collector/component v0.80.0 h1:yL+YJDnsAYF8Yviu4MceqNfQN16PDi8hIMUEllxNBlA= go.opentelemetry.io/collector/component v0.80.0/go.mod h1:4BMtyTmTmw7u3GCibdI+DY8S3q+Hk3W6CLTOk+WHoGU= +go.opentelemetry.io/collector/config/configauth v0.80.0 h1:+8zH36zhLNp/efwxoJwdp1ORIV/RkjoTp9I3OOzvpOE= +go.opentelemetry.io/collector/config/configauth v0.80.0/go.mod h1:F0W34vz4zbqFhxJjvpwDKIaKE2t1baGuhEqreTQJTI0= +go.opentelemetry.io/collector/config/configcompression v0.80.0 h1:lmSOh1Y+tYK/Nzr6ge7V3Oe2csxj3/Q/cUFuEvEQ2kU= +go.opentelemetry.io/collector/config/configcompression v0.80.0/go.mod h1:xhHm1sEH7BTECAJo1xn64NMxeIvZGKdVGdSKUUc+YuM= +go.opentelemetry.io/collector/config/configgrpc v0.80.0 h1:xpPu+FNnJR3fcqg+JxBvMBIA+xtn3iv4/bWIEQCwIT4= +go.opentelemetry.io/collector/config/configgrpc v0.80.0/go.mod h1:3xKnuF9F8JqnbXz8vEN7ySasEFATwAea/lGVld4BBRo= +go.opentelemetry.io/collector/config/confighttp v0.80.0 h1:B2wkgDTs8kmzLDhm3zEKAq7TvjEyEHVuL0pc75Sv0mo= +go.opentelemetry.io/collector/config/confighttp v0.80.0/go.mod h1:uj8coqg+AGHi3PAlVxDNIBjs5QrS499Ab+ywo6KwvBA= +go.opentelemetry.io/collector/config/confignet v0.80.0 h1:ouYD6fPs2DCVmqE4h6XpaSyZi9D/Q2j3oSjjPCVSoeo= +go.opentelemetry.io/collector/config/confignet v0.80.0/go.mod h1:unOg7BZvpt6T5xsf+LyeOQvUhD8ld/2AbfOsmUZ/bPM= +go.opentelemetry.io/collector/config/configopaque v0.80.0 h1:+Jwkd/X9jtbj3PkFk+p6H3GdD/qEREOJ47L+hs12ckw= +go.opentelemetry.io/collector/config/configopaque v0.80.0/go.mod h1:pM1oy6gasukw3H6jAvc9Q9OtFaaY2IbfeuwCPAjOgXc= go.opentelemetry.io/collector/config/configtelemetry v0.80.0 h1:IE/uyLy/roZAnj+jxiYY1QGjP1VYzzen7e1k6bVDBoA= go.opentelemetry.io/collector/config/configtelemetry v0.80.0/go.mod h1:KEYQRiYJdx38iZkvcLKBZWH9fK4NeafxBwGRrRKMgyA= +go.opentelemetry.io/collector/config/configtls v0.80.0 h1:0NwckEIIzXv2Zz+rLswHb37jsDmMnqo2O4/Fl/+2KFM= +go.opentelemetry.io/collector/config/configtls v0.80.0/go.mod h1:fO1VgdtrcgcVA3Y2vB/YQvTh2tNNFW0R0NjWrtvjTOQ= +go.opentelemetry.io/collector/config/internal v0.80.0 h1:9mOlmdHiaNKS5HTVJ1+COx/URDjzX0zvUKcbsLC6S8k= +go.opentelemetry.io/collector/config/internal v0.80.0/go.mod h1:RKcLV1gQxhgwx+6rlPYsvGMq1RZNne3UeOUZkHxJnIg= go.opentelemetry.io/collector/confmap v0.80.1-0.20230629234129-50c94c941969 h1:kpNiFFSdOBbKQubR+VXYiTN3iy9ZXYv58UiLk/xVdxE= go.opentelemetry.io/collector/confmap v0.80.1-0.20230629234129-50c94c941969/go.mod h1:iCTnTqGgZZJumhJxpY7rrJz9UQ/0zjPmsJz2Z7Tp4RY= go.opentelemetry.io/collector/consumer v0.80.0 h1:IQfkrNvZ6/ZkfQs/d6JENgfYPE3g3NcVcy1eC8nku58= go.opentelemetry.io/collector/consumer v0.80.0/go.mod h1://MinWlTMlL44B8WvUzLnYWBKgxfgGE8grpxoUiL9aQ= go.opentelemetry.io/collector/exporter v0.80.0 h1:BjwrMGhlrJ6IyhOOkPruzycCk7dv6vuf+VORzsVtcLI= go.opentelemetry.io/collector/exporter v0.80.0/go.mod h1:qcvFsxN8AaSxJ60WxSF0zGUTCMhGBWY3Coa9iiONWbg= +go.opentelemetry.io/collector/exporter/otlphttpexporter v0.80.0 h1:2rd+t/LCr7nOOBiHKaeSVbJKHmFbANkUMnumGYVCBCM= +go.opentelemetry.io/collector/exporter/otlphttpexporter v0.80.0/go.mod h1:vBz0hJe32gAEp8LWb8e7VwA96h6qlRxToy2acYV6lKw= go.opentelemetry.io/collector/extension v0.80.0 h1:Nx2QpwBAwcEcCVpqqqJIzpkhniGRhLC5Tr5s+e5H5p0= go.opentelemetry.io/collector/extension v0.80.0/go.mod h1:r61aYWq9NYZP22+LVqiLOEC0cHGksU+Ys5H+KWT3HgA= +go.opentelemetry.io/collector/extension/auth v0.80.0 h1:BElM8HXYVho2ZikMS8OpQQjmaMizB3qFGJ+kGZ4cyoI= +go.opentelemetry.io/collector/extension/auth v0.80.0/go.mod h1:wDpwb37PxV/aH/kecpPXtJqGSmiOYUyeLuQvRmWciAA= go.opentelemetry.io/collector/featuregate v1.0.0-rcv0013 h1:tiTUG9X/gEDN1oDYQOBVUFYQfhUG2CvgW9VhBc2uk1U= go.opentelemetry.io/collector/featuregate v1.0.0-rcv0013/go.mod h1:0mE3mDLmUrOXVoNsuvj+7dV14h/9HFl/Fy9YTLoLObo= go.opentelemetry.io/collector/pdata v1.0.0-rcv0014 h1:iT5qH0NLmkGeIdDtnBogYDx7L58t6CaWGL378DEo2QY= @@ -2509,12 +2531,14 @@ go.opentelemetry.io/collector/processor/memorylimiterprocessor v0.80.0 h1:ZK9GRI go.opentelemetry.io/collector/processor/memorylimiterprocessor v0.80.0/go.mod h1:Saosjjmi1q1vkLJ8EH/jTvhp37/tTPDa2Sa28G8jCqk= go.opentelemetry.io/collector/receiver v0.80.0 h1:+dJn57mWaksgKDnR06mAu5aReOcqg1pdgC6zqqLmtZ4= go.opentelemetry.io/collector/receiver v0.80.0/go.mod h1:DsyjwQZjlcbwoY4Kq4yx1tNL8Z4d8E7e4tbQYzBUw/g= +go.opentelemetry.io/collector/receiver/otlpreceiver v0.80.0 h1:oWWztTJlNOp81JE27DRaHKJnyAympUn5boDI7dVG78I= +go.opentelemetry.io/collector/receiver/otlpreceiver v0.80.0/go.mod h1:bb6RD5rDEwuruBw/yRlAQ4a3e0VpD1jKNztqbPnvC3k= go.opentelemetry.io/collector/semconv v0.81.0 h1:lCYNNo3powDvFIaTPP2jDKIrBiV1T92NK4QgL/aHYXw= go.opentelemetry.io/collector/semconv v0.81.0/go.mod h1:TlYPtzvsXyHOgr5eATi43qEMqwSmIziivJB2uctKswo= go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.40.0 h1:E4MMXDxufRnIHXhoTNOlNsdkWpC5HdLhfj84WNRKPkc= go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.40.0/go.mod h1:A8+gHkpqTfMKxdKWq1pp360nAs096K26CH5Sm2YHDdA= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0 h1:ZOLJc06r4CB42laIXg/7udr0pbZyuAihN10A/XuiQRY= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0/go.mod h1:5z+/ZWJQKXa9YT34fQNx5K8Hd1EoIhvtUygUQPqEOgQ= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.1-0.20230612162650-64be7e574a17 h1:mdcNStUIXngF/mH3xxAo4nbR4g65IXqLL1SvYMjz7JQ= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.1-0.20230612162650-64be7e574a17/go.mod h1:N2Nw/UmmvQn0yCnaUzvsWzTWIeffYIdFteg6mxqCWII= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0 h1:pginetY7+onl4qN1vl0xW/V/v6OBZ0vVdH+esuJgvmM= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0/go.mod h1:XiYsayHc36K3EByOO6nbAXnAWbrUxdjUROCEeeROOH8= go.opentelemetry.io/contrib/propagators/autoprop v0.42.0 h1:s2RzYOAqHVgG23q8fPWYChobUoZM6rJZ98EnylJr66w= diff --git a/pkg/otel/types.go b/pkg/otel/types.go index 1a7efb58a1..1c125a3c67 100644 --- a/pkg/otel/types.go +++ b/pkg/otel/types.go @@ -6,6 +6,7 @@ import ( "github.com/rancher/opni/pkg/util" "github.com/samber/lo" + "go.opentelemetry.io/collector/exporter/otlphttpexporter" "go.opentelemetry.io/collector/processor/batchprocessor" "go.opentelemetry.io/collector/processor/memorylimiterprocessor" "google.golang.org/protobuf/types/known/durationpb" @@ -42,7 +43,7 @@ type AggregatorConfig struct { type AggregatorOTELConfig struct { Processors *AggregatorOTELProcessors - // Exporters *AggregatorOTELExporters + Exporters *AggregatorOTELExporters } type AggregatorOTELProcessors struct { @@ -50,9 +51,9 @@ type AggregatorOTELProcessors struct { MemoryLimiter memorylimiterprocessor.Config } -// type AggregatorOTELExporters struct { -// OTLPHTTP otlphttpexporter.Config -// } +type AggregatorOTELExporters struct { + OTLPHTTP otlphttpexporter.Config +} type LoggingConfig struct { Enabled bool diff --git a/pkg/resources/collector/workloads.go b/pkg/resources/collector/workloads.go index 8781d74cfa..60663234f2 100644 --- a/pkg/resources/collector/workloads.go +++ b/pkg/resources/collector/workloads.go @@ -12,6 +12,8 @@ import ( "github.com/rancher/opni/pkg/resources" opnimeta "github.com/rancher/opni/pkg/util/meta" "github.com/samber/lo" + "go.opentelemetry.io/collector/exporter/exporterhelper" + "go.opentelemetry.io/collector/exporter/otlphttpexporter" "go.opentelemetry.io/collector/processor/batchprocessor" "go.opentelemetry.io/collector/processor/memorylimiterprocessor" appsv1 "k8s.io/api/apps/v1" @@ -92,15 +94,15 @@ func (r *Reconciler) getAggregatorConfig( MetadataCardinalityLimit: r.collector.Spec.OTELConfigSpec.Processors.Batch.MetadataCardinalityLimit, }, }, - // Exporters: &otel.AggregatorOTELExporters{ - // OTLPHTTP: otlphttpexporter.Config{ - // QueueSettings: exporterhelper.QueueSettings{ - // Enabled: r.collector.Spec.OTELConfigSpec.Exporters.OTLPHTTP.SendingQueue.Enabled, - // NumConsumers: r.collector.Spec.OTELConfigSpec.Exporters.OTLPHTTP.SendingQueue.NumConsumers, - // QueueSize: r.collector.Spec.OTELConfigSpec.Exporters.OTLPHTTP.SendingQueue.QueueSize, - // }, - // }, - // }, + Exporters: &otel.AggregatorOTELExporters{ + OTLPHTTP: otlphttpexporter.Config{ + QueueSettings: exporterhelper.QueueSettings{ + Enabled: r.collector.Spec.OTELConfigSpec.Exporters.OTLPHTTP.SendingQueue.Enabled, + NumConsumers: r.collector.Spec.OTELConfigSpec.Exporters.OTLPHTTP.SendingQueue.NumConsumers, + QueueSize: r.collector.Spec.OTELConfigSpec.Exporters.OTLPHTTP.SendingQueue.QueueSize, + }, + }, + }, }, } } diff --git a/plugins/logging/pkg/agent/drivers/kubernetes_manager/kubernetes_manager.go b/plugins/logging/pkg/agent/drivers/kubernetes_manager/kubernetes_manager.go index 0d2abfd07e..5862925ca6 100644 --- a/plugins/logging/pkg/agent/drivers/kubernetes_manager/kubernetes_manager.go +++ b/plugins/logging/pkg/agent/drivers/kubernetes_manager/kubernetes_manager.go @@ -336,6 +336,15 @@ func (m *KubernetesManagerDriver) buildEmptyCollector() *opnicorev1beta1.Collect Timeout: 15 * time.Second, }, }, + Exporters: opnicorev1beta1.OTELExporters{ + OTLPHTTP: opnicorev1beta1.OTLPHTTPExporterConfig{ + SendingQueue: opnicorev1beta1.OTLPHTTPSendingQueue{ + Enabled: true, + NumConsumers: 4, + QueueSize: 100, + }, + }, + }, }, }, } From da4a40eb26f9822e614da6c538f271ca278cdacc Mon Sep 17 00:00:00 2001 From: Joao Henri Date: Fri, 6 Oct 2023 11:18:06 -0300 Subject: [PATCH 07/21] fix template Signed-off-by: Joao Henri --- pkg/resources/collector/templates.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/resources/collector/templates.go b/pkg/resources/collector/templates.go index c84f519aee..856664b5e9 100644 --- a/pkg/resources/collector/templates.go +++ b/pkg/resources/collector/templates.go @@ -271,7 +271,7 @@ processors: batch: timeout: {{ .OTELConfig.Processors.Batch.Timeout }} send_batch_size: {{ .OTELConfig.Processors.Batch.SendBatchSize }} - send_batch_max_size: timeout: {{ .OTELConfig.Processors.Batch.SendBatchMaxSize }} + send_batch_max_size: {{ .OTELConfig.Processors.Batch.SendBatchMaxSize }} memory_limiter: limit_mib: {{ .OTELConfig.Processors.MemoryLimiter.MemoryLimitMiB }} spike_limit_mib: {{ .OTELConfig.Processors.MemoryLimiter.MemorySpikeLimitMiB }} From 802795a07e837ebd8fd34426de01c529cb49d843 Mon Sep 17 00:00:00 2001 From: Joao Henri Date: Fri, 6 Oct 2023 11:53:22 -0300 Subject: [PATCH 08/21] Make OTLPHTTP queue configurable Signed-off-by: Joao Henri --- pkg/resources/collector/templates.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/resources/collector/templates.go b/pkg/resources/collector/templates.go index 856664b5e9..2d0d9ef258 100644 --- a/pkg/resources/collector/templates.go +++ b/pkg/resources/collector/templates.go @@ -288,8 +288,9 @@ exporters: tls: insecure: true sending_queue: - num_consumers: 4 - queue_size: 100 + enabled: {{ .OTELConfig.Exporters.OTLPHTTP.SendingQueue.Enabled }} + num_consumers: {{ .OTELConfig.Exporters.OTLPHTTP.SendingQueue.NumConsumers }} + queue_size: {{ .OTELConfig.Exporters.OTLPHTTP.SendingQueue.QueueSize }} retry_on_failure: enabled: true {{ template "metrics-remotewrite-exporter" .}} From 067c2e658425a33d3dde183546880e4a61b401d4 Mon Sep 17 00:00:00 2001 From: Joao Henri Date: Fri, 6 Oct 2023 13:23:25 -0300 Subject: [PATCH 09/21] Add fine tuning for the collector-agent Signed-off-by: Joao Henri --- apis/core/v1beta1/collector_types.go | 60 ++++++--- apis/core/v1beta1/zz_generated.deepcopy.go | 122 ++++++++++++++---- config/crd/bases/core.opni.io_collectors.yaml | 94 ++++++++++---- go.mod | 8 +- go.sum | 9 +- pkg/otel/types.go | 15 +++ pkg/resources/collector/templates.go | 11 +- pkg/resources/collector/workloads.go | 47 +++++-- .../kubernetes_manager/kubernetes_manager.go | 26 +++- 9 files changed, 299 insertions(+), 93 deletions(-) diff --git a/apis/core/v1beta1/collector_types.go b/apis/core/v1beta1/collector_types.go index f5c52adbf1..8f56b5c94c 100644 --- a/apis/core/v1beta1/collector_types.go +++ b/apis/core/v1beta1/collector_types.go @@ -19,34 +19,52 @@ const ( ) type CollectorSpec struct { - opnimeta.ImageSpec `json:",inline,omitempty"` - AgentEndpoint string `json:"agentEndpoint,omitempty"` - SystemNamespace string `json:"systemNamespace,omitempty"` - LoggingConfig *corev1.LocalObjectReference `json:"loggingConfig,omitempty"` - MetricsConfig *corev1.LocalObjectReference `json:"metricsConfig,omitempty"` - ConfigReloader *ConfigReloaderSpec `json:"configReloader,omitempty"` - LogLevel string `json:"logLevel,omitempty"` - OTELConfigSpec *OTELConfigSpec `json:"otelCollectorSpec,omitempty"` + opnimeta.ImageSpec `json:",inline,omitempty"` + AgentEndpoint string `json:"agentEndpoint,omitempty"` + SystemNamespace string `json:"systemNamespace,omitempty"` + LoggingConfig *corev1.LocalObjectReference `json:"loggingConfig,omitempty"` + MetricsConfig *corev1.LocalObjectReference `json:"metricsConfig,omitempty"` + ConfigReloader *ConfigReloaderSpec `json:"configReloader,omitempty"` + LogLevel string `json:"logLevel,omitempty"` + AggregatorOTELConfigSpec *AggregatorOTELConfigSpec `json:"aggregatorOtelCollectorSpec,omitempty"` + NodeOTELConfigSpec *NodeOTELConfigSpec `json:"nodeOtelCollectorSpec,omitempty"` } type ConfigReloaderSpec struct { opnimeta.ImageSpec `json:",inline,omitempty"` } -type OTELConfigSpec struct { - Processors OTELProcessors `json:"processors,omitempty"` - Exporters OTELExporters `json:"exporters,omitempty"` +type AggregatorOTELConfigSpec struct { + Processors AggregatorOTELProcessors `json:"processors,omitempty"` + Exporters AggregatorOTELExporters `json:"exporters,omitempty"` } -type OTELProcessors struct { +type AggregatorOTELProcessors struct { Batch BatchProcessorConfig `json:"batch,omitempty"` MemoryLimiter MemoryLimiterProcessorConfig `json:"memoryLimiter,omitempty"` } -type OTELExporters struct { +type AggregatorOTELExporters struct { OTLPHTTP OTLPHTTPExporterConfig `json:"otlphttp,omitempty"` } +type NodeOTELConfigSpec struct { + Processors NodeOTELProcessors `json:"processors,omitempty"` + Exporters NodeOTELExporters `json:"exporters,omitempty"` +} + +type NodeOTELProcessors struct { + MemoryLimiter MemoryLimiterProcessorConfig `json:"memoryLimiter,omitempty"` +} + +type NodeOTELExporters struct { + OTLP OTLPExporterConfig `json:"otlp,omitempty"` +} + +// MemoryLimiterProcessorConfig has the attributes that we want to make +// available from memorylimiterexporter.Config. +// Also, we extend it with the JSON struct tags needed in order to kubebuilder +// and controller-gen work. type MemoryLimiterProcessorConfig struct { // CheckInterval is the time between measurements of memory usage for the // purposes of avoiding going over the limits. Defaults to zero, so no @@ -74,6 +92,10 @@ type MemoryLimiterProcessorConfig struct { MemorySpikePercentage uint32 `json:"spikeLimitPercentage,omitempty"` } +// BatchProcessorConfig has the attributes that we want to make +// available from batchprocessor.Config. +// Also, we extend it with the JSON struct tags needed in order to kubebuilder +// and controller-gen work. type BatchProcessorConfig struct { // Timeout sets the time after which a batch will be sent regardless of size. // When this is set to zero, batched data will be sent immediately. @@ -110,7 +132,11 @@ type BatchProcessorConfig struct { MetadataCardinalityLimit uint32 `json:"metadataCardinalityLimit,omitempty"` } -type OTLPHTTPSendingQueue struct { +// CollectorSendingQueue has the attributes that we want to make +// available from exporterhelper.QueueSettings. +// Also, we extend it with the JSON struct tags needed in order to kubebuilder +// and controller-gen work. +type CollectorSendingQueue struct { // Enabled indicates whether to not enqueue batches before sending to the consumerSender. Enabled bool `json:"enabled,omitempty"` // NumConsumers is the number of consumers from the queue. @@ -118,8 +144,12 @@ type OTLPHTTPSendingQueue struct { // QueueSize is the maximum number of batches allowed in queue at a given time. QueueSize int `json:"queueSize,omitempty"` } + +type OTLPExporterConfig struct { + SendingQueue CollectorSendingQueue `json:"sendingQueue,omitempty"` +} type OTLPHTTPExporterConfig struct { - SendingQueue OTLPHTTPSendingQueue `json:"sendingQueue,omitempty"` + SendingQueue CollectorSendingQueue `json:"sendingQueue,omitempty"` } // CollectorStatus defines the observed state of Collector diff --git a/apis/core/v1beta1/zz_generated.deepcopy.go b/apis/core/v1beta1/zz_generated.deepcopy.go index 33a40c1484..c8073091cb 100644 --- a/apis/core/v1beta1/zz_generated.deepcopy.go +++ b/apis/core/v1beta1/zz_generated.deepcopy.go @@ -12,6 +12,56 @@ import ( apiv1 "opensearch.opster.io/api/v1" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AggregatorOTELConfigSpec) DeepCopyInto(out *AggregatorOTELConfigSpec) { + *out = *in + in.Processors.DeepCopyInto(&out.Processors) + out.Exporters = in.Exporters +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AggregatorOTELConfigSpec. +func (in *AggregatorOTELConfigSpec) DeepCopy() *AggregatorOTELConfigSpec { + if in == nil { + return nil + } + out := new(AggregatorOTELConfigSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AggregatorOTELExporters) DeepCopyInto(out *AggregatorOTELExporters) { + *out = *in + out.OTLPHTTP = in.OTLPHTTP +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AggregatorOTELExporters. +func (in *AggregatorOTELExporters) DeepCopy() *AggregatorOTELExporters { + if in == nil { + return nil + } + out := new(AggregatorOTELExporters) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AggregatorOTELProcessors) DeepCopyInto(out *AggregatorOTELProcessors) { + *out = *in + in.Batch.DeepCopyInto(&out.Batch) + out.MemoryLimiter = in.MemoryLimiter +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AggregatorOTELProcessors. +func (in *AggregatorOTELProcessors) DeepCopy() *AggregatorOTELProcessors { + if in == nil { + return nil + } + out := new(AggregatorOTELProcessors) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AlertManagerSpec) DeepCopyInto(out *AlertManagerSpec) { *out = *in @@ -402,6 +452,21 @@ func (in *CollectorList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CollectorSendingQueue) DeepCopyInto(out *CollectorSendingQueue) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CollectorSendingQueue. +func (in *CollectorSendingQueue) DeepCopy() *CollectorSendingQueue { + if in == nil { + return nil + } + out := new(CollectorSendingQueue) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CollectorSpec) DeepCopyInto(out *CollectorSpec) { *out = *in @@ -421,11 +486,16 @@ func (in *CollectorSpec) DeepCopyInto(out *CollectorSpec) { *out = new(ConfigReloaderSpec) (*in).DeepCopyInto(*out) } - if in.OTELConfigSpec != nil { - in, out := &in.OTELConfigSpec, &out.OTELConfigSpec - *out = new(OTELConfigSpec) + if in.AggregatorOTELConfigSpec != nil { + in, out := &in.AggregatorOTELConfigSpec, &out.AggregatorOTELConfigSpec + *out = new(AggregatorOTELConfigSpec) (*in).DeepCopyInto(*out) } + if in.NodeOTELConfigSpec != nil { + in, out := &in.NodeOTELConfigSpec, &out.NodeOTELConfigSpec + *out = new(NodeOTELConfigSpec) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CollectorSpec. @@ -1169,82 +1239,82 @@ func (in *NatsStatus) DeepCopy() *NatsStatus { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *OTELConfigSpec) DeepCopyInto(out *OTELConfigSpec) { +func (in *NodeOTELConfigSpec) DeepCopyInto(out *NodeOTELConfigSpec) { *out = *in - in.Processors.DeepCopyInto(&out.Processors) + out.Processors = in.Processors out.Exporters = in.Exporters } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OTELConfigSpec. -func (in *OTELConfigSpec) DeepCopy() *OTELConfigSpec { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeOTELConfigSpec. +func (in *NodeOTELConfigSpec) DeepCopy() *NodeOTELConfigSpec { if in == nil { return nil } - out := new(OTELConfigSpec) + out := new(NodeOTELConfigSpec) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *OTELExporters) DeepCopyInto(out *OTELExporters) { +func (in *NodeOTELExporters) DeepCopyInto(out *NodeOTELExporters) { *out = *in - out.OTLPHTTP = in.OTLPHTTP + out.OTLP = in.OTLP } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OTELExporters. -func (in *OTELExporters) DeepCopy() *OTELExporters { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeOTELExporters. +func (in *NodeOTELExporters) DeepCopy() *NodeOTELExporters { if in == nil { return nil } - out := new(OTELExporters) + out := new(NodeOTELExporters) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *OTELProcessors) DeepCopyInto(out *OTELProcessors) { +func (in *NodeOTELProcessors) DeepCopyInto(out *NodeOTELProcessors) { *out = *in - in.Batch.DeepCopyInto(&out.Batch) out.MemoryLimiter = in.MemoryLimiter } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OTELProcessors. -func (in *OTELProcessors) DeepCopy() *OTELProcessors { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeOTELProcessors. +func (in *NodeOTELProcessors) DeepCopy() *NodeOTELProcessors { if in == nil { return nil } - out := new(OTELProcessors) + out := new(NodeOTELProcessors) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *OTLPHTTPExporterConfig) DeepCopyInto(out *OTLPHTTPExporterConfig) { +func (in *OTLPExporterConfig) DeepCopyInto(out *OTLPExporterConfig) { *out = *in out.SendingQueue = in.SendingQueue } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OTLPHTTPExporterConfig. -func (in *OTLPHTTPExporterConfig) DeepCopy() *OTLPHTTPExporterConfig { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OTLPExporterConfig. +func (in *OTLPExporterConfig) DeepCopy() *OTLPExporterConfig { if in == nil { return nil } - out := new(OTLPHTTPExporterConfig) + out := new(OTLPExporterConfig) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *OTLPHTTPSendingQueue) DeepCopyInto(out *OTLPHTTPSendingQueue) { +func (in *OTLPHTTPExporterConfig) DeepCopyInto(out *OTLPHTTPExporterConfig) { *out = *in + out.SendingQueue = in.SendingQueue } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OTLPHTTPSendingQueue. -func (in *OTLPHTTPSendingQueue) DeepCopy() *OTLPHTTPSendingQueue { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OTLPHTTPExporterConfig. +func (in *OTLPHTTPExporterConfig) DeepCopy() *OTLPHTTPExporterConfig { if in == nil { return nil } - out := new(OTLPHTTPSendingQueue) + out := new(OTLPHTTPExporterConfig) in.DeepCopyInto(out) return out } diff --git a/config/crd/bases/core.opni.io_collectors.yaml b/config/crd/bases/core.opni.io_collectors.yaml index 05031e610e..c345ee2eb2 100644 --- a/config/crd/bases/core.opni.io_collectors.yaml +++ b/config/crd/bases/core.opni.io_collectors.yaml @@ -28,6 +28,73 @@ spec: properties: agentEndpoint: type: string + aggregatorOtelCollectorSpec: + properties: + exporters: + properties: + otlphttp: + properties: + sendingQueue: + properties: + enabled: + type: boolean + numConsumers: + type: integer + queueSize: + type: integer + type: object + type: object + type: object + processors: + properties: + batch: + properties: + metadataCardinalityLimit: + format: int32 + type: integer + metadataKeys: + items: + type: string + type: array + sendBatchMaxSize: + format: int32 + type: integer + sendBatchSize: + default: 1000 + format: int32 + type: integer + timeout: + allOf: + - format: int64 + - format: duration + default: 15s + type: integer + type: object + memoryLimiter: + properties: + checkInterval: + allOf: + - format: int64 + - format: duration + default: 1s + type: integer + limitMib: + default: 1000 + format: int32 + type: integer + limitPercentage: + format: int32 + type: integer + spikeLimitMib: + default: 350 + format: int32 + type: integer + spikeLimitPercentage: + format: int32 + type: integer + type: object + type: object + type: object configReloader: properties: image: @@ -69,11 +136,11 @@ spec: type: string type: object x-kubernetes-map-type: atomic - otelCollectorSpec: + nodeOtelCollectorSpec: properties: exporters: properties: - otlphttp: + otlp: properties: sendingQueue: properties: @@ -88,29 +155,6 @@ spec: type: object processors: properties: - batch: - properties: - metadataCardinalityLimit: - format: int32 - type: integer - metadataKeys: - items: - type: string - type: array - sendBatchMaxSize: - format: int32 - type: integer - sendBatchSize: - default: 1000 - format: int32 - type: integer - timeout: - allOf: - - format: int64 - - format: duration - default: 15s - type: integer - type: object memoryLimiter: properties: checkInterval: diff --git a/go.mod b/go.mod index 3dae55be20..864907f220 100644 --- a/go.mod +++ b/go.mod @@ -115,6 +115,7 @@ require ( go.etcd.io/etcd/api/v3 v3.5.9 go.etcd.io/etcd/client/v3 v3.5.9 go.etcd.io/etcd/etcdctl/v3 v3.5.9 + go.opentelemetry.io/collector/exporter/otlpexporter v0.80.0 go.opentelemetry.io/collector/exporter/otlphttpexporter v0.80.0 go.opentelemetry.io/collector/pdata v1.0.0-rcv0014 go.opentelemetry.io/collector/processor/batchprocessor v0.80.0 @@ -169,6 +170,7 @@ require ( require ( github.com/go-ole/go-ole v1.2.6 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/mostynb/go-grpc-compression v1.1.19 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/shirou/gopsutil/v3 v3.23.5 // indirect github.com/tklauser/go-sysconf v0.3.11 // indirect @@ -176,7 +178,9 @@ require ( github.com/yusufpapurcu/wmi v1.2.3 // indirect go.opentelemetry.io/collector/config/configauth v0.80.0 // indirect go.opentelemetry.io/collector/config/configcompression v0.80.0 // indirect + go.opentelemetry.io/collector/config/configgrpc v0.80.0 // indirect go.opentelemetry.io/collector/config/confighttp v0.80.0 // indirect + go.opentelemetry.io/collector/config/confignet v0.80.0 // indirect go.opentelemetry.io/collector/config/configopaque v0.80.0 // indirect go.opentelemetry.io/collector/config/configtls v0.80.0 // indirect go.opentelemetry.io/collector/config/internal v0.80.0 // indirect @@ -186,7 +190,7 @@ require ( require ( cloud.google.com/go v0.110.7 // indirect cloud.google.com/go/compute v1.23.0 // indirect - cloud.google.com/go/compute/metadata v0.2.3 // indirect + cloud.google.com/go/compute/metadata v0.2.4-0.20230617002413-005d2dfb6b68 // indirect cloud.google.com/go/iam v1.1.1 // indirect cloud.google.com/go/storage v1.30.1 // indirect github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect @@ -453,7 +457,7 @@ require ( github.com/pborman/uuid v1.2.1 // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect - github.com/pierrec/lz4/v4 v4.1.16 // indirect + github.com/pierrec/lz4/v4 v4.1.17 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pkg/term v1.1.0 // indirect github.com/plar/go-adaptive-radix-tree v1.0.5 diff --git a/go.sum b/go.sum index e5a32b630f..92ecee5ec4 100644 --- a/go.sum +++ b/go.sum @@ -128,8 +128,9 @@ cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdi cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZEXYonfTBHHFPO/4UU= cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= -cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/compute/metadata v0.2.4-0.20230617002413-005d2dfb6b68 h1:aRVqY1p2IJaBGStWMsQMpkAa83cPkCDLl80eOj0Rbz4= +cloud.google.com/go/compute/metadata v0.2.4-0.20230617002413-005d2dfb6b68/go.mod h1:1a3eRNYX12fs5UABBIXS8HXVvQbX9hRB/RkEBPORpe8= cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbTuPSj0fYJcPls9TFlPNnHHY= cloud.google.com/go/contactcenterinsights v1.4.0/go.mod h1:L2YzkGbPsv+vMQMCADxJoT9YiTTnSEd6fEvCeHTYVck= cloud.google.com/go/container v1.6.0/go.mod h1:Xazp7GjJSeUYo688S+6J5V+n/t+G5sKBTFkKNudGRxg= @@ -2028,8 +2029,8 @@ github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5/go.mod h1:iIss55rK github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pierrec/lz4/v4 v4.1.16 h1:kQPfno+wyx6C5572ABwV+Uo3pDFzQ7yhyGchSyRda0c= -github.com/pierrec/lz4/v4 v4.1.16/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= +github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= @@ -2513,6 +2514,8 @@ go.opentelemetry.io/collector/consumer v0.80.0 h1:IQfkrNvZ6/ZkfQs/d6JENgfYPE3g3N go.opentelemetry.io/collector/consumer v0.80.0/go.mod h1://MinWlTMlL44B8WvUzLnYWBKgxfgGE8grpxoUiL9aQ= go.opentelemetry.io/collector/exporter v0.80.0 h1:BjwrMGhlrJ6IyhOOkPruzycCk7dv6vuf+VORzsVtcLI= go.opentelemetry.io/collector/exporter v0.80.0/go.mod h1:qcvFsxN8AaSxJ60WxSF0zGUTCMhGBWY3Coa9iiONWbg= +go.opentelemetry.io/collector/exporter/otlpexporter v0.80.0 h1:WQOuXDt40i50lyS3JFIBp0dKEAltt54oERk7klBB5LQ= +go.opentelemetry.io/collector/exporter/otlpexporter v0.80.0/go.mod h1:vfD1P2bcPeK9Df77wzWX897teeBcCvnxA9aNRxDM3ks= go.opentelemetry.io/collector/exporter/otlphttpexporter v0.80.0 h1:2rd+t/LCr7nOOBiHKaeSVbJKHmFbANkUMnumGYVCBCM= go.opentelemetry.io/collector/exporter/otlphttpexporter v0.80.0/go.mod h1:vBz0hJe32gAEp8LWb8e7VwA96h6qlRxToy2acYV6lKw= go.opentelemetry.io/collector/extension v0.80.0 h1:Nx2QpwBAwcEcCVpqqqJIzpkhniGRhLC5Tr5s+e5H5p0= diff --git a/pkg/otel/types.go b/pkg/otel/types.go index 1c125a3c67..deb07a21c4 100644 --- a/pkg/otel/types.go +++ b/pkg/otel/types.go @@ -6,6 +6,7 @@ import ( "github.com/rancher/opni/pkg/util" "github.com/samber/lo" + "go.opentelemetry.io/collector/exporter/otlpexporter" "go.opentelemetry.io/collector/exporter/otlphttpexporter" "go.opentelemetry.io/collector/processor/batchprocessor" "go.opentelemetry.io/collector/processor/memorylimiterprocessor" @@ -30,6 +31,7 @@ type NodeConfig struct { Metrics MetricsConfig Containerized bool LogLevel string + OTELConfig NodeOTELConfig } type AggregatorConfig struct { @@ -55,6 +57,19 @@ type AggregatorOTELExporters struct { OTLPHTTP otlphttpexporter.Config } +type NodeOTELConfig struct { + Processors *NodeOTELProcessors + Exporters *NodeOTELExporters +} + +type NodeOTELProcessors struct { + MemoryLimiter memorylimiterprocessor.Config +} + +type NodeOTELExporters struct { + OTLP otlpexporter.Config +} + type LoggingConfig struct { Enabled bool Receivers []string diff --git a/pkg/resources/collector/templates.go b/pkg/resources/collector/templates.go index 2d0d9ef258..67fee08cfe 100644 --- a/pkg/resources/collector/templates.go +++ b/pkg/resources/collector/templates.go @@ -194,15 +194,16 @@ exporters: tls: insecure: true sending_queue: - num_consumers: 4 - queue_size: 100 + enabled: {{ .OTELConfig.Exporters.OTLP.SendingQueue.Enabled }} + num_consumers: {{ .OTELConfig.Exporters.OTLP.SendingQueue.NumConsumers }} + queue_size: {{ .OTELConfig.Exporters.OTLP.SendingQueue.QueueSize }} retry_on_failure: enabled: true processors: memory_limiter: - limit_mib: 250 - spike_limit_mib: 50 - check_interval: 1s + limit_mib: {{ .OTELConfig.Processors.MemoryLimiter.MemoryLimitMiB }} + spike_limit_mib: {{ .OTELConfig.Processors.MemoryLimiter.MemorySpikeLimitMiB }} + check_interval: {{ .OTELConfig.Processors.MemoryLimiter.CheckInterval }} k8sattributes: passthrough: false pod_association: diff --git a/pkg/resources/collector/workloads.go b/pkg/resources/collector/workloads.go index 60663234f2..20ff21663e 100644 --- a/pkg/resources/collector/workloads.go +++ b/pkg/resources/collector/workloads.go @@ -13,6 +13,7 @@ import ( opnimeta "github.com/rancher/opni/pkg/util/meta" "github.com/samber/lo" "go.opentelemetry.io/collector/exporter/exporterhelper" + "go.opentelemetry.io/collector/exporter/otlpexporter" "go.opentelemetry.io/collector/exporter/otlphttpexporter" "go.opentelemetry.io/collector/processor/batchprocessor" "go.opentelemetry.io/collector/processor/memorylimiterprocessor" @@ -65,6 +66,26 @@ func (r *Reconciler) getDaemonConfig(loggingReceivers []string) otel.NodeConfig Metrics: lo.FromPtr(r.getMetricsConfig()), Containerized: true, LogLevel: r.collector.Spec.LogLevel, + OTELConfig: otel.NodeOTELConfig{ + Processors: &otel.NodeOTELProcessors{ + MemoryLimiter: memorylimiterprocessor.Config{ + CheckInterval: r.collector.Spec.NodeOTELConfigSpec.Processors.MemoryLimiter.CheckInterval, + MemoryLimitMiB: r.collector.Spec.NodeOTELConfigSpec.Processors.MemoryLimiter.MemoryLimitMiB, + MemorySpikeLimitMiB: r.collector.Spec.NodeOTELConfigSpec.Processors.MemoryLimiter.MemorySpikeLimitMiB, + MemoryLimitPercentage: r.collector.Spec.NodeOTELConfigSpec.Processors.MemoryLimiter.MemoryLimitPercentage, + MemorySpikePercentage: r.collector.Spec.NodeOTELConfigSpec.Processors.MemoryLimiter.MemorySpikePercentage, + }, + }, + Exporters: &otel.NodeOTELExporters{ + OTLP: otlpexporter.Config{ + QueueSettings: exporterhelper.QueueSettings{ + Enabled: r.collector.Spec.NodeOTELConfigSpec.Exporters.OTLP.SendingQueue.Enabled, + NumConsumers: r.collector.Spec.NodeOTELConfigSpec.Exporters.OTLP.SendingQueue.NumConsumers, + QueueSize: r.collector.Spec.NodeOTELConfigSpec.Exporters.OTLP.SendingQueue.QueueSize, + }, + }, + }, + }, } } @@ -80,26 +101,26 @@ func (r *Reconciler) getAggregatorConfig( OTELConfig: otel.AggregatorOTELConfig{ Processors: &otel.AggregatorOTELProcessors{ MemoryLimiter: memorylimiterprocessor.Config{ - CheckInterval: r.collector.Spec.OTELConfigSpec.Processors.MemoryLimiter.CheckInterval, - MemoryLimitMiB: r.collector.Spec.OTELConfigSpec.Processors.MemoryLimiter.MemoryLimitMiB, - MemorySpikeLimitMiB: r.collector.Spec.OTELConfigSpec.Processors.MemoryLimiter.MemorySpikeLimitMiB, - MemoryLimitPercentage: r.collector.Spec.OTELConfigSpec.Processors.MemoryLimiter.MemoryLimitPercentage, - MemorySpikePercentage: r.collector.Spec.OTELConfigSpec.Processors.MemoryLimiter.MemorySpikePercentage, + CheckInterval: r.collector.Spec.AggregatorOTELConfigSpec.Processors.MemoryLimiter.CheckInterval, + MemoryLimitMiB: r.collector.Spec.AggregatorOTELConfigSpec.Processors.MemoryLimiter.MemoryLimitMiB, + MemorySpikeLimitMiB: r.collector.Spec.AggregatorOTELConfigSpec.Processors.MemoryLimiter.MemorySpikeLimitMiB, + MemoryLimitPercentage: r.collector.Spec.AggregatorOTELConfigSpec.Processors.MemoryLimiter.MemoryLimitPercentage, + MemorySpikePercentage: r.collector.Spec.AggregatorOTELConfigSpec.Processors.MemoryLimiter.MemorySpikePercentage, }, Batch: batchprocessor.Config{ - Timeout: r.collector.Spec.OTELConfigSpec.Processors.Batch.Timeout, - SendBatchSize: r.collector.Spec.OTELConfigSpec.Processors.Batch.SendBatchSize, - SendBatchMaxSize: r.collector.Spec.OTELConfigSpec.Processors.Batch.SendBatchMaxSize, - MetadataKeys: r.collector.Spec.OTELConfigSpec.Processors.Batch.MetadataKeys, - MetadataCardinalityLimit: r.collector.Spec.OTELConfigSpec.Processors.Batch.MetadataCardinalityLimit, + Timeout: r.collector.Spec.AggregatorOTELConfigSpec.Processors.Batch.Timeout, + SendBatchSize: r.collector.Spec.AggregatorOTELConfigSpec.Processors.Batch.SendBatchSize, + SendBatchMaxSize: r.collector.Spec.AggregatorOTELConfigSpec.Processors.Batch.SendBatchMaxSize, + MetadataKeys: r.collector.Spec.AggregatorOTELConfigSpec.Processors.Batch.MetadataKeys, + MetadataCardinalityLimit: r.collector.Spec.AggregatorOTELConfigSpec.Processors.Batch.MetadataCardinalityLimit, }, }, Exporters: &otel.AggregatorOTELExporters{ OTLPHTTP: otlphttpexporter.Config{ QueueSettings: exporterhelper.QueueSettings{ - Enabled: r.collector.Spec.OTELConfigSpec.Exporters.OTLPHTTP.SendingQueue.Enabled, - NumConsumers: r.collector.Spec.OTELConfigSpec.Exporters.OTLPHTTP.SendingQueue.NumConsumers, - QueueSize: r.collector.Spec.OTELConfigSpec.Exporters.OTLPHTTP.SendingQueue.QueueSize, + Enabled: r.collector.Spec.AggregatorOTELConfigSpec.Exporters.OTLPHTTP.SendingQueue.Enabled, + NumConsumers: r.collector.Spec.AggregatorOTELConfigSpec.Exporters.OTLPHTTP.SendingQueue.NumConsumers, + QueueSize: r.collector.Spec.AggregatorOTELConfigSpec.Exporters.OTLPHTTP.SendingQueue.QueueSize, }, }, }, diff --git a/plugins/logging/pkg/agent/drivers/kubernetes_manager/kubernetes_manager.go b/plugins/logging/pkg/agent/drivers/kubernetes_manager/kubernetes_manager.go index 5862925ca6..ac8b793781 100644 --- a/plugins/logging/pkg/agent/drivers/kubernetes_manager/kubernetes_manager.go +++ b/plugins/logging/pkg/agent/drivers/kubernetes_manager/kubernetes_manager.go @@ -324,8 +324,8 @@ func (m *KubernetesManagerDriver) buildEmptyCollector() *opnicorev1beta1.Collect }, SystemNamespace: m.Namespace, AgentEndpoint: otel.AgentEndpoint(serviceName), - OTELConfigSpec: &opnicorev1beta1.OTELConfigSpec{ - Processors: opnicorev1beta1.OTELProcessors{ + AggregatorOTELConfigSpec: &opnicorev1beta1.AggregatorOTELConfigSpec{ + Processors: opnicorev1beta1.AggregatorOTELProcessors{ MemoryLimiter: opnicorev1beta1.MemoryLimiterProcessorConfig{ MemoryLimitMiB: 1000, MemorySpikeLimitMiB: 350, @@ -336,9 +336,27 @@ func (m *KubernetesManagerDriver) buildEmptyCollector() *opnicorev1beta1.Collect Timeout: 15 * time.Second, }, }, - Exporters: opnicorev1beta1.OTELExporters{ + Exporters: opnicorev1beta1.AggregatorOTELExporters{ OTLPHTTP: opnicorev1beta1.OTLPHTTPExporterConfig{ - SendingQueue: opnicorev1beta1.OTLPHTTPSendingQueue{ + SendingQueue: opnicorev1beta1.CollectorSendingQueue{ + Enabled: true, + NumConsumers: 4, + QueueSize: 100, + }, + }, + }, + }, + NodeOTELConfigSpec: &opnicorev1beta1.NodeOTELConfigSpec{ + Processors: opnicorev1beta1.NodeOTELProcessors{ + MemoryLimiter: opnicorev1beta1.MemoryLimiterProcessorConfig{ + MemoryLimitMiB: 250, + MemorySpikeLimitMiB: 50, + CheckInterval: 1 * time.Second, + }, + }, + Exporters: opnicorev1beta1.NodeOTELExporters{ + OTLP: opnicorev1beta1.OTLPExporterConfig{ + SendingQueue: opnicorev1beta1.CollectorSendingQueue{ Enabled: true, NumConsumers: 4, QueueSize: 100, From 9cd9fb672beff5ec7c5fc7171e8479dac3e943b9 Mon Sep 17 00:00:00 2001 From: Joao Henri Date: Fri, 6 Oct 2023 13:55:37 -0300 Subject: [PATCH 10/21] Refactor for readability Signed-off-by: Joao Henri --- pkg/resources/collector/workloads.go | 45 +++++++++++++++------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/pkg/resources/collector/workloads.go b/pkg/resources/collector/workloads.go index 20ff21663e..638408ee0f 100644 --- a/pkg/resources/collector/workloads.go +++ b/pkg/resources/collector/workloads.go @@ -57,6 +57,7 @@ func (r *Reconciler) aggregatorConfigMapName() string { } func (r *Reconciler) getDaemonConfig(loggingReceivers []string) otel.NodeConfig { + nodeOTELCfg := r.collector.Spec.NodeOTELConfigSpec return otel.NodeConfig{ Instance: r.collector.Name, Logs: otel.LoggingConfig{ @@ -69,19 +70,19 @@ func (r *Reconciler) getDaemonConfig(loggingReceivers []string) otel.NodeConfig OTELConfig: otel.NodeOTELConfig{ Processors: &otel.NodeOTELProcessors{ MemoryLimiter: memorylimiterprocessor.Config{ - CheckInterval: r.collector.Spec.NodeOTELConfigSpec.Processors.MemoryLimiter.CheckInterval, - MemoryLimitMiB: r.collector.Spec.NodeOTELConfigSpec.Processors.MemoryLimiter.MemoryLimitMiB, - MemorySpikeLimitMiB: r.collector.Spec.NodeOTELConfigSpec.Processors.MemoryLimiter.MemorySpikeLimitMiB, - MemoryLimitPercentage: r.collector.Spec.NodeOTELConfigSpec.Processors.MemoryLimiter.MemoryLimitPercentage, - MemorySpikePercentage: r.collector.Spec.NodeOTELConfigSpec.Processors.MemoryLimiter.MemorySpikePercentage, + CheckInterval: nodeOTELCfg.Processors.MemoryLimiter.CheckInterval, + MemoryLimitMiB: nodeOTELCfg.Processors.MemoryLimiter.MemoryLimitMiB, + MemorySpikeLimitMiB: nodeOTELCfg.Processors.MemoryLimiter.MemorySpikeLimitMiB, + MemoryLimitPercentage: nodeOTELCfg.Processors.MemoryLimiter.MemoryLimitPercentage, + MemorySpikePercentage: nodeOTELCfg.Processors.MemoryLimiter.MemorySpikePercentage, }, }, Exporters: &otel.NodeOTELExporters{ OTLP: otlpexporter.Config{ QueueSettings: exporterhelper.QueueSettings{ - Enabled: r.collector.Spec.NodeOTELConfigSpec.Exporters.OTLP.SendingQueue.Enabled, - NumConsumers: r.collector.Spec.NodeOTELConfigSpec.Exporters.OTLP.SendingQueue.NumConsumers, - QueueSize: r.collector.Spec.NodeOTELConfigSpec.Exporters.OTLP.SendingQueue.QueueSize, + Enabled: nodeOTELCfg.Exporters.OTLP.SendingQueue.Enabled, + NumConsumers: nodeOTELCfg.Exporters.OTLP.SendingQueue.NumConsumers, + QueueSize: nodeOTELCfg.Exporters.OTLP.SendingQueue.QueueSize, }, }, }, @@ -92,6 +93,8 @@ func (r *Reconciler) getDaemonConfig(loggingReceivers []string) otel.NodeConfig func (r *Reconciler) getAggregatorConfig( metricsCfg otel.MetricsConfig, ) otel.AggregatorConfig { + aggregatorOTELCfg := r.collector.Spec.AggregatorOTELConfigSpec + return otel.AggregatorConfig{ LogsEnabled: r.collector.Spec.LoggingConfig != nil, Metrics: metricsCfg, @@ -101,26 +104,26 @@ func (r *Reconciler) getAggregatorConfig( OTELConfig: otel.AggregatorOTELConfig{ Processors: &otel.AggregatorOTELProcessors{ MemoryLimiter: memorylimiterprocessor.Config{ - CheckInterval: r.collector.Spec.AggregatorOTELConfigSpec.Processors.MemoryLimiter.CheckInterval, - MemoryLimitMiB: r.collector.Spec.AggregatorOTELConfigSpec.Processors.MemoryLimiter.MemoryLimitMiB, - MemorySpikeLimitMiB: r.collector.Spec.AggregatorOTELConfigSpec.Processors.MemoryLimiter.MemorySpikeLimitMiB, - MemoryLimitPercentage: r.collector.Spec.AggregatorOTELConfigSpec.Processors.MemoryLimiter.MemoryLimitPercentage, - MemorySpikePercentage: r.collector.Spec.AggregatorOTELConfigSpec.Processors.MemoryLimiter.MemorySpikePercentage, + CheckInterval: aggregatorOTELCfg.Processors.MemoryLimiter.CheckInterval, + MemoryLimitMiB: aggregatorOTELCfg.Processors.MemoryLimiter.MemoryLimitMiB, + MemorySpikeLimitMiB: aggregatorOTELCfg.Processors.MemoryLimiter.MemorySpikeLimitMiB, + MemoryLimitPercentage: aggregatorOTELCfg.Processors.MemoryLimiter.MemoryLimitPercentage, + MemorySpikePercentage: aggregatorOTELCfg.Processors.MemoryLimiter.MemorySpikePercentage, }, Batch: batchprocessor.Config{ - Timeout: r.collector.Spec.AggregatorOTELConfigSpec.Processors.Batch.Timeout, - SendBatchSize: r.collector.Spec.AggregatorOTELConfigSpec.Processors.Batch.SendBatchSize, - SendBatchMaxSize: r.collector.Spec.AggregatorOTELConfigSpec.Processors.Batch.SendBatchMaxSize, - MetadataKeys: r.collector.Spec.AggregatorOTELConfigSpec.Processors.Batch.MetadataKeys, - MetadataCardinalityLimit: r.collector.Spec.AggregatorOTELConfigSpec.Processors.Batch.MetadataCardinalityLimit, + Timeout: aggregatorOTELCfg.Processors.Batch.Timeout, + SendBatchSize: aggregatorOTELCfg.Processors.Batch.SendBatchSize, + SendBatchMaxSize: aggregatorOTELCfg.Processors.Batch.SendBatchMaxSize, + MetadataKeys: aggregatorOTELCfg.Processors.Batch.MetadataKeys, + MetadataCardinalityLimit: aggregatorOTELCfg.Processors.Batch.MetadataCardinalityLimit, }, }, Exporters: &otel.AggregatorOTELExporters{ OTLPHTTP: otlphttpexporter.Config{ QueueSettings: exporterhelper.QueueSettings{ - Enabled: r.collector.Spec.AggregatorOTELConfigSpec.Exporters.OTLPHTTP.SendingQueue.Enabled, - NumConsumers: r.collector.Spec.AggregatorOTELConfigSpec.Exporters.OTLPHTTP.SendingQueue.NumConsumers, - QueueSize: r.collector.Spec.AggregatorOTELConfigSpec.Exporters.OTLPHTTP.SendingQueue.QueueSize, + Enabled: aggregatorOTELCfg.Exporters.OTLPHTTP.SendingQueue.Enabled, + NumConsumers: aggregatorOTELCfg.Exporters.OTLPHTTP.SendingQueue.NumConsumers, + QueueSize: aggregatorOTELCfg.Exporters.OTLPHTTP.SendingQueue.QueueSize, }, }, }, From 94e74a4dcc322443667f1b2154a0bca40b3f1a5a Mon Sep 17 00:00:00 2001 From: Joao Henri Date: Fri, 6 Oct 2023 15:22:54 -0300 Subject: [PATCH 11/21] Remove fields we don't want to export Signed-off-by: Joao Henri --- apis/core/v1beta1/collector_types.go | 26 +++++++------------ apis/core/v1beta1/zz_generated.deepcopy.go | 11 +++----- config/crd/bases/core.opni.io_collectors.yaml | 7 ----- pkg/resources/collector/workloads.go | 8 +++--- 4 files changed, 15 insertions(+), 37 deletions(-) diff --git a/apis/core/v1beta1/collector_types.go b/apis/core/v1beta1/collector_types.go index 8f56b5c94c..42778014b5 100644 --- a/apis/core/v1beta1/collector_types.go +++ b/apis/core/v1beta1/collector_types.go @@ -113,23 +113,6 @@ type BatchProcessorConfig struct { // Larger batches are split into smaller units. // Default value is 0, that means no maximum size. SendBatchMaxSize uint32 `json:"sendBatchMaxSize,omitempty"` - - // MetadataKeys is a list of client.Metadata keys that will be - // used to form distinct batchers. If this setting is empty, - // a single batcher instance will be used. When this setting - // is not empty, one batcher will be used per distinct - // combination of values for the listed metadata keys. - // - // Empty value and unset metadata are treated as distinct cases. - // - // Entries are case-insensitive. Duplicated entries will - // trigger a validation error. - MetadataKeys []string `json:"metadataKeys,omitempty"` - - // MetadataCardinalityLimit indicates the maximum number of - // batcher instances that will be created through a distinct - // combination of MetadataKeys. - MetadataCardinalityLimit uint32 `json:"metadataCardinalityLimit,omitempty"` } // CollectorSendingQueue has the attributes that we want to make @@ -145,9 +128,18 @@ type CollectorSendingQueue struct { QueueSize int `json:"queueSize,omitempty"` } +// CollectorSendingQueue has the attributes that we want to make +// available from otlpexporter.Config. +// Also, we extend it with the JSON struct tags needed in order to kubebuilder +// and controller-gen work. type OTLPExporterConfig struct { SendingQueue CollectorSendingQueue `json:"sendingQueue,omitempty"` } + +// CollectorSendingQueue has the attributes that we want to make +// available from otlphttpexporter.Config. +// Also, we extend it with the JSON struct tags needed in order to kubebuilder +// and controller-gen work. type OTLPHTTPExporterConfig struct { SendingQueue CollectorSendingQueue `json:"sendingQueue,omitempty"` } diff --git a/apis/core/v1beta1/zz_generated.deepcopy.go b/apis/core/v1beta1/zz_generated.deepcopy.go index c8073091cb..aac561f822 100644 --- a/apis/core/v1beta1/zz_generated.deepcopy.go +++ b/apis/core/v1beta1/zz_generated.deepcopy.go @@ -15,7 +15,7 @@ import ( // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AggregatorOTELConfigSpec) DeepCopyInto(out *AggregatorOTELConfigSpec) { *out = *in - in.Processors.DeepCopyInto(&out.Processors) + out.Processors = in.Processors out.Exporters = in.Exporters } @@ -48,7 +48,7 @@ func (in *AggregatorOTELExporters) DeepCopy() *AggregatorOTELExporters { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AggregatorOTELProcessors) DeepCopyInto(out *AggregatorOTELProcessors) { *out = *in - in.Batch.DeepCopyInto(&out.Batch) + out.Batch = in.Batch out.MemoryLimiter = in.MemoryLimiter } @@ -315,11 +315,6 @@ func (in *AuthSpec) DeepCopy() *AuthSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BatchProcessorConfig) DeepCopyInto(out *BatchProcessorConfig) { *out = *in - if in.MetadataKeys != nil { - in, out := &in.MetadataKeys, &out.MetadataKeys - *out = make([]string, len(*in)) - copy(*out, *in) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BatchProcessorConfig. @@ -489,7 +484,7 @@ func (in *CollectorSpec) DeepCopyInto(out *CollectorSpec) { if in.AggregatorOTELConfigSpec != nil { in, out := &in.AggregatorOTELConfigSpec, &out.AggregatorOTELConfigSpec *out = new(AggregatorOTELConfigSpec) - (*in).DeepCopyInto(*out) + **out = **in } if in.NodeOTELConfigSpec != nil { in, out := &in.NodeOTELConfigSpec, &out.NodeOTELConfigSpec diff --git a/config/crd/bases/core.opni.io_collectors.yaml b/config/crd/bases/core.opni.io_collectors.yaml index c345ee2eb2..1a54d8aacf 100644 --- a/config/crd/bases/core.opni.io_collectors.yaml +++ b/config/crd/bases/core.opni.io_collectors.yaml @@ -49,13 +49,6 @@ spec: properties: batch: properties: - metadataCardinalityLimit: - format: int32 - type: integer - metadataKeys: - items: - type: string - type: array sendBatchMaxSize: format: int32 type: integer diff --git a/pkg/resources/collector/workloads.go b/pkg/resources/collector/workloads.go index 638408ee0f..ae93a08c25 100644 --- a/pkg/resources/collector/workloads.go +++ b/pkg/resources/collector/workloads.go @@ -111,11 +111,9 @@ func (r *Reconciler) getAggregatorConfig( MemorySpikePercentage: aggregatorOTELCfg.Processors.MemoryLimiter.MemorySpikePercentage, }, Batch: batchprocessor.Config{ - Timeout: aggregatorOTELCfg.Processors.Batch.Timeout, - SendBatchSize: aggregatorOTELCfg.Processors.Batch.SendBatchSize, - SendBatchMaxSize: aggregatorOTELCfg.Processors.Batch.SendBatchMaxSize, - MetadataKeys: aggregatorOTELCfg.Processors.Batch.MetadataKeys, - MetadataCardinalityLimit: aggregatorOTELCfg.Processors.Batch.MetadataCardinalityLimit, + Timeout: aggregatorOTELCfg.Processors.Batch.Timeout, + SendBatchSize: aggregatorOTELCfg.Processors.Batch.SendBatchSize, + SendBatchMaxSize: aggregatorOTELCfg.Processors.Batch.SendBatchMaxSize, }, }, Exporters: &otel.AggregatorOTELExporters{ From ed4145fa7d2d7fd939ec234cd28ad4c1b64e56e5 Mon Sep 17 00:00:00 2001 From: Joao Henri Date: Fri, 6 Oct 2023 16:22:12 -0300 Subject: [PATCH 12/21] Add null safety Signed-off-by: Joao Henri --- pkg/resources/collector/workloads.go | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/pkg/resources/collector/workloads.go b/pkg/resources/collector/workloads.go index ae93a08c25..1cd0bf0aed 100644 --- a/pkg/resources/collector/workloads.go +++ b/pkg/resources/collector/workloads.go @@ -58,7 +58,7 @@ func (r *Reconciler) aggregatorConfigMapName() string { func (r *Reconciler) getDaemonConfig(loggingReceivers []string) otel.NodeConfig { nodeOTELCfg := r.collector.Spec.NodeOTELConfigSpec - return otel.NodeConfig{ + nodeConfig := otel.NodeConfig{ Instance: r.collector.Name, Logs: otel.LoggingConfig{ Enabled: r.collector.Spec.LoggingConfig != nil, @@ -67,7 +67,10 @@ func (r *Reconciler) getDaemonConfig(loggingReceivers []string) otel.NodeConfig Metrics: lo.FromPtr(r.getMetricsConfig()), Containerized: true, LogLevel: r.collector.Spec.LogLevel, - OTELConfig: otel.NodeOTELConfig{ + } + + if nodeOTELCfg != nil { + nodeConfig.OTELConfig = otel.NodeOTELConfig{ Processors: &otel.NodeOTELProcessors{ MemoryLimiter: memorylimiterprocessor.Config{ CheckInterval: nodeOTELCfg.Processors.MemoryLimiter.CheckInterval, @@ -86,22 +89,26 @@ func (r *Reconciler) getDaemonConfig(loggingReceivers []string) otel.NodeConfig }, }, }, - }, + } } + + return nodeConfig } func (r *Reconciler) getAggregatorConfig( metricsCfg otel.MetricsConfig, ) otel.AggregatorConfig { - aggregatorOTELCfg := r.collector.Spec.AggregatorOTELConfigSpec - - return otel.AggregatorConfig{ + aggregatorCfg := otel.AggregatorConfig{ LogsEnabled: r.collector.Spec.LoggingConfig != nil, Metrics: metricsCfg, AgentEndpoint: r.collector.Spec.AgentEndpoint, Containerized: true, LogLevel: r.collector.Spec.LogLevel, - OTELConfig: otel.AggregatorOTELConfig{ + } + + aggregatorOTELCfg := r.collector.Spec.AggregatorOTELConfigSpec + if aggregatorOTELCfg != nil { + aggregatorCfg.OTELConfig = otel.AggregatorOTELConfig{ Processors: &otel.AggregatorOTELProcessors{ MemoryLimiter: memorylimiterprocessor.Config{ CheckInterval: aggregatorOTELCfg.Processors.MemoryLimiter.CheckInterval, @@ -125,8 +132,10 @@ func (r *Reconciler) getAggregatorConfig( }, }, }, - }, + } } + + return aggregatorCfg } func (r *Reconciler) receiverConfig() (retData []byte, retReceivers []string, retErr error) { From 23abc140612ccb43cd8fbf4ecd10fccf5dcaf7b6 Mon Sep 17 00:00:00 2001 From: Joao Henri Date: Fri, 6 Oct 2023 16:50:44 -0300 Subject: [PATCH 13/21] Add default OTEL config on reconcile Signed-off-by: Joao Henri --- apis/core/v1beta1/collector_types.go | 52 +++++++++ apis/core/v1beta1/zz_generated.deepcopy.go | 2 +- pkg/resources/collector/workloads.go | 101 ++++++++++-------- .../kubernetes_manager/kubernetes_manager.go | 47 +------- 4 files changed, 113 insertions(+), 89 deletions(-) diff --git a/apis/core/v1beta1/collector_types.go b/apis/core/v1beta1/collector_types.go index 42778014b5..47a3924313 100644 --- a/apis/core/v1beta1/collector_types.go +++ b/apis/core/v1beta1/collector_types.go @@ -1,6 +1,7 @@ package v1beta1 import ( + "k8s.io/apimachinery/pkg/runtime" "time" opnimeta "github.com/rancher/opni/pkg/util/meta" @@ -164,6 +165,11 @@ type Collector struct { Status CollectorStatus `json:"status,omitempty"` } +func (c Collector) DeepCopyObject() runtime.Object { + //TODO implement me + panic("implement me") +} + //+kubebuilder:object:root=true // CollectorList contains a list of Collector @@ -187,6 +193,52 @@ func CollectorCRD() (*crd.CRD, error) { }, nil } +func NewDefaultAggregatorOTELConfigSpec() *AggregatorOTELConfigSpec { + return &AggregatorOTELConfigSpec{ + Processors: AggregatorOTELProcessors{ + MemoryLimiter: MemoryLimiterProcessorConfig{ + MemoryLimitMiB: 1000, + MemorySpikeLimitMiB: 350, + CheckInterval: 1 * time.Second, + }, + Batch: BatchProcessorConfig{ + SendBatchSize: 1000, + Timeout: 15 * time.Second, + }, + }, + Exporters: AggregatorOTELExporters{ + OTLPHTTP: OTLPHTTPExporterConfig{ + SendingQueue: CollectorSendingQueue{ + Enabled: true, + NumConsumers: 4, + QueueSize: 100, + }, + }, + }, + } +} + +func NewDefaultNodeOTELConfigSpec() *NodeOTELConfigSpec { + return &NodeOTELConfigSpec{ + Processors: NodeOTELProcessors{ + MemoryLimiter: MemoryLimiterProcessorConfig{ + MemoryLimitMiB: 250, + MemorySpikeLimitMiB: 50, + CheckInterval: 1 * time.Second, + }, + }, + Exporters: NodeOTELExporters{ + OTLP: OTLPExporterConfig{ + SendingQueue: CollectorSendingQueue{ + Enabled: true, + NumConsumers: 4, + QueueSize: 100, + }, + }, + }, + } +} + func init() { SchemeBuilder.Register(&Collector{}, &CollectorList{}) } diff --git a/apis/core/v1beta1/zz_generated.deepcopy.go b/apis/core/v1beta1/zz_generated.deepcopy.go index aac561f822..add88d8465 100644 --- a/apis/core/v1beta1/zz_generated.deepcopy.go +++ b/apis/core/v1beta1/zz_generated.deepcopy.go @@ -8,7 +8,7 @@ import ( "github.com/rancher/opni/pkg/util/meta" appsv1 "k8s.io/api/apps/v1" "k8s.io/api/core/v1" - runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime" apiv1 "opensearch.opster.io/api/v1" ) diff --git a/pkg/resources/collector/workloads.go b/pkg/resources/collector/workloads.go index 1cd0bf0aed..61acb489ea 100644 --- a/pkg/resources/collector/workloads.go +++ b/pkg/resources/collector/workloads.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "fmt" + opnicorev1beta1 "github.com/rancher/opni/apis/core/v1beta1" opniloggingv1beta1 "github.com/rancher/opni/apis/logging/v1beta1" monitoringv1beta1 "github.com/rancher/opni/apis/monitoring/v1beta1" "github.com/rancher/opni/pkg/otel" @@ -57,7 +58,6 @@ func (r *Reconciler) aggregatorConfigMapName() string { } func (r *Reconciler) getDaemonConfig(loggingReceivers []string) otel.NodeConfig { - nodeOTELCfg := r.collector.Spec.NodeOTELConfigSpec nodeConfig := otel.NodeConfig{ Instance: r.collector.Name, Logs: otel.LoggingConfig{ @@ -67,32 +67,38 @@ func (r *Reconciler) getDaemonConfig(loggingReceivers []string) otel.NodeConfig Metrics: lo.FromPtr(r.getMetricsConfig()), Containerized: true, LogLevel: r.collector.Spec.LogLevel, + OTELConfig: r.getDaemonOTELConfig(), } - if nodeOTELCfg != nil { - nodeConfig.OTELConfig = otel.NodeOTELConfig{ - Processors: &otel.NodeOTELProcessors{ - MemoryLimiter: memorylimiterprocessor.Config{ - CheckInterval: nodeOTELCfg.Processors.MemoryLimiter.CheckInterval, - MemoryLimitMiB: nodeOTELCfg.Processors.MemoryLimiter.MemoryLimitMiB, - MemorySpikeLimitMiB: nodeOTELCfg.Processors.MemoryLimiter.MemorySpikeLimitMiB, - MemoryLimitPercentage: nodeOTELCfg.Processors.MemoryLimiter.MemoryLimitPercentage, - MemorySpikePercentage: nodeOTELCfg.Processors.MemoryLimiter.MemorySpikePercentage, - }, + return nodeConfig +} + +func (r *Reconciler) getDaemonOTELConfig() otel.NodeOTELConfig { + nodeOTELCfg := r.collector.Spec.NodeOTELConfigSpec + if nodeOTELCfg == nil { + nodeOTELCfg = opnicorev1beta1.NewDefaultNodeOTELConfigSpec() + } + + return otel.NodeOTELConfig{ + Processors: &otel.NodeOTELProcessors{ + MemoryLimiter: memorylimiterprocessor.Config{ + CheckInterval: nodeOTELCfg.Processors.MemoryLimiter.CheckInterval, + MemoryLimitMiB: nodeOTELCfg.Processors.MemoryLimiter.MemoryLimitMiB, + MemorySpikeLimitMiB: nodeOTELCfg.Processors.MemoryLimiter.MemorySpikeLimitMiB, + MemoryLimitPercentage: nodeOTELCfg.Processors.MemoryLimiter.MemoryLimitPercentage, + MemorySpikePercentage: nodeOTELCfg.Processors.MemoryLimiter.MemorySpikePercentage, }, - Exporters: &otel.NodeOTELExporters{ - OTLP: otlpexporter.Config{ - QueueSettings: exporterhelper.QueueSettings{ - Enabled: nodeOTELCfg.Exporters.OTLP.SendingQueue.Enabled, - NumConsumers: nodeOTELCfg.Exporters.OTLP.SendingQueue.NumConsumers, - QueueSize: nodeOTELCfg.Exporters.OTLP.SendingQueue.QueueSize, - }, + }, + Exporters: &otel.NodeOTELExporters{ + OTLP: otlpexporter.Config{ + QueueSettings: exporterhelper.QueueSettings{ + Enabled: nodeOTELCfg.Exporters.OTLP.SendingQueue.Enabled, + NumConsumers: nodeOTELCfg.Exporters.OTLP.SendingQueue.NumConsumers, + QueueSize: nodeOTELCfg.Exporters.OTLP.SendingQueue.QueueSize, }, }, - } + }, } - - return nodeConfig } func (r *Reconciler) getAggregatorConfig( @@ -104,38 +110,43 @@ func (r *Reconciler) getAggregatorConfig( AgentEndpoint: r.collector.Spec.AgentEndpoint, Containerized: true, LogLevel: r.collector.Spec.LogLevel, + OTELConfig: r.getAggregatorOTELConfig(), } + return aggregatorCfg +} + +func (r *Reconciler) getAggregatorOTELConfig() otel.AggregatorOTELConfig { aggregatorOTELCfg := r.collector.Spec.AggregatorOTELConfigSpec - if aggregatorOTELCfg != nil { - aggregatorCfg.OTELConfig = otel.AggregatorOTELConfig{ - Processors: &otel.AggregatorOTELProcessors{ - MemoryLimiter: memorylimiterprocessor.Config{ - CheckInterval: aggregatorOTELCfg.Processors.MemoryLimiter.CheckInterval, - MemoryLimitMiB: aggregatorOTELCfg.Processors.MemoryLimiter.MemoryLimitMiB, - MemorySpikeLimitMiB: aggregatorOTELCfg.Processors.MemoryLimiter.MemorySpikeLimitMiB, - MemoryLimitPercentage: aggregatorOTELCfg.Processors.MemoryLimiter.MemoryLimitPercentage, - MemorySpikePercentage: aggregatorOTELCfg.Processors.MemoryLimiter.MemorySpikePercentage, - }, - Batch: batchprocessor.Config{ - Timeout: aggregatorOTELCfg.Processors.Batch.Timeout, - SendBatchSize: aggregatorOTELCfg.Processors.Batch.SendBatchSize, - SendBatchMaxSize: aggregatorOTELCfg.Processors.Batch.SendBatchMaxSize, - }, + if aggregatorOTELCfg == nil { + aggregatorOTELCfg = opnicorev1beta1.NewDefaultAggregatorOTELConfigSpec() + } + return otel.AggregatorOTELConfig{ + Processors: &otel.AggregatorOTELProcessors{ + MemoryLimiter: memorylimiterprocessor.Config{ + CheckInterval: aggregatorOTELCfg.Processors.MemoryLimiter.CheckInterval, + MemoryLimitMiB: aggregatorOTELCfg.Processors.MemoryLimiter.MemoryLimitMiB, + MemorySpikeLimitMiB: aggregatorOTELCfg.Processors.MemoryLimiter.MemorySpikeLimitMiB, + MemoryLimitPercentage: aggregatorOTELCfg.Processors.MemoryLimiter.MemoryLimitPercentage, + MemorySpikePercentage: aggregatorOTELCfg.Processors.MemoryLimiter.MemorySpikePercentage, }, - Exporters: &otel.AggregatorOTELExporters{ - OTLPHTTP: otlphttpexporter.Config{ - QueueSettings: exporterhelper.QueueSettings{ - Enabled: aggregatorOTELCfg.Exporters.OTLPHTTP.SendingQueue.Enabled, - NumConsumers: aggregatorOTELCfg.Exporters.OTLPHTTP.SendingQueue.NumConsumers, - QueueSize: aggregatorOTELCfg.Exporters.OTLPHTTP.SendingQueue.QueueSize, - }, + Batch: batchprocessor.Config{ + Timeout: aggregatorOTELCfg.Processors.Batch.Timeout, + SendBatchSize: aggregatorOTELCfg.Processors.Batch.SendBatchSize, + SendBatchMaxSize: aggregatorOTELCfg.Processors.Batch.SendBatchMaxSize, + }, + }, + Exporters: &otel.AggregatorOTELExporters{ + OTLPHTTP: otlphttpexporter.Config{ + QueueSettings: exporterhelper.QueueSettings{ + Enabled: aggregatorOTELCfg.Exporters.OTLPHTTP.SendingQueue.Enabled, + NumConsumers: aggregatorOTELCfg.Exporters.OTLPHTTP.SendingQueue.NumConsumers, + QueueSize: aggregatorOTELCfg.Exporters.OTLPHTTP.SendingQueue.QueueSize, }, }, - } + }, } - return aggregatorCfg } func (r *Reconciler) receiverConfig() (retData []byte, retReceivers []string, retErr error) { diff --git a/plugins/logging/pkg/agent/drivers/kubernetes_manager/kubernetes_manager.go b/plugins/logging/pkg/agent/drivers/kubernetes_manager/kubernetes_manager.go index ac8b793781..b5c9973cb9 100644 --- a/plugins/logging/pkg/agent/drivers/kubernetes_manager/kubernetes_manager.go +++ b/plugins/logging/pkg/agent/drivers/kubernetes_manager/kubernetes_manager.go @@ -6,7 +6,6 @@ import ( "fmt" "os" "sync" - "time" "github.com/cisco-open/k8s-objectmatcher/patch" "github.com/lestrrat-go/backoff/v2" @@ -322,48 +321,10 @@ func (m *KubernetesManagerDriver) buildEmptyCollector() *opnicorev1beta1.Collect ImageSpec: opnimeta.ImageSpec{ ImagePullPolicy: lo.ToPtr(corev1.PullAlways), }, - SystemNamespace: m.Namespace, - AgentEndpoint: otel.AgentEndpoint(serviceName), - AggregatorOTELConfigSpec: &opnicorev1beta1.AggregatorOTELConfigSpec{ - Processors: opnicorev1beta1.AggregatorOTELProcessors{ - MemoryLimiter: opnicorev1beta1.MemoryLimiterProcessorConfig{ - MemoryLimitMiB: 1000, - MemorySpikeLimitMiB: 350, - CheckInterval: 1 * time.Second, - }, - Batch: opnicorev1beta1.BatchProcessorConfig{ - SendBatchSize: 1000, - Timeout: 15 * time.Second, - }, - }, - Exporters: opnicorev1beta1.AggregatorOTELExporters{ - OTLPHTTP: opnicorev1beta1.OTLPHTTPExporterConfig{ - SendingQueue: opnicorev1beta1.CollectorSendingQueue{ - Enabled: true, - NumConsumers: 4, - QueueSize: 100, - }, - }, - }, - }, - NodeOTELConfigSpec: &opnicorev1beta1.NodeOTELConfigSpec{ - Processors: opnicorev1beta1.NodeOTELProcessors{ - MemoryLimiter: opnicorev1beta1.MemoryLimiterProcessorConfig{ - MemoryLimitMiB: 250, - MemorySpikeLimitMiB: 50, - CheckInterval: 1 * time.Second, - }, - }, - Exporters: opnicorev1beta1.NodeOTELExporters{ - OTLP: opnicorev1beta1.OTLPExporterConfig{ - SendingQueue: opnicorev1beta1.CollectorSendingQueue{ - Enabled: true, - NumConsumers: 4, - QueueSize: 100, - }, - }, - }, - }, + SystemNamespace: m.Namespace, + AgentEndpoint: otel.AgentEndpoint(serviceName), + AggregatorOTELConfigSpec: opnicorev1beta1.NewDefaultAggregatorOTELConfigSpec(), + NodeOTELConfigSpec: opnicorev1beta1.NewDefaultNodeOTELConfigSpec(), }, } } From ba4ba28a4c5408b8fd8f74d3f83410649ef90584 Mon Sep 17 00:00:00 2001 From: Joao Henri Date: Fri, 6 Oct 2023 17:01:14 -0300 Subject: [PATCH 14/21] remove unimplemented Collector DeepCopyObject Signed-off-by: Joao Henri --- apis/core/v1beta1/collector_types.go | 6 ------ apis/core/v1beta1/zz_generated.deepcopy.go | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/apis/core/v1beta1/collector_types.go b/apis/core/v1beta1/collector_types.go index 47a3924313..a22b065884 100644 --- a/apis/core/v1beta1/collector_types.go +++ b/apis/core/v1beta1/collector_types.go @@ -1,7 +1,6 @@ package v1beta1 import ( - "k8s.io/apimachinery/pkg/runtime" "time" opnimeta "github.com/rancher/opni/pkg/util/meta" @@ -165,11 +164,6 @@ type Collector struct { Status CollectorStatus `json:"status,omitempty"` } -func (c Collector) DeepCopyObject() runtime.Object { - //TODO implement me - panic("implement me") -} - //+kubebuilder:object:root=true // CollectorList contains a list of Collector diff --git a/apis/core/v1beta1/zz_generated.deepcopy.go b/apis/core/v1beta1/zz_generated.deepcopy.go index add88d8465..aac561f822 100644 --- a/apis/core/v1beta1/zz_generated.deepcopy.go +++ b/apis/core/v1beta1/zz_generated.deepcopy.go @@ -8,7 +8,7 @@ import ( "github.com/rancher/opni/pkg/util/meta" appsv1 "k8s.io/api/apps/v1" "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" + runtime "k8s.io/apimachinery/pkg/runtime" apiv1 "opensearch.opster.io/api/v1" ) From 5d32d155d96c22b7396c000a541b1a80dc548826 Mon Sep 17 00:00:00 2001 From: Joao Henri Date: Fri, 6 Oct 2023 21:10:54 +0000 Subject: [PATCH 15/21] Remove kubebuilder defaults Signed-off-by: Joao Henri --- apis/core/v1beta1/collector_types.go | 7 ------- config/crd/bases/core.opni.io_collectors.yaml | 20 +++---------------- 2 files changed, 3 insertions(+), 24 deletions(-) diff --git a/apis/core/v1beta1/collector_types.go b/apis/core/v1beta1/collector_types.go index a22b065884..2684ae6d43 100644 --- a/apis/core/v1beta1/collector_types.go +++ b/apis/core/v1beta1/collector_types.go @@ -69,18 +69,14 @@ type MemoryLimiterProcessorConfig struct { // CheckInterval is the time between measurements of memory usage for the // purposes of avoiding going over the limits. Defaults to zero, so no // checks will be performed. - // +kubebuilder:default="1s" - // +kubebuilder:validation:Format=duration CheckInterval time.Duration `json:"checkInterval,omitempty"` // MemoryLimitMiB is the maximum amount of memory, in MiB, targeted to be // allocated by the process. - // +kubebuilder:default=1000 MemoryLimitMiB uint32 `json:"limitMib,omitempty"` // MemorySpikeLimitMiB is the maximum, in MiB, spike expected between the // measurements of memory usage. - // +kubebuilder:default=350 MemorySpikeLimitMiB uint32 `json:"spikeLimitMib,omitempty"` // MemoryLimitPercentage is the maximum amount of memory, in %, targeted to be @@ -99,14 +95,11 @@ type MemoryLimiterProcessorConfig struct { type BatchProcessorConfig struct { // Timeout sets the time after which a batch will be sent regardless of size. // When this is set to zero, batched data will be sent immediately. - // +kubebuilder:default="15s" - // +kubebuilder:validation:Format=duration Timeout time.Duration `json:"timeout,omitempty"` // SendBatchSize is the size of a batch which after hit, will trigger it to be sent. // When this is set to zero, the batch size is ignored and data will be sent immediately // subject to only send_batch_max_size. - // +kubebuilder:default=1000 SendBatchSize uint32 `json:"sendBatchSize,omitempty"` // SendBatchMaxSize is the maximum size of a batch. It must be larger than SendBatchSize. diff --git a/config/crd/bases/core.opni.io_collectors.yaml b/config/crd/bases/core.opni.io_collectors.yaml index 1a54d8aacf..c75d69eab6 100644 --- a/config/crd/bases/core.opni.io_collectors.yaml +++ b/config/crd/bases/core.opni.io_collectors.yaml @@ -53,33 +53,24 @@ spec: format: int32 type: integer sendBatchSize: - default: 1000 format: int32 type: integer timeout: - allOf: - - format: int64 - - format: duration - default: 15s + format: int64 type: integer type: object memoryLimiter: properties: checkInterval: - allOf: - - format: int64 - - format: duration - default: 1s + format: int64 type: integer limitMib: - default: 1000 format: int32 type: integer limitPercentage: format: int32 type: integer spikeLimitMib: - default: 350 format: int32 type: integer spikeLimitPercentage: @@ -151,20 +142,15 @@ spec: memoryLimiter: properties: checkInterval: - allOf: - - format: int64 - - format: duration - default: 1s + format: int64 type: integer limitMib: - default: 1000 format: int32 type: integer limitPercentage: format: int32 type: integer spikeLimitMib: - default: 350 format: int32 type: integer spikeLimitPercentage: From ac83a53a15e318da950d405582bdd00084317421 Mon Sep 17 00:00:00 2001 From: Joao Henri Date: Fri, 6 Oct 2023 19:41:45 -0300 Subject: [PATCH 16/21] refactor fetch of OTELConfig Signed-off-by: Joao Henri --- pkg/resources/collector/workloads.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pkg/resources/collector/workloads.go b/pkg/resources/collector/workloads.go index 61acb489ea..c429fc3a1d 100644 --- a/pkg/resources/collector/workloads.go +++ b/pkg/resources/collector/workloads.go @@ -58,7 +58,7 @@ func (r *Reconciler) aggregatorConfigMapName() string { } func (r *Reconciler) getDaemonConfig(loggingReceivers []string) otel.NodeConfig { - nodeConfig := otel.NodeConfig{ + return otel.NodeConfig{ Instance: r.collector.Name, Logs: otel.LoggingConfig{ Enabled: r.collector.Spec.LoggingConfig != nil, @@ -69,8 +69,6 @@ func (r *Reconciler) getDaemonConfig(loggingReceivers []string) otel.NodeConfig LogLevel: r.collector.Spec.LogLevel, OTELConfig: r.getDaemonOTELConfig(), } - - return nodeConfig } func (r *Reconciler) getDaemonOTELConfig() otel.NodeOTELConfig { @@ -104,7 +102,7 @@ func (r *Reconciler) getDaemonOTELConfig() otel.NodeOTELConfig { func (r *Reconciler) getAggregatorConfig( metricsCfg otel.MetricsConfig, ) otel.AggregatorConfig { - aggregatorCfg := otel.AggregatorConfig{ + return otel.AggregatorConfig{ LogsEnabled: r.collector.Spec.LoggingConfig != nil, Metrics: metricsCfg, AgentEndpoint: r.collector.Spec.AgentEndpoint, @@ -112,8 +110,6 @@ func (r *Reconciler) getAggregatorConfig( LogLevel: r.collector.Spec.LogLevel, OTELConfig: r.getAggregatorOTELConfig(), } - - return aggregatorCfg } func (r *Reconciler) getAggregatorOTELConfig() otel.AggregatorOTELConfig { From 24c21cdce98ea5b8c8b9118697b881cea82be4af Mon Sep 17 00:00:00 2001 From: Joao Henri Date: Mon, 9 Oct 2023 11:22:00 -0300 Subject: [PATCH 17/21] Add kubebuilder defaults bacl Signed-off-by: Joao Henri --- apis/core/v1beta1/collector_types.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/apis/core/v1beta1/collector_types.go b/apis/core/v1beta1/collector_types.go index 2684ae6d43..eaf22f84da 100644 --- a/apis/core/v1beta1/collector_types.go +++ b/apis/core/v1beta1/collector_types.go @@ -73,10 +73,12 @@ type MemoryLimiterProcessorConfig struct { // MemoryLimitMiB is the maximum amount of memory, in MiB, targeted to be // allocated by the process. + // +kubebuilder:default=1000 MemoryLimitMiB uint32 `json:"limitMib,omitempty"` // MemorySpikeLimitMiB is the maximum, in MiB, spike expected between the // measurements of memory usage. + // +kubebuilder:default=350 MemorySpikeLimitMiB uint32 `json:"spikeLimitMib,omitempty"` // MemoryLimitPercentage is the maximum amount of memory, in %, targeted to be @@ -100,6 +102,7 @@ type BatchProcessorConfig struct { // SendBatchSize is the size of a batch which after hit, will trigger it to be sent. // When this is set to zero, the batch size is ignored and data will be sent immediately // subject to only send_batch_max_size. + // +kubebuilder:default=1000 SendBatchSize uint32 `json:"sendBatchSize,omitempty"` // SendBatchMaxSize is the maximum size of a batch. It must be larger than SendBatchSize. @@ -114,10 +117,13 @@ type BatchProcessorConfig struct { // and controller-gen work. type CollectorSendingQueue struct { // Enabled indicates whether to not enqueue batches before sending to the consumerSender. + // +kubebuilder:default=true Enabled bool `json:"enabled,omitempty"` // NumConsumers is the number of consumers from the queue. + // +kubebuilder:default=4 NumConsumers int `json:"numConsumers,omitempty"` // QueueSize is the maximum number of batches allowed in queue at a given time. + // +kubebuilder:default=100 QueueSize int `json:"queueSize,omitempty"` } From 6e54de4d19f8c30bcd2d3eaa73a3b0b24aa7ef82 Mon Sep 17 00:00:00 2001 From: Joao Henri Date: Mon, 9 Oct 2023 14:58:18 +0000 Subject: [PATCH 18/21] Fix sendingQueue templates for OTLP exporters Signed-off-by: Joao Henri --- apis/core/v1beta1/collector_types.go | 10 ++-------- pkg/resources/collector/templates.go | 12 ++++++------ 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/apis/core/v1beta1/collector_types.go b/apis/core/v1beta1/collector_types.go index eaf22f84da..d5b700462f 100644 --- a/apis/core/v1beta1/collector_types.go +++ b/apis/core/v1beta1/collector_types.go @@ -73,12 +73,10 @@ type MemoryLimiterProcessorConfig struct { // MemoryLimitMiB is the maximum amount of memory, in MiB, targeted to be // allocated by the process. - // +kubebuilder:default=1000 MemoryLimitMiB uint32 `json:"limitMib,omitempty"` // MemorySpikeLimitMiB is the maximum, in MiB, spike expected between the // measurements of memory usage. - // +kubebuilder:default=350 MemorySpikeLimitMiB uint32 `json:"spikeLimitMib,omitempty"` // MemoryLimitPercentage is the maximum amount of memory, in %, targeted to be @@ -102,7 +100,6 @@ type BatchProcessorConfig struct { // SendBatchSize is the size of a batch which after hit, will trigger it to be sent. // When this is set to zero, the batch size is ignored and data will be sent immediately // subject to only send_batch_max_size. - // +kubebuilder:default=1000 SendBatchSize uint32 `json:"sendBatchSize,omitempty"` // SendBatchMaxSize is the maximum size of a batch. It must be larger than SendBatchSize. @@ -117,17 +114,14 @@ type BatchProcessorConfig struct { // and controller-gen work. type CollectorSendingQueue struct { // Enabled indicates whether to not enqueue batches before sending to the consumerSender. - // +kubebuilder:default=true Enabled bool `json:"enabled,omitempty"` // NumConsumers is the number of consumers from the queue. - // +kubebuilder:default=4 NumConsumers int `json:"numConsumers,omitempty"` // QueueSize is the maximum number of batches allowed in queue at a given time. - // +kubebuilder:default=100 QueueSize int `json:"queueSize,omitempty"` } -// CollectorSendingQueue has the attributes that we want to make +// OTLPExporterConfig has the attributes that we want to make // available from otlpexporter.Config. // Also, we extend it with the JSON struct tags needed in order to kubebuilder // and controller-gen work. @@ -135,7 +129,7 @@ type OTLPExporterConfig struct { SendingQueue CollectorSendingQueue `json:"sendingQueue,omitempty"` } -// CollectorSendingQueue has the attributes that we want to make +// OTLPHTTPExporterConfig has the attributes that we want to make // available from otlphttpexporter.Config. // Also, we extend it with the JSON struct tags needed in order to kubebuilder // and controller-gen work. diff --git a/pkg/resources/collector/templates.go b/pkg/resources/collector/templates.go index 67fee08cfe..66269307fb 100644 --- a/pkg/resources/collector/templates.go +++ b/pkg/resources/collector/templates.go @@ -194,9 +194,9 @@ exporters: tls: insecure: true sending_queue: - enabled: {{ .OTELConfig.Exporters.OTLP.SendingQueue.Enabled }} - num_consumers: {{ .OTELConfig.Exporters.OTLP.SendingQueue.NumConsumers }} - queue_size: {{ .OTELConfig.Exporters.OTLP.SendingQueue.QueueSize }} + enabled: {{ .OTELConfig.Exporters.OTLP.QueueSettings.Enabled }} + num_consumers: {{ .OTELConfig.Exporters.OTLP.QueueSettings.NumConsumers }} + queue_size: {{ .OTELConfig.Exporters.OTLP.QueueSettings.QueueSize }} retry_on_failure: enabled: true processors: @@ -289,9 +289,9 @@ exporters: tls: insecure: true sending_queue: - enabled: {{ .OTELConfig.Exporters.OTLPHTTP.SendingQueue.Enabled }} - num_consumers: {{ .OTELConfig.Exporters.OTLPHTTP.SendingQueue.NumConsumers }} - queue_size: {{ .OTELConfig.Exporters.OTLPHTTP.SendingQueue.QueueSize }} + enabled: {{ .OTELConfig.Exporters.OTLPHTTP.QueueSettings.Enabled }} + num_consumers: {{ .OTELConfig.Exporters.OTLPHTTP.QueueSettings.NumConsumers }} + queue_size: {{ .OTELConfig.Exporters.OTLPHTTP.QueueSettings.QueueSize }} retry_on_failure: enabled: true {{ template "metrics-remotewrite-exporter" .}} From ffed8b472edbed547c0982e4a310941a31d717e9 Mon Sep 17 00:00:00 2001 From: Joao Henri Date: Mon, 9 Oct 2023 15:49:13 -0300 Subject: [PATCH 19/21] Make timeout and check interval in seconds Signed-off-by: Joao Henri --- apis/core/v1beta1/collector_types.go | 30 +++++++++++++++++----------- pkg/resources/collector/workloads.go | 7 ++++--- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/apis/core/v1beta1/collector_types.go b/apis/core/v1beta1/collector_types.go index d5b700462f..a61f0f5987 100644 --- a/apis/core/v1beta1/collector_types.go +++ b/apis/core/v1beta1/collector_types.go @@ -1,8 +1,6 @@ package v1beta1 import ( - "time" - opnimeta "github.com/rancher/opni/pkg/util/meta" "github.com/rancher/wrangler/pkg/crd" "github.com/rancher/wrangler/pkg/schemas/openapi" @@ -69,14 +67,17 @@ type MemoryLimiterProcessorConfig struct { // CheckInterval is the time between measurements of memory usage for the // purposes of avoiding going over the limits. Defaults to zero, so no // checks will be performed. - CheckInterval time.Duration `json:"checkInterval,omitempty"` + // +kubebuilder:default:=1 + CheckIntervalSeconds uint32 `json:"checkIntervalSeconds,omitempty"` // MemoryLimitMiB is the maximum amount of memory, in MiB, targeted to be // allocated by the process. + // +kubebuilder:default:=1000 MemoryLimitMiB uint32 `json:"limitMib,omitempty"` // MemorySpikeLimitMiB is the maximum, in MiB, spike expected between the // measurements of memory usage. + // +kubebuilder:default:=350 MemorySpikeLimitMiB uint32 `json:"spikeLimitMib,omitempty"` // MemoryLimitPercentage is the maximum amount of memory, in %, targeted to be @@ -95,11 +96,13 @@ type MemoryLimiterProcessorConfig struct { type BatchProcessorConfig struct { // Timeout sets the time after which a batch will be sent regardless of size. // When this is set to zero, batched data will be sent immediately. - Timeout time.Duration `json:"timeout,omitempty"` + // +kubebuilder:default:=15 + TimeoutSeconds uint32 `json:"timeoutSeconds,omitempty"` // SendBatchSize is the size of a batch which after hit, will trigger it to be sent. // When this is set to zero, the batch size is ignored and data will be sent immediately // subject to only send_batch_max_size. + // +kubebuilder:default:=1000 SendBatchSize uint32 `json:"sendBatchSize,omitempty"` // SendBatchMaxSize is the maximum size of a batch. It must be larger than SendBatchSize. @@ -114,10 +117,13 @@ type BatchProcessorConfig struct { // and controller-gen work. type CollectorSendingQueue struct { // Enabled indicates whether to not enqueue batches before sending to the consumerSender. + // +kubebuilder:default:=true Enabled bool `json:"enabled,omitempty"` // NumConsumers is the number of consumers from the queue. + // +kubebuilder:default:=4 NumConsumers int `json:"numConsumers,omitempty"` // QueueSize is the maximum number of batches allowed in queue at a given time. + // +kubebuilder:default:=100 QueueSize int `json:"queueSize,omitempty"` } @@ -184,13 +190,13 @@ func NewDefaultAggregatorOTELConfigSpec() *AggregatorOTELConfigSpec { return &AggregatorOTELConfigSpec{ Processors: AggregatorOTELProcessors{ MemoryLimiter: MemoryLimiterProcessorConfig{ - MemoryLimitMiB: 1000, - MemorySpikeLimitMiB: 350, - CheckInterval: 1 * time.Second, + MemoryLimitMiB: 1000, + MemorySpikeLimitMiB: 350, + CheckIntervalSeconds: 1, }, Batch: BatchProcessorConfig{ - SendBatchSize: 1000, - Timeout: 15 * time.Second, + SendBatchSize: 1000, + TimeoutSeconds: 15, }, }, Exporters: AggregatorOTELExporters{ @@ -209,9 +215,9 @@ func NewDefaultNodeOTELConfigSpec() *NodeOTELConfigSpec { return &NodeOTELConfigSpec{ Processors: NodeOTELProcessors{ MemoryLimiter: MemoryLimiterProcessorConfig{ - MemoryLimitMiB: 250, - MemorySpikeLimitMiB: 50, - CheckInterval: 1 * time.Second, + MemoryLimitMiB: 250, + MemorySpikeLimitMiB: 50, + CheckIntervalSeconds: 1, }, }, Exporters: NodeOTELExporters{ diff --git a/pkg/resources/collector/workloads.go b/pkg/resources/collector/workloads.go index c429fc3a1d..63669e1c1a 100644 --- a/pkg/resources/collector/workloads.go +++ b/pkg/resources/collector/workloads.go @@ -5,6 +5,7 @@ import ( "crypto/sha256" "encoding/hex" "fmt" + "time" opnicorev1beta1 "github.com/rancher/opni/apis/core/v1beta1" opniloggingv1beta1 "github.com/rancher/opni/apis/logging/v1beta1" @@ -80,7 +81,7 @@ func (r *Reconciler) getDaemonOTELConfig() otel.NodeOTELConfig { return otel.NodeOTELConfig{ Processors: &otel.NodeOTELProcessors{ MemoryLimiter: memorylimiterprocessor.Config{ - CheckInterval: nodeOTELCfg.Processors.MemoryLimiter.CheckInterval, + CheckInterval: time.Duration(nodeOTELCfg.Processors.MemoryLimiter.CheckIntervalSeconds) * time.Second, MemoryLimitMiB: nodeOTELCfg.Processors.MemoryLimiter.MemoryLimitMiB, MemorySpikeLimitMiB: nodeOTELCfg.Processors.MemoryLimiter.MemorySpikeLimitMiB, MemoryLimitPercentage: nodeOTELCfg.Processors.MemoryLimiter.MemoryLimitPercentage, @@ -120,14 +121,14 @@ func (r *Reconciler) getAggregatorOTELConfig() otel.AggregatorOTELConfig { return otel.AggregatorOTELConfig{ Processors: &otel.AggregatorOTELProcessors{ MemoryLimiter: memorylimiterprocessor.Config{ - CheckInterval: aggregatorOTELCfg.Processors.MemoryLimiter.CheckInterval, + CheckInterval: time.Duration(aggregatorOTELCfg.Processors.MemoryLimiter.CheckIntervalSeconds) * time.Second, MemoryLimitMiB: aggregatorOTELCfg.Processors.MemoryLimiter.MemoryLimitMiB, MemorySpikeLimitMiB: aggregatorOTELCfg.Processors.MemoryLimiter.MemorySpikeLimitMiB, MemoryLimitPercentage: aggregatorOTELCfg.Processors.MemoryLimiter.MemoryLimitPercentage, MemorySpikePercentage: aggregatorOTELCfg.Processors.MemoryLimiter.MemorySpikePercentage, }, Batch: batchprocessor.Config{ - Timeout: aggregatorOTELCfg.Processors.Batch.Timeout, + Timeout: time.Duration(aggregatorOTELCfg.Processors.Batch.TimeoutSeconds) * time.Second, SendBatchSize: aggregatorOTELCfg.Processors.Batch.SendBatchSize, SendBatchMaxSize: aggregatorOTELCfg.Processors.Batch.SendBatchMaxSize, }, From edaec527e4e881f091a7656389434de0998b8807 Mon Sep 17 00:00:00 2001 From: Joao Henri Date: Tue, 10 Oct 2023 03:07:57 +0000 Subject: [PATCH 20/21] Generate collector Signed-off-by: Joao Henri --- config/crd/bases/core.opni.io_collectors.yaml | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/config/crd/bases/core.opni.io_collectors.yaml b/config/crd/bases/core.opni.io_collectors.yaml index c75d69eab6..7ade24f160 100644 --- a/config/crd/bases/core.opni.io_collectors.yaml +++ b/config/crd/bases/core.opni.io_collectors.yaml @@ -37,10 +37,13 @@ spec: sendingQueue: properties: enabled: + default: true type: boolean numConsumers: + default: 4 type: integer queueSize: + default: 100 type: integer type: object type: object @@ -53,24 +56,29 @@ spec: format: int32 type: integer sendBatchSize: + default: 1000 format: int32 type: integer - timeout: - format: int64 + timeoutSeconds: + default: 15 + format: int32 type: integer type: object memoryLimiter: properties: - checkInterval: - format: int64 + checkIntervalSeconds: + default: 1 + format: int32 type: integer limitMib: + default: 1000 format: int32 type: integer limitPercentage: format: int32 type: integer spikeLimitMib: + default: 350 format: int32 type: integer spikeLimitPercentage: @@ -129,10 +137,13 @@ spec: sendingQueue: properties: enabled: + default: true type: boolean numConsumers: + default: 4 type: integer queueSize: + default: 100 type: integer type: object type: object @@ -141,16 +152,19 @@ spec: properties: memoryLimiter: properties: - checkInterval: - format: int64 + checkIntervalSeconds: + default: 1 + format: int32 type: integer limitMib: + default: 1000 format: int32 type: integer limitPercentage: format: int32 type: integer spikeLimitMib: + default: 350 format: int32 type: integer spikeLimitPercentage: From 51262a3d001cdb0e5e79909cb9f3c9f53f108d7e Mon Sep 17 00:00:00 2001 From: Joao Henri Date: Wed, 11 Oct 2023 12:26:22 -0300 Subject: [PATCH 21/21] Add log message on OTEL Collector defaults and use default for metrics Signed-off-by: Joao Henri --- pkg/resources/collector/workloads.go | 2 ++ .../pkg/agent/drivers/opni_manager_otel/otel_driver.go | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/resources/collector/workloads.go b/pkg/resources/collector/workloads.go index 63669e1c1a..f1b5e8998c 100644 --- a/pkg/resources/collector/workloads.go +++ b/pkg/resources/collector/workloads.go @@ -75,6 +75,7 @@ func (r *Reconciler) getDaemonConfig(loggingReceivers []string) otel.NodeConfig func (r *Reconciler) getDaemonOTELConfig() otel.NodeOTELConfig { nodeOTELCfg := r.collector.Spec.NodeOTELConfigSpec if nodeOTELCfg == nil { + r.logger.Warn("found no config for the daemon's OTEL Collector, falling back to default") nodeOTELCfg = opnicorev1beta1.NewDefaultNodeOTELConfigSpec() } @@ -116,6 +117,7 @@ func (r *Reconciler) getAggregatorConfig( func (r *Reconciler) getAggregatorOTELConfig() otel.AggregatorOTELConfig { aggregatorOTELCfg := r.collector.Spec.AggregatorOTELConfigSpec if aggregatorOTELCfg == nil { + r.logger.Warn("found no config for the aggregator's OTEL Collector, falling back to default") aggregatorOTELCfg = opnicorev1beta1.NewDefaultAggregatorOTELConfigSpec() } return otel.AggregatorOTELConfig{ diff --git a/plugins/metrics/pkg/agent/drivers/opni_manager_otel/otel_driver.go b/plugins/metrics/pkg/agent/drivers/opni_manager_otel/otel_driver.go index 3b48f2e9bd..931f9b175f 100644 --- a/plugins/metrics/pkg/agent/drivers/opni_manager_otel/otel_driver.go +++ b/plugins/metrics/pkg/agent/drivers/opni_manager_otel/otel_driver.go @@ -215,8 +215,10 @@ func (o *OTELNodeDriver) buildEmptyCollector() *opnicorev1beta1.Collector { ImageSpec: opnimeta.ImageSpec{ ImagePullPolicy: lo.ToPtr(corev1.PullAlways), }, - SystemNamespace: o.Namespace, - AgentEndpoint: otel.AgentEndpoint(serviceName), + SystemNamespace: o.Namespace, + AgentEndpoint: otel.AgentEndpoint(serviceName), + NodeOTELConfigSpec: opnicorev1beta1.NewDefaultNodeOTELConfigSpec(), + AggregatorOTELConfigSpec: opnicorev1beta1.NewDefaultAggregatorOTELConfigSpec(), }, } }