diff --git a/api/v1beta1/lokistack_types.go b/api/v1beta1/lokistack_types.go index 99887033..efd86939 100644 --- a/api/v1beta1/lokistack_types.go +++ b/api/v1beta1/lokistack_types.go @@ -423,12 +423,28 @@ type IngestionLimitSpec struct { // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:number",displayName="Max Global Streams per Tenant" MaxGlobalStreamsPerTenant int32 `json:"maxGlobalStreamsPerTenant,omitempty"` - // MaxLineSize defines the aximum line size on ingestion path. Units in Bytes. + // MaxLineSize defines the maximum line size on ingestion path. Units in Bytes. // // +optional // +kubebuilder:validation:Optional // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:number",displayName="Max Line Size" MaxLineSize int32 `json:"maxLineSize,omitempty"` + + // PerStreamRateLimit defines the maximum byte rate per second for each stream. Units in Bytes. + // + // +optional + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:number",displayName="Per Stream Rate Limit" + PerStreamRateLimit int32 `json:"perStreamRateLimit,omitempty"` + + // PerStreamRateLimitBurst defines the maximum burst bytes per stream. + // This value sets the upper limit for the stream rate limit. When exceeding + // this, the stream is limited. + // + // +optional + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:number",displayName="PerStreamRateLimitBurst" + PerStreamRateLimitBurst int32 `json:"perStreamRateLimitBurst,omitempty"` } // LimitsTemplateSpec defines the limits applied at ingestion or query path. diff --git a/bundle/manifests/loki-operator.clusterserviceversion.yaml b/bundle/manifests/loki-operator.clusterserviceversion.yaml index 82572186..50269451 100644 --- a/bundle/manifests/loki-operator.clusterserviceversion.yaml +++ b/bundle/manifests/loki-operator.clusterserviceversion.yaml @@ -128,12 +128,25 @@ spec: path: limits.global.ingestion.maxLabelValueLength x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number - - description: MaxLineSize defines the aximum line size on ingestion path. Units - in Bytes. + - description: MaxLineSize defines the maximum line size on ingestion path. + Units in Bytes. displayName: Max Line Size path: limits.global.ingestion.maxLineSize x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: PerStreamRateLimit defines the maximum byte rate per second for + each stream. Units in Bytes. + displayName: Per Stream Rate Limit + path: limits.global.ingestion.perStreamRateLimit + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number + - description: PerStreamRateLimitBurst defines the maximum burst bytes per stream. + This value sets the upper limit for the stream rate limit. When exceeding + this, the stream is limited. + displayName: PerStreamRateLimitBurst + path: limits.global.ingestion.perStreamRateLimitBurst + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number - description: MaxChunksPerQuery defines the maximum number of chunks that can be fetched by a single query. displayName: Max Chunk per Query @@ -191,12 +204,25 @@ spec: path: limits.tenants.ingestion.maxLabelValueLength x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number - - description: MaxLineSize defines the aximum line size on ingestion path. Units - in Bytes. + - description: MaxLineSize defines the maximum line size on ingestion path. + Units in Bytes. displayName: Max Line Size path: limits.tenants.ingestion.maxLineSize x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: PerStreamRateLimit defines the maximum byte rate per second for + each stream. Units in Bytes. + displayName: Per Stream Rate Limit + path: limits.tenants.ingestion.perStreamRateLimit + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number + - description: PerStreamRateLimitBurst defines the maximum burst bytes per stream. + This value sets the upper limit for the stream rate limit. When exceeding + this, the stream is limited. + displayName: PerStreamRateLimitBurst + path: limits.tenants.ingestion.perStreamRateLimitBurst + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number - description: MaxChunksPerQuery defines the maximum number of chunks that can be fetched by a single query. displayName: Max Chunk per Query diff --git a/bundle/manifests/loki.openshift.io_lokistacks.yaml b/bundle/manifests/loki.openshift.io_lokistacks.yaml index 56ff4aef..16e97fb8 100644 --- a/bundle/manifests/loki.openshift.io_lokistacks.yaml +++ b/bundle/manifests/loki.openshift.io_lokistacks.yaml @@ -87,10 +87,22 @@ spec: format: int32 type: integer maxLineSize: - description: MaxLineSize defines the aximum line size + description: MaxLineSize defines the maximum line size on ingestion path. Units in Bytes. format: int32 type: integer + perStreamRateLimit: + description: PerStreamRateLimit defines the maximum byte + rate per second for each stream. Units in Bytes. + format: int32 + type: integer + perStreamRateLimitBurst: + description: PerStreamRateLimitBurst defines the maximum + burst bytes per stream. This value sets the upper limit + for the stream rate limit. When exceeding this, the + stream is limited. + format: int32 + type: integer type: object queries: description: QueryLimits defines the limit applied on querying @@ -157,10 +169,22 @@ spec: format: int32 type: integer maxLineSize: - description: MaxLineSize defines the aximum line size + description: MaxLineSize defines the maximum line size on ingestion path. Units in Bytes. format: int32 type: integer + perStreamRateLimit: + description: PerStreamRateLimit defines the maximum + byte rate per second for each stream. Units in Bytes. + format: int32 + type: integer + perStreamRateLimitBurst: + description: PerStreamRateLimitBurst defines the maximum + burst bytes per stream. This value sets the upper + limit for the stream rate limit. When exceeding this, + the stream is limited. + format: int32 + type: integer type: object queries: description: QueryLimits defines the limit applied on querying diff --git a/config/crd/bases/loki.openshift.io_lokistacks.yaml b/config/crd/bases/loki.openshift.io_lokistacks.yaml index 72368157..a7d48c1e 100644 --- a/config/crd/bases/loki.openshift.io_lokistacks.yaml +++ b/config/crd/bases/loki.openshift.io_lokistacks.yaml @@ -68,7 +68,15 @@ spec: format: int32 type: integer maxLineSize: - description: MaxLineSize defines the aximum line size on ingestion path. Units in Bytes. + description: MaxLineSize defines the maximum line size on ingestion path. Units in Bytes. + format: int32 + type: integer + perStreamRateLimit: + description: PerStreamRateLimit defines the maximum byte rate per second for each stream. Units in Bytes. + format: int32 + type: integer + perStreamRateLimitBurst: + description: PerStreamRateLimitBurst defines the maximum burst bytes per stream. This value sets the upper limit for the stream rate limit. When exceeding this, the stream is limited. format: int32 type: integer type: object @@ -121,7 +129,15 @@ spec: format: int32 type: integer maxLineSize: - description: MaxLineSize defines the aximum line size on ingestion path. Units in Bytes. + description: MaxLineSize defines the maximum line size on ingestion path. Units in Bytes. + format: int32 + type: integer + perStreamRateLimit: + description: PerStreamRateLimit defines the maximum byte rate per second for each stream. Units in Bytes. + format: int32 + type: integer + perStreamRateLimitBurst: + description: PerStreamRateLimitBurst defines the maximum burst bytes per stream. This value sets the upper limit for the stream rate limit. When exceeding this, the stream is limited. format: int32 type: integer type: object diff --git a/config/manifests/bases/loki-operator.clusterserviceversion.yaml b/config/manifests/bases/loki-operator.clusterserviceversion.yaml index a1a9dd65..95b6ee58 100644 --- a/config/manifests/bases/loki-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/loki-operator.clusterserviceversion.yaml @@ -107,12 +107,25 @@ spec: path: limits.global.ingestion.maxLabelValueLength x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number - - description: MaxLineSize defines the aximum line size on ingestion path. Units - in Bytes. + - description: MaxLineSize defines the maximum line size on ingestion path. + Units in Bytes. displayName: Max Line Size path: limits.global.ingestion.maxLineSize x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: PerStreamRateLimit defines the maximum byte rate per second for + each stream. Units in Bytes. + displayName: Per Stream Rate Limit + path: limits.global.ingestion.perStreamRateLimit + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number + - description: PerStreamRateLimitBurst defines the maximum burst bytes per stream. + This value sets the upper limit for the stream rate limit. When exceeding + this, the stream is limited. + displayName: PerStreamRateLimitBurst + path: limits.global.ingestion.perStreamRateLimitBurst + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number - description: MaxChunksPerQuery defines the maximum number of chunks that can be fetched by a single query. displayName: Max Chunk per Query @@ -170,12 +183,25 @@ spec: path: limits.tenants.ingestion.maxLabelValueLength x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number - - description: MaxLineSize defines the aximum line size on ingestion path. Units - in Bytes. + - description: MaxLineSize defines the maximum line size on ingestion path. + Units in Bytes. displayName: Max Line Size path: limits.tenants.ingestion.maxLineSize x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: PerStreamRateLimit defines the maximum byte rate per second for + each stream. Units in Bytes. + displayName: Per Stream Rate Limit + path: limits.tenants.ingestion.perStreamRateLimit + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number + - description: PerStreamRateLimitBurst defines the maximum burst bytes per stream. + This value sets the upper limit for the stream rate limit. When exceeding + this, the stream is limited. + displayName: PerStreamRateLimitBurst + path: limits.tenants.ingestion.perStreamRateLimitBurst + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number - description: MaxChunksPerQuery defines the maximum number of chunks that can be fetched by a single query. displayName: Max Chunk per Query diff --git a/internal/manifests/internal/config/build_test.go b/internal/manifests/internal/config/build_test.go index abcc932b..c9cb32c0 100644 --- a/internal/manifests/internal/config/build_test.go +++ b/internal/manifests/internal/config/build_test.go @@ -13,15 +13,13 @@ func TestBuild_ConfigAndRuntimeConfig_NoRuntimeConfigGenerated(t *testing.T) { auth_enabled: true chunk_store_config: chunk_cache_config: - enable_fifocache: yes + enable_fifocache: true + fifocache: + max_size_bytes: 500MB compactor: compaction_interval: 2h shared_store: s3 working_directory: /tmp/loki/compactor -distributor: - ring: - kvstore: - store: memberlist frontend: tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 compress_responses: true @@ -35,10 +33,11 @@ frontend_worker: ingester: chunk_block_size: 262144 chunk_encoding: snappy - chunk_idle_period: 2h - chunk_retain_period: 1m - chunk_target_size: 1572864 + chunk_idle_period: 1h + chunk_retain_period: 30s + chunk_target_size: 1048576 lifecycler: + final_sleep: 0s heartbeat_period: 5s interface_names: - eth0 @@ -47,13 +46,11 @@ ingester: ring: replication_factor: 1 heartbeat_timeout: 1m - kvstore: - store: memberlist max_transfer_retries: 0 wal: - enabled: true - dir: /tmp/wal - replay_memory_ceiling: 5000 + enabled: true + dir: /tmp/wal + replay_memory_ceiling: 3750 ingester_client: grpc_client_config: max_recv_msg_size: 67108864 @@ -85,14 +82,14 @@ limits_config: max_entries_limit_per_query: 5000 max_global_streams_per_user: 0 max_chunks_per_query: 2000000 - max_query_length: 12000h + max_query_length: 721h max_query_parallelism: 32 max_query_series: 500 cardinality_limit: 100000 max_streams_matchers_per_query: 1000 max_cache_freshness_per_query: 10m - per_stream_rate_limit: 3000000 - per_stream_rate_limit_burst: 15000000 + per_stream_rate_limit: 5000000 + per_stream_rate_limit_burst: 10000000 memberlist: abort_if_cluster_join_fails: true bind_port: 7946 @@ -113,7 +110,11 @@ query_range: align_queries_with_step: true cache_results: true max_retries: 5 - results_cache: {} + results_cache: + cache: + enable_fifocache: true + fifocache: + max_size_bytes: 500MB split_queries_by_interval: 30m parallelise_shardable_queries: false schema_config: @@ -127,6 +128,8 @@ schema_config: store: boltdb-shipper server: graceful_shutdown_timeout: 5s + grpc_server_min_time_between_pings: '10s' + grpc_server_ping_without_stream_allowed: true grpc_server_max_concurrent_streams: 1000 grpc_server_max_recv_msg_size: 104857600 grpc_server_max_send_msg_size: 104857600 @@ -141,8 +144,8 @@ storage_config: cache_ttl: 24h resync_interval: 5m shared_store: s3 - index_gateway_client: - server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 + index_gateway_client: + server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 aws: s3: http://test.default.svc.cluster.local.:9000 bucketnames: loki @@ -228,15 +231,13 @@ func TestBuild_ConfigAndRuntimeConfig_BothGenerated(t *testing.T) { auth_enabled: true chunk_store_config: chunk_cache_config: - enable_fifocache: yes + enable_fifocache: true + fifocache: + max_size_bytes: 500MB compactor: compaction_interval: 2h shared_store: s3 working_directory: /tmp/loki/compactor -distributor: - ring: - kvstore: - store: memberlist frontend: tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 compress_responses: true @@ -250,10 +251,11 @@ frontend_worker: ingester: chunk_block_size: 262144 chunk_encoding: snappy - chunk_idle_period: 2h - chunk_retain_period: 1m - chunk_target_size: 1572864 + chunk_idle_period: 1h + chunk_retain_period: 30s + chunk_target_size: 1048576 lifecycler: + final_sleep: 0s heartbeat_period: 5s interface_names: - eth0 @@ -262,9 +264,11 @@ ingester: ring: replication_factor: 1 heartbeat_timeout: 1m - kvstore: - store: memberlist - max_transfer_retries: 60 + max_transfer_retries: 0 + wal: + enabled: true + dir: /tmp/wal + replay_memory_ceiling: 3750 ingester_client: grpc_client_config: max_recv_msg_size: 67108864 @@ -296,14 +300,14 @@ limits_config: max_entries_limit_per_query: 5000 max_global_streams_per_user: 0 max_chunks_per_query: 2000000 - max_query_length: 12000h + max_query_length: 721h max_query_parallelism: 32 max_query_series: 500 cardinality_limit: 100000 max_streams_matchers_per_query: 1000 max_cache_freshness_per_query: 10m - per_stream_rate_limit: 3000000 - per_stream_rate_limit_burst: 15000000 + per_stream_rate_limit: 5000000 + per_stream_rate_limit_burst: 10000000 memberlist: abort_if_cluster_join_fails: true bind_port: 7946 @@ -324,7 +328,11 @@ query_range: align_queries_with_step: true cache_results: true max_retries: 5 - results_cache: {} + results_cache: + cache: + enable_fifocache: true + fifocache: + max_size_bytes: 500MB split_queries_by_interval: 30m parallelise_shardable_queries: false schema_config: @@ -338,6 +346,8 @@ schema_config: store: boltdb-shipper server: graceful_shutdown_timeout: 5s + grpc_server_min_time_between_pings: '10s' + grpc_server_ping_without_stream_allowed: true grpc_server_max_concurrent_streams: 1000 grpc_server_max_recv_msg_size: 104857600 grpc_server_max_send_msg_size: 104857600 @@ -352,8 +362,8 @@ storage_config: cache_ttl: 24h resync_interval: 5m shared_store: s3 - index_gateway_client: - server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 + index_gateway_client: + server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 aws: s3: http://test.default.svc.cluster.local.:9000 bucketnames: loki diff --git a/internal/manifests/internal/config/loki-config.yaml b/internal/manifests/internal/config/loki-config.yaml index 957ef426..6caf3464 100644 --- a/internal/manifests/internal/config/loki-config.yaml +++ b/internal/manifests/internal/config/loki-config.yaml @@ -2,15 +2,13 @@ auth_enabled: true chunk_store_config: chunk_cache_config: - enable_fifocache: yes + enable_fifocache: true + fifocache: + max_size_bytes: 500MB compactor: compaction_interval: 2h shared_store: s3 working_directory: {{ .StorageDirectory }}/compactor -distributor: - ring: - kvstore: - store: memberlist frontend: tail_proxy_url: http://{{ .Querier.FQDN }}:{{ .Querier.Port }} compress_responses: true @@ -24,10 +22,11 @@ frontend_worker: ingester: chunk_block_size: 262144 chunk_encoding: snappy - chunk_idle_period: 2h - chunk_retain_period: 1m - chunk_target_size: 1572864 + chunk_idle_period: 1h + chunk_retain_period: 30s + chunk_target_size: 1048576 lifecycler: + final_sleep: 0s heartbeat_period: 5s interface_names: - eth0 @@ -36,13 +35,11 @@ ingester: ring: replication_factor: {{ .Stack.ReplicationFactor }} heartbeat_timeout: 1m - kvstore: - store: memberlist max_transfer_retries: 0 wal: - enabled: true - dir: {{ .WriteAheadLog.Directory }} - replay_memory_ceiling: {{ .WriteAheadLog.ReplayMemoryCeiling }} + enabled: true + dir: {{ .WriteAheadLog.Directory }} + replay_memory_ceiling: {{ .WriteAheadLog.ReplayMemoryCeiling }} ingester_client: grpc_client_config: max_recv_msg_size: 67108864 @@ -74,12 +71,14 @@ limits_config: max_entries_limit_per_query: {{ .Stack.Limits.Global.QueryLimits.MaxEntriesLimitPerQuery }} max_global_streams_per_user: {{ .Stack.Limits.Global.IngestionLimits.MaxGlobalStreamsPerTenant }} max_chunks_per_query: {{ .Stack.Limits.Global.QueryLimits.MaxChunksPerQuery }} - max_query_length: 12000h - max_query_parallelism: 16 + max_query_length: 721h + max_query_parallelism: 32 max_query_series: {{ .Stack.Limits.Global.QueryLimits.MaxQuerySeries }} cardinality_limit: 100000 max_streams_matchers_per_query: 1000 max_cache_freshness_per_query: 10m + per_stream_rate_limit: {{ .Stack.Limits.Global.IngestionLimits.PerStreamRateLimit }} + per_stream_rate_limit_burst: {{ .Stack.Limits.Global.IngestionLimits.PerStreamRateLimitBurst }} memberlist: abort_if_cluster_join_fails: true bind_port: {{ .GossipRing.Port }} @@ -100,7 +99,11 @@ query_range: align_queries_with_step: true cache_results: true max_retries: 5 - results_cache: {} + results_cache: + cache: + enable_fifocache: true + fifocache: + max_size_bytes: 500MB split_queries_by_interval: 30m parallelise_shardable_queries: false schema_config: @@ -114,6 +117,8 @@ schema_config: store: boltdb-shipper server: graceful_shutdown_timeout: 5s + grpc_server_min_time_between_pings: '10s' + grpc_server_ping_without_stream_allowed: true grpc_server_max_concurrent_streams: 1000 grpc_server_max_recv_msg_size: 104857600 grpc_server_max_send_msg_size: 104857600 @@ -129,7 +134,7 @@ storage_config: resync_interval: 5m shared_store: s3 index_gateway_client: - server_address: dns:///{{ .IndexGateway.FQDN }}:{{ .IndexGateway.Port }} + server_address: dns:///{{ .IndexGateway.FQDN }}:{{ .IndexGateway.Port }} aws: s3: {{ .ObjectStorage.Endpoint }} bucketnames: {{ .ObjectStorage.Buckets }} diff --git a/internal/manifests/internal/config/loki-runtime-config.yaml b/internal/manifests/internal/config/loki-runtime-config.yaml index a2d23ab7..a16b25a5 100644 --- a/internal/manifests/internal/config/loki-runtime-config.yaml +++ b/internal/manifests/internal/config/loki-runtime-config.yaml @@ -24,6 +24,12 @@ overrides: {{ if $l.MaxGlobalStreamsPerTenant }} max_global_streams_per_user: {{ $l.MaxGlobalStreamsPerTenant }} {{- end -}} + {{ if $l.PerStreamRateLimit }} + per_stream_rate_limit: {{ $l.PerStreamRateLimit }} + {{- end -}} + {{ if $l.PerStreamRateLimitBurst }} + per_stream_rate_limit_burst: {{ $l.PerStreamRateLimitBurst }} + {{- end -}} {{- end -}} {{- if $l := $spec.QueryLimits -}} {{ if $l.MaxEntriesLimitPerQuery }}