Kubernetes(LGTMP): Enable caches

Makefile

@@ -284,7 +284,7 @@ endef
 
 
 .PHONY: deploy-monolithic-mode-metrics
-deploy-monolithic-mode-metrics: deploy-grafana ## Deploy monolithic-mode metrics
+deploy-monolithic-mode-metrics: deploy-memcached ## Deploy monolithic-mode metrics
 	$(info ******************** deploy monolithic-mode metrics manifests ********************)
 	@$(KUSTOMIZE) build kubernetes/monolithic-mode/metrics | kubectl apply -f -
 	kubectl rollout status -n monitoring-system deployment/mimir --watch --timeout=600s
@@ -294,7 +294,7 @@ delete-monolithic-mode-metrics: delete-minio
 
 
 .PHONY: deploy-monolithic-mode-logs
-deploy-monolithic-mode-logs: deploy-grafana ## Deploy monolithic-mode logs
+deploy-monolithic-mode-logs: deploy-memcached ## Deploy monolithic-mode logs
 	$(info ******************** deploy monolithic-mode logs manifests ********************)
 	@$(KUSTOMIZE) build --enable-helm kubernetes/monolithic-mode/logs | kubectl apply -f -
 	kubectl rollout status -n logging-system statefulset/loki --watch --timeout=600s
@@ -304,7 +304,7 @@ delete-monolithic-mode-logs: delete-minio
 
 
 .PHONY: deploy-monolithic-mode-profiles
-deploy-monolithic-mode-profiles: deploy-grafana ## Deploy monolithic-mode profiles
+deploy-monolithic-mode-profiles: deploy-memcached ## Deploy monolithic-mode profiles
 	$(info ******************** deploy monolithic-mode profiles manifests ********************)
 	@$(KUSTOMIZE) build --enable-helm kubernetes/monolithic-mode/profiles | kubectl apply -f -
 	kubectl rollout status -n profiles-system statefulset/pyroscope --watch --timeout=600s
@@ -314,7 +314,7 @@ delete-monolithic-mode-profiles: delete-minio
 
 
 .PHONY: deploy-monolithic-mode-traces
-deploy-monolithic-mode-traces: deploy-grafana ## Deploy monolithic-mode traces
+deploy-monolithic-mode-traces: deploy-memcached ## Deploy monolithic-mode traces
 	$(info ******************** deploy monolithic-mode traces manifests ********************)
 	@$(KUSTOMIZE) build --enable-helm kubernetes/monolithic-mode/traces | kubectl apply -f -
 	kubectl rollout status -n tracing-system statefulset/tempo --watch --timeout=600s
@@ -324,7 +324,7 @@ delete-monolithic-mode-traces: delete-minio
 
 
 .PHONY: deploy-monolithic-mode-all-in-one
-deploy-monolithic-mode-all-in-one: deploy-grafana ## Deploy monolithic-mode all-in-one
+deploy-monolithic-mode-all-in-one: deploy-memcached ## Deploy monolithic-mode all-in-one
 	$(info ******************** deploy monolithic-mode all-in-one manifests ********************)
 	@$(KUSTOMIZE) build --enable-helm kubernetes/monolithic-mode/all-in-one | kubectl apply -f -
 	kubectl rollout status -n monitoring-system deployment/mimir --watch --timeout=600s
@@ -335,7 +335,7 @@ delete-monolithic-mode-all-in-one: delete-minio
 
 
 .PHONY: deploy-read-write-mode-metrics
-deploy-read-write-mode-metrics: deploy-grafana ## Deploy read-write-mode metrics
+deploy-read-write-mode-metrics: deploy-memcached ## Deploy read-write-mode metrics
 	$(info ******************** deploy read-write-mode metrics manifests ********************)
 	@$(KUSTOMIZE) build kubernetes/read-write-mode/metrics | kubectl apply -f -
 	kubectl rollout status -n monitoring-system deployment/mimir-write --watch --timeout=600s
@@ -345,7 +345,7 @@ delete-read-write-mode-metrics: delete-minio
 
 
 .PHONY: deploy-read-write-mode-logs
-deploy-read-write-mode-logs: deploy-grafana ## Deploy read-write-mode logs
+deploy-read-write-mode-logs: deploy-memcached ## Deploy read-write-mode logs
 	$(info ******************** deploy read-write-mode logs manifests ********************)
 	@$(KUSTOMIZE) build --enable-helm kubernetes/read-write-mode/logs | kubectl apply -f -
 	kubectl rollout status -n logging-system statefulset/loki-write --watch --timeout=600s
@@ -356,7 +356,7 @@ delete-read-write-mode-logs: delete-minio
 
 
 .PHONY: deploy-microservices-mode-logs
-deploy-microservices-mode-logs: deploy-grafana ## Deploy microservices-mode logs
+deploy-microservices-mode-logs: deploy-memcached ## Deploy microservices-mode logs
 	$(info ******************** deploy microservices-mode logs manifests ********************)
 	@$(KUSTOMIZE) build --enable-helm kubernetes/microservices-mode/logs | kubectl apply -f -
 	kubectl rollout status -n logging-system statefulset/loki-distributed-ingester --watch --timeout=600s
@@ -366,7 +366,7 @@ delete-microservices-mode-logs: delete-minio
 
 
 .PHONY: deploy-microservices-mode-metrics
-deploy-microservices-mode-metrics: deploy-grafana ## Deploy microservices-mode metrics
+deploy-microservices-mode-metrics: deploy-memcached ## Deploy microservices-mode metrics
 	$(info ******************** deploy microservices-mode metrics manifests ********************)
 	@$(KUSTOMIZE) build --enable-helm kubernetes/microservices-mode/metrics | kubectl apply -f -
 	kubectl rollout status -n monitoring-system statefulset/mimir-distributed-ingester --watch --timeout=600s
@@ -376,7 +376,7 @@ delete-microservices-mode-metrics: delete-minio
 
 
 .PHONY: deploy-microservices-mode-profiles
-deploy-microservices-mode-profiles: deploy-grafana ## Deploy microservices-mode profiles
+deploy-microservices-mode-profiles: deploy-memcached ## Deploy microservices-mode profiles
 	$(info ******************** deploy microservices-mode profiles manifests ********************)
 	@$(KUSTOMIZE) build --enable-helm kubernetes/microservices-mode/profiles | kubectl apply -f -
 	@$(call config_changes_trigger_pod_restart, "Go to http://localhost:8080/explore for the profiles.")
@@ -385,7 +385,7 @@ delete-microservices-mode-profiles: delete-minio
 
 
 .PHONY: deploy-microservices-mode-traces
-deploy-microservices-mode-traces: deploy-grafana ## Deploy microservices-mode traces
+deploy-microservices-mode-traces: deploy-memcached ## Deploy microservices-mode traces
 	$(info ******************** deploy microservices-mode traces manifests ********************)
 	@$(KUSTOMIZE) build --enable-helm kubernetes/microservices-mode/traces | kubectl apply -f -
 	kubectl rollout status -n tracing-system statefulset/tempo-distributed-ingester --watch --timeout=600s
@@ -397,7 +397,7 @@ delete-microservices-mode-traces: delete-minio
 ##@ Grafana Agent Integrations
 
 .PHONY: deploy-memcached
-deploy-memcached: ## Deploy integration memcached manifests
+deploy-memcached: deploy-grafana ## Deploy integration memcached manifests
 	$(info ******************** deploy integration memcached manifests ********************)
 	@$(KUSTOMIZE) build --enable-helm kubernetes/common/memcached | kubectl apply -f -
 delete-memcached:

docker-compose/common/config/loki/monolithic-mode-logs.yaml

@@ -94,4 +94,3 @@ chunk_store_config:
   chunk_cache_config:
     memcached_client:
       addresses: "dns+memcached:11211"
-

kubernetes/microservices-mode/logs/configs/loki-distributed.yaml

@@ -5,7 +5,9 @@ analytics:
  reporting_enabled: false
 
 chunk_store_config:
-  max_look_back_period: 0s
+  chunk_cache_config:
+    memcached_client:
+      addresses: "dns+memcached.memcached-system.svc:11211"
 
 # https://grafana.com/docs/loki/latest/configure/#use-environment-variables-in-the-configuration
 common:
@@ -64,12 +66,15 @@ memberlist:
 query_range:
   align_queries_with_step: true
   cache_results: true
-  max_retries: 5
   results_cache:
     cache:
-      embedded_cache:
-        enabled: true
-        ttl: 1h
+      memcached_client:
+        addresses: "dns+memcached.memcached-system.svc:11211"
+  cache_index_stats_results: true
+  index_stats_results_cache:
+    cache:
+      memcached_client:
+        addresses: "dns+memcached.memcached-system.svc:11211"
 
 ruler:
   ring:
@@ -94,9 +99,10 @@ schema_config:
     store: boltdb-shipper
 
 server:
-  grpc_listen_port: 9095
   http_listen_port: 3100
-  log_level: warn
+  grpc_listen_port: 9095
+  log_level: info
+  log_format: json
 
 storage_config:
   boltdb_shipper:
@@ -108,4 +114,4 @@ storage_config:
 
 table_manager:
   retention_deletes_enabled: false
-  retention_period: 0s
+  retention_period: 0s

kubernetes/microservices-mode/logs/k8s-all-in-one.yaml

@@ -31,15 +31,17 @@ metadata:
 ---
 apiVersion: v1
 data:
-  config.yaml: |2
+  config.yaml: |2-
 
     auth_enabled: false
 
     analytics:
      reporting_enabled: false
 
     chunk_store_config:
-      max_look_back_period: 0s
+      chunk_cache_config:
+        memcached_client:
+          addresses: "dns+memcached.memcached-system.svc:11211"
 
     # https://grafana.com/docs/loki/latest/configure/#use-environment-variables-in-the-configuration
     common:
@@ -98,12 +100,15 @@ data:
     query_range:
       align_queries_with_step: true
       cache_results: true
-      max_retries: 5
       results_cache:
         cache:
-          embedded_cache:
-            enabled: true
-            ttl: 1h
+          memcached_client:
+            addresses: "dns+memcached.memcached-system.svc:11211"
+      cache_index_stats_results: true
+      index_stats_results_cache:
+        cache:
+          memcached_client:
+            addresses: "dns+memcached.memcached-system.svc:11211"
 
     ruler:
       ring:
@@ -128,9 +133,10 @@ data:
         store: boltdb-shipper
 
     server:
-      grpc_listen_port: 9095
       http_listen_port: 3100
-      log_level: warn
+      grpc_listen_port: 9095
+      log_level: info
+      log_format: json
 
     storage_config:
       boltdb_shipper:

kubernetes/microservices-mode/metrics/configs/mimir-distributed.yaml

@@ -16,14 +16,30 @@ alertmanager_storage:
 
 blocks_storage:
   backend: s3
-  bucket_store:
-    sync_dir: /data/tsdb-sync
   s3:
     bucket_name: mimir-blocks
   tsdb:
     dir: /data/tsdb
-    head_compaction_interval: 15m
-    wal_replay_concurrency: 3
+    ship_interval: 1m
+    block_ranges_period: [ 2h ]
+    retention_period: 3h
+  bucket_store:
+    sync_dir: /data/tsdb-sync
+    index_cache:
+      backend: memcached
+      memcached:
+        addresses: dns+memcached.memcached-system.svc:11211
+
+    chunks_cache:
+      backend: memcached
+      memcached:
+        addresses: dns+memcached.memcached-system.svc:11211
+
+    metadata_cache:
+      backend: memcached
+      memcached:
+        addresses: dns+memcached.memcached-system.svc:11211
+
 
 common:
   storage:
@@ -50,6 +66,13 @@ compactor:
 frontend:
   parallelize_shardable_queries: true
   scheduler_address: mimir-distributed-query-scheduler-headless.monitoring-system.svc:9095
+  query_sharding_target_series_per_shard: 2500
+  cache_results: true
+  results_cache:
+    backend: memcached
+    memcached:
+      addresses: dns+memcached.memcached-system.svc:11211
+
 frontend_worker:
   grpc_client_config:
     max_send_msg_size: 419430400
@@ -73,6 +96,8 @@ limits:
   native_histograms_ingestion_enabled: true
 
 memberlist:
+  abort_if_cluster_join_fails: false
+  compression_enabled: false
   join_members:
   - dns+mimir-distributed-gossip-ring.monitoring-system.svc.cluster.local:7946
 
@@ -92,6 +117,10 @@ ruler_storage:
   backend: s3
   s3:
     bucket_name: mimir-ruler
+  cache:
+    backend: memcached
+    memcached:
+      addresses: dns+memcached.memcached-system.svc:11211
 
 runtime_config:
   file: /var/mimir/runtime.yaml

kubernetes/microservices-mode/metrics/configs/runtime.yaml

@@ -1,4 +1,4 @@
-# https://grafana.com/docs/mimir/latest/configure/about-runtime-configuration/
+# This file can be used to set overrides or other runtime config.
 ingester_limits: # limits that each ingester replica enforces
   max_ingestion_rate: 20000
   max_series: 1500000
@@ -14,7 +14,6 @@ overrides:
   anonymous: # limits for anonymous that the whole cluster enforces
     # ingestion_tenant_shard_size: 9
     max_global_series_per_user: 1500000
-    max_global_series_per_metric: 50000
     max_fetched_series_per_query: 100000
-    ruler_max_rules_per_rule_group: 100
-    ruler_max_rule_groups_per_tenant: 100
+    native_histograms_ingestion_enabled: true
+    ruler_max_rules_per_rule_group: 50

kubernetes/microservices-mode/metrics/k8s-all-in-one.yaml

@@ -10539,14 +10539,30 @@ data:
 
     blocks_storage:
       backend: s3
-      bucket_store:
-        sync_dir: /data/tsdb-sync
       s3:
         bucket_name: mimir-blocks
       tsdb:
         dir: /data/tsdb
-        head_compaction_interval: 15m
-        wal_replay_concurrency: 3
+        ship_interval: 1m
+        block_ranges_period: [ 2h ]
+        retention_period: 3h
+      bucket_store:
+        sync_dir: /data/tsdb-sync
+        index_cache:
+          backend: memcached
+          memcached:
+            addresses: dns+memcached.memcached-system.svc:11211
+
+        chunks_cache:
+          backend: memcached
+          memcached:
+            addresses: dns+memcached.memcached-system.svc:11211
+
+        metadata_cache:
+          backend: memcached
+          memcached:
+            addresses: dns+memcached.memcached-system.svc:11211
+
 
     common:
       storage:
@@ -10573,6 +10589,13 @@ data:
     frontend:
       parallelize_shardable_queries: true
       scheduler_address: mimir-distributed-query-scheduler-headless.monitoring-system.svc:9095
+      query_sharding_target_series_per_shard: 2500
+      cache_results: true
+      results_cache:
+        backend: memcached
+        memcached:
+          addresses: dns+memcached.memcached-system.svc:11211
+
     frontend_worker:
       grpc_client_config:
         max_send_msg_size: 419430400
@@ -10596,6 +10619,8 @@ data:
       native_histograms_ingestion_enabled: true
 
     memberlist:
+      abort_if_cluster_join_fails: false
+      compression_enabled: false
       join_members:
       - dns+mimir-distributed-gossip-ring.monitoring-system.svc.cluster.local:7946
 
@@ -10615,6 +10640,10 @@ data:
       backend: s3
       s3:
         bucket_name: mimir-ruler
+      cache:
+        backend: memcached
+        memcached:
+          addresses: dns+memcached.memcached-system.svc:11211
 
     runtime_config:
       file: /var/mimir/runtime.yaml
@@ -10651,7 +10680,7 @@ metadata:
 apiVersion: v1
 data:
   runtime.yaml: |-
-    # https://grafana.com/docs/mimir/latest/configure/about-runtime-configuration/
+    # This file can be used to set overrides or other runtime config.
     ingester_limits: # limits that each ingester replica enforces
       max_ingestion_rate: 20000
       max_series: 1500000
@@ -10667,10 +10696,9 @@ data:
       anonymous: # limits for anonymous that the whole cluster enforces
         # ingestion_tenant_shard_size: 9
         max_global_series_per_user: 1500000
-        max_global_series_per_metric: 50000
         max_fetched_series_per_query: 100000
-        ruler_max_rules_per_rule_group: 100
-        ruler_max_rule_groups_per_tenant: 100
+        native_histograms_ingestion_enabled: true
+        ruler_max_rules_per_rule_group: 50
 kind: ConfigMap
 metadata:
   labels: