Merge pull request #55 from qclaogui:logs-with-caches-enabled

Docker Compose(logs): Enable caches
qclaogui · Mar 19, 2024 · 06e4e1c · 06e4e1c
2 parents cfaed9e + de70275
commit 06e4e1c
Show file tree

Hide file tree

Showing 11 changed files with 162 additions and 76 deletions.
diff --git a/docker-compose/common/compose-include/memcached.yaml b/docker-compose/common/compose-include/memcached.yaml
@@ -7,6 +7,7 @@ services:
 
   memcached-exporter:
     labels:
+      - logs.agent.grafana.com/scrape=false
       - metrics.agent.grafana.com/scrape=true
       # - metrics.agent.grafana.com/job=memcached
       - metrics.agent.grafana.com/port=9150

diff --git a/docker-compose/common/config/agent-flow/modules/docker/logs/all.river b/docker-compose/common/config/agent-flow/modules/docker/logs/all.river
@@ -38,7 +38,7 @@ argument "keep_labels" {
 // get the all available containers.
 discovery.docker "dd_logs" {
 	host             = "unix:///var/run/docker.sock"
-	refresh_interval = "15s"
+	refresh_interval = "30s"
 
 	filter {
 		name   = "status"

diff --git a/docker-compose/common/config/agent-flow/modules/docker/metrics/all.river b/docker-compose/common/config/agent-flow/modules/docker/metrics/all.river
@@ -14,7 +14,7 @@ argument "tenant" {
 // get the all available containers.
 discovery.docker "dd_metrics" {
 	host             = "unix:///var/run/docker.sock"
-	refresh_interval = "15s"
+	refresh_interval = "30s"
 
 	filter {
 		name   = "status"

diff --git a/docker-compose/common/config/loki/microservices-mode-logs.yaml b/docker-compose/common/config/loki/microservices-mode-logs.yaml
@@ -26,44 +26,76 @@ common:
         insecure_skip_verify: true
 
 compactor:
+  working_directory: /tmp/compactor
   shared_store: s3
 
 memberlist:
   join_members:
   - loki-memberlist:7946
 
-frontend:
-  scheduler_address: query-scheduler:9095
-  tail_proxy_url: http://querier:3100
-frontend_worker:
-  scheduler_address: query-scheduler:9095
+query_range:
+  align_queries_with_step: true
+
+  cache_results: true
+  results_cache:
+    cache:
+      memcached_client:
+        consistent_hash: true
+        addresses: "dns+memcached:11211"
+
+  cache_index_stats_results: true
+  index_stats_results_cache:
+    cache:
+      memcached_client:
+        consistent_hash: true
+        addresses: "dns+memcached:11211"
+
+limits_config:
+  max_cache_freshness_per_query: 10m
+  enforce_metric_name: false
+  reject_old_samples: true
+  reject_old_samples_max_age: 30m
+  split_queries_by_interval: 15m
 
 ruler:
   storage:
+    type: s3
     s3:
       bucketnames: loki-ruler
 
+runtime_config:
+  file: /etc/loki/runtime.yaml
+
 schema_config:
   configs:
   - from: "2023-08-01"
     index:
-      period: 24h
       prefix: loki_index_
+      period: 24h
     object_store: s3
     store: tsdb
     schema: v12
 
 chunk_store_config:
   chunk_cache_config:
-    embedded_cache:
-      enabled: true
+    memcached_client:
+      addresses: "dns+memcached:11211"
 
-query_range:
-  align_queries_with_step: true
-  cache_results: true
-  max_retries: 5
-  results_cache:
-    cache:
-      embedded_cache:
-        enabled: true
-        ttl: 1h
+ingester:
+  lifecycler:
+    join_after: 10s
+    observe_period: 5s
+    ring:
+      replication_factor: 2
+  chunk_idle_period: 1m
+  max_chunk_age: 1m
+  chunk_retain_period: 30s
+  chunk_encoding: snappy
+  flush_op_timeout: 10s
+
+
+frontend:
+  scheduler_address: query-scheduler:9095
+  tail_proxy_url: http://querier:3100
+frontend_worker:
+  scheduler_address: query-scheduler:9095
diff --git a/docker-compose/common/config/loki/monolithic-mode-logs.yaml b/docker-compose/common/config/loki/monolithic-mode-logs.yaml
@@ -35,11 +35,39 @@ memberlist:
 
 query_range:
   align_queries_with_step: true
+
   cache_results: true
   results_cache:
     cache:
-      embedded_cache:
-        enabled: true
+      memcached_client:
+        consistent_hash: true
+        addresses: "dns+memcached:11211"
+
+  cache_index_stats_results: true
+  index_stats_results_cache:
+    cache:
+      memcached_client:
+        consistent_hash: true
+        addresses: "dns+memcached:11211"
+  # cache_volume_results: true
+  # cache_series_results: true
+  # instant_metric_query_split_align: true
+  # cache_instant_metric_results: true
+  # instant_metric_results_cache:
+  #   cache:
+  #     memcached_client:
+  #       consistent_hash: true
+  #       addresses: "dns+memcached:11211"
+  # series_results_cache:
+  #   cache:
+  #     memcached_client:
+  #       consistent_hash: true
+  #       addresses: dns+memcached:11211"
+  # volume_results_cache:
+  #   cache:
+  #     memcached_client:
+  #       consistent_hash: true
+  #       addresses: "dns+memcached:11211"
 
 limits_config:
   max_cache_freshness_per_query: 10m
@@ -53,17 +81,21 @@ ruler:
       bucketnames: loki-ruler
     type: s3
 
+runtime_config:
+  file: /etc/loki/runtime.yaml
+
 schema_config:
   configs:
   - from: "2023-08-01"
     index:
-      period: 24h
       prefix: loki_index_
+      period: 24h
     object_store: s3
     store: tsdb
     schema: v12
 
 chunk_store_config:
   chunk_cache_config:
-    embedded_cache:
-      enabled: true
+    memcached_client:
+      addresses: "dns+memcached:11211"
+
diff --git a/docker-compose/common/config/loki/read-write-mode-logs.yaml b/docker-compose/common/config/loki/read-write-mode-logs.yaml
@@ -35,35 +35,37 @@ memberlist:
 
 query_range:
   align_queries_with_step: true
+
   cache_results: true
   results_cache:
     cache:
-      embedded_cache:
-        enabled: true
+      memcached_client:
+        consistent_hash: true
+        addresses: "dns+memcached:11211"
+
+  cache_index_stats_results: true
+  index_stats_results_cache:
+    cache:
+      memcached_client:
+        consistent_hash: true
+        addresses: "dns+memcached:11211"
 
 limits_config:
+  max_cache_freshness_per_query: 10m
   enforce_metric_name: false
   reject_old_samples: true
   reject_old_samples_max_age: 30m
   split_queries_by_interval: 15m
 
-ingester:
-  lifecycler:
-    join_after: 10s
-    observe_period: 5s
-    ring:
-      replication_factor: 2
-  chunk_idle_period: 1m
-  max_chunk_age: 1m
-  chunk_retain_period: 30s
-  chunk_encoding: snappy
-  flush_op_timeout: 10s
-
 ruler:
   storage:
+    type: s3
     s3:
       bucketnames: loki-ruler
-    type: s3
+
+
+runtime_config:
+  file: /etc/loki/runtime.yaml
 
 schema_config:
   configs:
@@ -77,11 +79,17 @@ schema_config:
 
 chunk_store_config:
   chunk_cache_config:
-    embedded_cache:
-      enabled: true
-
-frontend:
-  compress_responses: true
+    memcached_client:
+      addresses: "dns+memcached:11211"
 
-query_scheduler:
-  max_outstanding_requests_per_tenant: 1024
+ingester:
+  lifecycler:
+    join_after: 10s
+    observe_period: 5s
+    ring:
+      replication_factor: 2
+  chunk_idle_period: 1m
+  max_chunk_age: 1m
+  chunk_retain_period: 30s
+  chunk_encoding: snappy
+  flush_op_timeout: 10s
diff --git a/docker-compose/common/config/loki/runtime.yaml b/docker-compose/common/config/loki/runtime.yaml
@@ -0,0 +1,10 @@
+# This file can be used to set overrides or other runtime config.
+overrides:
+  "fake": # limits for anonymous that the whole cluster enforces
+    ingestion_rate_mb: 1500000
+    max_streams_per_user: 100000
+    max_chunks_per_query: 100000
+  "anonymous": # limits for anonymous that the whole cluster enforces
+    ingestion_rate_mb: 1500000
+    max_streams_per_user: 100000
+    max_chunks_per_query: 100000
diff --git a/docker-compose/microservices-mode/logs/docker-compose.yaml b/docker-compose/microservices-mode/logs/docker-compose.yaml
@@ -9,6 +9,7 @@ version: '3.9'
 include:
   - path: ../../common/compose-include/minio.yaml
   - path: ../../common/compose-include/agent-collect-logs.yaml
+  - path: ../../common/compose-include/memcached.yaml
 
 x-labels: &loki-labels
   - logs.agent.grafana.com/log-format=json
@@ -48,9 +49,9 @@ services:
         condition: service_started
     image: &lokiImage ${LOKI_IMAGE:-docker.io/grafana/loki:latest}
     volumes:
-      - ../../common/config/loki/microservices-mode-logs.yaml:/etc/loki.yaml # Note: Loki use microservices-mode-logs.yaml
+      - ../../common/config/loki:/etc/loki
     command:
-      - -config.file=/etc/loki.yaml
+      - -config.file=/etc/loki/microservices-mode-logs.yaml
       - -target=distributor
       - -config.expand-env=true
     healthcheck:
@@ -72,9 +73,9 @@ services:
         condition: service_healthy
     image: *lokiImage
     volumes:
-      - ../../common/config/loki/microservices-mode-logs.yaml:/etc/loki.yaml # Note: Loki use microservices-mode-logs.yaml
+      - ../../common/config/loki:/etc/loki
     command:
-      - -config.file=/etc/loki.yaml
+      - -config.file=/etc/loki/microservices-mode-logs.yaml
       - -target=ingester
       - -config.expand-env=true
     deploy:
@@ -91,19 +92,19 @@ services:
         condition: service_started
     image: *lokiImage
     volumes:
-      - ../../common/config/loki/microservices-mode-logs.yaml:/etc/loki.yaml # Note: Loki use microservices-mode-logs.yaml
+      - ../../common/config/loki:/etc/loki
     command:
-      - -config.file=/etc/loki.yaml
+      - -config.file=/etc/loki/microservices-mode-logs.yaml
       - -target=query-frontend
       - -config.expand-env=true
 
   query-scheduler:
     labels: *loki-labels
     image: *lokiImage
     volumes:
-      - ../../common/config/loki/microservices-mode-logs.yaml:/etc/loki.yaml # Note: Loki use microservices-mode-logs.yaml
+      - ../../common/config/loki:/etc/loki
     command:
-      - -config.file=/etc/loki.yaml
+      - -config.file=/etc/loki/microservices-mode-logs.yaml
       - -target=query-scheduler
       - -config.expand-env=true
     deploy:
@@ -120,9 +121,9 @@ services:
         condition: service_started
     image: *lokiImage
     volumes:
-      - ../../common/config/loki/microservices-mode-logs.yaml:/etc/loki.yaml # Note: Loki use microservices-mode-logs.yaml
+      - ../../common/config/loki:/etc/loki
     command:
-      - -config.file=/etc/loki.yaml
+      - -config.file=/etc/loki/microservices-mode-logs.yaml
       - -target=querier
       - -config.expand-env=true
 
@@ -133,11 +134,11 @@ services:
         condition: service_healthy
     image: *lokiImage
     volumes:
-    - ../../common/config/loki/microservices-mode-logs.yaml:/etc/loki.yaml # Note: Loki use microservices-mode-logs.yaml
+      - ../../common/config/loki:/etc/loki
     command:
-    - -config.file=/etc/loki.yaml
-    - -target=ruler
-    - -config.expand-env=true
+      - -config.file=/etc/loki/microservices-mode-logs.yaml
+      - -target=ruler
+      - -config.expand-env=true
     networks:
       default:
         aliases:
@@ -147,9 +148,9 @@ services:
     labels: *loki-labels
     image: *lokiImage
     volumes:
-      - ../../common/config/loki/microservices-mode-logs.yaml:/etc/loki.yaml # Note: Loki use microservices-mode-logs.yaml
+      - ../../common/config/loki:/etc/loki
     command:
-      - -config.file=/etc/loki.yaml
+      - -config.file=/etc/loki/microservices-mode-logs.yaml
       - -target=compactor
       - -config.expand-env=true
     networks:

diff --git a/docker-compose/monolithic-mode/all-in-one/docker-compose.yaml b/docker-compose/monolithic-mode/all-in-one/docker-compose.yaml
@@ -110,9 +110,9 @@ services:
         condition: service_healthy
     image: ${LOKI_IMAGE:-docker.io/grafana/loki:latest}
     volumes:
-      - ../../common/config/loki/monolithic-mode-logs.yaml:/etc/config.yaml
+      - ../../common/config/loki:/etc/loki
     command:
-      - -config.file=/etc/config.yaml
+      - -config.file=/etc/loki/monolithic-mode-logs.yaml
       - -target=all
       - -config.expand-env=true
     environment: