[#361] Integrated Loki and Promtail into Deployment Stack

This commit introduces Loki and Promtail to the deployment stack,
enhancing the project's logging infrastructure by enabling advanced log
aggregation, visualization, and analysis capabilities. The integration
involves modifications across multiple configuration files and the
introduction of new templates for Loki and Promtail. Here's a breakdown
of the changes:

- config.mk Adjustments: Updated the prepare-config target to include
  steps for preparing Loki and Promtail configuration files, ensuring
  these services are configured as part of the deployment setup.
- Docker Compose Template Update: Extended the docker-compose.yml.tpl to
  define loki and promtail services, including their respective Docker
  images, volumes, and dependencies. This update ensures that both Loki
  for log aggregation and Promtail for log shipping are part of the
  containerized environment.
- Grafana Data Sources Configuration: Modified the
  grafana-provisioning/datasources/datasource.yml to include Loki as a
  data source, enabling Grafana to query log data collected by Loki. This
  allows for comprehensive log visualization alongside metrics within
  Grafana dashboards.
- Loki Configuration Template (loki.yml): Introduced a new configuration
  template for Loki, specifying its operational parameters such as
  server ports, ingester configuration, schema, storage, and retention
  policies. This configuration is designed to optimize Loki's log
  aggregation capabilities within the deployment environment.
- Promtail Configuration Template (promtail.yml): Added a new
  configuration template for Promtail, detailing how log files are
  located, scraped, and forwarded to Loki. The configuration includes the
  definition of job names, static configs, and labels for log
  categorization.

The integration of Loki and Promtail represents a significant
enhancement to the project's observability infrastructure. It enables
efficient log collection, storage, and querying, providing developers
and operators with powerful tools for troubleshooting, performance
monitoring, and operational analysis.

scripts/govtool/config.mk

@@ -18,7 +18,7 @@ docker_compose_file := $(target_config_dir)/docker-compose.yml
 cardano_config_provider := https://book.world.dev.cardano.org
 
 .PHONY: prepare-config
-prepare-config: clear generate-docker-compose-file enable-prometheus prepare-dbsync-secrets prepare-backend-config prepare-prometheus-config prepare-grafana-provisioning prepare-nginx-config
+prepare-config: clear generate-docker-compose-file enable-prometheus prepare-dbsync-secrets prepare-backend-config prepare-grafana-provisioning prepare-nginx-config
 
 .PHONY: clear
 clear:
@@ -68,8 +68,16 @@ prepare-backend-config:
 prepare-prometheus-config:
 	cp -a "$(template_config_dir)/prometheus.yml" "$(target_config_dir)/prometheus.yml"
 
+.PHONY: prepare-promtail-config
+prepare-promtail-config:
+	cp -a "$(template_config_dir)/promtail.yml" "$(target_config_dir)/promtail.yml"
+
+.PHONY: prepare-loki-config
+prepare-loki-config:
+	cp -a "$(template_config_dir)/loki.yml" "$(target_config_dir)/loki.yml"
+
 .PHONY: prepare-grafana-provisioning
-prepare-grafana-provisioning:
+prepare-grafana-provisioning: prepare-prometheus-config prepare-loki-config prepare-promtail-config
 	mkdir -p $(grafana_provisioning_dir)
 	cp -a $(template_config_dir)/grafana-provisioning/* $(grafana_provisioning_dir)
 	sed -e "s|<GRAFANA_SLACK_RECIPIENT>|$${GRAFANA_SLACK_RECIPIENT}|" \

scripts/govtool/config/templates/docker-compose.yml.tpl

@@ -36,6 +36,24 @@ services:
       - "traefik.http.routers.http-catchall.entrypoints=web"
       - "traefik.http.routers.http-catchall.middlewares=redirect-to-https"
 
+  loki:
+    image: grafana/loki:2.9.4
+    ports:
+      - "3100:3100"
+    command: -config.file=/etc/loki/loki.yml
+    volumes:
+      - loki-data:/loki
+      - /home/<DOCKER_USER>/config/loki.yml:/etc/loki/loki.yml
+
+  promtail:
+    image: grafana/promtail:2.9.4
+    volumes:
+      - /var/log:/var/log
+      - /home/<DOCKER_USER>/config/promtail.yml:/etc/promtail/promtail.yml
+    command: -config.file=/etc/promtail/promtail.yml
+    depends_on:
+      - loki
+
   prometheus:
     image: prom/prometheus:v2.47.1
     volumes:
@@ -229,3 +247,4 @@ volumes:
   prometheus-data:
   node-db:
   node-ipc:
+  loki-data:

scripts/govtool/config/templates/grafana-provisioning/datasources/datasource.yml

@@ -48,3 +48,21 @@ datasources:
   version: 1
   # <bool> allow users to edit datasources from the UI.
   editable: true
+
+  # <string, required> name of the datasource. Required
+- name: Loki
+  # <string, required> datasource type. Required
+  type: loki
+  # <string, required> access mode. direct or proxy. Required
+  access: browser
+  # <int> org id. will default to orgId 1 if not specified
+  orgId: 1
+  # <string> url
+  url: http://loki:3100
+  # <bool> enable/disable basic auth
+  basicAuth: false
+  # <bool> mark as default datasource. Max one per org
+  isDefault: false
+  version: 1
+  # <bool> allow users to edit datasources from the UI.
+  editable: true

scripts/govtool/config/templates/loki.yml

@@ -0,0 +1,56 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9095
+
+ingester:
+  wal:
+    enabled: false
+  lifecycler:
+    address: 127.0.0.1
+    ring:
+      kvstore:
+        store: inmemory
+      replication_factor: 1
+  chunk_idle_period: 15m
+  chunk_block_size: 262144
+  chunk_retain_period: 1m
+  max_transfer_retries: 0
+
+compactor:
+  working_directory: /loki/compactor
+  shared_store: filesystem
+
+schema_config:
+  configs:
+  - from: 2024-01-01
+    store: boltdb-shipper
+    object_store: filesystem
+    schema: v11
+    index:
+      prefix: index_
+      period: 24h
+
+storage_config:
+  boltdb_shipper:
+    active_index_directory: /loki/index
+    cache_location: /loki/index_cache
+    cache_ttl: 24h
+    shared_store: filesystem
+  filesystem:
+    directory: /loki/chunks
+
+limits_config:
+  enforce_metric_name: false
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+  max_cache_freshness_per_query: 10m
+  retention_period: 672h
+
+chunk_store_config:
+  max_look_back_period: 672h
+
+table_manager:
+  retention_deletes_enabled: true
+  retention_period: 672h

scripts/govtool/config/templates/promtail.yml

@@ -0,0 +1,18 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+- job_name: system
+  static_configs:
+  - targets:
+      - localhost
+    labels:
+      job: varlogs
+      __path__: /var/log/*log