diff --git a/examples/distributed-loki/1-prerequisites/config.yaml b/examples/distributed-loki/1-prerequisites/config.yaml new file mode 100644 index 0000000..70e6c2a --- /dev/null +++ b/examples/distributed-loki/1-prerequisites/config.yaml @@ -0,0 +1,123 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: loki-distributed + labels: + app.kubernetes.io/instance: loki + app.kubernetes.io/name: loki-distributed + app.kubernetes.io/version: 2.5.0 +data: + config.yaml: | + auth_enabled: false + common: + storage: + $(LOKI_STORE) + server: + http_listen_port: 3100 + http_server_read_timeout: 1m + http_server_write_timeout: 1m + log_level: error + chunk_store_config: + max_look_back_period: 0s + compactor: + compaction_interval: 5m + shared_store: filesystem + distributor: + ring: + kvstore: + store: memberlist + frontend: + compress_responses: true + log_queries_longer_than: 5s + tail_proxy_url: http://loki-distributed-querier:3100 + frontend_worker: + frontend_address: loki-distributed-query-frontend:9095 + ingester: + chunk_block_size: 262144 + chunk_encoding: snappy + chunk_idle_period: 30m + chunk_retain_period: 1m + lifecycler: + ring: + kvstore: + store: memberlist + replication_factor: 1 + max_transfer_retries: 0 + wal: + dir: /loki-store/wal + memberlist: + join_members: + - loki-distributed-memberlist + query_range: + align_queries_with_step: true + cache_results: true + max_retries: 5 + results_cache: + cache: + enable_fifocache: true + fifocache: + max_size_bytes: 500MB + validity: 24h + ruler: + alertmanager_url: https://alertmanager.xx + external_url: https://alertmanager.xx + ring: + kvstore: + store: memberlist + rule_path: /tmp/loki/scratch + storage: + local: + directory: /etc/loki/rules + type: local + schema_config: + configs: + - from: 2022-01-01 + store: boltdb-shipper + object_store: $(LOKI_STORE_NAME) + schema: v11 + index: + prefix: index_ + period: 24h + storage_config: + filesystem: + directory: /loki-store/chunks + boltdb_shipper: + active_index_directory: /loki-store/index + shared_store: filesystem + cache_location: /loki-store/cache + cache_ttl: 24h + table_manager: + retention_deletes_enabled: false + retention_period: 0s + limits_config: + ingestion_rate_strategy: global + ingestion_rate_mb: 4 + ingestion_burst_size_mb: 6 + max_label_name_length: 1024 + max_label_value_length: 2048 + max_label_names_per_series: 30 + reject_old_samples: true + reject_old_samples_max_age: 15m + creation_grace_period: 10m + enforce_metric_name: false + max_line_size: 256000 + max_line_size_truncate: false + max_entries_limit_per_query: 10000 + max_streams_per_user: 0 + max_global_streams_per_user: 0 + unordered_writes: true + max_chunks_per_query: 2000000 + max_query_length: 721h + max_query_parallelism: 32 + max_query_series: 10000 + cardinality_limit: 100000 + max_streams_matchers_per_query: 1000 + max_concurrent_tail_requests: 10 + retention_period: 24h + max_cache_freshness_per_query: 5m + max_queriers_per_tenant: 0 + per_stream_rate_limit: 3MB + per_stream_rate_limit_burst: 15MB + max_query_lookback: 0 + min_sharding_lookback: 0s + split_queries_by_interval: 1m diff --git a/examples/distributed-loki/1-prerequisites/service-account.yaml b/examples/distributed-loki/1-prerequisites/service-account.yaml new file mode 100644 index 0000000..c5a1934 --- /dev/null +++ b/examples/distributed-loki/1-prerequisites/service-account.yaml @@ -0,0 +1,14 @@ +kind: ServiceAccount +apiVersion: v1 +metadata: + name: loki-distributed + labels: + app.kubernetes.io/instance: loki + app.kubernetes.io/name: loki-distributed + app.kubernetes.io/version: 2.5.0 +secrets: + - name: loki-distributed-token + - name: loki-distributed-dockercfg +imagePullSecrets: + - name: loki-distributed-dockercfg +automountServiceAccountToken: true diff --git a/examples/distributed-loki/2-components/distributor-deployment.yaml b/examples/distributed-loki/2-components/distributor-deployment.yaml new file mode 100644 index 0000000..defdd49 --- /dev/null +++ b/examples/distributed-loki/2-components/distributor-deployment.yaml @@ -0,0 +1,101 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: loki-distributed-distributor + labels: + app.kubernetes.io/component: distributor + app.kubernetes.io/instance: loki + app.kubernetes.io/name: loki-distributed + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: 2.5.0 +spec: + replicas: 2 + selector: + matchLabels: + app.kubernetes.io/component: distributor + app.kubernetes.io/instance: loki + app.kubernetes.io/name: loki-distributed + template: + metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: distributor + app.kubernetes.io/instance: loki + app.kubernetes.io/name: loki-distributed + app.kubernetes.io/part-of: memberlist + spec: + restartPolicy: Always + serviceAccountName: loki-distributed + schedulerName: default-scheduler + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/component: distributor + app.kubernetes.io/instance: loki + app.kubernetes.io/name: loki-distributed + topologyKey: kubernetes.io/hostname + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/component: distributor + app.kubernetes.io/instance: loki + app.kubernetes.io/name: loki-distributed + topologyKey: failure-domain.beta.kubernetes.io/zone + terminationGracePeriodSeconds: 30 + containers: + - resources: {} + readinessProbe: + httpGet: + path: /ready + port: http + scheme: HTTP + initialDelaySeconds: 30 + timeoutSeconds: 1 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 3 + terminationMessagePath: /dev/termination-log + name: distributor + securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + ports: + - name: http + containerPort: 3100 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + - name: http-memberlist + containerPort: 7946 + protocol: TCP + imagePullPolicy: IfNotPresent + volumeMounts: + - name: config + mountPath: /etc/loki/config + terminationMessagePolicy: File + image: 'grafana/loki:2.5.0' + args: + - '-config.file=/etc/loki/config/config.yaml' + - '-target=distributor' + serviceAccount: loki-distributed + volumes: + - name: config + configMap: + name: loki-distributed + defaultMode: 420 + dnsPolicy: ClusterFirst + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + maxSurge: 0 + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 \ No newline at end of file diff --git a/examples/distributed-loki/2-components/ingester-statefulset.yaml b/examples/distributed-loki/2-components/ingester-statefulset.yaml new file mode 100644 index 0000000..041d1d7 --- /dev/null +++ b/examples/distributed-loki/2-components/ingester-statefulset.yaml @@ -0,0 +1,122 @@ +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: loki-distributed-ingester + labels: + app.kubernetes.io/component: ingester + app.kubernetes.io/instance: loki + app.kubernetes.io/name: loki-distributed + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: 2.5.0 +spec: + replicas: 2 + selector: + matchLabels: + app.kubernetes.io/component: ingester + app.kubernetes.io/instance: loki + app.kubernetes.io/name: loki-distributed + template: + metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ingester + app.kubernetes.io/instance: loki + app.kubernetes.io/name: loki-distributed + app.kubernetes.io/part-of: memberlist + spec: + restartPolicy: Always + serviceAccountName: loki-distributed + schedulerName: default-scheduler + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/component: ingester + app.kubernetes.io/instance: loki + app.kubernetes.io/name: loki-distributed + topologyKey: kubernetes.io/hostname + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/component: ingester + app.kubernetes.io/instance: loki + app.kubernetes.io/name: loki-distributed + topologyKey: failure-domain.beta.kubernetes.io/zone + terminationGracePeriodSeconds: 300 + containers: + - resources: {} + readinessProbe: + httpGet: + path: /ready + port: http + scheme: HTTP + initialDelaySeconds: 30 + timeoutSeconds: 1 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 3 + terminationMessagePath: /dev/termination-log + name: ingester + securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + ports: + - name: http + containerPort: 3100 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + - name: http-memberlist + containerPort: 7946 + protocol: TCP + imagePullPolicy: IfNotPresent + volumeMounts: + - name: config + mountPath: /etc/loki/config + - name: loki-store + mountPath: /loki-store + terminationMessagePolicy: File + image: 'grafana/loki:2.5.0' + args: + - '-config.file=/etc/loki/config/config.yaml' + - '-target=ingester' + serviceAccount: loki-distributed + volumes: + - name: config + configMap: + name: loki-distributed + defaultMode: 420 + dnsPolicy: ClusterFirst + volumeClaimTemplates: + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: loki-store + creationTimestamp: null + labels: + app.kubernetes.io/component: ingester + app.kubernetes.io/created-by: loki-distributed-ingester + app.kubernetes.io/instance: loki + app.kubernetes.io/name: loki-distributed + app.kubernetes.io/version: 2.5.0 + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi + volumeMode: Filesystem + serviceName: loki-distributed-ingester-headless + podManagementPolicy: Parallel + updateStrategy: + type: RollingUpdate + rollingUpdate: + partition: 0 + revisionHistoryLimit: 10 \ No newline at end of file diff --git a/examples/distributed-loki/2-components/querier-statefulset.yaml b/examples/distributed-loki/2-components/querier-statefulset.yaml new file mode 100644 index 0000000..f44f672 --- /dev/null +++ b/examples/distributed-loki/2-components/querier-statefulset.yaml @@ -0,0 +1,122 @@ +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: loki-distributed-querier + labels: + app.kubernetes.io/component: querier + app.kubernetes.io/instance: loki + app.kubernetes.io/name: loki-distributed + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: 2.5.0 +spec: + replicas: 2 + selector: + matchLabels: + app.kubernetes.io/component: querier + app.kubernetes.io/instance: loki + app.kubernetes.io/name: loki-distributed + template: + metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: querier + app.kubernetes.io/instance: loki + app.kubernetes.io/name: loki-distributed + app.kubernetes.io/part-of: memberlist + spec: + restartPolicy: Always + serviceAccountName: loki-distributed + schedulerName: default-scheduler + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/component: querier + app.kubernetes.io/instance: loki + app.kubernetes.io/name: loki-distributed + topologyKey: kubernetes.io/hostname + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/component: querier + app.kubernetes.io/instance: loki + app.kubernetes.io/name: loki-distributed + topologyKey: failure-domain.beta.kubernetes.io/zone + terminationGracePeriodSeconds: 30 + containers: + - resources: {} + readinessProbe: + httpGet: + path: /ready + port: http + scheme: HTTP + initialDelaySeconds: 30 + timeoutSeconds: 1 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 3 + terminationMessagePath: /dev/termination-log + name: querier + securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + ports: + - name: http + containerPort: 3100 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + - name: http-memberlist + containerPort: 7946 + protocol: TCP + imagePullPolicy: IfNotPresent + volumeMounts: + - name: config + mountPath: /etc/loki/config + - name: loki-store + mountPath: /loki-store + terminationMessagePolicy: File + image: 'grafana/loki:2.5.0' + args: + - '-config.file=/etc/loki/config/config.yaml' + - '-target=querier' + serviceAccount: loki-distributed + volumes: + - name: config + configMap: + name: loki-distributed + defaultMode: 420 + dnsPolicy: ClusterFirst + volumeClaimTemplates: + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: loki-store + creationTimestamp: null + labels: + app.kubernetes.io/component: querier + app.kubernetes.io/created-by: loki-distributed-querier + app.kubernetes.io/instance: loki + app.kubernetes.io/name: loki-distributed + app.kubernetes.io/version: 2.5.0 + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi + volumeMode: Filesystem + serviceName: loki-distributed-querier-headless + podManagementPolicy: Parallel + updateStrategy: + type: RollingUpdate + rollingUpdate: + partition: 0 + revisionHistoryLimit: 10 \ No newline at end of file diff --git a/examples/distributed-loki/2-components/query-frontend-deployment.yaml b/examples/distributed-loki/2-components/query-frontend-deployment.yaml new file mode 100644 index 0000000..9921615 --- /dev/null +++ b/examples/distributed-loki/2-components/query-frontend-deployment.yaml @@ -0,0 +1,86 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: loki-distributed-query-frontend + labels: + app.kubernetes.io/component: query-frontend + app.kubernetes.io/instance: loki + app.kubernetes.io/name: loki-distributed + app.kubernetes.io/version: 2.5.0 +spec: + replicas: 2 + selector: + matchLabels: + app.kubernetes.io/component: query-frontend + app.kubernetes.io/instance: loki + app.kubernetes.io/name: loki-distributed + template: + metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: query-frontend + app.kubernetes.io/instance: loki + app.kubernetes.io/name: loki-distributed + spec: + restartPolicy: Always + serviceAccountName: loki-distributed + schedulerName: default-scheduler + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/component: query-frontend + app.kubernetes.io/instance: loki + app.kubernetes.io/name: loki-distributed + topologyKey: kubernetes.io/hostname + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/component: query-frontend + app.kubernetes.io/instance: loki + app.kubernetes.io/name: loki-distributed + topologyKey: failure-domain.beta.kubernetes.io/zone + terminationGracePeriodSeconds: 30 + containers: + - resources: {} + terminationMessagePath: /dev/termination-log + name: query-frontend + securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + ports: + - name: http + containerPort: 3100 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + imagePullPolicy: IfNotPresent + volumeMounts: + - name: config + mountPath: /etc/loki/config + terminationMessagePolicy: File + image: 'grafana/loki:2.5.0' + args: + - '-config.file=/etc/loki/config/config.yaml' + - '-target=query-frontend' + serviceAccount: loki-distributed + volumes: + - name: config + configMap: + name: loki-distributed + defaultMode: 420 + dnsPolicy: ClusterFirst + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + maxSurge: 0 + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 \ No newline at end of file diff --git a/examples/distributed-loki/3-services/distributor-service.yaml b/examples/distributed-loki/3-services/distributor-service.yaml new file mode 100644 index 0000000..a93e2eb --- /dev/null +++ b/examples/distributed-loki/3-services/distributor-service.yaml @@ -0,0 +1,29 @@ +kind: Service +apiVersion: v1 +metadata: + name: loki-distributed-distributor + labels: + app.kubernetes.io/component: distributor + app.kubernetes.io/instance: loki + app.kubernetes.io/name: loki-distributed + app.kubernetes.io/version: 2.5.0 +spec: + ipFamilies: + - IPv4 + ports: + - name: http + protocol: TCP + port: 3100 + targetPort: http + - name: grpc + protocol: TCP + port: 9095 + targetPort: grpc + internalTrafficPolicy: Cluster + type: ClusterIP + ipFamilyPolicy: SingleStack + sessionAffinity: None + selector: + app.kubernetes.io/component: distributor + app.kubernetes.io/instance: loki + app.kubernetes.io/name: loki-distributed diff --git a/examples/distributed-loki/3-services/ingester-service.yaml b/examples/distributed-loki/3-services/ingester-service.yaml new file mode 100644 index 0000000..04f51dc --- /dev/null +++ b/examples/distributed-loki/3-services/ingester-service.yaml @@ -0,0 +1,62 @@ +kind: Service +apiVersion: v1 +metadata: + name: loki-distributed-ingester + labels: + app.kubernetes.io/component: ingester + app.kubernetes.io/instance: loki + app.kubernetes.io/name: loki-distributed + app.kubernetes.io/version: 2.5.0 +spec: + ipFamilies: + - IPv4 + ports: + - name: http + protocol: TCP + port: 3100 + targetPort: http + - name: grpc + protocol: TCP + port: 9095 + targetPort: grpc + internalTrafficPolicy: Cluster + type: ClusterIP + ipFamilyPolicy: SingleStack + sessionAffinity: None + selector: + app.kubernetes.io/component: ingester + app.kubernetes.io/instance: loki + app.kubernetes.io/name: loki-distributed +--- +kind: Service +apiVersion: v1 +metadata: + name: loki-distributed-ingester-headless + labels: + app.kubernetes.io/component: ingester + app.kubernetes.io/instance: loki + app.kubernetes.io/name: loki-distributed + prometheus.io/service-monitor: 'false' +spec: + clusterIP: None + ipFamilies: + - IPv4 + ports: + - name: http + protocol: TCP + port: 3100 + targetPort: http + - name: grpc + protocol: TCP + port: 9095 + targetPort: grpc + internalTrafficPolicy: Cluster + clusterIPs: + - None + type: ClusterIP + ipFamilyPolicy: SingleStack + sessionAffinity: None + selector: + app.kubernetes.io/component: ingester + app.kubernetes.io/instance: loki + app.kubernetes.io/name: loki-distributed \ No newline at end of file diff --git a/examples/distributed-loki/3-services/memberlist-service.yaml b/examples/distributed-loki/3-services/memberlist-service.yaml new file mode 100644 index 0000000..557f430 --- /dev/null +++ b/examples/distributed-loki/3-services/memberlist-service.yaml @@ -0,0 +1,27 @@ +kind: Service +apiVersion: v1 +metadata: + name: loki-distributed-memberlist + labels: + app.kubernetes.io/instance: loki + app.kubernetes.io/name: loki-distributed + app.kubernetes.io/version: 2.5.0 +spec: + clusterIP: None + ipFamilies: + - IPv4 + ports: + - name: http + protocol: TCP + port: 7946 + targetPort: http-memberlist + internalTrafficPolicy: Cluster + clusterIPs: + - None + type: ClusterIP + ipFamilyPolicy: SingleStack + sessionAffinity: None + selector: + app.kubernetes.io/instance: loki + app.kubernetes.io/name: loki-distributed + app.kubernetes.io/part-of: memberlist \ No newline at end of file diff --git a/examples/distributed-loki/3-services/querier-service.yaml b/examples/distributed-loki/3-services/querier-service.yaml new file mode 100644 index 0000000..b2fc0e3 --- /dev/null +++ b/examples/distributed-loki/3-services/querier-service.yaml @@ -0,0 +1,62 @@ +kind: Service +apiVersion: v1 +metadata: + name: loki-distributed-querier + labels: + app.kubernetes.io/component: querier + app.kubernetes.io/instance: loki + app.kubernetes.io/name: loki-distributed + app.kubernetes.io/version: 2.5.0 +spec: + ipFamilies: + - IPv4 + ports: + - name: http + protocol: TCP + port: 3100 + targetPort: http + - name: grpc + protocol: TCP + port: 9095 + targetPort: grpc + internalTrafficPolicy: Cluster + type: ClusterIP + ipFamilyPolicy: SingleStack + sessionAffinity: None + selector: + app.kubernetes.io/component: querier + app.kubernetes.io/instance: loki + app.kubernetes.io/name: loki-distributed +--- +kind: Service +apiVersion: v1 +metadata: + name: loki-distributed-querier-headless + labels: + app.kubernetes.io/component: querier + app.kubernetes.io/instance: loki + app.kubernetes.io/name: loki-distributed + prometheus.io/service-monitor: 'false' +spec: + clusterIP: None + ipFamilies: + - IPv4 + ports: + - name: http + protocol: TCP + port: 3100 + targetPort: http + - name: grpc + protocol: TCP + port: 9095 + targetPort: grpc + internalTrafficPolicy: Cluster + clusterIPs: + - None + type: ClusterIP + ipFamilyPolicy: SingleStack + sessionAffinity: None + selector: + app.kubernetes.io/component: querier + app.kubernetes.io/instance: loki + app.kubernetes.io/name: loki-distributed diff --git a/examples/distributed-loki/3-services/query-frontend-service.yaml b/examples/distributed-loki/3-services/query-frontend-service.yaml new file mode 100644 index 0000000..acd1bb7 --- /dev/null +++ b/examples/distributed-loki/3-services/query-frontend-service.yaml @@ -0,0 +1,36 @@ +kind: Service +apiVersion: v1 +metadata: + name: loki-distributed-query-frontend + labels: + app.kubernetes.io/component: query-frontend + app.kubernetes.io/instance: loki + app.kubernetes.io/name: loki-distributed + app.kubernetes.io/version: 2.5.0 +spec: + clusterIP: None + ipFamilies: + - IPv4 + ports: + - name: http + protocol: TCP + port: 3100 + targetPort: http + - name: grpc + protocol: TCP + port: 9095 + targetPort: grpc + - name: grpclb + protocol: TCP + port: 9096 + targetPort: grpc + internalTrafficPolicy: Cluster + clusterIPs: + - None + type: ClusterIP + ipFamilyPolicy: SingleStack + sessionAffinity: None + selector: + app.kubernetes.io/component: query-frontend + app.kubernetes.io/instance: loki + app.kubernetes.io/name: loki-distributed \ No newline at end of file diff --git a/examples/zero-click-loki/2-loki.yaml b/examples/zero-click-loki/2-loki.yaml index 1db523e..6ad418d 100644 --- a/examples/zero-click-loki/2-loki.yaml +++ b/examples/zero-click-loki/2-loki.yaml @@ -8,6 +8,10 @@ data: server: http_listen_port: 3100 grpc_listen_port: 9096 + http_server_read_timeout: 1m + http_server_write_timeout: 1m + log_level: error + target: all common: path_prefix: /loki-store storage: @@ -19,9 +23,27 @@ data: instance_addr: 127.0.0.1 kvstore: store: inmemory + compactor: + compaction_interval: 5m + frontend: + compress_responses: true + ingester: + chunk_encoding: snappy + chunk_retain_period: 1m + query_range: + align_queries_with_step: true + cache_results: true + max_retries: 5 + results_cache: + cache: + enable_fifocache: true + fifocache: + max_size_bytes: 500MB + validity: 24h + parallelise_shardable_queries: true schema_config: configs: - - from: 2020-10-24 + - from: 2022-01-01 store: boltdb-shipper object_store: filesystem schema: v11 @@ -35,6 +57,39 @@ data: active_index_directory: /loki-store/index shared_store: filesystem cache_location: /loki-store/boltdb-cache + cache_ttl: 24h + limits_config: + ingestion_rate_strategy: global + ingestion_rate_mb: 4 + ingestion_burst_size_mb: 6 + max_label_name_length: 1024 + max_label_value_length: 2048 + max_label_names_per_series: 30 + reject_old_samples: true + reject_old_samples_max_age: 15m + creation_grace_period: 10m + enforce_metric_name: false + max_line_size: 256000 + max_line_size_truncate: false + max_entries_limit_per_query: 10000 + max_streams_per_user: 0 + max_global_streams_per_user: 0 + unordered_writes: true + max_chunks_per_query: 2000000 + max_query_length: 721h + max_query_parallelism: 32 + max_query_series: 10000 + cardinality_limit: 100000 + max_streams_matchers_per_query: 1000 + max_concurrent_tail_requests: 10 + retention_period: 24h + max_cache_freshness_per_query: 5m + max_queriers_per_tenant: 0 + per_stream_rate_limit: 3MB + per_stream_rate_limit_burst: 15MB + max_query_lookback: 0 + min_sharding_lookback: 0s + split_queries_by_interval: 1m --- apiVersion: v1 kind: Pod @@ -56,7 +111,7 @@ spec: name: loki-config containers: - name: loki - image: grafana/loki:2.4.1 + image: grafana/loki:2.5.0 volumeMounts: - mountPath: "/loki-store" name: loki-store diff --git a/loki_distributed.md b/loki_distributed.md new file mode 100644 index 0000000..d8e1749 --- /dev/null +++ b/loki_distributed.md @@ -0,0 +1,82 @@ +# Scalable loki deployment + +This document aims to design the deployment of a scalable version of Loki for testing purposes. This is based on [microservice mode](https://grafana.com/docs/loki/next/fundamentals/architecture/deployment-modes/#microservices-mode) + +This requires [external storage to save chunks](https://grafana.com/docs/loki/latest/storage/#implementations---chunks). For a deployment without external storage, check [loki simple doc](./loki_simple.md). + +## Folders content + +You can go to the next sections to understand the configuration of this stack or directly go to [deploy section](##Deploy). + +### 1 - Prerequisites + +To run this stack you will need to configure: +- a [service account](./examples/distributed-loki/1-prerequisites/service-account.yaml) used for deployments and statefulSets +- a [configMap](./examples/distributed-loki/1-prerequisites/config.yaml) containing loki configuration used in each component + +### 2 - Components + +The following components will be generated: +- Deployments: + - [distributor](./examples/distributed-loki/2-components/distributor-deployment.yaml) + - [query front end](./examples/distributed-loki/2-components/query-frontend-deployment.yaml) +- StatefulSets: + - [ingester](./examples/distributed-loki/2-components/ingester-statefulset.yaml) + - [querier](./examples/distributed-loki/2-components/querier-statefulset.yaml) +These statefulSets uses `volumeClaimTemplates` that create a `PersistentVolumeClaim` for each pod. + +By default, you will get 2 pods of each component. You can change this number updating the `replicas` section of each yaml file independently. + +Check [official architecture documentation](https://grafana.com/docs/loki/next/fundamentals/architecture/components/) for more details. +![loki architecture representation](https://grafana.com/docs/loki/next/fundamentals/architecture/loki_architecture_components.svg) + +### 3 - Services + +Related services will be created for each components. +Note that `querier` and `ingester` has alternative services with `-headless` suffix. These services uses `clusterIP: None` parameter. No load balancing will be done using these since client usually use first IP returned by the DNS record. This can be usefull to list available IPs for each component. + +An additionnal service called `loki-distributed-memberlist` will match all the components and allow discoverability between them. + +## Storage configuration +Edit [loki ConfigMap](./examples/distributed-loki/1-prerequisites/config.yaml) and replace `$(LOKI_STORE_NAME)` and `$(LOKI_STORE)` with your proper external storage configuration. + +Example using `s3` bucket called `loki` in `us-east-1` region: +```yaml + config.yaml: | + ... + common: + storage: + s3: + s3: https://s3.us-east-1.amazonaws.com + bucketnames: loki + region: us-east-1 + access_key_id: XXXXXXXXXXXXXXXXXXXX + secret_access_key: YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY + s3forcepathstyle: true + ... + schema_config: + configs: + - ... + object_store: s3 +``` + +Check [official examples](https://grafana.com/docs/loki/latest/storage/#examples). + +/!\ Without external storage, components will not share the chunks and querier will crash as soon as cache is cleared /!\ +## Deploy + +Ensure [storage configuration](#storage-configuration) is up to date. +Run the following command to apply all yamls at once in `network-observability` namespace: +`kubectl apply -f ./examples/distributed-loki/ -n network-observability --recursive` + +This will create all the components and services described above. + +## Network Observability Operator + +To connect NOO with this config, you will have to update the `loki` section with the following urls assuming you used `network-observability` namespace: + +```yaml + loki: + url: 'http://loki-distributed-distributor.network-observability.svc.cluster.local:3100/' + querierUrl: 'http://loki-distributed-query-frontend.network-observability.svc.cluster.local:3100/' +``` \ No newline at end of file