From 7b4caa1ad53547f9e66a0ad7aec67eebaec43fe3 Mon Sep 17 00:00:00 2001 From: Edward Welch Date: Sat, 6 Apr 2024 20:57:48 +0000 Subject: [PATCH] force a schema be provided. fix the nginx gateway paths Signed-off-by: Edward Welch --- .../loki/ci/default-single-binary-values.yaml | 1 + production/helm/loki/ci/default-values.yaml | 1 + .../helm/loki/ci/distributed-values.yaml | 2 +- .../loki/ci/legacy-monitoring-values.yaml | 1 + production/helm/loki/templates/_helpers.tpl | 41 ++++++++++++------- production/helm/loki/templates/validate.yaml | 9 ++++ production/helm/loki/values.yaml | 25 +++++++---- 7 files changed, 55 insertions(+), 25 deletions(-) diff --git a/production/helm/loki/ci/default-single-binary-values.yaml b/production/helm/loki/ci/default-single-binary-values.yaml index f54c0c139aa6..0eaff13de2ab 100644 --- a/production/helm/loki/ci/default-single-binary-values.yaml +++ b/production/helm/loki/ci/default-single-binary-values.yaml @@ -4,6 +4,7 @@ loki: replication_factor: 1 image: tag: "main-5e53303" + useTestSchema: true deploymentMode: SingleBinary singleBinary: replicas: 1 diff --git a/production/helm/loki/ci/default-values.yaml b/production/helm/loki/ci/default-values.yaml index 25675a503cc9..5b482a3aed15 100644 --- a/production/helm/loki/ci/default-values.yaml +++ b/production/helm/loki/ci/default-values.yaml @@ -4,6 +4,7 @@ loki: replication_factor: 1 image: tag: "main-5e53303" + useTestSchema: true read: replicas: 1 write: diff --git a/production/helm/loki/ci/distributed-values.yaml b/production/helm/loki/ci/distributed-values.yaml index 93b75b46d047..24e38e748435 100644 --- a/production/helm/loki/ci/distributed-values.yaml +++ b/production/helm/loki/ci/distributed-values.yaml @@ -4,7 +4,7 @@ loki: replication_factor: 1 image: tag: "k195-51c54ad" - useBloomFilters: false + useTestSchema: true deploymentMode: Distributed backend: replicas: 0 diff --git a/production/helm/loki/ci/legacy-monitoring-values.yaml b/production/helm/loki/ci/legacy-monitoring-values.yaml index d39c3b3ecd90..b28ad756a9e3 100644 --- a/production/helm/loki/ci/legacy-monitoring-values.yaml +++ b/production/helm/loki/ci/legacy-monitoring-values.yaml @@ -4,6 +4,7 @@ loki: replication_factor: 1 image: tag: "main-5e53303" + useTestSchema: true read: replicas: 1 write: diff --git a/production/helm/loki/templates/_helpers.tpl b/production/helm/loki/templates/_helpers.tpl index 69aacf777351..d07267ca8145 100644 --- a/production/helm/loki/templates/_helpers.tpl +++ b/production/helm/loki/templates/_helpers.tpl @@ -724,7 +724,8 @@ http { } ######################################################## - # simple-scalable mode hosts and urls definitions. + # Configure backend targets + {{- $backendHost := include "loki.backendFullname" .}} {{- $readHost := include "loki.readFullname" .}} {{- $writeHost := include "loki.writeFullname" .}} @@ -733,12 +734,6 @@ http { {{- $backendHost = include "loki.readFullname" . }} {{- end }} - {{- if gt (int .Values.singleBinary.replicas) 0 }} - {{- $backendHost = include "loki.singleBinaryFullname" . }} - {{- $readHost = include "loki.singleBinaryFullname" .}} - {{- $writeHost = include "loki.singleBinaryFullname" .}} - {{- end }} - {{- $httpSchema := .Values.gateway.nginxConfig.schema }} {{- $writeUrl := printf "%s://%s.%s.svc.%s:3100" $httpSchema $writeHost .Release.Namespace .Values.global.clusterDomain }} @@ -755,26 +750,42 @@ http { {{- $backendUrl = .Values.gateway.nginxConfig.customBackendUrl }} {{- end }} - ######################################################### - # distributed mode hosts and urls definitions. + {{- $singleBinaryHost := include "loki.singleBinaryFullname" . }} + {{- $singleBinaryUrl := printf "%s://%s.%s.svc.%s:3100" $httpSchema $singleBinaryHost .Release.Namespace .Values.global.clusterDomain }} + {{- $distributorHost := include "loki.distributorFullname" .}} {{- $ingesterHost := include "loki.ingesterFullname" .}} {{- $queryFrontendHost := include "loki.queryFrontendFullname" .}} {{- $indexGatewayHost := include "loki.indexGatewayFullname" .}} {{- $rulerHost := include "loki.rulerFullname" .}} + {{- $compactorHost := include "loki.compactorFullname" .}} + {{- $schedulerHost := include "loki.querySchedulerFullname" .}} + {{- $distributorUrl := printf "%s://%s.%s.svc.%s:3100" $httpSchema $distributorHost .Release.Namespace .Values.global.clusterDomain -}} {{- $ingesterUrl := printf "%s://%s.%s.svc.%s:3100" $httpSchema $ingesterHost .Release.Namespace .Values.global.clusterDomain }} {{- $queryFrontendUrl := printf "%s://%s.%s.svc.%s:3100" $httpSchema $queryFrontendHost .Release.Namespace .Values.global.clusterDomain }} {{- $indexGatewayUrl := printf "%s://%s.%s.svc.%s:3100" $httpSchema $indexGatewayHost .Release.Namespace .Values.global.clusterDomain }} {{- $rulerUrl := printf "%s://%s.%s.svc.%s:3100" $httpSchema $rulerHost .Release.Namespace .Values.global.clusterDomain }} - - {{- if not "loki.deployment.isDistributed "}} + {{- $compactorUrl := printf "%s://%s.%s.svc.%s:3100" $httpSchema $compactorHost .Release.Namespace .Values.global.clusterDomain }} + {{- $schedulerUrl := printf "%s://%s.%s.svc.%s:3100" $httpSchema $schedulerHost .Release.Namespace .Values.global.clusterDomain }} + + {{- if eq (include "loki.deployment.isSingleBinary" .) "true"}} + {{- $distributorUrl = $singleBinaryUrl }} + {{- $ingesterUrl = $singleBinaryUrl }} + {{- $queryFrontendUrl = $singleBinaryUrl }} + {{- $indexGatewayUrl = $singleBinaryUrl }} + {{- $rulerUrl = $singleBinaryUrl }} + {{- $compactorUrl = $singleBinaryUrl }} + {{- $schedulerUrl = $singleBinaryUrl }} + {{- else if eq (include "loki.deployment.isScalable" .) "true"}} {{- $distributorUrl = $writeUrl }} {{- $ingesterUrl = $writeUrl }} {{- $queryFrontendUrl = $readUrl }} {{- $indexGatewayUrl = $backendUrl }} {{- $rulerUrl = $backendUrl }} + {{- $compactorUrl = $backendUrl }} + {{- $schedulerUrl = $backendUrl }} {{- end -}} # Distributor @@ -834,13 +845,13 @@ http { # Compactor location = /compactor/ring { - proxy_pass {{ $backendUrl }}$request_uri; + proxy_pass {{ $compactorUrl }}$request_uri; } location = /loki/api/v1/delete { - proxy_pass {{ $backendUrl }}$request_uri; + proxy_pass {{ $compactorUrl }}$request_uri; } location = /loki/api/v1/cache/generation_numbers { - proxy_pass {{ $backendUrl }}$request_uri; + proxy_pass {{ $compactorUrl }}$request_uri; } # IndexGateway @@ -850,7 +861,7 @@ http { # QueryScheduler location = /scheduler/ring { - proxy_pass {{ $backendUrl }}$request_uri; + proxy_pass {{ $schedulerUrl }}$request_uri; } # Config diff --git a/production/helm/loki/templates/validate.yaml b/production/helm/loki/templates/validate.yaml index fa1938316b60..93e2490636b4 100644 --- a/production/helm/loki/templates/validate.yaml +++ b/production/helm/loki/templates/validate.yaml @@ -30,3 +30,12 @@ {{- if and (gt $singleBinaryReplicas 0) $atLeastOneScalableReplica (ne .Values.deploymentMode "SingleBinary<->SimpleScalable") }} {{- fail "You have more than zero replicas configured for both the single binary and simple scalable targets. If this was intentional change the deploymentMode to the transitional 'SingleBinary<->SimpleScalable' mode"}} {{- end }} + +{{- if and (or (not (empty .Values.loki.schemaConfig)) (not (empty .Values.loki.structuredConfig.schema_config))) .Values.loki.useTestSchema }} +{{- fail "loki.useTestSchema must be false if loki.schemaConfig or loki.structuredConfig.schema_config are defined."}} +{{- end }} + + +{{- if and (empty .Values.loki.schemaConfig) (empty .Values.loki.structuredConfig.schema_config) (not .Values.loki.useTestSchema) }} +{{- fail "You must provide a schema_config for Loki, one is not provided as this will be individual for every Loki cluster. See https://grafana.com/docs/loki/latest/operations/storage/schema/ for schema information. For quick testing (with no persistence) add `--set loki.useTestSchema=true`"}} +{{- end }} \ No newline at end of file diff --git a/production/helm/loki/values.yaml b/production/helm/loki/values.yaml index 2615faf11012..65e6d7450ac2 100644 --- a/production/helm/loki/values.yaml +++ b/production/helm/loki/values.yaml @@ -192,16 +192,11 @@ loki: {{- if .Values.loki.schemaConfig }} schema_config: {{- toYaml .Values.loki.schemaConfig | nindent 2}} - {{- else }} + {{- end }} + + {{- if .Values.loki.useTestSchema }} schema_config: - configs: - - from: 2022-01-11 - store: boltdb-shipper - object_store: {{ .Values.loki.storage.type }} - schema: v12 - index: - prefix: loki_index_ - period: 24h + {{- toYaml .Values.loki.testSchemaConfig | nindent 2}} {{- end }} {{ include "loki.rulerConfig" . }} @@ -372,6 +367,18 @@ loki: default_validity: "12h" # -- Check https://grafana.com/docs/loki/latest/configuration/#schema_config for more info on how to configure schemas schemaConfig: {} + # -- a real Loki install requires a proper schemaConfig defined above this, however for testing or playing around + # you can enable useTestSchema + useTestSchema: false + testSchemaConfig: + configs: + - from: 2024-04-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h # -- Check https://grafana.com/docs/loki/latest/configuration/#ruler for more info on configuring ruler rulerConfig: {} # -- Structured loki configuration, takes precedence over `loki.config`, `loki.schemaConfig`, `loki.storageConfig`