Skip to content

Commit

Permalink
feat: Add pattern ingester support in SSD mode (#12685)
Browse files Browse the repository at this point in the history
  • Loading branch information
@cyriltovena
cyriltovena committed Apr 19, 2024
1 parent 4e04d07 commit 19bfef4
Show file tree
Hide file tree
Showing 6 changed files with 28 additions and 14 deletions.
16 changes: 15 additions & 1 deletion docs/sources/setup/install/helm/reference.md
View file
Expand Up @@ -5211,7 +5211,7 @@ null
"replication_factor": 3
},
"compactor": {},
"config": "{{- if .Values.enterprise.enabled}}\n{{- tpl .Values.enterprise.config . }}\n{{- else }}\nauth_enabled: {{ .Values.loki.auth_enabled }}\n{{- end }}\n\n{{- with .Values.loki.server }}\nserver:\n {{- toYaml . | nindent 2}}\n{{- end}}\n\nmemberlist:\n{{- if .Values.loki.memberlistConfig }}\n {{- toYaml .Values.loki.memberlistConfig | nindent 2 }}\n{{- else }}\n{{- if .Values.loki.extraMemberlistConfig}}\n{{- toYaml .Values.loki.extraMemberlistConfig | nindent 2}}\n{{- end }}\n join_members:\n - {{ include \"loki.memberlist\" . }}\n {{- with .Values.migrate.fromDistributed }}\n {{- if .enabled }}\n - {{ .memberlistService }}\n {{- end }}\n {{- end }}\n{{- end }}\n\n{{- with .Values.loki.ingester }}\ningester:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- if .Values.loki.commonConfig}}\ncommon:\n{{- toYaml .Values.loki.commonConfig | nindent 2}}\n storage:\n {{- include \"loki.commonStorageConfig\" . | nindent 4}}\n{{- end}}\n\n{{- with .Values.loki.limits_config }}\nlimits_config:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\nruntime_config:\n file: /etc/loki/runtime-config/runtime-config.yaml\n\n{{- with .Values.chunksCache }}\n{{- if .enabled }}\nchunk_store_config:\n chunk_cache_config:\n default_validity: {{ .defaultValidity }}\n background:\n writeback_goroutines: {{ .writebackParallelism }}\n writeback_buffer: {{ .writebackBuffer }}\n writeback_size_limit: {{ .writebackSizeLimit }}\n memcached:\n batch_size: {{ .batchSize }}\n parallelism: {{ .parallelism }}\n memcached_client:\n addresses: dnssrvnoa+_memcached-client._tcp.{{ template \"loki.fullname\" $ }}-chunks-cache.{{ $.Release.Namespace }}.svc\n consistent_hash: true\n timeout: {{ .timeout }}\n max_idle_conns: 72\n{{- end }}\n{{- end }}\n\n{{- if .Values.loki.schemaConfig }}\nschema_config:\n{{- toYaml .Values.loki.schemaConfig | nindent 2}}\n{{- end }}\n\n{{- if .Values.loki.useTestSchema }}\nschema_config:\n{{- toYaml .Values.loki.testSchemaConfig | nindent 2}}\n{{- end }}\n\n{{ include \"loki.rulerConfig\" . }}\n\n{{- if or .Values.tableManager.retention_deletes_enabled .Values.tableManager.retention_period }}\ntable_manager:\n retention_deletes_enabled: {{ .Values.tableManager.retention_deletes_enabled }}\n retention_period: {{ .Values.tableManager.retention_period }}\n{{- end }}\n\nquery_range:\n align_queries_with_step: true\n {{- with .Values.loki.query_range }}\n {{- tpl (. | toYaml) $ | nindent 4 }}\n {{- end }}\n {{- if .Values.resultsCache.enabled }}\n {{- with .Values.resultsCache }}\n cache_results: true\n results_cache:\n cache:\n default_validity: {{ .defaultValidity }}\n background:\n writeback_goroutines: {{ .writebackParallelism }}\n writeback_buffer: {{ .writebackBuffer }}\n writeback_size_limit: {{ .writebackSizeLimit }}\n memcached_client:\n consistent_hash: true\n addresses: dnssrvnoa+_memcached-client._tcp.{{ template \"loki.fullname\" $ }}-results-cache.{{ $.Release.Namespace }}.svc\n timeout: {{ .timeout }}\n update_interval: 1m\n {{- end }}\n {{- end }}\n\n{{- with .Values.loki.storage_config }}\nstorage_config:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.query_scheduler }}\nquery_scheduler:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.compactor }}\ncompactor:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.analytics }}\nanalytics:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.querier }}\nquerier:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.index_gateway }}\nindex_gateway:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.frontend }}\nfrontend:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.frontend_worker }}\nfrontend_worker:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.distributor }}\ndistributor:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\ntracing:\n enabled: {{ .Values.loki.tracing.enabled }}\n",
"config": "{{- if .Values.enterprise.enabled}}\n{{- tpl .Values.enterprise.config . }}\n{{- else }}\nauth_enabled: {{ .Values.loki.auth_enabled }}\n{{- end }}\n\n{{- with .Values.loki.server }}\nserver:\n {{- toYaml . | nindent 2}}\n{{- end}}\n\npattern_ingester:\n enabled: {{ .Values.loki.pattern_ingester.enabled }}\n\nmemberlist:\n{{- if .Values.loki.memberlistConfig }}\n {{- toYaml .Values.loki.memberlistConfig | nindent 2 }}\n{{- else }}\n{{- if .Values.loki.extraMemberlistConfig}}\n{{- toYaml .Values.loki.extraMemberlistConfig | nindent 2}}\n{{- end }}\n join_members:\n - {{ include \"loki.memberlist\" . }}\n {{- with .Values.migrate.fromDistributed }}\n {{- if .enabled }}\n - {{ .memberlistService }}\n {{- end }}\n {{- end }}\n{{- end }}\n\n{{- with .Values.loki.ingester }}\ningester:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- if .Values.loki.commonConfig}}\ncommon:\n{{- toYaml .Values.loki.commonConfig | nindent 2}}\n storage:\n {{- include \"loki.commonStorageConfig\" . | nindent 4}}\n{{- end}}\n\n{{- with .Values.loki.limits_config }}\nlimits_config:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\nruntime_config:\n file: /etc/loki/runtime-config/runtime-config.yaml\n\n{{- with .Values.chunksCache }}\n{{- if .enabled }}\nchunk_store_config:\n chunk_cache_config:\n default_validity: {{ .defaultValidity }}\n background:\n writeback_goroutines: {{ .writebackParallelism }}\n writeback_buffer: {{ .writebackBuffer }}\n writeback_size_limit: {{ .writebackSizeLimit }}\n memcached:\n batch_size: {{ .batchSize }}\n parallelism: {{ .parallelism }}\n memcached_client:\n addresses: dnssrvnoa+_memcached-client._tcp.{{ template \"loki.fullname\" $ }}-chunks-cache.{{ $.Release.Namespace }}.svc\n consistent_hash: true\n timeout: {{ .timeout }}\n max_idle_conns: 72\n{{- end }}\n{{- end }}\n\n{{- if .Values.loki.schemaConfig }}\nschema_config:\n{{- toYaml .Values.loki.schemaConfig | nindent 2}}\n{{- end }}\n\n{{- if .Values.loki.useTestSchema }}\nschema_config:\n{{- toYaml .Values.loki.testSchemaConfig | nindent 2}}\n{{- end }}\n\n{{ include \"loki.rulerConfig\" . }}\n\n{{- if or .Values.tableManager.retention_deletes_enabled .Values.tableManager.retention_period }}\ntable_manager:\n retention_deletes_enabled: {{ .Values.tableManager.retention_deletes_enabled }}\n retention_period: {{ .Values.tableManager.retention_period }}\n{{- end }}\n\nquery_range:\n align_queries_with_step: true\n {{- with .Values.loki.query_range }}\n {{- tpl (. | toYaml) $ | nindent 4 }}\n {{- end }}\n {{- if .Values.resultsCache.enabled }}\n {{- with .Values.resultsCache }}\n cache_results: true\n results_cache:\n cache:\n default_validity: {{ .defaultValidity }}\n background:\n writeback_goroutines: {{ .writebackParallelism }}\n writeback_buffer: {{ .writebackBuffer }}\n writeback_size_limit: {{ .writebackSizeLimit }}\n memcached_client:\n consistent_hash: true\n addresses: dnssrvnoa+_memcached-client._tcp.{{ template \"loki.fullname\" $ }}-results-cache.{{ $.Release.Namespace }}.svc\n timeout: {{ .timeout }}\n update_interval: 1m\n {{- end }}\n {{- end }}\n\n{{- with .Values.loki.storage_config }}\nstorage_config:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.query_scheduler }}\nquery_scheduler:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.compactor }}\ncompactor:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.analytics }}\nanalytics:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.querier }}\nquerier:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.index_gateway }}\nindex_gateway:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.frontend }}\nfrontend:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.frontend_worker }}\nfrontend_worker:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.distributor }}\ndistributor:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\ntracing:\n enabled: {{ .Values.loki.tracing.enabled }}\n",
"configObjectName": "{{ include \"loki.name\" . }}",
"configStorageType": "ConfigMap",
"containerSecurityContext": {
Expand Down Expand Up @@ -5269,6 +5269,9 @@ null
"timeout": "500ms"
}
},
"pattern_ingester": {
"enabled": false
},
"podAnnotations": {},
"podLabels": {},
"podSecurityContext": {
Expand Down Expand Up @@ -5627,6 +5630,17 @@ null
}
}
</pre>
</td>
</tr>
<tr>
<td>loki.pattern_ingester</td>
<td>object</td>
<td>Optional pattern ingester configuration</td>
<td><pre lang="json">
{
"enabled": false
}
</pre>
</td>
</tr>
<tr>
Expand Down
2 changes: 1 addition & 1 deletion pkg/loki/loki.go
View file
Expand Up @@ -695,7 +695,7 @@ func (t *Loki) setupModuleManager() error {
MemberlistKV: {Server},

Read: {QueryFrontend, Querier},
Write: {Ingester, Distributor},
Write: {Ingester, Distributor, PatternIngester},
Backend: {QueryScheduler, Ruler, Compactor, IndexGateway, BloomGateway, BloomCompactor},

All: {QueryScheduler, QueryFrontend, Querier, Ingester, PatternIngester, Distributor, Ruler, Compactor},
Expand Down
10 changes: 4 additions & 6 deletions production/helm/loki/CHANGELOG.md
View file
Expand Up @@ -13,6 +13,10 @@ Entries should include a reference to the pull request that introduced the chang

[//]: # (<AUTOMATED_UPDATES_LOCATOR> : do not remove this line. This locator is used by the CI pipeline to automatically create a changelog entry for each new Loki release. Add other chart versions and respective changelog entries bellow this line.)

## 6.2.2

- [FEATURE] Add support for enabling pattern ingester config via `loki.pattern_ingester.enabled`.

## 6.2.1

- [BUGFIX] Removed duplicate bucketNames from documentation and fixed key name `deploymentMode`
Expand Down Expand Up @@ -44,7 +48,6 @@ Entries should include a reference to the pull request that introduced the chang

- [CHANGE] Changed version of Loki to 2.9.6


## 5.46.0

- [CHANGE] Changed version of Loki to 2.9.5
Expand Down Expand Up @@ -202,7 +205,6 @@ Entries should include a reference to the pull request that introduced the chang

- [CHANGE] Changed version of Grafana Enterprise Logs to v1.8.4


## 5.32.0

- [CHANGE] Grafana dashboards are no longer created solely in scalable mode and with external cloud storage enabled.
Expand Down Expand Up @@ -454,12 +456,10 @@ Entries should include a reference to the pull request that introduced the chang

- [CHANGE] Changed version of Grafana Enterprise Logs to v1.7.2


## 5.4.0

- [CHANGE] Changed version of Loki to 2.8.2


- [CHANGE] Change default GEL and Loki versions to 1.7.1 and 2.8.1 respectively
- [BUGFIX] Fix dns port in network-policy

Expand All @@ -469,12 +469,10 @@ Entries should include a reference to the pull request that introduced the chang

- [BUGFIX] Add projected volume type to psp


## 4.9.0

- [CHANGE] Changed version of Loki to 2.7.5


- [BUGFIX] Fix role/PSP mapping

- [BUGFIX] Fix service/ingress mapping
Expand Down
2 changes: 1 addition & 1 deletion production/helm/loki/Chart.yaml
View file
Expand Up @@ -3,7 +3,7 @@ name: loki
description: Helm chart for Grafana Loki in simple, scalable mode
type: application
appVersion: 3.0.0
version: 6.2.1
version: 6.2.2
home: https://grafana.github.io/helm-charts
sources:
- https://github.com/grafana/loki
Expand Down
2 changes: 1 addition & 1 deletion production/helm/loki/README.md
View file
@@ -1,6 +1,6 @@
# loki

![Version: 6.2.1](https://img.shields.io/badge/Version-6.2.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 3.0.0](https://img.shields.io/badge/AppVersion-3.0.0-informational?style=flat-square)
![Version: 6.2.2](https://img.shields.io/badge/Version-6.2.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 3.0.0](https://img.shields.io/badge/AppVersion-3.0.0-informational?style=flat-square)

Helm chart for Grafana Loki in simple, scalable mode

Expand Down
10 changes: 6 additions & 4 deletions production/helm/loki/values.yaml
View file
Expand Up @@ -122,7 +122,6 @@ loki:
# The value will be passed through tpl.
generatedConfigObjectName: '{{ include "loki.name" . }}'


# -- Config file contents for Loki
# @default -- See values.yaml
config: |
Expand All @@ -137,6 +136,9 @@ loki:
{{- toYaml . | nindent 2}}
{{- end}}
pattern_ingester:
enabled: {{ .Values.loki.pattern_ingester.enabled }}
memberlist:
{{- if .Values.loki.memberlistConfig }}
{{- toYaml .Values.loki.memberlistConfig | nindent 2 }}
Expand Down Expand Up @@ -415,6 +417,9 @@ loki:
up_to: 3
# -- Optional compactor configuration
compactor: {}
# -- Optional pattern ingester configuration
pattern_ingester:
enabled: false
# -- Optional analytics configuration
analytics: {}
# -- Optional querier configuration
Expand Down Expand Up @@ -851,7 +856,6 @@ adminApi:
# -- Grace period to allow the admin-api to shutdown before it is killed
terminationGracePeriodSeconds: 60


######################################################################################################################
#
# Gateway and Ingress
Expand Down Expand Up @@ -1175,7 +1179,6 @@ ingress:
# - loki.example.com
# secretName: loki-distributed-tls


######################################################################################################################
#
# Migration
Expand All @@ -1192,7 +1195,6 @@ migrate:
# memberlist service DNS so the new deployment can join its ring.
memberlistService: ""


######################################################################################################################
#
# Single Binary Deployment
Expand Down

0 comments on commit 19bfef4

Please sign in to comment.