added VMCluster (VictoriaMetrics#178)

* update dependencies * initial cluster * update VM versions and fix images * update endpoint generation to support cluster * added cluster dashboard * update dashboards from upsource * update rules and runbook configuration * remove dublicate * update helm docs
krakazyabra · Jul 27, 2021 · abb802b · abb802b
1 parent 16e75d3
commit abb802b
Show file tree

Hide file tree

Showing 43 changed files with 7,178 additions and 4,960 deletions.
diff --git a/charts/victoria-metrics-k8s-stack/.gitignore b/charts/victoria-metrics-k8s-stack/.gitignore
@@ -0,0 +1 @@
+out/
diff --git a/charts/victoria-metrics-k8s-stack/Chart.lock b/charts/victoria-metrics-k8s-stack/Chart.lock
@@ -4,12 +4,12 @@ dependencies:
   version: 0.1.17
 - name: kube-state-metrics
   repository: https://prometheus-community.github.io/helm-charts
-  version: 3.2.2
+  version: 3.4.1
 - name: prometheus-node-exporter
   repository: https://prometheus-community.github.io/helm-charts
-  version: 1.18.2
+  version: 2.0.2
 - name: grafana
   repository: https://grafana.github.io/helm-charts
-  version: 6.12.1
-digest: sha256:58ad75ef412eed7eff3fbfc4d721c33d34ca1af838f60d7297de2388dc2d8b26
-generated: "2021-07-12T20:06:41.848769241Z"
+  version: 6.14.1
+digest: sha256:ef56bd6d0c02f87ffbf5f3ae2debf4a8d6a914c1cd46a999940ce1d62354e039
+generated: "2021-07-27T19:55:13.172435+08:00"
diff --git a/charts/victoria-metrics-k8s-stack/Chart.yaml b/charts/victoria-metrics-k8s-stack/Chart.yaml
@@ -2,22 +2,23 @@ apiVersion: v2
 name: victoria-metrics-k8s-stack
 description: Kubernetes monitoring on VictoriaMetrics stack. Includes VictoriaMetrics Operator, Grafana dashboards, ServiceScrapes and VMRules
 type: application
-version: 0.2.9
-appVersion: "1.16.0"
+version: 0.3.0
+appVersion: "1.63.0"
 
 dependencies:
-- name: victoria-metrics-operator
-  version: "0.1.*"
-  repository: https://victoriametrics.github.io/helm-charts
-- name: kube-state-metrics
-  version: "3.2.*"
-  repository: https://prometheus-community.github.io/helm-charts
-  condition: kube-state-metrics.enabled
-- name: prometheus-node-exporter
-  version: "1.18.*"
-  repository: https://prometheus-community.github.io/helm-charts
-  condition: prometheus-node-exporter.enabled
-- name: grafana
-  version: "6.12.*"
-  repository: https://grafana.github.io/helm-charts
-  condition: grafana.enabled
+  - name: victoria-metrics-operator
+    version: "0.1.*"
+    repository: https://victoriametrics.github.io/helm-charts
+    condition: operator.enabled
+  - name: kube-state-metrics
+    version: "3.4.*"
+    repository: https://prometheus-community.github.io/helm-charts
+    condition: kube-state-metrics.enabled
+  - name: prometheus-node-exporter
+    version: "2.0.*"
+    repository: https://prometheus-community.github.io/helm-charts
+    condition: prometheus-node-exporter.enabled
+  - name: grafana
+    version: "6.14.*"
+    repository: https://grafana.github.io/helm-charts
+    condition: grafana.enabled
diff --git a/charts/victoria-metrics-k8s-stack/README.md b/charts/victoria-metrics-k8s-stack/README.md
@@ -1,6 +1,6 @@
 # Helm Chart For Victoria Metrics kubernetes monitoring stack.
 
-![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)  ![Version: 0.2.9](https://img.shields.io/badge/Version-0.2.9-informational?style=flat-square)
+![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)  ![Version: 0.3.0](https://img.shields.io/badge/Version-0.3.0-informational?style=flat-square)
 
 Kubernetes monitoring on VictoriaMetrics stack. Includes VictoriaMetrics Operator, Grafana dashboards, ServiceScrapes and VMRules
 
@@ -252,6 +252,7 @@ Change the values according to the need of the environment in ``victoria-metrics
 | alertmanager.ingress.tls | list | `[]` |  |
 | alertmanager.monzoTemplate.enabled | bool | `true` |  |
 | alertmanager.spec.externalURL | string | `""` |  |
+| alertmanager.spec.image.tag | string | `"v0.22.2"` |  |
 | alertmanager.spec.routePrefix | string | `"/"` |  |
 | coreDns.enabled | bool | `true` |  |
 | coreDns.service.enabled | bool | `true` |  |
@@ -282,7 +283,7 @@ Change the values according to the need of the environment in ``victoria-metrics
 | defaultRules.rules.kubernetesSystem | bool | `true` |  |
 | defaultRules.rules.network | bool | `true` |  |
 | defaultRules.rules.node | bool | `true` |  |
-| defaultRules.runbookUrl | string | `"https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#"` |  |
+| defaultRules.runbookUrl | string | `"https://runbooks.prometheus-operator.dev/runbooks"` |  |
 | fullnameOverride | string | `""` |  |
 | grafana.additionalDataSources | list | `[]` |  |
 | grafana.dashboardProviders."dashboardproviders.yaml".apiVersion | int | `1` |  |
@@ -390,6 +391,7 @@ Change the values according to the need of the environment in ``victoria-metrics
 | operator.cleanupCRD | bool | `true` |  |
 | operator.cleanupSA.create | bool | `true` |  |
 | operator.cleanupSA.name | string | `""` |  |
+| operator.enabled | bool | `true` |  |
 | operator.kubectlImage.pullPolicy | string | `"IfNotPresent"` |  |
 | operator.kubectlImage.repository | string | `"gcr.io/google_containers/hyperkube"` |  |
 | operator.kubectlImage.tag | string | `"v1.16.0"` |  |
@@ -415,6 +417,7 @@ Change the values according to the need of the environment in ``victoria-metrics
 | vmagent.ingress.tls | list | `[]` |  |
 | vmagent.spec.externalLabels.cluster | string | `"cluster-name"` |  |
 | vmagent.spec.extraArgs."promscrape.streamParse" | string | `"true"` |  |
+| vmagent.spec.image.tag | string | `"v1.63.0"` |  |
 | vmagent.spec.scrapeInterval | string | `"25s"` |  |
 | vmalert.enabled | bool | `true` |  |
 | vmalert.ingress.annotations | object | `{}` |  |
@@ -426,6 +429,54 @@ Change the values according to the need of the environment in ``victoria-metrics
 | vmalert.ingress.pathType | string | `"Prefix"` |  |
 | vmalert.ingress.tls | list | `[]` |  |
 | vmalert.spec.evaluationInterval | string | `"15s"` |  |
+| vmalert.spec.image.tag | string | `"v1.63.0"` |  |
+| vmcluster.enabled | bool | `false` |  |
+| vmcluster.ingress.insert.annotations | object | `{}` |  |
+| vmcluster.ingress.insert.enabled | bool | `false` |  |
+| vmcluster.ingress.insert.extraPaths | list | `[]` |  |
+| vmcluster.ingress.insert.hosts[0] | string | `"vminsert.domain.com"` |  |
+| vmcluster.ingress.insert.labels | object | `{}` |  |
+| vmcluster.ingress.insert.path | string | `"/"` |  |
+| vmcluster.ingress.insert.pathType | string | `"Prefix"` |  |
+| vmcluster.ingress.insert.tls | list | `[]` |  |
+| vmcluster.ingress.select.annotations | object | `{}` |  |
+| vmcluster.ingress.select.enabled | bool | `false` |  |
+| vmcluster.ingress.select.extraPaths | list | `[]` |  |
+| vmcluster.ingress.select.hosts[0] | string | `"vmselect.domain.com"` |  |
+| vmcluster.ingress.select.labels | object | `{}` |  |
+| vmcluster.ingress.select.path | string | `"/"` |  |
+| vmcluster.ingress.select.pathType | string | `"Prefix"` |  |
+| vmcluster.ingress.select.tls | list | `[]` |  |
+| vmcluster.ingress.storage.annotations | object | `{}` |  |
+| vmcluster.ingress.storage.enabled | bool | `false` |  |
+| vmcluster.ingress.storage.extraPaths | list | `[]` |  |
+| vmcluster.ingress.storage.hosts[0] | string | `"vmstorage.domain.com"` |  |
+| vmcluster.ingress.storage.labels | object | `{}` |  |
+| vmcluster.ingress.storage.path | string | `"/"` |  |
+| vmcluster.ingress.storage.pathType | string | `"Prefix"` |  |
+| vmcluster.ingress.storage.tls | list | `[]` |  |
+| vmcluster.spec.replicationFactor | int | `2` |  |
+| vmcluster.spec.retentionPeriod | string | `"14"` |  |
+| vmcluster.spec.vminsert.image.tag | string | `"v1.63.0-cluster"` |  |
+| vmcluster.spec.vminsert.replicaCount | int | `2` |  |
+| vmcluster.spec.vminsert.resources.limits.cpu | string | `"1"` |  |
+| vmcluster.spec.vminsert.resources.limits.memory | string | `"1000Mi"` |  |
+| vmcluster.spec.vminsert.resources.requests.cpu | string | `"0.5"` |  |
+| vmcluster.spec.vminsert.resources.requests.memory | string | `"500Mi"` |  |
+| vmcluster.spec.vmselect.cacheMountPath | string | `"/select-cache"` |  |
+| vmcluster.spec.vmselect.image.tag | string | `"v1.63.0-cluster"` |  |
+| vmcluster.spec.vmselect.replicaCount | int | `2` |  |
+| vmcluster.spec.vmselect.resources.limits.cpu | string | `"1"` |  |
+| vmcluster.spec.vmselect.resources.limits.memory | string | `"1000Mi"` |  |
+| vmcluster.spec.vmselect.resources.requests.cpu | string | `"0.5"` |  |
+| vmcluster.spec.vmselect.resources.requests.memory | string | `"500Mi"` |  |
+| vmcluster.spec.vmselect.storage.volumeClaimTemplate.spec.resources.requests.storage | string | `"2Gi"` |  |
+| vmcluster.spec.vmstorage.image.tag | string | `"v1.63.0-cluster"` |  |
+| vmcluster.spec.vmstorage.replicaCount | int | `2` |  |
+| vmcluster.spec.vmstorage.resources.limits.cpu | string | `"1"` |  |
+| vmcluster.spec.vmstorage.resources.limits.memory | string | `"1500Mi"` |  |
+| vmcluster.spec.vmstorage.storage.volumeClaimTemplate.spec.resources.requests.storage | string | `"10Gi"` |  |
+| vmcluster.spec.vmstorage.storageDataPath | string | `"/vm-data"` |  |
 | vmsingle.enabled | bool | `true` |  |
 | vmsingle.ingress.annotations | object | `{}` |  |
 | vmsingle.ingress.enabled | bool | `false` |  |
@@ -435,6 +486,7 @@ Change the values according to the need of the environment in ``victoria-metrics
 | vmsingle.ingress.path | string | `"/"` |  |
 | vmsingle.ingress.pathType | string | `"Prefix"` |  |
 | vmsingle.ingress.tls | list | `[]` |  |
+| vmsingle.spec.image.tag | string | `"v1.63.0"` |  |
 | vmsingle.spec.replicaCount | int | `1` |  |
 | vmsingle.spec.retentionPeriod | string | `"14"` |  |
 | vmsingle.spec.storage.accessModes[0] | string | `"ReadWriteOnce"` |  |

diff --git a/charts/victoria-metrics-k8s-stack/hack/sync_grafana_dashboards.py b/charts/victoria-metrics-k8s-stack/hack/sync_grafana_dashboards.py
@@ -45,6 +45,11 @@ def new_representer(dumper, data):
         'destination': '../templates/grafana/dashboards',
         'type': 'json'
     },
+    {
+        'source': 'https://raw.githubusercontent.com/VictoriaMetrics/VictoriaMetrics/cluster/dashboards/clusterbytenant.json',
+        'destination': '../templates/grafana/dashboards',
+        'type': 'json'
+    },
 ]
 
 skip_list = [
@@ -63,6 +68,7 @@ def new_representer(dumper, data):
     'scheduler': ' .Values.kubeScheduler.enabled',
     'node-rsrc-use': ' (index .Values "prometheus-node-exporter" "enabled")',
     'node-cluster-rsrc-use': ' (index .Values "prometheus-node-exporter" "enabled")',
+    'clusterbytenant': '.Values.vmcluster.enabled'
 }
 
 # standard header

diff --git a/charts/victoria-metrics-k8s-stack/hack/sync_rules.py b/charts/victoria-metrics-k8s-stack/hack/sync_rules.py
@@ -95,12 +95,9 @@ def new_representer(dumper, data):
 }
 
 replacement_map = {
-    'https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#': {
+    'https://runbooks.prometheus-operator.dev/runbooks': {
         'replacement': '{{ .Values.defaultRules.runbookUrl }}',
         'init': ''},
-    'https://github.com/prometheus-operator/kube-prometheus/wiki/': {
-        'replacement': '{{ .Values.defaultRules.runbookUrl }}alert-name-',
-        'init': ''},
     'job="kube-state-metrics"': {
         'replacement': 'job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"',
         'limitGroup': ['kubernetes-apps'],

diff --git a/charts/victoria-metrics-k8s-stack/templates/_helpers.tpl b/charts/victoria-metrics-k8s-stack/templates/_helpers.tpl
@@ -82,57 +82,78 @@ app.kubernetes.io/name: {{ include "victoria-metrics-k8s-stack.name" . }}
 app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end }}
 
-{{/*
-VM endpoint
-*/}}
-{{- define "victoria-metrics-k8s-stack.vmEndpoint" -}}
+
+{{- define "victoria-metrics-k8s-stack.vmSelectEndpoint" -}}
 {{- if .Values.vmsingle.enabled -}}
-url: "http://{{ include "victoria-metrics-k8s-stack.vmsingleName" .}}.{{ .Release.Namespace }}.svc:{{ .Values.vmsingle.spec.port | default 8429 }}"
+{{ printf "http://%s.%s.svc:%d" (include "victoria-metrics-k8s-stack.vmsingleName" .) .Release.Namespace (.Values.vmsingle.spec.port | default 8429) }}
+{{- end }}
+{{- if .Values.vmcluster.enabled -}}
+{{ printf "http://%s-%s.%s.svc:%d/select/0/prometheus" "vmselect" (include "victoria-metrics-k8s-stack.fullname" .) .Release.Namespace (.Values.vmcluster.spec.vmselect.port | default 8481) }}
 {{- end }}
 {{- end }}
 
-
-{{/*
-Alermanager spec
-*/}}
-{{- define "victoria-metrics-k8s-stack.alertmanagerSpec" -}}
-{{ omit .Values.alertmanager.spec  "configMaps" "configSecret" | toYaml }}
-configSecret: {{ .Values.alertmanager.spec.configSecret | default (printf "%s-alertmanager" (include "victoria-metrics-k8s-stack.fullname" .)) }}
-{{- if or .Values.alertmanager.spec.configMaps .Values.alertmanager.monzoTemplate.enabled }}
-{{- $list := .Values.alertmanager.spec.configMaps | default (list "") }}
-{{- if .Values.alertmanager.monzoTemplate.enabled }}
-{{- $list = append $list (printf "%s-%s" (include "victoria-metrics-k8s-stack.fullname" $) "alertmanager-monzo-tpl" | trunc 63 | trimSuffix "-") }}
+{{- define "victoria-metrics-k8s-stack.vmInsertEndpoint" -}}
+{{- if .Values.vmsingle.enabled -}}
+{{ printf "http://%s.%s.svc:%d" (include "victoria-metrics-k8s-stack.vmsingleName" .) .Release.Namespace (.Values.vmsingle.spec.port | default 8429) }}
 {{- end }}
-configMaps:
-{{- range compact $list }}
-- {{ . }}
+{{- if .Values.vmcluster.enabled -}}
+{{ printf "http://%s-%s.%s.svc:%d/insert/0" "vminsert" (include "victoria-metrics-k8s-stack.fullname" .) .Release.Namespace (.Values.vmcluster.spec.vminsert.port | default 8480) }}
 {{- end }}
 {{- end }}
+
+
+{{/*
+VMAlert remotes 
+*/}}
+{{- define "victoria-metrics-k8s-stack.vmAlertRemotes" -}}
+remoteWrite:
+    - url: {{ include "victoria-metrics-k8s-stack.vmInsertEndpoint" . }}
+remoteRead:
+    - url: {{ include "victoria-metrics-k8s-stack.vmSelectEndpoint" . }}
+datasource:
+    - url: {{ include "victoria-metrics-k8s-stack.vmSelectEndpoint" . }}
+notifier:
+    - url: {{ printf "http://%s-%s.%s.svc:9093" "vmalertmanager" (include "victoria-metrics-k8s-stack.fullname" .) .Release.Namespace }}
 {{- end }}
 
 {{/*
 VMAlert spec
 */}}
 {{- define "victoria-metrics-k8s-stack.vmAlertSpec" -}}
-{{- $vmAlertStackRemoteWrite := dict "remoteWrite" ( include "victoria-metrics-k8s-stack.vmEndpoint" . | fromYaml ) -}}
-{{- $vmAlertStackRemoteRead := dict "remoteRead" ( include "victoria-metrics-k8s-stack.vmEndpoint" . | fromYaml ) -}}
-{{- $vmAlertStackDatasource := dict "datasource" ( include "victoria-metrics-k8s-stack.vmEndpoint" . | fromYaml ) -}}
-{{- $vmAlertStackNotifier := dict "notifier"  ( dict "url" ( printf "http://vmalertmanager-%s.%s.svc:9093" (include "victoria-metrics-k8s-stack.fullname" .) .Release.Namespace ) ) -}}
-{{ deepCopy .Values.vmalert.spec | mergeOverwrite $vmAlertStackRemoteWrite $vmAlertStackRemoteRead $vmAlertStackDatasource $vmAlertStackNotifier | toYaml }}
+{{ deepCopy .Values.vmalert.spec | mergeOverwrite (include "victoria-metrics-k8s-stack.vmAlertRemotes" . | fromYaml) | toYaml }}
 {{- end }}
 
 
 {{/*
-VM remoteWrite
+VM Agent remoteWrite
 */}}
 {{- define "victoria-metrics-k8s-stack.vmAgentRemoteWrite" -}}
 remoteWrite:
-    - url: "http://{{ .Values.vmsingle.name | default (printf "vmsingle-%s" (include "victoria-metrics-k8s-stack.fullname" .))}}.{{ .Release.Namespace }}.svc:{{ .Values.vmsingle.spec.port | default 8429 }}/api/v1/write"
+    - url: {{ include "victoria-metrics-k8s-stack.vmInsertEndpoint" . }}/api/v1/write
 {{- end }}
 
 {{/*
 VMAgent spec
 */}}
 {{- define "victoria-metrics-k8s-stack.vmAgentSpec" -}}
-{{ deepCopy .Values.vmagent.spec | mergeOverwrite ( include "victoria-metrics-k8s-stack.vmAgentRemoteWrite" . | fromYaml ) | toYaml }}
+{{ deepCopy .Values.vmagent.spec | mergeOverwrite ( include "victoria-metrics-k8s-stack.vmAgentRemoteWrite" . | fromYaml) | toYaml }}
+{{- end }}
+
+
+{{/*
+Alermanager spec
+*/}}
+{{- define "victoria-metrics-k8s-stack.alertmanagerSpec" -}}
+{{ omit .Values.alertmanager.spec  "configMaps" "configSecret" | toYaml }}
+configSecret: {{ .Values.alertmanager.spec.configSecret | default (printf "%s-alertmanager" (include "victoria-metrics-k8s-stack.fullname" .)) }}
+{{- if or .Values.alertmanager.spec.configMaps .Values.alertmanager.monzoTemplate.enabled }}
+{{- $list := .Values.alertmanager.spec.configMaps | default (list "") }}
+{{- if .Values.alertmanager.monzoTemplate.enabled }}
+{{- $list = append $list (printf "%s-%s" (include "victoria-metrics-k8s-stack.fullname" $) "alertmanager-monzo-tpl" | trunc 63 | trimSuffix "-") }}
+{{- end }}
+configMaps:
+{{- range compact $list }}
+- {{ . }}
+{{- end }}
 {{- end }}
+{{- end }}