From 1af98b3e8c2619083dc2735ac6bc8bc93741f183 Mon Sep 17 00:00:00 2001 From: Andrey Zhavoronkov Date: Wed, 19 Jun 2024 13:28:56 +0300 Subject: [PATCH 1/3] Updated helm configuration to allow use of an external clickhouse instance --- components/analytics/vector/vector.toml | 2 +- helm-chart/Chart.yaml | 2 +- helm-chart/templates/_helpers.tpl | 32 +++++++++++++++++-- .../analytics/cvat-analytics-secret.yml | 1 + .../cvat_backend/server/deployment.yml | 1 + .../worker_analyticsreports/deployment.yml | 1 + .../cvat_backend/worker_export/deployment.yml | 1 + helm-chart/values.yaml | 13 +++++--- 8 files changed, 44 insertions(+), 9 deletions(-) diff --git a/components/analytics/vector/vector.toml b/components/analytics/vector/vector.toml index e528c772d2a4..b8f2ec268b28 100644 --- a/components/analytics/vector/vector.toml +++ b/components/analytics/vector/vector.toml @@ -20,7 +20,7 @@ table = "events" auth.strategy = "basic" auth.user = "${CLICKHOUSE_USER}" auth.password = "${CLICKHOUSE_PASSWORD}" -endpoint = "http://${CLICKHOUSE_HOST}:8123" +endpoint = "http://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT}" request.concurrency = "adaptive" encoding.only_fields = [ "scope", diff --git a/helm-chart/Chart.yaml b/helm-chart/Chart.yaml index 58531f888760..1dcc3739a31f 100644 --- a/helm-chart/Chart.yaml +++ b/helm-chart/Chart.yaml @@ -16,7 +16,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.13.1 +version: 0.13.2 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm-chart/templates/_helpers.tpl b/helm-chart/templates/_helpers.tpl index 9fe032f32551..576445f5121e 100644 --- a/helm-chart/templates/_helpers.tpl +++ b/helm-chart/templates/_helpers.tpl @@ -126,8 +126,6 @@ The name of the service account to use for backend pods value: "{{ .Release.Name }}-vector" - name: DJANGO_LOG_SERVER_PORT value: "80" -- name: CLICKHOUSE_HOST - value: "{{ .Release.Name }}-clickhouse" {{- end }} - name: SMOKESCREEN_OPTS @@ -141,3 +139,33 @@ The name of the service account to use for backend pods value: "{{ .Release.Namespace }}" {{- end }} {{- end }} + +{{- define "cvat.sharedClickhouseEnv" }} +{{- if .Values.analytics.enabled}} +- name: CLICKHOUSE_HOST + valueFrom: + secretKeyRef: + name: cvat-analytics-secret + key: CLICKHOUSE_HOST +- name: CLICKHOUSE_PORT + valueFrom: + secretKeyRef: + name: cvat-analytics-secret + key: CLICKHOUSE_PORT +- name: CLICKHOUSE_DB + valueFrom: + secretKeyRef: + name: cvat-analytics-secret + key: CLICKHOUSE_DB +- name: CLICKHOUSE_USER + valueFrom: + secretKeyRef: + name: cvat-analytics-secret + key: CLICKHOUSE_USER +- name: CLICKHOUSE_PASSWORD + valueFrom: + secretKeyRef: + name: cvat-analytics-secret + key: CLICKHOUSE_PASSWORD +{{- end }} +{{- end }} diff --git a/helm-chart/templates/analytics/cvat-analytics-secret.yml b/helm-chart/templates/analytics/cvat-analytics-secret.yml index 4097adca1146..e95f7fc671c9 100644 --- a/helm-chart/templates/analytics/cvat-analytics-secret.yml +++ b/helm-chart/templates/analytics/cvat-analytics-secret.yml @@ -17,4 +17,5 @@ stringData: CLICKHOUSE_USER: {{ .Values.analytics.clickhouseUser }} CLICKHOUSE_PASSWORD: {{ .Values.analytics.clickhousePassword }} CLICKHOUSE_HOST: {{ tpl (.Values.analytics.clickhouseHost) . }} + CLICKHOUSE_PORT: {{ .Values.analytics.clickhousePort | quote }} {{- end }} diff --git a/helm-chart/templates/cvat_backend/server/deployment.yml b/helm-chart/templates/cvat_backend/server/deployment.yml index 8dff291957a2..a90358cd9543 100644 --- a/helm-chart/templates/cvat_backend/server/deployment.yml +++ b/helm-chart/templates/cvat_backend/server/deployment.yml @@ -59,6 +59,7 @@ spec: - name: ALLOWED_HOSTS value: {{ $localValues.envs.ALLOWED_HOSTS | squote}} {{ include "cvat.sharedBackendEnv" . | indent 10 }} + {{ include "cvat.sharedClickhouseEnv" . | indent 10 }} {{- with concat .Values.cvat.backend.additionalEnv $localValues.additionalEnv }} {{- toYaml . | nindent 10 }} {{- end }} diff --git a/helm-chart/templates/cvat_backend/worker_analyticsreports/deployment.yml b/helm-chart/templates/cvat_backend/worker_analyticsreports/deployment.yml index b4ae5456fc2b..ffc4997081eb 100644 --- a/helm-chart/templates/cvat_backend/worker_analyticsreports/deployment.yml +++ b/helm-chart/templates/cvat_backend/worker_analyticsreports/deployment.yml @@ -57,6 +57,7 @@ spec: args: ["run", "worker.analytics_reports"] env: {{ include "cvat.sharedBackendEnv" . | indent 10 }} + {{ include "cvat.sharedClickhouseEnv" . | indent 10 }} {{- with concat .Values.cvat.backend.additionalEnv $localValues.additionalEnv }} {{- toYaml . | nindent 10 }} {{- end }} diff --git a/helm-chart/templates/cvat_backend/worker_export/deployment.yml b/helm-chart/templates/cvat_backend/worker_export/deployment.yml index ab37061e7ddd..e96723c394b8 100644 --- a/helm-chart/templates/cvat_backend/worker_export/deployment.yml +++ b/helm-chart/templates/cvat_backend/worker_export/deployment.yml @@ -57,6 +57,7 @@ spec: args: ["run", "worker.export"] env: {{ include "cvat.sharedBackendEnv" . | indent 10 }} + {{ include "cvat.sharedClickhouseEnv" . | indent 10 }} {{- with concat .Values.cvat.backend.additionalEnv $localValues.additionalEnv }} {{- toYaml . | nindent 10 }} {{- end }} diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml index 5da777ce47a7..6d64ace2fff0 100644 --- a/helm-chart/values.yaml +++ b/helm-chart/values.yaml @@ -326,6 +326,7 @@ analytics: clickhouseUser: user clickhousePassword: user clickhouseHost: "{{ .Release.Name }}-clickhouse" + clickhousePort: 8123 vector: envFrom: @@ -360,6 +361,7 @@ clickhouse: enabled: false grafana: + envFromSecret: cvat-analytics-secret datasources: datasources.yaml: apiVersion: 1 @@ -368,13 +370,14 @@ grafana: type: 'grafana-clickhouse-datasource' isDefault: true jsonData: - defaultDatabase: cvat - port: 9000 - server: "{{ .Release.Name }}-clickhouse" - username: user + defaultDatabase: ${CLICKHOUSE_DB} + port: ${CLICKHOUSE_PORT} + server: ${CLICKHOUSE_HOST} + username: ${CLICKHOUSE_USER} tlsSkipVerify: false + protocol: http secureJsonData: - password: user + password: ${CLICKHOUSE_PASSWORD} editable: false dashboardProviders: dashboardproviders.yaml: From 4767a086daebe7d1ffbe8e28c73d76d629433ad1 Mon Sep 17 00:00:00 2001 From: Andrey Zhavoronkov Date: Tue, 25 Jun 2024 12:59:43 +0300 Subject: [PATCH 2/3] apply comment --- .../20240625_125622_andrey_update_helm_clickhouse_config.md | 4 ++++ helm-chart/Chart.yaml | 2 +- helm-chart/templates/_helpers.tpl | 2 +- helm-chart/values.yaml | 3 +++ 4 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 changelog.d/20240625_125622_andrey_update_helm_clickhouse_config.md diff --git a/changelog.d/20240625_125622_andrey_update_helm_clickhouse_config.md b/changelog.d/20240625_125622_andrey_update_helm_clickhouse_config.md new file mode 100644 index 000000000000..a2bb86025e89 --- /dev/null +++ b/changelog.d/20240625_125622_andrey_update_helm_clickhouse_config.md @@ -0,0 +1,4 @@ +### Added + +- [Helm] Ability to use an external ClickHouse instance + () diff --git a/helm-chart/Chart.yaml b/helm-chart/Chart.yaml index 1dcc3739a31f..3d556c034b0d 100644 --- a/helm-chart/Chart.yaml +++ b/helm-chart/Chart.yaml @@ -44,7 +44,7 @@ dependencies: - name: clickhouse version: "4.1.*" repository: https://charts.bitnami.com/bitnami - condition: analytics.enabled + condition: clickhouse.enabled - name: grafana version: "6.60.*" diff --git a/helm-chart/templates/_helpers.tpl b/helm-chart/templates/_helpers.tpl index 576445f5121e..dae7d8b61a52 100644 --- a/helm-chart/templates/_helpers.tpl +++ b/helm-chart/templates/_helpers.tpl @@ -141,7 +141,7 @@ The name of the service account to use for backend pods {{- end }} {{- define "cvat.sharedClickhouseEnv" }} -{{- if .Values.analytics.enabled}} +{{- if .Values.analytics.enabled }} - name: CLICKHOUSE_HOST valueFrom: secretKeyRef: diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml index 6d64ace2fff0..91e4493258f7 100644 --- a/helm-chart/values.yaml +++ b/helm-chart/values.yaml @@ -321,6 +321,7 @@ nuclio: # password: somepass analytics: + # Set clickhouse.enabled to false if you disable analytics or use an external database enabled: true clickhouseDb: cvat clickhouseUser: user @@ -348,6 +349,8 @@ vector: tag: "0.26.0-alpine" clickhouse: + # Set to false in case of external db usage + enabled: true shards: 1 replicaCount: 1 extraEnvVarsSecret: cvat-analytics-secret From 35077f530c3bf999d61c669a233c48ea73391a71 Mon Sep 17 00:00:00 2001 From: Andrey Zhavoronkov Date: Tue, 25 Jun 2024 14:32:06 +0300 Subject: [PATCH 3/3] fix remark --- .../20240625_125622_andrey_update_helm_clickhouse_config.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/20240625_125622_andrey_update_helm_clickhouse_config.md b/changelog.d/20240625_125622_andrey_update_helm_clickhouse_config.md index a2bb86025e89..c7135605e0c6 100644 --- a/changelog.d/20240625_125622_andrey_update_helm_clickhouse_config.md +++ b/changelog.d/20240625_125622_andrey_update_helm_clickhouse_config.md @@ -1,4 +1,4 @@ ### Added -- [Helm] Ability to use an external ClickHouse instance +- \[Helm\] Ability to use an external ClickHouse instance ()