diff --git a/README.md b/README.md index 8666a28..7d5ada8 100644 --- a/README.md +++ b/README.md @@ -190,10 +190,21 @@ ADD my-config.yml /data/cortex-tenant.yml ### Deploy on Kubernetes -`deploy/k8s` directory contains the deployment, service and configmap manifest files for deploying this on Kubernetes. You can overwrite the default config by editing the configuration parameters in the configmap manifest. +#### Using manifests + +`deploy/k8s/manifests` directory contains the deployment, service and configmap manifest files for deploying this on Kubernetes. You can overwrite the default config by editing the configuration parameters in the configmap manifest. + +```bash +kubectl apply -f deploy/k8s/manifests/cortex-tenant-deployment.yaml +kubectl apply -f deploy/k8s/manifests/cortex-tenant-service.yaml +kubectl apply -f deploy/k8s/manifests/config-file-configmap.yml +``` + +#### Using a Helm Chart + +`deploy/k8s/chart` directory contains a chart for deploying this on Kubernetes. You can use `deploy/k8s/chart/testing` directory to test the deployment using helmfile. ```bash -kubectl apply -f deploy/k8s/cortex-tenant-deployment.yaml -kubectl apply -f deploy/k8s/cortex-tenant-service.yaml -kubectl apply -f deploy/k8s/config-file-configmap.yml +helmfile -f deploy/k8s/chart/testing/helmfile.yaml template +helmfile -f deploy/k8s/chart/testing/helmfile.yaml apply ``` diff --git a/deploy/k8s/chart/Chart.yaml b/deploy/k8s/chart/Chart.yaml new file mode 100644 index 0000000..f7fc618 --- /dev/null +++ b/deploy/k8s/chart/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +description: A Helm Chart for cortex-tenant +name: cortex-tenant +version: 0.1.0 # This is the chart version +appVersion: 1.10.1 # version number of the application being deployed. +type: application diff --git a/deploy/k8s/chart/README.md b/deploy/k8s/chart/README.md new file mode 100644 index 0000000..67db690 --- /dev/null +++ b/deploy/k8s/chart/README.md @@ -0,0 +1,58 @@ +# cortex-tenant + +![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.10.1](https://img.shields.io/badge/AppVersion-1.10.1-informational?style=flat-square) + +A Helm Chart for cortex-tenant + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| affinity | object | `{}` | | +| autoscaling.maxReplica | int | `3` | | +| autoscaling.minReplica | int | `1` | | +| autoscaling.targetCPUUtilizationPercentage | int | `50` | | +| autoscaling.targetMemoryAverageValue | string | `"100Mi"` | | +| config.auth.enabled | bool | `false` | | +| config.auth.existingSecret | string | `nil` | | +| config.auth.password | string | `nil` | | +| config.auth.username | string | `nil` | | +| config.concurrency | int | `1000` | | +| config.enable_ipv6 | bool | `false` | | +| config.listen | string | `"0.0.0.0:8080"` | | +| config.listen_pprof | string | `"0.0.0.0:7008"` | | +| config.log_level | string | `"warn"` | | +| config.log_response_errors | bool | `true` | | +| config.max_connection_duration | string | `"0s"` | | +| config.metadata | bool | `false` | | +| config.target | string | `"http://cortex-distributor.cortex.svc:8080/api/v1/push"` | | +| config.tenant.accept_all | bool | `false` | | +| config.tenant.default | string | `"cortex-tenant-default"` | | +| config.tenant.header | string | `"X-Scope-OrgID"` | | +| config.tenant.label | string | `"tenant"` | | +| config.tenant.label_remove | bool | `false` | | +| config.tenant.prefix | string | `""` | | +| config.timeout | string | `"10s"` | | +| config.timeout_shutdown | string | `"10s"` | | +| envs | string | `nil` | | +| fullnameOverride | string | `nil` | | +| image.pullPolicy | string | `"IfNotPresent"` | | +| image.repository | string | `"ghcr.io/blind-oracle/cortex-tenant"` | | +| image.tag | string | `""` | | +| nameOverride | string | `nil` | | +| nodeSelector | object | `{}` | | +| podAnnotations | object | `{}` | | +| podDisruptionBudget.enabled | bool | `true` | | +| podDisruptionBudget.minAvailable | int | `1` | | +| podSecurityContext.fsGroup | int | `1000` | | +| podSecurityContext.runAsGroup | int | `1000` | | +| podSecurityContext.runAsNonRoot | bool | `true` | | +| podSecurityContext.runAsUser | int | `1000` | | +| resources.limits.memory | string | `"256Mi"` | | +| resources.requests.cpu | string | `"100m"` | | +| resources.requests.memory | string | `"128Mi"` | | +| securityContext | object | `{}` | | +| service.port | int | `8080` | | +| service.targetPort | int | `8080` | | +| service.type | string | `"ClusterIP"` | | +| tolerations | list | `[]` | | diff --git a/deploy/k8s/chart/templates/_helpers.tpl b/deploy/k8s/chart/templates/_helpers.tpl new file mode 100644 index 0000000..eee0b88 --- /dev/null +++ b/deploy/k8s/chart/templates/_helpers.tpl @@ -0,0 +1,51 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "cortex-tenant.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "cortex-tenant.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "cortex-tenant.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "cortex-tenant.labels" -}} +helm.sh/chart: {{ include "cortex-tenant.chart" . }} +{{ include "cortex-tenant.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "cortex-tenant.selectorLabels" -}} +app.kubernetes.io/name: {{ include "cortex-tenant.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/deploy/k8s/chart/templates/autoscaling.yaml b/deploy/k8s/chart/templates/autoscaling.yaml new file mode 100644 index 0000000..270e441 --- /dev/null +++ b/deploy/k8s/chart/templates/autoscaling.yaml @@ -0,0 +1,30 @@ +apiVersion: autoscaling/v2beta2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "cortex-tenant.fullname" . }} + labels: + {{- include "cortex-tenant.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "cortex-tenant.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplica }} + maxReplicas: {{ .Values.autoscaling.maxReplica }} + metrics: + {{- with .Values.autoscaling.targetMemoryAverageValue }} + - type: Resource + resource: + name: memory + target: + type: AverageValue + averageValue: {{ . }} + {{- end }} + {{- with .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ . }} + {{- end }} diff --git a/deploy/k8s/chart/templates/configmap.yaml b/deploy/k8s/chart/templates/configmap.yaml new file mode 100644 index 0000000..4d518cd --- /dev/null +++ b/deploy/k8s/chart/templates/configmap.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "cortex-tenant.fullname" . }}-configmap + labels: + {{- include "cortex-tenant.labels" . | nindent 4 }} +data: + cortex-tenant.yml: |- + listen: {{ .Values.config.listen }} + listen_pprof: {{ .Values.config.listen_pprof }} + target: {{ .Values.config.target }} + enable_ipv6: {{ .Values.config.enable_ipv6 }} + log_level: {{ .Values.config.log_level }} + timeout: {{ .Values.config.timeout }} + timeout_shutdown: {{ .Values.config.timeout_shutdown }} + concurrency: {{ .Values.config.concurrency }} + metadata: {{ .Values.config.metadata }} + log_response_errors: {{ .Values.config.log_response_errors }} + max_connection_duration: {{ .Values.config.max_connection_duration }} + {{- if .Values.config.auth.enabled }} + auth: + egress: + username: {{ .Values.config.auth.username }} + password: {{ .Values.config.auth.password }} + {{- end }} + tenant: + label: {{ .Values.config.tenant.label }} + prefix: {{ .Values.config.tenant.prefix }} + label_remove: {{ .Values.config.tenant.label_remove }} + header: {{ .Values.config.tenant.header }} + default: {{ .Values.config.tenant.default }} + accept_all: {{ .Values.config.tenant.accept_all }} diff --git a/deploy/k8s/chart/templates/deployment.yaml b/deploy/k8s/chart/templates/deployment.yaml new file mode 100644 index 0000000..3ea254e --- /dev/null +++ b/deploy/k8s/chart/templates/deployment.yaml @@ -0,0 +1,60 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + {{- include "cortex-tenant.labels" . | nindent 4 }} + name: {{ include "cortex-tenant.fullname" . }} +spec: + selector: + matchLabels: + {{- include "cortex-tenant.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + {{- with .Values.podAnnotations }} + {{- toYaml .Values.podAnnotations | nindent 8 }} + {{- end }} + labels: + {{- include "cortex-tenant.selectorLabels" . | nindent 8 }} + spec: + containers: + - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + env: + {{- range .Values.envs }} + - name: "{{ .name }}" + value: "{{ .value }}" + {{- end }} + envFrom: + - secretRef: + name: {{ .Values.config.auth.existingSecret }} + ports: + - name: http + containerPort: {{ .Values.service.targetPort }} + protocol: TCP + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - mountPath: /data/ + name: config-file + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + volumes: + - configMap: + name: {{ include "cortex-tenant.fullname" . }}-configmap + name: config-file + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/deploy/k8s/chart/templates/pdb.yaml b/deploy/k8s/chart/templates/pdb.yaml new file mode 100644 index 0000000..45b6cd4 --- /dev/null +++ b/deploy/k8s/chart/templates/pdb.yaml @@ -0,0 +1,13 @@ +{{- if .Values.podDisruptionBudget.enabled -}} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: {{ include "cortex-tenant.fullname" . }} + labels: + {{- include "cortex-tenant.labels" . | nindent 4 }} +spec: + minAvailable: {{ .Values.podDisruptionBudget.minAvailable }} + selector: + matchLabels: + {{- include "cortex-tenant.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/deploy/k8s/chart/templates/service.yaml b/deploy/k8s/chart/templates/service.yaml new file mode 100644 index 0000000..df7c77c --- /dev/null +++ b/deploy/k8s/chart/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + {{- include "cortex-tenant.labels" . | nindent 4 }} + name: {{ include "cortex-tenant.fullname" . }} +spec: + type: {{ .Values.service.type }} + ports: + - name: http + port: {{ .Values.service.port }} + targetPort: {{ .Values.service.targetPort }} + protocol: TCP + selector: + {{- include "cortex-tenant.selectorLabels" . | nindent 4 }} diff --git a/deploy/k8s/chart/testing/README.md b/deploy/k8s/chart/testing/README.md new file mode 100644 index 0000000..d32329c --- /dev/null +++ b/deploy/k8s/chart/testing/README.md @@ -0,0 +1,6 @@ +# How to test (for eg. with Minikube) + +```bash +helmfile template --output-dir . +helmfile apply +``` diff --git a/deploy/k8s/chart/testing/helmfile.yaml b/deploy/k8s/chart/testing/helmfile.yaml new file mode 100644 index 0000000..1455d0f --- /dev/null +++ b/deploy/k8s/chart/testing/helmfile.yaml @@ -0,0 +1,7 @@ +helmDefaults: + cleanupOnFail: true + +releases: + - name: cortex-tenant + namespace: test + chart: ../ diff --git a/deploy/k8s/chart/values.yaml b/deploy/k8s/chart/values.yaml new file mode 100644 index 0000000..cf0bde0 --- /dev/null +++ b/deploy/k8s/chart/values.yaml @@ -0,0 +1,128 @@ +nameOverride: +fullnameOverride: + +image: + repository: ghcr.io/blind-oracle/cortex-tenant # registry to pull + pullPolicy: IfNotPresent # policy when pulling images + tag: "" # Overrides the image tag (default is `.Chart.appVersion`) + +service: + type: ClusterIP + port: 8080 + targetPort: 8080 + +autoscaling: + minReplica: 1 # Min number of pod replica autoscaled + maxReplica: 3 # Max number of pod replica autoscaled + targetMemoryAverageValue: 100Mi + targetCPUUtilizationPercentage: 50 + +envs: + +config: + # Where to listen for incoming write requests from Prometheus + # env: CT_LISTEN + listen: 0.0.0.0:8080 + # Profiling API, leave empty to disable + # env: CT_LISTEN_PPROF + listen_pprof: 0.0.0.0:7008 + # Where to send the modified requests (Cortex) + # env: CT_TARGET + target: http://cortex-distributor.cortex.svc:8080/api/v1/push + # Whether to enable querying for IPv6 records + # env: CT_ENABLE_IPV6 + enable_ipv6: false + # Log level + # env: CT_LOG_LEVEL + log_level: warn + # HTTP request timeout + # env: CT_TIMEOUT + timeout: 10s + # Timeout to wait on shutdown to allow load balancers detect that we're going away. + # During this period after the shutdown command the /alive endpoint will reply with HTTP 503. + # Set to 0s to disable. + # env: CT_TIMEOUT_SHUTDOWN + timeout_shutdown: 10s + # Max number of parallel incoming HTTP requests to handle + # env: CT_CONCURRENCY + concurrency: 1000 + # Whether to forward metrics metadata from Prometheus to Cortex + # Since metadata requests have no timeseries in them - we cannot divide them into tenants + # So the metadata requests will be sent to the default tenant only, if one is not defined - they will be dropped + # env: CT_METADATA + metadata: false + # If true response codes from metrics backend will be logged to stdout. This setting can be used to suppress errors + # which can be quite verbose like 400 code - out-of-order samples or 429 on hitting ingestion limits + # Also, those are already reported by other services like Cortex/Mimir distributors and ingesters + # env: CT_LOG_RESPONSE_ERRORS + log_response_errors: true + # Maximum duration to keep outgoing connections alive (to Cortex/Mimir) + # Useful for resetting L4 load-balancer state + # Use 0 to keep them indefinitely + # env: CT_MAX_CONN_DURATION + max_connection_duration: 0s + + # Authentication (optional) + auth: + # Egress HTTP basic auth -> add `Authentication` header to outgoing requests + enabled: false + # env: CT_AUTH_EGRESS_USERNAME + # env: CT_AUTH_EGRESS_PASSWORD + username: + password: + # Secret should pass the CT_AUTH_EGRESS_USERNAME and CT_AUTH_EGRESS_PASSWORD env variables + existingSecret: + + tenant: + # Which label to look for the tenant information + # env: CT_TENANT_LABEL + label: tenant + # Optional hard-coded prefix with delimeter for all tenant values. + # Delimeters allowed for use: + # https://grafana.com/docs/mimir/latest/configure/about-tenant-ids/ + # env: CT_TENANT_PREFIX + prefix: "" + # Whether to remove the tenant label from the request + # env: CT_TENANT_LABEL_REMOVE + label_remove: false + # To which header to add the tenant ID + # env: CT_TENANT_HEADER + header: X-Scope-OrgID + # Which tenant ID to use if the label is missing in any of the timeseries + # If this is not set or empty then the write request with missing tenant label + # will be rejected with HTTP code 400 + # env: CT_TENANT_DEFAULT + default: cortex-tenant-default + # Enable if you want all metrics from Prometheus to be accepted with a 204 HTTP code + # regardless of the response from Cortex. This can lose metrics if Cortex is + # throwing rejections. + # env: CT_TENANT_ACCEPT_ALL + accept_all: false + +resources: # Resource limits and requests for simu + limits: + # cpu: 100m + memory: 256Mi + requests: + cpu: 100m + memory: 128Mi + +podDisruptionBudget: + enabled: true # If Pod disruption must be enabled + minAvailable: 1 # Number of min pods that must remain available + +podAnnotations: {} # Annotations for pods + +podSecurityContext: # [Security Context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context) + runAsNonRoot: true + runAsGroup: 1000 + runAsUser: 1000 + fsGroup: 1000 + +securityContext: {} # [Security Context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context) + +nodeSelector: {} # [Node Selection](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + +tolerations: [] # [Taints and Tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) + +affinity: {} # [Node Selection](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) diff --git a/deploy/k8s/config-file-configmap.yml b/deploy/k8s/manifests/config-file-configmap.yml similarity index 100% rename from deploy/k8s/config-file-configmap.yml rename to deploy/k8s/manifests/config-file-configmap.yml diff --git a/deploy/k8s/cortex-tenant-deployment.yaml b/deploy/k8s/manifests/cortex-tenant-deployment.yaml similarity index 100% rename from deploy/k8s/cortex-tenant-deployment.yaml rename to deploy/k8s/manifests/cortex-tenant-deployment.yaml diff --git a/deploy/k8s/cortex-tenant-service.yaml b/deploy/k8s/manifests/cortex-tenant-service.yaml similarity index 100% rename from deploy/k8s/cortex-tenant-service.yaml rename to deploy/k8s/manifests/cortex-tenant-service.yaml