From 4140a321a996345799cbd9da6e1a7f81b7a9629d Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Thu, 21 Oct 2021 14:40:39 +0000 Subject: [PATCH 1/5] feat: helm: add anti-affinity rule for deployments with multiple replicas --- README.md | 5 ++++- templates/coderd.yaml | 16 ++++++++++++++++ values.yaml | 11 +++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f5a7a840..8fdd72f6 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,10 @@ View [our docs](https://coder.com/docs/setup/installation) for detailed installa | certs | object | Certificate that will be mounted inside Coder services. | `{"secret":{"key":"","name":""}}` | | certs.secret.key | string | Key pointing to a certificate in the secret. | `""` | | certs.secret.name | string | Name of the secret. | `""` | -| coderd | object | Primary service responsible for all things Coder! | `{"builtinProviderServiceAccount":{"annotations":{},"labels":{}},"devurlsHost":"","image":"","oidc":{"enableRefresh":false,"redirectOptions":{}},"podSecurityContext":{"runAsNonRoot":true,"runAsUser":1000,"seccompProfile":{"type":"RuntimeDefault"}},"replicas":1,"resources":{"limits":{"cpu":"250m","memory":"512Mi"},"requests":{"cpu":"250m","memory":"512Mi"}},"satellite":{"accessURL":"","enable":false,"primaryURL":""},"securityContext":{"allowPrivilegeEscalation":false,"readOnlyRootFilesystem":true,"seccompProfile":{"type":"RuntimeDefault"}},"serviceAnnotations":{},"serviceNodePorts":{"http":null,"https":null},"serviceSpec":{"externalTrafficPolicy":"Local","loadBalancerIP":"","loadBalancerSourceRanges":[],"type":"LoadBalancer"},"superAdmin":{"passwordSecret":{"key":"password","name":""}},"tls":{"devurlsHostSecretName":"","hostSecretName":""},"trustProxyIP":false}` | +| coderd | object | Primary service responsible for all things Coder! | `{"antiAffinity":{"enable":false,"weight":100},"builtinProviderServiceAccount":{"annotations":{},"labels":{}},"devurlsHost":"","image":"","oidc":{"enableRefresh":false,"redirectOptions":{}},"podSecurityContext":{"runAsNonRoot":true,"runAsUser":1000,"seccompProfile":{"type":"RuntimeDefault"}},"replicas":1,"resources":{"limits":{"cpu":"250m","memory":"512Mi"},"requests":{"cpu":"250m","memory":"512Mi"}},"satellite":{"accessURL":"","enable":false,"primaryURL":""},"securityContext":{"allowPrivilegeEscalation":false,"readOnlyRootFilesystem":true,"seccompProfile":{"type":"RuntimeDefault"}},"serviceAnnotations":{},"serviceNodePorts":{"http":null,"https":null},"serviceSpec":{"externalTrafficPolicy":"Local","loadBalancerIP":"","loadBalancerSourceRanges":[],"type":"LoadBalancer"},"superAdmin":{"passwordSecret":{"key":"password","name":""}},"tls":{"devurlsHostSecretName":"","hostSecretName":""},"trustProxyIP":false}` | +| coderd.antiAffinity | object | Enables adding a anti-affinity of type preferredDuringSchedulingIgnoredDuringExecution to coderd if more than one replica is present. This is recommended to ensure that multiple coderd pods are not scheduled on the same node. See: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#more-practical-use-cases | `{"enable":false,"weight":100}` | +| coderd.antiAffinity.enable | bool | Should an anti-affinity rule ever be added? | `false` | +| coderd.antiAffinity.weight | int | Weight for the anti-affinity rule, if added. | `100` | | coderd.builtinProviderServiceAccount | object | Customize the built-in Kubernetes provider service account. | `{"annotations":{},"labels":{}}` | | coderd.builtinProviderServiceAccount.annotations | object | A KV mapping of annotations. See: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ | `{}` | | coderd.builtinProviderServiceAccount.labels | object | Add labels to the service account used for the built-in provider. | `{}` | diff --git a/templates/coderd.yaml b/templates/coderd.yaml index 706936e1..fc487016 100644 --- a/templates/coderd.yaml +++ b/templates/coderd.yaml @@ -47,6 +47,22 @@ spec: {{- if not .Values.coderd.satellite.enable }} serviceAccountName: coder {{- end }} + {{- $replicas := include "movedValue" (dict "Values" .Values "Key" "coderd.replicas" 1) | int -}} + {{- $enabled := .Values.coderd.antiAffinity.enable | default false -}} + {{- if and (gt $replicas 1) ($enabled) }} + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - {{ include "coder.serviceName" . }} + topologyKey: "kubernetes.io/hostname" + weight: {{- .Values.coderd.antiAffinity.weight | default 100 }} + {{- end }} {{- include "coder.services.nodeSelector" . | indent 6 }} {{- include "coder.serviceTolerations" . | indent 6 }} {{- if not .Values.coderd.satellite.enable }} diff --git a/values.yaml b/values.yaml index a3c736db..864c07da 100644 --- a/values.yaml +++ b/values.yaml @@ -151,6 +151,17 @@ coderd: # contains the super admin password. key: "password" + # coderd.antiAffinity -- Enables adding a anti-affinity of type + # preferredDuringSchedulingIgnoredDuringExecution to coderd + # if more than one replica is present. This is recommended to ensure that + # multiple coderd pods are not scheduled on the same node. + # See: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#more-practical-use-cases + antiAffinity: + # coderd.antiAffinity.enable -- Should an anti-affinity rule ever be added? + enable: false + # coderd.antiAffinity.weight -- Weight for the anti-affinity rule, if added. + weight: 100 + # envbox -- Required for running Docker inside containers. See requirements: # https://coder.com/docs/coder/v1.19/admin/workspace-management/cvms envbox: From 60c4ab353bd5af3de9235bea7228ffd3d7c87b11 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Thu, 21 Oct 2021 18:53:48 +0000 Subject: [PATCH 2/5] fixup! feat: helm: add anti-affinity rule for deployments with multiple replicas --- templates/coderd.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/coderd.yaml b/templates/coderd.yaml index fc487016..1a8e64f3 100644 --- a/templates/coderd.yaml +++ b/templates/coderd.yaml @@ -47,8 +47,8 @@ spec: {{- if not .Values.coderd.satellite.enable }} serviceAccountName: coder {{- end }} - {{- $replicas := include "movedValue" (dict "Values" .Values "Key" "coderd.replicas" 1) | int -}} - {{- $enabled := .Values.coderd.antiAffinity.enable | default false -}} + {{- $replicas := include "movedValue" (dict "Values" .Values "Key" "coderd.replicas" 1) | int }} + {{- $enabled := .Values.coderd.antiAffinity.enable | default false }} {{- if and (gt $replicas 1) ($enabled) }} affinity: podAntiAffinity: @@ -61,7 +61,7 @@ spec: values: - {{ include "coder.serviceName" . }} topologyKey: "kubernetes.io/hostname" - weight: {{- .Values.coderd.antiAffinity.weight | default 100 }} + weight: {{ .Values.coderd.antiAffinity.weight | default 100 }} {{- end }} {{- include "coder.services.nodeSelector" . | indent 6 }} {{- include "coder.serviceTolerations" . | indent 6 }} From 8dec33e68a1c9b997e498c1ee2781cefd38304bf Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Thu, 21 Oct 2021 19:47:56 +0000 Subject: [PATCH 3/5] fixup! feat: helm: add anti-affinity rule for deployments with multiple replicas --- templates/coderd.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/coderd.yaml b/templates/coderd.yaml index 1a8e64f3..7ccfc4d3 100644 --- a/templates/coderd.yaml +++ b/templates/coderd.yaml @@ -47,6 +47,7 @@ spec: {{- if not .Values.coderd.satellite.enable }} serviceAccountName: coder {{- end }} + {{- if hasKey .Values.coderd "antiAffinity" }} {{- $replicas := include "movedValue" (dict "Values" .Values "Key" "coderd.replicas" 1) | int }} {{- $enabled := .Values.coderd.antiAffinity.enable | default false }} {{- if and (gt $replicas 1) ($enabled) }} @@ -63,6 +64,7 @@ spec: topologyKey: "kubernetes.io/hostname" weight: {{ .Values.coderd.antiAffinity.weight | default 100 }} {{- end }} + {{- end }} {{- include "coder.services.nodeSelector" . | indent 6 }} {{- include "coder.serviceTolerations" . | indent 6 }} {{- if not .Values.coderd.satellite.enable }} From e2e62c0d7732b14335fe595034b445cf3dd6ad04 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 22 Oct 2021 11:26:27 +0000 Subject: [PATCH 4/5] feat: helm: simplify coderd affinity definition --- templates/_functions.tpl | 13 +++++++++++++ templates/coderd.yaml | 19 ++----------------- values.yaml | 26 +++++++++++++++++--------- 3 files changed, 32 insertions(+), 26 deletions(-) diff --git a/templates/_functions.tpl b/templates/_functions.tpl index b10db930..a4141922 100644 --- a/templates/_functions.tpl +++ b/templates/_functions.tpl @@ -25,3 +25,16 @@ resources: {{- end }} {{- end }} {{- end }} + +{{/* +coder.template.render -- renders a value that contains template. +Usage: +{{ include "coder.template.render" ( dict "value" .Values.path.to.the.Value "context" $) }} +*/}} +{{- define "coder.template.render" -}} + {{- if typeIs "string" .value }} + {{- tpl .value .context }} + {{- else }} + {{- tpl (.value | toYaml) .context }} + {{- end }} +{{- end -}} diff --git a/templates/coderd.yaml b/templates/coderd.yaml index 7ccfc4d3..3626bd77 100644 --- a/templates/coderd.yaml +++ b/templates/coderd.yaml @@ -47,23 +47,8 @@ spec: {{- if not .Values.coderd.satellite.enable }} serviceAccountName: coder {{- end }} - {{- if hasKey .Values.coderd "antiAffinity" }} - {{- $replicas := include "movedValue" (dict "Values" .Values "Key" "coderd.replicas" 1) | int }} - {{- $enabled := .Values.coderd.antiAffinity.enable | default false }} - {{- if and (gt $replicas 1) ($enabled) }} - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - podAffinityTerm: - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - {{ include "coder.serviceName" . }} - topologyKey: "kubernetes.io/hostname" - weight: {{ .Values.coderd.antiAffinity.weight | default 100 }} - {{- end }} + {{- if .Values.coderd.affinity }} + affinity: {{- include "coder.template.render" ( dict "value" .Values.coderd.affinity "context" $) | nindent 8 }} {{- end }} {{- include "coder.services.nodeSelector" . | indent 6 }} {{- include "coder.serviceTolerations" . | indent 6 }} diff --git a/values.yaml b/values.yaml index 864c07da..db151a96 100644 --- a/values.yaml +++ b/values.yaml @@ -151,16 +151,24 @@ coderd: # contains the super admin password. key: "password" - # coderd.antiAffinity -- Enables adding a anti-affinity of type - # preferredDuringSchedulingIgnoredDuringExecution to coderd - # if more than one replica is present. This is recommended to ensure that + # coderd.affinity -- allows specifying an affinity rule for the coderd + # deployment. If you set coderd.replicas to a value higher than 1, you + # may wish to also add a pod anti-affinity definition to ensure that # multiple coderd pods are not scheduled on the same node. - # See: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#more-practical-use-cases - antiAffinity: - # coderd.antiAffinity.enable -- Should an anti-affinity rule ever be added? - enable: false - # coderd.antiAffinity.weight -- Weight for the anti-affinity rule, if added. - weight: 100 + # Example: + # affinity: + # podAntiAffinity: + # preferredDuringSchedulingIgnoredDuringExecution: + # - podAffinityTerm: + # labelSelector: + # matchExpressions: + # - key: app + # operator: In + # values: + # - "coderd" + # topologyKey: kubernetes.io/hostname + # weight: 1 + affinity: {} # envbox -- Required for running Docker inside containers. See requirements: # https://coder.com/docs/coder/v1.19/admin/workspace-management/cvms From a6943782f387dd89aa220cc27d514962c84989b5 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 22 Oct 2021 11:40:10 +0000 Subject: [PATCH 5/5] fixup! feat: helm: simplify coderd affinity definition --- README.md | 6 ++---- values.yaml | 8 +++++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 8fdd72f6..2a70cac7 100644 --- a/README.md +++ b/README.md @@ -25,10 +25,8 @@ View [our docs](https://coder.com/docs/setup/installation) for detailed installa | certs | object | Certificate that will be mounted inside Coder services. | `{"secret":{"key":"","name":""}}` | | certs.secret.key | string | Key pointing to a certificate in the secret. | `""` | | certs.secret.name | string | Name of the secret. | `""` | -| coderd | object | Primary service responsible for all things Coder! | `{"antiAffinity":{"enable":false,"weight":100},"builtinProviderServiceAccount":{"annotations":{},"labels":{}},"devurlsHost":"","image":"","oidc":{"enableRefresh":false,"redirectOptions":{}},"podSecurityContext":{"runAsNonRoot":true,"runAsUser":1000,"seccompProfile":{"type":"RuntimeDefault"}},"replicas":1,"resources":{"limits":{"cpu":"250m","memory":"512Mi"},"requests":{"cpu":"250m","memory":"512Mi"}},"satellite":{"accessURL":"","enable":false,"primaryURL":""},"securityContext":{"allowPrivilegeEscalation":false,"readOnlyRootFilesystem":true,"seccompProfile":{"type":"RuntimeDefault"}},"serviceAnnotations":{},"serviceNodePorts":{"http":null,"https":null},"serviceSpec":{"externalTrafficPolicy":"Local","loadBalancerIP":"","loadBalancerSourceRanges":[],"type":"LoadBalancer"},"superAdmin":{"passwordSecret":{"key":"password","name":""}},"tls":{"devurlsHostSecretName":"","hostSecretName":""},"trustProxyIP":false}` | -| coderd.antiAffinity | object | Enables adding a anti-affinity of type preferredDuringSchedulingIgnoredDuringExecution to coderd if more than one replica is present. This is recommended to ensure that multiple coderd pods are not scheduled on the same node. See: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#more-practical-use-cases | `{"enable":false,"weight":100}` | -| coderd.antiAffinity.enable | bool | Should an anti-affinity rule ever be added? | `false` | -| coderd.antiAffinity.weight | int | Weight for the anti-affinity rule, if added. | `100` | +| coderd | object | Primary service responsible for all things Coder! | `{"affinity":{},"builtinProviderServiceAccount":{"annotations":{},"labels":{}},"devurlsHost":"","image":"","oidc":{"enableRefresh":false,"redirectOptions":{}},"podSecurityContext":{"runAsNonRoot":true,"runAsUser":1000,"seccompProfile":{"type":"RuntimeDefault"}},"replicas":1,"resources":{"limits":{"cpu":"250m","memory":"512Mi"},"requests":{"cpu":"250m","memory":"512Mi"}},"satellite":{"accessURL":"","enable":false,"primaryURL":""},"securityContext":{"allowPrivilegeEscalation":false,"readOnlyRootFilesystem":true,"seccompProfile":{"type":"RuntimeDefault"}},"serviceAnnotations":{},"serviceNodePorts":{"http":null,"https":null},"serviceSpec":{"externalTrafficPolicy":"Local","loadBalancerIP":"","loadBalancerSourceRanges":[],"type":"LoadBalancer"},"superAdmin":{"passwordSecret":{"key":"password","name":""}},"tls":{"devurlsHostSecretName":"","hostSecretName":""},"trustProxyIP":false}` | +| coderd.affinity | object | Allows specifying an affinity rule for the `coderd` deployment. If you set `coderd.replicas` to a value higher than 1, you may wish to also add a pod anti-affinity definition to ensure that multiple coderd pods are not scheduled on the same node. | `{}` | | coderd.builtinProviderServiceAccount | object | Customize the built-in Kubernetes provider service account. | `{"annotations":{},"labels":{}}` | | coderd.builtinProviderServiceAccount.annotations | object | A KV mapping of annotations. See: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ | `{}` | | coderd.builtinProviderServiceAccount.labels | object | Add labels to the service account used for the built-in provider. | `{}` | diff --git a/values.yaml b/values.yaml index db151a96..c1a83393 100644 --- a/values.yaml +++ b/values.yaml @@ -151,11 +151,13 @@ coderd: # contains the super admin password. key: "password" - # coderd.affinity -- allows specifying an affinity rule for the coderd - # deployment. If you set coderd.replicas to a value higher than 1, you + # coderd.affinity -- Allows specifying an affinity rule for the `coderd` + # deployment. If you set `coderd.replicas` to a value higher than 1, you # may wish to also add a pod anti-affinity definition to ensure that # multiple coderd pods are not scheduled on the same node. + affinity: {} # Example: + # ``` # affinity: # podAntiAffinity: # preferredDuringSchedulingIgnoredDuringExecution: @@ -168,7 +170,7 @@ coderd: # - "coderd" # topologyKey: kubernetes.io/hostname # weight: 1 - affinity: {} + # ``` # envbox -- Required for running Docker inside containers. See requirements: # https://coder.com/docs/coder/v1.19/admin/workspace-management/cvms