feat(chart): add hostAliases and initContainers

It helps to deploy CCM with load blancer as sidecar. Signed-off-by: Serge Logvinov <serge.logvinov@sinextra.dev>
sergelogvinov · Feb 25, 2024 · 9ba9ff2 · 9ba9ff2
1 parent 677e6cc
commit 9ba9ff2
Show file tree

Hide file tree

Showing 8 changed files with 144 additions and 9 deletions.
diff --git a/.gitignore b/.gitignore
@@ -3,6 +3,7 @@
 /charts/proxmox-cloud-controller-manager/values-dev.yaml
 /proxmox-cloud-controller-manager*
 /kubeconfig
+/kubeconfig*
 /proxmox-config.yaml
 #
 

diff --git a/charts/proxmox-cloud-controller-manager/Chart.yaml b/charts/proxmox-cloud-controller-manager/Chart.yaml
@@ -14,7 +14,7 @@ maintainers:
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.11
+version: 0.2.0
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.

diff --git a/charts/proxmox-cloud-controller-manager/README.md b/charts/proxmox-cloud-controller-manager/README.md
@@ -1,6 +1,6 @@
 # proxmox-cloud-controller-manager
 
-![Version: 0.1.11](https://img.shields.io/badge/Version-0.1.11-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.4.0](https://img.shields.io/badge/AppVersion-v0.4.0-informational?style=flat-square)
+![Version: 0.2.0](https://img.shields.io/badge/Version-0.2.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.4.0](https://img.shields.io/badge/AppVersion-v0.4.0-informational?style=flat-square)
 
 A Helm chart for Kubernetes
 
@@ -73,6 +73,8 @@ helm upgrade -i --namespace=kube-system -f proxmox-ccm.yaml \
 | config | object | `{"clusters":[]}` | Proxmox cluster config. |
 | serviceAccount | object | `{"annotations":{},"create":true,"name":""}` | Pods Service Account. ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ |
 | priorityClassName | string | `"system-cluster-critical"` | CCM pods' priorityClassName. |
+| initContainers | list | `[]` | Add additional init containers to the CCM pods. ref: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ |
+| hostAliases | list | `[]` | hostAliases Deployment pod host aliases ref: https://kubernetes.io/docs/tasks/network/customize-hosts-file-for-pods/ |
 | podAnnotations | object | `{}` | Annotations for data pods. ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ |
 | podSecurityContext | object | `{"fsGroup":10258,"fsGroupChangePolicy":"OnRootMismatch","runAsGroup":10258,"runAsNonRoot":true,"runAsUser":10258}` | Pods Security Context. ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod |
 | securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"seccompProfile":{"type":"RuntimeDefault"}}` | Container Security Context. ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod |

diff --git a/charts/proxmox-cloud-controller-manager/templates/deployment.yaml b/charts/proxmox-cloud-controller-manager/templates/deployment.yaml
@@ -35,6 +35,9 @@ spec:
       serviceAccountName: {{ include "proxmox-cloud-controller-manager.serviceAccountName" . }}
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      hostAliases:
+        {{- toYaml .Values.hostAliases | nindent 8 }}
+      initContainers: {{- toYaml .Values.initContainers | nindent 8 }}
       containers:
         - name: {{ .Chart.Name }}
           securityContext:
@@ -73,9 +76,18 @@ spec:
       nodeSelector:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-      {{- with .Values.affinity }}
       affinity:
+      {{- with .Values.affinity }}
         {{- toYaml . | nindent 8 }}
+      {{- else }}
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - podAffinityTerm:
+                labelSelector:
+                  matchLabels:
+                    {{- include "proxmox-cloud-controller-manager.selectorLabels" . | nindent 20 }}
+                topologyKey: topology.kubernetes.io/zone
+              weight: 1
       {{- end }}
       {{- with .Values.tolerations }}
       tolerations:

diff --git a/charts/proxmox-cloud-controller-manager/values.yaml b/charts/proxmox-cloud-controller-manager/values.yaml
@@ -61,6 +61,36 @@ serviceAccount:
 # -- CCM pods' priorityClassName.
 priorityClassName: system-cluster-critical
 
+# -- Add additional init containers to the CCM pods.
+# ref: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/
+initContainers: []
+  # - name: loadbalancer
+  #   restartPolicy: Always
+  #   image: ghcr.io/sergelogvinov/haproxy:2.8.3-alpine3.18
+  #   imagePullPolicy: IfNotPresent
+  #   env:
+  #     - name: SVC
+  #       value: "proxmox.domain.com"
+  #     - name: PORT
+  #       value: "8006"
+  #   securityContext:
+  #     runAsUser: 99
+  #     runAsGroup: 99
+  #   resources:
+  #     limits:
+  #       cpu: 50m
+  #       memory: 64Mi
+  #     requests:
+  #       cpu: 50m
+  #       memory: 32Mi
+
+# -- hostAliases Deployment pod host aliases
+# ref: https://kubernetes.io/docs/tasks/network/customize-hosts-file-for-pods/
+hostAliases: []
+  # - ip: 127.0.0.1
+  #   hostnames:
+  #     - proxmox.domain.com
+
 # -- Annotations for data pods.
 # ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/
 podAnnotations: {}

diff --git a/docs/deploy/cloud-controller-manager-talos.yml b/docs/deploy/cloud-controller-manager-talos.yml
@@ -5,7 +5,7 @@ kind: ServiceAccount
 metadata:
   name: proxmox-cloud-controller-manager
   labels:
-    helm.sh/chart: proxmox-cloud-controller-manager-0.1.11
+    helm.sh/chart: proxmox-cloud-controller-manager-0.2.0
     app.kubernetes.io/name: proxmox-cloud-controller-manager
     app.kubernetes.io/instance: proxmox-cloud-controller-manager
     app.kubernetes.io/version: "v0.4.0"
@@ -18,7 +18,7 @@ kind: ClusterRole
 metadata:
   name: system:proxmox-cloud-controller-manager
   labels:
-    helm.sh/chart: proxmox-cloud-controller-manager-0.1.11
+    helm.sh/chart: proxmox-cloud-controller-manager-0.2.0
     app.kubernetes.io/name: proxmox-cloud-controller-manager
     app.kubernetes.io/instance: proxmox-cloud-controller-manager
     app.kubernetes.io/version: "v0.4.0"
@@ -106,7 +106,7 @@ kind: Deployment
 metadata:
   name: proxmox-cloud-controller-manager
   labels:
-    helm.sh/chart: proxmox-cloud-controller-manager-0.1.11
+    helm.sh/chart: proxmox-cloud-controller-manager-0.2.0
     app.kubernetes.io/name: proxmox-cloud-controller-manager
     app.kubernetes.io/instance: proxmox-cloud-controller-manager
     app.kubernetes.io/version: "v0.4.0"
@@ -137,6 +137,10 @@ spec:
         runAsGroup: 10258
         runAsNonRoot: true
         runAsUser: 10258
+      hostAliases:
+        []
+      initContainers:
+        []
       containers:
         - name: proxmox-cloud-controller-manager
           securityContext:

diff --git a/docs/deploy/cloud-controller-manager.yml b/docs/deploy/cloud-controller-manager.yml
@@ -5,7 +5,7 @@ kind: ServiceAccount
 metadata:
   name: proxmox-cloud-controller-manager
   labels:
-    helm.sh/chart: proxmox-cloud-controller-manager-0.1.11
+    helm.sh/chart: proxmox-cloud-controller-manager-0.2.0
     app.kubernetes.io/name: proxmox-cloud-controller-manager
     app.kubernetes.io/instance: proxmox-cloud-controller-manager
     app.kubernetes.io/version: "v0.4.0"
@@ -18,7 +18,7 @@ kind: ClusterRole
 metadata:
   name: system:proxmox-cloud-controller-manager
   labels:
-    helm.sh/chart: proxmox-cloud-controller-manager-0.1.11
+    helm.sh/chart: proxmox-cloud-controller-manager-0.2.0
     app.kubernetes.io/name: proxmox-cloud-controller-manager
     app.kubernetes.io/instance: proxmox-cloud-controller-manager
     app.kubernetes.io/version: "v0.4.0"
@@ -106,7 +106,7 @@ kind: Deployment
 metadata:
   name: proxmox-cloud-controller-manager
   labels:
-    helm.sh/chart: proxmox-cloud-controller-manager-0.1.11
+    helm.sh/chart: proxmox-cloud-controller-manager-0.2.0
     app.kubernetes.io/name: proxmox-cloud-controller-manager
     app.kubernetes.io/instance: proxmox-cloud-controller-manager
     app.kubernetes.io/version: "v0.4.0"
@@ -137,6 +137,10 @@ spec:
         runAsGroup: 10258
         runAsNonRoot: true
         runAsUser: 10258
+      hostAliases:
+        []
+      initContainers:
+        []
       containers:
         - name: proxmox-cloud-controller-manager
           securityContext:

diff --git a/docs/loadbalancer.md b/docs/loadbalancer.md
@@ -0,0 +1,82 @@
+# Loadbalancer on top of the Proxmox cluster
+
+Set up a load balancer to distribute traffic across multiple proxmox nodes.
+We use the [haproxy](https://hub.docker.com/_/haproxy) image to create a simple load balancer on top of the proxmox cluster.
+First, we need to create a headless service and set endpoints.
+
+```yaml
+# proxmox-service.yaml
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: proxmox
+  namespace: kube-system
+spec:
+  clusterIP: None
+  ports:
+    - name: https
+      protocol: TCP
+      port: 8006
+      targetPort: 8006
+---
+apiVersion: v1
+kind: Endpoints
+metadata:
+  name: proxmox
+  namespace: kube-system
+subsets:
+  - addresses:
+      - ip: 192.168.0.1
+      - ip: 192.168.0.2
+    ports:
+      - port: 8006
+```
+
+Apply the configuration to the cluster.
+
+```bash
+kubectl apply -f proxmox-service.yaml
+```
+
+Second, we need to deploy proxmox CCM with sidecar load balancer.
+Haproxy will resolve the `proxmox.kube-system.svc.cluster.local` service and uses IPs from the endpoints to distribute traffic.
+Proxmox CCM will use the `proxmox.domain.com` domain to connect to the proxmox cluster wich is resolved to the load balancer IP (127.0.0.1).
+
+```yaml
+# Helm Chart values
+
+config:
+  clusters:
+    - region: cluster
+      url: https://proxmox.domain.com:8006/api2/json
+      insecure: true
+      token_id: kubernetes@pve!ccm
+      token_secret: 11111111-1111-1111-1111-111111111111
+
+hostAliases:
+  - ip: 127.0.0.1
+    hostnames:
+      - proxmox.domain.com
+
+initContainers:
+  - name: loadbalancer
+    restartPolicy: Always
+    image: ghcr.io/sergelogvinov/haproxy:2.8.6-alpine3.19
+    imagePullPolicy: IfNotPresent
+    env:
+      - name: SVC
+        value: proxmox.kube-system.svc.cluster.local
+      - name: PORT
+        value: "8006"
+    securityContext:
+      runAsUser: 99
+      runAsGroup: 99
+    resources:
+      limits:
+        cpu: 50m
+        memory: 64Mi
+      requests:
+        cpu: 50m
+        memory: 32Mi
+```