Add Autoscaler support to existing CSE methods and types (#678)

Signed-off-by: abarreiro <abarreiro@vmware.com>
vmware · Jun 14, 2024 · af1e038 · af1e038
1 parent 5379f18
commit af1e038
Show file tree

Hide file tree

Showing 12 changed files with 1,218 additions and 88 deletions.
diff --git a/.changes/v2.25.0/678-features.md b/.changes/v2.25.0/678-features.md
@@ -0,0 +1,2 @@
+* Added autoscaling capabilities when creating or updating CSE Kubernetes clusters, with `CseWorkerPoolSettings.Autoscaler` 
+  and `CseWorkerPoolUpdateInput.Autoscaler`, that allows to configure this mechanism on specific worker pools [GH-678]
diff --git a/govcd/cse.go b/govcd/cse.go
@@ -365,8 +365,8 @@ func (cluster *CseKubernetesCluster) Delete(timeout time.Duration) error {
 			return fmt.Errorf("could not retrieve the Kubernetes cluster with ID '%s': %s", cluster.ID, err)
 		}
 
-		markForDelete = traverseMapAndGet[bool](rde.DefinedEntity.Entity, "spec.vcdKe.markForDelete")
-		forceDelete = traverseMapAndGet[bool](rde.DefinedEntity.Entity, "spec.vcdKe.forceDelete")
+		markForDelete = traverseMapAndGet[bool](rde.DefinedEntity.Entity, "spec.vcdKe.markForDelete", ".")
+		forceDelete = traverseMapAndGet[bool](rde.DefinedEntity.Entity, "spec.vcdKe.forceDelete", ".")
 
 		if !markForDelete || !forceDelete {
 			// Mark the cluster for deletion

diff --git a/govcd/cse/4.1/autoscaler.tmpl b/govcd/cse/4.1/autoscaler.tmpl
@@ -0,0 +1,197 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cluster-autoscaler
+  namespace: kube-system
+  labels:
+    app: cluster-autoscaler
+spec:
+  selector:
+    matchLabels:
+      app: cluster-autoscaler
+  replicas: {{.AutoscalerReplicas}}
+  template:
+    metadata:
+      labels:
+        app: cluster-autoscaler
+      annotations:
+        cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
+    spec:
+      serviceAccountName: cluster-autoscaler
+      containers:
+      - image: k8s.gcr.io/autoscaling/cluster-autoscaler:{{.AutoscalerVersion}}
+        name: cluster-autoscaler
+        resources:
+          limits:
+            cpu: 100m
+            memory: 500Mi
+          requests:
+            cpu: 100m
+            memory: 500Mi
+        command:
+        - /cluster-autoscaler
+        - --v=4
+        - --stderrthreshold=info
+        - --cloud-provider=clusterapi
+        - --expendable-pods-priority-cutoff=-10
+        - --scale-down-delay-after-delete=10s
+        - --scale-down-delay-after-add=10s
+        - --scale-down-delay-after-failure=10s
+        - --expander=least-waste
+        - --node-group-auto-discovery=clusterapi:namespace={{.TargetNamespace}}
+        - --balance-similar-node-groups
+        - --skip-nodes-with-system-pods=false
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cluster-autoscaler
+  namespace: kube-system
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: cluster-autoscaler
+rules:
+  - apiGroups:
+    - ""
+    resources:
+    - namespaces
+    - persistentvolumeclaims
+    - persistentvolumes
+    - pods
+    - replicationcontrollers
+    - services
+    verbs:
+    - get
+    - list
+    - watch
+  - apiGroups:
+    - ""
+    resources:
+    - nodes
+    verbs:
+    - get
+    - list
+    - update
+    - watch
+  - apiGroups:
+    - ""
+    resources:
+    - pods/eviction
+    verbs:
+    - create
+  - apiGroups:
+    - policy
+    resources:
+    - poddisruptionbudgets
+    verbs:
+    - list
+    - watch
+  - apiGroups:
+    - storage.k8s.io
+    resources:
+    - csinodes
+    - storageclasses
+    - csidrivers
+    - csistoragecapacities
+    verbs:
+    - get
+    - list
+    - watch
+  - apiGroups:
+    - batch
+    resources:
+    - jobs
+    verbs:
+    - list
+    - watch
+  - apiGroups:
+    - apps
+    resources:
+    - daemonsets
+    - replicasets
+    - statefulsets
+    verbs:
+    - list
+    - watch
+  - apiGroups:
+    - ""
+    resources:
+    - events
+    verbs:
+    - create
+    - patch
+  - apiGroups:
+    - ""
+    resources:
+    - configmaps
+    verbs:
+    - create
+    - delete
+    - get
+    - update
+  - apiGroups:
+    - coordination.k8s.io
+    resources:
+    - leases
+    verbs:
+    - create
+    - get
+    - update
+  - apiGroups:
+    - cluster.x-k8s.io
+    resources:
+    - machinedeployments
+    - machinedeployments/scale
+    - machines
+    - machinesets
+    - machinepools
+    verbs:
+    - get
+    - list
+    - update
+    - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: cluster-autoscaler
+  namespace: kube-system
+  labels:
+    k8s-addon: cluster-autoscaler.addons.k8s.io
+    k8s-app: cluster-autoscaler
+rules:
+  - apiGroups:
+    - ""
+    resources:
+    - configmaps
+    verbs:
+    - create
+    - list
+    - watch
+  - apiGroups:
+    - ""
+    resources:
+    - configmaps
+    resourceNames:
+    - cluster-autoscaler-status
+    - cluster-autoscaler-priority-expander
+    verbs:
+    - delete
+    - get
+    - update
+    - watch
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: cluster-autoscaler
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-autoscaler
+subjects:
+- kind: ServiceAccount
+  name: cluster-autoscaler
+  namespace: kube-system
diff --git a/govcd/cse/4.1/capiyaml_workerpool.tmpl b/govcd/cse/4.1/capiyaml_workerpool.tmpl
@@ -19,9 +19,16 @@ kind: MachineDeployment
 metadata:
   name: "{{.NodePoolName}}"
   namespace: "{{.TargetNamespace}}"
+  {{- if and .AutoscalerMaxSize .AutoscalerMinSize}}
+  annotations:
+    cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "{{.AutoscalerMaxSize}}"
+    cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "{{.AutoscalerMinSize}}"
+  {{- end}}
 spec:
   clusterName: "{{.ClusterName}}"
+  {{- if .NodePoolMachineCount}}
   replicas: {{.NodePoolMachineCount}}
+  {{- end}}
   selector:
     matchLabels: null
   template: