feat: add keep-crd upgrade hook

The crds have been moved from the templates/ to crds/ folder. When helm upgrade is run, helm will delete the crds because they're no longer in the generated template. To prevent deletion, we patch the 2 CRDs with the "helm.sh/resource-policy": "keep". Helm will skip deletion of resources with these annotation. Also, converted the hooks to a job as helm kills the pod before kubectl exec is run. With job, helm waits until the pod is run and reaches completion. Signed-off-by: Anish Ramasekar <anish.ramasekar@gmail.com>
kubernetes-sigs · Jul 23, 2021 · 7d4d5ce · 7d4d5ce
1 parent 53b054f
commit 7d4d5ce
Show file tree

Hide file tree

Showing 2 changed files with 103 additions and 24 deletions.
diff --git a/manifest_staging/charts/secrets-store-csi-driver/templates/crds-upgrade-hook.yaml b/manifest_staging/charts/secrets-store-csi-driver/templates/crds-upgrade-hook.yaml
@@ -5,7 +5,8 @@ metadata:
 {{ include "sscd.labels" . | indent 2 }}
   annotations:
     helm.sh/hook: pre-install,pre-upgrade
-    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+    helm.sh/hook-delete-policy: "hook-succeeded,before-hook-creation"
+    helm.sh/hook-weight: "1"
 rules:
   - apiGroups: ["apiextensions.k8s.io"]
     resources: ["customresourcedefinitions"]
@@ -18,7 +19,8 @@ metadata:
 {{ include "sscd.labels" . | indent 2 }}
   annotations:
     helm.sh/hook: pre-install,pre-upgrade
-    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+    helm.sh/hook-delete-policy: "hook-succeeded,before-hook-creation"
+    helm.sh/hook-weight: "1"
 subjects:
   - kind: ServiceAccount
     name: {{ template "sscd.fullname" . }}-upgrade-crds
@@ -36,31 +38,34 @@ metadata:
 {{ include "sscd.labels" . | indent 2 }}
   annotations:
     helm.sh/hook: pre-install,pre-upgrade
-    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+    helm.sh/hook-delete-policy: "hook-succeeded,before-hook-creation"
+    helm.sh/hook-weight: "1"
 ---
-apiVersion: v1
-kind: Pod
+apiVersion: batch/v1
+kind: Job
 metadata:
   name: {{ template "sscd.fullname" . }}-upgrade-crds
   namespace: {{ .Release.Namespace }}
-  annotations:
-    helm.sh/hook: pre-install,pre-upgrade
-    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
 {{ include "sscd.labels" . | indent 2 }}
+  annotations:
+    helm.sh/hook: pre-upgrade
+    helm.sh/hook-weight: "1"
+    helm.sh/hook-delete-policy: "hook-succeeded,before-hook-creation"
 spec:
-  serviceAccountName: {{ template "sscd.fullname" . }}-upgrade-crds
-  restartPolicy: OnFailure
-  containers:
-    - name: crds-upgrade
-      image: "{{ .Values.linux.crds.image.repository }}:{{ .Values.linux.crds.image.tag }}"
-      args:
-      - apply
-      - -f
-      - crds/
-      imagePullPolicy: {{ .Values.linux.crds.image.pullPolicy }}
-  {{- if .Values.imagePullSecrets }}
-  imagePullSecrets:
-    {{ toYaml .Values.imagePullSecrets | indent 4 }}
-  {{- end }}
-  nodeSelector:
-    kubernetes.io/os: linux
+  backoffLimit: 0
+  template:
+    metadata:
+      name: crds-keep
+    spec:
+      serviceAccountName: {{ template "sscd.fullname" . }}-upgrade-crds
+      restartPolicy: Never
+      containers:
+      - name: crds-upgrade
+        image: "{{ .Values.linux.crds.image.repository }}:{{ .Values.linux.crds.image.tag }}"
+        args:
+        - apply
+        - -f
+        - crds/
+        imagePullPolicy: {{ .Values.linux.crds.image.pullPolicy }}
+      nodeSelector:
+        kubernetes.io/os: linux
diff --git a/manifest_staging/charts/secrets-store-csi-driver/templates/keep-crds-upgrade-hook.yaml b/manifest_staging/charts/secrets-store-csi-driver/templates/keep-crds-upgrade-hook.yaml
@@ -0,0 +1,74 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ template "sscd.fullname" . }}-keep-crds
+{{ include "sscd.labels" . | indent 2 }}
+  annotations:
+    helm.sh/hook: pre-upgrade
+    helm.sh/hook-delete-policy: "hook-succeeded,before-hook-creation"
+    helm.sh/hook-weight: "2"
+rules:
+  - apiGroups: ["apiextensions.k8s.io"]
+    resources: ["customresourcedefinitions"]
+    verbs: ["get", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ template "sscd.fullname" . }}-keep-crds
+{{ include "sscd.labels" . | indent 2 }}
+  annotations:
+    helm.sh/hook: pre-upgrade
+    helm.sh/hook-delete-policy: "hook-succeeded,before-hook-creation"
+    helm.sh/hook-weight: "2"
+subjects:
+  - kind: ServiceAccount
+    name: {{ template "sscd.fullname" . }}-keep-crds
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: {{ template "sscd.fullname" . }}-keep-crds
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ template "sscd.fullname" . }}-keep-crds
+  namespace: {{ .Release.Namespace }}
+{{ include "sscd.labels" . | indent 2 }}
+  annotations:
+    helm.sh/hook: pre-upgrade
+    helm.sh/hook-delete-policy: "hook-succeeded,before-hook-creation"
+    helm.sh/hook-weight: "2"
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ template "sscd.fullname" . }}-keep-crds
+  namespace: {{ .Release.Namespace }}
+{{ include "sscd.labels" . | indent 2 }}
+  annotations:
+    helm.sh/hook: pre-upgrade
+    helm.sh/hook-weight: "2"
+    helm.sh/hook-delete-policy: "hook-succeeded,before-hook-creation"
+spec:
+  backoffLimit: 0
+  template:
+    metadata:
+      name: crds-keep
+    spec:
+      serviceAccountName: {{ template "sscd.fullname" . }}-keep-crds
+      restartPolicy: Never
+      containers:
+      - name: crds-keep
+        image: "{{ .Values.linux.crds.image.repository }}:{{ .Values.linux.crds.image.tag }}"
+        args:
+        - patch
+        - crd
+        - secretproviderclasses.secrets-store.csi.x-k8s.io
+        - secretproviderclasspodstatuses.secrets-store.csi.x-k8s.io
+        - -p
+        - '{"metadata":{"annotations": {"helm.sh/resource-policy": "keep"}}}'
+        imagePullPolicy: {{ .Values.linux.crds.image.pullPolicy }}
+      nodeSelector:
+        kubernetes.io/os: linux