A generic cleanup utility for removing arbitrary files from nodes and/or resources from a K8s cluster.
This tool can be deployed as a DaemonSet/Job/Pod. Simply create your config files and apply it on your K8s cluster.
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: spectro-cleanup-role
namespace: kube-system
annotations:
"helm.sh/hook": pre-delete
labels:
app: {{ template "multus.name" . }}
{{- include "multus.labels" . | indent 4 }}
rules:
- apiGroups:
- ""
resources:
- configmaps
- serviceaccounts
verbs:
- '*'
- apiGroups:
- apps
resources:
- daemonsets
verbs:
- '*'
- apiGroups:
- rbac.authorization.k8s.io
resources:
- rolebindings
- roles
verbs:
- '*'
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: spectro-cleanup-rolebinding
namespace: kube-system
annotations:
"helm.sh/hook": pre-delete
labels:
app: {{ template "multus.name" . }}
{{- include "multus.labels" . | indent 4 }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: spectro-cleanup-role
subjects:
- kind: ServiceAccount
name: spectro-cleanup
namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: spectro-cleanup
namespace: kube-system
annotations:
"helm.sh/hook": pre-delete
labels:
app: {{ template "multus.name" . }}
{{- include "multus.labels" . | indent 4 }}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: spectro-cleanup-config
namespace: kube-system
annotations:
"helm.sh/hook": pre-delete
labels:
app: {{ template "multus.name" . }}
{{- include "multus.labels" . | indent 4 }}
data:
file-config.json: |-
[
"/host/etc/cni/net.d/00-multus.conf",
"/host/opt/cni/bin/multus"
]
resource-config.json: |-
[
{
"group": "",
"version": "v1",
"resource": "configmaps",
"name": "spectro-cleanup-config",
"namespace": "kube-system"
},
{
"group": "apps",
"version": "v1",
"resource": "daemonsets",
"name": "spectro-cleanup",
"namespace": "kube-system"
}
]
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: spectro-cleanup
namespace: kube-system
annotations:
"helm.sh/hook": pre-delete
labels:
name: spectro-cleanup
app: {{ template "multus.name" . }}
release: {{ .Release.Name }}
{{- include "multus.labels" . | indent 4 }}
spec:
selector:
matchLabels:
name: spectro-cleanup
template:
metadata:
labels:
name: spectro-cleanup
spec:
hostNetwork: true
nodeSelector:
kubernetes.io/arch: amd64
tolerations:
- operator: Exists
effect: NoSchedule
serviceAccountName: spectro-cleanup
containers:
- name: spectro-cleanup
image: gcr.io/spectro-images-public/release/spectro-cleanup:1.0.0
command: ["/cleanup"]
resources:
requests:
cpu: "10m"
memory: "25Mi"
limits:
cpu: "20m"
memory: "50Mi"
securityContext:
privileged: true
volumeMounts:
- name: spectro-cleanup-config
mountPath: /tmp/spectro-cleanup
- name: cni-bin
mountPath: /host/opt/cni/bin
- name: cni
mountPath: /host/etc/cni/net.d
volumes:
- name: spectro-cleanup-config
configMap:
name: spectro-cleanup-config
items:
- key: file-config.json
path: file-config.json
- key: resource-config.json
path: resource-config.json
- name: cni-bin
hostPath:
path: /opt/cni/bin
- name: cni
hostPath:
path: /etc/cni/net.dapiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: spectro-cleanup-role
namespace: kube-system
rules:
- apiGroups:
- ""
resources:
- configmaps
- serviceaccounts
verbs:
- '*'
- apiGroups:
- batch
resources:
- jobs
verbs:
- '*'
- apiGroups:
- rbac.authorization.k8s.io
resources:
- rolebindings
- roles
verbs:
- '*'
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: spectro-cleanup-rolebinding
namespace: kube-system
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: spectro-cleanup-role
subjects:
- kind: ServiceAccount
name: spectro-cleanup
namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: spectro-cleanup
namespace: kube-system
---
apiVersion: v1
kind: ConfigMap
metadata:
name: spectro-cleanup-config
namespace: kube-system
data:
# multus files we want to delete
file-config.json: |-
[
"/host/etc/cni/net.d/00-multus.conf",
"/host/opt/cni/bin/multus"
]
# spectro-cleanup resources we want to delete
resource-config.json: |-
[
{
"group": "",
"version": "v1",
"resource": "configmaps",
"name": "spectro-cleanup-config",
"namespace": "kube-system"
},
{
"group": "batch",
"version": "v1",
"resource": "jobs",
"name": "spectro-cleanup",
"namespace": "kube-system"
}
]
---
apiVersion: batch/v1
kind: Job
metadata:
name: spectro-cleanup
namespace: kube-system
spec:
template:
metadata:
labels:
name: spectro-cleanup
spec:
restartPolicy: Never
serviceAccountName: spectro-cleanup
containers:
- name: spectro-cleanup
image: gcr.io/spectro-images-public/release/spectro-cleanup:1.2.0
command: ["/cleanup"]
env:
- name: CLEANUP_DELAY_SECONDS
value: "10"
resources:
requests:
cpu: "10m"
memory: "25Mi"
limits:
cpu: "20m"
memory: "50Mi"
securityContext:
privileged: true
volumeMounts:
- name: spectro-cleanup-config
mountPath: /tmp/spectro-cleanup
- name: cni-bin
mountPath: /host/opt/cni/bin
- name: cni
mountPath: /host/etc/cni/net.d
volumes:
- name: spectro-cleanup-config
configMap:
name: spectro-cleanup-config
items:
- key: file-config.json
path: file-config.json
- key: resource-config.json
path: resource-config.json
- name: cni-bin
hostPath:
path: /opt/cni/bin
- name: cni
hostPath:
path: /etc/cni/net.dTo ensure that spectro-cleanup itself is cleaned up after its finished getting rid of your chosed files/resources on your cluster,
you'll need to ensure that the final objects in your resource-config.json are the spectro-cleanup configmaps and the daemonset/job/pod.
If there are any resources added to the resource-config.json after the two aformentioned spectro-cleanup resources, they will not be cleaned up.
You can also optionally configure a gRPC server to run as a part of spectro-cleanup. This server has a single endpoint, FinalizeCleanup.
When this server is configured, spectro-cleanup will be able to wait for a request that notifies it that it can finally clean itself up.
In this case, the CLEANUP_DELAY_SECONDS env var will have the fallback time to self destruct in the case that a request is never made to the FinalizeCleanup endpoint.
Below you can see an example of how to configure the gRPC server on your daemonset or job:
apiVersion: batch/v1
kind: Job
metadata:
name: validator-cleanup
annotations:
"helm.sh/hook": pre-delete
spec:
template:
metadata:
labels:
app: validator-cleanup-job
spec:
restartPolicy: Never
serviceAccountName: spectro-cleanup
containers:
- name: validator-cleanup
image: {{ required ".Values.cleanup.image is required!" .Values.cleanup.image }}
command: ["/cleanup"]
env:
- name: CLEANUP_DELAY_SECONDS
value: "300"
{{- if .Values.cleanup.grpcServerEnabled }}
- name: CLEANUP_GRPC_SERVER_ENABLED
value: "true"
- name: CLEANUP_GRPC_SERVER_PORT
value: {{ required ".Values.cleanup.port is required!" .Values.cleanup.port | toString | quote }}
{{- end }}
resources:
requests:
cpu: "10m"
memory: "50Mi"
limits:
cpu: "100m"
memory: "100Mi"
volumeMounts:
- name: validator-cleanup-config
mountPath: /tmp/spectro-cleanup
volumes:
- name: validator-cleanup-config
configMap:
name: validator-cleanup-config
items:
- key: resource-config.json
path: resource-config.json
The main things to note here are that all three of the CLEANUP_GRPC_SERVER_ENBALED, CLEANUP_GRPC_SERVER_PORT, and CLEANUP_DELAY_SECONDS env vars are set.
You can see more about how this configuration is setup in the validator repo.