Add clean up job to delete collector when collection is uninstalled

deploy/helm/sumologic/README.md

@@ -15,6 +15,7 @@ Parameter | Description | Default
 `image.pullPolicy` | Image pullPolicy for Sumo Logic docker container.  | `IfNotPresent`
 `nameOverride` | Used to override the Chart name. | `Nil`
 `sumologic.setupEnabled` | If enabled, a pre-install hook will create Collector and Sources in Sumo Logic. | `true`
+`sumologic.cleanUpEnabled` | If enabled, a pre-delete hook will destroy Collector in Sumo Logic. | `false`
 `sumologic.logs.enabled` | Set the enabled flag to false for disabling logs ingestion altogether. | `true`
 `sumologic.metrics.enabled` | Set the enabled flag to false for disabling metrics ingestion altogether. | `true`
 `sumologic.traces.enabled` | Set the enabled flag to true to enable tracing ingestion. _Tracing must be enabled for the account first. Please contact your Sumo representative for activation details_ | `false`

deploy/helm/sumologic/conf/cleanup/cleanup.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+
+# Fix URL to remove "v1" or "v1/"
+export SUMOLOGIC_BASE_URL=${SUMOLOGIC_BASE_URL%v1*}
+# Support proxy for terraform
+export HTTP_PROXY=${HTTP_PROXY:=""}
+export HTTPS_PROXY=${HTTPS_PROXY:=""}
+export NO_PROXY=${NO_PROXY:=""}
+
+cd /cleanup/ || exit 1
+
+terraform init
+
+# shellcheck disable=SC1083
+terraform import sumologic_collector.collector {{ template "terraform.collector.name" . }}
+# shellcheck disable=SC1083
+terraform import kubernetes_secret.sumologic_collection_secret {{ template "terraform.secret.fullname" . }}
+
+terraform destroy -auto-approve .
+
+# Cleanup env variables
+export SUMOLOGIC_BASE_URL=
+export SUMOLOGIC_ACCESSKEY=
+export SUMOLOGIC_ACCESSID=

deploy/helm/sumologic/conf/setup/resources.tf

@@ -22,7 +22,7 @@ resource "sumologic_collector" "collector" {
 
 resource "kubernetes_secret" "sumologic_collection_secret" {
   metadata {
-    name = "sumologic"
+    name = "{{ template "terraform.secret.name" }}"
     namespace = var.namespace_name
   }
 

deploy/helm/sumologic/conf/setup/setup.sh

@@ -29,8 +29,6 @@ function should_create_fields() {
 cp /etc/terraform/{locals,main,providers,resources,variables,fields}.tf /terraform/
 cd /terraform
 
-COLLECTOR_NAME="{{- if .Values.sumologic.collectorName }}{{ .Values.sumologic.collectorName }}{{- else}}{{ .Values.sumologic.clusterName }}{{- end}}"
-
 terraform init
 
 # Sumo Logic fields
@@ -59,22 +57,24 @@ else
     echo "Please refer to https://help.sumologic.com/Manage/Fields to manually create the fields after you have removed unused fields to free up capacity."
 fi
 
+readonly COLLECTOR_NAME="{{ template "terraform.collector.name" . }}"
+
 # Sumo Logic Collector and HTTP sources
-terraform import sumologic_collector.collector "$COLLECTOR_NAME"
+terraform import sumologic_collector.collector "${COLLECTOR_NAME}"
 
 {{- $ctx := .Values -}}
 {{- range $type, $sources := .Values.sumologic.sources }}
 {{- if eq (include "terraform.sources.component_enabled" (dict "Context" $ctx "Type" $type)) "true" }}
 {{- range $key, $source := $sources }}
 {{- if eq (include "terraform.sources.to_create" (dict "Context" $ctx "Type" $type "Name" $key)) "true" }}
-terraform import sumologic_http_source.{{ template "terraform.sources.name" (dict "Name" $key "Type" $type) }} "$COLLECTOR_NAME/{{ $source.name }}"
+terraform import sumologic_http_source.{{ template "terraform.sources.name" (dict "Name" $key "Type" $type) }} "${COLLECTOR_NAME}/{{ $source.name }}"
 {{- end }}
 {{- end }}
 {{- end }}
 {{- end }}
 
 # Kubernetes Secret
-terraform import kubernetes_secret.sumologic_collection_secret {{ .Release.Namespace }}/sumologic
+terraform import kubernetes_secret.sumologic_collection_secret {{ template "terraform.secret.fullname" . }}
 
 # Apply planned changes
 terraform apply -auto-approve \

deploy/helm/sumologic/templates/_helpers.tpl

@@ -179,8 +179,30 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
 {{- template "sumologic.fullname" . }}-scc
 {{- end -}}
 
+{{- define "sumologic.labels.app.cleanup" -}}
+{{- template "sumologic.labels.app" . }}
+{{- end -}}
+
+{{- define "sumologic.labels.app.cleanup.configmap" -}}
+{{- template "sumologic.labels.app.cleanup" . }}
+{{- end -}}
+
+{{/*
+Generate cleanup job helm.sh annotations. It takes weight as parameter.
+
+Example usage:
+
+{{ include "sumologic.annotations.app.cleanup.helmsh" "1" }}
+
+*/}}
+{{- define "sumologic.annotations.app.cleanup.helmsh" -}}
+helm.sh/hook: pre-delete
+helm.sh/hook-weight: {{ printf "\"%s\"" . }}
+helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+{{- end -}}
+
 {{/*
-Generate helm.sh annotations. It takes weight as parameter.
+Generate setup job helm.sh annotations. It takes weight as parameter.
 
 Example usage:
 
@@ -337,6 +359,14 @@ helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
 {{- template "sumologic.metadata.name.setup" . }}-scc
 {{- end -}}
 
+{{- define "sumologic.metadata.name.cleanup" -}}
+{{ template "sumologic.fullname" . }}-cleanup
+{{- end -}}
+
+{{- define "sumologic.metadata.name.cleanup.configmap" -}}
+{{ template "sumologic.metadata.name.cleanup" . }}
+{{- end -}}
+
 {{- define "sumologic.labels.logs" -}}
 sumologic.com/app: fluentd-logs
 sumologic.com/component: logs
@@ -499,6 +529,30 @@ Example:
 </store>
 {{- end -}}
 
+{{/*
+Returns the name of kubernetes secret.
+
+Example usage:
+
+{{ include "terraform.secret.name" }}
+
+*/}}
+{{- define "terraform.secret.name" -}}
+{{ printf "%s" "sumologic" }}
+{{- end -}}
+
+{{/*
+Returns the name of kubernetes secret prefixed with release namespace.
+
+Example usage:
+
+{{ include "terraform.secret.fullname" }}
+
+*/}}
+{{- define "terraform.secret.fullname" -}}
+{{ .Release.Namespace }}/{{ template "terraform.secret.name" . }}
+{{- end -}}
+
 {{/*
 Convert source name to terraform metric name:
  * converts all `-` to `_`
@@ -562,6 +616,18 @@ Example usage:
 {{ printf "%-41s = sumologic_http_source.%s.url" .Endpoint .Name }}
 {{- end -}}
 
+{{/*
+Returns the collector name.
+
+Example usage:
+
+{{ include "terraform.collector.name" . }}
+
+*/}}
+{{- define "terraform.collector.name" -}}
+{{- if .Values.sumologic.collectorName }}{{ .Values.sumologic.collectorName }}{{- else}}{{ .Values.sumologic.clusterName }}{{- end}}
+{{- end -}}
+
 {{/*
 Generate resource sections
 

deploy/helm/sumologic/templates/cleanup/cleanup-configmap.yaml

@@ -0,0 +1,14 @@
+{{- if .Values.sumologic.cleanUpEnabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "sumologic.metadata.name.cleanup.configmap" . }}
+  annotations:
+{{ include "sumologic.annotations.app.cleanup.helmsh" "0" | indent 4 }}
+  labels:
+    app: {{ template "sumologic.labels.app.cleanup.configmap" . }}
+    {{- include "sumologic.labels.common" . | nindent 4 }}
+data:
+  {{- (tpl (.Files.Glob "conf/cleanup/cleanup.sh").AsConfig .) | nindent 2 }}
+  {{- (tpl (.Files.Glob "conf/setup/{locals,main,providers,resources,variables}.tf").AsConfig .) | nindent 2 }}
+{{- end }}

deploy/helm/sumologic/templates/cleanup/cleanup-job.yaml

@@ -0,0 +1,70 @@
+{{- if .Values.sumologic.cleanUpEnabled }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ template "sumologic.metadata.name.cleanup" . }}
+  annotations:
+{{ include "sumologic.annotations.app.cleanup.helmsh" "0" | indent 4 }}
+  labels:
+    app: {{ template "sumologic.metadata.name.cleanup" . }}
+    {{- include "sumologic.labels.common" . | nindent 4 }}
+spec:
+  template:
+    metadata:
+      annotations:
+{{- if .Values.sumologic.podAnnotations }}
+{{ toYaml .Values.sumologic.podAnnotations | indent 8 }}
+{{- end }}
+      labels:
+{{- if .Values.sumologic.podLabels }}
+{{ toYaml .Values.sumologic.podLabels | indent 8 }}
+{{- end }}
+    spec:
+      restartPolicy: OnFailure
+      serviceAccountName: {{ template "sumologic.metadata.name.roles.serviceaccount" . }}
+      volumes:
+      - name: configmap
+        configMap:
+          name: {{ template "sumologic.metadata.name.cleanup.configmap" . }}
+          defaultMode: 0777
+      - name: cleanup
+        emptyDir: {}
+      initContainers:
+      - name: copy-files
+        image: busybox
+        command: ['sh', '-c', 'cp /configmap/* /cleanup']
+        volumeMounts:
+          - name: configmap
+            mountPath: /configmap
+          - name: cleanup
+            mountPath: /cleanup
+      containers:
+        - name: cleanup
+          image: "hashicorp/terraform:0.12.29"
+          imagePullPolicy: IfNotPresent
+          volumeMounts:
+          - name: cleanup
+            mountPath: /cleanup
+          command: ["/cleanup/cleanup.sh"]
+          {{- if .Values.sumologic.envFromSecret }}
+          envFrom:
+          - secretRef:
+              name: {{ .Values.sumologic.envFromSecret }}
+          {{ else }}
+          env:
+          - name: SUMOLOGIC_ACCESSID
+            value: {{ required "A valid .Values.sumologic.accessId entry required!" .Values.sumologic.accessId }}
+          - name: SUMOLOGIC_ACCESSKEY
+            value: {{ required "A valid .Values.sumologic.accessKey entry required!" .Values.sumologic.accessKey }}
+          - name: SUMOLOGIC_BASE_URL
+            value: {{ .Values.sumologic.endpoint }}
+          - name: HTTP_PROXY
+            value: {{ .Values.sumologic.httpProxy }}
+          - name: HTTPS_PROXY
+            value: {{ .Values.sumologic.httpsProxy }}
+          - name: NO_PROXY
+            value: {{ .Values.sumologic.noProxy }}
+          {{ end }}
+      securityContext:
+        runAsUser: 999
+{{- end }}

deploy/helm/sumologic/templates/setup/setup-job.yaml

@@ -3,7 +3,6 @@ apiVersion: batch/v1
 kind: Job
 metadata:
   name: {{ template "sumologic.metadata.name.setup.job" . }}
-  namespace: {{ .Release.Namespace }}
   annotations:
 {{ include "sumologic.annotations.app.setup.helmsh" "3" | indent 4 }}
   labels:

deploy/helm/sumologic/values.yaml

@@ -14,6 +14,9 @@ sumologic:
   ## If enabled, a pre-install hook will create Collector and Sources in Sumo Logic
   setupEnabled: true
 
+  ## If enabled, a pre-delete hook will destroy Collector in Sumo Logic
+  cleanUpEnabled: false
+
   ## If enabled, accessId and accessKey will be sourced from Secret Name given
   ## Be sure to include at least the following env variables in your secret
   ## (1) SUMOLOGIC_ACCESSID, (2) SUMOLOGIC_ACCESSKEY

tests/terraform/static/all_fields.output.yaml

@@ -295,8 +295,6 @@ data:
     cp /etc/terraform/{locals,main,providers,resources,variables,fields}.tf /terraform/
     cd /terraform
 
-    COLLECTOR_NAME="kubernetes"
-
     terraform init
 
     # Sumo Logic fields
@@ -325,19 +323,21 @@ data:
         echo "Please refer to https://help.sumologic.com/Manage/Fields to manually create the fields after you have removed unused fields to free up capacity."
     fi
 
+    readonly COLLECTOR_NAME="kubernetes"
+
     # Sumo Logic Collector and HTTP sources
-    terraform import sumologic_collector.collector "$COLLECTOR_NAME"
-    terraform import sumologic_http_source.default_events_source "$COLLECTOR_NAME/events"
-    terraform import sumologic_http_source.default_logs_source "$COLLECTOR_NAME/logs"
-    terraform import sumologic_http_source.apiserver_metrics_source "$COLLECTOR_NAME/apiserver-metrics"
-    terraform import sumologic_http_source.control_plane_metrics_source "$COLLECTOR_NAME/control-plane-metrics"
-    terraform import sumologic_http_source.controller_metrics_source "$COLLECTOR_NAME/kube-controller-manager-metrics"
-    terraform import sumologic_http_source.default_metrics_source "$COLLECTOR_NAME/(default-metrics)"
-    terraform import sumologic_http_source.kubelet_metrics_source "$COLLECTOR_NAME/kubelet-metrics"
-    terraform import sumologic_http_source.node_metrics_source "$COLLECTOR_NAME/node-exporter-metrics"
-    terraform import sumologic_http_source.scheduler_metrics_source "$COLLECTOR_NAME/kube-scheduler-metrics"
-    terraform import sumologic_http_source.state_metrics_source "$COLLECTOR_NAME/kube-state-metrics"
-    terraform import sumologic_http_source.test_source_metrics_source "$COLLECTOR_NAME/(Test source)"
+    terraform import sumologic_collector.collector "${COLLECTOR_NAME}"
+    terraform import sumologic_http_source.default_events_source "${COLLECTOR_NAME}/events"
+    terraform import sumologic_http_source.default_logs_source "${COLLECTOR_NAME}/logs"
+    terraform import sumologic_http_source.apiserver_metrics_source "${COLLECTOR_NAME}/apiserver-metrics"
+    terraform import sumologic_http_source.control_plane_metrics_source "${COLLECTOR_NAME}/control-plane-metrics"
+    terraform import sumologic_http_source.controller_metrics_source "${COLLECTOR_NAME}/kube-controller-manager-metrics"
+    terraform import sumologic_http_source.default_metrics_source "${COLLECTOR_NAME}/(default-metrics)"
+    terraform import sumologic_http_source.kubelet_metrics_source "${COLLECTOR_NAME}/kubelet-metrics"
+    terraform import sumologic_http_source.node_metrics_source "${COLLECTOR_NAME}/node-exporter-metrics"
+    terraform import sumologic_http_source.scheduler_metrics_source "${COLLECTOR_NAME}/kube-scheduler-metrics"
+    terraform import sumologic_http_source.state_metrics_source "${COLLECTOR_NAME}/kube-state-metrics"
+    terraform import sumologic_http_source.test_source_metrics_source "${COLLECTOR_NAME}/(Test source)"
 
     # Kubernetes Secret
     terraform import kubernetes_secret.sumologic_collection_secret sumologic/sumologic

tests/terraform/static/collector_fields.output.yaml

@@ -250,8 +250,6 @@ data:
     cp /etc/terraform/{locals,main,providers,resources,variables,fields}.tf /terraform/
     cd /terraform
 
-    COLLECTOR_NAME="kubernetes"
-
     terraform init
 
     # Sumo Logic fields
@@ -280,18 +278,20 @@ data:
         echo "Please refer to https://help.sumologic.com/Manage/Fields to manually create the fields after you have removed unused fields to free up capacity."
     fi
 
+    readonly COLLECTOR_NAME="kubernetes"
+
     # Sumo Logic Collector and HTTP sources
-    terraform import sumologic_collector.collector "$COLLECTOR_NAME"
-    terraform import sumologic_http_source.default_events_source "$COLLECTOR_NAME/events"
-    terraform import sumologic_http_source.default_logs_source "$COLLECTOR_NAME/logs"
-    terraform import sumologic_http_source.apiserver_metrics_source "$COLLECTOR_NAME/apiserver-metrics"
-    terraform import sumologic_http_source.control_plane_metrics_source "$COLLECTOR_NAME/control-plane-metrics"
-    terraform import sumologic_http_source.controller_metrics_source "$COLLECTOR_NAME/kube-controller-manager-metrics"
-    terraform import sumologic_http_source.default_metrics_source "$COLLECTOR_NAME/(default-metrics)"
-    terraform import sumologic_http_source.kubelet_metrics_source "$COLLECTOR_NAME/kubelet-metrics"
-    terraform import sumologic_http_source.node_metrics_source "$COLLECTOR_NAME/node-exporter-metrics"
-    terraform import sumologic_http_source.scheduler_metrics_source "$COLLECTOR_NAME/kube-scheduler-metrics"
-    terraform import sumologic_http_source.state_metrics_source "$COLLECTOR_NAME/kube-state-metrics"
+    terraform import sumologic_collector.collector "${COLLECTOR_NAME}"
+    terraform import sumologic_http_source.default_events_source "${COLLECTOR_NAME}/events"
+    terraform import sumologic_http_source.default_logs_source "${COLLECTOR_NAME}/logs"
+    terraform import sumologic_http_source.apiserver_metrics_source "${COLLECTOR_NAME}/apiserver-metrics"
+    terraform import sumologic_http_source.control_plane_metrics_source "${COLLECTOR_NAME}/control-plane-metrics"
+    terraform import sumologic_http_source.controller_metrics_source "${COLLECTOR_NAME}/kube-controller-manager-metrics"
+    terraform import sumologic_http_source.default_metrics_source "${COLLECTOR_NAME}/(default-metrics)"
+    terraform import sumologic_http_source.kubelet_metrics_source "${COLLECTOR_NAME}/kubelet-metrics"
+    terraform import sumologic_http_source.node_metrics_source "${COLLECTOR_NAME}/node-exporter-metrics"
+    terraform import sumologic_http_source.scheduler_metrics_source "${COLLECTOR_NAME}/kube-scheduler-metrics"
+    terraform import sumologic_http_source.state_metrics_source "${COLLECTOR_NAME}/kube-state-metrics"
 
     # Kubernetes Secret
     terraform import kubernetes_secret.sumologic_collection_secret sumologic/sumologic

tests/terraform/static/conditional_sources.output.yaml

@@ -177,8 +177,6 @@ data:
     cp /etc/terraform/{locals,main,providers,resources,variables,fields}.tf /terraform/
     cd /terraform
 
-    COLLECTOR_NAME="kubernetes"
-
     terraform init
 
     # Sumo Logic fields
@@ -207,8 +205,10 @@ data:
         echo "Please refer to https://help.sumologic.com/Manage/Fields to manually create the fields after you have removed unused fields to free up capacity."
     fi
 
+    readonly COLLECTOR_NAME="kubernetes"
+
     # Sumo Logic Collector and HTTP sources
-    terraform import sumologic_collector.collector "$COLLECTOR_NAME"
+    terraform import sumologic_collector.collector "${COLLECTOR_NAME}"
 
     # Kubernetes Secret
     terraform import kubernetes_secret.sumologic_collection_secret sumologic/sumologic