Add support for data persistence

deploy/helm/sumologic/conf/buffer.output.conf

@@ -2,4 +2,4 @@ compress gzip
 flush_interval "#{ENV['FLUSH_INTERVAL']}"
 flush_thread_count "#{ENV['NUM_THREADS']}"
 chunk_limit_size "#{ENV['CHUNK_LIMIT_SIZE']}"
-total_limit_size "#{ENV['TOTAL_LIMIT_SIZE']}"
+overflow_action drop_oldest_chunk

deploy/helm/sumologic/conf/events/events.conf

@@ -35,11 +35,12 @@
   verify_ssl "#{ENV['VERIFY_SSL']}"
   proxy_uri "#{ENV['PROXY_URI']}"
   <buffer>
-    {{- if eq .Values.sumologic.fluentd.buffer "file" }}
+    {{- if or .Values.sumologic.fluentd.persistence.enabled (eq .Values.sumologic.fluentd.buffer "file") }}
     @type file
     path /fluentd/buffer/events
     {{- else }}
     @type memory
+    total_limit_size "#{ENV['TOTAL_LIMIT_SIZE']}"
     {{- end }}
     @include buffer.output.conf
   </buffer>

deploy/helm/sumologic/conf/logs/logs.source.containers.conf

@@ -56,11 +56,12 @@
     @id sumologic.endpoint.logs
     @include logs.output.conf
     <buffer>
-      {{- if eq .Values.sumologic.fluentd.buffer "file" }}
+      {{- if or .Values.sumologic.fluentd.persistence.enabled (eq .Values.sumologic.fluentd.buffer "file") }}
       @type file
       path /fluentd/buffer/logs.containers
       {{- else }}
       @type memory
+      total_limit_size "#{ENV['TOTAL_LIMIT_SIZE']}"
       {{- end }}
       @include buffer.output.conf
     </buffer>

deploy/helm/sumologic/conf/logs/logs.source.systemd.conf

@@ -18,11 +18,12 @@
     @id sumologic.endpoint.logs.kubelet
     @include logs.output.conf
     <buffer>
-      {{- if eq .Values.sumologic.fluentd.buffer "file" }}
+      {{- if or .Values.sumologic.fluentd.persistence.enabled (eq .Values.sumologic.fluentd.buffer "file") }}
       @type file
       path /fluentd/buffer/logs.kubelet
       {{- else }}
       @type memory
+      total_limit_size "#{ENV['TOTAL_LIMIT_SIZE']}"
       {{- end }}
       @include buffer.output.conf
     </buffer>
@@ -53,11 +54,12 @@
     @id sumologic.endpoint.logs.systemd
     @include logs.output.conf
     <buffer>
-      {{- if eq .Values.sumologic.fluentd.buffer "file" }}
+      {{- if or .Values.sumologic.fluentd.persistence.enabled (eq .Values.sumologic.fluentd.buffer "file") }}
       @type file
       path /fluentd/buffer/logs.systemd
       {{- else }}
       @type memory
+      total_limit_size "#{ENV['TOTAL_LIMIT_SIZE']}"
       {{- end }}
       @include buffer.output.conf
     </buffer>

deploy/helm/sumologic/conf/metrics/metrics.conf

@@ -23,11 +23,12 @@
     endpoint "#{ENV['SUMO_ENDPOINT_METRICS_APISERVER']}"
     @include metrics.output.conf
     <buffer>
-      {{- if eq .Values.sumologic.fluentd.buffer "file" }}
+      {{- if or .Values.sumologic.fluentd.persistence.enabled (eq .Values.sumologic.fluentd.buffer "file") }}
       @type file
       path /fluentd/buffer/metrics.apiserver
       {{- else }}
       @type memory
+      total_limit_size "#{ENV['TOTAL_LIMIT_SIZE']}"
       {{- end }}
       @include buffer.output.conf
     </buffer>
@@ -39,11 +40,12 @@
     endpoint "#{ENV['SUMO_ENDPOINT_METRICS_KUBELET']}"
     @include metrics.output.conf
     <buffer>
-      {{- if eq .Values.sumologic.fluentd.buffer "file" }}
+      {{- if or .Values.sumologic.fluentd.persistence.enabled (eq .Values.sumologic.fluentd.buffer "file") }}
       @type file
       path /fluentd/buffer/metrics.kubelet
       {{- else }}
       @type memory
+      total_limit_size "#{ENV['TOTAL_LIMIT_SIZE']}"
       {{- end }}
       @include buffer.output.conf
     </buffer>
@@ -54,11 +56,12 @@
     endpoint "#{ENV['SUMO_ENDPOINT_METRICS_KUBELET']}"
     @include metrics.output.conf
     <buffer>
-      {{- if eq .Values.sumologic.fluentd.buffer "file" }}
+      {{- if or .Values.sumologic.fluentd.persistence.enabled (eq .Values.sumologic.fluentd.buffer "file") }}
       @type file
       path /fluentd/buffer/metrics.container
       {{- else }}
       @type memory
+      total_limit_size "#{ENV['TOTAL_LIMIT_SIZE']}"
       {{- end }}
       @include buffer.output.conf
     </buffer>
@@ -69,11 +72,12 @@
     endpoint "#{ENV['SUMO_ENDPOINT_METRICS_KUBE_CONTROLLER_MANAGER']}"
     @include metrics.output.conf
     <buffer>
-      {{- if eq .Values.sumologic.fluentd.buffer "file" }}
+      {{- if or .Values.sumologic.fluentd.persistence.enabled (eq .Values.sumologic.fluentd.buffer "file") }}
       @type file
       path /fluentd/buffer/metrics.controller
       {{- else }}
       @type memory
+      total_limit_size "#{ENV['TOTAL_LIMIT_SIZE']}"
       {{- end }}
       @include buffer.output.conf
     </buffer>
@@ -84,11 +88,12 @@
     endpoint "#{ENV['SUMO_ENDPOINT_METRICS_KUBE_SCHEDULER']}"
     @include metrics.output.conf
     <buffer>
-      {{- if eq .Values.sumologic.fluentd.buffer "file" }}
+      {{- if or .Values.sumologic.fluentd.persistence.enabled (eq .Values.sumologic.fluentd.buffer "file") }}
       @type file
       path /fluentd/buffer/metrics.scheduler
       {{- else }}
       @type memory
+      total_limit_size "#{ENV['TOTAL_LIMIT_SIZE']}"
       {{- end }}
       @include buffer.output.conf
     </buffer>
@@ -99,11 +104,12 @@
     endpoint "#{ENV['SUMO_ENDPOINT_METRICS_KUBE_STATE']}"
     @include metrics.output.conf
     <buffer>
-      {{- if eq .Values.sumologic.fluentd.buffer "file" }}
+      {{- if or .Values.sumologic.fluentd.persistence.enabled (eq .Values.sumologic.fluentd.buffer "file") }}
       @type file
       path /fluentd/buffer/metrics.state
       {{- else }}
       @type memory
+      total_limit_size "#{ENV['TOTAL_LIMIT_SIZE']}"
       {{- end }}
       @include buffer.output.conf
     </buffer>
@@ -114,11 +120,12 @@
     endpoint "#{ENV['SUMO_ENDPOINT_METRICS_NODE_EXPORTER']}"
     @include metrics.output.conf
     <buffer>
-      {{- if eq .Values.sumologic.fluentd.buffer "file" }}
+      {{- if or .Values.sumologic.fluentd.persistence.enabled (eq .Values.sumologic.fluentd.buffer "file") }}
       @type file
       path /fluentd/buffer/metrics.node
       {{- else }}
       @type memory
+      total_limit_size "#{ENV['TOTAL_LIMIT_SIZE']}"
       {{- end }}
       @include buffer.output.conf
     </buffer>
@@ -129,11 +136,12 @@
     endpoint "#{ENV['SUMO_ENDPOINT_METRICS']}"
     @include metrics.output.conf
     <buffer>
-      {{- if eq .Values.sumologic.fluentd.buffer "file" }}
+      {{- if or .Values.sumologic.fluentd.persistence.enabled (eq .Values.sumologic.fluentd.buffer "file") }}
       @type file
       path /fluentd/buffer/metrics.all
       {{- else }}
       @type memory
+      total_limit_size "#{ENV['TOTAL_LIMIT_SIZE']}"
       {{- end }}
       @include buffer.output.conf
     </buffer>

deploy/helm/sumologic/templates/events-deployment.yaml → deploy/helm/sumologic/templates/events-statefulset.yaml

@@ -1,6 +1,6 @@
 {{- if eq .Values.sumologic.eventCollectionEnabled true }}
 apiVersion: apps/v1
-kind: Deployment
+kind: StatefulSet
 metadata:
   name: {{ printf "%s-events" (include "sumologic.fullname" .) }}
   labels:
@@ -17,13 +17,13 @@ spec:
         {{- include "sumologic.labels.common" . | nindent 8 }}
     spec:
       serviceAccountName: {{ template "sumologic.fullname" . }}
-{{- if .Values.eventsDeployment.nodeSelector }}
+{{- if .Values.eventsStatefulset.nodeSelector }}
       nodeSelector:
-{{ toYaml .Values.eventsDeployment.nodeSelector | indent 8 }}
+{{ toYaml .Values.eventsStatefulset.nodeSelector | indent 8 }}
 {{- end }}
-{{- if .Values.eventsDeployment.tolerations }}
+{{- if .Values.eventsStatefulset.tolerations }}
       tolerations:
-{{ toYaml .Values.eventsDeployment.tolerations | indent 8 }}
+{{ toYaml .Values.eventsStatefulset.tolerations | indent 8 }}
 {{- end }}
       volumes:
       - name: pos-files
@@ -33,17 +33,23 @@ spec:
       - name: config-volume
         configMap:
           name: {{ printf "%s-events" (include "sumologic.fullname" .) }}
+      securityContext:
+        fsGroup: {{ .Values.sumologic.fluentd.securityContext.fsGroup }}
       containers:
       - name: fluentd-events
         image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
         imagePullPolicy: {{ .Values.image.pullPolicy }}
         resources:
-          {{- toYaml .Values.eventsDeployment.resources | nindent 10 }}
+          {{- toYaml .Values.eventsStatefulset.resources | nindent 10 }}
         volumeMounts:
         - name: config-volume
           mountPath: /fluentd/etc/
         - name: pos-files
           mountPath: /mnt/pos/
+{{- if .Values.sumologic.fluentd.persistence.enabled }}
+        - name: buffer
+          mountPath: "/fluentd/buffer"
+{{- end}}
         livenessProbe:
           httpGet:
             path: /fluentd.pod.healthcheck?json=%7B%22log%22%3A+%22health+check%22%7D
@@ -73,4 +79,15 @@ spec:
           value: {{ .Values.sumologic.chunkLimitSize | quote }}
         - name: TOTAL_LIMIT_SIZE
           value: {{ .Values.sumologic.totalLimitSize | quote }}
-{{- end }}
+{{- if .Values.sumologic.fluentd.persistence.enabled }}
+  volumeClaimTemplates:
+  - metadata:
+      name: buffer
+    spec:
+      accessModes: [{{ .Values.sumologic.fluentd.persistence.accessMode }}]
+      storageClassName: {{ .Values.sumologic.fluentd.persistence.storageClass }}
+      resources:
+        requests:
+          storage: {{ .Values.sumologic.fluentd.persistence.size }}
+{{- end }}
+{{- end }}

deploy/helm/sumologic/templates/deployment.yaml → deploy/helm/sumologic/templates/statefulset.yaml

@@ -1,5 +1,5 @@
 apiVersion: apps/v1
-kind: Deployment
+kind: StatefulSet
 metadata:
   name: {{ template "sumologic.fullname" . }}
   labels:
@@ -9,21 +9,21 @@ spec:
   selector:
     matchLabels:
       app: {{ template "sumologic.labels.app" . }}
-  replicas: {{ .Values.deployment.replicaCount }}
+  replicas: {{ .Values.statefulset.replicaCount }}
   template:
     metadata:
       labels:
         app: {{ template "sumologic.labels.app" . }}
         {{- include "sumologic.labels.common" . | nindent 8 }}
     spec:
       serviceAccountName: {{ template "sumologic.fullname" . }}
-{{- if .Values.deployment.nodeSelector }}
+{{- if .Values.statefulset.nodeSelector }}
       nodeSelector:
-{{ toYaml .Values.deployment.nodeSelector | indent 8 }}
+{{ toYaml .Values.statefulset.nodeSelector | indent 8 }}
 {{- end }}
-{{- if .Values.deployment.tolerations }}
+{{- if .Values.statefulset.tolerations }}
       tolerations:
-{{ toYaml .Values.deployment.tolerations | indent 8 }}
+{{ toYaml .Values.statefulset.tolerations | indent 8 }}
 {{- end }}
       volumes:
       - name: pos-files
@@ -33,12 +33,14 @@ spec:
       - name: config-volume
         configMap:
           name: {{ template "sumologic.fullname" . }}
+      securityContext:
+        fsGroup: {{ .Values.sumologic.fluentd.securityContext.fsGroup }}
       containers:
       - name: fluentd
         image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
         imagePullPolicy: {{ .Values.image.pullPolicy }}
         resources:
-          {{- toYaml .Values.deployment.resources | nindent 10 }}
+          {{- toYaml .Values.statefulset.resources | nindent 10 }}
         ports:
         - name: prom-write
           containerPort: 9888
@@ -64,6 +66,10 @@ spec:
           mountPath: /fluentd/etc/
         - name: pos-files
           mountPath: /mnt/pos/
+{{- if .Values.sumologic.fluentd.persistence.enabled }}
+        - name: buffer
+          mountPath: "/fluentd/buffer"
+{{- end}}
         env:
         - name: SUMO_ENDPOINT_METRICS
           valueFrom:
@@ -152,4 +158,15 @@ spec:
         - name: EXCLUDE_HOST_REGEX
           value: {{ .Values.sumologic.excludeHostRegex | quote }}
         - name: EXCLUDE_POD_REGEX
-          value: {{ .Values.sumologic.excludePodRegex | quote }}
+          value: {{ .Values.sumologic.excludePodRegex | quote }}
+{{- if .Values.sumologic.fluentd.persistence.enabled }}
+  volumeClaimTemplates:
+  - metadata:
+      name: buffer
+    spec:
+      accessModes: [{{ .Values.sumologic.fluentd.persistence.accessMode }}]
+      storageClassName: {{ .Values.sumologic.fluentd.persistence.storageClass }}
+      resources:
+        requests:
+          storage: {{ .Values.sumologic.fluentd.persistence.size }}
+{{- end }}

deploy/helm/sumologic/values.yaml

@@ -5,7 +5,7 @@ image:
 
 nameOverride: ""
 
-deployment:
+statefulset:
   nodeSelector: {}
   tolerations: {}
   replicaCount: 3
@@ -17,7 +17,7 @@ deployment:
       memory: 768Mi
       cpu: 0.5
 
-eventsDeployment:
+eventsStatefulset:
   nodeSelector: {}
   tolerations: {}
   resources:
@@ -173,6 +173,25 @@ sumologic:
       maxReplicas: 10
       targetCPUUtilizationPercentage: 50
 
+    securityContext:
+      ## The group ID of all processes in the statefulset containers. By default this needs to be fluent(999).
+      fsGroup: 999
+
+    ## Persist data to a persistent volume; When enabled, fluentd uses the file buffer instead of memory buffer.
+    persistence:
+      enabled: false
+
+      ## If defined, storageClassName: <storageClass>
+      ## If set to "-", storageClassName: "", which disables dynamic provisioning
+      ## If undefined (the default) or set to null, no storageClassName spec is
+      ##   set, choosing the default provisioner.  (gp2 on AWS, standard on
+      ##   GKE, Azure & OpenStack)
+      ##
+      # storageClass: "-"
+      # annotations: {}
+      accessMode: ReadWriteOnce
+      size: 10Gi
+
   k8sMetadataFilter:
     ## Option to control the enabling of metadata filter plugin watch.
     ## ref: https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter#configuration