Create fields automatically in setup job

deploy/helm/sumologic/conf/setup/fields.tf

@@ -0,0 +1,9 @@
+{{- range $value := .Values.sumologic.logs.fields }}
+resource "sumologic_field" {{ $value | quote }} {
+  count = var.create_fields ? 1 : 0
+
+  field_name = {{ $value | quote }}
+  data_type = "String"
+  state = "Enabled"
+}
+{{- end }}

deploy/helm/sumologic/conf/setup/setup.sh

@@ -1,21 +1,67 @@
 #!/bin/bash
-cp /etc/terraform/{locals,main,providers,resources,variables}.tf /terraform
-cd /terraform
 
 # Fix URL to remove "v1" or "v1/"
 export SUMOLOGIC_BASE_URL=${SUMOLOGIC_BASE_URL%v1*}
-
 # Support proxy for terraform
 export HTTP_PROXY=${HTTP_PROXY:=""}
 export HTTPS_PROXY=${HTTPS_PROXY:=""}
 export NO_PROXY=${NO_PROXY:=""}
 
+function remaining_fields() {
+    local RESPONSE="$(curl -XGET -s \
+        -u "${SUMOLOGIC_ACCESSID}:${SUMOLOGIC_ACCESSKEY}" \
+        "${SUMOLOGIC_BASE_URL}"v1/fields/quota)"
+
+    echo "${RESPONSE}" | jq '.remaining'
+}
+
+# Check if we'd have at least 10 fields remaining after additional fields
+# would be created for the collection
+function should_create_fields() {
+    local REMAINING=$(remaining_fields)
+    if [[ $(( REMAINING - {{ len .Values.sumologic.logs.fields }} )) -ge 10 ]] ; then
+        return 0
+    else
+        return 1
+    fi
+}
+
+cp /etc/terraform/{locals,main,providers,resources,variables,fields}.tf /terraform/
+cd /terraform
+
 COLLECTOR_NAME="{{- if .Values.sumologic.collectorName }}{{ .Values.sumologic.collectorName }}{{- else}}{{ .Values.sumologic.clusterName }}{{- end}}"
 
 terraform init
 
-# Sumo Collector and HTTP sources
+# Sumo Logic fields
+if should_create_fields ; then
+    readonly CREATE_FIELDS=1
+    readonly FIELDS_RESPONSE="$(curl -XGET -s \
+        -u "${SUMOLOGIC_ACCESSID}:${SUMOLOGIC_ACCESSKEY}" \
+        "${SUMOLOGIC_BASE_URL}"v1/fields | jq '.data[]' )"
+
+    declare -ra FIELDS=({{ include "helm-toolkit.utils.joinListWithSpaces" .Values.sumologic.logs.fields }})
+    for FIELD in "${FIELDS[@]}" ; do
+        FIELD_ID=$( echo "${FIELDS_RESPONSE}" | jq -r "select(.fieldName == \"${FIELD}\") | .fieldId" )
+        # Don't try to import non existing fields
+        if [[ -z "${FIELD_ID}" ]]; then
+            continue
+        fi
+
+        terraform import \
+            -var="create_fields=1" \
+            sumologic_field."${FIELD}" "${FIELD_ID}"
+    done
+else
+    readonly CREATE_FIELDS=0
+    echo "Couldn't automatically create fields"
+    echo "You do not have enough field capacity to create the required fields automatically."
+    echo "Please refer to https://help.sumologic.com/Manage/Fields to manually create the fields after you have removed unused fields to free up capacity."
+fi
+
+# Sumo Logic Collector and HTTP sources
 terraform import sumologic_collector.collector "$COLLECTOR_NAME"
+
 {{- $ctx := .Values -}}
 {{- range $type, $sources := .Values.sumologic.sources }}
 {{- if eq (include "terraform.sources.component_enabled" (dict "Context" $ctx "Type" $type)) "true" }}
@@ -30,7 +76,9 @@ terraform import sumologic_http_source.{{ template "terraform.sources.name" (dic
 # Kubernetes Secret
 terraform import kubernetes_secret.sumologic_collection_secret {{ .Release.Namespace }}/sumologic
 
-terraform apply -auto-approve
+# Apply planned changes
+terraform apply -auto-approve \
+    -var="create_fields=${CREATE_FIELDS}"
 
 # Cleanup env variables
 export SUMOLOGIC_BASE_URL=

deploy/helm/sumologic/conf/setup/variables.tf

@@ -15,4 +15,10 @@ variable "collector_name" {
 variable "namespace_name" {
   type  = string
   default = "{{ .Release.Namespace }}"
-}
+}
+
+variable "create_fields" {
+  description = "If set, terraform will attempt to create fields at Sumo Logic"
+  type        = bool
+  default     = true
+}

deploy/helm/sumologic/templates/NOTES.txt

@@ -11,4 +11,8 @@ Check the release status by running:
 
 {{- if eq .Values.fluentd.persistence.enabled  false }}
 WARNING: File persistence for fluentd is disabled. This might lead to loss of data in case of memory buffer overflow. We recommend turning this property on for production environments by setting fluentd.persistence.enabled=true
-{{- end }}
+{{- end }}
+
+We've tried to automatically create fields. In an unlikely scenario that this
+fails please refer to the following to create them manually:
+https://github.com/SumoLogic/sumologic-kubernetes-collection/blob/2b3ca63/deploy/docs/Installation_with_Helm.md#prerequisite

deploy/helm/sumologic/templates/_helpers.tpl

@@ -778,3 +778,19 @@ Example:
               key: {{ template "terraform.sources.config-map-variable" (dict "Type" $type "Context" $ctx "Name" $key) }}
 {{- end }}
 {{- end -}}
+
+{{/*
+Generate a space separated list of quoted values:
+
+Example:
+
+{{ include "helm-toolkit.utils.joinListWithSpaces" .Values.sumologic.logs.fields }}
+*/}}
+{{- define "helm-toolkit.utils.joinListWithSpaces" -}}
+{{- $local := dict "first" true -}}
+{{- range $k, $v := . -}}
+{{- if not $local.first }} {{ end -}}
+{{- $v | quote -}}
+{{- $_ := set $local "first" false -}}
+{{- end -}}
+{{- end -}}

deploy/helm/sumologic/values.yaml

@@ -101,6 +101,19 @@ sumologic:
   logs:
     enabled: true
 
+    ## Fields to be created at Sumo Logic to ensure logs are tagged with
+    ## relevant metadata.
+    ## https://help.sumologic.com/Manage/Fields#Manage_fields
+    fields:
+      - cluster
+      - container
+      - deployment
+      - host
+      - namespace
+      - node
+      - pod
+      - service
+
   ### Metrics configuration
   ## Set the enabled flag to false for disabling metrics ingestion altogether.
   metrics:

deploy/kubernetes/setup-sumologic.yaml.tmpl

@@ -81,6 +81,63 @@ data:
       done
       cd "${target}" && bash setup.sh
     done
+  fields.tf: |
+    resource "sumologic_field" "cluster" {
+      count = var.create_fields ? 1 : 0
+
+      field_name = "cluster"
+      data_type = "String"
+      state = "Enabled"
+    }
+    resource "sumologic_field" "container" {
+      count = var.create_fields ? 1 : 0
+
+      field_name = "container"
+      data_type = "String"
+      state = "Enabled"
+    }
+    resource "sumologic_field" "deployment" {
+      count = var.create_fields ? 1 : 0
+
+      field_name = "deployment"
+      data_type = "String"
+      state = "Enabled"
+    }
+    resource "sumologic_field" "host" {
+      count = var.create_fields ? 1 : 0
+
+      field_name = "host"
+      data_type = "String"
+      state = "Enabled"
+    }
+    resource "sumologic_field" "namespace" {
+      count = var.create_fields ? 1 : 0
+
+      field_name = "namespace"
+      data_type = "String"
+      state = "Enabled"
+    }
+    resource "sumologic_field" "node" {
+      count = var.create_fields ? 1 : 0
+
+      field_name = "node"
+      data_type = "String"
+      state = "Enabled"
+    }
+    resource "sumologic_field" "pod" {
+      count = var.create_fields ? 1 : 0
+
+      field_name = "pod"
+      data_type = "String"
+      state = "Enabled"
+    }
+    resource "sumologic_field" "service" {
+      count = var.create_fields ? 1 : 0
+
+      field_name = "service"
+      data_type = "String"
+      state = "Enabled"
+    }
   locals.tf: |
     locals {
       default_events_source                       = "events"
@@ -193,22 +250,67 @@ data:
     }
   setup.sh: |
     #!/bin/bash
-    cp /etc/terraform/{locals,main,providers,resources,variables}.tf /terraform
-    cd /terraform
 
     # Fix URL to remove "v1" or "v1/"
     export SUMOLOGIC_BASE_URL=${SUMOLOGIC_BASE_URL%v1*}
-
     # Support proxy for terraform
     export HTTP_PROXY=${HTTP_PROXY:=""}
     export HTTPS_PROXY=${HTTPS_PROXY:=""}
     export NO_PROXY=${NO_PROXY:=""}
 
+    function remaining_fields() {
+        local RESPONSE="$(curl -XGET -s \
+            -u "${SUMOLOGIC_ACCESSID}:${SUMOLOGIC_ACCESSKEY}" \
+            "${SUMOLOGIC_BASE_URL}"v1/fields/quota)"
+
+        echo "${RESPONSE}" | jq '.remaining'
+    }
+
+    # Check if we'd have at least 10 fields remaining after additional fields
+    # would be created for the collection
+    function should_create_fields() {
+        local REMAINING=$(remaining_fields)
+        if [[ $(( REMAINING - 8 )) -ge 10 ]] ; then
+            return 0
+        else
+            return 1
+        fi
+    }
+
+    cp /etc/terraform/{locals,main,providers,resources,variables,fields}.tf /terraform/
+    cd /terraform
+
     COLLECTOR_NAME="$COLLECTOR_NAME"
 
     terraform init
 
-    # Sumo Collector and HTTP sources
+    # Sumo Logic fields
+    if should_create_fields ; then
+        readonly CREATE_FIELDS=1
+        readonly FIELDS_RESPONSE="$(curl -XGET -s \
+            -u "${SUMOLOGIC_ACCESSID}:${SUMOLOGIC_ACCESSKEY}" \
+            "${SUMOLOGIC_BASE_URL}"v1/fields | jq '.data[]' )"
+
+        declare -ra FIELDS=("cluster" "container" "deployment" "host" "namespace" "node" "pod" "service")
+        for FIELD in "${FIELDS[@]}" ; do
+            FIELD_ID=$( echo "${FIELDS_RESPONSE}" | jq -r "select(.fieldName == \"${FIELD}\") | .fieldId" )
+            # Don't try to import non existing fields
+            if [[ -z "${FIELD_ID}" ]]; then
+                continue
+            fi
+
+            terraform import \
+                -var="create_fields=1" \
+                sumologic_field."${FIELD}" "${FIELD_ID}"
+        done
+    else
+        readonly CREATE_FIELDS=0
+        echo "Couldn't automatically create fields"
+        echo "You do not have enough field capacity to create the required fields automatically."
+        echo "Please refer to https://help.sumologic.com/Manage/Fields to manually create the fields after you have removed unused fields to free up capacity."
+    fi
+
+    # Sumo Logic Collector and HTTP sources
     terraform import sumologic_collector.collector "$COLLECTOR_NAME"
     terraform import sumologic_http_source.default_events_source "$COLLECTOR_NAME/events"
     terraform import sumologic_http_source.default_logs_source "$COLLECTOR_NAME/logs"
@@ -224,15 +326,17 @@ data:
     # Kubernetes Secret
     terraform import kubernetes_secret.sumologic_collection_secret $NAMESPACE/sumologic
 
-    terraform apply -auto-approve
+    # Apply planned changes
+    terraform apply -auto-approve \
+        -var="create_fields=${CREATE_FIELDS}"
 
     # Cleanup env variables
     export SUMOLOGIC_BASE_URL=
     export SUMOLOGIC_ACCESSKEY=
     export SUMOLOGIC_ACCESSID=
 
     bash /etc/terraform/custom.sh
-  variables.tf: |-
+  variables.tf: |
     variable "cluster_name" {
       type  = string
       default = "$CLUSTER_NAME"
@@ -247,6 +351,12 @@ data:
       type  = string
       default = "$NAMESPACE"
     }
+
+    variable "create_fields" {
+      description = "If set, terraform will attempt to create fields at Sumo Logic"
+      type        = bool
+      default     = true
+    }
 ---
 # Source: sumologic/templates/setup/setup-custom-configmap.yaml
 apiVersion: v1