Create fields automatically in setup job

deploy/helm/sumologic/conf/setup/fields.tf

@@ -0,0 +1,63 @@
+resource "sumologic_field" "cluster" {
+  count = var.create_fields ? 1 : 0
+
+  field_name = "cluster"
+  data_type = "String"
+  state = "Enabled"
+}
+
+resource "sumologic_field" "container" {
+  count = var.create_fields ? 1 : 0
+
+  field_name = "container"
+  data_type = "String"
+  state = "Enabled"
+}
+
+resource "sumologic_field" "deployment" {
+  count = var.create_fields ? 1 : 0
+
+  field_name = "deployment"
+  data_type = "String"
+  state = "Enabled"
+}
+
+resource "sumologic_field" "host" {
+  count = var.create_fields ? 1 : 0
+
+  field_name = "host"
+  data_type = "String"
+  state = "Enabled"
+}
+
+resource "sumologic_field" "namespace" {
+  count = var.create_fields ? 1 : 0
+
+  field_name = "namespace"
+  data_type = "String"
+  state = "Enabled"
+}
+
+resource "sumologic_field" "node" {
+  count = var.create_fields ? 1 : 0
+
+  field_name = "node"
+  data_type = "String"
+  state = "Enabled"
+}
+
+resource "sumologic_field" "pod" {
+  count = var.create_fields ? 1 : 0
+
+  field_name = "pod"
+  data_type = "String"
+  state = "Enabled"
+}
+
+resource "sumologic_field" "service" {
+  count = var.create_fields ? 1 : 0
+
+  field_name = "service"
+  data_type = "String"
+  state = "Enabled"
+}

deploy/helm/sumologic/conf/setup/setup.sh

@@ -1,36 +1,89 @@
 #!/bin/bash
-cp /etc/terraform/{locals,main,providers,resources,variables}.tf /terraform
-cd /terraform
 
 # Fix URL to remove "v1" or "v1/"
 export SUMOLOGIC_BASE_URL=${SUMOLOGIC_BASE_URL%v1*}
-
 # Support proxy for terraform
 export HTTP_PROXY=${HTTP_PROXY:=""}
 export HTTPS_PROXY=${HTTPS_PROXY:=""}
 export NO_PROXY=${NO_PROXY:=""}
 
+function remaining_fields() {
+    local RESPONSE="$(curl -XGET -s \
+        -u "${SUMOLOGIC_ACCESSID}:${SUMOLOGIC_ACCESSKEY}" \
+        "${SUMOLOGIC_BASE_URL}"v1/fields/quota)"
+
+    echo "${RESPONSE}" | jq '.remaining'
+}
+
+# Check if we'd have at least 10 fields remaining after 8 additional fields
+# would be created for the collection
+function should_create_fields() {
+    local REMAINING=$(remaining_fields)
+    if [[ $(( REMAINING - 8 )) -ge 10 ]] ; then
+        echo 1
+    else
+        echo 0
+    fi
+}
+
+cp /etc/terraform/{locals,main,providers,resources,variables,fields}.tf /terraform/
+cd /terraform
+
 COLLECTOR_NAME="{{- if .Values.sumologic.collectorName }}{{ .Values.sumologic.collectorName }}{{- else}}{{ .Values.sumologic.clusterName }}{{- end}}"
 
 terraform init
 
-# Sumo Collector and HTTP sources
-terraform import sumologic_collector.collector "$COLLECTOR_NAME"
+# Sumo Logic fields
+readonly CREATE_FIELDS="$(should_create_fields)"
+if [[ "${CREATE_FIELDS}" -eq 0 ]]; then
+    echo "Couldn't automatically create fields\n"
+    echo "There's not enough fields which we could use for collection fields creation"
+    echo "Please free some of them and rerun the setup job"
+else
+    readonly FIELDS_RESPONSE="$(curl -XGET -s \
+        -u "${SUMOLOGIC_ACCESSID}:${SUMOLOGIC_ACCESSKEY}" \
+        "${SUMOLOGIC_BASE_URL}"v1/fields | jq '.data[]' )"
+
+    declare -ra FIELDS=("cluster" "container" "deployment" "host" "namespace" "node" "pod" "service")
+    for FIELD in "${FIELDS[@]}" ; do
+        FIELD_ID=$( echo "${FIELDS_RESPONSE}" | jq -r "select(.fieldName == \"${FIELD}\") | .fieldId" )
+        # Don't try to import non existing fields
+        if [[ -z "${FIELD_ID}" ]]; then
+            continue
+        fi
+
+        terraform import \
+            -var="create_fields=${CREATE_FIELDS}" \
+            sumologic_field."${FIELD}" "${FIELD_ID}"
+    done
+fi
+
+# Sumo Logic Collector and HTTP sources
+terraform import \
+    -var="create_fields=${CREATE_FIELDS}" \
+    sumologic_collector.collector "$COLLECTOR_NAME"
+
 {{- $ctx := .Values -}}
 {{- range $type, $sources := .Values.sumologic.sources }}
 {{- if eq (include "terraform.sources.component_enabled" (dict "Context" $ctx "Type" $type)) "true" }}
 {{- range $key, $source := $sources }}
 {{- if eq (include "terraform.sources.to_create" (dict "Context" $ctx "Type" $type "Name" $key)) "true" }}
-terraform import sumologic_http_source.{{ template "terraform.sources.name" (dict "Name" $key "Type" $type) }} "$COLLECTOR_NAME/{{ $source.name }}"
+terraform import \
+    -var="create_fields=${CREATE_FIELDS}" \
+    sumologic_http_source.{{ template "terraform.sources.name" (dict "Name" $key "Type" $type) }} "$COLLECTOR_NAME/{{ $source.name }}"
 {{- end }}
 {{- end }}
 {{- end }}
 {{- end }}
 
 # Kubernetes Secret
-terraform import kubernetes_secret.sumologic_collection_secret {{ .Release.Namespace }}/sumologic
+terraform import \
+    -var="create_fields=${CREATE_FIELDS}" \
+    kubernetes_secret.sumologic_collection_secret {{ .Release.Namespace }}/sumologic
 
-terraform apply -auto-approve
+# Apply planned changes
+terraform apply -auto-approve \
+    -var="create_fields=${CREATE_FIELDS}"
 
 # Cleanup env variables
 export SUMOLOGIC_BASE_URL=

deploy/helm/sumologic/conf/setup/variables.tf

@@ -15,4 +15,9 @@ variable "collector_name" {
 variable "namespace_name" {
   type  = string
   default = "{{ .Release.Namespace }}"
-}
+}
+
+variable "create_fields" {
+  description = "If set, terraform will attempt to create fields at Sumo Logic"
+  type = bool
+}

deploy/helm/sumologic/templates/NOTES.txt

@@ -11,4 +11,8 @@ Check the release status by running:
 
 {{- if eq .Values.fluentd.persistence.enabled  false }}
 WARNING: File persistence for fluentd is disabled. This might lead to loss of data in case of memory buffer overflow. We recommend turning this property on for production environments by setting fluentd.persistence.enabled=true
-{{- end }}
+{{- end }}
+
+We've tried to automatically create fields. In an unlikely scenario that this
+fails please refer to the following to create them manually:
+https://github.com/SumoLogic/sumologic-kubernetes-collection/blob/2b3ca63/deploy/docs/Installation_with_Helm.md#prerequisite