Clean up collector keys in values.yaml (#1160)

deploy/docs/Terraform.md

@@ -4,12 +4,17 @@ Terraform is used by sumologic-kubernetes-collection during the setup process
 to automatically create HTTP sources and store their URLs in the Kubernetes secret.
 
 We are using two providers to perform those actions:
- * [Kubernetes Terraform provider](https://www.terraform.io/docs/providers/kubernetes/)
- * [Sumo Logic Terraform provider](https://www.terraform.io/docs/providers/sumologic/)
+
+- [Kubernetes Terraform provider](https://www.terraform.io/docs/providers/kubernetes/)
+- [Sumo Logic Terraform provider](https://www.terraform.io/docs/providers/sumologic/)
 
 ## Kubernetes Terraform provider
 
-[Kubernetes Terraform provider](https://www.terraform.io/docs/providers/kubernetes/) is responsible for creating the secret with the created HTTP source endpoints during setup process. The default configuration is expected to work in most cases, however, for self-hosted Kubernetes clusters there can be a few exceptions. For these cases we expose the provider configuration in `values.yaml`.
+[Kubernetes Terraform provider](https://www.terraform.io/docs/providers/kubernetes/)
+is responsible for creating the secret with the created HTTP source endpoints during setup process.
+The default configuration is expected to work in most cases, however, for self-hosted
+Kubernetes clusters there can be a few exceptions.
+For these cases we expose the provider configuration in `values.yaml`.
 
 ```yaml
 sumologic:
@@ -42,22 +47,24 @@ sumologic:
 ## Sumo Logic Terraform provider
 
 The [Sumo Logic Terraform provider](https://www.terraform.io/docs/providers/sumologic/) creates your HTTP sources.
-The related configuration section in the `values.yaml` file is under `sumologic.sources`:
+The related configuration section in the `values.yaml` file is under `sumologic.collector.sources`:
 
 ```yaml
 sumologic:
   # ...
-  sources:
+  collector:
     # ...
-    logs: # source type, can be one of: logs/metrics/events/traces
-      example-source: # source reference name
-        name: # name of the source (visible on the sumologic platform)
-        config-name: # name which be used in secret to store the url. This is backward-compatibility option
-        category: # this is backward compatibility property. It's deprecated and it's going to be removed in version 2.0
-                  # Sets source category to "${var.cluster_name}/${local.default_events_source}" if true
-                  # To overwrite category, please use `sumologic.sources[].properties.category`
-        properties: # Additional Terraform properties like fields or content_type
-                    # ref: https://www.terraform.io/docs/providers/sumologic/r/collector.html
+    sources:
+      # ...
+      logs: # source type, can be one of: logs/metrics/events/traces
+        example-source: # source reference name
+          name: # name of the source (visible on the sumologic platform)
+          config-name: # name which be used in secret to store the url. This is backward-compatibility option
+          category: # this is backward compatibility property. It's deprecated and it's going to be removed in version 2.0
+                    # Sets source category to "${var.cluster_name}/${local.default_events_source}" if true
+                    # To overwrite category, please use `sumologic.collector.sources[].properties.category`
+          properties: # Additional Terraform properties like fields or content_type
+                      # ref: https://www.terraform.io/docs/providers/sumologic/r/collector.html
 ```
 
 ### Usage
@@ -67,53 +74,61 @@ The variable name is built using the schema `SUMO_ENDPOINT_<source name>_<source
 where `<source name>` and `<source type>` are in uppercase and dashes are replaced with underscores.
 
 Examples:
- - `sumologic.sources.logs.example-source` becomes `SUMO_ENDPOINT_EXAMPLE_SOURCE_LOGS_SOURCE`
- - `sumologic.sources.traces.default` becomes `SUMO_ENDPOINT_DEFAULT_TRACES_SOURCE`
+
+- `sumologic.collector.sources.logs.example-source` becomes `SUMO_ENDPOINT_EXAMPLE_SOURCE_LOGS_SOURCE`
+- `sumologic.collector.sources.traces.default` becomes `SUMO_ENDPOINT_DEFAULT_TRACES_SOURCE`
 
 ### Properties
 
 You can set all of the source [properties](https://www.terraform.io/docs/providers/sumologic/r/http_source.html#argument-reference)
-using `sumologic.sources.<logs,traces,metrics,traces>.<source ref name>.properties`.
+using `sumologic.collector.sources.<logs,traces,metrics,traces>.<source ref name>.properties`.
 
 #### Processing Rules
 
-You can add [Processing Rules](https://help.sumologic.com/Manage/Collection/Processing-Rules) to an HTTP source via values.yaml. Below is an example of an exclude rule to filter `DEBUG` log messages.  All logs from Kubernetes (Systemd, container, and custom logs) will have this filter applied.
+You can add [Processing Rules](https://help.sumologic.com/Manage/Collection/Processing-Rules) to an HTTP source via values.yaml.
+Below is an example of an exclude rule to filter `DEBUG` log messages.
+All logs from Kubernetes (Systemd, container, and custom logs) will have this filter applied.
 
 ```yaml
 sumologic:
   # ...
-  sources:
+  collector:
     # ...
-    logs:
+    sources:
       # ...
-      default:
+      logs:
         # ...
-        name: logs
-        config-name: endpoint-logs
-        properties:
-          filters:
-            - name: "Test Exclude Debug"
-              filter_type: "Exclude"
-              regexp: ".*DEBUG.*"
+        default:
+          # ...
+          name: logs
+          config-name: endpoint-logs
+          properties:
+            filters:
+              - name: "Test Exclude Debug"
+                filter_type: "Exclude"
+                regexp: ".*DEBUG.*"
 ```
 
 #### Fields
 
-The configuration snippet below configures two [fields](https://help.sumologic.com/Manage/Fields), (`node` and `deployment`) for the default logs source:
+The configuration snippet below configures two [fields](https://help.sumologic.com/Manage/Fields),
+(`node` and `deployment`) for the default logs source:
 
 ```yaml
 sumologic:
   # ...
-  sources:
+  collector:
     # ...
-    logs:
+    sources:
       # ...
-      default:
+      logs:
         # ...
-        properties:
-          fields:
-            node: hornetq-livestream-9
-            deployment: sumologic
+        default:
+          # ...
+          properties:
+            fields:
+              node: hornetq-livestream-9
+              deployment: sumologic
 ```
 
 #### Terraform variables
@@ -122,16 +137,18 @@ You can use Terraform extrapolation for properties:
 
 ```yaml
 sumologic:
-  sources:
-    logs:
-      example-source:
-        properties:
-          category: "${var.cluster_name}/my-name"
+  collector:
+    sources:
+      logs:
+        example-source:
+          properties:
+            category: "${var.cluster_name}/my-name"
 ```
 
 List of available variables:
- * var.cluster_name
- * var.namespace_name
- * var.collector_name
+
+- `var.cluster_name`
+- `var.namespace_name`
+- `var.collector_name`
 
 **Note** You have to manually activate fields using the Sumo Logic service.

deploy/docs/additional_prometheus_configuration.md

@@ -216,7 +216,7 @@ fluentd:
     # ...
     output:
       # ...
-      my_source:  # It matches sumologic.sources.my_source
+      my_source:  # It matches sumologic.collector.sources.my_source
         tag: prometheus.metrics.YOUR_TAG  # tag used by Fluentd's match clausule
         id: sumologic.endpoint.metrics
 

deploy/docs/v2_migration_doc.md

@@ -73,6 +73,12 @@ the exact steps for migration.
 - We've separated our fluentd image from setup job image, hence `image` was migrated
   to `sumologic.setup.job.image` and to `fluentd.image`
 
+- `sumologic.sources` become `sumologic.collector.sources` as Sources are being
+  created under Collectors
+
+- `sumologic.setup.fields` become `sumologic.collector.fields` as Fields are
+  set on a Collector
+
 ## How to upgrade
 
 **Note: The below steps are using Helm 3. Helm 2 is not supported.**

deploy/helm/sumologic/conf/setup/locals.tf

@@ -1,6 +1,6 @@
 locals {
 {{- $ctx := .Values }}
-{{- range $type, $sources := .Values.sumologic.sources }}
+{{- range $type, $sources := .Values.sumologic.collector.sources }}
 {{- if eq (include "terraform.sources.component_enabled" (dict "Context" $ctx "Type" $type)) "true" }}
 {{- range $key, $source := $sources }}
 {{- if eq (include "terraform.sources.to_create" (dict "Context" $ctx "Type" $type "Name" $key)) "true" }}

deploy/helm/sumologic/conf/setup/resources.tf

@@ -1,7 +1,7 @@
 resource "sumologic_collector" "collector" {
     name  = var.collector_name
     fields  = {
-      {{- $fields := .Values.sumologic.setup.fields }}
+      {{- $fields := .Values.sumologic.collector.fields }}
       {{ include "terraform.generate-key" (dict "Name" "cluster" "Value" "var.cluster_name" "SkipEscaping" true "KeyLength" (include "terraform.max-key-length" $fields)) }}
       {{- range $name, $value := $fields }}
       {{ include "terraform.generate-key" (dict "Name" $name "Value" $value "KeyLength" (include "terraform.max-key-length" $fields)) }}
@@ -10,7 +10,7 @@ resource "sumologic_collector" "collector" {
 }
 
 {{- $ctx := .Values }}
-{{- range $type, $sources := .Values.sumologic.sources }}
+{{- range $type, $sources := .Values.sumologic.collector.sources }}
 {{- if eq (include "terraform.sources.component_enabled" (dict "Context" $ctx "Type" $type)) "true" }}
 {{- range $key, $source := $sources }}
 {{- if eq (include "terraform.sources.to_create" (dict "Context" $ctx "Type" $type "Name" $key)) "true" }}
@@ -28,7 +28,7 @@ resource "kubernetes_secret" "sumologic_collection_secret" {
 
   data = {
     {{- $ctx := .Values }}
-    {{- range $type, $sources := .Values.sumologic.sources }}
+    {{- range $type, $sources := .Values.sumologic.collector.sources }}
     {{- if eq (include "terraform.sources.component_enabled" (dict "Context" $ctx "Type" $type)) "true" }}
     {{- range $key, $source := $sources }}
     {{- if eq (include "terraform.sources.to_create" (dict "Context" $ctx "Type" $type "Name" $key)) "true" }}

deploy/helm/sumologic/conf/setup/setup.sh

@@ -63,7 +63,7 @@ readonly COLLECTOR_NAME="{{ template "terraform.collector.name" . }}"
 # Only import sources when collector exists.
 if terraform import sumologic_collector.collector "${COLLECTOR_NAME}"; then
 {{- $ctx := .Values -}}
-{{- range $type, $sources := .Values.sumologic.sources }}
+{{- range $type, $sources := .Values.sumologic.collector.sources }}
 {{- if eq (include "terraform.sources.component_enabled" (dict "Context" $ctx "Type" $type)) "true" }}
 {{- range $key, $source := $sources }}
 {{- if eq (include "terraform.sources.to_create" (dict "Context" $ctx "Type" $type "Name" $key)) "true" }}

deploy/helm/sumologic/templates/_helpers.tpl

@@ -754,9 +754,9 @@ Example usage:
 {{- $type := .Type -}}
 {{- $endpoint := .Endpoint -}}
 {{- if not $endpoint -}}
-{{- $source := (index $ctx.sumologic.sources $type "default") -}}
-{{- if (index $ctx.sumologic.sources $type .Name "config-name") -}}
-{{- $endpoint = index $ctx.sumologic.sources $type .Name "config-name" -}}
+{{- $source := (index $ctx.sumologic.collector.sources $type "default") -}}
+{{- if (index $ctx.sumologic.collector.sources $type .Name "config-name") -}}
+{{- $endpoint = index $ctx.sumologic.collector.sources $type .Name "config-name" -}}
 {{- else -}}
 {{- $endpoint = printf "endpoint-%s" (include "terraform.sources.name" (dict "Name" $name "Type" $type)) -}}
 {{- end -}}
@@ -818,8 +818,8 @@ Example Usage:
 {{- $ctx := .Context -}}
 {{- $name := .Name -}}
 {{- $value := true -}}
-{{- if and (hasKey $ctx.sumologic.sources $type) (hasKey (index $ctx.sumologic.sources $type) $name) (hasKey (index $ctx.sumologic.sources $type $name) "create") -}}
-{{- if not (index $ctx.sumologic.sources $type $name "create") -}}
+{{- if and (hasKey $ctx.sumologic.collector.sources $type) (hasKey (index $ctx.sumologic.collector.sources $type) $name) (hasKey (index $ctx.sumologic.collector.sources $type $name) "create") -}}
+{{- if not (index $ctx.sumologic.collector.sources $type $name "create") -}}
 {{- $value = false -}}
 {{- end -}}
 {{- end -}}
@@ -836,7 +836,7 @@ Example:
 {{- define "kubernetes.sources.envs" -}}
 {{- $ctx := .Context -}}
 {{- $type := .Type -}}
-{{- range $key, $source := (index .Context.sumologic.sources $type) }}
+{{- range $key, $source := (index .Context.sumologic.collector.sources $type) }}
         - name: {{ template "terraform.sources.endpoint" (include "terraform.sources.name" (dict "Name" $key "Type" $type)) }}
           valueFrom:
             secretKeyRef:

deploy/helm/sumologic/upgrade-2.0.0.sh

@@ -149,6 +149,40 @@ function migrate_prometheus_operator_to_kube_prometheus_stack() {
   yq d -i "${TEMP_FILE}" "prometheus-operator"
 }
 
+function migrate_sumologic_sources() {
+  # Nothing to migrate, return
+  if [[ -z $(yq r "${TEMP_FILE}" sumologic.sources) ]] ; then
+    return
+  fi
+
+  info "Migrating sumologic.sources to sumologic.collector.sources"
+  yq m -i \
+    "${TEMP_FILE}" \
+    <(
+      yq p \
+      <(yq r "${TEMP_FILE}" "sumologic.sources") \
+      "sumologic.collector.sources" \
+    )
+  yq d -i "${TEMP_FILE}" "sumologic.sources"
+}
+
+function migrate_sumologic_setup_fields() {
+  # Nothing to migrate, return
+  if [[ -z $(yq r "${TEMP_FILE}" sumologic.setup.fields) ]] ; then
+    return
+  fi
+
+  info "Migrating sumologic.setup.fields to sumologic.collector.fields"
+  yq m -i \
+    "${TEMP_FILE}" \
+    <(
+      yq p \
+      <(yq r "${TEMP_FILE}" "sumologic.setup.fields") \
+      "sumologic.collector.fields" \
+    )
+  yq d -i "${TEMP_FILE}" "sumologic.setup.fields"
+}
+
 function delete_migrated_unused_keys() {
   IFS=$'\n' read -r -d ' ' -a MAPPINGS_KEYS_TO_DELETE <<< "${KEYS_TO_DELETE}"
   readonly MAPPINGS_KEYS_TO_DELETE
@@ -214,14 +248,6 @@ function migrate_customer_keys() {
   echo
 }
 
-function migrate_pre_upgrade_hook() {
-  # Keep pre-upgrade hook
-  if [[ -n "$(yq r "${TEMP_FILE}" -- sumologic.setup)" ]]; then
-    info "Updating setup hooks (sumologic.setup.*.annotations[helm.sh/hook]) to 'pre-install,pre-upgrade'"
-    yq w -i "${TEMP_FILE}" -- 'sumologic.setup.*.annotations[helm.sh/hook]' 'pre-install,pre-upgrade'
-  fi
-}
-
 function get_regex() {
     # Get regex from old yaml file and strip `'` and `"` from beginning/end of it
     local write_index="${1}"
@@ -274,8 +300,9 @@ create_temp_file
 
 migrate_customer_keys
 
-migrate_pre_upgrade_hook
 migrate_prometheus_operator_to_kube_prometheus_stack
+migrate_sumologic_sources
+migrate_sumologic_setup_fields
 
 check_user_image