From ecea6a8c70714f6fde9a695057b07e8415e8d2b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20Ma=C5=82ek?= Date: Tue, 24 Nov 2020 18:39:21 +0100 Subject: [PATCH 1/4] Clean up collector keys in values.yaml --- deploy/docs/Terraform.md | 10 +-- .../additional_prometheus_configuration.md | 2 +- deploy/docs/v2_migration_doc.md | 6 ++ deploy/helm/sumologic/conf/setup/locals.tf | 2 +- deploy/helm/sumologic/conf/setup/resources.tf | 6 +- deploy/helm/sumologic/conf/setup/setup.sh | 2 +- deploy/helm/sumologic/templates/_helpers.tpl | 12 +-- deploy/helm/sumologic/upgrade-2.0.0.sh | 45 ++++++++-- deploy/helm/sumologic/values.yaml | 85 ++++++++++--------- .../sumologic_collector_fields.input.yaml | 36 ++++++++ .../static/sumologic_collector_fields.log | 6 ++ .../sumologic_collector_fields.output.yaml | 29 +++++++ .../static/sumologic_sources.input.yaml | 35 ++++++++ .../static/sumologic_sources.log | 6 ++ .../static/sumologic_sources.output.yaml | 36 ++++++++ 15 files changed, 250 insertions(+), 68 deletions(-) create mode 100644 tests/upgrade_v2_script/static/sumologic_collector_fields.input.yaml create mode 100644 tests/upgrade_v2_script/static/sumologic_collector_fields.log create mode 100644 tests/upgrade_v2_script/static/sumologic_collector_fields.output.yaml create mode 100644 tests/upgrade_v2_script/static/sumologic_sources.input.yaml create mode 100644 tests/upgrade_v2_script/static/sumologic_sources.log create mode 100644 tests/upgrade_v2_script/static/sumologic_sources.output.yaml diff --git a/deploy/docs/Terraform.md b/deploy/docs/Terraform.md index f8c93bf248..6507ff22e1 100644 --- a/deploy/docs/Terraform.md +++ b/deploy/docs/Terraform.md @@ -42,7 +42,7 @@ sumologic: ## Sumo Logic Terraform provider The [Sumo Logic Terraform provider](https://www.terraform.io/docs/providers/sumologic/) creates your HTTP sources. -The related configuration section in the `values.yaml` file is under `sumologic.sources`: +The related configuration section in the `values.yaml` file is under `sumologic.collector.sources`: ```yaml sumologic: @@ -55,7 +55,7 @@ sumologic: config-name: # name which be used in secret to store the url. This is backward-compatibility option category: # this is backward compatibility property. It's deprecated and it's going to be removed in version 2.0 # Sets source category to "${var.cluster_name}/${local.default_events_source}" if true - # To overwrite category, please use `sumologic.sources[].properties.category` + # To overwrite category, please use `sumologic.collector.sources[].properties.category` properties: # Additional Terraform properties like fields or content_type # ref: https://www.terraform.io/docs/providers/sumologic/r/collector.html ``` @@ -67,13 +67,13 @@ The variable name is built using the schema `SUMO_ENDPOINT__` and `` are in uppercase and dashes are replaced with underscores. Examples: - - `sumologic.sources.logs.example-source` becomes `SUMO_ENDPOINT_EXAMPLE_SOURCE_LOGS_SOURCE` - - `sumologic.sources.traces.default` becomes `SUMO_ENDPOINT_DEFAULT_TRACES_SOURCE` + - `sumologic.collector.sources.logs.example-source` becomes `SUMO_ENDPOINT_EXAMPLE_SOURCE_LOGS_SOURCE` + - `sumologic.collector.sources.traces.default` becomes `SUMO_ENDPOINT_DEFAULT_TRACES_SOURCE` ### Properties You can set all of the source [properties](https://www.terraform.io/docs/providers/sumologic/r/http_source.html#argument-reference) -using `sumologic.sources...properties`. +using `sumologic.collector.sources...properties`. #### Processing Rules diff --git a/deploy/docs/additional_prometheus_configuration.md b/deploy/docs/additional_prometheus_configuration.md index 502666ed78..aea84d6f15 100644 --- a/deploy/docs/additional_prometheus_configuration.md +++ b/deploy/docs/additional_prometheus_configuration.md @@ -216,7 +216,7 @@ fluentd: # ... output: # ... - my_source: # It matches sumologic.sources.my_source + my_source: # It matches sumologic.collector.sources.my_source tag: prometheus.metrics.YOUR_TAG # tag used by Fluentd's match clausule id: sumologic.endpoint.metrics diff --git a/deploy/docs/v2_migration_doc.md b/deploy/docs/v2_migration_doc.md index d67c2f04f3..9a97acf1be 100644 --- a/deploy/docs/v2_migration_doc.md +++ b/deploy/docs/v2_migration_doc.md @@ -73,6 +73,12 @@ the exact steps for migration. - We've separated our fluentd image from setup job image, hence `image` was migrated to `sumologic.setup.job.image` and to `fluentd.image` +- `sumologic.sources` become `sumologic.collector.sources` as Sources are being + created under Collectors + +- `sumologic.setup.fields` become `sumologic.collector.fields` as Fields are + set on a Collector + ## How to upgrade **Note: The below steps are using Helm 3. Helm 2 is not supported.** diff --git a/deploy/helm/sumologic/conf/setup/locals.tf b/deploy/helm/sumologic/conf/setup/locals.tf index 2f5f68a17f..bdeea24b3c 100644 --- a/deploy/helm/sumologic/conf/setup/locals.tf +++ b/deploy/helm/sumologic/conf/setup/locals.tf @@ -1,6 +1,6 @@ locals { {{- $ctx := .Values }} -{{- range $type, $sources := .Values.sumologic.sources }} +{{- range $type, $sources := .Values.sumologic.collector.sources }} {{- if eq (include "terraform.sources.component_enabled" (dict "Context" $ctx "Type" $type)) "true" }} {{- range $key, $source := $sources }} {{- if eq (include "terraform.sources.to_create" (dict "Context" $ctx "Type" $type "Name" $key)) "true" }} diff --git a/deploy/helm/sumologic/conf/setup/resources.tf b/deploy/helm/sumologic/conf/setup/resources.tf index 04d659ca9c..495357f3aa 100644 --- a/deploy/helm/sumologic/conf/setup/resources.tf +++ b/deploy/helm/sumologic/conf/setup/resources.tf @@ -1,7 +1,7 @@ resource "sumologic_collector" "collector" { name = var.collector_name fields = { - {{- $fields := .Values.sumologic.setup.fields }} + {{- $fields := .Values.sumologic.collector.fields }} {{ include "terraform.generate-key" (dict "Name" "cluster" "Value" "var.cluster_name" "SkipEscaping" true "KeyLength" (include "terraform.max-key-length" $fields)) }} {{- range $name, $value := $fields }} {{ include "terraform.generate-key" (dict "Name" $name "Value" $value "KeyLength" (include "terraform.max-key-length" $fields)) }} @@ -10,7 +10,7 @@ resource "sumologic_collector" "collector" { } {{- $ctx := .Values }} -{{- range $type, $sources := .Values.sumologic.sources }} +{{- range $type, $sources := .Values.sumologic.collector.sources }} {{- if eq (include "terraform.sources.component_enabled" (dict "Context" $ctx "Type" $type)) "true" }} {{- range $key, $source := $sources }} {{- if eq (include "terraform.sources.to_create" (dict "Context" $ctx "Type" $type "Name" $key)) "true" }} @@ -28,7 +28,7 @@ resource "kubernetes_secret" "sumologic_collection_secret" { data = { {{- $ctx := .Values }} - {{- range $type, $sources := .Values.sumologic.sources }} + {{- range $type, $sources := .Values.sumologic.collector.sources }} {{- if eq (include "terraform.sources.component_enabled" (dict "Context" $ctx "Type" $type)) "true" }} {{- range $key, $source := $sources }} {{- if eq (include "terraform.sources.to_create" (dict "Context" $ctx "Type" $type "Name" $key)) "true" }} diff --git a/deploy/helm/sumologic/conf/setup/setup.sh b/deploy/helm/sumologic/conf/setup/setup.sh index fca96f8dcd..eb51a6f36e 100755 --- a/deploy/helm/sumologic/conf/setup/setup.sh +++ b/deploy/helm/sumologic/conf/setup/setup.sh @@ -63,7 +63,7 @@ readonly COLLECTOR_NAME="{{ template "terraform.collector.name" . }}" # Only import sources when collector exists. if terraform import sumologic_collector.collector "${COLLECTOR_NAME}"; then {{- $ctx := .Values -}} -{{- range $type, $sources := .Values.sumologic.sources }} +{{- range $type, $sources := .Values.sumologic.collector.sources }} {{- if eq (include "terraform.sources.component_enabled" (dict "Context" $ctx "Type" $type)) "true" }} {{- range $key, $source := $sources }} {{- if eq (include "terraform.sources.to_create" (dict "Context" $ctx "Type" $type "Name" $key)) "true" }} diff --git a/deploy/helm/sumologic/templates/_helpers.tpl b/deploy/helm/sumologic/templates/_helpers.tpl index f19e638e2e..43364dc4ae 100644 --- a/deploy/helm/sumologic/templates/_helpers.tpl +++ b/deploy/helm/sumologic/templates/_helpers.tpl @@ -754,9 +754,9 @@ Example usage: {{- $type := .Type -}} {{- $endpoint := .Endpoint -}} {{- if not $endpoint -}} -{{- $source := (index $ctx.sumologic.sources $type "default") -}} -{{- if (index $ctx.sumologic.sources $type .Name "config-name") -}} -{{- $endpoint = index $ctx.sumologic.sources $type .Name "config-name" -}} +{{- $source := (index $ctx.sumologic.collector.sources $type "default") -}} +{{- if (index $ctx.sumologic.collector.sources $type .Name "config-name") -}} +{{- $endpoint = index $ctx.sumologic.collector.sources $type .Name "config-name" -}} {{- else -}} {{- $endpoint = printf "endpoint-%s" (include "terraform.sources.name" (dict "Name" $name "Type" $type)) -}} {{- end -}} @@ -818,8 +818,8 @@ Example Usage: {{- $ctx := .Context -}} {{- $name := .Name -}} {{- $value := true -}} -{{- if and (hasKey $ctx.sumologic.sources $type) (hasKey (index $ctx.sumologic.sources $type) $name) (hasKey (index $ctx.sumologic.sources $type $name) "create") -}} -{{- if not (index $ctx.sumologic.sources $type $name "create") -}} +{{- if and (hasKey $ctx.sumologic.collector.sources $type) (hasKey (index $ctx.sumologic.collector.sources $type) $name) (hasKey (index $ctx.sumologic.collector.sources $type $name) "create") -}} +{{- if not (index $ctx.sumologic.collector.sources $type $name "create") -}} {{- $value = false -}} {{- end -}} {{- end -}} @@ -836,7 +836,7 @@ Example: {{- define "kubernetes.sources.envs" -}} {{- $ctx := .Context -}} {{- $type := .Type -}} -{{- range $key, $source := (index .Context.sumologic.sources $type) }} +{{- range $key, $source := (index .Context.sumologic.collector.sources $type) }} - name: {{ template "terraform.sources.endpoint" (include "terraform.sources.name" (dict "Name" $key "Type" $type)) }} valueFrom: secretKeyRef: diff --git a/deploy/helm/sumologic/upgrade-2.0.0.sh b/deploy/helm/sumologic/upgrade-2.0.0.sh index 2833d29553..fea08a37ad 100755 --- a/deploy/helm/sumologic/upgrade-2.0.0.sh +++ b/deploy/helm/sumologic/upgrade-2.0.0.sh @@ -149,6 +149,40 @@ function migrate_prometheus_operator_to_kube_prometheus_stack() { yq d -i "${TEMP_FILE}" "prometheus-operator" } +function migrate_sumologic_sources() { + # Nothing to migrate, return + if [[ -z $(yq r "${TEMP_FILE}" sumologic.sources) ]] ; then + return + fi + + info "Migrating sumologic.sources to sumologic.collector.sources" + yq m -i \ + "${TEMP_FILE}" \ + <( + yq p \ + <(yq r "${TEMP_FILE}" "sumologic.sources") \ + "sumologic.collector.sources" \ + ) + yq d -i "${TEMP_FILE}" "sumologic.sources" +} + +function migrate_sumologic_setup_fields() { + # Nothing to migrate, return + if [[ -z $(yq r "${TEMP_FILE}" sumologic.setup.fields) ]] ; then + return + fi + + info "Migrating sumologic.setup.fields to sumologic.collector.fields" + yq m -i \ + "${TEMP_FILE}" \ + <( + yq p \ + <(yq r "${TEMP_FILE}" "sumologic.setup.fields") \ + "sumologic.collector.fields" \ + ) + yq d -i "${TEMP_FILE}" "sumologic.setup.fields" +} + function delete_migrated_unused_keys() { IFS=$'\n' read -r -d ' ' -a MAPPINGS_KEYS_TO_DELETE <<< "${KEYS_TO_DELETE}" readonly MAPPINGS_KEYS_TO_DELETE @@ -214,14 +248,6 @@ function migrate_customer_keys() { echo } -function migrate_pre_upgrade_hook() { - # Keep pre-upgrade hook - if [[ -n "$(yq r "${TEMP_FILE}" -- sumologic.setup)" ]]; then - info "Updating setup hooks (sumologic.setup.*.annotations[helm.sh/hook]) to 'pre-install,pre-upgrade'" - yq w -i "${TEMP_FILE}" -- 'sumologic.setup.*.annotations[helm.sh/hook]' 'pre-install,pre-upgrade' - fi -} - function get_regex() { # Get regex from old yaml file and strip `'` and `"` from beginning/end of it local write_index="${1}" @@ -274,8 +300,9 @@ create_temp_file migrate_customer_keys -migrate_pre_upgrade_hook migrate_prometheus_operator_to_kube_prometheus_stack +migrate_sumologic_sources +migrate_sumologic_setup_fields check_user_image diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index cb28d98f4a..e2049151dc 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -103,10 +103,52 @@ sumologic: ## Add custom annotations only to setup job pod podAnnotations: {} + collector: ## Configuration of additional collector fields ## https://help.sumologic.com/Manage/Fields#http-source-fields fields: {} + ## Configuration of http sources + ## See docs/Terraform.md for more information + ## name: source name visible in sumologic platform + ## config-name: This is mostly for backward compatibility + sources: + metrics: + default: + name: (default-metrics) + config-name: endpoint-metrics + apiserver: + name: apiserver-metrics + config-name: endpoint-metrics-apiserver + controller: + name: kube-controller-manager-metrics + config-name: endpoint-metrics-kube-controller-manager + scheduler: + name: kube-scheduler-metrics + config-name: endpoint-metrics-kube-scheduler + state: + name: kube-state-metrics + config-name: endpoint-metrics-kube-state + kubelet: + name: kubelet-metrics + config-name: endpoint-metrics-kubelet + node: + name: node-exporter-metrics + config-name: endpoint-metrics-node-exporter + control-plane: + name: control-plane-metrics + logs: + default: + name: logs + config-name: endpoint-logs + + ## Properties can be used to extend default settings, such as processing rules, fields etc + #properties: + # filters: + # - name: "Test Exclude Debug" + # filter_type: "Exclude" + # regexp: ".*DEBUG.*" + ### Logs configuration ## Set the enabled flag to false for disabling logs ingestion altogether. logs: @@ -130,47 +172,6 @@ sumologic: metrics: enabled: true - ## Configuration of http sources - ## See docs/Terraform.md for more information - ## name: source name visible in sumologic platform - ## config-name: This is mostly for backward compatibility - sources: - metrics: - default: - name: (default-metrics) - config-name: endpoint-metrics - apiserver: - name: apiserver-metrics - config-name: endpoint-metrics-apiserver - controller: - name: kube-controller-manager-metrics - config-name: endpoint-metrics-kube-controller-manager - scheduler: - name: kube-scheduler-metrics - config-name: endpoint-metrics-kube-scheduler - state: - name: kube-state-metrics - config-name: endpoint-metrics-kube-state - kubelet: - name: kubelet-metrics - config-name: endpoint-metrics-kubelet - node: - name: node-exporter-metrics - config-name: endpoint-metrics-node-exporter - control-plane: - name: control-plane-metrics - logs: - default: - name: logs - config-name: endpoint-logs - - ## Properties can be used to extend default settings, such as processing rules, fields etc - #properties: - # filters: - # - name: "Test Exclude Debug" - # filter_type: "Exclude" - # regexp: ".*DEBUG.*" - events: default: name: events @@ -636,7 +637,7 @@ fluentd: ## Output configuration ## tag: tag for fluentd match ## id: sumologic output @id - ## endpoint: [optional] [output key by default] It should match the sumologic.sources.metrics[].endpoint + ## endpoint: [optional] [output key by default] It should match the sumologic.collector.sources.metrics[].endpoint ## drop: [optional] [false by default] Change it to true to drop traffic for specific output ## weight: [optional [0 by default] Order of fluentd output (lower means higher priority) ## The weight has meaning in case of fluentd matches diff --git a/tests/upgrade_v2_script/static/sumologic_collector_fields.input.yaml b/tests/upgrade_v2_script/static/sumologic_collector_fields.input.yaml new file mode 100644 index 0000000000..ee20ce81ed --- /dev/null +++ b/tests/upgrade_v2_script/static/sumologic_collector_fields.input.yaml @@ -0,0 +1,36 @@ +sumologic: + setupEnabled: true + cleanUpEnabled: false + ## Exclude Kubernetes internal traffic from proxy + noProxy: kubernetes.default.svc + ## Collector name + collectorName: "dummy" + ## Cluster name: Note spaces are not allowed and will be replaced with dashes. + clusterName: "kubernetes" + ## If you set it to false, it would set EXCLUDE_NAMESPACE= and not add the fluentD/fluent-bit logs and Prometheus remotestorage metrics. + collectionMonitoring: true + ## Add custom labels to the following sumologic resources(fluentd sts, setup job, otelcol deployment) + podLabels: {} + ## Add custom annotations to the following sumologic resources(fluentd sts, setup job, otelcol deployment) + podAnnotations: {} + setup: + job: + image: + repository: sumologic/kubernetes-fluentd + tag: 1.3.0 + pullPolicy: IfNotPresent + resources: + limits: + memory: 256Mi + cpu: 2000m + requests: + memory: 64Mi + cpu: 200m + nodeSelector: {} + ## Add custom labels only to setup job pod + podLabels: {} + ## Add custom annotations only to setup job pod + podAnnotations: {} + ## Configuration of additional collector fields + ## https://help.sumologic.com/Manage/Fields#http-source-fields + fields: {} \ No newline at end of file diff --git a/tests/upgrade_v2_script/static/sumologic_collector_fields.log b/tests/upgrade_v2_script/static/sumologic_collector_fields.log new file mode 100644 index 0000000000..6b094fe43b --- /dev/null +++ b/tests/upgrade_v2_script/static/sumologic_collector_fields.log @@ -0,0 +1,6 @@ +[INFO] Mapping sumologic.setup.fields into sumologic.collector.fields + +[INFO] Updating setup hooks (sumologic.setup.*.annotations[helm.sh/hook]) to 'pre-install,pre-upgrade' + +Thank you for upgrading to v2.0.0 of the Sumo Logic Kubernetes Collection Helm chart. +A new yaml file has been generated for you. Please check the current directory for new_values.yaml. diff --git a/tests/upgrade_v2_script/static/sumologic_collector_fields.output.yaml b/tests/upgrade_v2_script/static/sumologic_collector_fields.output.yaml new file mode 100644 index 0000000000..5ba64832bd --- /dev/null +++ b/tests/upgrade_v2_script/static/sumologic_collector_fields.output.yaml @@ -0,0 +1,29 @@ +sumologic: + setupEnabled: true + cleanUpEnabled: false + noProxy: kubernetes.default.svc + collectorName: dummy + clusterName: kubernetes + collectionMonitoring: true + podLabels: {} + podAnnotations: {} + setup: + job: + image: + repository: sumologic/kubernetes-fluentd + tag: 1.3.0 + pullPolicy: IfNotPresent + resources: + limits: + memory: 256Mi + cpu: 2000m + requests: + memory: 64Mi + cpu: 200m + nodeSelector: {} + podLabels: {} + podAnnotations: {} + annotations: + helm.sh/hook: pre-install,pre-upgrade + collector: + fields: {} diff --git a/tests/upgrade_v2_script/static/sumologic_sources.input.yaml b/tests/upgrade_v2_script/static/sumologic_sources.input.yaml new file mode 100644 index 0000000000..28c8b77a84 --- /dev/null +++ b/tests/upgrade_v2_script/static/sumologic_sources.input.yaml @@ -0,0 +1,35 @@ +sumologic: + setupEnabled: true + setup: + fields: + - dummyfield1 + - dummyfield2 + sources: + metrics: + default: + name: (default-metrics) + config-name: endpoint-metrics + apiserver: + name: apiserver-metrics + config-name: endpoint-metrics-apiserver + controller: + name: kube-controller-manager-metrics + config-name: endpoint-metrics-kube-controller-manager + scheduler: + name: kube-scheduler-metrics + config-name: endpoint-metrics-kube-scheduler + state: + name: kube-state-metrics + config-name: endpoint-metrics-kube-state + kubelet: + name: kubelet-metrics + config-name: endpoint-metrics-kubelet + node: + name: node-exporter-metrics + config-name: endpoint-metrics-node-exporter + control-plane: + name: control-plane-metrics + logs: + default: + name: logs + config-name: endpoint-logs diff --git a/tests/upgrade_v2_script/static/sumologic_sources.log b/tests/upgrade_v2_script/static/sumologic_sources.log new file mode 100644 index 0000000000..d744f1dac1 --- /dev/null +++ b/tests/upgrade_v2_script/static/sumologic_sources.log @@ -0,0 +1,6 @@ + +[INFO] Migrating sumologic.sources to sumologic.collector.sources +[INFO] Migrating sumologic.setup.fields to sumologic.collector.fields + +Thank you for upgrading to v2.0.0 of the Sumo Logic Kubernetes Collection Helm chart. +A new yaml file has been generated for you. Please check the current directory for new_values.yaml. diff --git a/tests/upgrade_v2_script/static/sumologic_sources.output.yaml b/tests/upgrade_v2_script/static/sumologic_sources.output.yaml new file mode 100644 index 0000000000..eb7e2afe64 --- /dev/null +++ b/tests/upgrade_v2_script/static/sumologic_sources.output.yaml @@ -0,0 +1,36 @@ +sumologic: + setupEnabled: true + setup: {} + collector: + sources: + metrics: + default: + name: (default-metrics) + config-name: endpoint-metrics + apiserver: + name: apiserver-metrics + config-name: endpoint-metrics-apiserver + controller: + name: kube-controller-manager-metrics + config-name: endpoint-metrics-kube-controller-manager + scheduler: + name: kube-scheduler-metrics + config-name: endpoint-metrics-kube-scheduler + state: + name: kube-state-metrics + config-name: endpoint-metrics-kube-state + kubelet: + name: kubelet-metrics + config-name: endpoint-metrics-kubelet + node: + name: node-exporter-metrics + config-name: endpoint-metrics-node-exporter + control-plane: + name: control-plane-metrics + logs: + default: + name: logs + config-name: endpoint-logs + fields: + - dummyfield1 + - dummyfield2 From 2b740e241fa9920ff9e814e0b7c549a3b9fb25ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20Ma=C5=82ek?= Date: Wed, 25 Nov 2020 12:45:30 +0100 Subject: [PATCH 2/4] Fix dependent docs and review tweaks --- deploy/docs/Terraform.md | 101 ++++++++++-------- .../sumologic_collector_fields.input.yaml | 2 +- 2 files changed, 60 insertions(+), 43 deletions(-) diff --git a/deploy/docs/Terraform.md b/deploy/docs/Terraform.md index 6507ff22e1..c1271ce8b6 100644 --- a/deploy/docs/Terraform.md +++ b/deploy/docs/Terraform.md @@ -4,12 +4,17 @@ Terraform is used by sumologic-kubernetes-collection during the setup process to automatically create HTTP sources and store their URLs in the Kubernetes secret. We are using two providers to perform those actions: - * [Kubernetes Terraform provider](https://www.terraform.io/docs/providers/kubernetes/) - * [Sumo Logic Terraform provider](https://www.terraform.io/docs/providers/sumologic/) + +* [Kubernetes Terraform provider](https://www.terraform.io/docs/providers/kubernetes/) +* [Sumo Logic Terraform provider](https://www.terraform.io/docs/providers/sumologic/) ## Kubernetes Terraform provider -[Kubernetes Terraform provider](https://www.terraform.io/docs/providers/kubernetes/) is responsible for creating the secret with the created HTTP source endpoints during setup process. The default configuration is expected to work in most cases, however, for self-hosted Kubernetes clusters there can be a few exceptions. For these cases we expose the provider configuration in `values.yaml`. +[Kubernetes Terraform provider](https://www.terraform.io/docs/providers/kubernetes/) +is responsible for creating the secret with the created HTTP source endpoints during setup process. +The default configuration is expected to work in most cases, however, for self-hosted +Kubernetes clusters there can be a few exceptions. +For these cases we expose the provider configuration in `values.yaml`. ```yaml sumologic: @@ -47,17 +52,19 @@ The related configuration section in the `values.yaml` file is under `sumologic. ```yaml sumologic: # ... - sources: + collector: # ... - logs: # source type, can be one of: logs/metrics/events/traces - example-source: # source reference name - name: # name of the source (visible on the sumologic platform) - config-name: # name which be used in secret to store the url. This is backward-compatibility option - category: # this is backward compatibility property. It's deprecated and it's going to be removed in version 2.0 - # Sets source category to "${var.cluster_name}/${local.default_events_source}" if true - # To overwrite category, please use `sumologic.collector.sources[].properties.category` - properties: # Additional Terraform properties like fields or content_type - # ref: https://www.terraform.io/docs/providers/sumologic/r/collector.html + sources: + # ... + logs: # source type, can be one of: logs/metrics/events/traces + example-source: # source reference name + name: # name of the source (visible on the sumologic platform) + config-name: # name which be used in secret to store the url. This is backward-compatibility option + category: # this is backward compatibility property. It's deprecated and it's going to be removed in version 2.0 + # Sets source category to "${var.cluster_name}/${local.default_events_source}" if true + # To overwrite category, please use `sumologic.collector.sources[].properties.category` + properties: # Additional Terraform properties like fields or content_type + # ref: https://www.terraform.io/docs/providers/sumologic/r/collector.html ``` ### Usage @@ -67,8 +74,9 @@ The variable name is built using the schema `SUMO_ENDPOINT__` and `` are in uppercase and dashes are replaced with underscores. Examples: - - `sumologic.collector.sources.logs.example-source` becomes `SUMO_ENDPOINT_EXAMPLE_SOURCE_LOGS_SOURCE` - - `sumologic.collector.sources.traces.default` becomes `SUMO_ENDPOINT_DEFAULT_TRACES_SOURCE` + +* `sumologic.collector.sources.logs.example-source` becomes `SUMO_ENDPOINT_EXAMPLE_SOURCE_LOGS_SOURCE` +* `sumologic.collector.sources.traces.default` becomes `SUMO_ENDPOINT_DEFAULT_TRACES_SOURCE` ### Properties @@ -77,43 +85,50 @@ using `sumologic.collector.sources.. Date: Wed, 25 Nov 2020 15:36:31 +0100 Subject: [PATCH 3/4] Fix tests --- deploy/helm/sumologic/values.yaml | 23 +++--- tests/terraform/static/all_fields.input.yaml | 75 ++++++++++--------- .../static/collector_fields.input.yaml | 2 +- .../static/disable_default_metrics.input.yaml | 9 ++- .../upgrade_v2_script/static/image_rename.log | 1 - .../static/image_rename.output.yaml | 2 - .../static/sumologic_collector_fields.log | 3 +- .../sumologic_collector_fields.output.yaml | 2 - 8 files changed, 56 insertions(+), 61 deletions(-) diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index e2049151dc..5cde31e5c3 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -148,6 +148,17 @@ sumologic: # - name: "Test Exclude Debug" # filter_type: "Exclude" # regexp: ".*DEBUG.*" + events: + default: + name: events + config-name: endpoint-events + category: true + traces: + default: + name: traces + config-name: endpoint-traces + properties: + content_type: Zipkin ### Logs configuration ## Set the enabled flag to false for disabling logs ingestion altogether. @@ -172,18 +183,6 @@ sumologic: metrics: enabled: true - events: - default: - name: events - config-name: endpoint-events - category: true - traces: - default: - name: traces - config-name: endpoint-traces - properties: - content_type: Zipkin - ### Traces configuration ## This is experimental feature and may be unavailable for your account traces: diff --git a/tests/terraform/static/all_fields.input.yaml b/tests/terraform/static/all_fields.input.yaml index 9144ddde69..4b5b309392 100644 --- a/tests/terraform/static/all_fields.input.yaml +++ b/tests/terraform/static/all_fields.input.yaml @@ -1,38 +1,39 @@ sumologic: - sources: - metrics: - test_source: - name: "(Test source)" - properties: - description: "This is complex test" - category: "custom/category" - host_name: "localhost" - timezone: "local timezone" - automatic_date_parsing: false - multiline_processing_enabled: true - use_autoline_matching: false - manual_prefix_regexp: '.*' - force_timezone: true - default_date_formats: - - format: format_1 - locator: locator_1 - - format: format_2 - locator: locator_2 - filters: - - name: filter.no1 - filter_type: Include - regexp: '[a-z]+' - mask: 'some_mask' - - name: filter.no2 - filter_type: Exclude - regexp: '.+' - mask: 'another_mask' - cutoff_timestamp: 40 - cutoff_relative_time: 'some value' - fields: - some_field: some_value - another_field: other_value - content_type: 'application/json' - test_list: - - point_1 - - point_2 + collector: + sources: + metrics: + test_source: + name: "(Test source)" + properties: + description: "This is complex test" + category: "custom/category" + host_name: "localhost" + timezone: "local timezone" + automatic_date_parsing: false + multiline_processing_enabled: true + use_autoline_matching: false + manual_prefix_regexp: '.*' + force_timezone: true + default_date_formats: + - format: format_1 + locator: locator_1 + - format: format_2 + locator: locator_2 + filters: + - name: filter.no1 + filter_type: Include + regexp: '[a-z]+' + mask: 'some_mask' + - name: filter.no2 + filter_type: Exclude + regexp: '.+' + mask: 'another_mask' + cutoff_timestamp: 40 + cutoff_relative_time: 'some value' + fields: + some_field: some_value + another_field: other_value + content_type: 'application/json' + test_list: + - point_1 + - point_2 diff --git a/tests/terraform/static/collector_fields.input.yaml b/tests/terraform/static/collector_fields.input.yaml index 3946431eb6..51bcbfd900 100644 --- a/tests/terraform/static/collector_fields.input.yaml +++ b/tests/terraform/static/collector_fields.input.yaml @@ -1,5 +1,5 @@ sumologic: - setup: + collector: fields: test_fields: test_value here_is_very_long_field_name: another_value diff --git a/tests/terraform/static/disable_default_metrics.input.yaml b/tests/terraform/static/disable_default_metrics.input.yaml index b277f34949..e337299a2b 100644 --- a/tests/terraform/static/disable_default_metrics.input.yaml +++ b/tests/terraform/static/disable_default_metrics.input.yaml @@ -1,5 +1,6 @@ sumologic: - sources: - metrics: - default: - create: false + collector: + sources: + metrics: + default: + create: false diff --git a/tests/upgrade_v2_script/static/image_rename.log b/tests/upgrade_v2_script/static/image_rename.log index d7269f1c00..d0fef95995 100644 --- a/tests/upgrade_v2_script/static/image_rename.log +++ b/tests/upgrade_v2_script/static/image_rename.log @@ -8,7 +8,6 @@ [INFO] - fluentd.image.pullPolicy [INFO] - sumologic.setup.job.image.pullPolicy -[INFO] Updating setup hooks (sumologic.setup.*.annotations[helm.sh/hook]) to 'pre-install,pre-upgrade' [INFO] Migrating from image.tag '1.3.0' to sumologic.setup.job.image.tag '2.0.0' [INFO] Migrating from image.tag '1.3.0' to fluentd.image.tag '2.0.0' diff --git a/tests/upgrade_v2_script/static/image_rename.output.yaml b/tests/upgrade_v2_script/static/image_rename.output.yaml index bf7a6c4317..bbb1b1660f 100644 --- a/tests/upgrade_v2_script/static/image_rename.output.yaml +++ b/tests/upgrade_v2_script/static/image_rename.output.yaml @@ -15,8 +15,6 @@ sumologic: repository: sumologic/kubernetes-fluentd tag: 2.0.0 pullPolicy: IfNotPresent - annotations: - helm.sh/hook: pre-install,pre-upgrade setupEnabled: true cleanUpEnabled: false accessId: dummy diff --git a/tests/upgrade_v2_script/static/sumologic_collector_fields.log b/tests/upgrade_v2_script/static/sumologic_collector_fields.log index 6b094fe43b..b036a03839 100644 --- a/tests/upgrade_v2_script/static/sumologic_collector_fields.log +++ b/tests/upgrade_v2_script/static/sumologic_collector_fields.log @@ -1,6 +1,5 @@ -[INFO] Mapping sumologic.setup.fields into sumologic.collector.fields -[INFO] Updating setup hooks (sumologic.setup.*.annotations[helm.sh/hook]) to 'pre-install,pre-upgrade' +[INFO] Migrating sumologic.setup.fields to sumologic.collector.fields Thank you for upgrading to v2.0.0 of the Sumo Logic Kubernetes Collection Helm chart. A new yaml file has been generated for you. Please check the current directory for new_values.yaml. diff --git a/tests/upgrade_v2_script/static/sumologic_collector_fields.output.yaml b/tests/upgrade_v2_script/static/sumologic_collector_fields.output.yaml index 5ba64832bd..7d9c1ae68e 100644 --- a/tests/upgrade_v2_script/static/sumologic_collector_fields.output.yaml +++ b/tests/upgrade_v2_script/static/sumologic_collector_fields.output.yaml @@ -23,7 +23,5 @@ sumologic: nodeSelector: {} podLabels: {} podAnnotations: {} - annotations: - helm.sh/hook: pre-install,pre-upgrade collector: fields: {} From 604fe221ae3f3605a2bacb4bf0222abad8bcebcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20Ma=C5=82ek?= Date: Wed, 25 Nov 2020 18:15:38 +0100 Subject: [PATCH 4/4] Change aterisks to dashes in .md when using an unordered list --- deploy/docs/Terraform.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/deploy/docs/Terraform.md b/deploy/docs/Terraform.md index c1271ce8b6..6cb1051fef 100644 --- a/deploy/docs/Terraform.md +++ b/deploy/docs/Terraform.md @@ -5,8 +5,8 @@ to automatically create HTTP sources and store their URLs in the Kubernetes secr We are using two providers to perform those actions: -* [Kubernetes Terraform provider](https://www.terraform.io/docs/providers/kubernetes/) -* [Sumo Logic Terraform provider](https://www.terraform.io/docs/providers/sumologic/) +- [Kubernetes Terraform provider](https://www.terraform.io/docs/providers/kubernetes/) +- [Sumo Logic Terraform provider](https://www.terraform.io/docs/providers/sumologic/) ## Kubernetes Terraform provider @@ -75,8 +75,8 @@ where `` and `` are in uppercase and dashes are replac Examples: -* `sumologic.collector.sources.logs.example-source` becomes `SUMO_ENDPOINT_EXAMPLE_SOURCE_LOGS_SOURCE` -* `sumologic.collector.sources.traces.default` becomes `SUMO_ENDPOINT_DEFAULT_TRACES_SOURCE` +- `sumologic.collector.sources.logs.example-source` becomes `SUMO_ENDPOINT_EXAMPLE_SOURCE_LOGS_SOURCE` +- `sumologic.collector.sources.traces.default` becomes `SUMO_ENDPOINT_DEFAULT_TRACES_SOURCE` ### Properties @@ -147,8 +147,8 @@ sumologic: List of available variables: -* `var.cluster_name` -* `var.namespace_name` -* `var.collector_name` +- `var.cluster_name` +- `var.namespace_name` +- `var.collector_name` **Note** You have to manually activate fields using the Sumo Logic service.