From 7b1e73300d87687214284fe6814369a86d3fbf85 Mon Sep 17 00:00:00 2001 From: Dominik Rosiek Date: Thu, 7 Jan 2021 15:05:34 +0100 Subject: [PATCH] Fix upgrade-script for logs migrations for v1.0 --- deploy/helm/sumologic/upgrade-1.0.0.sh | 16 ++++---- .../key_mapping_one_to_multiple.input.yaml | 8 ++++ .../static/key_mapping_one_to_multiple.log | 40 +++++++++++++++++++ .../key_mapping_one_to_multiple.output.yaml | 36 +++++++++++++++++ .../static/key_mapping_one_to_one.input.yaml | 8 ---- .../static/key_mapping_one_to_one.log | 8 ---- .../static/key_mapping_one_to_one.output.yaml | 9 ----- 7 files changed, 92 insertions(+), 33 deletions(-) diff --git a/deploy/helm/sumologic/upgrade-1.0.0.sh b/deploy/helm/sumologic/upgrade-1.0.0.sh index 2156264962..340a8a6ec2 100755 --- a/deploy/helm/sumologic/upgrade-1.0.0.sh +++ b/deploy/helm/sumologic/upgrade-1.0.0.sh @@ -24,10 +24,6 @@ sumologic.addTimestamp:fluentd.logs.output.addTimestamp sumologic.chunkLimitSize:fluentd.buffer.chunkLimitSize sumologic.eventCollectionEnabled:fluentd.events.enabled sumologic.events.sourceCategory:fluentd.events.sourceCategory -sumologic.excludeContainerRegex:fluentd.logs.containers.excludeContainerRegex -sumologic.excludeHostRegex:fluentd.logs.containers.excludeHostRegex -sumologic.excludeNamespaceRegex:fluentd.logs.containers.excludeNamespaceRegex -sumologic.excludePodRegex:fluentd.logs.containers.excludePodRegex sumologic.fluentd.buffer:fluentd.buffer.type sumologic.fluentdLogLevel:fluentd.logLevel sumologic.flushInterval:fluentd.buffer.flushInterval @@ -39,10 +35,6 @@ sumologic.k8sMetadataFilter.watch:fluentd.logs.containers.k8sMetadataFilter.watc sumologic.logFormat:fluentd.logs.output.logFormat sumologic.numThreads:fluentd.buffer.numThreads sumologic.queueChunkLimitSize:fluentd.buffer.queueChunkLimitSize -sumologic.sourceCategory:fluentd.logs.containers.sourceCategory -sumologic.sourceCategoryPrefix:fluentd.logs.containers.sourceCategoryPrefix -sumologic.sourceCategoryReplaceDash:fluentd.logs.containers.sourceCategoryReplaceDash -sumologic.sourceName:fluentd.logs.containers.sourceName sumologic.timestampKey:fluentd.logs.output.timestampKey sumologic.totalLimitSize:fluentd.buffer.totalLimitSize sumologic.verifySsl:fluentd.verifySsl @@ -62,6 +54,14 @@ sumologic.fluentd.autoscaling.enabled:fluentd.logs.autoscaling.enabled:fluentd.m sumologic.fluentd.autoscaling.maxReplicas:fluentd.logs.autoscaling.maxReplicas:fluentd.metrics.autoscaling.maxReplicas sumologic.fluentd.autoscaling.minReplicas:fluentd.logs.autoscaling.minReplicas:fluentd.metrics.autoscaling.minReplicas sumologic.fluentd.autoscaling.targetCPUUtilizationPercentage:fluentd.logs.autoscaling.targetCPUUtilizationPercentage:fluentd.metrics.autoscaling.targetCPUUtilizationPercentage +sumologic.excludeContainerRegex:fluentd.logs.containers.excludeContainerRegex:fluentd.logs.default.excludeContainerRegex:fluentd.logs.systemd.excludeContainerRegex:fluentd.logs.kubelet.excludeContainerRegex +sumologic.excludeHostRegex:fluentd.logs.containers.excludeHostRegex:fluentd.logs.default.excludeHostRegex:fluentd.logs.systemd.excludeHostRegex:fluentd.logs.kubelet.excludeHostRegex +sumologic.excludeNamespaceRegex:fluentd.logs.containers.excludeNamespaceRegex:fluentd.logs.default.excludeNamespaceRegex:fluentd.logs.systemd.excludeNamespaceRegex:fluentd.logs.kubelet.excludeNamespaceRegex +sumologic.excludePodRegex:fluentd.logs.containers.excludePodRegex:fluentd.logs.default.excludePodRegex:fluentd.logs.systemd.excludePodRegex:fluentd.logs.kubelet.excludePodRegex +sumologic.sourceCategory:fluentd.logs.containers.sourceCategory:fluentd.logs.default.sourceCategory:fluentd.logs.systemd.sourceCategory:fluentd.logs.kubelet.sourceCategory +sumologic.sourceCategoryPrefix:fluentd.logs.containers.sourceCategoryPrefix:fluentd.logs.default.sourceCategoryPrefix:fluentd.logs.systemd.sourceCategoryPrefix:fluentd.logs.kubelet.sourceCategoryPrefix +sumologic.sourceCategoryReplaceDash:fluentd.logs.containers.sourceCategoryReplaceDash:fluentd.logs.default.sourceCategoryReplaceDash:fluentd.logs.systemd.sourceCategoryReplaceDash:fluentd.logs.kubelet.sourceCategoryReplaceDash +sumologic.sourceName:fluentd.logs.containers.sourceName:fluentd.logs.default.sourceName:fluentd.logs.systemd.sourceName:fluentd.logs.kubelet.sourceName " readonly KEY_MAPPINGS_EMPTY=" diff --git a/tests/upgrade_script/static/key_mapping_one_to_multiple.input.yaml b/tests/upgrade_script/static/key_mapping_one_to_multiple.input.yaml index 841899eb21..4dbf8cc40e 100644 --- a/tests/upgrade_script/static/key_mapping_one_to_multiple.input.yaml +++ b/tests/upgrade_script/static/key_mapping_one_to_multiple.input.yaml @@ -18,3 +18,11 @@ sumologic: maxReplicas: fluentd.logs.autoscaling.maxReplicas, fluentd.metrics.autoscaling.maxReplicas minReplicas: fluentd.logs.autoscaling.minReplicas, fluentd.metrics.autoscaling.minReplicas targetCPUUtilizationPercentage: fluentd.logs.autoscaling.targetCPUUtilizationPercentage, fluentd.metrics.autoscaling.targetCPUUtilizationPercentage + excludeContainerRegex: fluentd.logs.containers.excludeContainerRegex + excludeHostRegex: fluentd.logs.containers.excludeHostRegex + excludeNamespaceRegex: fluentd.logs.containers.excludeNamespaceRegex + excludePodRegex: fluentd.logs.containers.excludePodRegex + sourceCategory: fluentd.logs.containers.sourceCategory + sourceCategoryPrefix: fluentd.logs.containers.sourceCategoryPrefix + sourceCategoryReplaceDash: fluentd.logs.containers.sourceCategoryReplaceDash + sourceName: fluentd.logs.containers.sourceName diff --git a/tests/upgrade_script/static/key_mapping_one_to_multiple.log b/tests/upgrade_script/static/key_mapping_one_to_multiple.log index efe3697f09..eaf221e9f5 100644 --- a/tests/upgrade_script/static/key_mapping_one_to_multiple.log +++ b/tests/upgrade_script/static/key_mapping_one_to_multiple.log @@ -37,6 +37,46 @@ [INFO] Mapping sumologic.fluentd.autoscaling.targetCPUUtilizationPercentage into: [INFO] - fluentd.logs.autoscaling.targetCPUUtilizationPercentage [INFO] - fluentd.metrics.autoscaling.targetCPUUtilizationPercentage +[INFO] Mapping sumologic.excludeContainerRegex into: +[INFO] - fluentd.logs.containers.excludeContainerRegex +[INFO] - fluentd.logs.default.excludeContainerRegex +[INFO] - fluentd.logs.systemd.excludeContainerRegex +[INFO] - fluentd.logs.kubelet.excludeContainerRegex +[INFO] Mapping sumologic.excludeHostRegex into: +[INFO] - fluentd.logs.containers.excludeHostRegex +[INFO] - fluentd.logs.default.excludeHostRegex +[INFO] - fluentd.logs.systemd.excludeHostRegex +[INFO] - fluentd.logs.kubelet.excludeHostRegex +[INFO] Mapping sumologic.excludeNamespaceRegex into: +[INFO] - fluentd.logs.containers.excludeNamespaceRegex +[INFO] - fluentd.logs.default.excludeNamespaceRegex +[INFO] - fluentd.logs.systemd.excludeNamespaceRegex +[INFO] - fluentd.logs.kubelet.excludeNamespaceRegex +[INFO] Mapping sumologic.excludePodRegex into: +[INFO] - fluentd.logs.containers.excludePodRegex +[INFO] - fluentd.logs.default.excludePodRegex +[INFO] - fluentd.logs.systemd.excludePodRegex +[INFO] - fluentd.logs.kubelet.excludePodRegex +[INFO] Mapping sumologic.sourceCategory into: +[INFO] - fluentd.logs.containers.sourceCategory +[INFO] - fluentd.logs.default.sourceCategory +[INFO] - fluentd.logs.systemd.sourceCategory +[INFO] - fluentd.logs.kubelet.sourceCategory +[INFO] Mapping sumologic.sourceCategoryPrefix into: +[INFO] - fluentd.logs.containers.sourceCategoryPrefix +[INFO] - fluentd.logs.default.sourceCategoryPrefix +[INFO] - fluentd.logs.systemd.sourceCategoryPrefix +[INFO] - fluentd.logs.kubelet.sourceCategoryPrefix +[INFO] Mapping sumologic.sourceCategoryReplaceDash into: +[INFO] - fluentd.logs.containers.sourceCategoryReplaceDash +[INFO] - fluentd.logs.default.sourceCategoryReplaceDash +[INFO] - fluentd.logs.systemd.sourceCategoryReplaceDash +[INFO] - fluentd.logs.kubelet.sourceCategoryReplaceDash +[INFO] Mapping sumologic.sourceName into: +[INFO] - fluentd.logs.containers.sourceName +[INFO] - fluentd.logs.default.sourceName +[INFO] - fluentd.logs.systemd.sourceName +[INFO] - fluentd.logs.kubelet.sourceName [INFO] falco will be disabled. Change "falco.enabled" to "true" if you want to enable it diff --git a/tests/upgrade_script/static/key_mapping_one_to_multiple.output.yaml b/tests/upgrade_script/static/key_mapping_one_to_multiple.output.yaml index 6981a7fc31..c847382a1c 100644 --- a/tests/upgrade_script/static/key_mapping_one_to_multiple.output.yaml +++ b/tests/upgrade_script/static/key_mapping_one_to_multiple.output.yaml @@ -19,6 +19,42 @@ fluentd: minReplicas: fluentd.logs.autoscaling.minReplicas, fluentd.metrics.autoscaling.minReplicas targetCPUUtilizationPercentage: fluentd.logs.autoscaling.targetCPUUtilizationPercentage, fluentd.metrics.autoscaling.targetCPUUtilizationPercentage + containers: + excludeContainerRegex: fluentd.logs.containers.excludeContainerRegex + excludeHostRegex: fluentd.logs.containers.excludeHostRegex + excludeNamespaceRegex: fluentd.logs.containers.excludeNamespaceRegex + excludePodRegex: fluentd.logs.containers.excludePodRegex + sourceCategory: fluentd.logs.containers.sourceCategory + sourceCategoryPrefix: fluentd.logs.containers.sourceCategoryPrefix + sourceCategoryReplaceDash: fluentd.logs.containers.sourceCategoryReplaceDash + sourceName: fluentd.logs.containers.sourceName + default: + excludeContainerRegex: fluentd.logs.containers.excludeContainerRegex + excludeHostRegex: fluentd.logs.containers.excludeHostRegex + excludeNamespaceRegex: fluentd.logs.containers.excludeNamespaceRegex + excludePodRegex: fluentd.logs.containers.excludePodRegex + sourceCategory: fluentd.logs.containers.sourceCategory + sourceCategoryPrefix: fluentd.logs.containers.sourceCategoryPrefix + sourceCategoryReplaceDash: fluentd.logs.containers.sourceCategoryReplaceDash + sourceName: fluentd.logs.containers.sourceName + systemd: + excludeContainerRegex: fluentd.logs.containers.excludeContainerRegex + excludeHostRegex: fluentd.logs.containers.excludeHostRegex + excludeNamespaceRegex: fluentd.logs.containers.excludeNamespaceRegex + excludePodRegex: fluentd.logs.containers.excludePodRegex + sourceCategory: fluentd.logs.containers.sourceCategory + sourceCategoryPrefix: fluentd.logs.containers.sourceCategoryPrefix + sourceCategoryReplaceDash: fluentd.logs.containers.sourceCategoryReplaceDash + sourceName: fluentd.logs.containers.sourceName + kubelet: + excludeContainerRegex: fluentd.logs.containers.excludeContainerRegex + excludeHostRegex: fluentd.logs.containers.excludeHostRegex + excludeNamespaceRegex: fluentd.logs.containers.excludeNamespaceRegex + excludePodRegex: fluentd.logs.containers.excludePodRegex + sourceCategory: fluentd.logs.containers.sourceCategory + sourceCategoryPrefix: fluentd.logs.containers.sourceCategoryPrefix + sourceCategoryReplaceDash: fluentd.logs.containers.sourceCategoryReplaceDash + sourceName: fluentd.logs.containers.sourceName metrics: statefulset: affinity: fluentd.logs.statefulset.affinity, fluentd.metrics.statefulset.affinity diff --git a/tests/upgrade_script/static/key_mapping_one_to_one.input.yaml b/tests/upgrade_script/static/key_mapping_one_to_one.input.yaml index fe390827cc..f1b9c48db1 100644 --- a/tests/upgrade_script/static/key_mapping_one_to_one.input.yaml +++ b/tests/upgrade_script/static/key_mapping_one_to_one.input.yaml @@ -14,10 +14,6 @@ sumologic: eventCollectionEnabled: fluentd.events.enabled events: sourceCategory: fluentd.events.sourceCategory - excludeContainerRegex: fluentd.logs.containers.excludeContainerRegex - excludeHostRegex: fluentd.logs.containers.excludeHostRegex - excludeNamespaceRegex: fluentd.logs.containers.excludeNamespaceRegex - excludePodRegex: fluentd.logs.containers.excludePodRegex fluentd: buffer: fluentd.buffer.type fluentdLogLevel: fluentd.logLevel @@ -31,10 +27,6 @@ sumologic: logFormat: fluentd.logs.output.logFormat numThreads: fluentd.buffer.numThreads queueChunkLimitSize: fluentd.buffer.queueChunkLimitSize - sourceCategory: fluentd.logs.containers.sourceCategory - sourceCategoryPrefix: fluentd.logs.containers.sourceCategoryPrefix - sourceCategoryReplaceDash: fluentd.logs.containers.sourceCategoryReplaceDash - sourceName: fluentd.logs.containers.sourceName timestampKey: fluentd.logs.output.timestampKey totalLimitSize: fluentd.buffer.totalLimitSize verifySsl: fluentd.verifySsl diff --git a/tests/upgrade_script/static/key_mapping_one_to_one.log b/tests/upgrade_script/static/key_mapping_one_to_one.log index cf7cfbd348..7c83f0ac0a 100644 --- a/tests/upgrade_script/static/key_mapping_one_to_one.log +++ b/tests/upgrade_script/static/key_mapping_one_to_one.log @@ -8,20 +8,12 @@ [INFO] Mapping sumologic.chunkLimitSize into fluentd.buffer.chunkLimitSize [INFO] Mapping sumologic.eventCollectionEnabled into fluentd.events.enabled [INFO] Mapping sumologic.events.sourceCategory into fluentd.events.sourceCategory -[INFO] Mapping sumologic.excludeContainerRegex into fluentd.logs.containers.excludeContainerRegex -[INFO] Mapping sumologic.excludeHostRegex into fluentd.logs.containers.excludeHostRegex -[INFO] Mapping sumologic.excludeNamespaceRegex into fluentd.logs.containers.excludeNamespaceRegex -[INFO] Mapping sumologic.excludePodRegex into fluentd.logs.containers.excludePodRegex [INFO] Mapping sumologic.fluentd.buffer into fluentd.buffer.type [INFO] Mapping sumologic.fluentdLogLevel into fluentd.logLevel [INFO] Mapping sumologic.flushInterval into fluentd.buffer.flushInterval [INFO] Mapping sumologic.logFormat into fluentd.logs.output.logFormat [INFO] Mapping sumologic.numThreads into fluentd.buffer.numThreads [INFO] Mapping sumologic.queueChunkLimitSize into fluentd.buffer.queueChunkLimitSize -[INFO] Mapping sumologic.sourceCategory into fluentd.logs.containers.sourceCategory -[INFO] Mapping sumologic.sourceCategoryPrefix into fluentd.logs.containers.sourceCategoryPrefix -[INFO] Mapping sumologic.sourceCategoryReplaceDash into fluentd.logs.containers.sourceCategoryReplaceDash -[INFO] Mapping sumologic.sourceName into fluentd.logs.containers.sourceName [INFO] Mapping sumologic.timestampKey into fluentd.logs.output.timestampKey [INFO] Mapping sumologic.totalLimitSize into fluentd.buffer.totalLimitSize [INFO] Mapping sumologic.verifySsl into fluentd.verifySsl diff --git a/tests/upgrade_script/static/key_mapping_one_to_one.output.yaml b/tests/upgrade_script/static/key_mapping_one_to_one.output.yaml index 1d0d0c8402..fbe39ff055 100644 --- a/tests/upgrade_script/static/key_mapping_one_to_one.output.yaml +++ b/tests/upgrade_script/static/key_mapping_one_to_one.output.yaml @@ -18,15 +18,6 @@ fluentd: addTimestamp: fluentd.logs.output.addTimestamp logFormat: fluentd.logs.output.logFormat timestampKey: fluentd.logs.output.timestampKey - containers: - excludeContainerRegex: fluentd.logs.containers.excludeContainerRegex - excludeHostRegex: fluentd.logs.containers.excludeHostRegex - excludeNamespaceRegex: fluentd.logs.containers.excludeNamespaceRegex - excludePodRegex: fluentd.logs.containers.excludePodRegex - sourceCategory: fluentd.logs.containers.sourceCategory - sourceCategoryPrefix: fluentd.logs.containers.sourceCategoryPrefix - sourceCategoryReplaceDash: fluentd.logs.containers.sourceCategoryReplaceDash - sourceName: fluentd.logs.containers.sourceName buffer: chunkLimitSize: fluentd.buffer.chunkLimitSize type: fluentd.buffer.type