Fix upgrade-script for logs migrations for v1.0

SumoLogic · Jan 7, 2021 · 7b1e733 · 7b1e733
1 parent f1bf475
commit 7b1e733
Show file tree

Hide file tree

Showing 7 changed files with 92 additions and 33 deletions.
diff --git a/deploy/helm/sumologic/upgrade-1.0.0.sh b/deploy/helm/sumologic/upgrade-1.0.0.sh
@@ -24,10 +24,6 @@ sumologic.addTimestamp:fluentd.logs.output.addTimestamp
 sumologic.chunkLimitSize:fluentd.buffer.chunkLimitSize
 sumologic.eventCollectionEnabled:fluentd.events.enabled
 sumologic.events.sourceCategory:fluentd.events.sourceCategory
-sumologic.excludeContainerRegex:fluentd.logs.containers.excludeContainerRegex
-sumologic.excludeHostRegex:fluentd.logs.containers.excludeHostRegex
-sumologic.excludeNamespaceRegex:fluentd.logs.containers.excludeNamespaceRegex
-sumologic.excludePodRegex:fluentd.logs.containers.excludePodRegex
 sumologic.fluentd.buffer:fluentd.buffer.type
 sumologic.fluentdLogLevel:fluentd.logLevel
 sumologic.flushInterval:fluentd.buffer.flushInterval
@@ -39,10 +35,6 @@ sumologic.k8sMetadataFilter.watch:fluentd.logs.containers.k8sMetadataFilter.watc
 sumologic.logFormat:fluentd.logs.output.logFormat
 sumologic.numThreads:fluentd.buffer.numThreads
 sumologic.queueChunkLimitSize:fluentd.buffer.queueChunkLimitSize
-sumologic.sourceCategory:fluentd.logs.containers.sourceCategory
-sumologic.sourceCategoryPrefix:fluentd.logs.containers.sourceCategoryPrefix
-sumologic.sourceCategoryReplaceDash:fluentd.logs.containers.sourceCategoryReplaceDash
-sumologic.sourceName:fluentd.logs.containers.sourceName
 sumologic.timestampKey:fluentd.logs.output.timestampKey
 sumologic.totalLimitSize:fluentd.buffer.totalLimitSize
 sumologic.verifySsl:fluentd.verifySsl
@@ -62,6 +54,14 @@ sumologic.fluentd.autoscaling.enabled:fluentd.logs.autoscaling.enabled:fluentd.m
 sumologic.fluentd.autoscaling.maxReplicas:fluentd.logs.autoscaling.maxReplicas:fluentd.metrics.autoscaling.maxReplicas
 sumologic.fluentd.autoscaling.minReplicas:fluentd.logs.autoscaling.minReplicas:fluentd.metrics.autoscaling.minReplicas
 sumologic.fluentd.autoscaling.targetCPUUtilizationPercentage:fluentd.logs.autoscaling.targetCPUUtilizationPercentage:fluentd.metrics.autoscaling.targetCPUUtilizationPercentage
+sumologic.excludeContainerRegex:fluentd.logs.containers.excludeContainerRegex:fluentd.logs.default.excludeContainerRegex:fluentd.logs.systemd.excludeContainerRegex:fluentd.logs.kubelet.excludeContainerRegex
+sumologic.excludeHostRegex:fluentd.logs.containers.excludeHostRegex:fluentd.logs.default.excludeHostRegex:fluentd.logs.systemd.excludeHostRegex:fluentd.logs.kubelet.excludeHostRegex
+sumologic.excludeNamespaceRegex:fluentd.logs.containers.excludeNamespaceRegex:fluentd.logs.default.excludeNamespaceRegex:fluentd.logs.systemd.excludeNamespaceRegex:fluentd.logs.kubelet.excludeNamespaceRegex
+sumologic.excludePodRegex:fluentd.logs.containers.excludePodRegex:fluentd.logs.default.excludePodRegex:fluentd.logs.systemd.excludePodRegex:fluentd.logs.kubelet.excludePodRegex
+sumologic.sourceCategory:fluentd.logs.containers.sourceCategory:fluentd.logs.default.sourceCategory:fluentd.logs.systemd.sourceCategory:fluentd.logs.kubelet.sourceCategory
+sumologic.sourceCategoryPrefix:fluentd.logs.containers.sourceCategoryPrefix:fluentd.logs.default.sourceCategoryPrefix:fluentd.logs.systemd.sourceCategoryPrefix:fluentd.logs.kubelet.sourceCategoryPrefix
+sumologic.sourceCategoryReplaceDash:fluentd.logs.containers.sourceCategoryReplaceDash:fluentd.logs.default.sourceCategoryReplaceDash:fluentd.logs.systemd.sourceCategoryReplaceDash:fluentd.logs.kubelet.sourceCategoryReplaceDash
+sumologic.sourceName:fluentd.logs.containers.sourceName:fluentd.logs.default.sourceName:fluentd.logs.systemd.sourceName:fluentd.logs.kubelet.sourceName
 "
 
 readonly KEY_MAPPINGS_EMPTY="

diff --git a/tests/upgrade_script/static/key_mapping_one_to_multiple.input.yaml b/tests/upgrade_script/static/key_mapping_one_to_multiple.input.yaml
@@ -18,3 +18,11 @@ sumologic:
       maxReplicas: fluentd.logs.autoscaling.maxReplicas, fluentd.metrics.autoscaling.maxReplicas
       minReplicas: fluentd.logs.autoscaling.minReplicas, fluentd.metrics.autoscaling.minReplicas
       targetCPUUtilizationPercentage: fluentd.logs.autoscaling.targetCPUUtilizationPercentage, fluentd.metrics.autoscaling.targetCPUUtilizationPercentage
+  excludeContainerRegex: fluentd.logs.containers.excludeContainerRegex
+  excludeHostRegex: fluentd.logs.containers.excludeHostRegex
+  excludeNamespaceRegex: fluentd.logs.containers.excludeNamespaceRegex
+  excludePodRegex: fluentd.logs.containers.excludePodRegex
+  sourceCategory: fluentd.logs.containers.sourceCategory
+  sourceCategoryPrefix: fluentd.logs.containers.sourceCategoryPrefix
+  sourceCategoryReplaceDash: fluentd.logs.containers.sourceCategoryReplaceDash
+  sourceName: fluentd.logs.containers.sourceName
diff --git a/tests/upgrade_script/static/key_mapping_one_to_multiple.log b/tests/upgrade_script/static/key_mapping_one_to_multiple.log
@@ -37,6 +37,46 @@
 [INFO]    Mapping sumologic.fluentd.autoscaling.targetCPUUtilizationPercentage into:
 [INFO]    - fluentd.logs.autoscaling.targetCPUUtilizationPercentage
 [INFO]    - fluentd.metrics.autoscaling.targetCPUUtilizationPercentage
+[INFO]    Mapping sumologic.excludeContainerRegex into:
+[INFO]    - fluentd.logs.containers.excludeContainerRegex
+[INFO]    - fluentd.logs.default.excludeContainerRegex
+[INFO]    - fluentd.logs.systemd.excludeContainerRegex
+[INFO]    - fluentd.logs.kubelet.excludeContainerRegex
+[INFO]    Mapping sumologic.excludeHostRegex into:
+[INFO]    - fluentd.logs.containers.excludeHostRegex
+[INFO]    - fluentd.logs.default.excludeHostRegex
+[INFO]    - fluentd.logs.systemd.excludeHostRegex
+[INFO]    - fluentd.logs.kubelet.excludeHostRegex
+[INFO]    Mapping sumologic.excludeNamespaceRegex into:
+[INFO]    - fluentd.logs.containers.excludeNamespaceRegex
+[INFO]    - fluentd.logs.default.excludeNamespaceRegex
+[INFO]    - fluentd.logs.systemd.excludeNamespaceRegex
+[INFO]    - fluentd.logs.kubelet.excludeNamespaceRegex
+[INFO]    Mapping sumologic.excludePodRegex into:
+[INFO]    - fluentd.logs.containers.excludePodRegex
+[INFO]    - fluentd.logs.default.excludePodRegex
+[INFO]    - fluentd.logs.systemd.excludePodRegex
+[INFO]    - fluentd.logs.kubelet.excludePodRegex
+[INFO]    Mapping sumologic.sourceCategory into:
+[INFO]    - fluentd.logs.containers.sourceCategory
+[INFO]    - fluentd.logs.default.sourceCategory
+[INFO]    - fluentd.logs.systemd.sourceCategory
+[INFO]    - fluentd.logs.kubelet.sourceCategory
+[INFO]    Mapping sumologic.sourceCategoryPrefix into:
+[INFO]    - fluentd.logs.containers.sourceCategoryPrefix
+[INFO]    - fluentd.logs.default.sourceCategoryPrefix
+[INFO]    - fluentd.logs.systemd.sourceCategoryPrefix
+[INFO]    - fluentd.logs.kubelet.sourceCategoryPrefix
+[INFO]    Mapping sumologic.sourceCategoryReplaceDash into:
+[INFO]    - fluentd.logs.containers.sourceCategoryReplaceDash
+[INFO]    - fluentd.logs.default.sourceCategoryReplaceDash
+[INFO]    - fluentd.logs.systemd.sourceCategoryReplaceDash
+[INFO]    - fluentd.logs.kubelet.sourceCategoryReplaceDash
+[INFO]    Mapping sumologic.sourceName into:
+[INFO]    - fluentd.logs.containers.sourceName
+[INFO]    - fluentd.logs.default.sourceName
+[INFO]    - fluentd.logs.systemd.sourceName
+[INFO]    - fluentd.logs.kubelet.sourceName
 
 [INFO]    falco will be disabled. Change "falco.enabled" to "true" if you want to enable it
 

diff --git a/tests/upgrade_script/static/key_mapping_one_to_multiple.output.yaml b/tests/upgrade_script/static/key_mapping_one_to_multiple.output.yaml
@@ -19,6 +19,42 @@ fluentd:
       minReplicas: fluentd.logs.autoscaling.minReplicas, fluentd.metrics.autoscaling.minReplicas
       targetCPUUtilizationPercentage: fluentd.logs.autoscaling.targetCPUUtilizationPercentage,
         fluentd.metrics.autoscaling.targetCPUUtilizationPercentage
+    containers:
+      excludeContainerRegex: fluentd.logs.containers.excludeContainerRegex
+      excludeHostRegex: fluentd.logs.containers.excludeHostRegex
+      excludeNamespaceRegex: fluentd.logs.containers.excludeNamespaceRegex
+      excludePodRegex: fluentd.logs.containers.excludePodRegex
+      sourceCategory: fluentd.logs.containers.sourceCategory
+      sourceCategoryPrefix: fluentd.logs.containers.sourceCategoryPrefix
+      sourceCategoryReplaceDash: fluentd.logs.containers.sourceCategoryReplaceDash
+      sourceName: fluentd.logs.containers.sourceName
+    default:
+      excludeContainerRegex: fluentd.logs.containers.excludeContainerRegex
+      excludeHostRegex: fluentd.logs.containers.excludeHostRegex
+      excludeNamespaceRegex: fluentd.logs.containers.excludeNamespaceRegex
+      excludePodRegex: fluentd.logs.containers.excludePodRegex
+      sourceCategory: fluentd.logs.containers.sourceCategory
+      sourceCategoryPrefix: fluentd.logs.containers.sourceCategoryPrefix
+      sourceCategoryReplaceDash: fluentd.logs.containers.sourceCategoryReplaceDash
+      sourceName: fluentd.logs.containers.sourceName
+    systemd:
+      excludeContainerRegex: fluentd.logs.containers.excludeContainerRegex
+      excludeHostRegex: fluentd.logs.containers.excludeHostRegex
+      excludeNamespaceRegex: fluentd.logs.containers.excludeNamespaceRegex
+      excludePodRegex: fluentd.logs.containers.excludePodRegex
+      sourceCategory: fluentd.logs.containers.sourceCategory
+      sourceCategoryPrefix: fluentd.logs.containers.sourceCategoryPrefix
+      sourceCategoryReplaceDash: fluentd.logs.containers.sourceCategoryReplaceDash
+      sourceName: fluentd.logs.containers.sourceName
+    kubelet:
+      excludeContainerRegex: fluentd.logs.containers.excludeContainerRegex
+      excludeHostRegex: fluentd.logs.containers.excludeHostRegex
+      excludeNamespaceRegex: fluentd.logs.containers.excludeNamespaceRegex
+      excludePodRegex: fluentd.logs.containers.excludePodRegex
+      sourceCategory: fluentd.logs.containers.sourceCategory
+      sourceCategoryPrefix: fluentd.logs.containers.sourceCategoryPrefix
+      sourceCategoryReplaceDash: fluentd.logs.containers.sourceCategoryReplaceDash
+      sourceName: fluentd.logs.containers.sourceName
   metrics:
     statefulset:
       affinity: fluentd.logs.statefulset.affinity, fluentd.metrics.statefulset.affinity

diff --git a/tests/upgrade_script/static/key_mapping_one_to_one.input.yaml b/tests/upgrade_script/static/key_mapping_one_to_one.input.yaml
@@ -14,10 +14,6 @@ sumologic:
   eventCollectionEnabled: fluentd.events.enabled
   events:
     sourceCategory: fluentd.events.sourceCategory
-  excludeContainerRegex: fluentd.logs.containers.excludeContainerRegex
-  excludeHostRegex: fluentd.logs.containers.excludeHostRegex
-  excludeNamespaceRegex: fluentd.logs.containers.excludeNamespaceRegex
-  excludePodRegex: fluentd.logs.containers.excludePodRegex
   fluentd:
     buffer: fluentd.buffer.type
   fluentdLogLevel: fluentd.logLevel
@@ -31,10 +27,6 @@ sumologic:
   logFormat: fluentd.logs.output.logFormat
   numThreads: fluentd.buffer.numThreads
   queueChunkLimitSize: fluentd.buffer.queueChunkLimitSize
-  sourceCategory: fluentd.logs.containers.sourceCategory
-  sourceCategoryPrefix: fluentd.logs.containers.sourceCategoryPrefix
-  sourceCategoryReplaceDash: fluentd.logs.containers.sourceCategoryReplaceDash
-  sourceName: fluentd.logs.containers.sourceName
   timestampKey: fluentd.logs.output.timestampKey
   totalLimitSize: fluentd.buffer.totalLimitSize
   verifySsl: fluentd.verifySsl

diff --git a/tests/upgrade_script/static/key_mapping_one_to_one.log b/tests/upgrade_script/static/key_mapping_one_to_one.log
@@ -8,20 +8,12 @@
 [INFO]    Mapping sumologic.chunkLimitSize into fluentd.buffer.chunkLimitSize
 [INFO]    Mapping sumologic.eventCollectionEnabled into fluentd.events.enabled
 [INFO]    Mapping sumologic.events.sourceCategory into fluentd.events.sourceCategory
-[INFO]    Mapping sumologic.excludeContainerRegex into fluentd.logs.containers.excludeContainerRegex
-[INFO]    Mapping sumologic.excludeHostRegex into fluentd.logs.containers.excludeHostRegex
-[INFO]    Mapping sumologic.excludeNamespaceRegex into fluentd.logs.containers.excludeNamespaceRegex
-[INFO]    Mapping sumologic.excludePodRegex into fluentd.logs.containers.excludePodRegex
 [INFO]    Mapping sumologic.fluentd.buffer into fluentd.buffer.type
 [INFO]    Mapping sumologic.fluentdLogLevel into fluentd.logLevel
 [INFO]    Mapping sumologic.flushInterval into fluentd.buffer.flushInterval
 [INFO]    Mapping sumologic.logFormat into fluentd.logs.output.logFormat
 [INFO]    Mapping sumologic.numThreads into fluentd.buffer.numThreads
 [INFO]    Mapping sumologic.queueChunkLimitSize into fluentd.buffer.queueChunkLimitSize
-[INFO]    Mapping sumologic.sourceCategory into fluentd.logs.containers.sourceCategory
-[INFO]    Mapping sumologic.sourceCategoryPrefix into fluentd.logs.containers.sourceCategoryPrefix
-[INFO]    Mapping sumologic.sourceCategoryReplaceDash into fluentd.logs.containers.sourceCategoryReplaceDash
-[INFO]    Mapping sumologic.sourceName into fluentd.logs.containers.sourceName
 [INFO]    Mapping sumologic.timestampKey into fluentd.logs.output.timestampKey
 [INFO]    Mapping sumologic.totalLimitSize into fluentd.buffer.totalLimitSize
 [INFO]    Mapping sumologic.verifySsl into fluentd.verifySsl

diff --git a/tests/upgrade_script/static/key_mapping_one_to_one.output.yaml b/tests/upgrade_script/static/key_mapping_one_to_one.output.yaml
@@ -18,15 +18,6 @@ fluentd:
       addTimestamp: fluentd.logs.output.addTimestamp
       logFormat: fluentd.logs.output.logFormat
       timestampKey: fluentd.logs.output.timestampKey
-    containers:
-      excludeContainerRegex: fluentd.logs.containers.excludeContainerRegex
-      excludeHostRegex: fluentd.logs.containers.excludeHostRegex
-      excludeNamespaceRegex: fluentd.logs.containers.excludeNamespaceRegex
-      excludePodRegex: fluentd.logs.containers.excludePodRegex
-      sourceCategory: fluentd.logs.containers.sourceCategory
-      sourceCategoryPrefix: fluentd.logs.containers.sourceCategoryPrefix
-      sourceCategoryReplaceDash: fluentd.logs.containers.sourceCategoryReplaceDash
-      sourceName: fluentd.logs.containers.sourceName
   buffer:
     chunkLimitSize: fluentd.buffer.chunkLimitSize
     type: fluentd.buffer.type