fix: v2 migration script - info on new default value for fluent-bit.i…

…mage.pullPolicy key instead of just adding it

deploy/helm/sumologic/upgrade-2.0.0.sh

@@ -15,7 +15,9 @@ readonly MAX_YQ_VERSION=4.0.0
 readonly KEY_MAPPINGS="
 prometheus-operator.prometheusOperator.tlsProxy.enabled:kube-prometheus-stack.prometheusOperator.tls.enabled
 otelcol.deployment.image.name:otelcol.deployment.image.repository
-fluent-bit.image.fluent_bit:fluent-bit.image.repository
+fluent-bit.image.fluent_bit.repository:fluent-bit.image.repository
+fluent-bit.image.fluent_bit.tag:fluent-bit.image.tag
+
 "
 
 readonly KEY_VALUE_MAPPINGS="
@@ -649,8 +651,11 @@ function migrate_fluent_bit() {
     yq w -i "${TEMP_FILE}" 'fluent-bit.env.(name==CHART).name' FLUENTD_LOGS_SVC
   fi
 
-  if [[ -n "$(yq r "${TEMP_FILE}" -- 'fluent-bit.image.fluent_bit')" ]]; then
-    yq w -i "${TEMP_FILE}" 'fluent-bit.image.pullPolicy' IfNotPresent
+  local image_pull_policy
+  readonly image_pull_policy="$(yq r "${TEMP_FILE}" -- 'fluent-bit.image.pullPolicy')"
+  if [[ -n ${image_pull_policy} ]] && [[ ${image_pull_policy} != "IfNotPresent" ]]; then
+    info "Leaving the fluent-bit.image.pullPolicy set to '${image_pull_policy}'"
+    info "Please note that in v2.0 the fluent-bit.image.pullPolicy is set to 'IfNotPresent' by default"
   fi
 
   if [[ -n "$(yq r "${TEMP_FILE}" -- 'fluent-bit.service.flush')" ]]; then

tests/upgrade_v2_script/static/fluent_bit.input.yaml

@@ -39,6 +39,7 @@ fluent-bit:
     fluent_bit:
       repository: public.ecr.aws/sumologic/fluent-bit
       tag: 1.6.0
+    pullPolicy: Always
   service:
     flush: 5
   metrics:

tests/upgrade_v2_script/static/fluent_bit.log

@@ -1,5 +1,9 @@
+[INFO]    Mapping fluent-bit.image.fluent_bit.repository into fluent-bit.image.repository
+[INFO]    Mapping fluent-bit.image.fluent_bit.tag into fluent-bit.image.tag
 
 [INFO]    Migrating prometheus remote write urls
+[INFO]    Leaving the fluent-bit.image.pullPolicy set to 'Always'
+[INFO]    Please note that in v2.0 the fluent-bit.image.pullPolicy is set to 'IfNotPresent' by default
 [INFO]    Migrating "fluent-bit.backend.forward.host" from "${CHART}.${NAMESPACE}.svc.cluster.local." to "${FLUENTD_LOGS_SVC}.${NAMESPACE}.svc.cluster.local."
 [WARNING] You have an unexpected section in your fluent-bit configuration that we're not migrating automatically. It shouldn't be necessary in 2.0 but please consider whether you need to migrate this manually. Line: @INCLUDE fluent-bit-service.conf
 [INFO]    Migrating fluent-bit's input config from Multiline to Docker_Mode

tests/upgrade_v2_script/static/fluent_bit.output.yaml

@@ -36,10 +36,9 @@ fluent-bit:
     - effect: NoSchedule
       operator: Exists
   image:
-    fluent_bit:
-      repository: public.ecr.aws/sumologic/fluent-bit
-      tag: 1.6.0
-    pullPolicy: IfNotPresent
+    repository: public.ecr.aws/sumologic/fluent-bit
+    tag: 1.6.0
+    pullPolicy: Always
   service:
     labels:
       sumologic.com/scrape: "true"

tests/upgrade_v2_script/static/fluent_bit_no_image.input.yaml

@@ -0,0 +1,155 @@
+fluent-bit:
+  podLabels: {}
+  podAnnotations: {}
+  env:
+    - name: CHART
+      valueFrom:
+        configMapKeyRef:
+          name: sumologic-configmap
+          key: fluentdLogs
+    - name: NAMESPACE
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.namespace
+  extraVolumes:
+    - name: project2-db
+      emptyDir: {}
+    - name: project-db
+      emptyDir: {}
+    - name: project2-alerts
+      hostPath:
+        path: /var/project2/logs/alerts/alerts.json
+        type: File
+    - name: company-db
+      emptyDir: {}
+  extraVolumeMounts:
+    - name: project2-db
+      mountPath: /fluent-bit/project2
+    - name: project-db
+      mountPath: /fluent-bit/project
+    - name: project2-alerts
+      mountPath: /var/project2/logs/alerts/alerts.json
+      readOnly: true
+    - name: company-db
+      mountPath: /fluent-bit/company
+  tolerations:
+    - effect: NoSchedule
+      operator: Exists
+  service:
+    flush: 5
+  metrics:
+    enabled: true
+  trackOffsets: true
+
+  backend:
+    type: forward
+    forward:
+      ## NOTE: Requires trailing "." for fully-qualified name resolution
+      host: ${CHART}.${NAMESPACE}.svc.cluster.local.
+      port: 24321
+      tls: "off"
+      tls_verify: "on"
+      tls_debug: 1
+      shared_key:
+  input:
+    systemd:
+      enabled: true
+  parsers:
+    enabled: true
+    ## This regex matches the first line of a multiline log starting with a date of the format :  "2019-11-17 07:14:12" or "2019-11-17T07:14:12"
+    regex:
+      - name: multi_line
+        regex: (?<log>^{"log":"\d{4}-\d{1,2}-\d{1,2}.\d{2}:\d{2}:\d{2}.*)
+      - name: crio
+        regex: ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<log>.*)$
+        timeKey: time
+        timeFormat: "%Y-%m-%dT%H:%M:%S.%L%z"
+    json:
+      - name: test_json
+        timeKey: time
+        timeFormat: "%Y-%m-%dT%H:%M:%S.%L%z"
+  rawConfig: |-
+    @INCLUDE fluent-bit-service.conf
+
+    [INPUT]
+      Name             tail
+      Path             /var/log/containers/*.log
+      Multiline        On
+      Parser_Firstline multi_line
+      Tag              containers.*
+      Refresh_Interval 1
+      Rotate_Wait      60
+      Mem_Buf_Limit    5MB
+      Skip_Long_Lines  On
+      DB               /tail-db/tail-containers-state-sumo.db
+      DB.Sync          Normal
+
+    [OUTPUT]
+      Name            test-output-1
+
+    [FILTER]
+      Name            test-filter-1
+
+    [PARSER]
+      Name            test-logfmt-parser-1
+      Format          logfmt
+    [PARSER]
+      Name            test-regex-parser-1
+      Format          regex
+      Regex           .*
+    [INPUT]
+      Name            systemd
+      Tag             host.*
+      DB              /tail-db/systemd-state-sumo.db
+      Systemd_Filter  _SYSTEMD_UNIT=addon-config.service
+      Systemd_Filter  _SYSTEMD_UNIT=addon-run.service
+      Systemd_Filter  _SYSTEMD_UNIT=cfn-etcd-environment.service
+      Systemd_Filter  _SYSTEMD_UNIT=cfn-signal.service
+      Systemd_Filter  _SYSTEMD_UNIT=clean-ca-certificates.service
+      Systemd_Filter  _SYSTEMD_UNIT=containerd.service
+      Systemd_Filter  _SYSTEMD_UNIT=coreos-metadata.service
+      Systemd_Filter  _SYSTEMD_UNIT=coreos-setup-environment.service
+      Systemd_Filter  _SYSTEMD_UNIT=coreos-tmpfiles.service
+      Systemd_Filter  _SYSTEMD_UNIT=dbus.service
+      Systemd_Filter  _SYSTEMD_UNIT=docker.service
+      Systemd_Filter  _SYSTEMD_UNIT=efs.service
+      Systemd_Filter  _SYSTEMD_UNIT=etcd-member.service
+      Systemd_Filter  _SYSTEMD_UNIT=etcd.service
+      Systemd_Filter  _SYSTEMD_UNIT=etcd2.service
+      Systemd_Filter  _SYSTEMD_UNIT=etcd3.service
+      Systemd_Filter  _SYSTEMD_UNIT=etcdadm-check.service
+      Systemd_Filter  _SYSTEMD_UNIT=etcdadm-reconfigure.service
+      Systemd_Filter  _SYSTEMD_UNIT=etcdadm-save.service
+      Systemd_Filter  _SYSTEMD_UNIT=etcdadm-update-status.service
+      Systemd_Filter  _SYSTEMD_UNIT=flanneld.service
+      Systemd_Filter  _SYSTEMD_UNIT=format-etcd2-volume.service
+      Systemd_Filter  _SYSTEMD_UNIT=kube-node-taint-and-uncordon.service
+      Systemd_Filter  _SYSTEMD_UNIT=kubelet.service
+      Systemd_Filter  _SYSTEMD_UNIT=ldconfig.service
+      Systemd_Filter  _SYSTEMD_UNIT=locksmithd.service
+      Systemd_Filter  _SYSTEMD_UNIT=logrotate.service
+      Systemd_Filter  _SYSTEMD_UNIT=lvm2-monitor.service
+      Systemd_Filter  _SYSTEMD_UNIT=mdmon.service
+      Systemd_Filter  _SYSTEMD_UNIT=nfs-idmapd.service
+      Systemd_Filter  _SYSTEMD_UNIT=nfs-mountd.service
+      Systemd_Filter  _SYSTEMD_UNIT=nfs-server.service
+      Systemd_Filter  _SYSTEMD_UNIT=nfs-utils.service
+      Systemd_Filter  _SYSTEMD_UNIT=node-problem-detector.service
+      Systemd_Filter  _SYSTEMD_UNIT=ntp.service
+      Systemd_Filter  _SYSTEMD_UNIT=oem-cloudinit.service
+      Systemd_Filter  _SYSTEMD_UNIT=rkt-gc.service
+      Systemd_Filter  _SYSTEMD_UNIT=rkt-metadata.service
+      Systemd_Filter  _SYSTEMD_UNIT=rpc-idmapd.service
+      Systemd_Filter  _SYSTEMD_UNIT=rpc-mountd.service
+      Systemd_Filter  _SYSTEMD_UNIT=rpc-statd.service
+      Systemd_Filter  _SYSTEMD_UNIT=rpcbind.service
+      Systemd_Filter  _SYSTEMD_UNIT=set-aws-environment.service
+      Systemd_Filter  _SYSTEMD_UNIT=system-cloudinit.service
+      Systemd_Filter  _SYSTEMD_UNIT=systemd-timesyncd.service
+      Systemd_Filter  _SYSTEMD_UNIT=update-ca-certificates.service
+      Systemd_Filter  _SYSTEMD_UNIT=user-cloudinit.service
+      Systemd_Filter  _SYSTEMD_UNIT=var-lib-etcd2.service
+      Max_Entries     1000
+      Read_From_Tail  true
+
+    @INCLUDE fluent-bit-output.conf

tests/upgrade_v2_script/static/fluent_bit_no_image.log

@@ -0,0 +1,10 @@
+
+[INFO]    Migrating prometheus remote write urls
+[INFO]    Migrating "fluent-bit.backend.forward.host" from "${CHART}.${NAMESPACE}.svc.cluster.local." to "${FLUENTD_LOGS_SVC}.${NAMESPACE}.svc.cluster.local."
+[WARNING] You have an unexpected section in your fluent-bit configuration that we're not migrating automatically. It shouldn't be necessary in 2.0 but please consider whether you need to migrate this manually. Line: @INCLUDE fluent-bit-service.conf
+[INFO]    Migrating fluent-bit's input config from Multiline to Docker_Mode
+[INFO]    Migrating fluent-bit's input config from Parser_Firstline to Docker_Mode_Parser
+[WARNING] You have an unexpected section in your fluent-bit configuration that we're not migrating automatically. It shouldn't be necessary in 2.0 but please consider whether you need to migrate this manually. Line: @INCLUDE fluent-bit-output.conf
+
+Thank you for upgrading to v2.0.0 of the Sumo Logic Kubernetes Collection Helm chart.
+A new yaml file has been generated for you. Please check the current directory for new_values.yaml.

tests/upgrade_v2_script/static/fluent_bit_no_image.output.yaml

@@ -0,0 +1,145 @@
+fluent-bit:
+  podLabels: {}
+  podAnnotations: {}
+  env:
+    - name: FLUENTD_LOGS_SVC
+      valueFrom:
+        configMapKeyRef:
+          name: sumologic-configmap
+          key: fluentdLogs
+    - name: NAMESPACE
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.namespace
+  extraVolumes:
+    - name: project2-db
+      emptyDir: {}
+    - name: project-db
+      emptyDir: {}
+    - name: project2-alerts
+      hostPath:
+        path: /var/project2/logs/alerts/alerts.json
+        type: File
+    - name: company-db
+      emptyDir: {}
+  extraVolumeMounts:
+    - name: project2-db
+      mountPath: /fluent-bit/project2
+    - name: project-db
+      mountPath: /fluent-bit/project
+    - name: project2-alerts
+      mountPath: /var/project2/logs/alerts/alerts.json
+      readOnly: true
+    - name: company-db
+      mountPath: /fluent-bit/company
+  tolerations:
+    - effect: NoSchedule
+      operator: Exists
+  service:
+    labels:
+      sumologic.com/scrape: "true"
+  config:
+    inputs: |-
+      [INPUT]
+          Name                  tail
+          Path                  /var/log/containers/*.log
+          Docker_Mode           On
+          Docker_Mode_Parser    multi_line
+          Tag                   containers.*
+          Refresh_Interval      1
+          Rotate_Wait           60
+          Mem_Buf_Limit         5MB
+          Skip_Long_Lines       On
+          DB                    /tail-db/tail-containers-state-sumo.db
+          DB.Sync               Normal
+      [INPUT]
+          Name                  systemd
+          Tag                   host.*
+          DB                    /tail-db/systemd-state-sumo.db
+          Systemd_Filter        _SYSTEMD_UNIT=addon-config.service
+          Systemd_Filter        _SYSTEMD_UNIT=addon-run.service
+          Systemd_Filter        _SYSTEMD_UNIT=cfn-etcd-environment.service
+          Systemd_Filter        _SYSTEMD_UNIT=cfn-signal.service
+          Systemd_Filter        _SYSTEMD_UNIT=clean-ca-certificates.service
+          Systemd_Filter        _SYSTEMD_UNIT=containerd.service
+          Systemd_Filter        _SYSTEMD_UNIT=coreos-metadata.service
+          Systemd_Filter        _SYSTEMD_UNIT=coreos-setup-environment.service
+          Systemd_Filter        _SYSTEMD_UNIT=coreos-tmpfiles.service
+          Systemd_Filter        _SYSTEMD_UNIT=dbus.service
+          Systemd_Filter        _SYSTEMD_UNIT=docker.service
+          Systemd_Filter        _SYSTEMD_UNIT=efs.service
+          Systemd_Filter        _SYSTEMD_UNIT=etcd-member.service
+          Systemd_Filter        _SYSTEMD_UNIT=etcd.service
+          Systemd_Filter        _SYSTEMD_UNIT=etcd2.service
+          Systemd_Filter        _SYSTEMD_UNIT=etcd3.service
+          Systemd_Filter        _SYSTEMD_UNIT=etcdadm-check.service
+          Systemd_Filter        _SYSTEMD_UNIT=etcdadm-reconfigure.service
+          Systemd_Filter        _SYSTEMD_UNIT=etcdadm-save.service
+          Systemd_Filter        _SYSTEMD_UNIT=etcdadm-update-status.service
+          Systemd_Filter        _SYSTEMD_UNIT=flanneld.service
+          Systemd_Filter        _SYSTEMD_UNIT=format-etcd2-volume.service
+          Systemd_Filter        _SYSTEMD_UNIT=kube-node-taint-and-uncordon.service
+          Systemd_Filter        _SYSTEMD_UNIT=kubelet.service
+          Systemd_Filter        _SYSTEMD_UNIT=ldconfig.service
+          Systemd_Filter        _SYSTEMD_UNIT=locksmithd.service
+          Systemd_Filter        _SYSTEMD_UNIT=logrotate.service
+          Systemd_Filter        _SYSTEMD_UNIT=lvm2-monitor.service
+          Systemd_Filter        _SYSTEMD_UNIT=mdmon.service
+          Systemd_Filter        _SYSTEMD_UNIT=nfs-idmapd.service
+          Systemd_Filter        _SYSTEMD_UNIT=nfs-mountd.service
+          Systemd_Filter        _SYSTEMD_UNIT=nfs-server.service
+          Systemd_Filter        _SYSTEMD_UNIT=nfs-utils.service
+          Systemd_Filter        _SYSTEMD_UNIT=node-problem-detector.service
+          Systemd_Filter        _SYSTEMD_UNIT=ntp.service
+          Systemd_Filter        _SYSTEMD_UNIT=oem-cloudinit.service
+          Systemd_Filter        _SYSTEMD_UNIT=rkt-gc.service
+          Systemd_Filter        _SYSTEMD_UNIT=rkt-metadata.service
+          Systemd_Filter        _SYSTEMD_UNIT=rpc-idmapd.service
+          Systemd_Filter        _SYSTEMD_UNIT=rpc-mountd.service
+          Systemd_Filter        _SYSTEMD_UNIT=rpc-statd.service
+          Systemd_Filter        _SYSTEMD_UNIT=rpcbind.service
+          Systemd_Filter        _SYSTEMD_UNIT=set-aws-environment.service
+          Systemd_Filter        _SYSTEMD_UNIT=system-cloudinit.service
+          Systemd_Filter        _SYSTEMD_UNIT=systemd-timesyncd.service
+          Systemd_Filter        _SYSTEMD_UNIT=update-ca-certificates.service
+          Systemd_Filter        _SYSTEMD_UNIT=user-cloudinit.service
+          Systemd_Filter        _SYSTEMD_UNIT=var-lib-etcd2.service
+          Max_Entries           1000
+          Read_From_Tail        true
+    outputs: |-
+      [OUTPUT]
+          Name                  forward
+          Match                 *
+          Host                  ${FLUENTD_LOGS_SVC}.${NAMESPACE}.svc.cluster.local.
+          Port                  24321
+          tls                   off
+          tls.verify            on
+          tls.debug             1
+      [OUTPUT]
+          Name                  test-output-1
+    customParsers: |-
+      [PARSER]
+          Name                  multi_line
+          Format                regex
+          Regex                 (?<log>^{"log":"\d{4}-\d{1,2}-\d{1,2}.\d{2}:\d{2}:\d{2}.*)
+      [PARSER]
+          Name                  crio
+          Format                regex
+          Regex                 ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<log>.*)$
+          Time_Key              time
+          Time_Format           %Y-%m-%dT%H:%M:%S.%L%z
+      [PARSER]
+          Name                  test_json
+          Format                json
+          Time_Key              time
+          Time_Format           %Y-%m-%dT%H:%M:%S.%L%z
+      [PARSER]
+          Name                  test-logfmt-parser-1
+          Format                logfmt
+      [PARSER]
+          Name                  test-regex-parser-1
+          Format                regex
+          Regex                 .*
+    filters: |-
+      [FILTER]
+          Name                  test-filter-1