diff --git a/deploy/helm/sumologic/upgrade-2.0.0.sh b/deploy/helm/sumologic/upgrade-2.0.0.sh index 3de68c08ef..5c2c9845b5 100755 --- a/deploy/helm/sumologic/upgrade-2.0.0.sh +++ b/deploy/helm/sumologic/upgrade-2.0.0.sh @@ -666,7 +666,7 @@ function migrate_fluent_bit() { yq w -i "${TEMP_FILE}" 'fluent-bit.extraVolumes[0].name' 'tail-db' local CONFIG_KEY_WIDTH - readonly CONFIG_KEY_WIDTH="16" + readonly CONFIG_KEY_WIDTH="22" local OUTPUTS="" # Migrate fluent-bit.backend @@ -826,6 +826,8 @@ function migrate_fluent_bit() { readonly RAWCONFIG="$(yq r "${TEMP_FILE}" 'fluent-bit.rawConfig')" local SECTION="" + local KEY="" + local VALUE="" shopt -s extglob while IFS= read -r line do @@ -866,27 +868,41 @@ function migrate_fluent_bit() { else # Trim whitespace line="${line##+([[:space:]])}" + if [[ -n "${line}" ]]; then - case "${SECTION}" in - "INPUT") - INPUTS="$(printf "%s\n %s" "${INPUTS}" "${line}")" - ;; - - "OUTPUT") - OUTPUTS="$(printf "%s\n %s" "${OUTPUTS}" "${line}")" - ;; - - "PARSER") - PARSERS="$(printf "%s\n %s" "${PARSERS}" "${line}")" - ;; - - "FILTER") - FILTERS="$(printf "%s\n %s" "${FILTERS}" "${line}")" - ;; - - *) - ;; - esac + IFS=' ' read -ra ADDR <<< "${line}" + if [[ ${#ADDR[@]} -eq 2 ]]; then + KEY="${ADDR[0]}" + VALUE="${ADDR[1]}" + + case "${SECTION}" in + "INPUT") + if [[ "${KEY}" == "Multiline" && -n "${VALUE}" ]]; then + info "Migrating fluent-bit's input config from Multiline to Docker_Mode" + KEY="Docker_Mode" + elif [[ "${KEY}" == "Parser_Firstline" && -n "${VALUE}" ]]; then + info "Migrating fluent-bit's input config from Parser_Firstline to Docker_Mode_Parser" + KEY="Docker_Mode_Parser" + fi + INPUTS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${INPUTS}" "${KEY}" "${VALUE}")" + ;; + + "OUTPUT") + OUTPUTS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${OUTPUTS}" "${KEY}" "${VALUE}")" + ;; + + "PARSER") + PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "${KEY}" "${VALUE}")" + ;; + + "FILTER") + FILTERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${FILTERS}" "${KEY}" "${VALUE}")" + ;; + + *) + ;; + esac + fi fi fi done < <(echo "${RAWCONFIG}") diff --git a/tests/upgrade_v2_script/static/fluent_bit.log b/tests/upgrade_v2_script/static/fluent_bit.log index 25cf071b44..ad9dcb07c3 100644 --- a/tests/upgrade_v2_script/static/fluent_bit.log +++ b/tests/upgrade_v2_script/static/fluent_bit.log @@ -2,6 +2,8 @@ [INFO] Migrating prometheus remote write urls [INFO] Migrating "fluent-bit.backend.forward.host" from "${CHART}.${NAMESPACE}.svc.cluster.local." to "${FLUENTD_LOGS_SVC}.${NAMESPACE}.svc.cluster.local." [WARNING] You have an unexpected section in your fluent-bit configuration that we're not migrating automatically. It shouldn't be necessary in 2.0 but please consider whether you need to migrate this manually. Line: @INCLUDE fluent-bit-service.conf +[INFO] Migrating fluent-bit's input config from Multiline to Docker_Mode +[INFO] Migrating fluent-bit's input config from Parser_Firstline to Docker_Mode_Parser [WARNING] You have an unexpected section in your fluent-bit configuration that we're not migrating automatically. It shouldn't be necessary in 2.0 but please consider whether you need to migrate this manually. Line: @INCLUDE fluent-bit-output.conf Thank you for upgrading to v2.0.0 of the Sumo Logic Kubernetes Collection Helm chart. diff --git a/tests/upgrade_v2_script/static/fluent_bit.output.yaml b/tests/upgrade_v2_script/static/fluent_bit.output.yaml index 95c45faba5..75752ed9f3 100644 --- a/tests/upgrade_v2_script/static/fluent_bit.output.yaml +++ b/tests/upgrade_v2_script/static/fluent_bit.output.yaml @@ -30,105 +30,105 @@ fluent-bit: config: inputs: |- [INPUT] - Name tail - Path /var/log/containers/*.log - Multiline On - Parser_Firstline multi_line - Tag containers.* - Refresh_Interval 1 - Rotate_Wait 60 - Mem_Buf_Limit 5MB - Skip_Long_Lines On - DB /tail-db/tail-containers-state-sumo.db - DB.Sync Normal + Name tail + Path /var/log/containers/*.log + Docker_Mode On + Docker_Mode_Parser multi_line + Tag containers.* + Refresh_Interval 1 + Rotate_Wait 60 + Mem_Buf_Limit 5MB + Skip_Long_Lines On + DB /tail-db/tail-containers-state-sumo.db + DB.Sync Normal [INPUT] - Name systemd - Tag host.* - DB /tail-db/systemd-state-sumo.db - Systemd_Filter _SYSTEMD_UNIT=addon-config.service - Systemd_Filter _SYSTEMD_UNIT=addon-run.service - Systemd_Filter _SYSTEMD_UNIT=cfn-etcd-environment.service - Systemd_Filter _SYSTEMD_UNIT=cfn-signal.service - Systemd_Filter _SYSTEMD_UNIT=clean-ca-certificates.service - Systemd_Filter _SYSTEMD_UNIT=containerd.service - Systemd_Filter _SYSTEMD_UNIT=coreos-metadata.service - Systemd_Filter _SYSTEMD_UNIT=coreos-setup-environment.service - Systemd_Filter _SYSTEMD_UNIT=coreos-tmpfiles.service - Systemd_Filter _SYSTEMD_UNIT=dbus.service - Systemd_Filter _SYSTEMD_UNIT=docker.service - Systemd_Filter _SYSTEMD_UNIT=efs.service - Systemd_Filter _SYSTEMD_UNIT=etcd-member.service - Systemd_Filter _SYSTEMD_UNIT=etcd.service - Systemd_Filter _SYSTEMD_UNIT=etcd2.service - Systemd_Filter _SYSTEMD_UNIT=etcd3.service - Systemd_Filter _SYSTEMD_UNIT=etcdadm-check.service - Systemd_Filter _SYSTEMD_UNIT=etcdadm-reconfigure.service - Systemd_Filter _SYSTEMD_UNIT=etcdadm-save.service - Systemd_Filter _SYSTEMD_UNIT=etcdadm-update-status.service - Systemd_Filter _SYSTEMD_UNIT=flanneld.service - Systemd_Filter _SYSTEMD_UNIT=format-etcd2-volume.service - Systemd_Filter _SYSTEMD_UNIT=kube-node-taint-and-uncordon.service - Systemd_Filter _SYSTEMD_UNIT=kubelet.service - Systemd_Filter _SYSTEMD_UNIT=ldconfig.service - Systemd_Filter _SYSTEMD_UNIT=locksmithd.service - Systemd_Filter _SYSTEMD_UNIT=logrotate.service - Systemd_Filter _SYSTEMD_UNIT=lvm2-monitor.service - Systemd_Filter _SYSTEMD_UNIT=mdmon.service - Systemd_Filter _SYSTEMD_UNIT=nfs-idmapd.service - Systemd_Filter _SYSTEMD_UNIT=nfs-mountd.service - Systemd_Filter _SYSTEMD_UNIT=nfs-server.service - Systemd_Filter _SYSTEMD_UNIT=nfs-utils.service - Systemd_Filter _SYSTEMD_UNIT=node-problem-detector.service - Systemd_Filter _SYSTEMD_UNIT=ntp.service - Systemd_Filter _SYSTEMD_UNIT=oem-cloudinit.service - Systemd_Filter _SYSTEMD_UNIT=rkt-gc.service - Systemd_Filter _SYSTEMD_UNIT=rkt-metadata.service - Systemd_Filter _SYSTEMD_UNIT=rpc-idmapd.service - Systemd_Filter _SYSTEMD_UNIT=rpc-mountd.service - Systemd_Filter _SYSTEMD_UNIT=rpc-statd.service - Systemd_Filter _SYSTEMD_UNIT=rpcbind.service - Systemd_Filter _SYSTEMD_UNIT=set-aws-environment.service - Systemd_Filter _SYSTEMD_UNIT=system-cloudinit.service - Systemd_Filter _SYSTEMD_UNIT=systemd-timesyncd.service - Systemd_Filter _SYSTEMD_UNIT=update-ca-certificates.service - Systemd_Filter _SYSTEMD_UNIT=user-cloudinit.service - Systemd_Filter _SYSTEMD_UNIT=var-lib-etcd2.service - Max_Entries 1000 - Read_From_Tail true + Name systemd + Tag host.* + DB /tail-db/systemd-state-sumo.db + Systemd_Filter _SYSTEMD_UNIT=addon-config.service + Systemd_Filter _SYSTEMD_UNIT=addon-run.service + Systemd_Filter _SYSTEMD_UNIT=cfn-etcd-environment.service + Systemd_Filter _SYSTEMD_UNIT=cfn-signal.service + Systemd_Filter _SYSTEMD_UNIT=clean-ca-certificates.service + Systemd_Filter _SYSTEMD_UNIT=containerd.service + Systemd_Filter _SYSTEMD_UNIT=coreos-metadata.service + Systemd_Filter _SYSTEMD_UNIT=coreos-setup-environment.service + Systemd_Filter _SYSTEMD_UNIT=coreos-tmpfiles.service + Systemd_Filter _SYSTEMD_UNIT=dbus.service + Systemd_Filter _SYSTEMD_UNIT=docker.service + Systemd_Filter _SYSTEMD_UNIT=efs.service + Systemd_Filter _SYSTEMD_UNIT=etcd-member.service + Systemd_Filter _SYSTEMD_UNIT=etcd.service + Systemd_Filter _SYSTEMD_UNIT=etcd2.service + Systemd_Filter _SYSTEMD_UNIT=etcd3.service + Systemd_Filter _SYSTEMD_UNIT=etcdadm-check.service + Systemd_Filter _SYSTEMD_UNIT=etcdadm-reconfigure.service + Systemd_Filter _SYSTEMD_UNIT=etcdadm-save.service + Systemd_Filter _SYSTEMD_UNIT=etcdadm-update-status.service + Systemd_Filter _SYSTEMD_UNIT=flanneld.service + Systemd_Filter _SYSTEMD_UNIT=format-etcd2-volume.service + Systemd_Filter _SYSTEMD_UNIT=kube-node-taint-and-uncordon.service + Systemd_Filter _SYSTEMD_UNIT=kubelet.service + Systemd_Filter _SYSTEMD_UNIT=ldconfig.service + Systemd_Filter _SYSTEMD_UNIT=locksmithd.service + Systemd_Filter _SYSTEMD_UNIT=logrotate.service + Systemd_Filter _SYSTEMD_UNIT=lvm2-monitor.service + Systemd_Filter _SYSTEMD_UNIT=mdmon.service + Systemd_Filter _SYSTEMD_UNIT=nfs-idmapd.service + Systemd_Filter _SYSTEMD_UNIT=nfs-mountd.service + Systemd_Filter _SYSTEMD_UNIT=nfs-server.service + Systemd_Filter _SYSTEMD_UNIT=nfs-utils.service + Systemd_Filter _SYSTEMD_UNIT=node-problem-detector.service + Systemd_Filter _SYSTEMD_UNIT=ntp.service + Systemd_Filter _SYSTEMD_UNIT=oem-cloudinit.service + Systemd_Filter _SYSTEMD_UNIT=rkt-gc.service + Systemd_Filter _SYSTEMD_UNIT=rkt-metadata.service + Systemd_Filter _SYSTEMD_UNIT=rpc-idmapd.service + Systemd_Filter _SYSTEMD_UNIT=rpc-mountd.service + Systemd_Filter _SYSTEMD_UNIT=rpc-statd.service + Systemd_Filter _SYSTEMD_UNIT=rpcbind.service + Systemd_Filter _SYSTEMD_UNIT=set-aws-environment.service + Systemd_Filter _SYSTEMD_UNIT=system-cloudinit.service + Systemd_Filter _SYSTEMD_UNIT=systemd-timesyncd.service + Systemd_Filter _SYSTEMD_UNIT=update-ca-certificates.service + Systemd_Filter _SYSTEMD_UNIT=user-cloudinit.service + Systemd_Filter _SYSTEMD_UNIT=var-lib-etcd2.service + Max_Entries 1000 + Read_From_Tail true outputs: |- [OUTPUT] - Name forward - Match * - Host ${FLUENTD_LOGS_SVC}.${NAMESPACE}.svc.cluster.local. - Port 24321 - tls off - tls.verify on - tls.debug 1 + Name forward + Match * + Host ${FLUENTD_LOGS_SVC}.${NAMESPACE}.svc.cluster.local. + Port 24321 + tls off + tls.verify on + tls.debug 1 [OUTPUT] - Name test-output-1 + Name test-output-1 customParsers: |- [PARSER] - Name multi_line - Format regex - Regex (?^{"log":"\d{4}-\d{1,2}-\d{1,2}.\d{2}:\d{2}:\d{2}.*) + Name multi_line + Format regex + Regex (?^{"log":"\d{4}-\d{1,2}-\d{1,2}.\d{2}:\d{2}:\d{2}.*) [PARSER] - Name crio - Format regex - Regex ^(?