Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

v2 migration script - add fluent bit multiline migration #1325

Merged
merged 1 commit into from
Jan 13, 2021
Merged

v2 migration script - add fluent bit multiline migration #1325

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
58 changes: 37 additions & 21 deletions deploy/helm/sumologic/upgrade-2.0.0.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -666,7 +666,7 @@ function migrate_fluent_bit() {
yq w -i "${TEMP_FILE}" 'fluent-bit.extraVolumes[0].name' 'tail-db'

local CONFIG_KEY_WIDTH
readonly CONFIG_KEY_WIDTH="16"
readonly CONFIG_KEY_WIDTH="22"

local OUTPUTS=""
# Migrate fluent-bit.backend
Expand Down Expand Up @@ -826,6 +826,8 @@ function migrate_fluent_bit() {
readonly RAWCONFIG="$(yq r "${TEMP_FILE}" 'fluent-bit.rawConfig')"

local SECTION=""
local KEY=""
local VALUE=""
shopt -s extglob
while IFS= read -r line
do
Expand Down Expand Up @@ -866,27 +868,41 @@ function migrate_fluent_bit() {
else
# Trim whitespace
line="${line##+([[:space:]])}"

if [[ -n "${line}" ]]; then
case "${SECTION}" in
"INPUT")
INPUTS="$(printf "%s\n %s" "${INPUTS}" "${line}")"
;;

"OUTPUT")
OUTPUTS="$(printf "%s\n %s" "${OUTPUTS}" "${line}")"
;;

"PARSER")
PARSERS="$(printf "%s\n %s" "${PARSERS}" "${line}")"
;;

"FILTER")
FILTERS="$(printf "%s\n %s" "${FILTERS}" "${line}")"
;;

*)
;;
esac
IFS=' ' read -ra ADDR <<< "${line}"
if [[ ${#ADDR[@]} -eq 2 ]]; then
KEY="${ADDR[0]}"
VALUE="${ADDR[1]}"

case "${SECTION}" in
"INPUT")
if [[ "${KEY}" == "Multiline" && -n "${VALUE}" ]]; then
info "Migrating fluent-bit's input config from Multiline to Docker_Mode"
KEY="Docker_Mode"
elif [[ "${KEY}" == "Parser_Firstline" && -n "${VALUE}" ]]; then
info "Migrating fluent-bit's input config from Parser_Firstline to Docker_Mode_Parser"
KEY="Docker_Mode_Parser"
fi
INPUTS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${INPUTS}" "${KEY}" "${VALUE}")"
;;

"OUTPUT")
OUTPUTS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${OUTPUTS}" "${KEY}" "${VALUE}")"
;;

"PARSER")
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "${KEY}" "${VALUE}")"
;;

"FILTER")
FILTERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${FILTERS}" "${KEY}" "${VALUE}")"
;;

*)
;;
esac
fi
fi
fi
done < <(echo "${RAWCONFIG}")
Expand Down
2 changes: 2 additions & 0 deletions tests/upgrade_v2_script/static/fluent_bit.log
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
[INFO] Migrating prometheus remote write urls
[INFO] Migrating "fluent-bit.backend.forward.host" from "${CHART}.${NAMESPACE}.svc.cluster.local." to "${FLUENTD_LOGS_SVC}.${NAMESPACE}.svc.cluster.local."
[WARNING] You have an unexpected section in your fluent-bit configuration that we're not migrating automatically. It shouldn't be necessary in 2.0 but please consider whether you need to migrate this manually. Line: @INCLUDE fluent-bit-service.conf
[INFO] Migrating fluent-bit's input config from Multiline to Docker_Mode
[INFO] Migrating fluent-bit's input config from Parser_Firstline to Docker_Mode_Parser
[WARNING] You have an unexpected section in your fluent-bit configuration that we're not migrating automatically. It shouldn't be necessary in 2.0 but please consider whether you need to migrate this manually. Line: @INCLUDE fluent-bit-output.conf

Thank you for upgrading to v2.0.0 of the Sumo Logic Kubernetes Collection Helm chart.
Expand Down
180 changes: 90 additions & 90 deletions tests/upgrade_v2_script/static/fluent_bit.output.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,105 +30,105 @@ fluent-bit:
config:
inputs: |-
[INPUT]
Name tail
Path /var/log/containers/*.log
Multiline On
Parser_Firstline multi_line
Tag containers.*
Refresh_Interval 1
Rotate_Wait 60
Mem_Buf_Limit 5MB
Skip_Long_Lines On
DB /tail-db/tail-containers-state-sumo.db
DB.Sync Normal
Name tail
Path /var/log/containers/*.log
Docker_Mode On
Docker_Mode_Parser multi_line
Tag containers.*
Refresh_Interval 1
Rotate_Wait 60
Mem_Buf_Limit 5MB
Skip_Long_Lines On
DB /tail-db/tail-containers-state-sumo.db
DB.Sync Normal
[INPUT]
Name systemd
Tag host.*
DB /tail-db/systemd-state-sumo.db
Systemd_Filter _SYSTEMD_UNIT=addon-config.service
Systemd_Filter _SYSTEMD_UNIT=addon-run.service
Systemd_Filter _SYSTEMD_UNIT=cfn-etcd-environment.service
Systemd_Filter _SYSTEMD_UNIT=cfn-signal.service
Systemd_Filter _SYSTEMD_UNIT=clean-ca-certificates.service
Systemd_Filter _SYSTEMD_UNIT=containerd.service
Systemd_Filter _SYSTEMD_UNIT=coreos-metadata.service
Systemd_Filter _SYSTEMD_UNIT=coreos-setup-environment.service
Systemd_Filter _SYSTEMD_UNIT=coreos-tmpfiles.service
Systemd_Filter _SYSTEMD_UNIT=dbus.service
Systemd_Filter _SYSTEMD_UNIT=docker.service
Systemd_Filter _SYSTEMD_UNIT=efs.service
Systemd_Filter _SYSTEMD_UNIT=etcd-member.service
Systemd_Filter _SYSTEMD_UNIT=etcd.service
Systemd_Filter _SYSTEMD_UNIT=etcd2.service
Systemd_Filter _SYSTEMD_UNIT=etcd3.service
Systemd_Filter _SYSTEMD_UNIT=etcdadm-check.service
Systemd_Filter _SYSTEMD_UNIT=etcdadm-reconfigure.service
Systemd_Filter _SYSTEMD_UNIT=etcdadm-save.service
Systemd_Filter _SYSTEMD_UNIT=etcdadm-update-status.service
Systemd_Filter _SYSTEMD_UNIT=flanneld.service
Systemd_Filter _SYSTEMD_UNIT=format-etcd2-volume.service
Systemd_Filter _SYSTEMD_UNIT=kube-node-taint-and-uncordon.service
Systemd_Filter _SYSTEMD_UNIT=kubelet.service
Systemd_Filter _SYSTEMD_UNIT=ldconfig.service
Systemd_Filter _SYSTEMD_UNIT=locksmithd.service
Systemd_Filter _SYSTEMD_UNIT=logrotate.service
Systemd_Filter _SYSTEMD_UNIT=lvm2-monitor.service
Systemd_Filter _SYSTEMD_UNIT=mdmon.service
Systemd_Filter _SYSTEMD_UNIT=nfs-idmapd.service
Systemd_Filter _SYSTEMD_UNIT=nfs-mountd.service
Systemd_Filter _SYSTEMD_UNIT=nfs-server.service
Systemd_Filter _SYSTEMD_UNIT=nfs-utils.service
Systemd_Filter _SYSTEMD_UNIT=node-problem-detector.service
Systemd_Filter _SYSTEMD_UNIT=ntp.service
Systemd_Filter _SYSTEMD_UNIT=oem-cloudinit.service
Systemd_Filter _SYSTEMD_UNIT=rkt-gc.service
Systemd_Filter _SYSTEMD_UNIT=rkt-metadata.service
Systemd_Filter _SYSTEMD_UNIT=rpc-idmapd.service
Systemd_Filter _SYSTEMD_UNIT=rpc-mountd.service
Systemd_Filter _SYSTEMD_UNIT=rpc-statd.service
Systemd_Filter _SYSTEMD_UNIT=rpcbind.service
Systemd_Filter _SYSTEMD_UNIT=set-aws-environment.service
Systemd_Filter _SYSTEMD_UNIT=system-cloudinit.service
Systemd_Filter _SYSTEMD_UNIT=systemd-timesyncd.service
Systemd_Filter _SYSTEMD_UNIT=update-ca-certificates.service
Systemd_Filter _SYSTEMD_UNIT=user-cloudinit.service
Systemd_Filter _SYSTEMD_UNIT=var-lib-etcd2.service
Max_Entries 1000
Read_From_Tail true
Name systemd
Tag host.*
DB /tail-db/systemd-state-sumo.db
Systemd_Filter _SYSTEMD_UNIT=addon-config.service
Systemd_Filter _SYSTEMD_UNIT=addon-run.service
Systemd_Filter _SYSTEMD_UNIT=cfn-etcd-environment.service
Systemd_Filter _SYSTEMD_UNIT=cfn-signal.service
Systemd_Filter _SYSTEMD_UNIT=clean-ca-certificates.service
Systemd_Filter _SYSTEMD_UNIT=containerd.service
Systemd_Filter _SYSTEMD_UNIT=coreos-metadata.service
Systemd_Filter _SYSTEMD_UNIT=coreos-setup-environment.service
Systemd_Filter _SYSTEMD_UNIT=coreos-tmpfiles.service
Systemd_Filter _SYSTEMD_UNIT=dbus.service
Systemd_Filter _SYSTEMD_UNIT=docker.service
Systemd_Filter _SYSTEMD_UNIT=efs.service
Systemd_Filter _SYSTEMD_UNIT=etcd-member.service
Systemd_Filter _SYSTEMD_UNIT=etcd.service
Systemd_Filter _SYSTEMD_UNIT=etcd2.service
Systemd_Filter _SYSTEMD_UNIT=etcd3.service
Systemd_Filter _SYSTEMD_UNIT=etcdadm-check.service
Systemd_Filter _SYSTEMD_UNIT=etcdadm-reconfigure.service
Systemd_Filter _SYSTEMD_UNIT=etcdadm-save.service
Systemd_Filter _SYSTEMD_UNIT=etcdadm-update-status.service
Systemd_Filter _SYSTEMD_UNIT=flanneld.service
Systemd_Filter _SYSTEMD_UNIT=format-etcd2-volume.service
Systemd_Filter _SYSTEMD_UNIT=kube-node-taint-and-uncordon.service
Systemd_Filter _SYSTEMD_UNIT=kubelet.service
Systemd_Filter _SYSTEMD_UNIT=ldconfig.service
Systemd_Filter _SYSTEMD_UNIT=locksmithd.service
Systemd_Filter _SYSTEMD_UNIT=logrotate.service
Systemd_Filter _SYSTEMD_UNIT=lvm2-monitor.service
Systemd_Filter _SYSTEMD_UNIT=mdmon.service
Systemd_Filter _SYSTEMD_UNIT=nfs-idmapd.service
Systemd_Filter _SYSTEMD_UNIT=nfs-mountd.service
Systemd_Filter _SYSTEMD_UNIT=nfs-server.service
Systemd_Filter _SYSTEMD_UNIT=nfs-utils.service
Systemd_Filter _SYSTEMD_UNIT=node-problem-detector.service
Systemd_Filter _SYSTEMD_UNIT=ntp.service
Systemd_Filter _SYSTEMD_UNIT=oem-cloudinit.service
Systemd_Filter _SYSTEMD_UNIT=rkt-gc.service
Systemd_Filter _SYSTEMD_UNIT=rkt-metadata.service
Systemd_Filter _SYSTEMD_UNIT=rpc-idmapd.service
Systemd_Filter _SYSTEMD_UNIT=rpc-mountd.service
Systemd_Filter _SYSTEMD_UNIT=rpc-statd.service
Systemd_Filter _SYSTEMD_UNIT=rpcbind.service
Systemd_Filter _SYSTEMD_UNIT=set-aws-environment.service
Systemd_Filter _SYSTEMD_UNIT=system-cloudinit.service
Systemd_Filter _SYSTEMD_UNIT=systemd-timesyncd.service
Systemd_Filter _SYSTEMD_UNIT=update-ca-certificates.service
Systemd_Filter _SYSTEMD_UNIT=user-cloudinit.service
Systemd_Filter _SYSTEMD_UNIT=var-lib-etcd2.service
Max_Entries 1000
Read_From_Tail true
outputs: |-
[OUTPUT]
Name forward
Match *
Host ${FLUENTD_LOGS_SVC}.${NAMESPACE}.svc.cluster.local.
Port 24321
tls off
tls.verify on
tls.debug 1
Name forward
Match *
Host ${FLUENTD_LOGS_SVC}.${NAMESPACE}.svc.cluster.local.
Port 24321
tls off
tls.verify on
tls.debug 1
[OUTPUT]
Name test-output-1
Name test-output-1
customParsers: |-
[PARSER]
Name multi_line
Format regex
Regex (?<log>^{"log":"\d{4}-\d{1,2}-\d{1,2}.\d{2}:\d{2}:\d{2}.*)
Name multi_line
Format regex
Regex (?<log>^{"log":"\d{4}-\d{1,2}-\d{1,2}.\d{2}:\d{2}:\d{2}.*)
[PARSER]
Name crio
Format regex
Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<log>.*)$
Time_Key time
Time_Format %Y-%m-%dT%H:%M:%S.%L%z
Name crio
Format regex
Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<log>.*)$
Time_Key time
Time_Format %Y-%m-%dT%H:%M:%S.%L%z
[PARSER]
Name test_json
Format json
Time_Key time
Time_Format %Y-%m-%dT%H:%M:%S.%L%z
Name test_json
Format json
Time_Key time
Time_Format %Y-%m-%dT%H:%M:%S.%L%z
[PARSER]
Name test-logfmt-parser-1
Format logfmt
Name test-logfmt-parser-1
Format logfmt
[PARSER]
Name test-regex-parser-1
Format regex
Regex .*
Name test-regex-parser-1
Format regex
Regex .*
filters: |-
[FILTER]
Name test-filter-1
Name test-filter-1