Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add v2 migration for fluent bit configs #1304

Merged
merged 1 commit into from
Jan 8, 2021
Merged

Add v2 migration for fluent bit configs #1304

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 28 additions & 25 deletions deploy/helm/sumologic/upgrade-2.0.0.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -635,44 +635,47 @@ function migrate_fluent_bit() {
yq w -i "${TEMP_FILE}" 'fluent-bit.extraVolumes[0].hostPath.type' 'DirectoryOrCreate'
yq w -i "${TEMP_FILE}" 'fluent-bit.extraVolumes[0].name' 'tail-db'

local CONFIG_KEY_WIDTH
readonly CONFIG_KEY_WIDTH="16"

local OUTPUTS=""
# Migrate fluent-bit.backend
if [[ -n "$(yq r "${TEMP_FILE}" -- 'fluent-bit.backend')" ]]; then
OUTPUTS="$(printf "%s[OUTPUT]\n" "${OUTPUTS}")"
if [[ -n "$(yq r "${TEMP_FILE}" -- 'fluent-bit.backend.type')" ]]; then
OUTPUTS="$(printf "%s\n %-14s%s\n" "${OUTPUTS}" "Name" "forward")"
OUTPUTS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s\n" "${OUTPUTS}" "Name" "forward")"
fi
if [[ -n "$(yq r "${TEMP_FILE}" -- 'fluent-bit.backend.forward')" ]]; then
OUTPUTS="$(printf "%s\n %-14s%s\n" "${OUTPUTS}" "Match" "*")"
OUTPUTS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s\n" "${OUTPUTS}" "Match" "*")"

local HOST
readonly HOST="$(yq r "${TEMP_FILE}" -- 'fluent-bit.backend.forward.host')"
if [[ -n "${HOST}" ]]; then
OUTPUTS="$(printf "%s\n %-14s%s\n" "${OUTPUTS}" "Host" "${HOST}")"
OUTPUTS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s\n" "${OUTPUTS}" "Host" "${HOST}")"
fi

local PORT
readonly PORT="$(yq r "${TEMP_FILE}" -- 'fluent-bit.backend.forward.port')"
if [[ -n "${PORT}" ]]; then
OUTPUTS="$(printf "%s\n %-14s%s\n" "${OUTPUTS}" "Port" "${PORT}")"
OUTPUTS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s\n" "${OUTPUTS}" "Port" "${PORT}")"
fi

local TLS
readonly TLS="$(yq r "${TEMP_FILE}" -- 'fluent-bit.backend.forward.tls')"
if [[ -n "${TLS}" ]]; then
OUTPUTS="$(printf "%s\n %-14s%s\n" "${OUTPUTS}" "tls" "${TLS}")"
OUTPUTS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s\n" "${OUTPUTS}" "tls" "${TLS}")"
fi

local TLS_VERIFY
readonly TLS_VERIFY="$(yq r "${TEMP_FILE}" -- 'fluent-bit.backend.forward.tls_verify')"
if [[ -n "${TLS_VERIFY}" ]]; then
OUTPUTS="$(printf "%s\n %-14s%s\n" "${OUTPUTS}" "tls.verify" "${TLS_VERIFY}")"
OUTPUTS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s\n" "${OUTPUTS}" "tls.verify" "${TLS_VERIFY}")"
fi

local TLS_DEBUG
readonly TLS_DEBUG="$(yq r "${TEMP_FILE}" -- 'fluent-bit.backend.forward.tls_debug')"
if [[ -n "${TLS_DEBUG}" ]]; then
OUTPUTS="$(printf "%s\n %-14s%s\n" "${OUTPUTS}" "tls.debug" "${TLS_DEBUG}")"
OUTPUTS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s\n" "${OUTPUTS}" "tls.debug" "${TLS_DEBUG}")"
fi
fi
fi
Expand All @@ -690,23 +693,23 @@ function migrate_fluent_bit() {
PARSERS="$(printf "[PARSER]\n")"
fi

PARSERS="$(printf "%s\n %-14s%s" "${PARSERS}" "Name" "${name}")"
PARSERS="$(printf "%s\n %-14s%s" "${PARSERS}" "Format" "regex")"
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Name" "${name}")"
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Format" "regex")"

local REGEX
REGEX="$(yq r "${TEMP_FILE}" -- "fluent-bit.parsers.regex.(name==${name}).regex" )"
PARSERS="$(printf "%s\n %-14s%s" "${PARSERS}" "Regex" "${REGEX}")"
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Regex" "${REGEX}")"

local TIMEKEY
TIMEKEY="$(yq r "${TEMP_FILE}" -- "fluent-bit.parsers.regex.(name==${name}).timeKey" )"
if [[ -n "${TIMEKEY}" ]]; then
PARSERS="$(printf "%s\n %-14s%s" "${PARSERS}" "Time_Key" "${TIMEKEY}")"
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Time_Key" "${TIMEKEY}")"
fi

local TIMEFORMAT
TIMEFORMAT="$(yq r "${TEMP_FILE}" -- "fluent-bit.parsers.regex.(name==${name}).timeFormat" )"
if [[ -n "${TIMEFORMAT}" ]]; then
PARSERS="$(printf "%s\n %-14s%s" "${PARSERS}" "Time_Format" "${TIMEFORMAT}")"
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Time_Format" "${TIMEFORMAT}")"
fi
done

Expand All @@ -718,25 +721,25 @@ function migrate_fluent_bit() {
PARSERS="$(printf "[PARSER]\n")"
fi

PARSERS="$(printf "%s\n %-14s%s" "${PARSERS}" "Name" "${name}")"
PARSERS="$(printf "%s\n %-14s%s" "${PARSERS}" "Format" "json")"
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Name" "${name}")"
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Format" "json")"

local TIMEKEY
TIMEKEY="$(yq r "${TEMP_FILE}" -- "fluent-bit.parsers.json.(name==${name}).timeKey" )"
if [[ -n "${TIMEKEY}" ]]; then
PARSERS="$(printf "%s\n %-14s%s" "${PARSERS}" "Time_Key" "${TIMEKEY}")"
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Time_Key" "${TIMEKEY}")"
fi

local TIMEFORMAT
TIMEFORMAT="$(yq r "${TEMP_FILE}" -- "fluent-bit.parsers.json.(name==${name}).timeFormat" )"
if [[ -n "${TIMEFORMAT}" ]]; then
PARSERS="$(printf "%s\n %-14s%s" "${PARSERS}" "Time_Format" "${TIMEFORMAT}")"
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Time_Format" "${TIMEFORMAT}")"
fi

local TIMEKEEP
TIMEKEEP="$(yq r "${TEMP_FILE}" -- "fluent-bit.parsers.json.(name==${name}).timeKeep" )"
if [[ -n "${TIMEKEEP}" ]]; then
PARSERS="$(printf "%s\n %-14s%s" "${PARSERS}" "Time_Keep" "${TIMEKEEP}")"
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Time_Keep" "${TIMEKEEP}")"
fi

local DECODE_FIELD_AS
Expand All @@ -747,7 +750,7 @@ function migrate_fluent_bit() {
if [[ -z "${DECODE_FIELD}" ]]; then
DECODE_FIELD="log"
fi
PARSERS="$(printf "%s\n %-14s%s %s" "${PARSERS}" "Decode_Field_As" "${DECODE_FIELD_AS}" "${DECODE_FIELD}")"
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s %s" "${PARSERS}" "Decode_Field_As" "${DECODE_FIELD_AS}" "${DECODE_FIELD}")"
fi
done

Expand All @@ -759,34 +762,34 @@ function migrate_fluent_bit() {
PARSERS="$(printf "[PARSER]\n")"
fi

PARSERS="$(printf "%s\n %-14s%s" "${PARSERS}" "Name" "${name}")"
PARSERS="$(printf "%s\n %-14s%s" "${PARSERS}" "Format" "logfmt")"
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Name" "${name}")"
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Format" "logfmt")"

local REGEX
REGEX="$(yq r "${TEMP_FILE}" -- "fluent-bit.parsers.regex.(name==${name}).regex" )"
PARSERS="$(printf "%s\n %-14s%s" "${PARSERS}" "Regex" "${REGEX}")"
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Regex" "${REGEX}")"

local TIMEKEY
TIMEKEY="$(yq r "${TEMP_FILE}" -- "fluent-bit.parsers.regex.(name==${name}).timeKey" )"
if [[ -n "${TIMEKEY}" ]]; then
PARSERS="$(printf "%s\n %-14s%s" "${PARSERS}" "Time_Key" "${TIMEKEY}")"
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Time_Key" "${TIMEKEY}")"
fi

local TIMEFORMAT
TIMEFORMAT="$(yq r "${TEMP_FILE}" -- "fluent-bit.parsers.regex.(name==${name}).timeFormat" )"
if [[ -n "${TIMEFORMAT}" ]]; then
PARSERS="$(printf "%s\n %-14s%s" "${PARSERS}" "Time_Format" "${TIMEFORMAT}")"
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Time_Format" "${TIMEFORMAT}")"
fi
done
fi

local INPUTS=""
local OUTPUTS=""
local FILTERS=""
local RAWCONFIG
readonly RAWCONFIG="$(yq r "${TEMP_FILE}" 'fluent-bit.rawConfig')"

local SECTION=""
shopt -s extglob
while IFS= read -r line
do
if [[ "${line}" =~ ^\[.* ]]; then
Expand Down Expand Up @@ -825,7 +828,7 @@ function migrate_fluent_bit() {
continue
else
# Trim whitespace
line="$(echo "${line}" | awk '{$1=$1;print}')"
line="${line##+([[:space:]])}"

case "${SECTION}" in
"INPUT")
Expand Down
170 changes: 89 additions & 81 deletions tests/upgrade_v2_script/static/fluent_bit.output.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,97 +30,105 @@ fluent-bit:
config:
inputs: |-
[INPUT]
Name tail
Path /var/log/containers/*.log
Multiline On
Name tail
Path /var/log/containers/*.log
Multiline On
Parser_Firstline multi_line
Tag containers.*
Tag containers.*
Refresh_Interval 1
Rotate_Wait 60
Mem_Buf_Limit 5MB
Skip_Long_Lines On
DB /tail-db/tail-containers-state-sumo.db
DB.Sync Normal
Rotate_Wait 60
Mem_Buf_Limit 5MB
Skip_Long_Lines On
DB /tail-db/tail-containers-state-sumo.db
DB.Sync Normal
[INPUT]
Name systemd
Tag host.*
DB /tail-db/systemd-state-sumo.db
Systemd_Filter _SYSTEMD_UNIT=addon-config.service
Systemd_Filter _SYSTEMD_UNIT=addon-run.service
Systemd_Filter _SYSTEMD_UNIT=cfn-etcd-environment.service
Systemd_Filter _SYSTEMD_UNIT=cfn-signal.service
Systemd_Filter _SYSTEMD_UNIT=clean-ca-certificates.service
Systemd_Filter _SYSTEMD_UNIT=containerd.service
Systemd_Filter _SYSTEMD_UNIT=coreos-metadata.service
Systemd_Filter _SYSTEMD_UNIT=coreos-setup-environment.service
Systemd_Filter _SYSTEMD_UNIT=coreos-tmpfiles.service
Systemd_Filter _SYSTEMD_UNIT=dbus.service
Systemd_Filter _SYSTEMD_UNIT=docker.service
Systemd_Filter _SYSTEMD_UNIT=efs.service
Systemd_Filter _SYSTEMD_UNIT=etcd-member.service
Systemd_Filter _SYSTEMD_UNIT=etcd.service
Systemd_Filter _SYSTEMD_UNIT=etcd2.service
Systemd_Filter _SYSTEMD_UNIT=etcd3.service
Systemd_Filter _SYSTEMD_UNIT=etcdadm-check.service
Systemd_Filter _SYSTEMD_UNIT=etcdadm-reconfigure.service
Systemd_Filter _SYSTEMD_UNIT=etcdadm-save.service
Systemd_Filter _SYSTEMD_UNIT=etcdadm-update-status.service
Systemd_Filter _SYSTEMD_UNIT=flanneld.service
Systemd_Filter _SYSTEMD_UNIT=format-etcd2-volume.service
Systemd_Filter _SYSTEMD_UNIT=kube-node-taint-and-uncordon.service
Systemd_Filter _SYSTEMD_UNIT=kubelet.service
Systemd_Filter _SYSTEMD_UNIT=ldconfig.service
Systemd_Filter _SYSTEMD_UNIT=locksmithd.service
Systemd_Filter _SYSTEMD_UNIT=logrotate.service
Systemd_Filter _SYSTEMD_UNIT=lvm2-monitor.service
Systemd_Filter _SYSTEMD_UNIT=mdmon.service
Systemd_Filter _SYSTEMD_UNIT=nfs-idmapd.service
Systemd_Filter _SYSTEMD_UNIT=nfs-mountd.service
Systemd_Filter _SYSTEMD_UNIT=nfs-server.service
Systemd_Filter _SYSTEMD_UNIT=nfs-utils.service
Systemd_Filter _SYSTEMD_UNIT=node-problem-detector.service
Systemd_Filter _SYSTEMD_UNIT=ntp.service
Systemd_Filter _SYSTEMD_UNIT=oem-cloudinit.service
Systemd_Filter _SYSTEMD_UNIT=rkt-gc.service
Systemd_Filter _SYSTEMD_UNIT=rkt-metadata.service
Systemd_Filter _SYSTEMD_UNIT=rpc-idmapd.service
Systemd_Filter _SYSTEMD_UNIT=rpc-mountd.service
Systemd_Filter _SYSTEMD_UNIT=rpc-statd.service
Systemd_Filter _SYSTEMD_UNIT=rpcbind.service
Systemd_Filter _SYSTEMD_UNIT=set-aws-environment.service
Systemd_Filter _SYSTEMD_UNIT=system-cloudinit.service
Systemd_Filter _SYSTEMD_UNIT=systemd-timesyncd.service
Systemd_Filter _SYSTEMD_UNIT=update-ca-certificates.service
Systemd_Filter _SYSTEMD_UNIT=user-cloudinit.service
Systemd_Filter _SYSTEMD_UNIT=var-lib-etcd2.service
Max_Entries 1000
Read_From_Tail true
Name systemd
Tag host.*
DB /tail-db/systemd-state-sumo.db
Systemd_Filter _SYSTEMD_UNIT=addon-config.service
Systemd_Filter _SYSTEMD_UNIT=addon-run.service
Systemd_Filter _SYSTEMD_UNIT=cfn-etcd-environment.service
Systemd_Filter _SYSTEMD_UNIT=cfn-signal.service
Systemd_Filter _SYSTEMD_UNIT=clean-ca-certificates.service
Systemd_Filter _SYSTEMD_UNIT=containerd.service
Systemd_Filter _SYSTEMD_UNIT=coreos-metadata.service
Systemd_Filter _SYSTEMD_UNIT=coreos-setup-environment.service
Systemd_Filter _SYSTEMD_UNIT=coreos-tmpfiles.service
Systemd_Filter _SYSTEMD_UNIT=dbus.service
Systemd_Filter _SYSTEMD_UNIT=docker.service
Systemd_Filter _SYSTEMD_UNIT=efs.service
Systemd_Filter _SYSTEMD_UNIT=etcd-member.service
Systemd_Filter _SYSTEMD_UNIT=etcd.service
Systemd_Filter _SYSTEMD_UNIT=etcd2.service
Systemd_Filter _SYSTEMD_UNIT=etcd3.service
Systemd_Filter _SYSTEMD_UNIT=etcdadm-check.service
Systemd_Filter _SYSTEMD_UNIT=etcdadm-reconfigure.service
Systemd_Filter _SYSTEMD_UNIT=etcdadm-save.service
Systemd_Filter _SYSTEMD_UNIT=etcdadm-update-status.service
Systemd_Filter _SYSTEMD_UNIT=flanneld.service
Systemd_Filter _SYSTEMD_UNIT=format-etcd2-volume.service
Systemd_Filter _SYSTEMD_UNIT=kube-node-taint-and-uncordon.service
Systemd_Filter _SYSTEMD_UNIT=kubelet.service
Systemd_Filter _SYSTEMD_UNIT=ldconfig.service
Systemd_Filter _SYSTEMD_UNIT=locksmithd.service
Systemd_Filter _SYSTEMD_UNIT=logrotate.service
Systemd_Filter _SYSTEMD_UNIT=lvm2-monitor.service
Systemd_Filter _SYSTEMD_UNIT=mdmon.service
Systemd_Filter _SYSTEMD_UNIT=nfs-idmapd.service
Systemd_Filter _SYSTEMD_UNIT=nfs-mountd.service
Systemd_Filter _SYSTEMD_UNIT=nfs-server.service
Systemd_Filter _SYSTEMD_UNIT=nfs-utils.service
Systemd_Filter _SYSTEMD_UNIT=node-problem-detector.service
Systemd_Filter _SYSTEMD_UNIT=ntp.service
Systemd_Filter _SYSTEMD_UNIT=oem-cloudinit.service
Systemd_Filter _SYSTEMD_UNIT=rkt-gc.service
Systemd_Filter _SYSTEMD_UNIT=rkt-metadata.service
Systemd_Filter _SYSTEMD_UNIT=rpc-idmapd.service
Systemd_Filter _SYSTEMD_UNIT=rpc-mountd.service
Systemd_Filter _SYSTEMD_UNIT=rpc-statd.service
Systemd_Filter _SYSTEMD_UNIT=rpcbind.service
Systemd_Filter _SYSTEMD_UNIT=set-aws-environment.service
Systemd_Filter _SYSTEMD_UNIT=system-cloudinit.service
Systemd_Filter _SYSTEMD_UNIT=systemd-timesyncd.service
Systemd_Filter _SYSTEMD_UNIT=update-ca-certificates.service
Systemd_Filter _SYSTEMD_UNIT=user-cloudinit.service
Systemd_Filter _SYSTEMD_UNIT=var-lib-etcd2.service
Max_Entries 1000
Read_From_Tail true
outputs: |-
[OUTPUT]
Name test-output-1
Name forward
Match *
Host ${CHART}.${NAMESPACE}.svc.cluster.local.
Port 24321
tls off
tls.verify on
tls.debug 1
[OUTPUT]
Name test-output-1
customParsers: |-
[PARSER]
Name multi_line
Format regex
Regex (?<log>^{"log":"\d{4}-\d{1,2}-\d{1,2}.\d{2}:\d{2}:\d{2}.*)
Name multi_line
Format regex
Regex (?<log>^{"log":"\d{4}-\d{1,2}-\d{1,2}.\d{2}:\d{2}:\d{2}.*)
[PARSER]
Name crio
Format regex
Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<log>.*)$
Time_Key time
Time_Format %Y-%m-%dT%H:%M:%S.%L%z
Name crio
Format regex
Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<log>.*)$
Time_Key time
Time_Format %Y-%m-%dT%H:%M:%S.%L%z
[PARSER]
Name test_json
Format json
Time_Key time
Time_Format %Y-%m-%dT%H:%M:%S.%L%z
Name test_json
Format json
Time_Key time
Time_Format %Y-%m-%dT%H:%M:%S.%L%z
[PARSER]
Name test-logfmt-parser-1
Format logfmt
Name test-logfmt-parser-1
Format logfmt
[PARSER]
Name test-regex-parser-1
Format regex
Regex .*
Name test-regex-parser-1
Format regex
Regex .*
filters: |-
[FILTER]
Name test-filter-1
Name test-filter-1