Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Don't add empty lines to Fluent-bit config #1318

Merged
merged 1 commit into from
Jan 12, 2021
Merged

Don't add empty lines to Fluent-bit config #1318

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
91 changes: 46 additions & 45 deletions deploy/helm/sumologic/upgrade-2.0.0.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -673,46 +673,46 @@ function migrate_fluent_bit() {
if [[ -n "$(yq r "${TEMP_FILE}" -- 'fluent-bit.backend')" ]]; then
OUTPUTS="$(printf "%s[OUTPUT]\n" "${OUTPUTS}")"
if [[ -n "$(yq r "${TEMP_FILE}" -- 'fluent-bit.backend.type')" ]]; then
OUTPUTS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s\n" "${OUTPUTS}" "Name" "forward")"
OUTPUTS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s\n" "${OUTPUTS}" "Name" "forward")"
fi
if [[ -n "$(yq r "${TEMP_FILE}" -- 'fluent-bit.backend.forward')" ]]; then
OUTPUTS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s\n" "${OUTPUTS}" "Match" "*")"
OUTPUTS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s\n" "${OUTPUTS}" "Match" "*")"

local HOST
readonly HOST="$(yq r "${TEMP_FILE}" -- 'fluent-bit.backend.forward.host')"
if [[ -n "${HOST}" ]]; then
# shellcheck disable=SC2016
if [[ "${HOST}" == '${CHART}.${NAMESPACE}.svc.cluster.local.' ]]; then
info 'Migrating "fluent-bit.backend.forward.host" from "${CHART}.${NAMESPACE}.svc.cluster.local." to "${FLUENTD_LOGS_SVC}.${NAMESPACE}.svc.cluster.local."'
OUTPUTS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s\n" "${OUTPUTS}" "Host" '${FLUENTD_LOGS_SVC}.${NAMESPACE}.svc.cluster.local.')"
OUTPUTS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s\n" "${OUTPUTS}" "Host" '${FLUENTD_LOGS_SVC}.${NAMESPACE}.svc.cluster.local.')"
else
warning "Unexpected \"fluent-bit.backend.forward.host\" value: ${HOST}. Leaving it as is in new fluent-bit configuration"
OUTPUTS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s\n" "${OUTPUTS}" "Host" "${HOST}")"
OUTPUTS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s\n" "${OUTPUTS}" "Host" "${HOST}")"
fi
fi

local PORT
readonly PORT="$(yq r "${TEMP_FILE}" -- 'fluent-bit.backend.forward.port')"
if [[ -n "${PORT}" ]]; then
OUTPUTS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s\n" "${OUTPUTS}" "Port" "${PORT}")"
OUTPUTS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s\n" "${OUTPUTS}" "Port" "${PORT}")"
fi

local TLS
readonly TLS="$(yq r "${TEMP_FILE}" -- 'fluent-bit.backend.forward.tls')"
if [[ -n "${TLS}" ]]; then
OUTPUTS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s\n" "${OUTPUTS}" "tls" "${TLS}")"
OUTPUTS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s\n" "${OUTPUTS}" "tls" "${TLS}")"
fi

local TLS_VERIFY
readonly TLS_VERIFY="$(yq r "${TEMP_FILE}" -- 'fluent-bit.backend.forward.tls_verify')"
if [[ -n "${TLS_VERIFY}" ]]; then
OUTPUTS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s\n" "${OUTPUTS}" "tls.verify" "${TLS_VERIFY}")"
OUTPUTS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s\n" "${OUTPUTS}" "tls.verify" "${TLS_VERIFY}")"
fi

local TLS_DEBUG
readonly TLS_DEBUG="$(yq r "${TEMP_FILE}" -- 'fluent-bit.backend.forward.tls_debug')"
if [[ -n "${TLS_DEBUG}" ]]; then
OUTPUTS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s\n" "${OUTPUTS}" "tls.debug" "${TLS_DEBUG}")"
OUTPUTS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s\n" "${OUTPUTS}" "tls.debug" "${TLS_DEBUG}")"
fi
fi
fi
Expand All @@ -730,23 +730,23 @@ function migrate_fluent_bit() {
PARSERS="$(printf "[PARSER]\n")"
fi

PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Name" "${name}")"
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Format" "regex")"
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Name" "${name}")"
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Format" "regex")"

local REGEX
REGEX="$(yq r "${TEMP_FILE}" -- "fluent-bit.parsers.regex.(name==${name}).regex" )"
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Regex" "${REGEX}")"
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Regex" "${REGEX}")"

local TIMEKEY
TIMEKEY="$(yq r "${TEMP_FILE}" -- "fluent-bit.parsers.regex.(name==${name}).timeKey" )"
if [[ -n "${TIMEKEY}" ]]; then
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Time_Key" "${TIMEKEY}")"
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Time_Key" "${TIMEKEY}")"
fi

local TIMEFORMAT
TIMEFORMAT="$(yq r "${TEMP_FILE}" -- "fluent-bit.parsers.regex.(name==${name}).timeFormat" )"
if [[ -n "${TIMEFORMAT}" ]]; then
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Time_Format" "${TIMEFORMAT}")"
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Time_Format" "${TIMEFORMAT}")"
fi
done

Expand All @@ -758,25 +758,25 @@ function migrate_fluent_bit() {
PARSERS="$(printf "[PARSER]\n")"
fi

PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Name" "${name}")"
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Format" "json")"
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Name" "${name}")"
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Format" "json")"

local TIMEKEY
TIMEKEY="$(yq r "${TEMP_FILE}" -- "fluent-bit.parsers.json.(name==${name}).timeKey" )"
if [[ -n "${TIMEKEY}" ]]; then
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Time_Key" "${TIMEKEY}")"
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Time_Key" "${TIMEKEY}")"
fi

local TIMEFORMAT
TIMEFORMAT="$(yq r "${TEMP_FILE}" -- "fluent-bit.parsers.json.(name==${name}).timeFormat" )"
if [[ -n "${TIMEFORMAT}" ]]; then
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Time_Format" "${TIMEFORMAT}")"
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Time_Format" "${TIMEFORMAT}")"
fi

local TIMEKEEP
TIMEKEEP="$(yq r "${TEMP_FILE}" -- "fluent-bit.parsers.json.(name==${name}).timeKeep" )"
if [[ -n "${TIMEKEEP}" ]]; then
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Time_Keep" "${TIMEKEEP}")"
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Time_Keep" "${TIMEKEEP}")"
fi

local DECODE_FIELD_AS
Expand All @@ -787,7 +787,7 @@ function migrate_fluent_bit() {
if [[ -z "${DECODE_FIELD}" ]]; then
DECODE_FIELD="log"
fi
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s %s" "${PARSERS}" "Decode_Field_As" "${DECODE_FIELD_AS}" "${DECODE_FIELD}")"
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s %s" "${PARSERS}" "Decode_Field_As" "${DECODE_FIELD_AS}" "${DECODE_FIELD}")"
fi
done

Expand All @@ -799,23 +799,23 @@ function migrate_fluent_bit() {
PARSERS="$(printf "[PARSER]\n")"
fi

PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Name" "${name}")"
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Format" "logfmt")"
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Name" "${name}")"
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Format" "logfmt")"

local REGEX
REGEX="$(yq r "${TEMP_FILE}" -- "fluent-bit.parsers.regex.(name==${name}).regex" )"
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Regex" "${REGEX}")"
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Regex" "${REGEX}")"

local TIMEKEY
TIMEKEY="$(yq r "${TEMP_FILE}" -- "fluent-bit.parsers.regex.(name==${name}).timeKey" )"
if [[ -n "${TIMEKEY}" ]]; then
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Time_Key" "${TIMEKEY}")"
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Time_Key" "${TIMEKEY}")"
fi

local TIMEFORMAT
TIMEFORMAT="$(yq r "${TEMP_FILE}" -- "fluent-bit.parsers.regex.(name==${name}).timeFormat" )"
if [[ -n "${TIMEFORMAT}" ]]; then
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Time_Format" "${TIMEFORMAT}")"
PARSERS="$(printf "%s\n %-${CONFIG_KEY_WIDTH}s%s" "${PARSERS}" "Time_Format" "${TIMEFORMAT}")"
fi
done
fi
Expand Down Expand Up @@ -866,27 +866,28 @@ function migrate_fluent_bit() {
else
# Trim whitespace
line="${line##+([[:space:]])}"

case "${SECTION}" in
"INPUT")
INPUTS="$(printf "%s\n %s" "${INPUTS}" "${line}")"
;;

"OUTPUT")
OUTPUTS="$(printf "%s\n %s" "${OUTPUTS}" "${line}")"
;;

"PARSER")
PARSERS="$(printf "%s\n %s" "${PARSERS}" "${line}")"
;;

"FILTER")
FILTERS="$(printf "%s\n %s" "${FILTERS}" "${line}")"
;;

*)
;;
esac
if [[ -n "${line}" ]]; then
case "${SECTION}" in
"INPUT")
INPUTS="$(printf "%s\n %s" "${INPUTS}" "${line}")"
;;

"OUTPUT")
OUTPUTS="$(printf "%s\n %s" "${OUTPUTS}" "${line}")"
;;

"PARSER")
PARSERS="$(printf "%s\n %s" "${PARSERS}" "${line}")"
;;

"FILTER")
FILTERS="$(printf "%s\n %s" "${FILTERS}" "${line}")"
;;

*)
;;
esac
fi
fi
done < <(echo "${RAWCONFIG}")

Expand Down
5 changes: 5 additions & 0 deletions tests/upgrade_v2_script/static/fluent_bit.input.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ fluent-bit:
timeFormat: "%Y-%m-%dT%H:%M:%S.%L%z"
rawConfig: |-
@INCLUDE fluent-bit-service.conf

[INPUT]
Name tail
Path /var/log/containers/*.log
Expand All @@ -65,10 +66,13 @@ fluent-bit:
Skip_Long_Lines On
DB /tail-db/tail-containers-state-sumo.db
DB.Sync Normal

[OUTPUT]
Name test-output-1

[FILTER]
Name test-filter-1

[PARSER]
Name test-logfmt-parser-1
Format logfmt
Expand Down Expand Up @@ -130,4 +134,5 @@ fluent-bit:
Systemd_Filter _SYSTEMD_UNIT=var-lib-etcd2.service
Max_Entries 1000
Read_From_Tail true

@INCLUDE fluent-bit-output.conf
Loading