Skip to content

Bug 1734951: rsyslog with MERGE_JSON_LOG=true does not handle non-JSON data #222

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Aug 2, 2019
Merged

Bug 1734951: rsyslog with MERGE_JSON_LOG=true does not handle non-JSON data #222

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions files/rsyslog/20-viaq-templates.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,11 @@ template(name="viaq_template_nl" type="list") {

template(name="prefix_index_template" type="list") {
property(name="$.viaq_index_prefix")
property(name="$!@timestamp" dateFormat="rfc3339" position.from="1" position.to="4")
property(name="$.ts_for_index" dateFormat="rfc3339" position.from="1" position.to="4")
constant(value=".")
property(name="$!@timestamp" dateFormat="rfc3339" position.from="6" position.to="7")
property(name="$.ts_for_index" dateFormat="rfc3339" position.from="6" position.to="7")
constant(value=".")
property(name="$!@timestamp" dateFormat="rfc3339" position.from="9" position.to="10")
property(name="$.ts_for_index" dateFormat="rfc3339" position.from="9" position.to="10")
}

template(name="index_template" type="string" string="%$.viaq_index_name%")
Expand Down
60 changes: 36 additions & 24 deletions files/rsyslog/60-mmk8s.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,36 +16,48 @@ if ((strlen($!CONTAINER_NAME) > 0) and (strlen($!CONTAINER_ID_FULL) > 0)) or
if ((strlen($!metadata) > 0) and (strlen($!metadata!filename) > 0) and ($!metadata!filename startswith "/var/log/containers/")) then {
if $msg startswith "{" then {
action(name="parse_multiline_docker_json" type="mmnormalize" rulebase="/etc/rsyslog.d/multiline-json.rulebase")
foreach ($.ii in $!multilinejson) do {
if strlen($!@timestamp) == 0 then {
set $!@timestamp = $.ii!time;
}
if strlen($!stream) == 0 then {
set $!stream = $.ii!stream;
}
if strlen($!log) == 0 then {
set $!log = $.ii!log;
} else {
reset $!log = $!log & $.ii!log;
if strlen($!originalmsg) > 0 then {
# parsing failed
unset $!originalmsg;
unset $!unparsed-data;
} else {
foreach ($.ii in $!multilinejson) do {
if strlen($!@timestamp) == 0 then {
set $!@timestamp = $.ii!time;
}
if strlen($!stream) == 0 then {
set $!stream = $.ii!stream;
}
if strlen($!log) == 0 then {
set $!log = $.ii!log;
} else {
reset $!log = $!log & $.ii!log;
}
}
unset $!multilinejson;
}
unset $!multilinejson;
} else {
action(name="parse_multiline_crio_file" type="mmnormalize" rulebase="/etc/rsyslog.d/crio.rulebase")
foreach ($.ii in $!multilinecrio) do {
if strlen($!@timestamp) == 0 then {
set $!@timestamp = $.ii!time;
}
if strlen($!stream) == 0 then {
set $!stream = $.ii!stream;
}
if strlen($!log) == 0 then {
set $!log = $.ii!log;
} else {
reset $!log = $!log & $.ii!log;
if strlen($!originalmsg) > 0 then {
# parsing failed
unset $!originalmsg;
unset $!unparsed-data;
} else {
foreach ($.ii in $!multilinecrio) do {
if strlen($!@timestamp) == 0 then {
set $!@timestamp = $.ii!time;
}
if strlen($!stream) == 0 then {
set $!stream = $.ii!stream;
}
if strlen($!log) == 0 then {
set $!log = $.ii!log;
} else {
reset $!log = $!log & $.ii!log;
}
}
unset $!multilinecrio;
}
unset $!multilinecrio;
}
}
action(type="mmkubernetes"
Expand Down
25 changes: 21 additions & 4 deletions files/rsyslog/65-viaq-formatting.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,11 @@ if strlen($!MESSAGE) > 0 then {
set $!pipeline_metadata!collector!original_raw_message = $!MESSAGE;
if (strlen(`echo $MERGE_JSON_LOG`) > 0) and (`echo $MERGE_JSON_LOG` == "true") then {
action(name="parse_json_journald" type="mmnormalize" ruleBase="/etc/rsyslog.d/parse_json.rulebase" variable="$!MESSAGE")
if strlen($!originalmsg) > 0 then {
# parsing failed - not json - just continue
unset $!originalmsg;
unset $!unparsed-data;
}
}
# ensure that $!message is set and $!MESSAGE is unset
# if rsyslog is case sensitive, then $!MESSAGE == $!message
Expand All @@ -25,21 +30,22 @@ if strlen($!MESSAGE) > 0 then {
}
unset $!MESSAGE;
}
unset $!originalmsg;
unset $!unparsed-data;
} else {
if strlen($!log) > 0 then {
set $!pipeline_metadata!collector!original_raw_message = $!log;
if (strlen(`echo $MERGE_JSON_LOG`) > 0) and (`echo $MERGE_JSON_LOG` == "true") then {
action(name="parse_json_containers" type="mmnormalize" ruleBase="/etc/rsyslog.d/parse_json.rulebase" variable="$!log")
if strlen($!originalmsg) > 0 then {
# parsing failed - not json - just continue
unset $!originalmsg;
unset $!unparsed-data;
}
}
if strlen($!message) == 0 then {
set $!message = $!log;
}
unset $!log;
}
unset $!originalmsg;
unset $!unparsed-data;
}

if strlen($!_MACHINE_ID) > 0 then {
Expand Down Expand Up @@ -262,6 +268,17 @@ if strlen($!level) > 0 then {
}
unset $!stream;

# normalize timestamp for use in index name, but preserve whatever
# timestamp the user may have set in the log e.g. when using
# MERGE_JSON_LOG=true and the user has set "@timestamp" in the log
# we must have something in YYYY-MM-DD format for the index suffix
set $.tssec = parse_time($!@timestamp);
if $.tssec == 0 then {
set $.ts_for_index = exec_template("cnvt_to_viaq_timestamp");
} else {
set $.ts_for_index = format_time($.tssec, "date-rfc3339");
}

# add eventrouter

if (strlen($!kubernetes) > 0) and (strlen($!kubernetes!namespace_name) > 0) then {
Expand Down
5 changes: 5 additions & 0 deletions files/rsyslog/67-mmexternal.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,11 @@ module(load="mmexternal")

if (strlen(`echo $SKIP_EMPTY`) > 0) and (`echo $SKIP_EMPTY` == "true") then {
action(name="skip_empty" type="mmnormalize" ruleBase="/etc/rsyslog.d/parse_json_skip_empty.rulebase" path="$!openshift_logging_all")
if strlen($!originalmsg) > 0 then {
# parsing failed - not json - just continue
unset $!originalmsg;
unset $!unparsed-data;
}
} else if (strlen(`echo $USE_MMEXTERNAL`) > 0) and (`echo $USE_MMEXTERNAL` == "true") then {
action(name="undefined_field" type="mmexternal" binary="/usr/local/bin/undefined_field" interface.input="fulljson")
}
Expand Down