Parsing based on new line and delimiter

[dpkkumar01]

I have the following log pattern in my system & i would like to know is there any option to parse this message using liblognorm

Log Format:

---

RecordType:Submit
RecepientID:2328288id23
MediaTypes:image/jpeg,image/jpeg,image/jpeg,application/smil
MediaSizes:31214,31900,31214,364
ContentType:multipart/related

---

RecordType:Submit
RecepientID:2328232id23
MediaTypes:image/jpeg,image/jpeg,image/jpeg,application/smil
MediaSizes:31214,31900,31214,364
ContentType:multipart/related

---

RecordType:Submit
RecepientID:23282353id23
MediaTypes:image/jpeg,image/jpeg,image/jpeg,application/smil
MediaSizes:31214,31900,31214,364
ContentType:multipart/related

---

Image:

[manios]

Hi @dpkkumar01 ,

Yes, you can use Rsyslog with Liblognorm to parse your message. You will need three files:

1. rsyslog.conf : rsyslog configuration
2. issue308.rule: Liblognorm rule file to parse your message
3. multiline.log: your input file where you have your logs

Given rsyslog.conf :

#  /etc/rsyslog.conf	Configuration file for rsyslog.
#
#			For more information see
#			/usr/share/doc/rsyslog-doc/html/rsyslog_conf.html
#
#  Default logging rules can be found in /etc/rsyslog.d/50-default.conf


#################
#### MODULES ####
#################

module(load="imuxsock") # provides support for local system logging
# module(load="imklog")   # provides kernel logging support
module(load="immark")  # provides --MARK-- message capability
# $ModLoad imuxsock # provides support for local system logging
# $ModLoad imklog   # provides kernel logging support (previously done by rklogd)
#$ModLoad immark  # provides --MARK-- message capability

# provides UDP syslog reception
# $ModLoad imudp
# $UDPServerRun 514

# provides TCP syslog reception
# $ModLoad imtcp
# $InputTCPServerRun 514
module(load="builtin:omfile")
module(load="mmnormalize") # parser using liblognorm
module(load="imfile")

###########################
#### GLOBAL DIRECTIVES ####
###########################

#
# Use traditional timestamp format.
# To enable high precision timestamps, comment out the following line.
#
$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat

# Filter duplicated messages
$RepeatedMsgReduction on

# Some messages are over 10k, so increase max message size
$MaxMessageSize 30k

#
# Set the default permissions for all log files.
#
$FileOwner root
$FileGroup root
$FileCreateMode 0640
$DirCreateMode 0755
$Umask 0022
$PrivDropToUser syslog
$PrivDropToGroup syslog

# General globals
global(net.enableDNS="off")

# Remove Control Chars
global(parser.escapeControlCharactersOnReceive="off" )

#
# Where to place spool files
#
$WorkDirectory /var/spool/rsyslog

global(workDirectory="/var/spool/rsyslog")
#
# Include all config files in /etc/rsyslog.d/
#
# $IncludeConfig /etc/rsyslog.d/*.conf



#################
#### Inputs  ####
#################

# File 1
input(type="imfile"
      File="/opt/multiline.log"
      Tag="multilos"
      PersistStateInterval="1"
      freshStartTail="off"
      startmsg.regex="[*]+")

#################
### Templates ###
#################

# this is for formatting our syslog in JSON with @timestamp for output to Elasticsearch
template(name="log-json-template"
  type="list") {
    constant(value="{")
      constant(value="\"@version\":\"1")
      constant(value="\",\"@timestamp\":\"")		property(name="timegenerated" dateFormat="rfc3339")
      constant(value="\",\"host\":\"")			    property(name="hostname") 
      constant(value="\",\"type\":\"syslog")
      constant(value="\",\"syslog_timestamp\":\"")	property(name="timereported" dateFormat="rfc3164"  format="json")
      constant(value="\",\"syslog_hostname\":\"")   property(name="hostname" format="json")
      constant(value="\",\"syslog_program\":\"")    property(name="programname" format="json")
      constant(value="\",\"syslog_message\":\"")    property(name="msg" format="json")
      constant(value="\",\"received_at\":\"")		property(name="timegenerated" dateFormat="rfc3339")
      constant(value="\",\"received_from\":\"")	    property(name="fromhost" format="json")
      constant(value="\",\"logi:\":")             property(name="$!")
      constant(value="}\n")
}

#################
#### Actions ####
#################

if ($syslogtag contains 'multilos') then {

    # Parse simple JSON message with liblognorm (in order to have parsed message to json first level)
    action(type="mmnormalize" rulebase="/etc/rsyslog.d/issue308.rule")

    action(type="omfile" File="/tmp/ml-parsed" template="log-json-template")
}

, rule file issue308.rule :

version=2

type=@Logline:%-:char-to{"extradata":":"}%:%..:string-to{"extradata":"\\n"}%

rule=:%-:string-to{"extradata":"\\n"}%\\n%recordType:@Logline%\\n%recepientID:@Logline%\\n%mediaTypes:@Logline%\\n%mediaSizes:@Logline%\\n%contentType:rest%

and input log file:

************************************************************
RecordType:Submit
RecepientID:2328288id23
MediaTypes:image/jpeg,image/jpeg,image/jpeg,application/smil
MediaSizes:31214,31900,31214,364
ContentType:multipart/related
************************************************************
RecordType:Submit
RecepientID:2328232id23
MediaTypes:image/jpeg,image/jpeg,image/jpeg,application/smil
MediaSizes:31214,31900,31214,364
ContentType:multipart/related
************************************************************
RecordType:Submit
RecepientID:23282353id23
MediaTypes:image/jpeg,image/jpeg,image/jpeg,application/smil
MediaSizes:31214,31900,31214,364
ContentType:multipart/related
************************************************************

then Rsyslog will output to /tmp/ml-parsed the following:

{"@version":"1","@timestamp":"2018-10-11T17:20:25.369215+00:00","host":"cd27d80528c2","type":"syslog","syslog_timestamp":"Oct 11 17:20:25","syslog_hostname":"cd27d80528c2","syslog_program":"multilos","syslog_message":"************************************************************\\nRecordType:Submit\\nRecepientID:2328288id23\\nMediaTypes:image\/jpeg,image\/jpeg,image\/jpeg,application\/smil\\nMediaSizes:31214,31900,31214,364\\nContentType:multipart\/related","received_at":"2018-10-11T17:20:25.369215+00:00","received_from":"","logi:":{ "contentType": "ContentType:multipart\/related", "mediaSizes": "31214,31900,31214,364", "mediaTypes": "image\/jpeg,image\/jpeg,image\/jpeg,application\/smil", "recepientID": "2328288id23", "recordType": "Submit" }}
{"@version":"1","@timestamp":"2018-10-11T17:20:25.369279+00:00","host":"cd27d80528c2","type":"syslog","syslog_timestamp":"Oct 11 17:20:25","syslog_hostname":"cd27d80528c2","syslog_program":"multilos","syslog_message":"************************************************************\\nRecordType:Submit\\nRecepientID:2328232id23\\nMediaTypes:image\/jpeg,image\/jpeg,image\/jpeg,application\/smil\\nMediaSizes:31214,31900,31214,364\\nContentType:multipart\/related","received_at":"2018-10-11T17:20:25.369279+00:00","received_from":"","logi:":{ "contentType": "ContentType:multipart\/related", "mediaSizes": "31214,31900,31214,364", "mediaTypes": "image\/jpeg,image\/jpeg,image\/jpeg,application\/smil", "recepientID": "2328232id23", "recordType": "Submit" }}
{"@version":"1","@timestamp":"2018-10-11T17:20:25.369415+00:00","host":"cd27d80528c2","type":"syslog","syslog_timestamp":"Oct 11 17:20:25","syslog_hostname":"cd27d80528c2","syslog_program":"multilos","syslog_message":"************************************************************\\nRecordType:Submit\\nRecepientID:23282353id23\\nMediaTypes:image\/jpeg,image\/jpeg,image\/jpeg,application\/smil\\nMediaSizes:31214,31900,31214,364\\nContentType:multipart\/related","received_at":"2018-10-11T17:20:25.369415+00:00","received_from":"","logi:":{ "contentType": "ContentType:multipart\/related", "mediaSizes": "31214,31900,31214,364", "mediaTypes": "image\/jpeg,image\/jpeg,image\/jpeg,application\/smil", "recepientID": "23282353id23", "recordType": "Submit" }}

Pretty printed output for legibility:

{
    "@version": "1",
    "@timestamp": "2018-10-11T17:20:25.369215+00:00",
    "host": "cd27d80528c2",
    "type": "syslog",
    "syslog_timestamp": "Oct 11 17:20:25",
    "syslog_hostname": "cd27d80528c2",
    "syslog_program": "multilos",
    "syslog_message": "************************************************************\\nRecordType:Submit\\nRecepientID:2328288id23\\nMediaTypes:image\/jpeg,image\/jpeg,image\/jpeg,application\/smil\\nMediaSizes:31214,31900,31214,364\\nContentType:multipart\/related",
    "received_at": "2018-10-11T17:20:25.369215+00:00",
    "received_from": "",
    "logi:": {
        "contentType": "ContentType:multipart\/related",
        "mediaSizes": "31214,31900,31214,364",
        "mediaTypes": "image\/jpeg,image\/jpeg,image\/jpeg,application\/smil",
        "recepientID": "2328288id23",
        "recordType": "Submit"
    }
}
{
    "@version": "1",
    "@timestamp": "2018-10-11T17:20:25.369279+00:00",
    "host": "cd27d80528c2",
    "type": "syslog",
    "syslog_timestamp": "Oct 11 17:20:25",
    "syslog_hostname": "cd27d80528c2",
    "syslog_program": "multilos",
    "syslog_message": "************************************************************\\nRecordType:Submit\\nRecepientID:2328232id23\\nMediaTypes:image\/jpeg,image\/jpeg,image\/jpeg,application\/smil\\nMediaSizes:31214,31900,31214,364\\nContentType:multipart\/related",
    "received_at": "2018-10-11T17:20:25.369279+00:00",
    "received_from": "",
    "logi:": {
        "contentType": "ContentType:multipart\/related",
        "mediaSizes": "31214,31900,31214,364",
        "mediaTypes": "image\/jpeg,image\/jpeg,image\/jpeg,application\/smil",
        "recepientID": "2328232id23",
        "recordType": "Submit"
    }
}
{
    "@version": "1",
    "@timestamp": "2018-10-11T17:20:25.369415+00:00",
    "host": "cd27d80528c2",
    "type": "syslog",
    "syslog_timestamp": "Oct 11 17:20:25",
    "syslog_hostname": "cd27d80528c2",
    "syslog_program": "multilos",
    "syslog_message": "************************************************************\\nRecordType:Submit\\nRecepientID:23282353id23\\nMediaTypes:image\/jpeg,image\/jpeg,image\/jpeg,application\/smil\\nMediaSizes:31214,31900,31214,364\\nContentType:multipart\/related",
    "received_at": "2018-10-11T17:20:25.369415+00:00",
    "received_from": "",
    "logi:": {
        "contentType": "ContentType:multipart\/related",
        "mediaSizes": "31214,31900,31214,364",
        "mediaTypes": "image\/jpeg,image\/jpeg,image\/jpeg,application\/smil",
        "recepientID": "23282353id23",
        "recordType": "Submit"
    }
}

I hope this helps,
Christos