solace_syslog-ng.conf

# AAaron's Solace Configuration for syslog-ng
# Copyright 2016-2021
# Last tested againt syslog-ng v3.5.6
#
# Either rename this file to 'syslog-ng.conf' and replace in /etc/syslog-ng/, or copy into /etc/syslog-ng/conf.d/

# Useful: https://docs.solace.com/System-and-Software-Maintenance/Monitoring-Events-Using-Syslog.htm
# Useful for debugging: syslog-ng --syntax-only --debug


@version:3.5

options {
    flush-lines (0);        # send Syslog messages to log immediately
    flush-timeout (2000);   # 2 seconds... probably ignored though due to above
    time-reopen (10);
    log-fifo-size (20000);
    chain-hostnames (off);
    use-dns (no);
    use-fqdn (no);
    create-dirs (yes);      # force it to create directory structure
    keep-hostname (yes);    # use the hostname from the Syslog message
    perm (0644);            # world readable logs
    dir-perm (0755);        # world readable/searchable directories
    frac-digits (3);        # millisecond timestamps
};

# Where am I gonna send my logs?  Amend to just "solace" if you want
@define logpath "/var/log/solace.syslog-ng"
@define routerlogpath "/var/log/solace.syslog-ng/${HOST}"
@define vpnlogpath "/var/log/solace.syslog-ng/${HOST}/${SOL_VPN}"



### Sources

# By separating Solace devices by dev, test, and prod, I could implement different rules & filtering below
# (e.g. logging to a different location, or escalating some events to higher severities).
# Simply point the Solace external syslog host to the appropriate port, based on the classification of that router
source s_tcp_dev       { tcp(ip(0.0.0.0) port(51410)) };
source s_tcp_test      { tcp(ip(0.0.0.0) port(51411)) };
source s_tcp_prd       { tcp(ip(0.0.0.0) port(51412)) };

# This will be used to scan the output of sec and inject those logs back into the alertable log
# not used right now
#source s_sec_correlate { file("`logpath`/sec.out.log" follow_freq(2)); };


### Parsers

# For SYSTEM events, parse the message into the component parts 
parser p_solace_event {
    csv-parser(columns("SOL_EVENTTYPE", "SOL_EVENT", "SOL_BODY")
    delimiters(" ")
    flags(greedy, escape-none)
    template("${MSG}"));
};
# For VPN and CLIENT events, include the additional VPN name
parser p_solace_vpn_event {
    csv-parser(columns("SOL_EVENTTYPE", "SOL_EVENT", "SOL_VPN", "SOL_BODY")
    delimiters(" ")
    flags(greedy, escape-none)
    template("${MSG}"));
};



### Templates

# For command logs, don't need to include the severity level, everything is the same
template t_command_format   { template("${ISODATE} ${HOST} ${PROGRAM}: ${MSG}\n");          template_escape(no); };
# For event logs, include the severity level.  Make the log output format look similar to the router's
template t_event_format  { template("${ISODATE} <${LEVEL}> ${HOST} ${PROGRAM}: ${MSG}\n"); template_escape(no); };



### Destinations

# Where do the command logs go? Include router name as part of directory
# This will just be configuration?  I.e. don't log 'show' commands if Solace has all command logging enabled
destination d_command {       file("`routerlogpath`/command.log"  template(t_command_format) ); };
# Just the 'show' commands... useful when tracking/debugging monitoring apps.
# No need to roll these longtime, or just keep 1 or 2, 
destination d_show_command {  file("`routerlogpath`/show.log"     template(t_command_format) ); };

# This log file will contain events that I want watched by my monitoring framework/application
destination d_alerts {        file("`logpath`/alerts.log"         template(t_event_format) ); };
# this one isn't being used now
#destination d_correlate_in {  file("`logpath`/correlate.in.log"   template(t_event_format) ); };

# This is the "raw" event log
destination d_event {         file("`routerlogpath`/event.log"    template(t_event_format) ); };
# Filter out all the 'authorization' logs from the event log... most of the time it's just monitoring apps
destination d_auth_event {    file("`routerlogpath`/auth.log"     template(t_event_format) ); };

# The 'system' log file, which is a subset of event.log.  Not that useful IMHO
destination d_system {        file("`routerlogpath`/system.log"   template(t_event_format) ); };
# You could even have events separated by VPN name if you wish? Makes it easy for logs that churn a lot
destination d_vpn_event {     file("`vpnlogpath`/event.log"       template(t_event_format) ); };



### Standard filters - Solace uses facilities local3 for events and local1 for commands

filter f_command {       facility(local1); };
filter f_event {         facility(local3); };
filter f_system {        facility(local4); };  # system.log - this is a subset of event.log, so ignore
filter f_show_command {  facility(local1) and (
                         (message('CLI/') and (message('> show ') or message('# show '))) or
                         ((message('SEMP/') or message('SEMP2/')) and (message('> show ') or message('  show '))));
};



### Custom filters - i.e. rules to capture the messages for custom processing

filter f_warn_and_higher {    facility(local3) and level(warning..emerg); };
filter f_notice_and_higher {  facility(local3) and level(notice..emerg); };

filter f_up_or_clear {        facility(local3) and level(info..notice) and (
                                  match("_UP:" value("SOL_EVENT")) or
                                  match("_CLEAR:" value("SOL_EVENT"))); };

#filter f_system_routing_cspf_nbr { match('SYSTEM_ROUTING_CSPF_NBR_' value("SOL_EVENT")); };
# Want to correlate (suppress) multiple lost logging events
#filter f_system_logging_lost {     match('SYSTEM_LOGGING_LOST_EVENTS:' value("SOL_EVENT")); };
#filter f_mw_vpn_solcache_all { match('VPN_SOLCACHE_' value("SOL_EVENT")) and not match('DELETE_MESSAGE:' value("SOL_EVENT")); };


filter f_system_eventable {        match('SYSTEM:' value("SOL_EVENTTYPE")) and (
                                       filter(f_notice_and_higher) or
                                       filter(f_up_or_clear)); };
filter f_auth_event {              match('SYSTEM_AUTHENTICATION_SESSION_OPENED' value("SOL_EVENT")) or
                                   match('SYSTEM_AUTHENTICATION_SESSION_CLOSED' value("SOL_EVENT")) or
                                   match('SYSTEM_AUTHENTICATION_SESSION_DENIED' value("SOL_EVENT")) or
                                   match('SYSTEM_SSL_CONNECTION_REJECTED' value("SOL_EVENT")) or
                                   match('SYSTEM_CLIENT_CONNECT_AUTH_FAIL' value("SOL_EVENT")); };
filter f_vpn_or_client_event {     match('VPN:' value("SOL_EVENTTYPE")) or match('CLIENT:' value("SOL_EVENTTYPE")); };
filter f_vpn_or_client_eventable { filter(f_vpn_or_client_event) and (
                                       filter(f_warn_and_higher) or
                                       filter(f_up_or_clear)); };

## Don't want these events getting put into the 'alertable' log (because they're WARN level) - essentially, they will be suppressed
# Perhaps that's not what you want though? Also: the SOLCACHE logs are meant to
# get sent to the correlation log for further processing, so maybe allow them?
filter f_vpn_vpn_state_change {  match('VPN_VPN_STATE_CHANGE:' value("SOL_EVENT")); };
filter f_client_bind_failed {    match('CLIENT_CLIENT_BIND_FAILED:' value("SOL_EVENT")); };
#filter f_vpn_solcache        {   match('VPN_SOLCACHE_' value("SOL_EVENT")); };

## This is now where I start to define the things that must be correlated
# My original config tried to ensure a success within a time window using correlation
filter f_auth_denied {           match('SYSTEM_AUTHENTICATION_SESSION_DENIED' value("SOL_EVENT")); };



### Log statements
### application of filter and rewrite rules and choice of destination to write too  - Note: Order is VERY important
### Defaults - These must be actioned first - note there is NO flags(final) entry as we want to execute all of these

## The following are just meant for the "raw archive", and not to be monitored directly

# if it's a show command, put it in a separate log
log { source(s_tcp_prd); source(s_tcp_test); source(s_tcp_dev);
      filter(f_show_command);
      destination(d_show_command);
      flags(flow-control,final);
};
# otherwise, it's a valid command, so save it
log { source(s_tcp_prd); source(s_tcp_test); source(s_tcp_dev);
      filter(f_command);
      destination(d_command);
      flags(flow-control,final);
};
# the boring system log, 
log { source(s_tcp_prd); source(s_tcp_test); source(s_tcp_dev);
      filter(f_system);
      destination(d_system);
      flags(flow-control);
};
# Anything being written out by the SEC process into the 'outbound' file needs to be alerted on
# log { source(s_sec_correlate);
#       destination(d_alerts);
#       };


# this prevents the SYSTEM_AUTHENTICATION logs from monitoring apps junking up the event.log by writing to a different file and then 'final' flag
log { source(s_tcp_prd); source(s_tcp_test); source(s_tcp_dev);
      parser(p_solace_event);
      filter(f_auth_event);
      destination(d_auth_event);
      #flags(flow-control,final);
      # don't stop maybe alert later
      flags(flow-control);
};

## And this just writes all event logs (the 'raw') to the standard event log
log { source(s_tcp_prd); source(s_tcp_test); source(s_tcp_dev);
      filter(f_event);
      destination(d_event);
      flags(flow-control);
};
# OPTIONAL!
# This is a special directory-per-VPN... useful in large environments with many VPNs per appliance, but could get noisy... turn this off if lots of VPNs and logging is struggling
#log { source(s_tcp_prd); source(s_tcp_test); source(s_tcp_dev);
#      parser(p_solace_vpn_event);
#      filter(f_vpn_or_client_event);
#      destination(d_vpn_event);
#      flags(flow-control);
#};


##################################################################################
## All stuff that's alertable now

## This matches all the "cannot bind to queue" WARN events and doesn't log them to the 'alertable' log
log { source(s_tcp_prd); source(s_tcp_test); source(s_tcp_dev);
      parser(p_solace_event);
      filter(f_client_bind_failed);
      flags(flow-control,final);
};
# don't log VPN state changes
log { source(s_tcp_prd); source(s_tcp_test); source(s_tcp_dev);
      parser(p_solace_event);
      filter(f_vpn_vpn_state_change);
      flags(flow-control,final);
};

##################################################################################3
# here is where we will add special rules for logs that we want to correlate, and put into correlate log
log { source(s_tcp_prd); source(s_tcp_test); source(s_tcp_dev);
      parser(p_solace_event);
      filter(f_auth_denied);
      destination(d_correlate_in);
      flags(flow-control,final);
};


## This will filter all SYSTEM messages, then output to the alertable log file
log { source(s_tcp_prd); source(s_tcp_test); source(s_tcp_dev);
      parser(p_solace_event);
      filter(f_system_eventable);
      destination(d_alerts);
      flags(flow-control,final);
};


######################################################################################

## All VPN and client related warns and etc. now handled here, dumped into alertable file


## All other warning,err,alert,emerg messages not captured by the custom log statements i.e. to ensure we capture the stuff we already capture
#  Note that this includes all the warnings and such specifically for the app teams
log { source(s_tcp_prd); source(s_tcp_test); source(s_tcp_dev);
      parser(p_solace_event);
      filter(f_vpn_or_client_eventable);
      destination(d_alerts);
      flags(flow-control,final);
};