filter-syslog.conf

# vim:set sw=2 ts=2 sts=2 et
# Parse syslog input
filter {
  if [type] == "syslog" {
    # General syslog message cleanup
    mutate {
      replace => [ "host", "%{logsource}" ]
      add_field => { "level" => "%{severity_label}" }
      # "\n" newline notation in substitution results in "\\n" in output.
      # Using a string with a literal newline works as desired.
      gsub => [ "message", "#012", '
' ]
      add_tag => [ "syslog", "es" ]
    }

    # Strip "message repeated" preamble
    if [message] =~ /^message repeated \d+ times:/ {
      grok {
        match => [
          "message",
          "^message repeated %{NUMBER:repeated} times: \[\s*%{GREEDYDATA:message}\]$"
        ]
        overwrite => [ "message" ]
        named_captures_only => true
      }
    }

    # Mark kernel messages forwarded because of hhvm as hhvm messages
    if [program] == "kernel" and [message] =~ /hhvm/ {
      mutate {
        replace => [ "program",  "hhvm" ]
      }
    }

    if [program] == "hhvm" {
      mutate {
        replace => [ "type",  "hhvm" ]
        # Strip leading newline from hhvm messages
        strip => [ "message" ]
      }

      if [message] =~ /^(?m){.*}$/ {
        # Parse json encoded fatal errors
        json {
          source => "message"
          add_tag => [ "hhvm-json" ]
        }
        mutate {
          replace => [
            "message", "%{message} in %{file} on line %{line}",
            "level", "Fatal"
            ]
        }
      } else {
        grok {
          match => [
            "message",
            "^%{LOGLEVEL:level}"
          ]
          overwrite => [ "level" ]
          named_captures_only => true
        }
      }
    } # end [program] == "hhvm"

    if [program] == "hhvm-fatal" {
      # Join sequential lines into a single event
      multiline {
        pattern => "^Host: "
        negate => true
        what => "previous"
      }

      mutate {
        replace => [
          "type",  "hhvm",
          "level", "Fatal"
        ]
      }
    } # end [program] == "hhvm-fatal"

    if [program] == "apache2" {
      mutate {
        replace => [ "type",  "apache2" ]
      }

      # Parse typical apache error format:
      # [channel:level] [pid N:tid N] MSG? [client HOST:PORT] MSG, referer: URL
      grok {
        match => [
          "message",
          "^\[(%{WORD:channel}?:)?%{LOGLEVEL:level}\]\s+(\[pid %{POSINT}(:tid %{POSINT:thread})?\]\s+)?(?<message_prefix>[^\[]+)?(\[client %{IP:clientip}(:%{POSINT:clientport})?\]\s+)?%{DATA:message}(,\s+referer:\s+%{NOTSPACE:referrer})?$"
        ]
        overwrite => [ "message", "level" ]
        named_captures_only => true
      }

      if [message_prefix] {
        mutate {
          "replace" => [ "message", "%{message_prefix}%{message}" ]
          "remove_field" => [ "message_prefix" ]
        }
      }

    } # end [program] == "apache2"

    if [program] == "mediawiki" {
      mutate {
        replace => [ "type",  "mediawiki" ]
      }
      if [message] =~ /^{.*}$/ {
        mutate {
          # Remove syslog added fields
          remove_field => [
              "facility",
              "facility_label",
              "logsource",
              "priority",
              "program",
              "severity",
              "severity_label",
              "timestamp"
          ]
        }
        # Parse message as json to unpack logstash record
        json {
          source => "message"
        }
      } else {
        # Mark up the message as JSON that was cut off by the syslog transport
        mutate {
          add_field => { "channel" => "jsonTruncated" }
          add_tag => [ "syslog_truncated" ]
        }
      }
    } # end [program] == "mediawiki"
  }
}