add handling for Azure linux syslog files in CSV form

configfiles/1801-preprocess-azure.conf

@@ -0,0 +1,67 @@
+# SOF-ELK® Configuration File
+# (C)2021 Lewes Technology Consulting, LLC
+#
+# This file parses CSV-formatted Azure linux syslog logs
+# At this time, the only known CSV logs from Azure that will be handled are of this type
+# This preprocessor normalizes the content to match other syslog pipelines
+# The resulting event will be handled as a normal syslog event
+
+filter {
+  if [type] == "azure" {
+
+    if "csv" in [tags] {
+      ### Azure Linux Logs, in CSV format
+      csv {
+        separator => ","
+        skip_empty_rows => "true"
+        columns => [ "partition_key", "row_key", "datetime", "deployment_id", "deployment_id_type","event_time", "event_time_type", "facility", "facility_type","fluentd_ingest_timestamp", "fluentd_ingest_timestamp_type", "host", "host_type", "ignore", "ignore_type", "msg", "msg_type", "n", "n_type", "precise_timestamp", "precise_timestamp_type", "row_index", "row_index_type", "sending_host", "sending_host_type", "severity", "severity_type", "timestamp", "timestamp_type", "hostname", "hostname_type", "ident", "ident_type", "pid", "pid_type" ]
+        remove_field => "message"
+        target => "raw"
+        add_tag => [ "azure_linux_syslog" ]
+      }
+
+      mutate {
+        rename => {
+          "[raw][event_time]" => "event_time"
+          "[raw][facility]" => "facility"
+          "[raw][severity]" => "severity"
+          "[raw][host]" => "syslog_hostname"
+          "[raw][ident]" => "syslog_program"
+          "[raw][pid]" => "syslog_pid"
+          "[raw][msg]" => "message"
+        }
+      }
+
+      if [event_time] == "EventTime" {
+        drop {}  # drop the first line that contains the column names.
+      }
+
+      date {
+        match => [ "event_time", "ISO8601" ]
+      }
+
+      # convert facility and severity back to integers
+      translate {
+        field => "facility"
+        destination => "facility_int"
+        dictionary_path => "/usr/local/sof-elk/lib/dictionaries/syslog_facility2int.yaml"
+      }
+      translate {
+        field => "severity"
+        destination => "severity_int"
+        dictionary_path => "/usr/local/sof-elk/lib/dictionaries/syslog_severity2int.yaml"
+      }
+
+      mutate {
+        remove_field => [ "raw", "event_time", "facility", "severity" ]
+        replace => { "type" => "syslog" }
+      }
+      mutate {
+        rename => {
+          "facility_int" => "facility"
+          "severity_int" => "severity"
+        }
+      }
+    }
+  }
+}

lib/dictionaries/syslog_facility2int.yaml

@@ -0,0 +1,24 @@
+"kernel": 0
+"user": 1
+"mail": 2
+"daemon": 3
+"auth": 4
+"syslog": 5
+"lpr": 6
+"news": 7
+"uucp": 8
+"cron": 9
+"authpriv": 10
+"ftp": 11
+"ntp": 12
+"security": 13
+"console": 14
+"solaris-cron": 15
+"local0": 16
+"local1": 17
+"local2": 18
+"local3": 19
+"local4": 20
+"local5": 21
+"local6": 22
+"local7": 23

lib/dictionaries/syslog_severity2int.yaml

@@ -0,0 +1,8 @@
+"emerg": 0
+"alert": 1
+"crit": 2
+"err": 3
+"warning": 4
+"notice": 5
+"info": 6
+"debug": 7