improved cisco ACI processor

config/cisco_aci.json

@@ -0,0 +1,3 @@
+{ 
+"F0532":{ "fault_name":"fltEthpmIfPortDownInfraEpg", "msg_explanation":"This fault occurs when a port is down and is in use for epg", "msg_recommendation":"Check the port connectivity or VPC configuration" }
+}

config/processors/syslog_audit_cisco.aci.conf

@@ -26,49 +26,35 @@ filter {
 
   # 2. We now apply a grok pattern to extract known fields
   grok {
-    match => { "actual_msg" => "%{SYSLOGTIMESTAMP:[[tmp][dateoriginal]]} %{GREEDYDATA:[[tmp][device]]} \%LOG_LOCAL%{INT:[[tmp][loglocal]]}-%{INT:[[tmp][severity]]}-%{WORD:[[tmp][module]]} %{NOTSPACE:[[tmp][msg_details]]} %{GREEDYDATA:[[tmp][error_message]]}" }
+    match => { "actual_msg" => "%{SYSLOGTIMESTAMP:[[tmp][dateoriginal]]} %{GREEDYDATA:[[tmp][device]]} \%LOG_LOCAL%{INT:[[tmp][loglocal]]}-%{INT:[[tmp][severity]]}-SYSTEM_MSG \[%{WORD:[[tmp][code]]}](?:\[%{WORD:[[tmp][lifecycle]]}])?\[%{DATA:[[tmp][rule]]}]\[%{WORD:[[tmp][det_severity]]}\]\[%{DATA:[[tmp][effected_dn]]}] %{GREEDYDATA:[[tmp][error_message]]}" }
     timeout_millis => 500
   }
-
-  # 3. Inside msg_details: faults are stateful and we'll find 4 fields that contain categories of this message.
-  # events are stateless and the second field will be missing. because of that we can't dissect it, so we'll add a separator
-  # we'll also clean the observer name
-  mutate {
-    gsub =>
-      [ "[tmp][msg_details]", "\]", ",",
-        "[tmp][msg_details]", "\[", "",
-        "[tmp][host]", "\{name\=", "",
-        "[tmp][host]", "\}", "",
-        "[tmp][host]", "\..*", ""
-      ]
-  }
-
-  mutate {
-    split => {"[tmp][msg_details]" => ","}
-  }
-
+
   # we now know if this is fault or event by the number of fields that msg_details contains. we can look into the last field
-
-  if [tmp][msg_details][3]
-  {
+  if [tmp][lifecycle] {
     mutate {
-      add_field => {"[tmp][msg_type]" => "fault"}
-      add_field => {"[tmp][code]" => "%{[[tmp][msg_details][0]]}"}
-      add_field => {"[tmp][lifecycle]" => "%{[[tmp][msg_details][1]]}"}
-      add_field => {"[tmp][det_severity]" => "%{[[tmp][msg_details][2]]}"}
-      add_field => {"[tmp][rule]" => "%{[[tmp][msg_details][3]]}"}
+      add_field => {
+        "[tmp][msg_type]" => "alert"
+      }
     }
   }
-  else
-  {
+  else {
     mutate {
-      add_field => {"[tmp][msg_type]" => "event"}
-      add_field => {"[tmp][code]" => "%{[[tmp][msg_details][0]]}"}
-      add_field => {"[tmp][det_severity]" => "%{[[tmp][msg_details][1]]}"}
-      add_field => {"[tmp][rule]" => "%{[[tmp][msg_details][2]]}"}
+      add_field => {
+        "[tmp][msg_type]" => "event"
+      }
     }
   }
-
+
+  # 3. clean the observer name
+  mutate {
+    gsub =>
+      [ "[tmp][host]", "\{name\=", "",
+        "[tmp][host]", "\}", "",
+        "[tmp][host]", "\..*", ""
+      ]
+  }
+
   # 4. pure ACI messages differ from ACI messages from nexus subsystems. we know nexus messages if rule = [sys]. in that case we need additional parsing
   if "[sys]" in [tmp][rule]
   {
@@ -87,36 +73,26 @@ filter {
   # a) Parse fields that belong to ECS into ECS fields
   mutate {
     rename => {"[tmp][device]" => "[host][hostname]"}
-    rename => {"pri" => "[log][syslog][priority]"}
     rename => {"[tmp][code]" => "[error][code]"}
     rename => {"[tmp][msg_type]" => "[event][kind]" }
     rename => {"[tmp][det_severity]" => "[log][level]"}
     rename => {"[tmp][severity]" => "[log][syslog][severity][code]"}
     rename => {"[tmp][error_message]" => "[error][message]"}
     rename => {"[tmp][host]" => "[observer][hostname]"}
     rename => {"[tmp][module]" => "[event][type]"}
-    rename => {"[tmp][rule]" => "[user][tmp]"}
     rename => {"[tmp][lifecycle]" => "[event][action]"}
     rename => {"[tmp][loglocal]" => "[log][syslog][facility][code]"}
     rename => {"[tmp][nexus_loglocal]" => "[log][syslog][facility][name]"}
   }
 
-  if [user][tmp] =~ "-" {
-    mutate {
-      split => { "[user][tmp]" => "-" }
-    }
-    if [user][tmp][1] {
-      mutate {
-        rename => {"[user][tmp][1]" => "[user][name]"}
-      }
-    }
-  }
-
   # b) Parse fields that don't belong to ECS into Labels (as per https://www.elastic.co/guide/en/ecs/current/ecs-custom-fields-in-ecs.html#_the_labels_field)
-  # c) Drop unused/unwanted fields
+
   mutate {
-    remove_field => [ "[tmp]", "[user][tmp]", "actual_msg" ]
+    rename => {
+      "[tmp][effected_dn]" => "[labels][effected_dn]"
+    }
   }
+
 
   # 6. Proceed to hardcoded evaluations
   # [event][category]
@@ -141,7 +117,40 @@ filter {
     }
   }
 
-  # 7. Convert fields (i.e. extract site, appliance type, etc)
+ # 7. Translate full msg, explanation, recommendation
+  translate {
+    id => "cisco-aci-errors"
+    source => "[error][code]"
+    target => "[tmp][error_translation]"
+    dictionary_path => "${LOGSTASH_HOME}/config/cisco_aci.json" # ** Must set full "/path/to/lookup.json" to your lookup file **
+    refresh_interval => 3000
+    fallback => '{"key1":"not_found"}' 
+  }
+
+  # a) because a fallback value can only contain a string, additional processing is done to ensure that failed lookups store values in proper fields
+  if [tmp][error_translation] == '{"key1":"not_found"}' {
+    json { 
+      source => "[tmp][error_translation]" 
+      target => "[tmp][error_translation]"
+    }
+    mutate {
+      remove_field => ["[tmp][error_translation]"]
+    }
+    mutate {
+      add_field => {"[tmp][error_translation][fault_name]" => "not_found_fault_name"}
+      add_field => {"[tmp][error_translation][msg_recommendation]" => "not_found_msg_recommendation"}
+      rename => {"[tmp][rule]" => "[tmp][error_translation][msg_explanation]"}
+    }
+  }
+
+  # b) add proper fields from [tmp] translated
+  mutate {
+    add_field => { "[error][stack_trace]" => "%{[[tmp][error_translation][fault_name]]}"}
+    add_field => { "[event][reason]" => "%{[[tmp][error_translation][msg_explanation]]}"}
+    add_field => { "[event][recommendation]" => "%{[[tmp][error_translation][msg_recommendation]]}"}
+  }
+
+  # 8. Convert fields (i.e. extract site, appliance type, etc)
   if "" in [network][name] or ![network][name]  {
     mutate {
       add_field => {"[network][name]" => "%{[[host][hostname]]}" }
@@ -156,7 +165,13 @@ filter {
       "[network][name]", "([a-z]*)([0-9].*)", "\1"
     ]
   }
+
+  # 9) Drop unused/unwanted fields
+  mutate {
+    remove_field => [ "[tmp]", "pri", "actual_msg" ]
+  }
 }
+
 output {
   pipeline { send_to => [enrichments] }
 }