Filterlog config changes

Security-Onion-Solutions · Oct 9, 2020 · 69a04de · 69a04de
1 parent 930ec33
commit 69a04de
Show file tree

Hide file tree

Showing 4 changed files with 85 additions and 33 deletions.
diff --git a/salt/elasticsearch/files/ingest/common b/salt/elasticsearch/files/ingest/common
@@ -21,44 +21,33 @@
 		"properties": ["ip", "country_iso_code", "country_name", "continent_name", "region_iso_code", "region_name", "city_name", "timezone", "location"]
 	}
     },
-    { 
-	"split":  {
-		"field": "_index",
-		"target_field": "index_name_prefix",
-		"separator": "-"
-	}
-    },
-    { 
-	"date_index_name": {
-        	"field": "@timestamp",
-	        "index_name_prefix": "{{index_name_prefix.0}}-{{index_name_prefix.1}}-",
-        	"date_rounding": "d",
-	        "ignore_failure": true,
-        	"index_name_format": "yyyy.MM.dd"
-        }
-    },  
-    { "set":         { "if": "ctx.event?.severity == 1",   "field": "event.severity_label", "value": "low", "override": true }  },
-    { "set":         { "if": "ctx.event?.severity == 2",   "field": "event.severity_label", "value": "medium", "override": true }  },
-    { "set":         { "if": "ctx.event?.severity == 3",   "field": "event.severity_label", "value": "high", "override": true }  },
-    { "set":         { "if": "ctx.event?.severity == 4",   "field": "event.severity_label", "value": "critical", "override": true }  },
-    { "rename":         { "field": "fields.category",              "target_field": "event.category",                  "ignore_failure": true, "ignore_missing": true  } },
-    { "rename":         { "field": "fields.module",              "target_field": "event.module",                  "ignore_failure": true, "ignore_missing": true  } },
-    { "rename":         { "field": "module",              "target_field": "event.module",                  "ignore_failure": true, "ignore_missing": true  } },
-    { "rename":         { "field": "dataset",             "target_field": "event.dataset",                "ignore_failure": true, "ignore_missing": true  } },
-    { "rename":         { "field": "category",            "target_field": "event.category",              "ignore_failure": true, "ignore_missing": true  } },
-    { "rename":         { "field": "message2.community_id", "target_field": "network.community_id",  "ignore_failure": true,  "ignore_missing": true  } },
-    { "lowercase":         { "field": "event.dataset", "ignore_failure": true,  "ignore_missing": true  } },    
-    { "lowercase":         { "field": "network.transport", "ignore_failure": true,  "ignore_missing": true  } },
+    { "set":             { "if": "ctx.event?.severity == 1",   "field": "event.severity_label", "value": "low", "override": true }  },
+    { "set":             { "if": "ctx.event?.severity == 2",   "field": "event.severity_label", "value": "medium", "override": true }  },
+    { "set":             { "if": "ctx.event?.severity == 3",   "field": "event.severity_label", "value": "high", "override": true }  },
+    { "set":             { "if": "ctx.event?.severity == 4",   "field": "event.severity_label", "value": "critical", "override": true }  },
+    { "rename":          { "field": "fields.category",              "target_field": "event.category",                  "ignore_failure": true, "ignore_missing": true  } },
+    { "rename":          { "field": "fields.module",              "target_field": "event.module",                  "ignore_failure": true, "ignore_missing": true  } },
+    { "rename":          { "field": "module",              "target_field": "event.module",                  "ignore_failure": true, "ignore_missing": true  } },
+    { "rename":          { "field": "dataset",             "target_field": "event.dataset",                "ignore_failure": true, "ignore_missing": true  } },
+    { "rename":          { "field": "category",            "target_field": "event.category",              "ignore_failure": true, "ignore_missing": true  } },
+    { "rename":          { "field": "message2.community_id", "target_field": "network.community_id",  "ignore_failure": true,  "ignore_missing": true  } },
+    { "lowercase":       { "field": "event.dataset", "ignore_failure": true,  "ignore_missing": true  } },
     { "convert":         { "field": "destination.port", "type": "integer",  "ignore_failure": true,  "ignore_missing": true  } },
     { "convert":         { "field": "source.port", "type": "integer",  "ignore_failure": true,  "ignore_missing": true  } },
     { "convert":         { "field": "log.id.uid", "type": "string",  "ignore_failure": true,  "ignore_missing": true  } },
     { "convert":         { "field": "agent.id", "type": "string",  "ignore_failure": true,  "ignore_missing": true  } },
     { "convert":         { "field": "event.severity", "type": "integer",  "ignore_failure": true,  "ignore_missing": true  } },
-    { 
-        "remove": {
-                "field": [ "index_name_prefix", "message2", "type", "fields" ],
-                "ignore_failure": true
+    { "remove": 	 { "field": [ "message2", "type", "fields" ], "ignore_failure": true } },
+    {
+        "date_index_name": {
+                "field": "@timestamp",
+                "index_name_prefix": "{{ _index }}-",
+                "date_rounding": "d",
+                "ignore_failure": true,
+                "index_name_format": "yyyy.MM.dd"
         }
     }
   ]
+}
+  ]
 }
diff --git a/salt/elasticsearch/files/ingest/filterlog b/salt/elasticsearch/files/ingest/filterlog
@@ -0,0 +1,60 @@
+{
+  "description" : "filterlog",
+  "processors" : [
+      {
+        "dissect": {
+                "field": "real_message",
+
+                "pattern" : "%{rule.uuid},%{rule.sub_uuid},%{firewall.anchor},%{firewall.tracker_id},%{interface.name},%{rule.reason},%{rule.action},%{network.direction},%{ip.version},%{firewall.sub_message}",
+                "on_failure" : [ {"set" : {"field" : "error.message","value" : "{{ _ingest.on_failure_message }}"}}]
+        }
+     },
+     {
+        "dissect": {
+                "if": "ctx.ip.version == '4'",
+                "field": "firewall.sub_message",
+                "pattern" : "%{ip.tos},%{ip.ecn},%{ip.ttl},%{ip.id},%{ip.offset},%{ip.flags},%{network.transport_id},%{network.transport},%{data.length},%{source.ip},%{destination.ip},%{ip_sub_msg}",
+                "on_failure" : [ {"set" : {"field" : "error.message","value" : "{{ _ingest.on_failure_message }}"}}]
+       }
+     },
+     {
+        "dissect": {
+                "if": "ctx.ip?.version == '6'",
+                "field": "firewall.sub_message",
+                "pattern" : "%{network.class},%{network.flow_label},%{network.hop_limit},%{network.transport},%{network.transport_id},%{data.length},%{source.ip},%{destination.ip},%{ip_sub_msg}",
+                "on_failure" : [ {"set" : {"field" : "error.message","value" : "{{ _ingest.on_failure_message }}"}}]
+        }
+     },
+     {
+        "dissect": {
+                "if": "ctx.network?.transport == 'tcp'",
+                "field": "ip_sub_msg",
+                "pattern" : "%{source.port},%{destination.port},%{data.length},%{tcp.flags},",
+                "on_failure" : [ {"set" : {"field" : "error.message","value" : "{{ _ingest.on_failure_message }}"}}]
+        }
+     },
+     {
+        "dissect": {
+                "if": "ctx.protocol == 'udp'",
+                "field": "ip_sub_msg",
+                "pattern" : "%{source.port},%{destination.port},%{data.length}",
+                "on_failure" : [ {"set" : {"field" : "error.message","value" : "{{ _ingest.on_failure_message }}"}}]
+        }
+     },
+     {
+        "split": {
+                "if": "ctx.ip.version =='6' && ctx.network?.transport == 'Options'",
+                "field": "ip_sub_msg",
+                "target_field": "ip.options",
+                "separator" : ",",
+                "on_failure" : [ {"set" : {"field" : "error.message","value" : "{{ _ingest.on_failure_message }}"}}]
+        }
+     },
+     { "set":         { "field": "_index",        "value": "so-firewall", "override": true           } },
+     { "set":         { "if": "ctx.network?.transport_id == '0'", "field": "network.transport",        "value": "icmp", "override": true           } },
+     { "set":         { "field": "event.module",        "value": "pfsense", "override": true           } },
+     { "set":         { "field": "event.dataset",        "value": "firewall", "override": true           } },
+     { "remove":      { "field": ["real_message", "ip_sub_msg", "firewall.sub_message"],     "ignore_failure": true                                                                  } },
+     { "append":      { "field": "tags", "value": ["pfsense"]                                   } }
+  ]
+}
diff --git a/salt/elasticsearch/files/ingest/syslog b/salt/elasticsearch/files/ingest/syslog
@@ -12,6 +12,9 @@
                 "ignore_failure": true
         }
     },
+    { "grok":           { "field": "message",       "patterns": ["<%{INT:syslog.priority}>%{DATA:syslog.timestamp} %{WORD:source.application}: %{GREEDYDATA:real_message}"], "ignore_failure": false } },
+    { "set": { "if": "ctx.source.application == 'filterlog'", "field": "dataset", "value": "firewall" } },
+    { "pipeline": { "if": "ctx.dataset == 'firewall'", "name": "filterlog" } },
     { "pipeline":       { "name": "common"                                             } }
   ]
 }
diff --git a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja
@@ -5,7 +5,7 @@
 {%- endif %}
 {% set FEATURES = salt['pillar.get']('elastic:features', False) %}
 output {
-  if "firewall" in [tags] {
+  if [dataset] =~ "firewall" {
     elasticsearch {
       hosts => "{{ ES }}"
       index => "so-firewall-%{+YYYY.MM.dd}"