work in progress for idaholab#393, fuzzy matching for manufacturers b…

…ased on OUI to NetBox list is not very good (broken)

logstash/pipelines/enrichment/21_netbox.conf

@@ -13,15 +13,15 @@ filter {
 
   ruby {
     id => "ruby_determine_netbox_suitability"
-    # $logtypes = {"suricata"=>["alert"], "zeek"=>["conn", "known_hosts", "known_services", "notice", "signatures", "software", "weird"]}
-    init => "logtypesStr = ENV['LOGSTASH_NETBOX_ENRICHMENT_DATASETS'] || 'suricata.alert,zeek.conn,zeek.known_hosts,zeek.known_services,zeek.notice,zeek.signatures,zeek.software,zeek.weird' ; logtypesArr = logtypesStr.gsub(/\s+/, '').split(','); $logtypes = logtypesArr.group_by { |logtype| logtype.split('.').first }.transform_values { |values| values.map { |v| v.split('.')[1] } }"
+    # @logtypes = {"suricata"=>["alert"], "zeek"=>["conn", "known_hosts", "known_services", "notice", "signatures", "software", "weird"]}
+    init => "logtypesStr = ENV['LOGSTASH_NETBOX_ENRICHMENT_DATASETS'] || 'suricata.alert,zeek.conn,zeek.known_hosts,zeek.known_services,zeek.notice,zeek.signatures,zeek.software,zeek.weird' ; logtypesArr = logtypesStr.gsub(/\s+/, '').split(','); @logtypes = logtypesArr.group_by { |logtype| logtype.split('.').first }.transform_values { |values| values.map { |v| v.split('.')[1] } }"
     code => "
       provider = event.get('[event][provider]').to_s
       dataset = event.get('[event][dataset]').to_s
-      if ($logtypes.is_a?(Hash) &&
-          !$logtypes.empty? &&
-          ($logtypes.has_key?('all') ||
-           (!provider.empty? && !dataset.empty? && $logtypes.has_key?(provider) && $logtypes[provider].is_a?(Array) && $logtypes[provider].include?(dataset))))
+      if (@logtypes.is_a?(Hash) &&
+          !@logtypes.empty? &&
+          (@logtypes.has_key?('all') ||
+           (!provider.empty? && !dataset.empty? && @logtypes.has_key?(provider) && @logtypes[provider].is_a?(Array) && @logtypes[provider].include?(dataset))))
       then
         event.set('[@metadata][do_netbox_enrichment]', true)
       end

logstash/pipelines/enrichment/23_severity.conf

@@ -38,15 +38,15 @@ filter {
     if ([source][geo][country_iso_code]) or ([destination][geo][country_iso_code]) or ([dns][GEO]) {
       ruby {
         id => "ruby_add_field_severity_geo"
-        init => "countriesStr = ENV['SENSITIVE_COUNTRY_CODES'] || 'AM,AZ,BY,CN,CU,DZ,GE,HK,IL,IN,IQ,IR,KG,KP,KZ,LY,MD,MO,PK,RU,SD,SS,SY,TJ,TM,TW,UA,UZ' ; $countries = countriesStr.gsub(/\s+/, '').upcase.split(',')"
+        init => "countriesStr = ENV['SENSITIVE_COUNTRY_CODES'] || 'AM,AZ,BY,CN,CU,DZ,GE,HK,IL,IN,IQ,IR,KG,KP,KZ,LY,MD,MO,PK,RU,SD,SS,SY,TJ,TM,TW,UA,UZ' ; @countries = countriesStr.gsub(/\s+/, '').upcase.split(',')"
         code => "
           srcGEOs = event.get('[source][geo][country_iso_code]')
           dstGEOs = event.get('[destination][geo][country_iso_code]')
           dnsGEOs = event.get('[dns][GEO]')
           allGEOs = [srcGEOs.nil? ? [] : (srcGEOs.kind_of?(Array) ? srcGEOs : [srcGEOs]),
                      dstGEOs.nil? ? [] : (dstGEOs.kind_of?(Array) ? dstGEOs : [dstGEOs]),
                      dnsGEOs.nil? ? [] : (dnsGEOs.kind_of?(Array) ? dnsGEOs : [dnsGEOs])].flatten
-          if (!((allGEOs & $countries).empty?)) then
+          if (!((allGEOs & @countries).empty?)) then
             sevtags = Array.new unless (sevtags = event.get('[event][severity_tags]'))
             if !sevtags.kind_of?(Array) then
               newtags = Array.new
@@ -193,13 +193,13 @@ filter {
     if ([event][freq_score_v1]) or ([event][freq_score_v2]) {
       ruby {
         id => "ruby_add_field_severity_domain_entropy"
-        init => "$freqSeverityThreshold = ENV['FREQ_SEVERITY_THRESHOLD'] || '3.0'"
+        init => "@freqSeverityThreshold = ENV['FREQ_SEVERITY_THRESHOLD'] || '3.0'"
         code => "
           freqs1 = event.get('[event][freq_score_v1]')
           freqs2 = event.get('[event][freq_score_v2]')
           lowestFreqScore = [freqs1.nil? ? 100 : (freqs1.kind_of?(Array) ? freqs1.min : freqs1),
                              freqs2.nil? ? 100 : (freqs2.kind_of?(Array) ? freqs2.min : freqs2)].min
-          if (lowestFreqScore < Float($freqSeverityThreshold)) then
+          if (lowestFreqScore < Float(@freqSeverityThreshold)) then
             sevtags = Array.new unless (sevtags = event.get('[event][severity_tags]'))
             if !sevtags.kind_of?(Array) then
               newtags = Array.new
@@ -216,13 +216,13 @@ filter {
     if ([totDataBytes]) or ([network][bytes]) {
       ruby {
         id => "ruby_add_field_severity_total_bytes"
-        init => "mbSeverityThreshold = ENV['TOTAL_MEGABYTES_SEVERITY_THRESHOLD'] || '1000' ; $bytesSeverityThreshold = Integer(mbSeverityThreshold) * 1000000"
+        init => "mbSeverityThreshold = ENV['TOTAL_MEGABYTES_SEVERITY_THRESHOLD'] || '1000' ; @bytesSeverityThreshold = Integer(mbSeverityThreshold) * 1000000"
         code => "
           totDataBytes = event.get('[totDataBytes]')
           totBytes = event.get('[network][bytes]')
           highBytes = [totDataBytes.nil? ? 0 : Integer(totDataBytes),
                        totBytes.nil? ? 0 : Integer(totBytes)].max
-          if ($bytesSeverityThreshold > 0) and (highBytes >= $bytesSeverityThreshold) then
+          if (@bytesSeverityThreshold > 0) and (highBytes >= @bytesSeverityThreshold) then
             sevtags = Array.new unless (sevtags = event.get('[event][severity_tags]'))
             if !sevtags.kind_of?(Array) then
               newtags = Array.new
@@ -239,9 +239,9 @@ filter {
     if ([length]) {
       ruby {
         id => "ruby_add_field_severity_duration"
-        init => "secSeverityThreshold = ENV['CONNECTION_SECONDS_SEVERITY_THRESHOLD'] || '3600' ; $msSeverityThreshold = Integer(secSeverityThreshold) * 1000"
+        init => "secSeverityThreshold = ENV['CONNECTION_SECONDS_SEVERITY_THRESHOLD'] || '3600' ; @msSeverityThreshold = Integer(secSeverityThreshold) * 1000"
         code => "
-          if ($msSeverityThreshold > 0) and (event.get('[length]') >= $msSeverityThreshold) then
+          if (@msSeverityThreshold > 0) and (event.get('[length]') >= @msSeverityThreshold) then
             sevtags = Array.new unless (sevtags = event.get('[event][severity_tags]'))
             if !sevtags.kind_of?(Array) then
               newtags = Array.new

logstash/pipelines/suricata/11_suricata_logs.conf

@@ -82,7 +82,7 @@ filter {
 
         ruby {
           id => "ruby_suricata_timestamp_calc"
-          init => "require 'time'; require 'date';"
+          init => "require 'time'; require 'date'"
           code => "
             timpStamp = DateTime.parse(event.get('[suricata][timestamp]')).to_time
             timeStampMs = (1000*timpStamp.to_f).round(0)

logstash/pipelines/zeek/10_zeek_prep.conf

@@ -26,8 +26,8 @@ filter {
   # Zeek logs we're going to ignore
   ruby {
     id => "ruby_zeek_log_type_determine_drop"
-    init => "logtypesStr = ENV['LOGSTASH_ZEEK_IGNORED_LOGS'] || 'analyzer,broker,bsap_ip_unknown,bsap_serial_unknown,capture_loss,cluster,config,ecat_arp_info,loaded_scripts,packet_filter,png,print,prof,reporter,stats,stderr,stdout' ; $logtypes = logtypesStr.gsub(/\s+/, '').split(',');"
-    code => "event.set('[@metadata][drop_zeek_log]', true) if $logtypes.include?(event.get('[log_source]').to_s)"
+    init => "logtypesStr = ENV['LOGSTASH_ZEEK_IGNORED_LOGS'] || 'analyzer,broker,bsap_ip_unknown,bsap_serial_unknown,capture_loss,cluster,config,ecat_arp_info,loaded_scripts,packet_filter,png,print,prof,reporter,stats,stderr,stdout' ; @logtypes = logtypesStr.gsub(/\s+/, '').split(',')"
+    code => "event.set('[@metadata][drop_zeek_log]', true) if @logtypes.include?(event.get('[log_source]').to_s)"
   }
   if [@metadata][drop_zeek_log] { drop { id => "drop_zeek_ignored_source" } }