Merge pull request #231 from michalpurzynski/master

The Lua code for parsing and transforming Bro logs inside the Heka sandbox
mozilla · Feb 2, 2015 · eb7a528 · eb7a528
2 parents 6b16292 + 8236da6
commit eb7a528
Show file tree

Hide file tree

Showing 18 changed files with 1,880 additions and 0 deletions.
diff --git a/examples/heka-lua-bro/bro_capture_loss.lua b/examples/heka-lua-bro/bro_capture_loss.lua
@@ -0,0 +1,93 @@
+-- This Source Code Form is subject to the terms of the Mozilla Public
+-- License, v. 2.0. If a copy of the MPL was not distributed with this
+-- file, You can obtain one at http://mozilla.org/MPL/2.0/.
+-- Copyright (c) 2014 Mozilla Corporation
+--
+-- Contributors:
+-- Anthony Verez averez@mozilla.com
+-- Jeff Bryner jbryner@mozilla.com
+-- Michal Purzynski mpurzynski@mozilla.com
+
+local l=require "lpeg"
+local string=require "string"
+l.locale(l) --add locale entries in the lpeg table
+local space = l.space^0  --define a space constant
+local sep = l.P"\t"
+local elem = l.C((1-sep)^0)
+grammar = l.Ct(elem * (sep * elem)^0) -- split on tabs, return as table
+
+function toString(value)
+    if value == "-" then
+        return nil
+    end
+    return value
+end
+
+function nilToString(value)
+    if value == nil then
+        return ""
+    end
+    return value
+end
+
+function toNumber(value)
+    if value == "-" then
+        return nil
+    end
+    return tonumber(value)
+end
+
+function truncate(value)
+    -- truncate the URI if too long (heka limited to 63KiB messages)
+    if toString(value) then
+        if string.len(value) > 10000 then
+            return toString(string.sub(value, 1, 10000)) .. "[truncated]"
+        else
+            return value
+        end
+    end
+end
+
+function lastField(value)
+    -- remove last "\n" if there's one
+    if value ~= nil and string.len(value) > 1 and string.sub(value, -2) == "\n" then
+        return string.sub(value, 1, -2)
+    end
+    return value
+end
+
+function process_message()
+    local log = read_message("Payload")
+    --set a default msg that heka's
+    --message matcher can ignore via a message matcher:
+    -- message_matcher = "( Type!='heka.all-report' && Type != 'IGNORE' )"
+    local msg = {
+        Type = "IGNORE",
+        Fields={}
+    }
+    local matches = grammar:match(log)
+    if not matches then
+        --return 0 to not propogate errors to heka's log.
+        --return a message with IGNORE type to not match heka's message matcher
+        inject_message(msg)
+        return 0
+    end
+    if string.sub(log,1,1)=='#' then
+        --it's a comment line
+        inject_message(msg)
+        return 0
+    end
+
+    msg['Type'] = 'brocaptureloss'
+    msg['Logger'] = 'nsm'
+    msg.Fields['ts'] = toString(matches[1])
+    msg.Fields['ts_delta_float'] = toNumber(matches[2])
+    msg.Fields['peer'] = toNumber(matches[3])
+    msg.Fields['gaps_int'] = toNumber(matches[4])
+    msg.Fields['acks_int'] = toNumber(matches[5])
+    msg.Fields['percent_lost_float'] = toNumber(lastField(matches[6])) -- remove last "\n"
+    msg.Fields['summary'] = nilToString(msg.Fields['percent_lost_float']) .. "% loss on ".. nilToString(msg.Fields['host']) .. " in the last " .. nilToString(msg.Fields['ts_delta_float']) .. "  seconds"
+    inject_message(msg)
+    return 0
+end
+
diff --git a/examples/heka-lua-bro/bro_conn.lua b/examples/heka-lua-bro/bro_conn.lua
@@ -0,0 +1,95 @@
+-- This Source Code Form is subject to the terms of the Mozilla Public
+-- License, v. 2.0. If a copy of the MPL was not distributed with this
+-- file, You can obtain one at http://mozilla.org/MPL/2.0/.
+-- Copyright (c) 2014 Mozilla Corporation
+--
+-- Contributors:
+-- Anthony Verez averez@mozilla.com
+-- Jeff Bryner jbryner@mozilla.com
+-- Michal Purzynski mpurzynski@mozilla.com
+
+local l=require "lpeg"
+local string=require "string"
+l.locale(l) --add locale entries in the lpeg table
+local space = l.space^0  --define a space constant
+local sep = l.P"\t"
+local elem = l.C((1-sep)^0)
+grammar = l.Ct(elem * (sep * elem)^0) -- split on tabs, return as table
+
+function toString(value)
+    if value == "-" then
+        return ""
+    end
+    return value
+end
+
+function toNumber(value)
+    if value == "-" then
+        return 0
+    end
+    return tonumber(value)
+end
+
+function lastField(value)
+    -- remove last "\n" if there's one
+    if value ~= nil and string.len(value) > 1 and string.sub(value, -2) == "\n" then
+        return string.sub(value, 1, -2)
+    end
+    return value
+end
+
+function process_message()
+    local log = read_message("Payload")
+    --set a default msg that heka's
+    --message matcher can ignore via a message matcher:
+    -- message_matcher = "( Type!='heka.all-report' && Type != 'IGNORE' )"
+    local msg = {
+        Type = "IGNORE",
+        Fields={}
+    } 
+    local matches = grammar:match(log)
+    if not matches then
+        --return 0 to not propogate errors to heka's log.
+        --return a message with IGNORE type to not match heka's message matcher
+        inject_message(msg)
+        return 0 
+    end
+    if string.sub(log,1,1)=='#' then
+        --it's a comment line
+        inject_message(msg)
+        return 0
+    end
+
+    -- avoid logging UDP/TCP connections for UDP, remove that if you care
+    if matches[8] == "dns" then 
+        inject_message(msg)
+        return 0 
+    end
+
+    msg['Type']='broconnection'
+    msg['Logger']='nsm'
+    msg['ts'] = toString(matches[1])
+    msg.Fields['uid'] = toString(matches[2])
+    msg.Fields['sourceipaddress'] = toString(matches[3])
+    msg.Fields['sourceport'] = toNumber(matches[4])
+    msg.Fields['destinationipaddress'] = toString(matches[5])
+    msg.Fields['destinationport'] = toNumber(matches[6])
+    msg.Fields['protocol'] = toString(matches[7])
+    msg.Fields['service'] = toString(matches[8])
+    msg.Fields['duration_float'] = toNumber(matches[9])
+    msg.Fields['originbytes_int'] = toNumber(matches[10])
+    msg.Fields['responsebytes_int'] = toNumber(matches[11])
+    msg.Fields['connectionstate'] = toString(matches[12])
+    msg.Fields['local_origin'] = toString(matches[13])
+    msg.Fields['missedbytes_int'] = toNumber(matches[14])
+    msg.Fields['history'] = toString(matches[15])
+    msg.Fields['originpkts_int'] = toNumber(matches[16])
+    msg.Fields['originipbytes_int'] = toNumber(matches[17])
+    msg.Fields['responsepackets_int'] = toNumber(matches[18])
+    msg.Fields['responseipbytes_int'] = toNumber(matches[19])
+    msg.Fields['tunnelparents'] = lastField(toString(matches[20]))
+    msg['Payload'] = toString(msg.Fields['sourceipaddress']) .. ":" .. toString(msg.Fields['sourceport']) .. " -> " .. toString(msg.Fields['destinationipaddress']) .. ":" .. toString(msg.Fields['destinationport']) .. " " .. toString(msg.Fields['history']) .. " " .. toString(msg.Fields['originipbytes_int']) .. " bytes / " .. toString(msg.Fields['responseipbytes_int']) .. " bytes"
+    inject_message(msg)
+    return 0
+end
+
diff --git a/examples/heka-lua-bro/bro_dhcp.lua b/examples/heka-lua-bro/bro_dhcp.lua
@@ -0,0 +1,97 @@
+-- This Source Code Form is subject to the terms of the Mozilla Public
+-- License, v. 2.0. If a copy of the MPL was not distributed with this
+-- file, You can obtain one at http://mozilla.org/MPL/2.0/.
+-- Copyright (c) 2014 Mozilla Corporation
+--
+-- Contributors:
+-- Anthony Verez averez@mozilla.com
+-- Jeff Bryner jbryner@mozilla.com
+-- Michal Purzynski mpurzynski@mozilla.com
+
+local l=require "lpeg"
+local string=require "string"
+l.locale(l) --add locale entries in the lpeg table
+local space = l.space^0  --define a space constant
+local sep = l.P"\t"
+local elem = l.C((1-sep)^0)
+grammar = l.Ct(elem * (sep * elem)^0) -- split on tabs, return as table
+
+function toString(value)
+    if value == "-" then
+        return nil
+    end
+    return value
+end
+
+function nilToString(value)
+    if value == nil then
+        return ""
+    end
+    return value
+end
+
+function toNumber(value)
+    if value == "-" then
+        return nil
+    end
+    return tonumber(value)
+end
+
+function truncate(value)
+    -- truncate the URI if too long (heka limited to 63KiB messages)
+    if toString(value) then
+        if string.len(value) > 10000 then
+            return toString(string.sub(value, 1, 10000)) .. "[truncated]"
+        else
+            return value
+        end
+    end
+end
+
+function lastField(value)
+    -- remove last "\n" if there's one
+    if value ~= nil and string.len(value) > 1 and string.sub(value, -2) == "\n" then
+        return string.sub(value, 1, -2)
+    end
+    return value
+end
+
+function process_message()
+    local log = read_message("Payload")
+    --set a default msg that heka's
+    --message matcher can ignore via a message matcher:
+    -- message_matcher = "( Type!='heka.all-report' && Type != 'IGNORE' )"
+    local msg = {
+        Type = "IGNORE",
+        Fields={}
+    }
+    local matches = grammar:match(log)
+    if not matches then
+        --return 0 to not propogate errors to heka's log.
+        --return a message with IGNORE type to not match heka's message matcher
+        inject_message(msg)
+        return 0
+    end
+    if string.sub(log,1,1)=='#' then
+        --it's a comment line
+        inject_message(msg)
+        return 0
+    end
+
+    msg['Type'] = 'brodhcp'
+    msg['Logger'] = 'nsm'
+    msg.Fields['ts'] = toString(matches[1])
+    msg.Fields['uid'] = toString(matches[2])
+    msg.Fields['sourceipaddress'] = toString(matches[3])
+    msg.Fields['sourceport'] = toNumber(matches[4])
+    msg.Fields['destinationipaddress'] = toString(matches[5])
+    msg.Fields['destinationport'] = toNumber(matches[6])
+    msg.Fields['mac'] = toString(matches[7])
+    msg.Fields['assigned_ip'] = toString(matches[8])
+    msg.Fields['lease_time_float'] = toNumber(matches[9])
+    msg.Fields['trans_id_int'] = toNumber(lastField(matches[10])) -- remove last "\n"
+    msg.Fields['summary'] = nilToString(msg.Fields['assigned_ip']) .. " assigned to " .. nilToString(msg.Fields['mac'])
+    inject_message(msg)
+    return 0
+end
+