Skip to content

Commit

Permalink
Merge pull request mozilla#348 from pwnbus/standardize_bro_intel
Browse files Browse the repository at this point in the history 
Standardize other bro_* categories
  • Loading branch information
@jeffbryner
jeffbryner committed Jun 28, 2016
2 parents 3568cc4 + 5765bdf commit 1ae54e2
Show file tree
Hide file tree
Showing 7 changed files with 41 additions and 41 deletions.
2 changes: 1 addition & 1 deletion alerts/bro_intel_dashboard.json
View file
Expand Up @@ -38,7 +38,7 @@
"1": { "1": {
"type": "querystring", "type": "querystring",
"field": "category", "field": "category",
"query": "bro_intel", "query": "brointel",
"mandate": "must", "mandate": "must",
"active": true, "active": true,
"alias": "", "alias": "",
Expand Down
2 changes: 1 addition & 1 deletion alerts/bro_intel_pyes.py
View file
Expand Up @@ -18,7 +18,7 @@ def main(self):
# Configure filters using pyes # Configure filters using pyes
must = [ must = [
pyes.TermFilter('_type', 'event'), pyes.TermFilter('_type', 'event'),
pyes.TermFilter('category', 'bro_intel'), pyes.TermFilter('category', 'brointel'),
pyes.ExistsFilter('seenindicator') pyes.ExistsFilter('seenindicator')
] ]
self.filtersManual(date_timedelta, must=must) self.filtersManual(date_timedelta, must=must)
Expand Down
42 changes: 21 additions & 21 deletions cron/broAlerts.py
View file
Expand Up @@ -30,7 +30,7 @@ def initLogger():
logger.level=logging.INFO logger.level=logging.INFO
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
formatter.formatTime = loggerTimeStamp formatter.formatTime = loggerTimeStamp
if options.output=='syslog': if options.output=='syslog':
logger.addHandler(SysLogHandler(address=(options.sysloghostname,options.syslogport))) logger.addHandler(SysLogHandler(address=(options.sysloghostname,options.syslogport)))
else: else:
sh=logging.StreamHandler(sys.stderr) sh=logging.StreamHandler(sys.stderr)
Expand All @@ -45,15 +45,15 @@ def toUTC(suspectedDate,localTimeZone="US/Pacific"):
objDate=parse(suspectedDate,fuzzy=True) objDate=parse(suspectedDate,fuzzy=True)
elif type(suspectedDate)==datetime: elif type(suspectedDate)==datetime:
objDate=suspectedDate objDate=suspectedDate

if objDate.tzinfo is None: if objDate.tzinfo is None:
objDate=pytz.timezone(localTimeZone).localize(objDate) objDate=pytz.timezone(localTimeZone).localize(objDate)
objDate=utc.normalize(objDate) objDate=utc.normalize(objDate)
else: else:
objDate=utc.normalize(objDate) objDate=utc.normalize(objDate)
if objDate is not None: if objDate is not None:
objDate=utc.normalize(objDate) objDate=utc.normalize(objDate)

return objDate return objDate


def flattenDict(dictIn): def flattenDict(dictIn):
Expand All @@ -68,7 +68,7 @@ def alertToMessageQueue(alertDict):
channel = connection.channel() channel = connection.channel()
#declare the exchanges #declare the exchanges
channel.exchange_declare(exchange=options.alertexchange,type='topic', durable=True) channel.exchange_declare(exchange=options.alertexchange,type='topic', durable=True)

#cherry pick items from the alertDict to send to the alerts messageQueue #cherry pick items from the alertDict to send to the alerts messageQueue
mqAlert=dict(severity='INFO',category='') mqAlert=dict(severity='INFO',category='')
if 'severity' in alertDict.keys(): if 'severity' in alertDict.keys():
Expand All @@ -80,7 +80,7 @@ def alertToMessageQueue(alertDict):
if 'eventtimestamp' in alertDict.keys(): if 'eventtimestamp' in alertDict.keys():
mqAlert['eventtimestamp']=alertDict['eventtimestamp'] mqAlert['eventtimestamp']=alertDict['eventtimestamp']
mqAlert['summary']=alertDict['summary'] mqAlert['summary']=alertDict['summary']
channel.basic_publish(exchange=options.alertexchange,routing_key=options.alertqueue,body=json.dumps(mqAlert)) channel.basic_publish(exchange=options.alertexchange,routing_key=options.alertqueue,body=json.dumps(mqAlert))
except Exception as e: except Exception as e:
logger.error('Exception while sending alert to message queue: {0}'.format(e)) logger.error('Exception while sending alert to message queue: {0}'.format(e))


Expand All @@ -93,15 +93,15 @@ def alertToES(es,alertDict):


def esBroIntelEvents(): def esBroIntelEvents():
begindateUTC= toUTC(datetime.now() - timedelta(minutes=30)) begindateUTC= toUTC(datetime.now() - timedelta(minutes=30))
enddateUTC= toUTC(datetime.now()) enddateUTC= toUTC(datetime.now())
#search for events within the date range that haven't already been alerted (i.e. given an alerttimestamp) #search for events within the date range that haven't already been alerted (i.e. given an alerttimestamp)
qDate=pyes.RangeQuery(qrange=pyes.ESRange('utctimestamp',from_value=begindateUTC,to_value=enddateUTC)) qDate=pyes.RangeQuery(qrange=pyes.ESRange('utctimestamp',from_value=begindateUTC,to_value=enddateUTC))
qType=pyes.TermFilter('_type','event') qType=pyes.TermFilter('_type','event')
qEvents=pyes.TermsFilter('category',['brointel']) qEvents=pyes.TermsFilter('category',['brointel'])
qalerted=pyes.ExistsFilter('alerttimestamp') qalerted=pyes.ExistsFilter('alerttimestamp')
q=pyes.ConstantScoreQuery(pyes.MatchAllQuery()) q=pyes.ConstantScoreQuery(pyes.MatchAllQuery())
q.filters.append(pyes.BoolFilter( q.filters.append(pyes.BoolFilter(
must=[qType, must=[qType,
qDate, qDate,
qEvents, qEvents,
pyes.ExistsFilter('seenindicator') pyes.ExistsFilter('seenindicator')
Expand All @@ -117,11 +117,11 @@ def esBroXSSEvents():
enddateUTC= toUTC(datetime.now()) enddateUTC= toUTC(datetime.now())
qDate = pyes.RangeQuery(qrange=pyes.ESRange('utctimestamp', from_value=begindateUTC, to_value=enddateUTC)) qDate = pyes.RangeQuery(qrange=pyes.ESRange('utctimestamp', from_value=begindateUTC, to_value=enddateUTC))
qType = pyes.TermFilter('_type', 'event') qType = pyes.TermFilter('_type', 'event')
qEvents = pyes.TermFilter("category","bro_xss_log") qEvents = pyes.TermFilter("category","broxsslog")
qalerted = pyes.ExistsFilter('alerttimestamp') qalerted = pyes.ExistsFilter('alerttimestamp')
q=pyes.ConstantScoreQuery(pyes.MatchAllQuery()) q=pyes.ConstantScoreQuery(pyes.MatchAllQuery())
q.filters.append(pyes.BoolFilter( q.filters.append(pyes.BoolFilter(
must=[qType, must=[qType,
qDate, qDate,
qEvents, qEvents,
pyes.ExistsFilter('uri') pyes.ExistsFilter('uri')
Expand Down Expand Up @@ -161,7 +161,7 @@ def esRunSearch(es, query, aggregateField, detailLimit=5):
return indicatorList return indicatorList


except pyes.exceptions.NoServerAvailable: except pyes.exceptions.NoServerAvailable:
logger.error('Elastic Search server could not be reached, check network connectivity') logger.error('Elastic Search server could not be reached, check network connectivity')




def createAlerts(es, indicatorCounts, threshold, description): def createAlerts(es, indicatorCounts, threshold, description):
Expand Down Expand Up @@ -195,14 +195,14 @@ def createAlerts(es, indicatorCounts, threshold, description):
# append the relevant events in text format to avoid errant ES issues. # append the relevant events in text format to avoid errant ES issues.
# should be able to just set eventsource to i['events'] but different versions of ES 1.0 complain # should be able to just set eventsource to i['events'] but different versions of ES 1.0 complain
alert['eventsource'].append(flattenDict(e)) alert['eventsource'].append(flattenDict(e))

logger.debug(alert['summary']) logger.debug(alert['summary'])
logger.debug(alert['events']) logger.debug(alert['events'])
logger.debug(alert) logger.debug(alert)

# save alert to alerts index, update events index with alert ID for cross reference # save alert to alerts index, update events index with alert ID for cross reference
alertResult = alertToES(es, alert) alertResult = alertToES(es, alert)

##logger.debug(alertResult) ##logger.debug(alertResult)
# for each event in this list of indicatorCounts # for each event in this list of indicatorCounts
# update with the alertid/index # update with the alertid/index
Expand All @@ -212,28 +212,28 @@ def createAlerts(es, indicatorCounts, threshold, description):
e['_source']['alerts'] = [] e['_source']['alerts'] = []
e['_source']['alerts'].append(dict(index=alertResult['_index'], type=alertResult['_type'], id=alertResult['_id'])) e['_source']['alerts'].append(dict(index=alertResult['_index'], type=alertResult['_type'], id=alertResult['_id']))
e['_source']['alerttimestamp'] = toUTC(datetime.now()).isoformat() e['_source']['alerttimestamp'] = toUTC(datetime.now()).isoformat()

es.update(e['_index'], e['_type'], e['_id'], document=e['_source']) es.update(e['_index'], e['_type'], e['_id'], document=e['_source'])

alertToMessageQueue(alert) alertToMessageQueue(alert)
except ValueError as e: except ValueError as e:
logger.error("Exception %r when creating alerts " % e) logger.error("Exception %r when creating alerts " % e)

def main(): def main():
logger.debug('starting') logger.debug('starting')
logger.debug(options) logger.debug(options)
es=pyes.ES((list('{0}'.format(s) for s in options.esservers))) es=pyes.ES((list('{0}'.format(s) for s in options.esservers)))
# search for brointel # search for brointel
#indicatorCounts=esSearch(es) #indicatorCounts=esSearch(es)
#createAlerts(es,indicatorCounts) #createAlerts(es,indicatorCounts)

indicatorCounts=esRunSearch(es,esBroIntelEvents(),'seenindicator', 50) indicatorCounts=esRunSearch(es,esBroIntelEvents(),'seenindicator', 50)
createAlerts(es,indicatorCounts, 5, 'bro intel match') createAlerts(es,indicatorCounts, 5, 'bro intel match')

# search for xss events # search for xss events
indicatorCounts=esRunSearch(es,esBroXSSEvents(),'cluster_client_ip', 50) indicatorCounts=esRunSearch(es,esBroXSSEvents(),'cluster_client_ip', 50)
createAlerts(es,indicatorCounts, 5, 'bro xss') createAlerts(es,indicatorCounts, 5, 'bro xss')

logger.debug('finished') logger.debug('finished')


def initConfig(): def initConfig():
Expand All @@ -249,7 +249,7 @@ def initConfig():
options.syslogport=getConfig('syslogport',514,options.configfile) #syslog port options.syslogport=getConfig('syslogport',514,options.configfile) #syslog port
#elastic search server settings #elastic search server settings
options.esservers=list(getConfig('esservers','http://localhost:9200',options.configfile).split(',')) options.esservers=list(getConfig('esservers','http://localhost:9200',options.configfile).split(','))

if __name__ == '__main__': if __name__ == '__main__':
parser=OptionParser() parser=OptionParser()
parser.add_option("-c", dest='configfile' , default=sys.argv[0].replace('.py','.conf'), help="configuration file to use") parser.add_option("-c", dest='configfile' , default=sys.argv[0].replace('.py','.conf'), help="configuration file to use")
Expand Down
4 changes: 2 additions & 2 deletions examples/demo/sampleevents/alertcreating-bro-intel.json
View file
@@ -1,14 +1,14 @@
[ [
{ {
"category": "bro_intel", "category": "brointel",
"processid": "0", "processid": "0",
"severity": "7", "severity": "7",
"tags": ["nsm","bro","intel"], "tags": ["nsm","bro","intel"],
"hostname": "nsm5", "hostname": "nsm5",
"summary": "Bro intel match: <randomipaddress>", "summary": "Bro intel match: <randomipaddress>",
"file": "nsm", "file": "nsm",
"details": { "details": {
"category": "bro_intel", "category": "brointel",
"destinationipaddress": "0.0.82.27", "destinationipaddress": "0.0.82.27",
"seenwhere": "Intel::ADDR", "seenwhere": "Intel::ADDR",
"uid": "C4RdjhyE2jkvRH54d", "uid": "C4RdjhyE2jkvRH54d",
Expand Down
24 changes: 12 additions & 12 deletions examples/es-docs/bro_intel.json
View file
@@ -1,6 +1,6 @@
[ [
{ {
"category": "bro_intel", "category": "brointel",
"processid": "0", "processid": "0",
"receivedtimestamp": "2014-07-16T21:32:07.502716+00:00", "receivedtimestamp": "2014-07-16T21:32:07.502716+00:00",
"severity": "7", "severity": "7",
Expand All @@ -11,7 +11,7 @@
"summary": "Bro intel match: 0.0.139.213", "summary": "Bro intel match: 0.0.139.213",
"eventsource": "nsm", "eventsource": "nsm",
"details": { "details": {
"category": "bro_intel", "category": "brointel",
"destinationipaddress": "0.0.82.27", "destinationipaddress": "0.0.82.27",
"seenwhere": "Intel::ADDR", "seenwhere": "Intel::ADDR",
"uid": "C4RdjhyE2jkvRH54d", "uid": "C4RdjhyE2jkvRH54d",
Expand All @@ -28,7 +28,7 @@
} }
}, },
{ {
"category": "bro_intel", "category": "brointel",
"processid": "0", "processid": "0",
"receivedtimestamp": "2014-07-16T21:32:07.499594+00:00", "receivedtimestamp": "2014-07-16T21:32:07.499594+00:00",
"severity": "7", "severity": "7",
Expand All @@ -39,7 +39,7 @@
"summary": "Bro intel match: 0.0.139.213", "summary": "Bro intel match: 0.0.139.213",
"eventsource": "nsm", "eventsource": "nsm",
"details": { "details": {
"category": "bro_intel", "category": "brointel",
"destinationipaddress": "0.0.82.28", "destinationipaddress": "0.0.82.28",
"seenwhere": "Intel::ADDR", "seenwhere": "Intel::ADDR",
"uid": "Ce58I13SIYMCYbcAw4", "uid": "Ce58I13SIYMCYbcAw4",
Expand All @@ -56,7 +56,7 @@
} }
}, },
{ {
"category": "bro_intel", "category": "brointel",
"processid": "0", "processid": "0",
"receivedtimestamp": "2014-07-16T21:32:07.499594+00:00", "receivedtimestamp": "2014-07-16T21:32:07.499594+00:00",
"severity": "7", "severity": "7",
Expand All @@ -67,7 +67,7 @@
"summary": "Bro intel match: 0.0.139.213", "summary": "Bro intel match: 0.0.139.213",
"eventsource": "nsm", "eventsource": "nsm",
"details": { "details": {
"category": "bro_intel", "category": "brointel",
"destinationipaddress": "0.0.82.28", "destinationipaddress": "0.0.82.28",
"seenwhere": "Intel::ADDR", "seenwhere": "Intel::ADDR",
"uid": "Ce58I13SIYMCYbcAw4", "uid": "Ce58I13SIYMCYbcAw4",
Expand All @@ -84,7 +84,7 @@
} }
}, },
{ {
"category": "bro_intel", "category": "brointel",
"processid": "0", "processid": "0",
"receivedtimestamp": "2014-07-16T21:32:07.499594+00:00", "receivedtimestamp": "2014-07-16T21:32:07.499594+00:00",
"severity": "7", "severity": "7",
Expand All @@ -95,7 +95,7 @@
"summary": "Bro intel match: 0.0.139.213", "summary": "Bro intel match: 0.0.139.213",
"eventsource": "nsm", "eventsource": "nsm",
"details": { "details": {
"category": "bro_intel", "category": "brointel",
"destinationipaddress": "0.0.82.28", "destinationipaddress": "0.0.82.28",
"seenwhere": "Intel::ADDR", "seenwhere": "Intel::ADDR",
"uid": "Ce58I13SIYMCYbcAw4", "uid": "Ce58I13SIYMCYbcAw4",
Expand All @@ -112,7 +112,7 @@
} }
}, },
{ {
"category": "bro_intel", "category": "brointel",
"processid": "0", "processid": "0",
"receivedtimestamp": "2014-07-16T21:32:07.499594+00:00", "receivedtimestamp": "2014-07-16T21:32:07.499594+00:00",
"severity": "7", "severity": "7",
Expand All @@ -123,7 +123,7 @@
"summary": "Bro intel match: 0.0.139.213", "summary": "Bro intel match: 0.0.139.213",
"eventsource": "nsm", "eventsource": "nsm",
"details": { "details": {
"category": "bro_intel", "category": "brointel",
"destinationipaddress": "0.0.82.28", "destinationipaddress": "0.0.82.28",
"seenwhere": "Intel::ADDR", "seenwhere": "Intel::ADDR",
"uid": "Ce58I13SIYMCYbcAw4", "uid": "Ce58I13SIYMCYbcAw4",
Expand All @@ -140,7 +140,7 @@
} }
}, },
{ {
"category": "bro_intel", "category": "brointel",
"processid": "0", "processid": "0",
"receivedtimestamp": "2014-07-16T21:32:07.499594+00:00", "receivedtimestamp": "2014-07-16T21:32:07.499594+00:00",
"severity": "7", "severity": "7",
Expand All @@ -151,7 +151,7 @@
"summary": "Bro intel match: 0.0.139.213", "summary": "Bro intel match: 0.0.139.213",
"eventsource": "nsm", "eventsource": "nsm",
"details": { "details": {
"category": "bro_intel", "category": "brointel",
"destinationipaddress": "0.0.82.28", "destinationipaddress": "0.0.82.28",
"seenwhere": "Intel::ADDR", "seenwhere": "Intel::ADDR",
"uid": "Ce58I13SIYMCYbcAw4", "uid": "Ce58I13SIYMCYbcAw4",
Expand Down
4 changes: 2 additions & 2 deletions examples/heka-lua-bro/bro_known_certs.lua
View file
Expand Up @@ -16,7 +16,7 @@ local elem = lpeg.C((1-sep)^0)
local grammar = -lpeg.P"#" * lpeg.Ct(elem * (sep * elem)^0) -- ignore comment, split on tabs, return as table local grammar = -lpeg.P"#" * lpeg.Ct(elem * (sep * elem)^0) -- ignore comment, split on tabs, return as table


local msg = { local msg = {
Type = "bro_known_certs", Type = "broknowncerts",
Logger = "nsm", Logger = "nsm",
Fields = { Fields = {
-- Initializing our fields -- Initializing our fields
Expand All @@ -28,7 +28,7 @@ local msg = {
['serial'] = nil, ['serial'] = nil,
summary = nil, summary = nil,
severity = "INFO", severity = "INFO",
category = "bro_known_certs", category = "broknowncerts",
tags = "nsm,bro,known_certs" tags = "nsm,bro,known_certs"
} }
} }
Expand Down
4 changes: 2 additions & 2 deletions examples/heka-lua-bro/bro_socks.lua
View file
Expand Up @@ -16,7 +16,7 @@ local elem = lpeg.C((1-sep)^0)
local grammar = -lpeg.P"#" * lpeg.Ct(elem * (sep * elem)^0) -- ignore comment, split on tabs, return as table local grammar = -lpeg.P"#" * lpeg.Ct(elem * (sep * elem)^0) -- ignore comment, split on tabs, return as table


local msg = { local msg = {
Type = "bro_socks", Type = "brosocks",
Logger = "nsm", Logger = "nsm",
Fields = { Fields = {
-- Initializing our fields -- Initializing our fields
Expand All @@ -38,7 +38,7 @@ local msg = {
['summary'] = nil, ['summary'] = nil,
summary = nil, summary = nil,
severity = "INFO", severity = "INFO",
category = "bro_socks", category = "brosocks",
tags = "nsm,bro,socks" tags = "nsm,bro,socks"
} }
} }
Expand Down

0 comments on commit 1ae54e2

Please sign in to comment.