Skip to content
Permalink
Browse files

Merge remote-tracking branch 'origin/master' into python_3_upgrade

  • Loading branch information...
pwnbus committed Jul 8, 2019
2 parents dd23f77 + 592b613 commit 6e0ee8d0a01ae8f175930e79b88c0f625fb1d510
Showing with 59 additions and 28 deletions.
  1. +16 −16 benchmarking/workers/json2Mozdef.py
  2. +5 −1 mq/plugins/broFixup.py
  3. +38 −11 tests/mq/plugins/test_broFixup.py
@@ -20,33 +20,33 @@
import time

httpsession = FuturesSession(max_workers=5)
httpsession.trust_env=False # turns of needless .netrc check for creds
httpsession.trust_env = False # turns of needless .netrc check for creds
# a = requests.adapters.HTTPAdapter(max_retries=2)
# httpsession.mount('http://', a)


logger = logging.getLogger(sys.argv[0])
logger.level=logging.DEBUG
logger.level = logging.DEBUG

formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')


def postLogs(logcache):
# post logs asynchronously with requests workers and check on the results
# expects a queue object from the multiprocessing library
posts=[]
posts = []
try:
while not logcache.empty():
postdata=logcache.get_nowait()
postdata = logcache.get_nowait()
if len(postdata) > 0:
url=options.url
a=httpsession.get_adapter(url)
a.max_retries=3
r=httpsession.post(url,data=postdata)
url = options.url
a = httpsession.get_adapter(url)
a.max_retries = 3
r = httpsession.post(url,data=postdata)
posts.append((r,postdata,url))
except Empty as e:
pass
for p,postdata,url in posts:
for p, postdata, url in posts:
try:
if p.result().status_code >= 500:
logger.error("exception posting to %s %r [will retry]\n" % (url, p.result().status_code))
@@ -63,12 +63,12 @@ def postLogs(logcache):
if __name__ == '__main__':
parser=OptionParser()
parser.add_option("-u", dest='url', default='http://localhost:8080/events/', help="mozdef events URL to use when posting events")
(options,args) = parser.parse_args()
sh=logging.StreamHandler(sys.stdout)
(options, args) = parser.parse_args()
sh = logging.StreamHandler(sys.stdout)
sh.setFormatter(formatter)
logger.addHandler(sh)
# create a list of logs we can append json to and call for a post when we want.
logcache=Queue()
logcache = Queue()
try:
for i in range(0,10):

@@ -84,14 +84,14 @@ def postLogs(logcache):
tags=[],
details=[]
)
alog['details']=dict(success=True,username='mozdef')
alog['tags']=['mozdef','stresstest']
alog['details'] = dict(success=True, username='mozdef')
alog['tags'] = ['mozdef', 'stresstest']

logcache.put(json.dumps(alog))
if not logcache.empty():
time.sleep(.001)
try:
postingProcess=Process(target=postLogs,args=(logcache,),name="json2MozdefStressTest")
postingProcess = Process(target=postLogs, args=(logcache,), name="json2MozdefStressTest")
postingProcess.start()
except OSError as e:
if e.errno == 35: # resource temporarily unavailable.
@@ -102,7 +102,7 @@ def postLogs(logcache):

while not logcache.empty():
try:
postingProcess=Process(target=postLogs,args=(logcache,),name="json2MozdefStressTest")
postingProcess = Process(target=postLogs, args=(logcache,), name = "json2MozdefStressTest")
postingProcess.start()
except OSError as e:
if e.errno == 35: # resource temporarily unavailable.
@@ -227,8 +227,12 @@ def onMessage(self, message, metadata):
return (newmessage, metadata)

if logtype == 'dhcp':
if 'assigned_addr' not in newmessage['details']:
newmessage['details']['assigned_addr'] = '0.0.0.0'
if 'mac' not in newmessage['details']:
newmessage['details']['mac'] = '00:00:00:00:00:00'
newmessage['summary'] = (
'{assigned_ip} assigned to '
'{assigned_addr} assigned to '
'{mac}'
).format(**newmessage['details'])
return (newmessage, metadata)
@@ -546,16 +546,17 @@ def test_dhcp_log(self):
'customendpoint': 'bro'
}
MESSAGE = {
"ts":1505701256.181043,
"uid":"Cbs59u2x6KXu85dsOi",
"id.orig_h":"10.26.40.65",
"id.orig_p":68,
"id.resp_h":"10.26.40.1",
"id.resp_p":67,
"mac":"00:25:90:9b:67:b2",
"assigned_ip":"10.26.40.65",
"lease_time":86400.0,
"trans_id":1504605887
"ts": 1561756317.104897,
"uids": ["C6uJBE1z3CKfrA9FE4", "CdCBtl1fKEIMNvebrb", "CNJJ9g1HgefKR09ied", "CuXKNM1R5MEJ9GsMIi", "CMIYsm2weaHvzBRJIi", "C0vslbmXr3Psyy5Ff", "Ct0BRQ2Y84MWhag1Ik", "C5BNK71HlfhlXf8Pq", "C5ZrPG3DfQNzsiUMi2", "CMJHze3BH9o7yg9yM6", "CMSyg03ZZcdic8pTMc"],
"client_addr": "10.251.255.10",
"server_addr": "10.251.24.1",
"mac": "f0:18:98:55:0e:0e",
"host_name": "aliczekkroliczek",
"domain": "ala.ma.kota",
"assigned_addr": "10.251.30.202",
"lease_time": 43200.0,
"msg_types": ["DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "OFFER", "OFFER", "OFFER", "DISCOVER", "DISCOVER", "DISCOVER", "OFFER", "OFFER", "OFFER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "OFFER", "OFFER", "OFFER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "OFFER", "OFFER", "OFFER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "OFFER", "OFFER", "OFFER", "OFFER", "OFFER", "OFFER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "OFFER", "OFFER", "OFFER", "OFFER", "OFFER", "OFFER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "OFFER", "OFFER", "OFFER", "DISCOVER", "OFFER", "OFFER", "OFFER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "OFFER", "DISCOVER", "OFFER", "OFFER", "OFFER", "OFFER", "OFFER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "DISCOVER", "OFFER", "OFFER", "OFFER", "DISCOVER", "DISCOVER", "OFFER", "OFFER", "OFFER"],
"duration": 34.037004
}
event['MESSAGE'] = json.dumps(MESSAGE)

@@ -568,7 +569,33 @@ def test_dhcp_log(self):
if not key.startswith('id.'):
assert key in result['details']
assert MESSAGE[key] == result['details'][key]
assert result['summary'] == '10.26.40.65 assigned to 00:25:90:9b:67:b2'
assert result['summary'] == '10.251.30.202 assigned to f0:18:98:55:0e:0e'

def test_dhcp_log2(self):
event = {
'category': 'bro',
'SOURCE': 'bro_dhcp',
'customendpoint': 'bro'
}
MESSAGE = {
"ts": 1561607456.803827,
"uids": ["CsXuIb2HTmDaPrPvT7"],
"host_name": "nsm2",
"msg_types": ["DISCOVER", "DISCOVER"],
"duration": 17.778322
}
event['MESSAGE'] = json.dumps(MESSAGE)

result, metadata = self.plugin.onMessage(event, self.metadata)
self.verify_defaults(result)
self.verify_metadata(metadata)
assert toUTC(MESSAGE['ts']).isoformat() == result['utctimestamp']
assert toUTC(MESSAGE['ts']).isoformat() == result['timestamp']
for key in MESSAGE.keys():
if not key.startswith('id.'):
assert key in result['details']
assert MESSAGE[key] == result['details'][key]
assert result['summary'] == '0.0.0.0 assigned to 00:00:00:00:00:00'

def test_ftp_log(self):
event = {

0 comments on commit 6e0ee8d

Please sign in to comment.
You can’t perform that action at this time.