Skip to content
Permalink
Browse files

Merge pull request #1105 from mozilla/fixup_keys_references

Remove .keys() call during key exists comparison
  • Loading branch information...
pwnbus committed Feb 27, 2019
2 parents c9a3d7a + e16ec57 commit 4190c8d5c5fb79485f56d6cca86e0227daf5336e
@@ -38,9 +38,9 @@
'options': {'queue': 'celery-default', "exchange": "celery-default"},
}
# add optional parameters:
if 'args' in ALERTS[alert].keys():
if 'args' in ALERTS[alert]:
CELERYBEAT_SCHEDULE[alert]['args']=ALERTS[alert]['args']
if 'kwargs' in ALERTS[alert].keys():
if 'kwargs' in ALERTS[alert]:
CELERYBEAT_SCHEDULE[alert]['kwargs']=ALERTS[alert]['kwargs']

# Load logging config
@@ -179,13 +179,13 @@ def alertToMessageQueue(self, alertDict):
try:
# cherry pick items from the alertDict to send to the alerts messageQueue
mqAlert = dict(severity='INFO', category='')
if 'severity' in alertDict.keys():
if 'severity' in alertDict:
mqAlert['severity'] = alertDict['severity']
if 'category' in alertDict.keys():
if 'category' in alertDict:
mqAlert['category'] = alertDict['category']
if 'utctimestamp' in alertDict.keys():
if 'utctimestamp' in alertDict:
mqAlert['utctimestamp'] = alertDict['utctimestamp']
if 'eventtimestamp' in alertDict.keys():
if 'eventtimestamp' in alertDict:
mqAlert['eventtimestamp'] = alertDict['eventtimestamp']
mqAlert['summary'] = alertDict['summary']
self.log.debug(mqAlert)
@@ -441,7 +441,7 @@ def tagEventsAlert(self, events, alertResultES):
"""
try:
for event in events:
if 'alerts' not in event['_source'].keys():
if 'alerts' not in event['_source']:
event['_source']['alerts'] = []
event['_source']['alerts'].append({
'index': alertResultES['_index'],
@@ -41,9 +41,9 @@ def onAggregation(self, aggreg):

summary += ' sample hosts that hit it: '
for e in aggreg['events'][:3]:
if 'details' in e['_source'].keys() \
and 'sourceipaddress' in e['_source']['details'].keys() \
and 'seenwhere' in e['_source']['details'].keys():
if 'details' in e['_source'] \
and 'sourceipaddress' in e['_source']['details'] \
and 'seenwhere' in e['_source']['details']:
interestingaddres = ''
# someone talking to a bad guy, I want to know who
# someone resolving bad guy's domain name, I want to know who
@@ -45,9 +45,9 @@ def initConfiguration(self):
def onMessage(self, message):
# here is where you do something with the incoming alert message
doclink = 'unknown'
if message['category'] in self.options.docs.keys():
if message['category'] in self.options.docs:
doclink = self.options.docs[message['category']]
if 'summary' in message.keys():
if 'summary' in message:
headers = {
'Content-type': 'application/json',
}
@@ -137,11 +137,11 @@ def formatAlert(jsonDictIn):
severity = 'INFO'
summary = ''
category = ''
if 'severity' in jsonDictIn.keys():
if 'severity' in jsonDictIn:
severity = jsonDictIn['severity']
if 'summary' in jsonDictIn.keys():
if 'summary' in jsonDictIn:
summary = jsonDictIn['summary']
if 'category' in jsonDictIn.keys():
if 'category' in jsonDictIn:
category = jsonDictIn['category']

return colorify('{0}: {1} {2}'.format(
@@ -184,7 +184,7 @@ def join_channels(client, *params):
if not options.join:
return
for chan in options.join.split(","):
if chan in options.channelkeys.keys():
if chan in options.channelkeys:
client.join(chan, options.channelkeys[chan])
else:
client.join(chan)
@@ -311,7 +311,7 @@ def on_message(self, body, message):
# process valid message
# see where we send this alert
ircchannel = options.alertircchannel
if 'ircchannel' in bodyDict.keys():
if 'ircchannel' in bodyDict:
if bodyDict['ircchannel'] in options.join.split(","):
ircchannel = bodyDict['ircchannel']

@@ -68,7 +68,7 @@ def on_message(self, body, message):
# process valid message
# see where we send this alert
channel = options.default_alert_channel
if 'ircchannel' in body_dict.keys():
if 'ircchannel' in body_dict:
if body_dict['ircchannel'] in options.channels:
channel = body_dict['ircchannel']

@@ -437,7 +437,7 @@ def fetch_auth0_logs(config, headers, fromid):
ret = r.json()

# Sometimes API give us the requested totals.. sometimes not.
if (type(ret) is dict) and ('logs' in ret.keys()):
if type(ret) is dict and 'logs' in ret:
have_totals = True
all_msgs = ret['logs']
else:
@@ -298,7 +298,7 @@ def broadcastAttacker(attacker):
# generate an 'alert' structure for this attacker:
mqAlert = dict(severity='NOTICE', category='attacker')

if 'datecreated' in attacker.keys():
if 'datecreated' in attacker:
mqAlert['utctimestamp'] = attacker['datecreated'].isoformat()

mqAlert['summary'] = 'New Attacker: {0} events: {1}, alerts: {2}'.format(attacker['indicators'], attacker['eventscount'], attacker['alertscount'])
@@ -359,19 +359,19 @@ def updateAttackerGeoIP(mozdefdb, attackerID, eventDictionary):
# "continent": "EU"
# }
# logger.debug(eventDictionary)
if 'details' in eventDictionary.keys():
if 'details' in eventDictionary:
if 'sourceipgeolocation' in eventDictionary['details']:
attackers=mozdefdb['attackers']
attacker = attackers.find_one({'_id': attackerID})
if attacker is not None:
attacker['geocoordinates'] = dict(countrycode='',
longitude=0,
latitude=0)
if 'country_code' in eventDictionary['details']['sourceipgeolocation'].keys():
if 'country_code' in eventDictionary['details']['sourceipgeolocation']:
attacker['geocoordinates']['countrycode'] = eventDictionary['details']['sourceipgeolocation']['country_code']
if 'longitude' in eventDictionary['details']['sourceipgeolocation'].keys():
if 'longitude' in eventDictionary['details']['sourceipgeolocation']:
attacker['geocoordinates']['longitude'] = eventDictionary['details']['sourceipgeolocation']['longitude']
if 'latitude' in eventDictionary['details']['sourceipgeolocation'].keys():
if 'latitude' in eventDictionary['details']['sourceipgeolocation']:
attacker['geocoordinates']['latitude'] = eventDictionary['details']['sourceipgeolocation']['latitude']
attackers.save(attacker)
else:
@@ -88,8 +88,8 @@ def esSearch(es, macassignments=None):
for r in results:
fields = re.search(usermacre,r['_source']['summary'])
if fields:
if '{0} {1}'.format(fields.group('username'),fields.group('macaddress')) not in correlations.keys():
if fields.group('macaddress')[0:8].lower() in macassignments.keys():
if '{0} {1}'.format(fields.group('username'),fields.group('macaddress')) not in correlations:
if fields.group('macaddress')[0:8].lower() in macassignments:
entity=macassignments[fields.group('macaddress')[0:8].lower()]
else:
entity='unknown'
@@ -82,7 +82,7 @@ def process_events(mozmsg, duo_events, etype, state):
return

# Care for API v2
if isinstance(duo_events, dict) and "authlogs" in duo_events.keys():
if isinstance(duo_events, dict) and "authlogs" in duo_events:
offset = duo_events["metadata"]["next_offset"]
if offset is not None:
state["{}_offset".format(etype)] = offset
@@ -137,7 +137,7 @@ def main():

# fix up the event craziness to a flatter format
events=[]
if 'items' in response.keys():
if 'items' in response:
for i in response['items']:
# flatten the sub dict/lists to pull out the good parts
event=dict(category='google')
@@ -157,17 +157,17 @@ def main():

# find important keys
# and adjust their location/name
if 'ipaddress' in details.keys():
if 'ipaddress' in details:
# it's the source ip
details['sourceipaddress']=details['ipaddress']
del details['ipaddress']

if 'id_time' in details.keys():
if 'id_time' in details:
event['timestamp']=details['id_time']
event['utctimestamp']=details['id_time']
if 'events_name' in details.keys():
if 'events_name' in details:
event['summary']+= details['events_name'] + ' '
if 'actor_email' in details.keys():
if 'actor_email' in details:
event['summary']+= details['actor_email'] + ' '

event['details']=details
@@ -102,13 +102,13 @@ def main():
healthlog['details']['total_messages_ready'] = 0
healthlog['tags'] = ['mozdef', 'status']
for m in mq:
if 'message_stats' in m.keys() and isinstance(m['message_stats'], dict):
if 'messages_ready' in m.keys():
if 'message_stats' in m and isinstance(m['message_stats'], dict):
if 'messages_ready' in m:
mready = m['messages_ready']
healthlog['details']['total_messages_ready'] += m['messages_ready']
else:
mready = 0
if 'messages_unacknowledged' in m.keys():
if 'messages_unacknowledged' in m:
munack = m['messages_unacknowledged']
else:
munack = 0
@@ -118,13 +118,13 @@ def main():
messages_ready=mready,
messages_unacknowledged=munack)

if 'deliver_details' in m['message_stats'].keys():
if 'deliver_details' in m['message_stats']:
queueinfo['deliver_eps'] = round(m['message_stats']['deliver_details']['rate'], 2)
healthlog['details']['total_deliver_eps'] += round(m['message_stats']['deliver_details']['rate'], 2)
if 'deliver_no_ack_details' in m['message_stats'].keys():
if 'deliver_no_ack_details' in m['message_stats']:
queueinfo['deliver_eps'] = round(m['message_stats']['deliver_no_ack_details']['rate'], 2)
healthlog['details']['total_deliver_eps'] += round(m['message_stats']['deliver_no_ack_details']['rate'], 2)
if 'publish_details' in m['message_stats'].keys():
if 'publish_details' in m['message_stats']:
queueinfo['publish_eps'] = round(m['message_stats']['publish_details']['rate'], 2)
healthlog['details']['total_publish_eps'] += round(m['message_stats']['publish_details']['rate'], 2)
healthlog['details']['queues'].append(queueinfo)
@@ -85,15 +85,15 @@ def main():
if r.status_code == 200:
oktaevents = json.loads(r.text)
for event in oktaevents:
if 'published' in event.keys():
if 'published' in event:
if toUTC(event['published']) > toUTC(state.data['lastrun']):
try:
mozdefEvent = dict()
mozdefEvent['utctimestamp']=toUTC(event['published']).isoformat()
mozdefEvent['receivedtimestamp']=toUTC(datetime.now()).isoformat()
mozdefEvent['category'] = 'okta'
mozdefEvent['tags'] = ['okta']
if 'action' in event.keys() and 'message' in event['action'].keys():
if 'action' in event and 'message' in event['action']:
mozdefEvent['summary'] = event['action']['message']
mozdefEvent['details'] = event
# Actor parsing
@@ -102,14 +102,14 @@ def main():
# This means the last instance of each attribute in all actors will be recorded in mozdef
# while others will be discarded
# Which ends up working out well in Okta's case.
if 'actors' in event.keys():
if 'actors' in event:
for actor in event['actors']:
if 'ipAddress' in actor.keys():
if 'ipAddress' in actor:
if netaddr.valid_ipv4(actor['ipAddress']):
mozdefEvent['details']['sourceipaddress'] = actor['ipAddress']
if 'login' in actor.keys():
if 'login' in actor:
mozdefEvent['details']['username'] = actor['login']
if 'requestUri' in actor.keys():
if 'requestUri' in actor:
mozdefEvent['details']['source_uri'] = actor['requestUri']

# We are renaming action to activity because there are
@@ -118,22 +118,22 @@ def makeEvents():
for event in events[target:target + 1]:
event['timestamp'] = pytz.timezone('UTC').localize(datetime.utcnow()).isoformat()
# remove stored times
if 'utctimestamp' in event.keys():
if 'utctimestamp' in event:
del event['utctimestamp']
if 'receivedtimestamp' in event.keys():
if 'receivedtimestamp' in event:
del event['receivedtimestamp']

# add demo to the tags so it's clear it's not real data.
if 'tags' not in event.keys():
if 'tags' not in event:
event['tags'] = list()

event['tags'].append('demodata')

# replace potential <randomipaddress> with a random ip address
if 'summary' in event.keys() and '<randomipaddress>' in event['summary']:
if 'summary' in event and '<randomipaddress>' in event['summary']:
randomIP = genRandomIPv4()
event['summary'] = event['summary'].replace("<randomipaddress>", randomIP)
if 'details' not in event.keys():
if 'details' not in event:
event['details'] = dict()
event['details']['sourceipaddress'] = randomIP
event['details']['sourceipv4address'] = randomIP
@@ -182,28 +182,28 @@ def makeAlerts():
for event in events[target:target + 1]:
event['timestamp'] = pytz.timezone('UTC').localize(datetime.utcnow()).isoformat()
# remove stored times
if 'utctimestamp' in event.keys():
if 'utctimestamp' in event:
del event['utctimestamp']
if 'receivedtimestamp' in event.keys():
if 'receivedtimestamp' in event:
del event['receivedtimestamp']

# add demo to the tags so it's clear it's not real data.
if 'tags' not in event.keys():
if 'tags' not in event:
event['tags'] = list()

event['tags'].append('demodata')
event['tags'].append('demoalert')

# replace potential <randomipaddress> with a random ip address
if 'summary' in event.keys() and '<randomipaddress>' in event['summary']:
if 'summary' in event and '<randomipaddress>' in event['summary']:
randomIP = genRandomIPv4()
event['summary'] = event['summary'].replace("<randomipaddress>", randomIP)
if 'details' not in event.keys():
if 'details' not in event:
event['details'] = dict()
event['details']['sourceipaddress'] = randomIP
event['details']['sourceipv4address'] = randomIP

if 'duplicate' in event.keys():
if 'duplicate' in event:
# send this event multiple times to trigger an alert
for x in range(0, int(event['duplicate'])):
logcache.put(json.dumps(event))
@@ -252,28 +252,28 @@ def makeAttackers():
for event in events[target:target + 1]:
event['timestamp'] = pytz.timezone('UTC').localize(datetime.utcnow()).isoformat()
# remove stored times
if 'utctimestamp' in event.keys():
if 'utctimestamp' in event:
del event['utctimestamp']
if 'receivedtimestamp' in event.keys():
if 'receivedtimestamp' in event:
del event['receivedtimestamp']

# add demo to the tags so it's clear it's not real data.
if 'tags' not in event.keys():
if 'tags' not in event:
event['tags'] = list()

event['tags'].append('demodata')
event['tags'].append('demoalert')

# replace potential <randomipaddress> with a random ip address
if 'summary' in event.keys() and '<randomipaddress>' in event['summary']:
if 'summary' in event and '<randomipaddress>' in event['summary']:
randomIP = genAttackerIPv4()
event['summary'] = event['summary'].replace("<randomipaddress>", randomIP)
if 'details' not in event.keys():
if 'details' not in event:
event['details'] = dict()
event['details']['sourceipaddress'] = randomIP
event['details']['sourceipv4address'] = randomIP

if 'duplicate' in event.keys():
if 'duplicate' in event:
# send this event multiple times to trigger an alert
for x in range(0, int(event['duplicate'])):
logcache.put(json.dumps(event))
@@ -1,15 +1,15 @@
def isCEF(aDict):
# determine if this is a CEF event
# could be an event posted to the /cef http endpoint
if 'endpoint' in aDict.keys() and aDict['endpoint'] == 'cef':
if 'endpoint' in aDict and aDict['endpoint'] == 'cef':
return True
# maybe it snuck in some other way
# check some key CEF indicators (the header fields)
if 'fields' in aDict.keys() and isinstance(aDict['fields'], dict):
if 'fields' in aDict and isinstance(aDict['fields'], dict):
lowerKeys = [s.lower() for s in aDict['fields'].keys()]
if 'devicevendor' in lowerKeys and 'deviceproduct' in lowerKeys and 'deviceversion' in lowerKeys:
return True
if 'details' in aDict.keys() and isinstance(aDict['details'], dict):
if 'details' in aDict and isinstance(aDict['details'], dict):
lowerKeys = [s.lower() for s in aDict['details'].keys()]
if 'devicevendor' in lowerKeys and 'deviceproduct' in lowerKeys and 'deviceversion' in lowerKeys:
return True
Oops, something went wrong.

0 comments on commit 4190c8d

Please sign in to comment.
You can’t perform that action at this time.
You signed in with another tab or window. Reload to refresh your session. You signed out in another tab or window. Reload to refresh your session.