Permalink
Browse files

Naming Convention and Logging Changes.

  • Loading branch information...
Phrozyn committed Oct 4, 2017
1 parent ede4866 commit 1fd73353551dbda986bc457f97167929fbba20b9
Showing with 123 additions and 139 deletions.
  1. +0 −16 alerts/alertPlugins.ini
  2. +16 −0 alerts/alert_plugins.ini
  3. 0 alerts/{alertWorker.conf → alert_worker.conf}
  4. 0 alerts/{alertWorker.py → alert_worker.py}
  5. 0 alerts/{amoFailedLogins.py → amo_failed_logins.py}
  6. 0 alerts/{bugzillaauthbruteforce.conf → bugzilla_auth_bruteforce.conf}
  7. +1 −1 alerts/{bugzillaauthbruteforce.py → bugzilla_auth_bruteforce.py}
  8. 0 alerts/{fxaAlerts.py → fxa_alerts.py}
  9. 0 alerts/{hostScannerAlerts.py → host_scanner_alerts.py}
  10. 0 alerts/{httpauthbruteforce.conf → http_auth_bruteforce.conf}
  11. +1 −1 alerts/{httpauthbruteforce.py → http_auth_bruteforce.py}
  12. 0 alerts/{httperrors.conf → http_errors.conf}
  13. +1 −1 alerts/{httperrors.py → http_errors.py}
  14. 0 alerts/{ldapAdd.py → ldap_add.py}
  15. 0 alerts/{ldapDelete.py → ldap_delete.py}
  16. 0 alerts/{ldapGroup.py → ldap_group.py}
  17. 0 alerts/{ldapLockout.py → ldap_lockout.py}
  18. 0 alerts/{sshbruteforce_bro.conf → ssh_bruteforce_bro.conf}
  19. +1 −1 alerts/{sshbruteforce_bro.py → ssh_bruteforce_bro.py}
  20. 0 alerts/{sshioc.py → ssh_ioc.py}
  21. 0 alerts/{sshkey.conf → ssh_key.conf}
  22. +2 −2 alerts/{sshkey.py → ssh_key.py}
  23. 0 alerts/{supervisord.alerts.ini → supervisord_alerts.ini}
  24. +1 −16 config/50-mozdef-filter.conf
  25. +1 −1 config/logrotate-mongod
  26. +1 −1 config/logrotate-mozdef
  27. +2 −2 config/logrotate-nginx
  28. +1 −1 config/logrotate-supervisord
  29. 0 mq/{esworker.cloudtrail.conf → esworker_cloudtrail.conf}
  30. 0 mq/{esworker.cloudtrail.py → esworker_cloudtrail.py}
  31. 0 mq/{esworker.conf → esworker_eventtask.conf}
  32. 0 mq/{esworker.py → esworker_eventtask.py}
  33. 0 mq/{esworker.papertrail.conf → esworker_papertrail.conf}
  34. +1 −1 mq/{esworker.papertrail.py → esworker_papertrail.py}
  35. 0 mq/{esworker.sqs.conf → esworker_sqs.conf}
  36. +1 −1 mq/{esworker.sqs.py → esworker_sqs.py}
  37. +25 −0 mq/eventtask.ini
  38. +0 −25 mq/mqwSyslog.ini
  39. 0 mq/plugins/{mozillaLocation.conf → mozilla_location.conf}
  40. +42 −42 mq/plugins/{mozillaLocation.py → mozilla_location.py}
  41. 0 rest/plugins/{facebookThreatExchange.conf → facebook_threatexchange.conf}
  42. +1 −1 rest/plugins/{facebookThreatExchange.py → facebook_threatexchange.py}
  43. 0 rest/plugins/{vpcblackhole.conf → vpc_blackhole.conf}
  44. +10 −12 rest/plugins/{vpcblackhole.py → vpc_blackhole.py}
  45. +3 −3 systemdfiles/alert/mozdefalertplugins.service
  46. +2 −2 systemdfiles/alert/mozdefalerts.service
  47. +1 −1 systemdfiles/alert/mozdefbot.service
  48. +1 −1 systemdfiles/alert/mozdefloginput.service
  49. +1 −1 systemdfiles/consumer/mozdefloginput.service
  50. +6 −5 systemdfiles/consumer/{mozdefmqwsyslog.service → mworker-eventtask.service}
  51. +1 −1 systemdfiles/web/mozdefrestapi.service
View

This file was deleted.

Oops, something went wrong.
View
@@ -0,0 +1,16 @@
[uwsgi]
chdir = /opt/mozdef/envs/mozdef/alerts/
uid = mozdef
mule = alert_worker.py
pyargv = -c /opt/mozdef/envs/mozdef/alerts/alert_worker.conf
log-syslog = alertplugins-worker
log-drain = generated 0 bytes
socket = /opt/mozdef/envs/mozdef/alerts/alert_plugins.socket
virtualenv = /opt/mozdef/envs/mozdef/
master-fifo = /opt/mozdef/envs/mozdef/alerts/alert_plugins.fifo
procname-master = [m]
procname-prefix = [alertplugins]
never-swap
pidfile= /var/run/mozdef-alerts/alert_plugins.pid
vacuum = true
enable-threads
File renamed without changes.
File renamed without changes.
File renamed without changes.
@@ -15,7 +15,7 @@
class AlertBugzillaPBruteforce(AlertTask):
def main(self):
self.parse_config('bugzillaauthbruteforce.conf', ['url'])
self.parse_config('bugzilla_auth_bruteforce.conf', ['url'])
search_query = SearchQuery(minutes=15)
search_query.add_must([
File renamed without changes.
File renamed without changes.
File renamed without changes.
@@ -15,7 +15,7 @@
class AlertHTTPBruteforce(AlertTask):
def main(self):
self.parse_config('httpauthbruteforce.conf', ['url'])
self.parse_config('http_auth_bruteforce.conf', ['url'])
search_query = SearchQuery(minutes=15)
search_query.add_must([
File renamed without changes.
@@ -15,7 +15,7 @@
class AlertHTTPErrors(AlertTask):
def main(self):
self.parse_config('httperrors.conf', ['url'])
self.parse_config('http_errors.conf', ['url'])
search_query = SearchQuery(minutes=15)
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
@@ -15,7 +15,7 @@
class AlertSSHManyConns(AlertTask):
def main(self):
self.parse_config('sshbruteforce_bro.conf', ['url'])
self.parse_config('ssh_bruteforce_bro.conf', ['url'])
search_query = SearchQuery(minutes=15)
File renamed without changes.
File renamed without changes.
@@ -13,7 +13,7 @@
import re
# This alert consumes data produced by the MIG sshkey module and mig-runner.
# sshkey related events are compared against a whitelist which is the
# ssh key related events are compared against a whitelist which is the
# alerts configuration file (sshkey.conf). The format of this whitelist
# is as follows:
#
@@ -30,7 +30,7 @@ def __init__(self):
self._whitelist = []
AlertTask.__init__(self)
self._parse_whitelist('sshkey.conf')
self._parse_whitelist('ssh_key.conf')
def main(self):
search_query = SearchQuery(minutes=30)
File renamed without changes.
@@ -1,23 +1,8 @@
if $programname == 'mozdefbot-worker' then /var/log/mozdef/mozdefbot.log
if $programname == 'loginput-worker' then /var/log/mozdef/loginput.log
if $programname == 'infosecsqs-worker' then /var/log/mozdef/infosecsqs.log
if $programname == 'restapi-worker' then /var/log/mozdef/restapi.log
if $programname == 'syslog-worker' then /var/log/mozdef/syslog.log
if $programname == 'nubis-worker' then /var/log/mozdef/nubis.log
if $programname == 'bro-worker' then /var/log/mozdef/bro.log
if $programname == 'migsqs-worker' then /var/log/mozdef/migsqs.log
if $programname == 'parsyssqs-worker' then /var/log/mozdef/parsyssqs.log
if $programname == 'autoland-worker' then /var/log/mozdef/autoland.log
if $programname == 'contegix-worker' then /var/log/mozdef/contegix.log
if $programname == 'deis-worker' then /var/log/mozdef/deis.log
if $programname == 'releng-worker' then /var/log/mozdef/releng.log
if $programname == 'fxa-worker' then /var/log/mozdef/fxa.log
if $programname == 'httpobs-worker' then /var/log/mozdef/httpobs.log
if $programname == 'riskheatmap-worker' then /var/log/mozdef/riskheatmap.log
if $programname == 'ssosqs-worker' then /var/log/mozdef/sso.log
if $programname == 'cloudtrail-worker' then /var/log/mozdef/cloudtrail.log
if $programname == 'eventtask-worker' then /var/log/mozdef/eventtask.log
if $programname == 'alertplugins-worker' then /var/log/mozdef/alertplugins.log
if $programname == 'contegix-auditd-worker' then /var/log/mozdef/contegix-auditd.log
if $programname == 'mongod.3002' then /var/log/mozdef/mongo/meteor-mongo.log
if $programname == 'mongod' then /var/log/mozdef/mongo/mongo.log
if $programname == 'kibana4' then /var/log/mozdef/kibana.log
View
@@ -1,6 +1,6 @@
/var/log/mozdef/mongo/*.log
{
rotate 17
rotate 4
weekly
missingok
notifempty
View
@@ -1,6 +1,6 @@
/var/log/mozdef/*.log
{
rotate 17
rotate 4
weekly
missingok
notifempty
View
@@ -1,7 +1,7 @@
/var/log/mozdef/nginx/*.log {
/var/log/mozdef/nginx/*.error_log {
weekly
missingok
rotate 17
rotate 4
compress
delaycompress
notifempty
@@ -2,7 +2,7 @@
{
copytruncate
dateext
rotate 17
rotate 4
weekly
missingok
notifempty
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
@@ -9,7 +9,7 @@
# Aaron Meihm ameihm@mozilla.com
# Reads from papertrail using the API and inserts log data into ES in
# the same manner as esworker.py
# the same manner as esworker_eventtask.py
import json
File renamed without changes.
@@ -11,7 +11,7 @@
# kombu's support for SQS is buggy
# so this version uses boto
# to read an SQS queue and put events into elastic search
# in the same manner as esworker.py
# in the same manner as esworker_eventtask.py
import json
View
@@ -0,0 +1,25 @@
[uwsgi]
chdir = /opt/mozdef/envs/mozdef/mq/
uid = mozdef
mule = esworker_eventtask.py
mule = esworker_eventtask.py
mule = esworker_eventtask.py
mule = esworker_eventtask.py
mule = esworker_eventtask.py
mule = esworker_eventtask.py
mule = esworker_eventtask.py
mule = esworker_eventtask.py
mule = esworker_eventtask.py
mule = esworker_eventtask.py
pyargv = -c /opt/mozdef/envs/mozdef/mq/esworker_eventtask.conf
log-syslog = eventtask-worker
log-drain = generated 0 bytes
socket = /opt/mozdef/envs/mozdef/mq/eventtask.socket
virtualenv = /opt/mozdef/envs/mozdef/
procname-master = [m]
procname-prefix = [eventtask]
master-fifo = /opt/mozdef/envs/mozdef/mq/eventtask.fifo
never-swap
pidfile = /var/run/mozdefmqwSyslog/eventtask.pid
vacuum = true
enable-threads
View

This file was deleted.

Oops, something went wrong.
File renamed without changes.
@@ -1,42 +1,42 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Copyright (c) 2017 Mozilla Corporation
#
# Contributors:
# Arzhel Younsi arzhel@mozilla.com
# Jeff Bryner jbryner@mozilla.com
# Brandon Myers bmyers@mozilla.com
import os
from configlib import getConfig
class message(object):
def __init__(self):
'''
this plugin takes a source hostname of form
host.private.site.mozilla.com
extracts the site, adds it and compares the site
to a list of known datacenters or offices and adds that metadata
'''
self.registration = ['network', 'netflow']
self.priority = 5
config_location = os.path.join(os.path.dirname(os.path.abspath(__file__)), "mozillaLocation.conf")
self.dc_code_list = getConfig('dc_code_list', '', config_location).split(',')
self.offices_code_list = getConfig('offices_code_list', '', config_location).split(',')
def onMessage(self, message, metadata):
if 'details' in message.keys() and 'hostname' in message['details'].keys():
hostnamesplit = str.lower(message['details']['hostname'].encode('ascii', 'ignore')).split('.')
if len(hostnamesplit) == 5:
if 'mozilla' == hostnamesplit[-2]:
message['details']['site'] = hostnamesplit[-3]
if message['details']['site'] in self.dc_code_list:
message['details']['sitetype'] = 'datacenter'
elif message['details']['site'] in self.offices_code_list:
message['details']['sitetype'] = 'office'
else:
message['details']['sitetype'] = 'unknown'
return (message, metadata)
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Copyright (c) 2017 Mozilla Corporation
#
# Contributors:
# Arzhel Younsi arzhel@mozilla.com
# Jeff Bryner jbryner@mozilla.com
# Brandon Myers bmyers@mozilla.com
import os
from configlib import getConfig
class message(object):
def __init__(self):
'''
this plugin takes a source hostname of form
host.private.site.mozilla.com
extracts the site, adds it and compares the site
to a list of known datacenters or offices and adds that metadata
'''
self.registration = ['network', 'netflow']
self.priority = 5
config_location = os.path.join(os.path.dirname(os.path.abspath(__file__)), "mozilla_location.conf")
self.dc_code_list = getConfig('dc_code_list', '', config_location).split(',')
self.offices_code_list = getConfig('offices_code_list', '', config_location).split(',')
def onMessage(self, message, metadata):
if 'details' in message.keys() and 'hostname' in message['details'].keys():
hostnamesplit = str.lower(message['details']['hostname'].encode('ascii', 'ignore')).split('.')
if len(hostnamesplit) == 5:
if 'mozilla' == hostnamesplit[-2]:
message['details']['site'] = hostnamesplit[-3]
if message['details']['site'] in self.dc_code_list:
message['details']['sitetype'] = 'datacenter'
elif message['details']['site'] in self.offices_code_list:
message['details']['sitetype'] = 'office'
else:
message['details']['sitetype'] = 'unknown'
return (message, metadata)
@@ -64,7 +64,7 @@ def __init__(self):
# set my own conf file
# relative path to the rest index.py file
self.configfile = './plugins/facebookThreatExchange.conf'
self.configfile = './plugins/facebook_threatexchange.conf'
self.options = None
if os.path.exists(self.configfile):
sys.stdout.write('found conf file {0}\n'.format(self.configfile))
File renamed without changes.
@@ -43,18 +43,17 @@ def __init__(self):
(i.e. blockip matches /blockip)
set the priority if you have a preference for order of plugins
0 goes first, 100 is assumed/default if not sent
Plugins will register in Meteor with attributes:
name: (as below)
description: (as below)
priority: (as below)
file: "plugins.filename" where filename.py is the plugin code.
Plugin gets sent main rest options as:
self.restoptions
self.restoptions['configfile'] will be the .conf file
used by the restapi's index.py file.
'''
self.registration = ['blockip']
@@ -64,13 +63,12 @@ def __init__(self):
# set my own conf file
# relative path to the rest index.py file
self.configfile = './plugins/vpcblackhole.conf'
self.configfile = './plugins/vpc_blackhole.conf'
self.options = None
self.multioptions = []
if os.path.exists(self.configfile):
sys.stdout.write('found conf file {0}\n'.format(self.configfile))
self.initConfiguration()
def initConfiguration(self):
myparser = ConfigParser.ConfigParser()
@@ -81,7 +79,7 @@ def initConfiguration(self):
cur_options = myparser.options(cur_section)
if cur_options is not None:
self.multioptions.append({ 'region': myparser.get(cur_section, 'region'), 'aws_access_key_id': myparser.get(cur_section, 'aws_access_key_id'), 'aws_secret_access_key': myparser.get(cur_section, 'aws_secret_access_key') } )
def addBlackholeEntry(self,
ipaddress=None):
try:
@@ -154,7 +152,7 @@ def addBlackholeEntry(self,
else:
sys.stdout.write('Skipping route table {0} in the VPC {1} - blackhole ENI could not be found\n'.format(rt_id, vpc_id))
continue
except Exception as e:
sys.stderr.write('Error while creating a blackhole entry %s: %r\n' % (ipaddress, e))
@@ -163,16 +161,16 @@ def onMessage(self, request, response):
'''
request: http://bottlepy.org/docs/dev/api.html#the-request-object
response: http://bottlepy.org/docs/dev/api.html#the-response-object
'''
'''
# format/validate request.json:
ipaddress = None
CIDR = None
sendToBHVPC = False
# loop through the fields of the form
# and fill in our values
try:
try:
for i in request.json:
# were we checked?
if self.name in i.keys():
@@ -197,5 +195,5 @@ def onMessage(self, request, response):
sys.stdout.write ('Blackholed {0}\n'.format(ipaddress))
except Exception as e:
sys.stderr.write('Error handling request.json %r \n'% (e))
return (request, response)
Oops, something went wrong.

0 comments on commit 1fd7335

Please sign in to comment.