Skip to content
Permalink
Browse files

Merge branch 'master' into themed-css

  • Loading branch information...
Phrozyn committed Mar 28, 2019
2 parents eb03ec8 + 0d13b62 commit d3ecd92c348ab7e86cd7f56b2a90e538c7a6acf6
Showing with 1,860 additions and 716 deletions.
  1. +1 −1 .gitignore
  2. +23 −0 CHANGELOG
  3. +6 −4 Makefile
  4. 0 alerts/actions/__init__.py
  5. 0 alerts/{plugins → actions}/dashboard_geomodel.json
  6. 0 alerts/{plugins → actions}/dashboard_geomodel.py
  7. 0 alerts/{plugins → actions}/pagerDutyTriggerEvent.conf
  8. +2 −2 alerts/{plugins → actions}/pagerDutyTriggerEvent.py
  9. +16 −0 alerts/alert_actions.ini
  10. 0 alerts/{alert_worker.conf → alert_actions_worker.conf}
  11. +10 −9 alerts/{alert_worker.py → alert_actions_worker.py}
  12. +0 −16 alerts/alert_plugins.ini
  13. +2 −2 alerts/celeryconfig.py
  14. +1 −1 alerts/duo_authfail.py
  15. +8 −18 alerts/generic_alert_loader.py
  16. +4 −0 alerts/get_watchlist.conf
  17. +90 −0 alerts/get_watchlist.py
  18. +34 −8 alerts/lib/alerttask.py
  19. +4 −0 alerts/lib/config.py
  20. +3 −3 alerts/multiple_intel_hits.py
  21. +20 −21 alerts/proxy_drop_executable.py
  22. +38 −28 alerts/proxy_drop_ip.py
  23. +20 −26 alerts/proxy_drop_non_standard_port.py
  24. +14 −29 alerts/proxy_exfil_domains.py
  25. +5 −5 bot/irc/mozdefbot.py
  26. +1 −1 bot/slack/mozdefbot.py
  27. +1 −1 config/50-mozdef-filter.conf
  28. +1 −1 cron/auth02mozdef.py
  29. +5 −5 cron/collectAttackers.py
  30. +2 −2 cron/correlateUserMacAddress.py
  31. +26 −6 cron/duo_logpull.py
  32. +5 −5 cron/google2mozdef.py
  33. +6 −6 cron/healthAndStatus.py
  34. +6 −6 cron/okta2mozdef.py
  35. +6 −0 cron/rotateIndexes.py
  36. +3 −3 docker/compose/docker-compose-cloudy-mozdef.yml
  37. +1 −1 docker/compose/docker-compose-user-env.yml
  38. +7 −8 docker/compose/docker-compose.yml
  39. +1 −1 docker/compose/{mozdef_alertplugins → mozdef_alertactions}/Dockerfile
  40. 0 ...ozdef_alertplugins/files/alert_worker.conf → mozdef_alertactions/files/alert_actions_worker.conf}
  41. +4 −0 docker/compose/mozdef_alerts/files/config.py
  42. +5 −4 docker/compose/mozdef_bootstrap/files/initial_setup.py
  43. +1 −1 docker/compose/mozdef_cron/files/backup.conf
  44. +7 −8 docker/compose/mozdef_meteor/Dockerfile
  45. +5 −3 docker/compose/rabbitmq/Dockerfile
  46. +10 −0 docker/compose/rabbitmq/files/erlang.repo
  47. +10 −0 docker/compose/rabbitmq/files/rabbitmq-server.repo
  48. +5 −5 docs/source/installation.rst
  49. +17 −17 examples/demo/sampleData2MozDef.py
  50. +166 −0 meteor/client/addwatchItem.html
  51. +63 −0 meteor/client/addwatchItem.js
  52. +4 −0 meteor/client/layout.js
  53. +4 −0 meteor/client/main.js
  54. +6 −2 meteor/client/menu.html
  55. +0 −1 meteor/client/mozdef.html
  56. +11 −6 meteor/client/mozdef.js
  57. +14 −0 meteor/client/router.js
  58. +166 −0 meteor/client/watchItem.html
  59. +63 −0 meteor/client/watchItem.js
  60. +54 −0 meteor/client/watchlistTable.html
  61. +40 −0 meteor/client/watchlistTable.js
  62. +21 −0 meteor/imports/collections.js
  63. +12 −0 meteor/server/methods.js
  64. +11 −0 mozdef_util/HISTORY.rst
  65. +25 −15 mozdef_util/mozdef_util/bulk_queue.py
  66. +6 −8 mozdef_util/mozdef_util/elasticsearch_client.py
  67. +20 −16 mozdef_util/mozdef_util/plugin_set.py
  68. +3 −3 mozdef_util/mozdef_util/utilities/is_cef.py
  69. +1 −1 mozdef_util/setup.py
  70. +18 −21 mq/esworker_cloudtrail.py
  71. +7 −11 mq/esworker_eventtask.py
  72. +7 −12 mq/esworker_papertrail.py
  73. +1 −8 mq/esworker_sns_sqs.py
  74. +6 −11 mq/esworker_sqs.py
  75. +1 −1 mq/plugins/alert_information.py
  76. +8 −8 mq/plugins/auditdFixup.py
  77. +5 −5 mq/plugins/broFixup.py
  78. +1 −0 mq/plugins/cloudtrail.py
  79. +4 −4 mq/plugins/complianceitems.py
  80. +1 −1 mq/plugins/customDocType.py
  81. +1 −1 mq/plugins/dropMessage.py
  82. +6 −6 mq/plugins/fluentdSqsFixup.py
  83. +8 −8 mq/plugins/fxaFixup.py
  84. +3 −3 mq/plugins/geoip.py
  85. +3 −3 mq/plugins/github_webhooks.py
  86. +2 −2 mq/plugins/guardDuty.py
  87. +13 −13 mq/plugins/ipFixup.py
  88. +2 −2 mq/plugins/mozilla_location.py
  89. +3 −3 mq/plugins/observium.py
  90. +47 −31 mq/plugins/parse_sshd.py
  91. +2 −2 mq/plugins/rt_flow.py
  92. +3 −3 mq/plugins/snmptt.py
  93. +3 −3 mq/plugins/sshdFindIP.py
  94. +5 −5 mq/plugins/vulnerability.py
  95. +2 −1 requirements.txt
  96. +63 −4 rest/index.py
  97. +1 −1 rest/plugins/cymon.py
  98. +6 −6 rest/plugins/fqdnblocklist.py
  99. +6 −6 rest/plugins/ipblocklist.py
  100. +2 −2 rest/plugins/vpc_blackhole.py
  101. +3 −0 rest/plugins/watchlist.conf
  102. +157 −0 rest/plugins/watchlist.py
  103. +3 −3 systemdfiles/alert/{mozdefalertplugins.service → mozdefalertactions.service}
  104. +1 −1 tests/alerts/{plugins → actions}/test_dashboard_geomodel.py
  105. +4 −3 tests/alerts/alert_test_suite.py
  106. +1 −1 tests/alerts/test_duo_authfail.py
  107. +23 −29 tests/alerts/test_proxy_drop_executable.py
  108. +25 −50 tests/alerts/test_proxy_drop_ip.py
  109. +26 −46 tests/alerts/test_proxy_drop_non_standard_port.py
  110. +35 −31 tests/alerts/test_proxy_exfil_domains.py
  111. +1 −1 tests/mozdef_util/query_models/query_test_suite.py
  112. +8 −8 tests/mozdef_util/query_models/test_aggregation.py
  113. +15 −15 tests/mozdef_util/query_models/test_search_query.py
  114. +9 −9 tests/mozdef_util/test_bulk_queue.py
  115. +16 −16 tests/mozdef_util/test_elasticsearch_client.py
  116. +57 −1 tests/mq/plugins/test_broFixup.py
  117. +101 −1 tests/mq/plugins/test_parse_sshd.py
  118. +1 −1 tests/mq/test_esworker_sns_sqs.py
  119. +2 −2 tests/rest/test_rest_index.py
  120. +6 −2 tests/unit_test_suite.py
@@ -16,4 +16,4 @@ alerts/generic_alerts
cloudy_mozdef/aws_parameters.json
cloudy_mozdef/aws_parameters.sh
docs/source/_build
docs/source/_static
docs/source/_static
@@ -0,0 +1,23 @@
# Changelog
All notable changes to this project will be documented in this file.

The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)

## [v1.37] - 2019-03-01
### Added
- Watchlist - use the UI to quickly add a term (username, IP, command, etc.) that MozDef alerts on
- Generic Deadman - use a simple config file to validate that expected events are appearing in a given time window (and alert an Error when they do not)

### Changed
- Improve error handling on Slack bot
- Improve Slack bot alert format for better readibility
- Minor UI adjustments

### Fixed
- Some Duo events were not correctly displaying the source IP address. It is now always the access device IP
- Fixed defaults for Slack bot to ensure more consistency each time it loads
- Added checks on sending SQS messages to only accept intra-account messages
- Improved docker performance and disk space requirements

[Unreleased Changes]: https://github.com/mozilla/MozDef/compare/v1.37...HEAD
[Releases prior to v1.37](https://github.com/mozilla/MozDef/releases)
@@ -5,7 +5,7 @@
#

ROOT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
DKR_IMAGES := mozdef_alertplugins mozdef_alerts mozdef_base mozdef_bootstrap mozdef_meteor mozdef_rest \
DKR_IMAGES := mozdef_alertactions mozdef_alerts mozdef_base mozdef_bootstrap mozdef_meteor mozdef_rest \
mozdef_mq_worker mozdef_loginput mozdef_cron mozdef_elasticsearch mozdef_mongodb \
mozdef_syslog mozdef_nginx mozdef_tester mozdef_rabbitmq mozdef_kibana
BUILD_MODE := build ## Pass `pull` in order to pull images instead of building them
@@ -33,9 +33,11 @@ run-cloudy-mozdef: ## Run the MozDef containers necessary to run in AWS (`cloudy

.PHONY: run-env-mozdef
run-env-mozdef: ## Run the MozDef containers with a user specified env file. Run with make 'run-env-mozdef -e ENV=my.env'
$(shell test -f $(ENV) || touch $(ENV))
$(eval ENV_FILE:=$(abspath $(ENV)))
docker-compose -f docker/compose/docker-compose.yml -f docker/compose/docker-compose-user-env.yml -p $(NAME) up -d
ifneq ("$(wildcard $(ENV))","") #Check for existence of ENV
ENV_FILE=$(abspath $(ENV)) docker-compose -f docker/compose/docker-compose.yml -f docker/compose/docker-compose-user-env.yml -p $(NAME) up -d
else
@echo $(ENV) not found.
endif

restart-cloudy-mozdef:
docker-compose -f docker/compose/docker-compose-cloudy-mozdef.yml -p $(NAME) restart
No changes.
@@ -45,9 +45,9 @@ def initConfiguration(self):
def onMessage(self, message):
# here is where you do something with the incoming alert message
doclink = 'unknown'
if message['category'] in self.options.docs.keys():
if message['category'] in self.options.docs:
doclink = self.options.docs[message['category']]
if 'summary' in message.keys():
if 'summary' in message:
headers = {
'Content-type': 'application/json',
}
@@ -0,0 +1,16 @@
[uwsgi]
chdir = /opt/mozdef/envs/mozdef/alerts/
uid = mozdef
mule = alert_actions_worker.py
pyargv = -c /opt/mozdef/envs/mozdef/alerts/alert_actions_worker.conf
log-syslog = alertactions-worker
log-drain = generated 0 bytes
socket = /opt/mozdef/envs/mozdef/alerts/alert_actions.socket
virtualenv = /opt/mozdef/envs/python/
master-fifo = /opt/mozdef/envs/mozdef/alerts/alert_actions.fifo
procname-master = [m]
procname-prefix = [alertactions]
never-swap
pidfile= /var/run/mozdef-alerts/alert_actions.pid
vacuum = true
enable-threads
@@ -5,8 +5,9 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Copyright (c) 2014 Mozilla Corporation
#
# Alert Worker to listen for alerts and call python plugins
# for user-controlled reaction to alerts.
# Alert Worker to listen for alerts and call python actions
# for a user-controlled reaction to alerts.
# This worker receives a copy of an alert.

import json
import os
@@ -16,15 +17,15 @@
from kombu.mixins import ConsumerMixin

from lib.alert_plugin_set import AlertPluginSet
from lib.config import ALERT_PLUGINS
from lib.config import ALERT_ACTIONS

from mozdef_util.utilities.logger import logger, initLogger


class alertConsumer(ConsumerMixin):
'''read in alerts,
compare them to plugins
and send alerts to plugins as requested
compare them to actions
and send alerts to actions as requested
'''

def __init__(self, mqAlertsConnection, alertQueue, alertExchange):
@@ -57,7 +58,7 @@ def on_message(self, body, message):
logger.exception("alertworker exception: unknown body type received %r" % body)
return
# process valid message
bodyDict = plugin_set.run_plugins(bodyDict)
bodyDict = action_set.run_plugins(bodyDict)

message.ack()
except ValueError as e:
@@ -75,7 +76,7 @@ def main():
)
mqAlertConn = Connection(mqConnString)

# Exchange for alerts we pass to plugins
# Exchange for alerts we pass to actions
alertExchange = Exchange(name=options.alertExchange,
type='topic',
durable=True,
@@ -140,7 +141,7 @@ def initConfig():
(options, args) = parser.parse_args()
initConfig()
initLogger(options)
plugin_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'plugins'))
plugin_set = AlertPluginSet(plugin_dir, ALERT_PLUGINS)
action_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'actions'))
action_set = AlertPluginSet(action_dir, ALERT_ACTIONS)

main()

This file was deleted.

Oops, something went wrong.
@@ -38,9 +38,9 @@
'options': {'queue': 'celery-default', "exchange": "celery-default"},
}
# add optional parameters:
if 'args' in ALERTS[alert].keys():
if 'args' in ALERTS[alert]:
CELERYBEAT_SCHEDULE[alert]['args']=ALERTS[alert]['args']
if 'kwargs' in ALERTS[alert].keys():
if 'kwargs' in ALERTS[alert]:
CELERYBEAT_SCHEDULE[alert]['kwargs']=ALERTS[alert]['kwargs']

# Load logging config
@@ -16,7 +16,7 @@ def main(self):
TermMatch('category', 'authentication'),
ExistsMatch('details.sourceipaddress'),
ExistsMatch('details.username'),
PhraseMatch('details.result', 'FRAUD')
PhraseMatch('details.result', 'fraud')
])

self.filtersManual(search_query)
@@ -9,12 +9,14 @@

from lib.alerttask import AlertTask
from mozdef_util.query_models import SearchQuery, TermMatch, QueryStringMatch
from mozdef_util.utilities.dot_dict import DotDict
from mozdef_util.utilities.logger import logger
import hjson
import logging
import sys
import traceback
import glob
import os
from os.path import basename

# Minimum data needed for an alert (this is an example alert json)
'''
@@ -55,23 +57,6 @@
'''


logger = logging.getLogger()
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')


class DotDict(dict):
'''dict.item notation for dict()'s'''
__getattr__ = dict.__getitem__
__setattr__ = dict.__setitem__
__delattr__ = dict.__delitem__

def __init__(self, dct):
for key, value in dct.items():
if hasattr(value, 'keys'):
value = DotDict(value)
self[key] = value


class AlertGenericLoader(AlertTask):
required_fields = [
"search_string",
@@ -103,11 +88,16 @@ def load_configs(self):
try:
cfg = DotDict(hjson.load(fd))
self.validate_alert(cfg)
# We set the alert name to the filename (excluding .json)
alert_name = basename(f).replace('.json', '')
cfg['custom_alert_name'] = alert_name
self.configs.append(cfg)
except Exception:
logger.error("Loading rule file {} failed".format(f))

def process_alert(self, alert_config):
# Set instance variable to populate event attributes about an alert
self.custom_alert_name = "{0}:{1}".format(self.classname(), alert_config['custom_alert_name'])
search_query = SearchQuery(minutes=int(alert_config.time_window))
terms = []
for i in alert_config.filters:
@@ -0,0 +1,4 @@
[options]
# set the following to your protected endpoint api_url
api_url = http://localhost:8081/getwatchlist
jwt_secret = secret
@@ -0,0 +1,90 @@
#!/usr/bin/env python

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Copyright (c) 2014 Mozilla Corporation


from lib.alerttask import AlertTask
from mozdef_util.query_models import SearchQuery, QueryStringMatch
import requests
import json
import logging
from requests_jwt import JWTAuth
import sys

logger = logging.getLogger()
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')


class AlertWatchList(AlertTask):
def main(self):
self.parse_config('get_watchlist.conf', ['api_url', 'jwt_secret'])

jwt_token = JWTAuth(self.config.jwt_secret)
jwt_token.set_header_format('Bearer %s')

# Connect to rest api and grab response
r = requests.get(self.config.api_url, auth=jwt_token)
status = r.status_code
index = 0
if status == 200:
status = r.status_code
response = r.text
terms_list = json.loads(response)
while index < len(terms_list):
term = terms_list[index]
term = '"{}"'.format(term)
self.watchterm = term
index += 1
self.process_alert(term)
else:
sys.stderr.write('The watchlist request failed. Status {0}.\n'.format(status))

def process_alert(self, term):
search_query = SearchQuery(minutes=20)
content = QueryStringMatch(str(term))
search_query.add_must(content)
self.filtersManual(search_query)
self.searchEventsSimple()
self.walkEvents()

# Set alert properties
def onEvent(self, event):
category = 'watchlist'
tags = ['watchtarget']
severity = 'WARNING'

ev = event['_source']
user = ''
sourceipaddress = ''
hostname = ''
source_data = ''
user_data = ''

# If the event severity is below what we want, just ignore
# the event.
if 'details' not in ev:
return None
if 'details' in ev:
if 'sourceipaddress' in ev['details']:
sourceipaddress = ev['details']['sourceipaddress']
source_data = 'from {}'.format(sourceipaddress)
if 'username' in ev['details'] or 'originaluser' in ev['details'] or 'user' in ev['details']:
if 'username' in ev['details']:
user = ev['details']['username']
user_data = 'by {}'.format(user)
elif 'originaluser' in ev['details']:
user = ev['details']['originaluser']
user_data = 'by {}'.format(user)
elif 'user' in ev['details']:
user = ev['details']['user']
user_data = 'by {}'.format(user)
if 'hostname' in ev:
hostname = ev['hostname']
else:
return None

summary = 'Watchlist term {} detected {} {} on {}'.format(self.watchterm, user_data, source_data, hostname)
return self.createAlertDict(summary, category, tags, [event], severity)
Oops, something went wrong.

0 comments on commit d3ecd92

Please sign in to comment.
You can’t perform that action at this time.