Merge 3fb74c7 into 937ad29

.gitignore

@@ -62,3 +62,6 @@ secmonkey.env
 *.key
 postgres-data/
 docker-compose.override.yml
+.cache/
+dart/pubspec.lock
+celerybeat-schedule

.travis.yml

@@ -48,11 +48,12 @@ matrix:
         - coverage run -a -m py.test security_monkey/tests/auditors || exit 1
         - coverage run -a -m py.test security_monkey/tests/watchers || exit 1
         - coverage run -a -m py.test security_monkey/tests/core || exit 1
+        - coverage run -a -m py.test security_monkey/tests/scheduling || exit 1
         - coverage run -a -m py.test security_monkey/tests/views || exit 1
         - coverage run -a -m py.test security_monkey/tests/interface || exit 1
         - coverage run -a -m py.test security_monkey/tests/utilities || exit 1
         - bandit -r -ll -ii -x security_monkey/tests .
-        - pylint -E -d E1101,E0611,F0401 --ignore=service.py,datastore.py,datastore_utils.py,watcher.py security_monkey
+        - pylint -E -d E1101,E0611,F0401 --ignore=service.py,datastore.py,datastore_utils.py,watcher.py,test_celery_scheduler.py security_monkey
 
       after_success:
         - coveralls

celeryconfig.py

@@ -0,0 +1,25 @@
+"""
+.. module: celeryconfig
+    :platform: Unix
+    :synopsis: Use this file to set up the Celery configuration for task scheduling.
+
+.. version:: $$VERSION$$
+.. moduleauthor:: Mike Grima <mgrima@netflix.com>
+
+"""
+# Broker source: Place yours here:
+broker_url = 'redis://localhost:6379/0'     # Default assume Redis on localhost
+
+# List of modules to import when the Celery worker starts.
+imports = ('security_monkey.task_scheduler.tasks',)
+
+# How many processes per worker instance?
+worker_concurrency = 10
+
+enable_utc = True
+
+###########################
+# IMPORTANT: This helps avoid memory leak issues - do not change this number!
+worker_max_tasks_per_child = 1
+############################
+

dart/lib/component/settings/watcher_config_component/watcher_config_component.html

@@ -1,7 +1,7 @@
 <div>
   <div class="col-lg-12" ng-switch="isLoaded">
     <div class="panel panel-info">
-      <div class="panel-heading">Users <span class="badge pull-right">{{ items_displayed() }} of {{ totalItems }}</span></div>
+      <div class="panel-heading">Watcher Configuration <span class="badge pull-right">{{ items_displayed() }} of {{ totalItems }}</span></div>
       <div class="panel-body" ng-switch-when="false" ng-switch="isError">
         <p ng-switch-when="false">Loading . . .</p>
         <div ng-switch-when="true" class="alert alert-danger">

security_monkey/__init__.py

@@ -23,7 +23,7 @@
 import stat
 
 ### VERSION ###
-__version__ = '0.9.3'
+__version__ = '0.9.3'   # TODO update this to 0.9.4!
 
 ### FLASK ###
 from flask import Flask
@@ -33,6 +33,7 @@
 from flask_login import LoginManager
 import os
 
+
 app = Flask(__name__, static_url_path='/static')
 
 # If SECURITY_MONKEY_SETTINGS is set, then use that.
@@ -68,7 +69,7 @@
     ARN_PARTITION = 'aws-us-gov'
     AWS_DEFAULT_REGION = 'us-gov-west-1'
 
-ARN_PREFIX= 'arn:' + ARN_PARTITION
+ARN_PREFIX = 'arn:' + ARN_PARTITION
 
 db = SQLAlchemy(app)
 
@@ -106,8 +107,6 @@ def csrf_error(reason):
 security = Security(app, user_datastore)
 
 
-
-
 @security.send_mail_task
 def send_email(msg):
     """

security_monkey/account_manager.py

@@ -20,13 +20,11 @@
 .. version:: $$VERSION$$
 .. moduleauthor:: Bridgewater OSS <opensource@bwater.com>
 
-
 """
 from datastore import Account, AccountType, AccountTypeCustomValues, User
 from security_monkey import app, db
 from security_monkey.common.utils import find_modules
 import psycopg2
-import time
 import traceback
 
 from security_monkey.exceptions import AccountNameExists
@@ -346,4 +344,31 @@ def delete_account_by_name(name):
     db.session.expunge(account)
     delete_account_by_id(account_id)
 
+
+def bulk_disable_accounts(account_names):
+    """Bulk disable accounts"""
+    for account_name in account_names:
+        account = Account.query.filter(Account.name == account_name).first()
+        if account:
+            app.logger.debug("Disabling account %s", account.name)
+            account.active = False
+            db.session.add(account)
+
+    db.session.commit()
+    db.session.close()
+
+
+def bulk_enable_accounts(account_names):
+    """Bulk enable accounts"""
+    for account_name in account_names:
+        account = Account.query.filter(Account.name == account_name).first()
+        if account:
+            app.logger.debug("Enabling account %s", account.name)
+            account.active = True
+            db.session.add(account)
+
+    db.session.commit()
+    db.session.close()
+
+
 find_modules('account_managers')

security_monkey/alerter.py

@@ -47,18 +47,17 @@ def report_content(content):
 
 class Alerter(object):
 
-    def __init__(self, watchers_auditors=[], account=None, debug=False):
+    def __init__(self, watchers_auditors=None, account=None, debug=False):
         """
         envs are list of environments where we care about changes
         """
-
         self.account = account
         self.notifications = ""
         self.new = []
         self.delete = []
         self.changed = []
-        self.watchers_auditors = watchers_auditors
-        users = User.query.filter(User.accounts.any(name=account)).filter(User.change_reports=='ALL').all()
+        self.watchers_auditors = watchers_auditors if watchers_auditors else []
+        users = User.query.filter(User.accounts.any(name=account)).filter(User.change_reports == 'ALL').all()
         self.emails = [user.email for user in users]
         self.team_emails = app.config.get('SECURITY_TEAM_EMAIL', [])
 
@@ -73,12 +72,14 @@ def report(self):
         """
         Collect change summaries from watchers defined and send out an email
         """
-        changed_watchers = [watcher_auditor.watcher for watcher_auditor in self.watchers_auditors if watcher_auditor.watcher.is_changed()]
+        changed_watchers = [watcher_auditor.watcher
+                            for watcher_auditor in self.watchers_auditors if watcher_auditor.watcher.is_changed()]
         has_issues = has_new_issue = has_unjustified_issue = False
         for watcher in changed_watchers:
             (has_issues, has_new_issue, has_unjustified_issue) = watcher.issues_found()
             if has_issues:
-                users = User.query.filter(User.accounts.any(name=self.account)).filter(User.change_reports=='ISSUES').all()
+                users = User.query.filter(
+                    User.accounts.any(name=self.account)).filter(User.change_reports == 'ISSUES').all()  # noqa
                 new_emails = [user.email for user in users]
                 self.emails.extend(new_emails)
                 break

security_monkey/datastore.py

@@ -83,6 +83,7 @@ class Account(db.Model):
     custom_fields = relationship("AccountTypeCustomValues", lazy="immediate", cascade="all, delete, delete-orphan")
     unique_const = UniqueConstraint('account_type_id', 'identifier')
 
+    type = relationship("AccountType", backref="account_type")
     exceptions = relationship("ExceptionLogs", backref="account", cascade="all, delete, delete-orphan")
 
     def getCustom(self, name):

security_monkey/manage.py

@@ -16,19 +16,17 @@
 
 from flask.ext.script import Manager, Command, Option, prompt_pass
 
-from security_monkey.common.s3_canonical import get_canonical_ids, fetch_id
-from security_monkey.datastore import clear_old_exceptions, store_exception, AccountType
+from security_monkey.account_manager import bulk_disable_accounts, bulk_enable_accounts
+from security_monkey.common.s3_canonical import get_canonical_ids
+from security_monkey.datastore import clear_old_exceptions, store_exception, AccountType, ItemAudit
 
-from security_monkey import app, db
+from security_monkey import app, db, jirasync
 from security_monkey.common.route53 import Route53Service
 
 from flask.ext.migrate import Migrate, MigrateCommand
 
-from security_monkey.scheduler import run_change_reporter as sm_run_change_reporter
-from security_monkey.scheduler import find_changes as sm_find_changes
-from security_monkey.scheduler import audit_changes as sm_audit_changes
-from security_monkey.scheduler import disable_accounts as sm_disable_accounts
-from security_monkey.scheduler import enable_accounts as sm_enable_accounts
+from security_monkey.task_scheduler.tasks import manual_run_change_reporter, manual_run_change_finder
+from security_monkey.task_scheduler.tasks import audit_changes as sm_audit_changes
 from security_monkey.backup import backup_config_to_json as sm_backup_config_to_json
 from security_monkey.common.utils import find_modules, load_plugins
 from security_monkey.datastore import Account
@@ -67,17 +65,27 @@ def drop_db():
 @manager.option('-a', '--accounts', dest='accounts', type=unicode, default=u'all')
 def run_change_reporter(accounts):
     """ Runs Reporter """
-    account_names = _parse_accounts(accounts)
-    sm_run_change_reporter(account_names)
+    try:
+        account_names = _parse_accounts(accounts)
+    except KeyError as e:
+        app.logger.error("The passed in account: {} does not exist in Security Monkey's database.".format(e.message))
+        return -1
+
+    manual_run_change_reporter(account_names)
 
 
 @manager.option('-a', '--accounts', dest='accounts', type=unicode, default=u'all')
 @manager.option('-m', '--monitors', dest='monitors', type=unicode, default=u'all')
 def find_changes(accounts, monitors):
     """ Runs watchers """
     monitor_names = _parse_tech_names(monitors)
-    account_names = _parse_accounts(accounts)
-    sm_find_changes(account_names, monitor_names)
+    try:
+        account_names = _parse_accounts(accounts)
+    except KeyError as e:
+        app.logger.error("The passed in account: {} does not exist in Security Monkey's database.".format(e.message))
+        return -1
+
+    manual_run_change_finder(account_names, monitor_names)
 
 
 @manager.option('-a', '--accounts', dest='accounts', type=unicode, default=u'all')
@@ -87,7 +95,12 @@ def find_changes(accounts, monitors):
 def audit_changes(accounts, monitors, send_report, skip_batch):
     """ Runs auditors """
     monitor_names = _parse_tech_names(monitors)
-    account_names = _parse_accounts(accounts)
+    try:
+        account_names = _parse_accounts(accounts)
+    except KeyError as e:
+        app.logger.error("The passed in account: {} does not exist in Security Monkey's database.".format(e.message))
+        return -1
+
     sm_audit_changes(account_names, monitor_names, send_report, skip_batch=skip_batch)
 
 
@@ -96,8 +109,12 @@ def audit_changes(accounts, monitors, send_report, skip_batch):
 def delete_unjustified_issues(accounts, monitors):
     """ Allows us to delete unjustified issues. """
     monitor_names = _parse_tech_names(monitors)
-    account_names = _parse_accounts(accounts)
-    from security_monkey.datastore import ItemAudit
+    try:
+        _parse_accounts(accounts)
+    except KeyError as e:
+        app.logger.error("The passed in account: {} does not exist in Security Monkey's database.".format(e.message))
+        return -1
+
     issues = ItemAudit.query.filter_by(justified=False).all()
     for issue in issues:
         del issue.sub_items[:]
@@ -111,22 +128,18 @@ def delete_unjustified_issues(accounts, monitors):
 def backup_config_to_json(accounts, monitors, outputfolder):
     """ Saves the most current item revisions to a json file. """
     monitor_names = _parse_tech_names(monitors)
-    account_names = _parse_accounts(accounts)
-    sm_backup_config_to_json(account_names, monitor_names, outputfolder)
-
+    try:
+        account_names = _parse_accounts(accounts)
+    except KeyError as e:
+        app.logger.error("The passed in account: {} does not exist in Security Monkey's database.".format(e.message))
+        return -1
 
-@manager.command
-def start_scheduler():
-    """ Starts the python scheduler to run the watchers and auditors """
-    from security_monkey import scheduler
-    scheduler.setup_scheduler()
-    scheduler.scheduler.start()
+    sm_backup_config_to_json(account_names, monitor_names, outputfolder)
 
 
 @manager.command
 def sync_jira():
     """ Syncs issues with Jira """
-    from security_monkey import jirasync
     if jirasync:
         app.logger.info('Syncing issues with Jira')
         jirasync.sync_issues()
@@ -140,9 +153,9 @@ def clear_expired_exceptions():
     Clears out the exception logs table of all exception entries that have expired past the TTL.
     :return:
     """
-    print("Clearing out exceptions that have an expired TTL...")
+    app.logger.info("Clearing out exceptions that have an expired TTL...")
     clear_old_exceptions()
-    print("Completed clearing out exceptions that have an expired TTL.")
+    app.logger.info("Completed clearing out exceptions that have an expired TTL.")
 
 
 @manager.command
@@ -232,15 +245,25 @@ def create_user(email, role):
 @manager.option('-a', '--accounts', dest='accounts', type=unicode, default=u'all')
 def disable_accounts(accounts):
     """ Bulk disables one or more accounts """
-    account_names = _parse_accounts(accounts)
-    sm_disable_accounts(account_names)
+    try:
+        account_names = _parse_accounts(accounts)
+    except KeyError as e:
+        app.logger.error("The passed in account: {} does not exist in Security Monkey's database.".format(e.message))
+        return -1
+
+    bulk_disable_accounts(account_names)
 
 
 @manager.option('-a', '--accounts', dest='accounts', type=unicode, default=u'all')
 def enable_accounts(accounts):
     """ Bulk enables one or more accounts """
-    account_names = _parse_accounts(accounts, active=False)
-    sm_enable_accounts(account_names)
+    try:
+        account_names = _parse_accounts(accounts)
+    except KeyError as e:
+        app.logger.error("The passed in account: {} does not exist in Security Monkey's database.".format(e.message))
+        return -1
+
+    bulk_enable_accounts(account_names)
 
 
 @manager.option('-t', '--tech_name', dest='tech_name', type=str, required=True)
@@ -467,15 +490,25 @@ def _parse_tech_names(tech_str):
 
 
 def _parse_accounts(account_str, active=True):
+    """Parse the account ID or name. This will raise a KeyError if it can't find it."""
     if account_str == 'all':
         accounts = Account.query.filter(Account.third_party == False).filter(Account.active == active).all()
         accounts = [account.name for account in accounts]
         return accounts
     else:
         names_or_ids = account_str.split(',')
         accounts = Account.query.all()
-        accounts = {account.identifier: account.name for account in accounts}
-        names = map(lambda n: accounts.get(n, n), names_or_ids)
+        accounts_by_id = {account.identifier: account.name for account in accounts}
+        accounts_by_name = {account.name: account.identifier for account in accounts}
+
+        # Verify that the account name exists (raise a KeyError if it doesn't):
+        names = []
+        for n in names_or_ids:
+            if not accounts_by_id.get(n):
+                _ = accounts_by_name[n]
+
+            names.append(n)
+
         return names