providence.py

'''
Copyright (c) 2015, Salesforce.com, Inc.
All rights reserved.

Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:

* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.

* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.

* Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
'''

#-- Load libraries
import datetime
import imp
import os, sys
import getpass
import json
import pytz
import logging
import argparse

from uuid import uuid4
from apscheduler.scheduler import Scheduler
from alerts import Alert
from Empire.creds import CredentialManager
#-- Load configuration
#-- This method may change in the future
import config
#-- remember command line settings
import settings

def set_global_config():
    configuration = config.Configuration('config.json')
    config.providence_configuration = configuration
    return configuration

def set_logging_from_configuration(configuration):
    #-- Setup Logging
    logging.basicConfig(filename=configuration.get(('logging','filename')),
                        filemode='w',
                        level=logging.DEBUG if configuration.get(('logging','loglevel')) == 'debug' else logging.INFO,
                        format=configuration.get(('logging','stringfmt')), 
                        datefmt=configuration.get(('logging','datefmt')))
    console = logging.StreamHandler()
    console.setLevel(logging.INFO)
    formatter = logging.Formatter(configuration.get(('logging','formatter')))
    console.setFormatter(formatter)
    logging.getLogger('').addHandler(console)

if __name__ == "__main__":
    configuration = set_global_config()
    set_logging_from_configuration(configuration)
    logger = logging.getLogger(__name__)


    parser = argparse.ArgumentParser(description='Providence Monitor Framework')
    parser.add_argument('--tests','-t', help="run plugin tests", action='store_true')
    parser.add_argument('--mode', help="specify production for production mode, or anything otherwise. Database will be reset if not in production, and providence will start from the current commit")
    parser.add_argument('--p4change', help="specify the p4 change number to debug. Must not be in production mode")
    parser.add_argument('--timestamp', help="timestamp in PST to pull commits from, in the format YYYY-MM-DD HH:MM:SS")
    args = parser.parse_args()

    settings.init(args.mode, args.p4change)

    #-- Basic Credentials setup
    credentials_file = configuration.get('credentials_file')
    credential_key = os.environ.get('CREDENTIAL_KEY')
    if credential_key is None:
        credential_key = getpass.getpass('Credential Key:')
    credential_manager = CredentialManager(credentials_file, credential_key)
    config.credential_manager = credential_manager

    from models import Base
    from db import engine
    #-- reset db if not in production or timestamp specified
    if not settings.in_production() or args.timestamp:
        Base.metadata.drop_all(engine)
    Base.metadata.create_all(engine)
    from repos import repotracker
    from plugins import RepoWatcher, Plugins
    from plugins.base import PluginTestError

#-- Load plugins
    loaded_plugins = Plugins(configuration)

    if args.tests:
        print "=======================  Loading plugins ======================="
        plugins = loaded_plugins.enabled_plugins()

        print "=======================  Running Plugin Tests ======================="
        for plugin_area in plugins:
            for plugin in plugins[plugin_area]:
                print "Running test for ", plugin.__module__
                try:
                    plugin.test()
                except PluginTestError, pte:
                    print "Test Failed for ", plugin.__module__
                    print pte.message
                    sys.exit(1)
        print "=======================  Tests Successful ======================="
        sys.exit(0)
    
    def run_watchers(startTime=None):
    # run watcher plugins
        logger.info("Running watchers")

        plugins = loaded_plugins.enabled_plugins()

        repositories = loaded_plugins.get_repositories(plugins["repositories"])        
        watchers = loaded_plugins.get_watchers(plugins["watchers"])
        tracker = repotracker.RepoTracker();

        for repository_name, repository_data in repositories.items():
            repository_watchers_by_path = watchers.get(repository_name)
            logger.info("In repo %s", repository_name)
            if repository_watchers_by_path is None:
                continue
            for repository_path, repo_watchers in repository_watchers_by_path.items():
                repository_db_identifier = repository_name
                if repository_path is not None:
                    repository_db_identifier += "::" + repository_path
                def commit_started_callback(repo_commit):
                    if repo_watchers:
                        for repo_watcher in repo_watchers:
                            plugin = repo_watcher.plugin
                            if hasattr(plugin, 'commit_started'):
                                plugin.commit_started(repository_name, repo_commit)
                def commit_finished_callback(repo_commit):
                    if repo_watchers:
                        for repo_watcher in repo_watchers:
                            plugin = repo_watcher.plugin
                            if hasattr(plugin, 'commit_finished'):
                                plugin.commit_finished(repository_name, repo_commit)
                    if repo_commit.identifier:
                        new_identifier = repo_commit.identifier
                        tracker.update_identifier(repository_db_identifier, new_identifier)
                def patch_callback(repo_patch):
                    if repo_watchers:
                        for repo_watcher in repo_watchers:
                            plugin = repo_watcher.plugin
                            if hasattr(plugin, 'patch'):
                                plugin.patch(repository_name, repo_patch)

                last_run_completed = tracker.last_run_completed(repository_db_identifier)
                if repository_data.get("check-every-x-minutes"):
                    now = datetime.datetime.utcnow()
                    if last_run_completed:
                        if (now - last_run_completed) < datetime.timedelta(minutes=repository_data.get("check-every-x-minutes")):
                            pass;
                try:
                    last_identifier = tracker.last_identifier(repository_db_identifier)
                    if not last_identifier and startTime:
                        last_identifier = startTime + " PST"
                    repository_data["source"].processSinceIdentifier(last_identifier, 
                                                                     commit_started_callback=commit_started_callback,
                                                                     patch_callback=patch_callback,
                                                                     commit_finished_callback=commit_finished_callback,
                                                                     path=repository_path);

                    tracker.update_last_run_completed(repository_db_identifier, datetime.datetime.utcnow())
                except Exception, e:
                    logger.exception("Repo Error: name=%s error=%s", repository_name, e, exc_info=True)

    def run_hourly():
    # run hourly plugins
        hour = datetime.datetime.now().hour
        logger.info("Running hourly")
        plugins = loaded_plugins.enabled_plugins()

        hourly_plugins = plugins.get("hourly")
        for plugin in hourly_plugins:
            try:
                hour = 1
                plugin.run_hourly(hour)
            except Exception, e:
                logger.exception("Exception running hourly: %s" % (plugin))

    def run_seven_minutes():
    # run seven minute plugins
        hour = datetime.datetime.now().hour
        logger.info("Running 7 minutes")

        plugins = loaded_plugins.enabled_plugins()
        seven_minute_plugins = plugins.get("seven_minutes")

        for plugin in seven_minute_plugins:
            try:
                plugin.run_seven_minutes()
            except Exception, e:
                logger.exception("Exception running 7 minutes: %s" % (plugin))
    
    if args.timestamp:
        run_watchers(args.timestamp)
    else:
        run_watchers()
    # run_seven_minutes()
    # run_hourly()

    sched = Scheduler(standalone=True)
    watcher_interval = "*/" + configuration.get(("cron", "watcher_interval"))
    sched.add_cron_job(run_watchers, minute=watcher_interval);
    # un-comment the following two lines if you'd like to use seven-minute or hourly plugins
    # sched.add_cron_job(run_seven_minutes, minute="*/7");
    # sched.add_cron_job(run_hourly, hour="*", minute="5");
    try:
        sched.start()
    except (KeyboardInterrupt, SystemExit):
        pass