oasis-data-updater

#!/usr/bin/env python2
# (ignore bad script name) pylint: disable=C0103
"""
A script for automatically updating and releasing CA certificates, CRLs,
and VO data, and releasing them on OASIS.
Designed to be run from cron.
Dies after a timeout if unsuccessful.
Lock file prevents duplicate processes from running.
Emails osg-software mailing list on failure.
"""
#############################################################################
# Defaults
DEFAULT_CERT_DIR = '/stage/oasis/mis/certificates'
# ^ NOTE: If this does not end in 'certificates', then osg-ca-manage setupCA
# will break!
DEFAULT_VO_DIR = '/stage/oasis/mis/vodata'
DEFAULT_WN_CLIENT = '/home/login/ouser.mis/osg-wn-client'
DEFAULT_LOCKFILE_PATH = '/home/login/ouser.mis/.oasis-data-updater.lock'
DEFAULT_TIMEOUT_MINS = 60
DEFAULT_NOTIFY_ADDRS = ['osg-software@opensciencegrid.org']
#
#############################################################################

import errno
import fcntl
import glob
import os
import pwd
import re
import signal
import smtplib
import socket
import subprocess
import sys
import time
import traceback

from optparse import OptionParser

from email.mime.text import MIMEText

verbose = False

class AlarmException(Exception):
    "Raised in the alarm handler"
    pass

class LockException(Exception):
    "Raised if another process has the lock"
    pass

# don't complain about unused arguments, this is the form of a signal handler
# pylint: disable=W0613
def alarm_handler(signum, frame):
    "Handles SIGALRM"
    raise AlarmException()


class CalledProcessError(Exception):
    """Raised by run_subprocess if it exits nonzero."""
    def __init__(self, process, returncode, output=None):
        Exception.__init__(self)
        self.process = process
        self.returncode = returncode
        self.output = output

    def __str__(self):
        return ("Error in called process(%s): subprocess returned %s.\nOutput: %s" %
                (str(self.process), str(self.returncode), str(self.output)))

    def __repr__(self):
        return str(repr(self.process),
                   repr(self.returncode),
                   repr(self.output))


def run_subprocess(command):
    """Runs command as a subprocess, returning its output/error, and raising
    an exception on failure"""
    if verbose:
        print "Running " + str(command)
    proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    output = proc.communicate()[0]
    if verbose:
        print "Output: " + output

    if proc.returncode != 0:
        raise CalledProcessError(command, proc.returncode, output)
    else:
        return output


def get_options(argv):
    "Parse, validate, and transform command-line options."
    parser = OptionParser("%prog [options]\n")

    parser.add_option("--timeout", default=DEFAULT_TIMEOUT_MINS, metavar="MINUTES", type="int",
                      help="The maximum duration this script should run for, in minutes")
    parser.add_option("--notify", metavar="EMAIL", action="append", default=None,
                      help="An email address to send a notification to in case of failure. "
                      "Can be specified multiple times for multiple recipients.")
    parser.add_option("--ignore-fetch-crl", default=False, action="store_true",
                      help="Ignore download and verification errors from fetch-crl.")
    parser.add_option("--verbose", default=False, action="store_true")
    parser.add_option("--nomail", default=False, action="store_true",
                      help="Do not send email, but print what we would send.")
    parser.add_option("--lockfile", metavar="PATH", default=DEFAULT_LOCKFILE_PATH,
                      help="Where to place the lock file that prevents two "
                      "copies of this script from running simultaneously.")
    parser.add_option("--wn-client", metavar="DIR", default=DEFAULT_WN_CLIENT,
                      help="The path to the worker-node tarball client installation to use.")
    parser.add_option("--cert-dir", metavar="DIR", default=DEFAULT_CERT_DIR,
                      help="The directory to install certificates into. "
                      "The last component must be 'certificates'.")
    parser.add_option("--vo-dir", metavar="DIR", default=DEFAULT_VO_DIR,
                      help="The directory to install VO data into.")
    parser.add_option("--oasis-update-retries",
                      help="Ignored, for compatibility reasons.")
    parser.add_option("--oasis-update-retry-wait",
                      help="Ignored, for compatibility reasons.")

    options = parser.parse_args(argv[1:])[0] # raises SystemExit(2) on error

    if not options.notify:
        options.notify = DEFAULT_NOTIFY_ADDRS

    if options.verbose:
        global verbose
        verbose = options.verbose

    return options


def acquire_lock(lockfile_path):
    "Get the lock for this script using Unix file locks"
    filehandle = open(lockfile_path, 'w')
    filedescriptor = filehandle.fileno()
    # Get an exclusive lock on the file (LOCK_EX) in non-blocking mode
    # (LOCK_NB), which causes the operation to raise IOError if some other
    # process already has the lock.
    try:
        fcntl.flock(filedescriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
    except IOError, err:
        if err.errno == errno.EWOULDBLOCK:
            raise LockException()
        else:
            raise
    return filehandle


def release_lock(lockfile_path, filehandle):
    "Release the lock for this script using Unix file locks"
    filedescriptor = filehandle.fileno()
    fcntl.flock(filedescriptor, fcntl.LOCK_UN)
    filehandle.close()


def send_notification(recipients, subject, text, nomail):
    username = pwd.getpwuid(os.getuid()).pw_name
    hostname = socket.gethostname()
    from_address = "%s@%s" % (username, hostname)

    msg = MIMEText(text)

    msg['Subject'] = subject
    msg['To'] = ', '.join(recipients)
    msg['From'] = from_address

    if nomail:
        print msg.as_string()
    else:
        smtp = smtplib.SMTP('localhost')
        smtp.sendmail(from_address, recipients, msg.as_string())
        smtp.quit()


def type_of_exception(exception_object):
    "Return the class name of an exception as a string"
    if isinstance(exception_object, Exception):
        return str(exception_object.__class__.__name__)


def cas_exist(cert_dir):
    """Return True if we already have CAs in cert_dir (and do not need to run
    osg-ca-manage setupCA)

    """
    cas = glob.glob(os.path.join(cert_dir, '*.0'))
    if cas:
        return True
    else:
        return False


def setup_cas(wn_client, osgrun, cert_dir):
    "Run osg-ca-manage setupCA"

    # Unfortunately, if we specify a location to osg-ca-manage setupCA, it
    # always wants to create a symlink from
    # $OSG_LOCATION/etc/grid-security/certificates to that location.  Since we
    # do not want to mess up the tarball install we're using, we must first
    # save the symlink that's already there, then run osg-ca-manage, then
    # restore it.
    certs_link = os.path.join(wn_client, 'etc/grid-security/certificates')
    certs_link_save = certs_link + '.save'

    # Note that in a proper tarball install, certs_link should already exist
    # but handle its nonexistence gracefully too.
    # Note the need to use 'lexists' since 'exists' returns False if the path
    # is a broken symlink.
    if os.path.lexists(certs_link):
        if os.path.lexists(certs_link_save):
            os.unlink(certs_link_save)
        os.rename(certs_link, certs_link_save)

    # osg-ca-manage always puts the certs into a subdirectory called 'certificates'
    # under the location specified here. So specify the parent of cert_dir as --location.
    command = [osgrun, 'osg-ca-manage']
    command += ['setupCA']
    command += ['--location', os.path.dirname(cert_dir)]
    command += ['--url', 'osg']
    try:
        run_subprocess(command)
    finally:
        if os.path.lexists(certs_link_save):
            if os.path.lexists(certs_link):
                os.unlink(certs_link)
            os.rename(certs_link_save, certs_link)


def update_cas(osgrun, cert_dir):
    run_subprocess([osgrun, 'osg-ca-manage', '--cert-dir', cert_dir, 'refreshCA'])


def update_crls(osgrun, cert_dir):
    "Run fetch-crl and return a list of non-fatal issues it finds"
    command = [osgrun, 'fetch-crl']
    command += ['--infodir', cert_dir]
    command += ['--out', cert_dir]
    command += ['--agingtolerance', '24'] # 24 hours
    global verbose
    if verbose:
        command += ['-vvv']
    else:
        command += ['--quiet']

    try:
        run_subprocess(command)
    except CalledProcessError, err:
        if (re.search(r'CRL verification failed', err.output) or re.search(r'Download error', err.output)):
            # These errors aren't actually fatal; we'll send a less alarming
            # notification about them.
            return err.output
        else:
            raise


def update_vos(osgrun, vo_dir):
    "Run osg-update-vos to update VO data"
    run_subprocess([osgrun, 'osg-update-vos', '--location', vo_dir])


def publish_updates():
    """Run osg-batch-update to queue release of the update to OASIS.

    """
    run_subprocess(['osg-batch-update'])


def main():
    """A wrapper around the rest of the logic to make sure this script can be
    safely used as a cron job, that is:
    (a) Ensure script does not run for too long
    (b) Ensure script does not run while another instance is running
    (c) Send mail if anything goes wrong

    """
    script_name = os.path.basename(sys.argv[0])

    options = get_options(sys.argv)
    osgrun = os.path.join(options.wn_client, "osgrun")

    try:
        lockfile_filehandle = acquire_lock(options.lockfile)
    except LockException:
        send_notification(options.notify, '%s already running; not starting a second time' % script_name, "",
                          options.nomail)
        return 1
    try:
        try:
            signal.signal(signal.SIGALRM, alarm_handler)
            signal.alarm(options.timeout * 60)

            if not cas_exist(options.cert_dir):
                setup_cas(options.wn_client, osgrun, options.cert_dir)
            update_cas(osgrun, options.cert_dir)
            fetch_crl_errs = update_crls(osgrun, options.cert_dir)
            update_vos(osgrun, options.vo_dir)
            publish_updates()
            if fetch_crl_errs and not options.ignore_fetch_crl:
                send_notification(options.notify, '%s fetch-crl errors' % (script_name),
                                  "Fetch-CRL reported the following errors:\n%s\n" % fetch_crl_errs,
                                  options.nomail)
        except AlarmException:
            send_notification(options.notify, '%s timed out' % (script_name),
                              "Traceback follows:\n%s\n" % traceback.format_exc(),
                              options.nomail)
            return 1
        except Exception, err:
            send_notification(options.notify, '%s died with exception %s' % (script_name, type_of_exception(err)),
                              "Traceback follows:\n%s\n" % traceback.format_exc(),
                              options.nomail)
            raise
    finally:
        release_lock(options.lockfile, lockfile_filehandle)
        signal.alarm(0)

    return 0

if __name__ == "__main__":
    sys.exit(main())