clobberer.py

#!/usr/bin/python
# vim:sts=4 sw=4
import sys
import shutil
import urllib2
import urllib
import json
import os
import time
import socket
import logging

log = logging.getLogger()

if os.name == 'nt':
    from win32file import RemoveDirectory, DeleteFile, \
        SetFileAttributesW, \
        FILE_ATTRIBUTE_NORMAL, FILE_ATTRIBUTE_DIRECTORY
    from win32api import FindFiles

clobber_suffix = '.deleteme'


def get_hostname():
    return socket.gethostname().split('.')[0]


def ts_to_str(ts):
    if ts is None:
        return None
    return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(ts))


def write_file(ts, fn):
    assert isinstance(ts, int)
    f = open(fn, "w")
    f.write(str(ts))
    f.close()


def read_file(fn):
    if not os.path.exists(fn):
        return None

    data = open(fn).read().strip()
    try:
        return int(data)
    except ValueError:
        return None


def safe_join(parent, path):
    """Returns $parent/$path
    Raises IOError in case the resulting path ends up outside of parent for
    some reason (e.g. by using ../../..)
    """
    while os.path.isabs(path):
        path = path.lstrip("/")
        drive, tail = os.path.splitdrive(path)
        path = tail
        path = os.path.normpath(path)
    if path.startswith("../"):
        raise IOError("unsafe join of %s/%s" % (parent, path))
    return os.path.join(parent, path)


def rmdir_recursive_windows(dir):
    """Windows-specific version of rmdir_recursive that handles
    path lengths longer than MAX_PATH.
    """

    dir = os.path.realpath(dir)
    # Make sure directory is writable
    SetFileAttributesW('\\\\?\\' + dir, FILE_ATTRIBUTE_NORMAL)

    for ffrec in FindFiles('\\\\?\\' + dir + '\\*.*'):
        file_attr = ffrec[0]
        name = ffrec[8].decode(sys.getfilesystemencoding())
        if name == '.' or name == '..':
            continue
        full_name = os.path.join(dir, name)

        if file_attr & FILE_ATTRIBUTE_DIRECTORY:
            rmdir_recursive_windows(full_name)
        else:
            SetFileAttributesW('\\\\?\\' + full_name, FILE_ATTRIBUTE_NORMAL)
            DeleteFile('\\\\?\\' + full_name)
    RemoveDirectory('\\\\?\\' + dir)


def rmdir_recursive(dir):
    """This is a replacement for shutil.rmtree that works better under
    windows. Thanks to Bear at the OSAF for the code.
    (Borrowed from buildbot.slave.commands)"""
    if os.name == 'nt':
        rmdir_recursive_windows(dir)
        return

    if not os.path.exists(dir):
        # This handles broken links
        if os.path.islink(dir):
            os.remove(dir)
        return

    if os.path.islink(dir):
        os.remove(dir)
        return

    # Verify the directory is read/write/execute for the current user
    os.chmod(dir, 0o700)

    for name in os.listdir(dir):
        full_name = os.path.join(dir, name)
        # on Windows, if we don't have write permission we can't remove
        # the file/directory either, so turn that on
        if os.name == 'nt':
            if not os.access(full_name, os.W_OK):
                # I think this is now redundant, but I don't have an NT
                # machine to test on, so I'm going to leave it in place
                # -warner
                os.chmod(full_name, 0o600)

        if os.path.isdir(full_name):
            rmdir_recursive(full_name)
        else:
            # Don't try to chmod links
            if not os.path.islink(full_name):
                os.chmod(full_name, 0o700)
            os.remove(full_name)
    os.rmdir(dir)


def do_clobber(dir, dryrun=False, skip=None):
    try:
        for f in os.listdir(dir):
            if skip is not None and f in skip:
                log.info("Skipping %s", f)
                continue
            clobber_path = f + clobber_suffix
            if os.path.isfile(f):
                log.info("Removing %s", f)
                if not dryrun:
                    if os.path.exists(clobber_path):
                        os.unlink(clobber_path)
                    # Prevent repeated moving.
                    if f.endswith(clobber_suffix):
                        os.unlink(f)
                    else:
                        shutil.move(f, clobber_path)
                        os.unlink(clobber_path)
            elif os.path.isdir(f):
                log.info("Removing %s/", f)
                if not dryrun:
                    if os.path.exists(clobber_path):
                        rmdir_recursive(clobber_path)
                    # Prevent repeated moving.
                    if f.endswith(clobber_suffix):
                        rmdir_recursive(f)
                    else:
                        shutil.move(f, clobber_path)
                        rmdir_recursive(clobber_path)
    except Exception as e:
        log.error("[clobber failed]: %s", e.message)
        sys.exit(1)


def get_clobber_times(clobber_url):
    log.info("Checking clobber URL: %s", clobber_url)
    data = urllib2.urlopen(clobber_url, timeout=30).read()
    return json.loads(data)


def legacy_get_clobber_times(clobber_url, branch, buildername, builddir, slave, master=None):
    params = dict(branch=branch, buildername=buildername,
                  builddir=builddir, slave=slave, master=master)
    url = "%s?%s" % (clobber_url, urllib.urlencode(params))
    # The timeout arg was added to urlopen() at Python 2.6
    # Deprecate this test when esr17 reaches EOL
    if sys.version_info[:2] < (2, 6):
        data = urllib2.urlopen(url).read().strip()
    else:
        data = urllib2.urlopen(url, timeout=30).read().strip()

    processed_data = data.strip().split(':')

    if len(processed_data) < 2:
        # We received an empty result, time to bail
        return {'result': []}

    builddir_result, lastclobber, who = processed_data
    lastclobber = int(lastclobber)
    # return a dictionary that mimicks the new endpoint
    # json format
    return {
        'result': [{
            'builddir': builddir_result,
            'lastclobber': lastclobber,
            'who': who,
            'slave': None
        }]
    }


def process_clobber_times(server_clobber_times, args):
    now = int(time.time())

    root_dir = os.path.abspath(args.dir)

    for clobber_time in server_clobber_times['result']:
        slave = clobber_time['slave']
        builddir = clobber_time['builddir']
        server_clobber_date = clobber_time['lastclobber']
        who = clobber_time['who']

        builder_dir = safe_join(root_dir, builddir)
        if not os.path.isdir(builder_dir):
            log.info("%s doesn't exist, skipping", builder_dir)
            continue
        os.chdir(builder_dir)

        our_clobber_date = read_file("last-clobber") or 0

        clobber = False
        clobber_type = None

        log.info("%s:Our last clobber date: %s", builddir, ts_to_str(our_clobber_date))
        log.info("%s:Server clobber date:   %s", builddir, ts_to_str(server_clobber_date))

        # If we don't have a last clobber date, then this is probably a fresh build.
        # We should only do a forced server clobber if we know when our last clobber
        # was, and if the server date is more recent than that.
        if server_clobber_date is not None and our_clobber_date is not None:
            # If the server is giving us a clobber date, compare the server's idea of
            # the clobber date to our last clobber date also skip slave specific
            # clobbers that aren't for the current host
            if (server_clobber_date > our_clobber_date) and (slave is None or slave == args.slave):
                # If the server's clobber date is greater than our last clobber date,
                # then we should clobber.
                clobber = True
                clobber_type = "forced"
                # We should also update our clobber date to match the server's
                our_clobber_date = server_clobber_date
                if who:
                    log.info("%s:Server is forcing a clobber, initiated by %s", builddir, who)
                else:
                    log.info("%s:Server is forcing a clobber", builddir)

        if not clobber:
            # Disable periodic clobbers for builders that aren't args.builddir
            if builddir != args.builddir:
                continue

            # Next, check if more than the periodic_clobber_time period has passed since
            # our last clobber
            if our_clobber_date is None:
                # We've never been clobbered
                # Set our last clobber time to now, so that we'll clobber
                # properly after periodic_clobber_time
                clobber_type = "purged"
                our_clobber_date = now
                write_file(our_clobber_date, "last-clobber")
            elif args.periodic_clobber_time and now > our_clobber_date + args.periodic_clobber_time:
                # periodic_clobber_time has passed since our last clobber
                clobber = True
                clobber_type = "periodic"
                # Update our clobber date to now
                our_clobber_date = now
                log.info("%s:More than %s seconds have passed since our last clobber",
                         (builddir, args.periodic_clobber_time))

        if clobber:
            # Finally, perform a clobber if we're supposed to
            log.info("%s:Clobbering...", builddir)
            do_clobber(builder_dir, args.dryrun, args.skip)
            write_file(our_clobber_date, "last-clobber")

        # If this is the build dir for the current job, display the clobber type in TBPL.
        # Note in the case of purged clobber, we output the clobber type even though no
        # clobber was performed this time.
        if clobber_type and builddir == args.builddir:
            log.info("Tinderboxlog.info: %s clobber" % clobber_type)


def make_argparser():
    import optparse
    parser = optparse.OptionParser(
        "%prog [options] clobberURL branch buildername builddir slave master"
    )
    parser.add_option("-n", "--dry-run", dest="dryrun", action="store_true",
                      default=False, help="don't actually delete anything")
    parser.add_option("-t", "--periodic", dest="period", type=float,
                      default=None, help="hours between periodic clobbers")
    parser.add_option('-s', '--skip', help='do not delete this file/directory',
                      action='append', dest='skip', default=['last-clobber'])
    parser.add_option('-d', '--dir', help='clobber this directory',
                      dest='dir', default='.')
    parser.add_option('--builddir', help='a specific builddir for periodic clobbers')
    parser.add_option('-v', '--verbose', help='be more verbose',
                      dest='loglevel', action='store_const',
                      default=logging.INFO, const=logging.DEBUG)
    parser.add_option('--url', dest='clobber_url', help='URL to clobberer service')
    parser.add_option('--slave', default=get_hostname(),
                      help='Name of the slave being clobbered (defaults to hostname)')
    return parser


def main(args, legacy=None):
    if args.period:
        args.periodic_clobber_time = args.period * 3600
    else:
        args.periodic_clobber_time = None
    # Legacy support
    if legacy and len(legacy) > 1:
        log.info("Entering legacy mode.")
        server_clobber_times = legacy_get_clobber_times(*legacy)
    elif not args.clobber_url:
        # we can't set required=True because of legacy support
        raise Exception("--url required")
    else:
        server_clobber_times = get_clobber_times(args.clobber_url)

    process_clobber_times(server_clobber_times, args)

if __name__ == "__main__":
    parser = make_argparser()
    parsed_args, legacy = parser.parse_args()
    logging.basicConfig(level=parsed_args.loglevel, format="%(asctime)s - %(message)s")
    main(parsed_args, legacy=legacy)