buildfarm-run

#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2006-2011 TUBITAK/UEKAE
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# Please read the COPYING file.

import os
os.environ["LC_ALL"] = "C"

import sys
import copy
import shutil
import traceback
import cStringIO

import pisi.context as ctx
import pisi.api
from pisi.file import File

from buildfarm import logger, mailer, queuemanager, pisiinterface, utils
from buildfarm.config import configuration as conf

def build_packages():
    qmgr = queuemanager.QueueManager()

    qmgr.transfer_all_packages_to_work_queue()
    queue = copy.copy(qmgr.get_work_queue())

    if len(queue) == 0:
        logger.info("Work Queue is empty...")
        sys.exit(1)

    # We'll set home variable just after build process to make revdep work right
    home_dir = os.environ["HOME"]

    package_list = []

    # Compiling current work queue

    # Use a/b/pspec.xml instead of full path in e-mails
    stripped_work_queue = [utils.get_package_component_path(pkg) for pkg in queue]

    logger.raw("QUEUE")
    logger.info("Starting to compile the following packages:%s" % stripped_work_queue)

    mailer.info("I'm starting to compile packages in the following order:\n\n%s" % "\n".join(stripped_work_queue))
    logger.raw()

    for pspec in queue:
        #utils.mount_tmpfs()

        package_name = utils.get_package_name_from_path(pspec)
        package_name_with_component = utils.get_package_component_path(pspec)
        logfile = os.path.join(utils.get_package_log_directory(),
                               utils.get_package_logfile_name(pspec))
        build_output = open(logfile, "w")
        logger.raw()

        log_msg = "Building %s (%d/%d)" % (package_name, int(queue.index(pspec)+1), len(queue))
        logger.info(log_msg)

        # This is here because farm captures the build output
        pisi_api = pisiinterface.PisiApi(stdout = build_output,
                                         stderr = build_output,
                                         output_dir = conf.workdir)
        try:
            # First let's build the package
            pisi_api.build(pspec)

            # These lists contain full path to the .pisi files
            new_binary_packages = pisi_api.get_new_packages()
            new_debug_packages = pisi_api.get_new_debug_packages()

            # Get the dictionary of delta packages (a-b-c.pisi -> a-b-c-d.delta.pisi)
            new_delta_packages = pisi_api.get_delta_package_map()

            # Build was successful, let's install the packages
            pisi_api.install(new_binary_packages)

        except Exception, e:
            # Finalize api to generate logs
            pisi_api.close()

            # Transfer source package to wait queue in case of a build error
            qmgr.transferToWaitQueue(pspec)

            # Cleanup possibly built .pisi packages
            utils.delete_pisi_files_from(conf.workdir)

            errmsg = "Error occurred in package %s:\n %s" % (package_name_with_component, e)
            logger.error(errmsg)
            mailer.error(e, pspec)
        else:
            # Installation was successful
            pisi_api.close()
            qmgr.removeFromWorkQueue(pspec)
            move_packages(new_binary_packages, new_debug_packages, new_delta_packages)
            package_list += [os.path.basename(pkg) for pkg in new_binary_packages]
        finally:
            # Unmount tmp_dir if any
            # utils.umount_tmpfs()
            pass

    # Use a/b/pspec.xml instead of full path in e-mails
    stripped_wait_queue = [utils.get_package_component_path(pkg) for pkg in qmgr.get_wait_queue()]

    if stripped_wait_queue:
        logger.info("Wait Queue: %s" % stripped_wait_queue)
        mailer.info("Queue finished with problems and those packages couldn't be compiled:\n\n%s\n\n\nNew binary packages are;\n\n%s\n\nnow in repository" % ("\n".join(stripped_wait_queue), "\n".join(package_list)))
    else:
        mailer.info("Queue finished without a problem.\n\n\nNew binary packages are:\n\n%s\n\n" % ("\n".join(package_list)))
    logger.raw()
    logger.raw()

    # Remove obsoleted packages
    logger.info("Removing obsoleted packages, this may take a while..")
    removed_packages = utils.remove_obsoleted_packages()
    if removed_packages:
        logger.info("Removed the following obsoleted packages:\n%s" % "\n".join(removed_packages))

    # Set index paths
    paths = [utils.get_compiled_packages_directory()]
    if ctx.config.values.build.generatedebug:
        paths.append(utils.get_compiled_debug_packages_directory())

    for directory in paths:
        pisi.api.index([utils.get_local_repository_url(), directory],
                       output=os.path.join(directory, "pisi-index.xml"),
                       skip_sources=True,
                       skip_signing=True,
                       compression=File.COMPRESSION_TYPE_XZ)

    # Run hooks
    # print "\nRunning hooks"""
    # utils.run_hooks()

    # Check packages containing binaries and libraries broken by any package update
    print "\nChecking binary consistency with revdep-rebuild"
    os.environ["HOME"] = home_dir
    os.system("/usr/bin/revdep-rebuild --force")

def move_packages(new_binary_packages, new_debug_packages, new_delta_packages):

    def move_package(package, target):
        logger.info("Moving %s into %s" % (package, target))
        try:
            shutil.move(os.path.join(conf.workdir, package), target)
        except shutil.Error:
            pass

    logger.info("New binary package(s): %s" % [os.path.basename(pkg) \
                                                for pkg in new_binary_packages])
    logger.info("New debug package(s): %s" % [os.path.basename(pkg) \
                                                for pkg in new_debug_packages])
    logger.info("New delta package(s): %s" % [os.path.basename(pkg) \
                                                for pkg in sum(new_delta_packages.values(), [])])

    for package in new_binary_packages:
        move_package(package, utils.get_compiled_packages_directory())

        for pkg in new_delta_packages[package]:
            move_package(pkg, utils.get_compiled_packages_directory())

    for package in new_debug_packages:
        move_package(package, utils.get_compiled_debug_packages_directory())

        for pkg in new_delta_packages[package]:
            move_package(pkg, utils.get_compiled_debug_packages_directory())


### Entry point ###

def handle_exception(exception, value, trace_back):
    string_io = cStringIO.StringIO()
    traceback.print_tb(trace_back, file = string_io)
    string_io.seek(0)

    logger.error(str(exception))
    logger.error(str(value))
    logger.error(string_io.read())

if __name__ == "__main__":
    sys.excepthook = handle_exception

    # Check for available disk space, fail if any
    if not utils.is_there_free_space():
        print "Error: There's no sufficient disk space, exiting."

    # Just to make sure
    utils.create_directories()

    build_packages()