csbuild

#!/usr/bin/env python3

# Copyright (C) 2014-2015 Red Hat, Inc.
#
# This file is part of csmock.
#
# csmock is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# any later version.
#
# csmock is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with csmock.  If not, see <http://www.gnu.org/licenses/>.

# standard imports
import argparse
import inspect
import os
import re
import shutil
import subprocess
import sys
import tempfile

# external imports
import git

# local imports
import csmock.common.cflags
from csmock.common.util   import add_paired_flag
from csmock.common.util   import shell_quote


CSBUILD_TRAVIS_MIRROR = \
        "deb https://kdudka.fedorapeople.org/csbuild trusty contrib"

RUN_SCAN_SH = "/usr/share/csbuild/scripts/run-scan.sh"

DEFAULT_ADDED_EXIT_CODE = 7

DEFAULT_BASE_FAIL_EXIT_CODE = 0

DEFAULT_CSWRAP_TIMEOUT = 30

DEFAULT_EMBED_CONTEXT = 3

DEFAULT_GCC_WARNING_LEVEL = 2

DEFAULT_HELP_SUFFIX = " (default: %(default)s)"

HELP_CMD_COMMON = "  It runs in @BUILD" + "DIR@ (which defaults to $PWD \
if --build-dir is not specified).  @SRC" + "DIR@ expands to $PWD at the \
time of csbuild's invocation."

TOOL_NAME = sys.argv[0]


class StatusWriter:
    def __init__(self):
        self.color_n = ""
        self.color_r = ""
        self.color_g = ""
        self.color_y = ""
        self.color_b = ""
        self.color_opt = "--no-color"
        self.csgrep_args = "--invert-match --event \"internal warning\""

    def enable_colors(self):
        self.color_n = "\033[0m"
        self.color_r = "\033[1;31m"
        self.color_g = "\033[1;32m"
        self.color_y = "\033[1;33m"
        self.color_b = "\033[1;34m"
        self.color_opt = "--color"

    def die(self, msg, ec=1):
        sys.stderr.write("%s: %sfatal error%s: %s\n" %
                         (TOOL_NAME, self.color_r, self.color_n, msg))
        sys.exit(ec)

    def emit_warning(self, msg):
        sys.stderr.write("%s: %swarning%s: %s\n" %
                         (TOOL_NAME, self.color_y, self.color_n, msg))
        sys.stderr.flush()

    def emit_status(self, msg):
        sys.stderr.write("%s: %sstatus%s: %s\n" %
                         (TOOL_NAME, self.color_g, self.color_n, msg))
        sys.stderr.flush()

    def print_stats(self, err_file):
        os.system("csgrep --mode=evtstat %s %s \"%s\"" %
                  (self.csgrep_args, self.color_opt, err_file))

    def print_defect_list(self, err_file, title):
        hline = "=" * len(title)
        print("\n%s%s\n%s%s" % (self.color_b, title, hline, self.color_n))
        sys.stdout.flush()

        # pass the --[no-]color option to csgrep
        os.system("csgrep %s %s \"%s\"" %
                  (self.csgrep_args, self.color_opt, err_file))

# FIXME: global instance
sw = StatusWriter()


def scan_or_die(cmd, what, fail_exit_code=1):
    sw.emit_status("running %s..." % what)
    sys.stderr.write("+ %s\n" % cmd)
    ret = os.system(cmd)

    signal = os.WTERMSIG(ret)
    if signal != 0:
        sw.die("%s signalled by signal %d" % (what, signal))

    status = os.WEXITSTATUS(ret)
    if status == 125 or (what == "prep" and status != 0):
        sw.die("%s failed: %s" % (what, cmd), ec=fail_exit_code)
    if status not in [0, 7]:
        sw.die("%s failed with exit code %d" % (what, status))

    sw.emit_status("%s succeeded" % what)
    return status


def stable_commit_ref(repo, ref):
    if hasattr(repo, "rev_parse"):
        commit = repo.rev_parse(ref)
    else:
        # repo.rev_parse() is not implemented on Ubuntu 12.04.5 LTS
        p = subprocess.Popen(["git", "rev-parse", ref], stdout=subprocess.PIPE)
        (out, _) = p.communicate()
        if p.returncode != 0:
            raise Exception("git rev-parse failed")
        commit = out.decode("utf8").strip()

    if "HEAD" in ref:
        # if HEAD is used in ref, we have have to checkout by hash (because
        # HEAD is going to change after checkout or git-bisect, which would
        # invalidate ref)
        return commit

    return ref


def do_git_checkout(repo, commit):
    sw.emit_status("checking out %s" % commit)
    try:
        repo.git.checkout(commit)
    except git.exc.GitCommandError as e:
        # if 'git checkout' fails, report it in a user-friendly way
        err = e.stderr
        err = re.sub("[^']*'error: ", "", err)
        err = re.sub("'$", "", err)
        sw.die(err)


def encode_paired_flag(args, flag):
    value = getattr(args, flag.replace("-", "_"))
    if value is None:
        return ""
    if value:
        return " --" + flag
    return " --no-" + flag


def encode_csbuild_args(args):
    cmd = " -c %s" % shell_quote(args.build_cmd)

    if args.git_bisect:
        cmd += " --git-bisect"

    if args.added_exit_code != DEFAULT_ADDED_EXIT_CODE:
        cmd += " --added-exit-code %d" % args.added_exit_code

    if args.base_fail_exit_code != DEFAULT_BASE_FAIL_EXIT_CODE:
        cmd += " --base-fail-exit-code %d" % args.base_fail_exit_code

    if args.cswrap_timeout != DEFAULT_CSWRAP_TIMEOUT:
        cmd += " --cswrap-timeout %d" % args.cswrap_timeout

    if args.embed_context != DEFAULT_EMBED_CONTEXT:
        cmd += " -U%d" % args.embed_context

    if args.gcc_warning_level != DEFAULT_GCC_WARNING_LEVEL:
        cmd += " -w%d" % args.gcc_warning_level

    cmd += csmock.common.cflags.encode_custom_flag_opts(args)
    for flag in args.clang_add_flag:
        cmd += " --clang-add-flag='%s'" % flag

    cmd += encode_paired_flag(args, "print-current")
    cmd += encode_paired_flag(args, "print-added")
    cmd += encode_paired_flag(args, "print-fixed")
    cmd += encode_paired_flag(args, "clean")
    cmd += encode_paired_flag(args, "color")
    return cmd


def print_yml_pair(name, value):
    print("%s: %s" % (name, value))


def print_yml_section(name):
    print("\n%s:" % name)


def print_yml_item(item):
    print("    - %s" % item)


def gen_travis_yml(args):
    print_yml_pair("language", "cpp")
    print_yml_pair("compiler", "gcc")

    # before_install
    print_yml_section("before_install")
    if "https://" in CSBUILD_TRAVIS_MIRROR:
        print_yml_item("sudo apt-get update -qq")
        print_yml_item("sudo apt-get install -qq apt-transport-https")
    print_yml_item("echo \"%s\" | sudo tee -a /etc/apt/sources.list" %
                   CSBUILD_TRAVIS_MIRROR)
    print_yml_item("sudo apt-get update -qq")

    # install
    print_yml_section("install")
    print_yml_item("sudo apt-get install -qq -y --force-yes csbuild")
    print_yml_item("sudo apt-get install %s" % args.install)

    # script
    print_yml_section("script")
    if args.prep_cmd is not None:
        print_yml_item(args.prep_cmd)
    print_yml_item("test -z \"$TRAVIS_COMMIT_RANGE\" \
|| csbuild --git-commit-range \"$TRAVIS_COMMIT_RANGE\"" +
                   encode_csbuild_args(args))

    # all OK
    return 0


def cmd_subst(cmd, src_dir, build_dir):
    if cmd is None:
        return None
    cmd = cmd.replace("@SRC" + "DIR@", src_dir)
    cmd = cmd.replace("@BUILD" + "DIR@", build_dir)
    return cmd


# argparse._VersionAction would write to stderr, which breaks help2man
class VersionPrinter(argparse.Action):
    def __init__(self, option_strings, dest=None, default=None, help=None):
        super(VersionPrinter, self).__init__(
            option_strings=option_strings, dest=dest, default=default, nargs=0,
            help=help)

    def __call__(self, parser, namespace, values, option_string=None):
        print("@VERSION@")
        sys.exit(0)


def main():
    # initialize argument parser
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "-c", "--build-cmd", required=True,
        help="Shell command used to build the sources." + HELP_CMD_COMMON)

    # optional arguments
    parser.add_argument(
        "-g", "--git-commit-range",
        help="range of git revisions for a differential scan in format OLD_REV..NEW_REV")

    parser.add_argument(
        "--git-bisect", action="store_true",
        help="if a new defect is added, use git-bisect to identify the cause \
WARNING: The given command must (re)compile all sources for this option to work!")

    parser.add_argument(
        "--added-exit-code", type=int, default=DEFAULT_ADDED_EXIT_CODE,
        help="exit code to return if there is any defect added in the new version" \
                + DEFAULT_HELP_SUFFIX)

    parser.add_argument(
        "--base-fail-exit-code", type=int, default=DEFAULT_BASE_FAIL_EXIT_CODE,
        help="exit code to return if the base scan fails" + DEFAULT_HELP_SUFFIX)

    add_paired_flag(
        parser, "print-current",
        help="print all defects in the current version (default unless -g is given) \
WARNING: The given command must (re)compile all sources for this option to work!")

    add_paired_flag(
        parser, "print-added",
        help="print defects added in the new version (default if -g is given)")

    add_paired_flag(
        parser, "print-fixed",
        help="print defects fixed in the new version \
WARNING: The given command must (re)compile all sources for this option to work!")

    add_paired_flag(
        parser, "clean",
        help="clean the temporary directory with results (and @BUILD" + "DIR@) \
on exit (default)")

    parser.add_argument(
        "--cswrap-timeout", type=int, default=DEFAULT_CSWRAP_TIMEOUT,
        help="maximal amount of time in seconds taken by analysis of a single module" \
                + DEFAULT_HELP_SUFFIX)

    add_paired_flag(
        parser, "color",
        help="use colorized console output (default if connected to a tty)")

    parser.add_argument(
        "--gen-travis-yml", action="store_true",
        help="generate the .travis.yml file for Travis CI (requires --install)")

    parser.add_argument(
        "--install",
        help="space-separated list of packages to install with --gen-travis-yml")

    parser.add_argument(
        "--prep-cmd",
        help="Shell command to run before the build." + HELP_CMD_COMMON)

    parser.add_argument(
        "--build-dir",
        help="Directory to be created to run the prep and build commands in \
(optional).  If not specified, the commands run in $PWD.  @BUILD" + "DIR@ in \
the operand of --build-cmd and --prep-cmd expands to the --build-dir operand \
converted to an absolute path.  Use --no-clean to preserve @BUILD" + "DIR@ \
on exit.")

    parser.add_argument(
        "-w", "--gcc-warning-level", type=int, default=DEFAULT_GCC_WARNING_LEVEL,
        help="Adjust GCC warning level.  -w0 means no additional warnings, \
-w1 appends -Wall and -Wextra, and -w2 enables some other useful warnings" \
+ DEFAULT_HELP_SUFFIX)

    csmock.common.cflags.add_custom_flag_opts(parser)

    parser.add_argument(
        "--clang-add-flag", action="append", default=[],
        help="append the given flag when invoking clang static analyzer \
(can be used multiple times)")

    parser.add_argument(
        "-U", "--embed-context", type=int, default=DEFAULT_EMBED_CONTEXT,
        help="embed a number of lines of context from the source file for the \
key event" + DEFAULT_HELP_SUFFIX)

    # needed for help2man
    parser.add_argument(
        "--version", action=VersionPrinter,
        help="print the version of csbuild and exit")

    # parse command-line arguments
    args = parser.parse_args()

    if args.gen_travis_yml:
        if args.install is None:
            parser.error("--install is required with --gen-travis-yml")
        if args.git_commit_range is not None:
            parser.error("--git-commit-range makes no sense with --gen-travis-yml")
        ret = gen_travis_yml(args)
        sys.exit(ret)
    elif args.install is not None:
        parser.error("--install makes sense only with --gen-travis-yml")

    # initialize color escape sequences if enabled
    if args.color is None:
        args.color = sys.stdout.isatty() and sys.stderr.isatty()
    if args.color:
        sw.enable_colors()

    # check whether we are in a git repository
    try:
        git_repo_args = inspect.getfullargspec(git.Repo.__init__).args
        if 'search_parent_directories' in git_repo_args:
            # search_parent_directories=True was the default behavior until
            # the arg was actually introduced (and we would like to keep it)
            repo = git.Repo(".", search_parent_directories=True)
        else:
            repo = git.Repo(".")
        repo_dir = repo.working_dir
        sw.emit_status("using git repository: %s" % repo_dir)
    except:
        repo = None

    diff_scan = args.git_commit_range is not None
    if diff_scan:
        # parse git commit range
        tokenized = args.git_commit_range.split("...")
        if len(tokenized) != 2:
            tokenized = args.git_commit_range.split("..")
        if len(tokenized) != 2:
            parser.error("not a range of git revisions: " + args.git_commit_range)

        if repo is None:
            parser.error("failed to open git repository: .")

        try:
            old_commit = stable_commit_ref(repo, tokenized[0])
            new_commit = stable_commit_ref(repo, tokenized[1])
        except:
            parser.error("failed to resolve the range of git revisions: " +
                         args.git_commit_range)

        if hasattr(repo.is_dirty, "__call__") and repo.is_dirty():
            sw.emit_warning("git repository is dirty: %s" % repo_dir)

    # initialize defaults where necessary
    if args.print_current is None:
        args.print_current = not diff_scan
    if args.print_added is None:
        args.print_added = diff_scan
    if args.print_fixed is None:
        args.print_fixed = False
    if args.clean is None:
        args.clean = True

    # check for possible conflict of command-line options
    if not diff_scan:
        if args.git_bisect \
                or (args.added_exit_code != DEFAULT_ADDED_EXIT_CODE) \
                or (args.base_fail_exit_code != DEFAULT_BASE_FAIL_EXIT_CODE) \
                or args.print_added or args.print_fixed:
            parser.error("options --git-bisect, --added-exit-code, --print-added, \
--base-fail-exit-code, and --print-fixed make sense only with --git-commit-range")

    # prepare environment
    env = {}
    env["CSWRAP_TIMEOUT"] = "%d" % args.cswrap_timeout
    env["CSWRAP_TIMEOUT_FOR"] = "clang:clang++:cppcheck"

    if args.clang_add_flag:
        # propagate custom clang flags
        clang_opts = csmock.common.cflags.serialize_flags(args.clang_add_flag)
        clang_opts_env = os.getenv("CSCLNG_ADD_OPTS")
        if clang_opts_env:
            # prepend ${CSCLNG_ADD_OPTS} from env
            clang_opts = "%s:%s" % (clang_opts_env, clang_opts)
        env["CSCLNG_ADD_OPTS"] = clang_opts

    # resolve compiler flags
    flags = csmock.common.cflags.flags_by_warning_level(args.gcc_warning_level)
    flags.append_custom_flags(args)
    flags.write_to_env(env)

    # resolve src_dir and build_dir
    src_dir = os.getcwd()
    if args.build_dir is None:
        build_dir = src_dir
    else:
        build_dir = os.path.abspath(args.build_dir)
        try:
            # create build_dir and enter the diretory
            os.mkdir(build_dir, 0o755)
            os.chdir(build_dir)
            sw.emit_status("entered build directory: %s" % build_dir)
        except OSError as err:
            parser.error("failed to create %s: %s" % (build_dir, err))

    # substitute prep_cmd and build_cmd
    prep_cmd = cmd_subst(args.prep_cmd, src_dir, build_dir)
    build_cmd = cmd_subst(args.build_cmd, src_dir, build_dir)

    # serialize environment
    cmd_prefix = ""
    for var in env:
        cmd_prefix += "%s='%s' " % (var, env[var])

    # append path of the run-scan.sh script
    cmd_prefix += RUN_SCAN_SH

    # create a temporary directory for the results
    res_dir = tempfile.mkdtemp(prefix="csbuild")
    cmd_prefix += " " + shell_quote(res_dir)

    if prep_cmd is not None:
        # run the command given by --prep-cmd through run-scan.sh
        scan_or_die(cmd_prefix + " " + shell_quote(prep_cmd), "prep")

    # chain all filters, starting with --embed-context propagation
    filter_cmd = "csgrep --embed-context %d" % args.embed_context
    if repo is not None:
        # if we are in a git repo, filter out results out of the repository
        filter_cmd += ' | csgrep --path "^%s/" --strip-path-prefix "%s/"' \
                % (repo_dir, repo_dir)

    # prepare template for running build_cmd through the run-scan.sh script
    cmd = "%s %s %s" % (cmd_prefix,
        shell_quote(build_cmd),
        shell_quote(filter_cmd))

    curr = "%s/current.err" % res_dir

    if diff_scan:
        # scan base revision first
        # TODO: handle checkout failures
        do_git_checkout(repo, old_commit)
        scan_or_die(cmd, "base scan", fail_exit_code=args.base_fail_exit_code)
        sw.print_stats(curr)
        base = "%s/base.err" % res_dir
        shutil.move(curr, base)
        cmd += " %s" % shell_quote(base)
        do_git_checkout(repo, new_commit)

    # scan the current version
    ret = scan_or_die(cmd, "scan")
    sw.print_stats(curr)

    # acknowledge the overall status
    if diff_scan:
        if ret == 0:
            sw.emit_status("no new defects found!")
        else:
            sw.emit_warning("new defects found!")

    res_added = "%s/added.err" % res_dir
    if args.git_bisect and os.path.getsize(res_added) > 0:
        # new defects found and we are asked to git-bisect the cause
        res_dir_gb = "%s/git-bisect" % res_dir
        os.mkdir(res_dir_gb)
        cmd = cmd.replace(res_dir, res_dir_gb, 1)
        sw.emit_status("running git-bisect...")
        cmd = "git bisect start %s %s \
&& git bisect run $SHELL -c %s \
&& git bisect reset" \
                % (new_commit, old_commit, shell_quote(cmd))
        sys.stderr.write("+ %s\n" % cmd)
        os.system(cmd)

    # print the results selected by the command-line options
    if args.print_current:
        sw.print_defect_list("%s/current.err" % res_dir, "CURRENT DEFECTS")
    if args.print_fixed:
        sw.print_defect_list("%s/fixed.err" % res_dir, "FIXED DEFECTS")
    if args.print_added:
        sw.print_defect_list(res_added, "ADDED DEFECTS")

    if args.clean:
        # purge the temporary directory
        shutil.rmtree(res_dir)
        if args.build_dir is not None:
            assert build_dir != src_dir
            shutil.rmtree(build_dir)
    else:
        if args.build_dir is not None:
            dst = "%s/csbuild" % build_dir
            shutil.move(res_dir, dst)
            res_dir = dst
        print("\nScan results: %s\n" % res_dir)

    if ret != 0:
        # return the required exit code if new defects were found
        sys.exit(args.added_exit_code)

if __name__ == '__main__':
    main()