build.py

#!/usr/bin/python2.7
import os, sys, platform, shutil
import re, threading, time, json
from os import path
from hashlib import sha1
from multiprocessing import cpu_count


config_file = "build.config.py"
cache_dir = ".buildcache"
object_dir = path.join(cache_dir, "obj")
cache_file = path.join(cache_dir, "cache.json")
max_workers = cpu_count()


config = {
    "compiler" : "gcc",
    "output"   : "a.out",
    "source"   : [ "src" ],
    "include"  : [],
    "cflags"   : [],
    "lflags"   : [],
    "run"      : "./{output}"
}


Hint, Warn, Error = range(3)
log_prefix = {
    Hint: "\x1b[32mHint:\x1b[0m",
    Warn: "\x1b[33mWarn:\x1b[0m",
    Error: "\x1b[31;1mError:\x1b[0m"
}


log_lock = threading.Lock()

def log(msg, mode=Hint):
    log_lock.acquire()
    print log_prefix[mode], msg
    log_lock.release()


def error(msg):
    log(msg, mode=Error)
    os._exit(1)


def load_config(filename):
    """ loads the given config file into the `config` global dict """
    if not path.exists(filename):
        error("config file does not exist: '%s'" % filename)

    d = {
        "opt": sys.argv,
        "platform": platform.system(),
        "error": error,
        "log": log,
        "Hint": Hint,
        "Warn": Warn,
        "Error": Error
    }
    execfile(filename, d)
    config.update(d)

    if len(config["source"]) == 0:
        error("no source directories specified in config")


def load_cache(cache_file):
    if not path.exists(cache_file):
        return { "hashes": [], "cmd": "" }
    with open(cache_file) as fp:
        log("loaded cache")
        return json.load(fp)


def update_cache(cache_file, obj):
    with open(cache_file, "wb") as fp:
        json.dump(obj, fp, indent=2)
    log("updated cache")


def resolve_file(filename, dir):
    """ finds the actual location of an included file """
    f = path.join(dir, filename)
    if path.exists(f):
        return short_name(f)

    for dir in config["include"]:
        f = path.join(dir, filename)
        if path.exists(f):
            return short_name(f)


file_info_cache = {}

def get_file_info(filename):
    """ returns a dict of file info for the given file """
    if filename in file_info_cache:
        return file_info_cache[filename]

    hash = sha1()
    includes = []

    with open(filename) as fp:
        for line in fp.readlines():
            # get includes
            if "#include" in line:
                match = re.match('^\s*#include\s+"(.*?)"', line)
                if match:
                    includes.append( match.group(1) )
            # update hash
            hash.update(line)
            hash.update("\n")

    res = { "hash": hash.hexdigest(), "includes": includes }
    file_info_cache[filename] = res
    return res


def short_name(filename):
    """ returns the filename relative to the current path """
    n = len(path.abspath("."))
    return path.abspath(filename)[n+1:]


def get_deep_hash(filename):
    """ creates a hash from the file and all its includes """
    h = sha1()
    processed = set()
    files = [ resolve_file(filename, ".") ]

    while len(files) > 0:
        f = files.pop()
        info = get_file_info(f)
        processed.add(f)

        # update hash
        h.update(info["hash"])

        # add includes
        for x in info["includes"]:
            resolved = resolve_file(x, path.dirname(f))
            if resolved:
                if resolved not in processed:
                    files.append(resolved)
            else:
                log("could not resolve file '%s'" % x, mode=Warn)

    return h.hexdigest()


def build_deep_hash_dict(cfiles):
    """ returns a dict mapping each cfile to its hash """
    res = {}
    for f in cfiles:
        res[f] = get_deep_hash(f)
    return res


def get_cfiles():
    """ returns all .h and .c files in source directories """
    res = []
    for dir in config["source"]:
        for root, dirs, files in os.walk(dir):
            for file in files:
                if file.endswith((".c", ".h")):
                    f = path.join(root, file)
                    res.append( short_name(f) )
    return res


def build_compile_cmd():
    """ creates the command used to compile files """
    lst = [
        config["compiler"],
        " ".join(map(lambda x: "-I" + x, config["include"])),
        " ".join(config["cflags"]),
        "-c", "{infile}", "-o", "{outfile}"
    ]
    return " ".join(lst)


def obj_name(filename):
    """ creates the object file name for a given filename """
    filename = re.sub("[^\w]+", "_", filename)
    return filename[:-2] + "_" + sha1(filename).hexdigest()[:8] + ".o"


def compile(cmd, filename):
    """ compiles the given file into an object file using the cmd """
    log("compiling '%s'" % filename)

    outfile = path.join(object_dir, obj_name(filename))

    res = os.system(cmd.format(infile=filename, outfile=outfile))
    if res != 0:
        error("failed to compile '%s'" % filename)


def link():
    """ links objects and outputs the final binary """
    log("linking")
    lst = [
        config["compiler"],
        "-o", config["output"],
        path.join(object_dir, "*"),
        " ".join(config["lflags"])
    ]
    cmd = " ".join(lst)
    res = os.system(cmd)
    if res != 0:
        error("failed to link")


def parallel(func, workers=4):
    """ runs func on multiple threads and waits for them all to finish """
    threads = []
    for i in range(workers):
        t = threading.Thread(target=func)
        threads.append(t)
        t.start()
    for t in threads:
        t.join()



if __name__ == "__main__":

    start_time = time.time()

    load_config(config_file)
    run_at_exit = False
    output_dir =  path.join(".", path.dirname(config["output"]))
    cache = load_cache(cache_file)
    cmd = build_compile_cmd()

    if "run" in sys.argv:
        run_at_exit = True

    if cache["cmd"] != cmd:
        sys.argv.append("clean")

    if "clean" in sys.argv:
        log("performing clean build")
        shutil.rmtree(cache_dir, ignore_errors=True)
        cache = load_cache(cache_file)


    if not path.exists(object_dir):
        os.makedirs(object_dir)

    if not path.exists(output_dir):
        os.makedirs(output_dir)


    if "pre" in config:
        config["pre"]()


    cfiles = get_cfiles()
    hashes = build_deep_hash_dict(cfiles)


    # delete object files for cfiles that no longer exist
    obj_files = set(map(obj_name, cfiles))
    for f in os.listdir(object_dir):
        if f not in obj_files:
            os.remove(path.join(object_dir, f))


    # build list of all .c files that need compiling
    pending = []
    for f in cfiles:
        if f.endswith(".c"):
            if f not in cache["hashes"] or cache["hashes"][f] != hashes[f]:
                pending.append(f)


    # compile files until there are none left
    def worker():
        while True:
            try:
                f = pending.pop()
            except:
                break
            compile(cmd, f)


    parallel(worker, workers=max_workers)


    link()
    update_cache(cache_file, { "hashes": hashes, "cmd": cmd })

    if "post" in config:
        config["post"]()


    log("done [%.2fs]" % (time.time() - start_time))


    if run_at_exit:
        log("running")
        cmd = config["run"].format(output=config["output"])
        os.system(cmd)