build

#!/usr/bin/env python

usage_string = '''
USAGE

    ./build [-verbose] [-showdep[=target|all]] [target ...]

DESCRIPTION

    This script will compile and link the MRtrix3 source tree. It relies on the
    configuration file produced by ./configure - please ensure you have run
    this first.

    In most cases, a simple invocation is all that is required:

      $ ./build

    If no targets are provided, the command will default to building all
    applications by scanning through the cmd/ folder.

    The target executables will be located in the bin/ folder, and the shared
    library (if requested - the default) will be located in the lib/ folder (or
    in bin/ on Windows). All intermediate temporary files will be located
    within the tmp/ folder.

SPECIAL TARGETS

    clean
       used to remove all compiler-generated files, including objects,
       executables, and shared libraries.

    bash
       used to update the bash completion script. Note that automatic updating
       of this script can be enabled at the configure stage, by running
       './configure -dev' prior to invoking './build'.

    doc
       used to update the command documentation, so that any modifications to
       the inline documentation in commands can be propagated through to the
       user documentation site.

    select name
       used to switch between configs / builds. This stores the current config
       and all compiler-generated files in a folder (called "build.oldname"),
       and restores the config in "build.name". If the named config does not
       already exist, an empty one is created. If "name" is not given the
       currently active config name is reported.

PARALLELISED BUILD

    By default, 'build' will use all available cores to run a parallel build.
    In some instances, this can cause problems (notably out of RAM errors). You
    can control how many jobs 'build' will run concurrently using the
    NUMBER_OF_PROCESSORS environment variable. For example:

      $ NUMBER_OF_PROCESSORS=1 ./build

OPTIONS

    -verbose
       print each command as it is being invoked

    -nowarnings
       do not print out non-fatal compiler messages (warnings, etc)

    -showdep[=target|all]
       print the list of dependencies for every target (with -showdep=target;
       the default) or for all files

    -tree
       [only used with -showdep] print full dependency tree for each file

    -persistent
       keep trying to build regardless of failures, until none of the remaining
       jobs succeed.

    -nopaginate
       do not feed error log to the paginator, even if running in a TTY
'''



############################################################################
#                          COMMON DEFINITIONS                              #
############################################################################

import platform, sys, os, time, threading, subprocess, copy, codecs, glob, atexit, tempfile, shutil

# on Windows, need to use MSYS2 version of python - not MinGW version:
if sys.executable[0].isalpha() and sys.executable[1] == ':':
  python_cmd = subprocess.check_output ([ 'cygpath.exe', '-w', '/usr/bin/python' ]).splitlines()[0].strip()
  sys.exit (subprocess.call ([ python_cmd ] + sys.argv))

global todo, headers, object_deps, file_flags, lock, print_lock, stop, main_cindex
global include_paths

bin_dir = 'bin'
cmd_dir = 'cmd'
lib_dir = 'core'
misc_dir = 'src'
script_dir = os.path.join('lib', 'mrtrix3')
tmp_dir = 'tmp'

cpp_suffix = '.cpp'
h_suffix = '.h'
libname = 'mrtrix'

include_paths = [ misc_dir ]
config_file = None


system = None
dependencies = 0
dep_recursive = False
verbose = False
persistent = False
nowarnings = False
paginate = True
targets = []
use_multiple_cores = True


todo, headers, object_deps, file_flags = {}, {}, {}, {}
lock = threading.Lock()
print_lock = threading.Lock()
stop = False
error_stream = None
main_cindex = 0

logfile = open ('build.log', 'wb')


bcolors = {
    "candidate" : '\033[94m',
    "no known conversion" : '\033[94m',
    "expected" : '\033[93m',
    "^" : '\033[91m',
    "static assertion" : '\033[91m',
    "Linking" : '\033[01;32m',
    "In function" : '\033[01;32m',
    "WARNING" : '\033[95m',
    "Warning" : '\033[95m',
    "warning" : '\033[95m',
    "required from" : '\033[94m',
    "In instantiation of" : '\033[01;32m',
    "In member" : '\033[01;32m',
    "ERROR" : '\033[01;95m',
    "error" : '\033[01;31m',
    "failed" : '\033[91m',
    "note" : '\033[94m'}








def colorize(s):
  out = []
  for l in s.splitlines():
    for st in bcolors.keys():
      if st in l:
        l = l.replace (st, bcolors[st] + st) + '\033[0m'
        break
    out.append(l + '\n')
  return out




def pipe_errors_to_less_handler():
  global error_stream
  if len (error_stream):
    try:
      [ fid, name ] = tempfile.mkstemp()
      try:
        fid = os.fdopen (fid, 'wb')
        [ fid.write (x.encode (errors='ignore')) for x in colorize(error_stream) ]
        fid.close()
        os.system ("less -RX " + name)
      except Exception as e:
        sys.stderr.write (str (e))
      os.unlink (name)
    except Exception as e:
      sys.stderr.write (str (e))
    except:
      raise






def disp (msg):
  print_lock.acquire()
  logfile.write (msg.encode (errors='ignore'))
  sys.stdout.write (msg)
  print_lock.release()

def log (msg):
  print_lock.acquire()
  logfile.write (msg.encode (errors='ignore'))
  if verbose:
    sys.stderr.write (msg)
  print_lock.release()

def error (msg):
  global error_stream
  print_lock.acquire()
  logfile.write (msg.encode (errors='ignore'))
  if error_stream is not None:
    error_stream += msg
  else:
    sys.stderr.write (msg)
  print_lock.release()



def split_path (filename):
  return filename.replace ('\\', '/').split ('/')


def get_real_name (path):
  if os.path.islink (path): return os.readlink (path)
  else: return (path)





def list_untracked_bin_files (folder = '.'):
  process = subprocess.Popen ([ 'git', '-C', folder, 'ls-files', '-x', '.*', '-o', bin_dir ], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  ( files, stderr ) = process.communicate()
  if process.returncode == 0:
    return files.decode(errors='ignore').splitlines()
  else:
    return []




def modify_path (name, tmp=False, in_main=False, strip=None, add=None):
  if strip is not None:
    name = name[:-len(strip)]
  if add is not None:
    name = name + add
  for project_dir in mrtrix_dir:
    relname = os.path.normpath (os.path.relpath (name, project_dir))
    if relname.startswith ('.'):
      continue
    if tmp:
      relname = os.path.join (tmp_dir, relname)
    elif not os.path.relpath (relname, tmp_dir).startswith ('.'):
      relname = os.sep.join (split_path (relname)[1:])
    name = os.path.normpath (os.path.join (project_dir, relname))
    return name




############################################################################
#                          COMMAND-LINE PARSING                            #
############################################################################

command_doc = False
bash_completion = False

for arg in sys.argv[1:]:
  if '-help'.startswith (arg):
    print (usage_string)
    exit (0)
  elif '-verbose'.startswith(arg):
    verbose = True
  elif '-persistent'.startswith(arg):
    persistent = True
  elif '-nowarnings'.startswith(arg):
    nowarnings = True
  elif '-showdep'.startswith(arg):
    dependencies = 1
  elif arg.startswith ('-showdep='):
    if arg[9:] == 'target': dependencies = 1
    elif arg[9:] == 'all': dependencies = 2
    else:
      sys.stderr.write ('invalid specified for option "-showdep" (expected target, all)\n')
      sys.exit (1)
  elif '-tree'.startswith(arg):
    dep_recursive = True
  elif '-nopaginate'.startswith(arg):
    paginate = False
  elif arg[0] == '-':
    sys.stderr.write ('unknown command-line option "' + arg + '"\n')
    sys.exit (1)
  elif arg == 'bash':
    bash_completion = True
  elif arg == 'doc':
    command_doc = True
  elif arg == 'clean':
    targets = [ 'clean' ]
  else:
    targets.append(arg)




if paginate and sys.stderr.isatty():
  error_stream = ''
  atexit.register (pipe_errors_to_less_handler)




############################################################################
#                            PROJECT DETECTION                             #
############################################################################

mrtrix_dir = [ '.' ]
build_script = sys.argv[0]
separate_project = False

while os.path.abspath (os.path.dirname (get_real_name (build_script))) != os.path.abspath (mrtrix_dir[-1]):
  if not separate_project:
    log ('compiling separate project against:' + os.linesep)
  separate_project = True
  build_script = os.path.normpath (os.path.join (mrtrix_dir[-1], get_real_name (build_script)))
  project_dir = os.path.dirname (build_script)
  mrtrix_dir += [ project_dir ]
  include_paths += [ os.path.join (project_dir, misc_dir) ]
  log ('    ' + project_dir + os.linesep + os.linesep)





############################################################################
#                             CONFIG HANDLING                              #
############################################################################

def get_active_build (folder):
  active_config = glob.glob (os.path.join (folder, 'build.*.active'))
  if len (active_config) > 1:
    sys.stderr.write ('ERROR: more than one config is currently marked as active!\n')
    sys.exit (1)
  if len (active_config) == 1:
    active_config = active_config[0]
    if not os.path.isdir (active_config):
      raise Exception ('ERROR: active config (' + active_config + ') is not a folder')
    if len (os.listdir (active_config)):
      raise Exception ('ERROR: active config folder (' + active_config + ') is not empty')
    return active_config[:-len('.active')]
  else:
    os.mkdir (os.path.join (folder, 'build.default.active'))
    return os.path.join (folder, 'build.default')



def store_current_build (folder = '.'):
  stored_config = get_active_build (folder)
  os.rename (stored_config + '.active', stored_config)
  print ('in "' + folder + '": storing "' + stored_config + '"...')
  for f in [ tmp_dir, 'config' ] + list_untracked_bin_files (folder) + glob.glob ('lib/libmrtrix*'):
    if os.path.exists (os.path.join (folder, f)):
      os.renames (os.path.join (folder, f), os.path.join (stored_config, f))



def restore_build (name, folder = '.'):
  stored_config = os.path.join (folder, 'build.' + name)
  active_config = stored_config + '.active'
  if os.path.isdir (stored_config):
    print ('in "' + folder + '": restoring "' + stored_config + '"...')
    os.rename (stored_config, active_config)
    for root, dirs, files in os.walk(active_config, topdown=False):
      for name in files:
        os.renames (os.path.join (root, name), os.path.join (folder, os.path.relpath (root, active_config), name))
      for name in dirs:
        if os.path.isdir (os.path.join (root, name)):
          os.rmdir (os.path.join(root, name))
  else:
    if os.path.exists (stored_config):
      raise Exception ('ERROR config to be restored (' + stored_config + ') is not a folder')
    print ('in "' + folder + '": creating empty "' + stored_config + '"...')

  if not os.path.isdir (active_config):
    os.mkdir (active_config)





def activate_build (name, folders):
  for folder in folders:
    active_config = get_active_build (folder)
    if os.path.realpath (active_config) == os.path.realpath (os.path.join (folder, 'build.' + name)):
      continue

    store_current_build (folder)
    restore_build (name, folder)






if len (targets) and targets[0] == 'select':

  if len(targets) == 1:
    active_config = os.path.basename (get_active_build ('.'))
    print ('current config is "' + active_config + '"')
    for folder in mrtrix_dir[1:]:
      other_config = os.path.basename (get_active_build (folder))
      if active_config != other_config:
        print ('WARNING: folder "' + folder + '" contains config "' + other_config + '"')
    sys.exit (0)

  if len(targets) != 2:
    sys.stderr.write ('ERROR: select target expects a single configuration name\n')
    sys.exit (1)

  activate_build (targets[1], mrtrix_dir)

  sys.exit (0)







active_config = os.path.basename (get_active_build ('.'))[6:]
log ('active config is ' + active_config + '\n\n')
active_config_core = os.path.basename (get_active_build (mrtrix_dir[-1]))[6:]
if active_config_core != active_config:
  print ('active config differs from core - switching to core active config')
activate_build (active_config_core, mrtrix_dir)




############################################################################
#                            BUILD CLEAN                                   #
############################################################################

if 'clean' in targets:

  for f in [ tmp_dir, 'dev' ]:
    if not os.path.isdir (f): continue
    for root, dirs, files in os.walk(f, topdown=False):
      for name in files:
        fname = os.path.join(root, name)
        print ('delete file:', fname)
        os.remove (fname)
      for name in dirs:
        print ('delete folder:', os.path.join (root, name))
        os.rmdir (os.path.join(root, name))
    print ('delete folder:', root)
    os.rmdir (root)

  try:
    for fname in list_untracked_bin_files():
      print ('delete file:', fname)
      os.remove (fname)
  except:
    pass

  try:
    for fname in glob.glob (os.path.join ('lib', 'libmrtrix*')):
      if os.path.isfile (fname):
        print ('delete file:', fname)
        os.remove (fname)
  except:
    pass

  sys.exit (0)







############################################################################
#                          LOAD CONFIGURATION FILE                         #
############################################################################

if config_file is None:
  config_file = os.path.normpath (os.path.join (mrtrix_dir[-1], 'config'))

try:
  log ('reading configuration from "' + config_file + '"...' + os.linesep)
  exec (codecs.open (config_file, mode='r', encoding='utf-8').read())
except IOError:
  sys.stderr.write ('''no configuration file found!
please run "./configure" prior to invoking this script

''')
  sys.exit (1)

if separate_project:
  cpp_flags += [ '-DMRTRIX_PROJECT' ]

environ = os.environ.copy()
environ.update ({ 'PATH': PATH })

target_bin_dir = os.path.join (mrtrix_dir[0], bin_dir);

system = platform.system().lower()
if system.startswith('mingw') or system.startswith('msys'):
  target_lib_dir = os.path.join (mrtrix_dir[-1], bin_dir)
else:
  target_lib_dir = os.path.join (mrtrix_dir[-1], 'lib')
lib_dir = os.path.join (mrtrix_dir[-1], lib_dir)

if ld_enabled and len(runpath):
  ld_flags += [ runpath+os.path.relpath (target_lib_dir,target_bin_dir) ]





############################################################################
#                          GET VERSION INFORMATION                         #
############################################################################

if ld_enabled:
  ld_flags.insert(0, '-l' + libname)
  libname = lib_prefix + libname + lib_suffix


# other settings:
include_paths += [ lib_dir, cmd_dir ]
cpp_flags += [ '-I' + entry for entry in include_paths ]
ld_flags += [ '-L' + target_lib_dir ]

moc_cpp_suffix = '_moc' + cpp_suffix
moc_obj_suffix = '_moc' + obj_suffix




# remove any files that might have been left over from older installations in
# different locations:
if os.path.isdir ('release'):
  disp ('WARNING: removing \'release/\' folder - most likely left over from a previous installation\n')
  shutil.rmtree ('release')

for entry in glob.glob (os.path.normpath (os.path.join (target_lib_dir, '*' + lib_suffix))):
  if os.path.basename (entry) != libname:
    disp ('WARNING: removing "' + entry + '" - most likely left over from a previous installation\n')
    os.remove (entry)


###########################################################################
#                           TODO LIST ENTRY                               #
###########################################################################

class Entry:
  def __init__ (self, name):
    global todo
    name = os.path.normpath (name)
    if name in todo.keys(): return
    todo[name] = self


    self.name = name
    self.cmd = []
    self.deps = set()
    self.action = '--'
    self.timestamp = mtime (self.name)
    self.dep_timestamp = self.timestamp
    self.currently_being_processed = False

    if is_executable (self.name): self.set_executable()
    elif is_icon (self.name): self.set_icon()
    elif is_object (self.name): self.set_object()
    elif is_library (self.name): self.set_library()
    elif is_moc (self.name): self.set_moc()
    elif not os.path.exists (self.name):
      raise Exception ('unknown target "' + self.name + '"')


    [ Entry(item) for item in self.deps ]
    dep_timestamp = [ todo[item].timestamp for item in todo.keys() if item in self.deps and not is_library(item) ]
    dep_timestamp += [ todo[item].dep_timestamp for item in todo.keys() if item in self.deps and not is_library(item) ]
    if len(dep_timestamp):
      self.dep_timestamp = max(dep_timestamp)



  def execute (self, cindex, formatstr):
    folder = os.path.dirname (self.name)
    try:
      os.makedirs (folder)
    except OSError as e:
      if not os.path.isdir (folder):
        error ('ERROR: can''t create target folder "' + folder + '": ' + os.strerror (e.errno))
        exit (1)

    if self.action == 'RCC':
      with codecs.open (self.cmd[1], mode='w', encoding='utf-8') as fd:
        fd.write ('<!DOCTYPE RCC><RCC version="1.0">\n<qresource>\n')
        for entry in self.deps:
          entry = os.path.basename (entry)
          if not entry.startswith ('config'):
            fd.write ('<file>' + entry + '</file>\n')
        fd.write ('</qresource>\n</RCC>\n')
    if len(self.cmd) > 0:
      return execute (formatstr.format (cindex, self.action, self.name), self.cmd)
    else:
      return None


  def set_executable (self):
    self.action = 'LB'
    if len(exe_suffix) > 0 and self.name.endswith(exe_suffix): cc_file = self.name[:-len(exe_suffix)]
    else: cc_file = self.name
    cc_file = modify_path (os.path.join (cmd_dir, os.sep.join (split_path(cc_file)[1:])), add=cpp_suffix)
    self.deps = list_cmd_deps(cc_file)
    if separate_project:
      self.deps = self.deps.union ([ os.path.join (tmp_dir, misc_dir, 'project_version' + obj_suffix) ])

    skip = False
    flags = []
    if 'Q' in file_flags[cc_file]: flags += qt_ldflags

    if not skip:
      if not ld_enabled:
        self.deps = self.deps.union (list_lib_deps())

      self.cmd = fillin (ld, {
        'LDFLAGS': [ s.replace ('LIBNAME', os.path.basename (self.name)) for s in ld_flags ] + flags,
        'OBJECTS': self.deps,
        'EXECUTABLE': [ self.name ] })

      try:
        if ld_use_shell: self.cmd = [ 'sh', '-c', ' '.join(self.cmd) ]
      except NameError: pass

      if ld_enabled:
        self.deps.add (os.path.normpath (os.path.join (target_lib_dir, libname)))



  def set_object (self):
    self.action = 'CC'
    cc_file = self.name[:-len(obj_suffix)] + cpp_suffix

    flags = copy.copy (eigen_cflags)

    if is_moc (cc_file):
      src_header = modify_path (cc_file, strip=moc_cpp_suffix, add=h_suffix)
      list_headers (src_header)
      file_flags[cc_file] = file_flags[src_header]
    elif is_icon (cc_file):
      src_header = modify_path (cc_file, strip=cpp_suffix, add=h_suffix)
      list_headers (src_header)
      file_flags[cc_file] = file_flags[src_header]
    else:
      cc_file = modify_path (cc_file, tmp=False)
      self.deps = self.deps.union (list_headers (cc_file))
    self.deps.add (config_file)

    self.deps.add( cc_file )

    skip = False
    if 'Q' in file_flags[cc_file]: flags += qt_cflags

    if not skip:
      self.cmd = fillin (cpp, {
        'CFLAGS': cpp_flags + flags,
        'OBJECT': [ self.name ],
        'SRC': [ cc_file ] })


  def set_moc (self):
    self.action = 'MOC'
    src_file = modify_path (self.name, strip=moc_cpp_suffix, add=h_suffix)
    self.deps = set([ src_file ])
    self.deps = self.deps.union (list_headers (src_file))
    self.deps.add (config_file)
    self.cmd = [ moc ]
    self.cmd += [ src_file, '-o', self.name ]


  def set_library (self):
    if not ld_enabled:
      error ('ERROR: shared library generation is disabled in this configuration')
      exit (1)

    self.action = 'LD'
    self.deps = list_lib_deps()

    self.cmd = fillin (ld_lib, {
      'LDLIB_FLAGS': [ s.replace ('LIBNAME', os.path.basename (self.name)) for s in ld_lib_flags ],
      'OBJECTS': [ item for item in self.deps ],
      'LIB': [ self.name ] })

    try:
      if ld_use_shell: self.cmd = [ 'sh', '-c', ' '.join(self.cmd) ]
    except NameError: pass


  def set_icon (self):
    self.action = 'RCC'
    with codecs.open (modify_path (self.name, strip=cpp_suffix, add=h_suffix), mode='r', encoding='utf-8') as fd:
      for line in fd:
        if line.startswith ('//RCC:'):
          for entry in line[6:].strip().split():
            self.deps = self.deps.union (glob.glob (os.path.normpath (os.path.join (mrtrix_dir[-1], 'icons', entry))))
    self.deps.add (config_file)
    qrc_file = os.path.normpath (os.path.join (mrtrix_dir[-1], 'icons', os.path.basename (os.path.dirname(self.name)) + '.qrc'))
    self.cmd = [ rcc, qrc_file, '-o', self.name ]


  def need_rebuild (self):
    return self.timestamp == float("inf") or self.timestamp < self.dep_timestamp

  def display (self, indent=''):
    show_rebuild = lambda x: x+' [REBUILD]' if todo[x].need_rebuild() else x
    sys.stdout.write (indent + '[' + self.action + '] ' + show_rebuild (self.name) + ':\n')
    sys.stdout.write (indent + '  timestamp: ' + str(self.timestamp))
    if len(self.deps):
      sys.stdout.write (', dep timestamp: ' + str(self.dep_timestamp) + ', diff: ' + str(self.timestamp-self.dep_timestamp))
    sys.stdout.write ('\n')
    if len(self.cmd):
      sys.stdout.write (indent + '  command:   ' + ' '.join(self.cmd) + '\n')
    if len(self.deps):
      sys.stdout.write (indent + '  deps:      ')
      if dep_recursive:
        sys.stdout.write ('\n')
        for x in self.deps:
          todo[x].display (indent + '    ')
      else:
        sys.stdout.write ((indent+'\n             ').join([ show_rebuild(x) for x in self.deps ]) + '\n')






###########################################################################
#                         FUNCTION DEFINITIONS                            #
###########################################################################


def default_targets():
  if not os.path.isdir (cmd_dir):
    sys.stderr.write ('ERROR: no "cmd" folder - unable to determine default targets' + os.linesep)
    sys.exit (1)
  for entry in os.listdir (cmd_dir):
    if entry.endswith(cpp_suffix):
      targets.append (os.path.normpath (os.path.join (target_bin_dir, entry[:-len(cpp_suffix)] + exe_suffix)))
  return targets

def is_executable (target):
  return os.path.normpath (split_path (target)[0]) == os.path.normpath (bin_dir) and not is_moc (target) and not is_library (target)

def is_library (target):
  return target.endswith (lib_suffix) and split_path(target)[-1].startswith (lib_prefix)

def is_object (target):
  return target.endswith (obj_suffix)

def is_moc (target):
  return target.endswith (moc_cpp_suffix)

def is_icon (target):
  return os.path.basename (target) == 'icons'+cpp_suffix

def mtime (target):
  if not os.path.exists (target): return float('inf')
  return os.stat(target).st_mtime


def fillin (template, keyvalue):
  cmd = []
  for item in template:
    if item in keyvalue:
      cmd += keyvalue[item]
    else:
      cmd += [ item ]
  return cmd




def execute (message, cmd, working_dir=None):
  disp (message + os.linesep)
  log (' '.join(cmd) + os.linesep)

  try:
    process = subprocess.Popen (cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=environ, cwd=working_dir)
    ( stdout, stderr ) = process.communicate()
    if process.returncode != 0:
      if error_stream is not None:
        disp ('ERROR: ' + message + os.linesep)
      error ('\nERROR: ' + message + '\n\n' + ' '.join(cmd) + '\n\nfailed with output\n\n' + stderr.decode (errors='ignore'))
      return 1

    errstr = None
    if len(stdout) or len(stderr):
      errstr = '\n' + ' '.join(cmd) + ':\n\n'

    if len(stdout):
      errstr += stdout.decode (errors='ignore') + '\n\n'
    if len(stderr):
      errstr += stderr.decode (errors='ignore') + '\n\n'

    if errstr and not nowarnings:
      disp (errstr);

  except OSError:
    error (cmd[0] + ': command not found')
    return 1
  except:
    error ('unexpected exception: ' + str(sys.exc_info()))
    raise


def print_deps (current_file, indent=''):
  current_file = os.path.normpath (current_file)
  sys.stdout.write (indent + current_file)
  if current_file in file_flags:
    if len(file_flags[current_file]):
      sys.stdout.write (' [' + file_flags[current_file] + ']')
  sys.stdout.write (os.linesep)
  if len(todo[current_file].deps):
    for entry in todo[current_file].deps:
      print_deps (entry, indent + '    ')




def is_GUI_target (current_file):
  if 'gui' in split_path (current_file):
    return True
  if current_file in file_flags:
    if 'Q' in file_flags[current_file]:
      return True
  if len(todo[current_file].deps):
    for entry in todo[current_file].deps:
      if is_GUI_target (entry):
        return True
  return False



def list_headers (current_file):
  global headers, file_flags
  current_file = os.path.normpath (current_file)

  if current_file not in headers.keys():
    headers[current_file] = set()

    if current_file not in file_flags:
      file_flags[current_file] = ''

    if 'gui' in split_path (current_file):
      if 'Q' not in file_flags[current_file]:
        file_flags[current_file] += 'Q'
    if not os.path.exists (current_file):
      if os.path.basename(current_file) == 'icons'+cpp_suffix:
        return headers[current_file]
      sys.stderr.write ('ERROR: cannot find file "' + current_file + '"' + os.linesep)
      sys.exit(1)
    with codecs.open (current_file, mode='r', encoding='utf-8') as fd:
      for line in fd:
        line = line.strip()
        if line.startswith('#include'):
          line = line[8:].split ('//')[0].split ('/*')[0].strip()
          if line[0] == '"':
            line = line[1:].rstrip('"')
            for path in include_paths:
              if os.path.exists (os.path.join (path, line)):
                line = os.path.normpath (os.path.join (path, line))
                headers[current_file].add (line)
                [ headers[current_file].add(entry) for entry in list_headers(line) ]
                break
            else:
              sys.stderr.write ('ERROR: cannot find header file \"' + line + '\" (from file \"' + current_file + '\")' + os.linesep)
              sys.exit(1)
        elif line == 'Q_OBJECT':
          if 'M' not in file_flags[current_file]:
            file_flags[current_file] += 'M'

    for entry in headers[current_file]:
      for c in file_flags[entry]:
        if c != 'M' and c not in file_flags[current_file]:
          file_flags[current_file] += c

  return headers[current_file]






def list_cmd_deps (file_cc):
  global object_deps, file_flags
  file_cc = os.path.normpath (file_cc)

  if file_cc not in object_deps.keys():
    object_deps[file_cc] = set([ modify_path (file_cc, tmp=True, strip=cpp_suffix, add=obj_suffix) ])
    for entry in list_headers (file_cc):
      if os.path.abspath(entry).startswith(os.path.abspath(lib_dir)): continue
      if 'M' in file_flags[entry]:
        object_deps[file_cc] = object_deps[file_cc].union ([ modify_path (entry, tmp=True, strip=h_suffix, add=moc_obj_suffix) ])
      entry_cc = entry[:-len(h_suffix)] + cpp_suffix
      if os.path.exists (entry_cc):
        object_deps[file_cc] = object_deps[file_cc].union (list_cmd_deps(entry_cc))
      if os.path.basename (entry) == 'icons'+h_suffix:
        object_deps[file_cc].add (modify_path (entry, tmp=True, strip=h_suffix, add=obj_suffix))
    if file_cc not in file_flags: file_flags[file_cc] = ''
    for entry in headers[file_cc]:
      for c in file_flags[entry]:
        if c != 'M' and c not in file_flags[file_cc]:
          file_flags[file_cc] += c

  return object_deps[file_cc]



def list_lib_deps ():
  deps = set()
  for root, dirs, files in os.walk (lib_dir):
    for current_file in files:
      if current_file[0] == '.': continue
      if current_file.endswith (cpp_suffix):
        deps.add (modify_path (os.path.join (root, current_file), tmp=True, strip=cpp_suffix, add=obj_suffix))

  return (deps)



def build_next (id):
  global todo, lock, stop, main_cindex
  total_count = len(todo)
  cindex = 0
  formatstr = '({:>'+str(len(str(total_count)))+'}/'+str(total_count)+') [{}] {}'

  try:
    while not stop:
      current = None
      lock.acquire()
      if len(todo):
        for item in todo.keys():
          if todo[item].currently_being_processed: continue
          unsatisfied_deps = set(todo[item].deps).intersection (todo.keys())
          if not len(unsatisfied_deps):
            todo[item].currently_being_processed = True
            current = item
            main_cindex+=1
            cindex = main_cindex
            break
      else: stop = max (stop, 1)
      lock.release()

      if stop: return
      if current == None:
        time.sleep (0.01)
        continue

      target = todo[current]
      if target.execute(cindex, formatstr):
        todo[item].currently_being_processed = False
        stop = 2
        return

      lock.acquire()
      del todo[current]
      lock.release()

  except:
    stop = 2
    return

  stop = max(stop, 1)



def start_html (fid, title, left, up, home, right):
  fid.write ('<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">\n<html>\n<head>\n<meta http-equiv="Content-Type" content="text/html;charset=iso-8859-1">\n')
  fid.write ('<title>MRtrix documentation</title>\n<link rel="stylesheet" href="../stylesheet.css" type="text/css" media=screen>\n</head>\n<body>\n\n')
  fid.write ('<table class=nav>\n<tr>\n<td><a href="' + left + '.html"><img src="../left.png"></a></td>\n')
  fid.write ('<td><a href="' + up + '.html"><img src="../up.png"></a></td>\n')
  fid.write ('<td><a href="' + home + '.html"><img src="../home.png"></a></td>\n')
  fid.write ('<th>' + title + '</th>\n')
  fid.write ('<td><a href="' + right + '.html"><img src="../right.png"></a></td>\n</tr>\n</table>\n')





def get_git_version (folder):
  log ('''
getting git version in folder "''' + folder + '"... ')

  try:
    process = subprocess.Popen ([ 'git', 'describe', '--abbrev=8', '--dirty', '--always' ], cwd=folder, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    ( git_version, stderr ) = process.communicate()
    if process.returncode == 0:
      git_version = git_version.decode(errors='ignore').strip()
      log (git_version + '\n')
      return git_version
  except:
    pass


  log ('not found\n')
  return 'unknown'


def update_git_version (folder, git_version_file, contents):
  git_version = get_git_version (folder)
  version_file_contents = contents.replace ('%%%', git_version)
  current_version_file_contents = ''
  try:
    with open (git_version_file) as fd:
      current_version_file_contents = fd.read()
  except:
    pass

  if not current_version_file_contents or (version_file_contents != current_version_file_contents and git_version != 'unknown'):
    log ('version file "' + git_version_file + '" is out of date - updating\n')
    with open (git_version_file, 'w') as fd:
      fd.write (version_file_contents)


def update_bash_completion ():
  # Only attempt to generate completion file if POSIX compliant
  if os.name != 'posix': return

  # Check whether completion generation script exists
  script_path = os.path.join (mrtrix_dir[-1], 'generate_bash_completion.py')
  completion_path = os.path.join (mrtrix_dir[-1], 'mrtrix_bash_completion')
  if not os.path.isfile (script_path):
    disp (colorize('WARNING: Skipping bash completion generation. Could not find script at ' + script_path + '\n')[0])
    return

  # Check whether both command files and completion file exist and completion file is newer than commands
  if not os.path.isdir (target_bin_dir):
    return
  else:
    # Only look at relevant executables in bin file
    commands = [comm for comm in os.listdir (target_bin_dir) if os.access( os.path.join( target_bin_dir, comm ), os.X_OK)]
    if not commands:
      return
    else:
      command_freshness_time = max (commands, key = lambda x: time.ctime ( os.path.getmtime( os.path.join( target_bin_dir, x ) ) ) )
      if os.path.isfile (completion_path) and time.ctime ( os.path.getmtime(completion_path) ) >= command_freshness_time:
        return

  execute ('[SH] ' + completion_path, [ script_path, "-c", completion_path, "-m", target_bin_dir ] )




def update_user_docs ():
  # Only attempt to update docs if POSIX compliant
  if os.name != 'posix': return

  scripts_dir = os.path.abspath(os.path.join (mrtrix_dir[-1], 'docs' ))
  script_path = os.path.join (scripts_dir, 'generate_user_docs.sh')

  # Check whether generate docs script exists
  if not os.path.isfile (script_path):
    disp (colorize('WARNING: Skipping user documentation generation. Could not find script at ' + script_path + '\n')[0])
    return

  # Check whether commands dir exists
  if not os.path.isdir (target_bin_dir):
    return

  import re

  # Fetch relevant executables in bin file
  commands = [comm for comm in os.listdir (target_bin_dir) if os.access( os.path.join( target_bin_dir, comm ), os.X_OK) and not re.match (r'^\w+__debug', comm)]

  # Fetch relevant user docs
  rst_files = []

  for root, subfolders, files in os.walk (scripts_dir):
    for rst_file in files:
      if rst_file.endswith('.rst'):
        rst_files.append(os.path.join(root, rst_file))

  if not commands:
    return

  if rst_files:
    command_freshness_time = max (commands, key = lambda x: time.ctime ( os.path.getmtime( os.path.join( target_bin_dir, x ) ) ) )
    docs_freshness_time = min (rst_files, key = lambda x: time.ctime ( os.path.getmtime( x ) ) )
    if docs_freshness_time >= command_freshness_time:
      return

  if execute ('[DOC] generating user documentation (./docs)', [ script_path ], scripts_dir):
    sys.exit (1)


###########################################################################
#                              SCRIPT VERSION                             #
###########################################################################

with open (os.path.join(mrtrix_dir[-1], script_dir, '_version.py'),'w') as vfile:
  vfile.write('__version__ = "'+get_git_version (mrtrix_dir[-1])+'"\n')


###########################################################################
#                              START BUILDING                             #
###########################################################################


if len(targets) == 0:
  targets = default_targets()



# get git version info:
update_git_version (mrtrix_dir[-1], os.path.join (lib_dir, 'version.cpp'), '''
namespace MR {
  namespace App {
    const char* mrtrix_version = "%%%";
    const char* build_date = __DATE__;
  }
}
''')

update_git_version (mrtrix_dir[-1], os.path.join (mrtrix_dir[-1], misc_dir, 'exec_version.cpp'), '''
namespace MR {
  namespace App {
    extern const char* executable_uses_mrtrix_version;
    void set_executable_uses_mrtrix_version () { executable_uses_mrtrix_version = "%%%"; }
  }
}
''')

if separate_project:
  if not os.path.exists (misc_dir):
    os.mkdir (misc_dir)
  git_version_file = os.path.join (misc_dir, 'project_version.cpp')
  update_git_version ('.', git_version_file, '''
namespace MR {
  namespace App {
    extern const char* project_version;
    extern const char* project_build_date;
    void set_project_version () {
      project_version = "%%%";
      project_build_date = __DATE__;
    }
  }
}
''')

  if not os.path.exists (git_version_file):
    with open (git_version_file, 'w'):
      pass








log ('''
compiling TODO list...
''')

[ Entry(item) for item in targets ]



# for nogui config, remove GUI elements from targets and todo list:
if nogui:
  nogui_targets = []
  for entry in targets:
    if not is_GUI_target (entry):
      nogui_targets.append (entry)
  targets = nogui_targets

  nogui_todo = {}
  for item in todo.keys():
    if not is_GUI_target (todo[item].name):
       nogui_todo[item] = todo[item]
  todo = nogui_todo





log ('building targets: ' + ' '.join (targets) + os.linesep)


if dependencies==1:
  sys.stderr.write ('''
Printing dependencies for targets:

''')
  for entry in targets:
    todo[entry].display()
  sys.exit (0)
elif dependencies==2:
  sys.stderr.write ('''
Printing dependencies for all files:

''')
  for entry in todo:
    todo[entry].display()
  sys.exit (0)

todo_tmp = {}
for item in todo.keys():
  if todo[item].action != '--' and todo[item].need_rebuild():
    todo_tmp[item] = todo[item]
todo = todo_tmp

log ('TODO list contains ' + str(len(todo)) + ''' items

''')


#for entry in todo.values(): entry.display()
try: num_processors = int(os.environ['NUMBER_OF_PROCESSORS'])
except:
  try: num_processors = os.sysconf('SC_NPROCESSORS_ONLN')
  except: num_processors = 1

while len(todo):

  stop = False
  main_cindex = 0
  num_todo_previous = len(todo)

  log ('''

  launching ''' + str(num_processors) + ''' threads

  ''')

  threads = []
  for i in range (1, num_processors):
    t = threading.Thread (target=build_next, args=(i,));
    t.start()
    threads.append (t)

  build_next(0)

  for t in threads: t.join()

  if not persistent:
    break

  if len(todo) == num_todo_previous:
    disp ('''
stopping despite errors as no jobs completed successfully

''')
    break

  if len(todo):
    disp ('''
retrying as running in persistent mode

''')
    if error_stream is not None:
      error_stream = ''


# generate development-specific files (if needed)
# i.e. bash completion and user documentation
if not separate_project and stop <= 1:
  if bash_completion:
    update_bash_completion()
  if command_doc:
    update_user_docs()

sys.exit (stop > 1)