xenia-build

#!/usr/bin/env python

# Copyright 2015 Ben Vanik. All Rights Reserved.

"""Main build script and tooling for xenia.

Run with --help or no arguments for possible commands.
"""

__author__ = 'ben.vanik@gmail.com (Ben Vanik)'


import argparse
import os
import re
import shutil
import string
import subprocess
import sys


self_path = os.path.dirname(os.path.abspath(__file__))


def main():
  # Add self to the root search path.
  sys.path.insert(0, os.path.abspath(os.path.dirname(__file__)))

  # Augment path to include our fancy things.
  os.environ['PATH'] += os.pathsep + os.pathsep.join([
      self_path,
      os.path.abspath('build_tools/'),
      ])

  # Check git exists.
  if not has_bin('git'):
    print('ERROR: git must be installed and on PATH.')
    sys.exit(1)
    return

  # Check python version.
  if not sys.version_info[:2] == (2, 7):
    # TODO(benvanik): allow things to work with 3, but warn on clang-format.
    print('ERROR: Python 2.7 must be installed and on PATH')
    sys.exit(1)
    return

  # Grab Visual Studio version and execute shell to set up environment.
  if sys.platform == 'win32':
    vs_version = import_vs_environment()
    if vs_version != 2015:
      print('ERROR: Visual Studio 2015 not found!')
      print('Ensure you have the VS140COMNTOOLS environment variable!')
      sys.exit(1)
      return

  # Setup main argument parser and common arguments.
  parser = argparse.ArgumentParser(prog='xenia-build')

  # Grab all commands and populate the argument parser for each.
  subparsers = parser.add_subparsers(title='subcommands',
                                     dest='subcommand')
  commands = discover_commands(subparsers)

  # If the user passed no args, die nicely.
  if len(sys.argv) == 1:
    parser.print_help()
    sys.exit(1)
    return

  # Gather any arguments that we want to pass to child processes.
  command_args = sys.argv[1:]
  pass_args = []
  try:
    pass_index = command_args.index('--')
    pass_args = command_args[pass_index + 1:]
    command_args = command_args[:pass_index]
  except:
    pass

  # Parse command name and dispatch.
  args = vars(parser.parse_args(command_args))
  command_name = args['subcommand']
  try:
    command = commands[command_name]
    return_code = command.execute(args, pass_args, os.getcwd())
  except Exception as e:
    raise
    return_code = 1
  sys.exit(return_code)


# TODO(benvanik): move to build_tools utils module.
def import_vs_environment():
  """Finds the installed Visual Studio version and imports
  interesting environment variables into os.environ.

  Returns:
    A version such as 2015 or None if no VS is found.
  """
  version = 0
  tools_path = ''
  if 'VS140COMNTOOLS' in os.environ:
    version = 2015
    tools_path = os.environ['VS140COMNTOOLS']
  if version == 0:
    return None
  tools_path = os.path.join(tools_path, '..\\..\\vc\\vcvarsall.bat')

  args = [tools_path, '&&', 'set']
  popen = subprocess.Popen(
      args, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
  variables, _ = popen.communicate()
  envvars_to_save = (
      'devenvdir',
      'include',
      'lib',
      'libpath',
      'path',
      'pathext',
      'systemroot',
      'temp',
      'tmp',
      'windowssdkdir',
      )
  for line in variables.splitlines():
    for envvar in envvars_to_save:
      if re.match(envvar + '=', line.lower()):
        var, setting = line.split('=', 1)
        if envvar == 'path':
          setting = os.path.dirname(sys.executable) + os.pathsep + setting
        os.environ[var.upper()] = setting
        break

  os.environ['VSVERSION'] = str(version)
  return version


def has_bin(bin):
  """Checks whether the given binary is present.
  """
  for path in os.environ["PATH"].split(os.pathsep):
    path = path.strip('"')
    exe_file = os.path.join(path, bin)
    if os.path.isfile(exe_file) and os.access(exe_file, os.X_OK):
      return True
    exe_file = exe_file + '.exe'
    if os.path.isfile(exe_file) and os.access(exe_file, os.X_OK):
      return True
  return None


def shell_call(command, throw_on_error=True, stdout_path=None):
  """Executes a shell command.

  Args:
    command: Command to execute, as a list of parameters.
    throw_on_error: Whether to throw an error or return the status code.
    stdout_path: File path to write stdout output to.

  Returns:
    If throw_on_error is False the status code of the call will be returned.
  """
  stdout_file = None
  if stdout_path:
    stdout_file = open(stdout_path, 'w')
  result = 0
  try:
    if throw_on_error:
      result = 1
      subprocess.check_call(command, shell=False, stdout=stdout_file)
      result = 0
    else:
      result = subprocess.call(command, shell=False, stdout=stdout_file)
  finally:
    if stdout_file:
      stdout_file.close()
  return result


def git_submodule_update():
  """Runs a full recursive git submodule init and update.

  Older versions of git do not support 'update --init --recursive'. We could
  check and run it on versions that do support it and speed things up a bit.
  """
  if True:
    shell_call([
        'git',
        'submodule',
        'update',
        '--init',
        '--recursive',
        ])
  else:
    shell_call([
        'git',
        'submodule',
        'init',
        ])
    shell_call([
        'git',
        'submodule',
        'foreach',
        '--recursive',
        'git',
        'submodule',
        'init',
        ])
    shell_call([
        'git',
        'submodule',
        'update',
        '--recursive',
        ])


def get_clang_format_binary():
  """Finds a clang-format binary. Aborts if none is found.

  Returns:
    A path to the clang-format executable.
  """
  attempts = [
      'C:\\Program Files (x86)\\LLVM\\bin\\clang-format.exe',
      'clang-format-3.7',
      'clang-format',
      ]
  for binary in attempts:
    if os.path.exists(binary):
      return binary
  print 'ERROR: clang-format is not on PATH'
  print 'LLVM is available from http://llvm.org/releases/download.html'
  print 'At least version 3.7 is required.'
  print 'See docs/style_guide.md for instructions on how to get it.'
  sys.exit(1)


def run_premake(target_os, action):
  """Runs premake on the main project with the given format.

  Args:
    target_os: target --os to pass to premake.
    action: action to preform.
  """
  shell_call([
      'python',
      os.path.join('build_tools', 'premake'),
      '--file=premake5.lua',
      '--os=%s' % (target_os),
      '--cc=clang',
      '--test-suite-mode=combined',
      '--verbose',
      action,
      ])


def run_premake_clean():
  """Runs a premake clean operation.
  """
  if sys.platform == 'darwin':
    run_premake('macosx', 'clean')
  elif sys.platform == 'win32':
    run_premake('windows', 'clean')
  else:
    run_premake('linux', 'clean')


def run_platform_premake():
  """Runs all gyp configurations.
  """
  if sys.platform == 'darwin':
    run_premake('macosx', 'xcode')
  elif sys.platform == 'win32':
    run_premake('windows', 'vs2015')
  else:
    run_premake('linux', 'gmake')


def get_build_bin_path(args):
  """Returns the path of the bin/ path with build results based on the
  configuration specified in the parsed arguments.

  Args:
    args: Parsed arguments.

  Returns:
    A full path for the bin folder.
  """
  if sys.platform == 'darwin':
    platform = 'macosx'
  elif sys.platform == 'win32':
    platform = 'windows'
  else:
    platform = 'linux'
  return os.path.join(self_path, 'build', 'bin', platform, args['config'])


def discover_commands(subparsers):
  """Looks for all commands and returns a dictionary of them.
  In the future commands could be discovered on disk.

  Args:
    subparsers: Argument subparsers parent used to add command parsers.

  Returns:
    A dictionary containing name-to-Command mappings.
  """
  commands = {
      'setup': SetupCommand(subparsers),
      'pull': PullCommand(subparsers),
      'premake': PremakeCommand(subparsers),
      'build': BuildCommand(subparsers),
      'gentests': GenTestsCommand(subparsers),
      'test': TestCommand(subparsers),
      'clean': CleanCommand(subparsers),
      'nuke': NukeCommand(subparsers),
      'lint': LintCommand(subparsers),
      'format': FormatCommand(subparsers),
      }
  if sys.platform == 'win32':
    commands['devenv'] = DevenvCommand(subparsers)
  return commands


class Command(object):
  """Base type for commands.
  """

  def __init__(self, subparsers, name, help_short=None, help_long=None,
               *args, **kwargs):
    """Initializes a command.

    Args:
      subparsers: Argument subparsers parent used to add command parsers.
      name: The name of the command exposed to the management script.
      help_short: Help text printed alongside the command when queried.
      help_long: Extended help text when viewing command help.
    """
    self.name = name
    self.help_short = help_short
    self.help_long = help_long

    self.parser = subparsers.add_parser(name,
                                        help=help_short,
                                        description=help_long)
    self.parser.set_defaults(command_handler=self)

  def execute(self, args, pass_args, cwd):
    """Executes the command.

    Args:
      args: Arguments hash for the command.
      pass_args: Arguments list to pass to child commands.
      cwd: Current working directory.

    Returns:
      Return code of the command.
    """
    return 1


class SetupCommand(Command):
  """'setup' command."""

  def __init__(self, subparsers, *args, **kwargs):
    super(SetupCommand, self).__init__(
        subparsers,
        name='setup',
        help_short='Setup the build environment.',
        *args, **kwargs)

  def execute(self, args, pass_args, cwd):
    print('Setting up the build environment...')
    print('')

    # Setup submodules.
    print('- git submodule init / update...')
    git_submodule_update()
    print('')

    print('- running premake...')
    run_platform_premake()
    print('')

    print('Success!')
    return 0


class PullCommand(Command):
  """'pull' command."""

  def __init__(self, subparsers, *args, **kwargs):
    super(PullCommand, self).__init__(
        subparsers,
        name='pull',
        help_short='Pulls the repo and all dependencies and rebases changes.',
        *args, **kwargs)
    self.parser.add_argument('--merge', action='store_true',
                             help='Merges on master instead of rebasing.')

  def execute(self, args, pass_args, cwd):
    print('Pulling...')
    print('')

    print('- switching to master...')
    shell_call([
        'git',
        'checkout',
        'master',
        ])
    print('')

    print('- pulling self...')
    if args['merge']:
      shell_call([
          'git',
          'pull',
          ])
    else:
      shell_call([
          'git',
          'pull',
          '--rebase',
          ])
    print('')

    print('- pulling dependencies...')
    git_submodule_update()
    print('')

    print('- running premake...')
    run_platform_premake()
    print('')

    print('Success!')
    return 0


class PremakeCommand(Command):
  """'premake' command."""

  def __init__(self, subparsers, *args, **kwargs):
    super(PremakeCommand, self).__init__(
        subparsers,
        name='premake',
        help_short='Runs premake to update all projects.',
        *args, **kwargs)

  def execute(self, args, pass_args, cwd):
    print('Running premake...')
    print('')

    # Update premake.
    run_platform_premake()

    print('Success!')
    return 0


class BaseBuildCommand(Command):
  """Base command for things that require building."""

  def __init__(self, subparsers, *args, **kwargs):
    super(BaseBuildCommand, self).__init__(
        subparsers,
        *args, **kwargs)
    self.parser.add_argument(
        '--config', choices=['checked', 'debug', 'release'], default='debug',
        help='Chooses the build configuration.')
    self.parser.add_argument(
        '--target', action='append', default=[],
        help='Builds only the given target(s).')
    self.parser.add_argument(
        '--force', action='store_true',
        help='Forces a full rebuild.')

  def execute(self, args, pass_args, cwd):
    print('- running premake...')
    run_platform_premake()
    print('')

    print('- building (%s):%s...' % (
        'all' if not len(args['target']) else ' '.join(args['target']),
        args['config']))
    if sys.platform == 'win32':
      result = shell_call([
          'devenv',
          '/nologo',
          'build/xenia.sln',
          '/rebuild' if args['force'] else '/build',
          args['config'],
          ] + [('/project ', target) for target in args['target']] +
          pass_args, throw_on_error=False)
    else:
      # TODO(benvanik): other platforms.
      print('ERROR: don\'t know how to build on this platform.')
    print('')
    if result != 0:
      print('ERROR: build failed with one or more errors.')
      return result
    return 0


class BuildCommand(BaseBuildCommand):
  """'build' command."""

  def __init__(self, subparsers, *args, **kwargs):
    super(BuildCommand, self).__init__(
        subparsers,
        name='build',
        help_short='Builds the project.',
        *args, **kwargs)

  def execute(self, args, pass_args, cwd):
    print('Building %s...' % (args['config']))
    print('')

    result = super(BuildCommand, self).execute(args, pass_args, cwd)
    if not result:
      print('Success!')
    return result


class TestCommand(BaseBuildCommand):
  """'test' command."""

  def __init__(self, subparsers, *args, **kwargs):
    super(TestCommand, self).__init__(
        subparsers,
        name='test',
        help_short='Runs automated tests that have been built with `xb build`.',
        help_long='''
        To pass arguments to the test executables separate them with `--`.
        For example, you can run only the instr_foo.s tests with:
          $ xb test -- instr_foo
        ''',
        *args, **kwargs)
    self.parser.add_argument(
        '--no-build', action='store_true',
        help='Don\'t build before running tests.')
    self.parser.add_argument(
        '--continue', action='store_true',
        help='Don\'t stop when a test errors, but continue running all.')

  def execute(self, args, pass_args, cwd):
    print('Testing...')
    print('')

    # The test executables that will be built and run.
    test_targets = args['target'] or [
        'xenia-cpu-frontend-tests',
        ]
    args['target'] = test_targets

    # Build all targets (if desired).
    if not args['no_build']:
      result = super(TestCommand, self).execute(args, [], cwd)
      if result:
        print('Failed to build, aborting test run.')
        return result

    # Ensure all targets exist before we run.
    test_executables = [
        os.path.join(get_build_bin_path(args), test_target)
        for test_target in test_targets]
    for test_executable in test_executables:
      if not has_bin(test_executable):
        print('ERROR: Unable to find %s - build it.' % (test_executable))
        return 1

    # Run tests.
    any_failed = False
    for test_executable in test_executables:
      print('- %s' % (test_executable))
      result = shell_call([
          test_executable,
          ] + pass_args,
          throw_on_error=False)
      if result:
        any_failed = True
        if args['continue']:
          print('ERROR: test failed but continuing due to --continue.')
        else:
          print('ERROR: test failed, aborting, use --continue to keep going.')
          return result

    if any_failed:
      print('ERROR: one or more tests failed.')
      result = 1
    return result


class GenTestsCommand(Command):
  """'gentests' command."""

  def __init__(self, subparsers, *args, **kwargs):
    super(GenTestsCommand, self).__init__(
        subparsers,
        name='gentests',
        help_short='Generates test binaries.',
        help_long='''
        Generates test binaries (under src/xenia/cpu/frontend/testing/bin/).
        Run after modifying test .s files.
        ''',
        *args, **kwargs)

  def execute(self, args, pass_args, cwd):
    print('Generating test binaries...')
    print('')

    binutils_path = os.path.join('third_party', 'binutils-ppc-cygwin')
    ppc_as = os.path.join(binutils_path, 'powerpc-none-elf-as')
    ppc_ld = os.path.join(binutils_path, 'powerpc-none-elf-ld')
    ppc_objdump = os.path.join(binutils_path, 'powerpc-none-elf-objdump')
    ppc_nm = os.path.join(binutils_path, 'powerpc-none-elf-nm')

    test_src = os.path.join('src', 'xenia', 'cpu', 'frontend', 'testing')
    test_bin = os.path.join(test_src, 'bin')

    # Ensure the test output path exists.
    if not os.path.exists(test_bin):
      os.mkdir(test_bin)

    src_files = [os.path.join(root, name)
       for root, dirs, files in os.walk('src')
       for name in files
       if name.endswith(('.s'))]

    def make_unix_path(p):
      """Forces a unix path separator style, as required by binutils.
      """
      return string.replace(p, os.sep, '/')

    any_errors = False
    for src_file in src_files:
      print('- %s' % (src_file))
      src_name = os.path.splitext(os.path.basename(src_file))[0]
      obj_file = os.path.join(test_bin, src_name) + '.o'
      shell_call([
          ppc_as,
          '-a32',
          '-be',
          '-mregnames',
          '-mpower7',
          '-maltivec',
          '-mvsx',
          '-mvmx128',
          '-R',
          '-o%s' % (make_unix_path(obj_file)),
          make_unix_path(src_file),
          ])
      dis_file = os.path.join(test_bin, src_name) + '.dis'
      shell_call([
          ppc_objdump,
          '--adjust-vma=0x100000',
          '-Mpower7',
          '-Mvmx128',
          '-D',
          '-EB',
          make_unix_path(obj_file),
          ], stdout_path=dis_file)
      # Eat the first 4 lines to kill the file path that'll differ across machines.
      with open(dis_file) as f:
        dis_file_lines = f.readlines()
      with open(dis_file, 'w') as f:
        f.writelines(dis_file_lines[4:])
      shell_call([
          ppc_ld,
          '-A powerpc:common32',
          '-melf32ppc',
          '-EB',
          '-nostdlib',
          '--oformat=binary',
          '-Ttext=0x80000000',
          '-e0x80000000',
          '-o%s' % (make_unix_path(os.path.join(test_bin, src_name) + '.bin')),
          make_unix_path(obj_file),
          ])
      shell_call([
          ppc_nm,
          '--numeric-sort',
          make_unix_path(obj_file),
          ], stdout_path=os.path.join(test_bin, src_name) + '.map')

    if any_errors:
      print('ERROR: failed to build one or more tests.')
      return 1

    return 0


class CleanCommand(Command):
  """'clean' command."""

  def __init__(self, subparsers, *args, **kwargs):
    super(CleanCommand, self).__init__(
        subparsers,
        name='clean',
        help_short='Removes intermediate files and build outputs.',
        *args, **kwargs)

  def execute(self, args, pass_args, cwd):
    print('Cleaning build artifacts...')
    print('')

    print('- premake clean...')
    run_premake_clean()
    print('')

    print('Success!')
    return 0


class NukeCommand(Command):
  """'nuke' command."""

  def __init__(self, subparsers, *args, **kwargs):
    super(NukeCommand, self).__init__(
        subparsers,
        name='nuke',
        help_short='Removes all build/ output.',
        *args, **kwargs)

  def execute(self, args, pass_args, cwd):
    print('Cleaning build artifacts...')
    print('')

    print('- removing build/...')
    if os.path.isdir('build/'):
      shutil.rmtree('build/')
    print('')

    print('- git reset to master...')
    shell_call([
        'git',
        'checkout',
        '--hard',
        'master',
        ])
    print('')

    print('- running premake...')
    run_platform_premake()
    print('')

    print('Success!')
    return 0


class LintCommand(Command):
  """'lint' command."""

  def __init__(self, subparsers, *args, **kwargs):
    super(LintCommand, self).__init__(
        subparsers,
        name='lint',
        help_short='Checks for lint errors with clang-format.',
        *args, **kwargs)
    self.parser.add_argument(
        '--all', action='store_true',
        help='Lint all files, not just those changed.')
    self.parser.add_argument(
        '--origin', action='store_true',
        help='Lints all files changed relative to origin/master.')

  def execute(self, args, pass_args, cwd):
    clang_format_binary = get_clang_format_binary()

    difftemp = '.difftemp.txt'

    if args['all']:
      all_files = [os.path.join(root, name)
         for root, dirs, files in os.walk('src')
         for name in files
         if name.endswith(('.cc', '.c', '.h', '.inl'))]
      print('- linting %d files' % (len(all_files)))
      any_errors = False
      for file_path in all_files:
        if os.path.exists(difftemp): os.remove(difftemp)
        ret = shell_call([
            clang_format_binary,
            '-output-replacements-xml',
            '-style=file',
            file_path,
            ], throw_on_error=False, stdout_path=difftemp)
        with open(difftemp) as f:
          had_errors = '<replacement ' in f.read()
        if os.path.exists(difftemp): os.remove(difftemp)
        if had_errors:
          any_errors = True
          print('')
          print(file_path)
          shell_call([
              clang_format_binary,
              '-style=file',
              file_path,
              ], throw_on_error=False, stdout_path=difftemp)
          shell_call([
              'python',
              'tools/diff.py',
              file_path,
              difftemp,
              difftemp,
              ])
          shell_call([
              'type' if sys.platform=='win32' else 'cat',
              difftemp,
              ])
          if os.path.exists(difftemp): os.remove(difftemp)
          print('')
      print('')
      if any_errors:
        print('ERROR: 1+ diffs. Stage changes and run \'xb format\' to fix.')
        return 1
      else:
        print('Linting completed successfully.')
        return 0
    else:
      print('- git-clang-format --diff')
      if os.path.exists(difftemp): os.remove(difftemp)
      ret = shell_call([
          'python',
          'third_party/clang-format/git-clang-format',
          '--binary=%s' % (clang_format_binary),
          '--commit=%s' % ('origin/master' if args['origin'] else 'HEAD'),
          '--diff',
          ], throw_on_error=False, stdout_path=difftemp)
      with open(difftemp) as f:
        not_modified = 'no modified files' in f.read()
        f.close()
      if os.path.exists(difftemp): os.remove(difftemp)
      if not not_modified:
        any_errors = True
        print('')
        shell_call([
            'python',
            'third_party/clang-format/git-clang-format',
            '--binary=%s' % (clang_format_binary),
            '--commit=%s' % ('origin/master' if args['origin'] else 'HEAD'),
            '--diff',
            ])
        print('ERROR: 1+ diffs. Stage changes and run \'xb format\' to fix.')
        return 1
      else:
        print('Linting completed successfully.')
        return 0


class FormatCommand(Command):
  """'format' command."""

  def __init__(self, subparsers, *args, **kwargs):
    super(FormatCommand, self).__init__(
        subparsers,
        name='format',
        help_short='Reformats staged code with clang-format.',
        *args, **kwargs)
    self.parser.add_argument(
        '--all', action='store_true',
        help='Format all files, not just those changed.')
    self.parser.add_argument(
        '--origin', action='store_true',
        help='Formats all files changed relative to origin/master.')

  def execute(self, args, pass_args, cwd):
    clang_format_binary = get_clang_format_binary()

    if args['all']:
      all_files = [os.path.join(root, name)
         for root, dirs, files in os.walk('src/')
         for name in files
         if name.endswith(('.cc', '.c', '.h', '.inl'))]
      print('- clang-format [%d files]' % (len(all_files)))
      any_errors = False
      for file_path in all_files:
        ret = shell_call([
            clang_format_binary,
            '-i',
            '-style=file',
            file_path,
            ], throw_on_error=False)
        if ret:
          any_errors = True
      print('')
      if any_errors:
        print('ERROR: 1+ clang-format calls failed.')
        print('Ensure all files are staged.')
        return 1
      else:
        print('Formatting completed successfully.')
        return 0
    else:
      print('- git-clang-format')
      shell_call([
          'python',
          'third_party/clang-format/git-clang-format',
          '--binary=%s' % (clang_format_binary),
          '--commit=%s' % ('origin/master' if args['origin'] else 'HEAD'),
          ])
      print('')

    return 0


class DevenvCommand(Command):
  """'devenv' command."""

  def __init__(self, subparsers, *args, **kwargs):
    super(DevenvCommand, self).__init__(
        subparsers,
        name='devenv',
        help_short='Launches Visual Studio with the sln.',
        *args, **kwargs)

  def execute(self, args, pass_args, cwd):
    print('Launching Visual Studio...')
    print('')

    print('- running premake...')
    run_platform_premake()
    print('')

    print('- launching devenv...')
    shell_call([
        'devenv',
        'build\\xenia.sln',
        ])
    print('')

    return 0


if __name__ == '__main__':
  main()