Use yaml for buildbot steps

.travis/test_pillars/buildbot/master.sls

@@ -4,6 +4,5 @@ buildbot:
     'http-pass': 'TEST_BUILDBOT_HTTP_PASS'
     'change-pass': 'TEST_BUILDBOT_CHANGE_PASS'
     'gh-doc-token': 'TEST_BUILDBOT_GH_DOC_TOKEN'
-    'gh-status-token': 'TEST_BUILDBOT_GH_STATUS_TOKEN'
     'gh-hook-secret': 'TEST_BUILDBOT_GH_HOOK_SECRET'
     'homu-secret': 'TEST_BUILDBOT_HOMU_SECRET'

buildbot/master/files/config/environments.py

@@ -0,0 +1,119 @@
+from passwords import GITHUB_DOC_TOKEN
+
+
+class Environment(dict):
+    """
+    Wrapper that allows 'adding' environment dictionaries
+    to make it easy to build up environments piece by piece.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super(Environment, self).__init__(*args, **kwargs)
+
+    def __add__(self, other):
+        assert type(self) == type(other)
+        combined = self.copy()
+        combined.update(other)  # other takes precedence over self
+        return Environment(combined)
+
+
+doc = Environment({
+    'CARGO_HOME': '{{ common.servo_home }}/.cargo',
+    'SERVO_CACHE_DIR': '{{ common.servo_home }}/.servo',
+    'SHELL': '/bin/bash',
+    'TOKEN': GITHUB_DOC_TOKEN,
+})
+
+build_common = Environment({
+    'RUST_BACKTRACE': '1',
+})
+
+build_windows = build_common + Environment({
+    'MSYS': 'winsymlinks=lnk',
+    'MSYSTEM': 'MINGW64',
+    'PATH': ';'.join([
+        r'C:\msys64\mingw64\bin',
+        r'C:\msys64\usr\bin',
+        r'C:\Windows\system32',
+        r'C:\Windows',
+        r'C:\Windows\System32\Wbem',
+        r'C:\Windows\System32\WindowsPowerShell\v1.0',
+        r'C:\Program Files\Amazon\cfn-bootstrap',
+    ]),
+})
+
+build_mac = build_common + Environment({
+    'CARGO_HOME': '/Users/servo/.cargo',
+    'CCACHE': '/usr/local/bin/ccache',
+    'SERVO_CACHE_DIR': '/Users/servo/.servo',
+})
+
+
+build_linux = build_common + Environment({
+    'CARGO_HOME': '{{ common.servo_home }}/.cargo',
+    'CCACHE': '/usr/bin/ccache',
+    'DISPLAY': ':0',
+    'SERVO_CACHE_DIR': '{{ common.servo_home }}/.servo',
+    'SHELL': '/bin/bash',
+})
+
+build_linux_headless = build_linux + Environment({
+    'SERVO_HEADLESS': '1',
+})
+
+build_android = build_linux + Environment({
+    'ANDROID_NDK': '{{ common.servo_home }}/android/ndk/current/',
+    'ANDROID_SDK': '{{ common.servo_home }}/android/sdk/current/',
+    'ANDROID_TOOLCHAIN': '{{ common.servo_home }}/android/toolchain/current/',
+    'PATH': ':'.join([
+        '/usr/local/sbin',
+        '/usr/local/bin',
+        '/usr/bin',
+        '/usr/sbin',
+        '/sbin',
+        '/bin',
+        '{{ common.servo_home }}/android/sdk/current/platform-tools',
+        '{{ common.servo_home }}/android/toolchain/current/bin',
+    ]),
+})
+
+build_arm = build_linux + Environment({
+    'EXPAT_NO_PKG_CONFIG': '1',
+    'FONTCONFIG_NO_PKG_CONFIG': '1',
+    'FREETYPE2_NO_PKG_CONFIG': '1',
+    'PKG_CONFIG_ALLOW_CROSS': '1',
+})
+
+
+build_arm32 = build_arm + Environment({
+    'BUILD_TARGET': 'arm-unknown-linux-gnueabihf',
+    'CC': 'arm-linux-gnueabihf-gcc',
+    'CXX': 'arm-linux-gnueabihf-g++',
+    'PATH': ':'.join([
+        '{{ common.servo_home }}/bin',
+        '/usr/local/sbin',
+        '/usr/local/bin',
+        '/usr/bin',
+        '/usr/sbin',
+        '/sbin',
+        '/bin',
+    ]),
+    'PKG_CONFIG_PATH': '/usr/lib/arm-linux-gnueabihf/pkgconfig',
+})
+
+build_arm64 = build_arm + Environment({
+    'BUILD_TARGET': 'aarch64-unknown-linux-gnu',
+    'CC': 'aarch64-linux-gnu-gcc',
+    'CXX': 'aarch64-linux-gnu-g++',
+    'PATH': ':'.join([
+        '{{ common.servo_home }}/bin',
+        '/usr/local/sbin',
+        '/usr/local/bin',
+        '/usr/bin',
+        '/usr/sbin',
+        '/sbin',
+        '/bin',
+    ]),
+    'PKG_CONFIG_PATH': '/usr/lib/aarch64-linux-gnu/pkgconfig',
+    'SERVO_RUSTC_WITH_GOLD': 'False',
+})

buildbot/master/files/config/factories.py

@@ -0,0 +1,126 @@
+import os.path
+import re
+
+from buildbot.plugins import steps, util
+from buildbot.process import buildstep
+from buildbot.status.results import SUCCESS
+import yaml
+
+import environments as envs
+
+
+SERVO_REPO = "https://github.com/servo/servo"
+
+
+class ServoFactory(util.BuildFactory):
+    """
+    Build factory which checks out the servo repo as the first build step.
+    """
+
+    def __init__(self, build_steps):
+        """
+        Takes a list of Buildbot steps.
+        Prefer using DynamicServoFactory to using this class directly.
+        """
+        all_steps = [
+            steps.Git(repourl=SERVO_REPO, mode="full", method="clobber"),
+        ] + build_steps
+        # util.BuildFactory is an old-style class so we cannot use super()
+        # but must hardcode the superclass here
+        util.BuildFactory.__init__(self, all_steps)
+
+
+class BadConfigurationStep(buildstep.BuildStep):
+    """
+    Step which immediately fails the build due to a bad configuration.
+    """
+
+    haltOnFailure = True
+    flunkOnFailure = True
+
+    def __init__(self, exception):
+        self.exception = exception
+
+    def run(self):
+        raise Exception("Bad configuration, unable to convert to steps" +
+                        str(self.exception))
+
+
+class DynamicServoFactory(ServoFactory):
+    """
+    Smart factory which takes a list of shell commands from a yaml file
+    and creates the appropriate Buildbot Steps. Uses heuristics to infer
+    Step type, if there are any logfiles, etc.
+    """
+
+    def __init__(self, builder_name, environment):
+        self.environment = environment
+        try:
+            config_dir = os.path.dirname(os.path.realpath(__file__))
+            yaml_path = os.path.join(config_dir, 'steps.yml')
+            with open(yaml_path) as steps_file:
+                builder_steps = yaml.safe_load(steps_file)
+            commands = builder_steps[builder_name]
+            dynamic_steps = [self.make_step(command) for command in commands]
+        except Exception as e:  # Bad step configuration, fail build
+            print(str(e))
+            dynamic_steps = [BadConfigurationStep(e)]
+
+        # TODO: windows compatibility (use a custom script for this?)
+        pkill_step = [steps.ShellCommand(command=["pkill", "-x", "servo"],
+                                         decodeRC={0: SUCCESS, 1: SUCCESS})]
+
+        # util.BuildFactory is an old-style class so we cannot use super()
+        # but must hardcode the superclass here
+        ServoFactory.__init__(self, pkill_step + dynamic_steps)
+
+    def make_step(self, command):
+        step_kwargs = {}
+        step_kwargs['env'] = self.environment
+
+        command = command.split(' ')
+        step_kwargs['command'] = command
+
+        step_class = steps.ShellCommand
+        args = iter(command)
+        for arg in args:
+            # Change Step class to capture warnings as needed
+            # (steps.Compile and steps.Test catch warnings)
+            if arg == './mach':
+                mach_arg = next(args)
+                if re.match('build(-.*)?', mach_arg):
+                    step_class = steps.Compile
+                elif re.match('test-.*', mach_arg):
+                    step_class = steps.Test
+
+            # Capture any logfiles
+            elif re.match('--log-.*', arg):
+                logfile = next(args)
+                if 'logfiles' not in step_kwargs:
+                    step_kwargs['logfiles'] = {}
+                step_kwargs['logfiles'][logfile] = logfile
+
+        return step_class(**step_kwargs)
+
+
+doc = ServoFactory([
+    # This is not dynamic because a) we need to pass the logEnviron kwarg
+    # and b) changes to the documentation build are already encapsulated
+    # in the upload_docs.sh script; any further changes should go through
+    # the saltfs repo to avoid leaking the token.
+    steps.ShellCommand(command=["etc/ci/upload_docs.sh"],
+                       env=envs.doc,
+                       # important not to leak token
+                       logEnviron=False),
+])
+
+windows = ServoFactory([
+    # TODO: convert this to use DynamicServoFactory
+    # We need to run each command in a bash login shell, which breaks the
+    # heuristics used by DynamicServoFactory.make_step
+    steps.Compile(command=["bash", "-l", "-c", "./mach build -d -v"],
+                  env=envs.build_windows),
+    steps.Compile(command=["bash", "-l", "-c", "./mach test-unit"],
+                  env=envs.build_windows),
+    # TODO: run lockfile_changed.sh and manifest_changed.sh scripts
+])