Use a YAML file to drive dynamic builders

As a stepping-stone to having builder steps live in the main Servo repo,
switch to a simple data format to express builder configuration (lists
of strings accessed via yaml.safe_load) instead of needing to run or
import arbitrary untrusted Python code.

Currently, the YAML file is still part of the master configuration, but
it is now ready to be moved to the main Servo repo.

buildbot/master/files/config/factories.py

@@ -1,8 +1,10 @@
+import os.path
 import re
 
 from buildbot.plugins import steps, util
 from buildbot.process import buildstep
 from buildbot.status.results import SUCCESS
+import yaml
 
 import environments as envs
 
@@ -46,14 +48,19 @@ def run(self):
 
 class DynamicServoFactory(ServoFactory):
     """
-    Smart factory which takes a list of shell commands and creates the
-    appropriate Buildbot Steps. Uses heuristics to infer Step type, if
-    there are any logfiles, etc.
+    Smart factory which takes a list of shell commands from a yaml file
+    and creates the appropriate Buildbot Steps. Uses heuristics to infer
+    Step type, if there are any logfiles, etc.
     """
 
-    def __init__(self, environment, commands):
+    def __init__(self, builder_name, environment):
         self.environment = environment
         try:
+            config_dir = os.path.dirname(os.path.realpath(__file__))
+            yaml_path = os.path.join(config_dir, 'steps.yml')
+            with open(yaml_path) as steps_file:
+                builder_steps = yaml.safe_load(steps_file)
+            commands = builder_steps[builder_name]
             dynamic_steps = [self.make_step(command) for command in commands]
         except Exception as e:  # Bad step configuration, fail build
             print(str(e))
@@ -116,77 +123,3 @@ def make_step(self, command):
     steps.Compile(command=["bash", "-l", "-c", "./mach test-unit"],
                   env=envs.build_windows),
 ])
-
-mac_rel_wpt = DynamicServoFactory(envs.build_mac, [
-    "./mach test-tidy --no-progress",
-    "./mach build --release",
-    "./mach test-wpt-failure",
-    "./mach test-wpt --release --processes 4 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log",
-    "./mach build-cef --release",
-    "bash ./etc/ci/lockfile_changed.sh",
-    "bash ./etc/ci/manifest_changed.sh",
-])
-
-mac_dev_unit = DynamicServoFactory(envs.build_mac, [
-    "./mach build --dev",
-    "./mach test-unit",
-    "./mach build-cef",
-    "./mach build-geckolib",
-    "bash ./etc/ci/lockfile_changed.sh",
-    "bash ./etc/ci/manifest_changed.sh",
-])
-
-mac_rel_css = DynamicServoFactory(envs.build_mac, [
-    "./mach build --release",
-    "./mach test-css --release --processes 4 --log-raw test-css.log --log-errorsummary css-errorsummary.log",
-    "./mach build-geckolib --release",
-    "bash ./etc/ci/lockfile_changed.sh",
-    "bash ./etc/ci/manifest_changed.sh",
-])
-
-linux_dev = DynamicServoFactory(envs.build_linux_headless, [
-    "./mach test-tidy --no-progress",
-    "./mach test-tidy --no-progress --self-test",
-    "./mach build --dev",
-    "./mach test-compiletest",
-    "./mach test-unit",
-    "./mach build-cef",
-    "./mach build-geckolib",
-    "bash ./etc/ci/lockfile_changed.sh",
-    "bash ./etc/ci/manifest_changed.sh",
-])
-
-linux_rel = DynamicServoFactory(envs.build_linux_headless, [
-    "./mach build --release",
-    "./mach test-wpt-failure",
-    "./mach test-wpt --release --processes 24 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log",
-    "./mach test-css --release --processes 16 --log-raw test-css.log --log-errorsummary css-errorsummary.log",
-    "./mach build-cef --release",
-    "./mach build-geckolib --release",
-    "bash ./etc/ci/lockfile_changed.sh",
-    "bash ./etc/ci/manifest_changed.sh",
-    "bash ./etc/ci/check_no_unwrap.sh",
-])
-
-android = DynamicServoFactory(envs.build_android, [
-    "./mach build --android --dev",
-    "bash ./etc/ci/lockfile_changed.sh",
-    "bash ./etc/ci/manifest_changed.sh",
-    "python ./etc/ci/check_dynamic_symbols.py",
-])
-
-android_nightly = DynamicServoFactory(envs.build_android, [
-    "./mach build --android --release",
-    "./mach package -r",
-    ("s3cmd put "
-     "./target/arm-linux-androideabi/release/servo.apk "
-     "s3://servo-rust/nightly/servo.apk"),
-])
-
-arm32 = DynamicServoFactory(envs.build_arm32, [
-    "./mach build --rel --target=arm-unknown-linux-gnueabihf",
-])
-
-arm64 = DynamicServoFactory(envs.build_arm64, [
-    "./mach build --rel --target=aarch64-unknown-linux-gnu",
-])

buildbot/master/files/config/master.cfg

@@ -1,6 +1,7 @@
 from buildbot.plugins import buildslave, changes, schedulers, util
 from buildbot.status import html, status_push, web, words
 
+import environments as envs
 import factories
 from passwords import HTTP_USERNAME, HTTP_PASSWORD
 from passwords import SLAVE_PASSWORD, CHANGE_PASSWORD
@@ -116,83 +117,48 @@ def branch_priority(builder, requests):
     return requests[0]
 
 
-c['builders'] = []
-c['builders'].append(util.BuilderConfig(
-    name="linux-dev",
-    slavenames=LINUX_SLAVES,
-    factory=factories.linux_dev,
-    nextBuild=branch_priority,
-    category="auto",
-))
-c['builders'].append(util.BuilderConfig(
-    name="linux-rel",
-    slavenames=LINUX_SLAVES,
-    factory=factories.linux_rel,
-    nextBuild=branch_priority,
-    category="auto",
-))
-c['builders'].append(util.BuilderConfig(
-    name="android",
-    slavenames=CROSS_SLAVES,
-    factory=factories.android,
-    nextBuild=branch_priority,
-    category="auto",
-))
-c['builders'].append(util.BuilderConfig(
-    name="arm32",
-    slavenames=CROSS_SLAVES,
-    factory=factories.arm32,
-    nextBuild=branch_priority,
-    category="auto",
-))
-c['builders'].append(util.BuilderConfig(
-    name="arm64",
-    slavenames=CROSS_SLAVES,
-    factory=factories.arm64,
-    nextBuild=branch_priority,
-    category="auto",
-))
-c['builders'].append(util.BuilderConfig(
-    name="mac-rel-wpt",
-    slavenames=MAC_SLAVES,
-    factory=factories.mac_rel_wpt,
-    nextBuild=branch_priority,
-    category="auto",
-))
-c['builders'].append(util.BuilderConfig(
-    name="mac-dev-unit",
-    slavenames=MAC_SLAVES,
-    factory=factories.mac_dev_unit,
-    nextBuild=branch_priority,
-    category="auto",
-))
-c['builders'].append(util.BuilderConfig(
-    name="mac-rel-css",
-    slavenames=MAC_SLAVES,
-    factory=factories.mac_rel_css,
-    nextBuild=branch_priority,
-    category="auto",
-))
-c['builders'].append(util.BuilderConfig(
-    name="windows",
-    slavenames=WINDOWS_SLAVES,
-    factory=factories.windows,
-    nextBuild=branch_priority,
-    category="auto",
-))
-c['builders'].append(util.BuilderConfig(
-    name="doc",
-    slavenames=LINUX_SLAVES,
-    factory=factories.doc,
-    category="auto",
-))
-c['builders'].append(util.BuilderConfig(
-    name="android-nightly",
-    slavenames=CROSS_SLAVES,
-    factory=factories.android_nightly,
-    nextBuild=branch_priority,
-    category="auto",
-))
+class DynamicServoBuilder(util.BuilderConfig):
+    """
+    Builder which uses DynamicServoFactory to run steps from a yaml file.
+    """
+    def __init__(self, name, slavenames, environment):
+        # util.BuilderConfig is an old-style class so we cannot use super()
+        # but must hardcode the superclass here
+        util.BuilderConfig.__init__(
+            self,
+            name=name,
+            slavenames=slavenames,
+            factory=factories.DynamicServoFactory(name, environment),
+            nextBuild=branch_priority,
+            category="auto",
+        )
+
+
+c['builders'] = [
+    DynamicServoBuilder("linux-dev", LINUX_SLAVES, envs.build_linux_headless),
+    DynamicServoBuilder("linux-rel", LINUX_SLAVES, envs.build_linux_headless),
+    DynamicServoBuilder("android", CROSS_SLAVES, envs.build_android),
+    DynamicServoBuilder("arm32", CROSS_SLAVES, envs.build_arm32),
+    DynamicServoBuilder("arm64", CROSS_SLAVES, envs.build_arm64),
+    DynamicServoBuilder("mac-dev-unit", MAC_SLAVES, envs.build_mac),
+    DynamicServoBuilder("mac-rel-wpt", MAC_SLAVES, envs.build_mac),
+    DynamicServoBuilder("mac-rel-css", MAC_SLAVES, envs.build_mac),
+    DynamicServoBuilder("android-nightly", CROSS_SLAVES, envs.build_android),
+    # The below builders are not dynamic but rather have hard-coded factories
+    util.BuilderConfig(
+        name="windows",
+        slavenames=WINDOWS_SLAVES,
+        factory=factories.windows,
+        nextBuild=branch_priority,
+        category="auto",
+    ),
+    util.BuilderConfig(
+        name="doc",
+        slavenames=LINUX_SLAVES,
+        factory=factories.doc,
+        category="auto",
+    ),
+]
 
 
 ##################

buildbot/master/files/config/steps.yml

@@ -0,0 +1,62 @@
+mac-rel-wpt:
+  - ./mach test-tidy --no-progress
+  - ./mach build --release
+  - ./mach test-wpt-failure
+  - ./mach test-wpt --release --processes 4 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log
+  - ./mach build-cef --release
+  - bash ./etc/ci/lockfile_changed.sh
+  - bash ./etc/ci/manifest_changed.sh
+
+mac-dev-unit:
+  - ./mach build --dev
+  - ./mach test-unit
+  - ./mach build-cef
+  - ./mach build-geckolib
+  - bash ./etc/ci/lockfile_changed.sh
+  - bash ./etc/ci/manifest_changed.sh
+
+mac-rel-css:
+  - ./mach build --release
+  - ./mach test-css --release --processes 4 --log-raw test-css.log --log-errorsummary css-errorsummary.log
+  - ./mach build-geckolib --release
+  - bash ./etc/ci/lockfile_changed.sh
+  - bash ./etc/ci/manifest_changed.sh
+
+linux-dev:
+  - ./mach test-tidy --no-progress
+  - ./mach test-tidy --no-progress --self-test
+  - ./mach build --dev
+  - ./mach test-compiletest
+  - ./mach test-unit
+  - ./mach build-cef
+  - ./mach build-geckolib
+  - bash ./etc/ci/lockfile_changed.sh
+  - bash ./etc/ci/manifest_changed.sh
+
+linux-rel:
+  - ./mach build --release
+  - ./mach test-wpt-failure
+  - ./mach test-wpt --release --processes 24 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log
+  - ./mach test-css --release --processes 16 --log-raw test-css.log --log-errorsummary css-errorsummary.log
+  - ./mach build-cef --release
+  - ./mach build-geckolib --release
+  - bash ./etc/ci/lockfile_changed.sh
+  - bash ./etc/ci/manifest_changed.sh
+  - bash ./etc/ci/check_no_unwrap.sh
+
+android:
+  - ./mach build --android --dev
+  - bash ./etc/ci/lockfile_changed.sh
+  - bash ./etc/ci/manifest_changed.sh
+  - python ./etc/ci/check_dynamic_symbols.py
+
+android-nightly:
+  - ./mach build --android --release
+  - ./mach package -r
+  - s3cmd put target/arm-linux-androideabi/release/servo.apk s3://servo-rust/nightly/servo.apk
+
+arm32:
+  - ./mach build --rel --target=arm-unknown-linux-gnueabihf
+
+arm64:
+  - ./mach build --rel --target=aarch64-unknown-linux-gnu

buildbot/master/init.sls

@@ -7,6 +7,7 @@ buildbot-master:
       - service_identity == 14.0.0
       - txgithub == 15.0.0
       - boto == 2.38.0
+      - pyyaml == 3.11
     - require:
       - pkg: pip
   service.running: