Add test to check Buildbot master config

This test can be run via `./test.py sls.buildbot.master`. It will only
work properly once the Buildbot master states have been run, and is
meant to be run either in Vagrant or on Travis, not locally.

Running `./test.py` without arguments will only run the linting tests, since
those can be run anywhere with just the source code.

For now, hard-code this test to be run on the servo-master1 node on Travis.
This should be replaced with a more intelligent test dispatcher in `test.py`
which uses the top file via Salt's state.show_top in the future, but hard
coding this for now will reduce Buildbot's fragility.

.travis.yml

@@ -20,6 +20,8 @@ matrix:
       os: linux
       sudo: required
       dist: trusty
+      language: python
+      python: 3.5
     - env: SALT_NODE_ID=test # Not a Salt node, runs test suite instead
       os: linux
       sudo: required

.travis/dispatch.sh

@@ -20,4 +20,9 @@ else
     sudo salt-call --id="${SALT_NODE_ID}" --retcode-passthrough state.show_highstate
     # Full on installation test
     sudo salt-call --id="${SALT_NODE_ID}" --retcode-passthrough --log-level=warning state.highstate
+
+    # TODO: don't hard-code this
+    if [ "${SALT_NODE_ID}" = "servo-master1" ]; then
+        ./test.py sls.buildbot.master
+    fi
 fi

test.py

@@ -7,35 +7,48 @@
 from tests.util import color, GREEN, RED, Failure, project_path
 
 
-def is_python_script(path):
-    return path.endswith('.py')
+def is_python_script(dir_entry):
+    return dir_entry.name.endswith('.py') and dir_entry.is_file()
+
+
+def run_tests(tests):
+    any_failures = False
+
+    for test_spec in tests:
+        test_dir = os.path.join(project_path(), 'tests', *test_spec.split('.'))
+
+        python_scripts = filter(is_python_script, os.scandir(test_dir))
+        tests = sorted([entry.name for entry in python_scripts])
+
+        for test in tests:
+            test_mod_name = 'tests.{}.{}'.format(test_spec, test[:-3])
+            test_mod = importlib.import_module(test_mod_name)
+            if not hasattr(test_mod, 'run'):  # Not a test script
+                continue
+
+            try:
+                result = test_mod.run()
+            except Exception as e:
+                message = 'Test \'{}\' raised an exception:'.format(test)
+                result = Failure(message, str(e))
+
+            if result.is_success():
+                print('[ {} ] {}'.format(color(GREEN, 'PASS'), result.message))
+            else:
+                any_failures = True
+                print('[ {} ] {}'.format(color(RED, 'FAIL'), result.message))
+                for line in result.output.splitlines():
+                    print('         {}'.format(line))
+
+    return 1 if any_failures else 0
 
 
 def main():
-    ANY_FAILURES = False
-
-    test_dir = os.path.join(project_path(), 'tests')
-    tests = sorted(filter(is_python_script, os.listdir(test_dir)))
-    for test in tests:
-        test_mod = importlib.import_module('tests.{}'.format(test[:-3]))
-        if not hasattr(test_mod, 'run'):  # Not a test script
-            continue
-
-        try:
-            result = test_mod.run()
-        except Exception as e:
-            result = Failure('Test \'{}\' raised an exception:'.format(test),
-                             str(e))
-
-        if result.is_success():
-            print('[ {} ] {}'.format(color(GREEN, 'PASS'), result.message))
-        else:
-            ANY_FAILURES = True
-            print('[ {} ] {}'.format(color(RED, 'FAIL'), result.message))
-            for line in result.output.splitlines():
-                print('         {}'.format(line))
-
-    return 1 if ANY_FAILURES else 0
+    tests = ['lint']  # Only tests that are always safe and meaningful to run
+    if len(sys.argv) > 1:
+        tests = sys.argv[1:]
+
+    return run_tests(tests)
 
 if __name__ == '__main__':
     sys.exit(main())

tests/shebang.py → tests/lint/shebang.py

@@ -1,7 +1,7 @@
 import os
 import stat
 
-from .util import display_path, paths, Failure, Success
+from tests.util import display_path, paths, Failure, Success
 
 
 SHEBANG = """\

tests/trailing_whitespace.py → tests/lint/trailing_whitespace.py

@@ -1,7 +1,7 @@
 import itertools
 import re
 
-from .util import colon, color, display_path, paths, Failure, Success
+from tests.util import colon, color, display_path, paths, Failure, Success
 
 
 def display_trailing_whitespace(whitespace):

tests/sls/buildbot/master/config.py

@@ -0,0 +1,17 @@
+import subprocess
+
+from tests.util import Failure, Success
+
+
+def run():
+    command = ['sudo',  # To get access to buildbot files owned by servo
+               'buildbot', 'checkconfig', '/home/servo/buildbot/master']
+    ret = subprocess.run(command,
+                         stdout=subprocess.PIPE,
+                         stderr=subprocess.PIPE,
+                         universal_newlines=True)
+
+    if ret.returncode == 0:
+        return Success("Buildbot master config passed checkconfig")
+    else:
+        return Failure("Buildbot master config check failed:", ret.stderr)