Skip to content

/ saltfs

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use yaml for buildbot steps #325

Merged
merged 12 commits into from Apr 20, 2016
+492 −332
Merged

Use yaml for buildbot steps #325

Changes from 1 commit
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
caa1d41
Remove gonk
aneeshusa Apr 16, 2016
36b16ff
Temporarily remove pkill steps
aneeshusa Apr 16, 2016
ceff384
Remove gzip steps
aneeshusa Apr 16, 2016
451f3f9
Create separate file for environments
aneeshusa Apr 16, 2016
85979c4
Create separate file for steps
aneeshusa Apr 16, 2016
d895550
Re-add pkill step
aneeshusa Apr 18, 2016
90937e6
Remove GITHUB_STATUS_TOKEN
aneeshusa Apr 17, 2016
056ed34
Clean up master.cfg to pass flake8
aneeshusa Apr 17, 2016
cf3adb1
Use a YAML file to drive dynamic builders
aneeshusa Apr 18, 2016
9d71e2d
Tweak testing steps
aneeshusa Apr 18, 2016
bab18c3
Add test for buildbot configuration passing flake8
aneeshusa Apr 18, 2016
acf76a1
Add flake8 lint test for other python files
aneeshusa Apr 19, 2016
File filter...
Filter file types
Clear filters
Jump to…
Jump to file
Failed to load files.

Always

Just for now

Prev Next

Create separate file for environments 

The existing mechanism for chaining environment dictionaries is a hack
that Guido has noted as abuse of the '**' mechanism:
https://mail.python.org/pipermail/python-dev/2010-April/099459.html

Create a subclass that allows 'adding' in a single expression, as
copy() + update() takes two lines.

Use str.join for listing PATHs to keep lines short. This also exposed
an error in the Windows PATH, with a missing semicolon between
r'C:\msys64\usr\bin' and r'C:\Windows\system32'.

Move the environments into a separate file to keep them all together.
Rename the various environment variables for clarity and consistency.

The environments.py file now passes flake8; this should be added to our
test suite.
  • Loading branch information
@aneeshusa
aneeshusa committed Apr 20, 2016
commit 451f3f974da71983c040e322cbf591eb74a640b9
119 buildbot/master/files/config/environments.py
View file
Open in desktop
@@ -0,0 +1,119 @@
from passwords import GITHUB_DOC_TOKEN


class Environment(dict):
"""
Wrapper that allows 'adding' environment dictionaries
to make it easy to build up environments piece by piece.
"""

def __init__(self, *args, **kwargs):
super(Environment, self).__init__(*args, **kwargs)

def __add__(self, other):
assert type(self) == type(other)
combined = self.copy()
combined.update(other) # other takes precedence over self
return Environment(combined)


doc = Environment({
'CARGO_HOME': '{{ common.servo_home }}/.cargo',
'SERVO_CACHE_DIR': '{{ common.servo_home }}/.servo',
'SHELL': '/bin/bash',
'TOKEN': GITHUB_DOC_TOKEN,
})

build_common = Environment({
'RUST_BACKTRACE': '1',
})

build_windows = build_common + Environment({
'MSYS': 'winsymlinks=lnk',
'MSYSTEM': 'MINGW64',
'PATH': ';'.join([
r'C:\msys64\mingw64\bin',
r'C:\msys64\usr\bin',
r'C:\Windows\system32',
r'C:\Windows',
r'C:\Windows\System32\Wbem',
r'C:\Windows\System32\WindowsPowerShell\v1.0',
r'C:\Program Files\Amazon\cfn-bootstrap',
]),
})

build_mac = build_common + Environment({
'CARGO_HOME': '/Users/servo/.cargo',
'CCACHE': '/usr/local/bin/ccache',
'SERVO_CACHE_DIR': '/Users/servo/.servo',
})


build_linux = build_common + Environment({
'CARGO_HOME': '{{ common.servo_home }}/.cargo',
'CCACHE': '/usr/bin/ccache',
'DISPLAY': ':0',
'SERVO_CACHE_DIR': '{{ common.servo_home }}/.servo',
'SHELL': '/bin/bash',
})

build_linux_headless = build_linux + Environment({
'SERVO_HEADLESS': '1',
})

build_android = build_linux + Environment({
'ANDROID_NDK': '{{ common.servo_home }}/android/ndk/current/',
'ANDROID_SDK': '{{ common.servo_home }}/android/sdk/current/',
'ANDROID_TOOLCHAIN': '{{ common.servo_home }}/android/toolchain/current/',
'PATH': ':'.join([
'/usr/local/sbin',
'/usr/local/bin',
'/usr/bin',
'/usr/sbin',
'/sbin',
'/bin',
'{{ common.servo_home }}/android/sdk/current/platform-tools',
'{{ common.servo_home }}/android/toolchain/current/bin',
]),
})

build_arm = build_linux + Environment({
'EXPAT_NO_PKG_CONFIG': '1',
'FONTCONFIG_NO_PKG_CONFIG': '1',
'FREETYPE2_NO_PKG_CONFIG': '1',
'PKG_CONFIG_ALLOW_CROSS': '1',
})


build_arm32 = build_arm + Environment({
'BUILD_TARGET': 'arm-unknown-linux-gnueabihf',
'CC': 'arm-linux-gnueabihf-gcc',
'CXX': 'arm-linux-gnueabihf-g++',
'PATH': ':'.join([
'{{ common.servo_home }}/bin',
'/usr/local/sbin',
'/usr/local/bin',
'/usr/bin',
'/usr/sbin',
'/sbin',
'/bin',
]),
'PKG_CONFIG_PATH': '/usr/lib/arm-linux-gnueabihf/pkgconfig',
})

build_arm64 = build_arm + Environment({
'BUILD_TARGET': 'aarch64-unknown-linux-gnu',
'CC': 'aarch64-linux-gnu-gcc',
'CXX': 'aarch64-linux-gnu-g++',
'PATH': ':'.join([
'{{ common.servo_home }}/bin',
'/usr/local/sbin',
'/usr/local/bin',
'/usr/bin',
'/usr/sbin',
'/sbin',
'/bin',
]),
'PKG_CONFIG_PATH': '/usr/lib/aarch64-linux-gnu/pkgconfig',
'SERVO_RUSTC_WITH_GOLD': 'False',
})
148 buildbot/master/files/config/master.cfg
View file
Open in desktop
@@ -3,8 +3,9 @@

from buildbot.plugins import *

import environments
from passwords import HTTP_USERNAME, HTTP_PASSWORD, SLAVE_PASSWORD, CHANGE_PASSWORD
from passwords import GITHUB_DOC_TOKEN, GITHUB_STATUS_TOKEN, HOMU_BUILDBOT_SECRET
from passwords import GITHUB_STATUS_TOKEN, HOMU_BUILDBOT_SECRET

SERVO_REPO = "https://github.com/servo/servo"
LINUX_RESERVED_SLAVES = ["servo-linux1", "servo-linux2"]
@@ -83,161 +84,108 @@ def create_servo_factory(commands):
] + commands)


common_test_env = {
'RUST_BACKTRACE': '1'
}

linux_test_env = dict({
'DISPLAY': ':0',
'CARGO_HOME': '{{ common.servo_home }}/.cargo',
'CCACHE': '/usr/bin/ccache',
'SERVO_CACHE_DIR': '{{ common.servo_home }}/.servo',
'SHELL': '/bin/bash'
}, **common_test_env)
linux_headless_env = dict({'SERVO_HEADLESS': '1'}, **linux_test_env)

mac_test_env = dict({
'CARGO_HOME': '/Users/servo/.cargo',
'CCACHE': '/usr/local/bin/ccache',
'SERVO_CACHE_DIR': '/Users/servo/.servo'
}, **common_test_env)

linux_dev_factory = create_servo_factory([
steps.ShellCommand(command=["./mach", "test-tidy", "--no-progress"], env=linux_headless_env),
steps.ShellCommand(command=["./mach", "test-tidy", "--no-progress", "--self-test"], env=linux_headless_env),
steps.Compile(command=["./mach", "build", "--dev"], env=linux_headless_env),
steps.ShellCommand(command=["./mach", "test-compiletest"], env=linux_headless_env),
steps.ShellCommand(command=["./mach", "test-unit"], env=linux_headless_env),
steps.Compile(command=["./mach", "build-cef"], env=linux_headless_env),
steps.Compile(command=["./mach", "build-geckolib"], env=linux_headless_env),
steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=linux_headless_env),
steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=linux_headless_env),
steps.ShellCommand(command=["./mach", "test-tidy", "--no-progress"], env=environments.build_linux_headless),
steps.ShellCommand(command=["./mach", "test-tidy", "--no-progress", "--self-test"], env=environments.build_linux_headless),
steps.Compile(command=["./mach", "build", "--dev"], env=environments.build_linux_headless),
steps.ShellCommand(command=["./mach", "test-compiletest"], env=environments.build_linux_headless),
steps.ShellCommand(command=["./mach", "test-unit"], env=environments.build_linux_headless),
steps.Compile(command=["./mach", "build-cef"], env=environments.build_linux_headless),
steps.Compile(command=["./mach", "build-geckolib"], env=environments.build_linux_headless),
steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=environments.build_linux_headless),
steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=environments.build_linux_headless),
])

linux_rel_factory = create_servo_factory([
steps.Compile(command=["./mach", "build", "--release"], env=linux_headless_env),
steps.ShellCommand(command=["./mach", "test-wpt-failure"], env=linux_headless_env),
steps.Compile(command=["./mach", "build", "--release"], env=environments.build_linux_headless),
steps.ShellCommand(command=["./mach", "test-wpt-failure"], env=environments.build_linux_headless),
steps.ShellCommand(command=["./mach", "test-wpt", "--release", "--processes", "24",
"--log-raw", "test-wpt.log",
"--log-errorsummary", "wpt_errorsummary.log"],
env=linux_headless_env,
env=environments.build_linux_headless,
logfiles={"test-wpt.log": "test-wpt.log",
"wpt_errorsummary.log": "wpt_errorsummary.log"}),
steps.ShellCommand(command=["./mach", "test-css", "--release", "--processes", "16",
"--log-raw", "test-css.log",
"--log-errorsummary", "css_errorsummary.log"],
env=linux_headless_env,
env=environments.build_linux_headless,
logfiles={"test-css.log": "test-css.log",
"css_errorsummary.log": "css_errorsummary.log"}),
steps.Compile(command=["./mach", "build-cef", "--release"], env=linux_headless_env),
steps.Compile(command=["./mach", "build-geckolib", "--release"], env=linux_headless_env),
steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=linux_headless_env),
steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=linux_headless_env),
steps.ShellCommand(command=["bash", "./etc/ci/check_no_unwrap.sh"], env=linux_headless_env),
steps.Compile(command=["./mach", "build-cef", "--release"], env=environments.build_linux_headless),
steps.Compile(command=["./mach", "build-geckolib", "--release"], env=environments.build_linux_headless),
steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=environments.build_linux_headless),
steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=environments.build_linux_headless),
steps.ShellCommand(command=["bash", "./etc/ci/check_no_unwrap.sh"], env=environments.build_linux_headless),
])

mac_rel_wpt_factory = create_servo_factory([
steps.ShellCommand(command=["./mach", "test-tidy", "--no-progress"]),
steps.Compile(command=["./mach", "build", "--release"], env=mac_test_env),
steps.ShellCommand(command=["./mach", "test-wpt-failure"], env=mac_test_env),
steps.Compile(command=["./mach", "build", "--release"], env=environments.build_mac),
steps.ShellCommand(command=["./mach", "test-wpt-failure"], env=environments.build_mac),
steps.ShellCommand(command=["./mach", "test-wpt", "--release", "--processes", "4",
"--log-raw", "test-wpt.log",
"--log-errorsummary", "wpt_errorsummary.log"],
env=mac_test_env,
env=environments.build_mac,
logfiles={"test-wpt.log": "test-wpt.log",
"wpt_errorsummary.log": "wpt_errorsummary.log"}),
steps.ShellCommand(command=["./mach", "build-cef", "--release"], env=mac_test_env),
steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=mac_test_env),
steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=mac_test_env),
steps.ShellCommand(command=["./mach", "build-cef", "--release"], env=environments.build_mac),
steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=environments.build_mac),
steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=environments.build_mac),
])

mac_dev_unit_factory = create_servo_factory([
steps.Compile(command=["./mach", "build", "--dev"], env=mac_test_env),
steps.ShellCommand(command=["./mach", "test-unit"], env=mac_test_env),
steps.Compile(command=["./mach", "build-cef"], env=mac_test_env),
steps.Compile(command=["./mach", "build-geckolib"], env=mac_test_env),
steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=mac_test_env),
steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=mac_test_env),
steps.Compile(command=["./mach", "build", "--dev"], env=environments.build_mac),
steps.ShellCommand(command=["./mach", "test-unit"], env=environments.build_mac),
steps.Compile(command=["./mach", "build-cef"], env=environments.build_mac),
steps.Compile(command=["./mach", "build-geckolib"], env=environments.build_mac),
steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=environments.build_mac),
steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=environments.build_mac),
])

mac_rel_css_factory = create_servo_factory([
steps.Compile(command=["./mach", "build", "--release"], env=mac_test_env),
steps.Compile(command=["./mach", "build", "--release"], env=environments.build_mac),
steps.ShellCommand(command=["./mach", "test-css", "--release", "--processes", "4",
"--log-raw", "test-css.log",
"--log-errorsummary", "css_errorsummary.log"],
env=mac_test_env,
env=environments.build_mac,
logfiles={"test-css.log": "test-css.log",
"css_errorsummary.log": "css_errorsummary.log"}),
steps.ShellCommand(command=["./mach", "build-geckolib", "--release"], env=mac_test_env),
steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=mac_test_env),
steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=mac_test_env),
steps.ShellCommand(command=["./mach", "build-geckolib", "--release"], env=environments.build_mac),
steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=environments.build_mac),
steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=environments.build_mac),
])

android_compile_env = dict({'ANDROID_SDK': '{{ common.servo_home }}/android/sdk/current/',
'ANDROID_NDK': '{{ common.servo_home }}/android/ndk/current/',
'ANDROID_TOOLCHAIN': '{{ common.servo_home }}/android/toolchain/current/',
'PATH': '/usr/local/sbin:/usr/local/bin:/usr/bin:/usr/sbin:/sbin:/bin:{{ common.servo_home }}/android/sdk/current/platform-tools:{{ common.servo_home }}/android/toolchain/current/bin'},
**linux_test_env)
android_factory = create_servo_factory([
steps.Compile(command=["./mach", "build", "--android", "--dev"], env=android_compile_env),
steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=android_compile_env),
steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=android_compile_env),
steps.ShellCommand(command=["python", "./etc/ci/check_dynamic_symbols.py"], env=android_compile_env),
steps.Compile(command=["./mach", "build", "--android", "--dev"], env=environments.build_android),
steps.ShellCommand(command=["bash", "./etc/ci/lockfile_changed.sh"], env=environments.build_android),
steps.ShellCommand(command=["bash", "./etc/ci/manifest_changed.sh"], env=environments.build_android),
steps.ShellCommand(command=["python", "./etc/ci/check_dynamic_symbols.py"], env=environments.build_android),
])

android_nightly_factory = create_servo_factory([
steps.Compile(command=["./mach", "build", "--android", "--release"], env=android_compile_env),
steps.Compile(command=["./mach", "package", "-r"], env=android_compile_env),
steps.Compile(command=["./mach", "build", "--android", "--release"], env=environments.build_android),
steps.Compile(command=["./mach", "package", "-r"], env=environments.build_android),
steps.ShellCommand(command=["s3cmd", "put",
"{{ common.servo_home }}/buildbot/slave/android-nightly/build/target/arm-linux-androideabi/release/servo.apk",
"s3://servo-rust/nightly/servo.apk"]),
])

arm32_compile_env = dict({'PATH': '{{ common.servo_home }}/bin:/usr/local/sbin:/usr/local/bin:/usr/bin:/usr/sbin:/sbin:/bin',
'BUILD_TARGET': 'arm-unknown-linux-gnueabihf',
'PKG_CONFIG_ALLOW_CROSS': '1',
'PKG_CONFIG_PATH': '/usr/lib/arm-linux-gnueabihf/pkgconfig',
'EXPAT_NO_PKG_CONFIG': '1',
'FREETYPE2_NO_PKG_CONFIG': '1',
'FONTCONFIG_NO_PKG_CONFIG': '1',
'CC': 'arm-linux-gnueabihf-gcc',
'CXX': 'arm-linux-gnueabihf-g++'},
**linux_test_env)
arm32_factory = create_servo_factory([
steps.Compile(command=["./mach", "build", "--rel", "--target=arm-unknown-linux-gnueabihf"], env=arm32_compile_env),
steps.Compile(command=["./mach", "build", "--rel", "--target=arm-unknown-linux-gnueabihf"], env=environments.build_arm32),
])

arm64_compile_env = dict({'PATH': '{{ common.servo_home }}/bin:/usr/local/sbin:/usr/local/bin:/usr/bin:/usr/sbin:/sbin:/bin',
'BUILD_TARGET': 'aarch64-unknown-linux-gnu',
'PKG_CONFIG_ALLOW_CROSS': '1',
'PKG_CONFIG_PATH': '/usr/lib/aarch64-linux-gnu/pkgconfig',
'EXPAT_NO_PKG_CONFIG': '1',
'FREETYPE2_NO_PKG_CONFIG': '1',
'FONTCONFIG_NO_PKG_CONFIG': '1',
'CC': 'aarch64-linux-gnu-gcc',
'SERVO_RUSTC_WITH_GOLD': 'False',
'CXX': 'aarch64-linux-gnu-g++'},
**linux_test_env)
arm64_factory = create_servo_factory([
steps.Compile(command=["./mach", "build", "--rel", "--target=aarch64-unknown-linux-gnu"], env=arm64_compile_env),
steps.Compile(command=["./mach", "build", "--rel", "--target=aarch64-unknown-linux-gnu"], env=environments.build_arm64),
])

windows_compile_env = dict({'PATH': r'C:\msys64\mingw64\bin;C:\msys64\usr\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0;C:\Program Files\Amazon\cfn-bootstrap',
'MSYSTEM': 'MINGW64',
'MSYS': 'winsymlinks=lnk'},
**common_test_env)

windows_factory = create_servo_factory([
steps.Compile(command=["bash", "-l", "-c", "./mach build -d -v"], env=windows_compile_env),
steps.Compile(command=["bash", "-l", "-c", "./mach test-unit"], env=windows_compile_env),
steps.Compile(command=["bash", "-l", "-c", "./mach build -d -v"], env=environments.build_windows),
steps.Compile(command=["bash", "-l", "-c", "./mach test-unit"], env=environments.build_windows),
])

doc_factory = create_servo_factory([
steps.ShellCommand(command=["etc/ci/upload_docs.sh"],
env={'TOKEN': GITHUB_DOC_TOKEN,
'CARGO_HOME': '{{ common.servo_home }}/.cargo',
'SERVO_CACHE_DIR': '{{ common.servo_home }}/.servo',
'SHELL': '/bin/bash'},
env=environments.doc,
# important not to leak token
logEnviron=False),
])
ProTip! Use n and p to navigate between commits in a pull request.
You can’t perform that action at this time.