Add linux and osx nightly builds

.travis/test_pillars/buildbot/master.sls

@@ -6,3 +6,5 @@ buildbot:
     'gh-doc-token': 'TEST_BUILDBOT_GH_DOC_TOKEN'
     'gh-hook-secret': 'TEST_BUILDBOT_GH_HOOK_SECRET'
     'homu-secret': 'TEST_BUILDBOT_HOMU_SECRET'
+    's3-upload-access-key-id': 'TEST_BUILDBOT_S3_UPLOAD_ACCESS_KEY_ID'
+    's3-upload-secret-access-key': 'TEST_BUILDBOT_S3_UPLOAD_SECRET_ACCESS_KEY'

buildbot/master/files/config/factories.py

@@ -1,3 +1,4 @@
+import copy
 import os.path
 import re
 
@@ -8,6 +9,7 @@
 import yaml
 
 import environments as envs
+from passwords import S3_UPLOAD_ACCESS_KEY_ID, S3_UPLOAD_SECRET_ACCESS_KEY
 
 
 SERVO_REPO = "https://github.com/servo/servo"
@@ -78,7 +80,7 @@ def __init__(self, builder_name, environment):
 
     def make_step(self, command):
         step_kwargs = {}
-        step_kwargs['env'] = self.environment
+        step_env = copy.deepcopy(self.environment)
 
         command = command.split(' ')
         step_kwargs['command'] = command
@@ -102,6 +104,13 @@ def make_step(self, command):
                     step_kwargs['logfiles'] = {}
                 step_kwargs['logfiles'][logfile] = logfile
 
+            # Provide environment variables for s3cmd
+            elif arg == './etc/ci/upload_nightly.sh':
+                step_kwargs['logEnviron'] = False
+                step_env['AWS_ACCESS_KEY_ID'] = S3_UPLOAD_ACCESS_KEY_ID
+                step_env['AWS_SECRET_ACCESS_KEY'] = S3_UPLOAD_SECRET_ACCESS_KEY
+
+        step_kwargs['env'] = self.environment
         return step_class(**step_kwargs)
 
 
@@ -158,7 +167,7 @@ def run(self):
 
     def make_step(self, command):
         step_kwargs = {}
-        step_kwargs['env'] = self.environment
+        step_env = copy.deepcopy(self.environment)
 
         command = command.split(' ')
         step_kwargs['command'] = command
@@ -182,6 +191,13 @@ def make_step(self, command):
                     step_kwargs['logfiles'] = {}
                 step_kwargs['logfiles'][logfile] = logfile
 
+            # Provide environment variables for s3cmd
+            elif arg == './etc/ci/upload_nightly.sh':
+                step_kwargs['logEnviron'] = False
+                step_env['AWS_ACCESS_KEY_ID'] = S3_UPLOAD_ACCESS_KEY_ID
+                step_env['AWS_SECRET_ACCESS_KEY'] = S3_UPLOAD_SECRET_ACCESS_KEY
+
+        step_kwargs['env'] = self.environment
         return step_class(**step_kwargs)
 
 

buildbot/master/files/config/master.cfg

@@ -85,23 +85,27 @@ c['schedulers'].append(schedulers.SingleBranchScheduler(
 c['schedulers'].append(schedulers.ForceScheduler(
     name="force",
     builderNames=[
+        "android",
+        "android-nightly",
+        "arm32",
+        "arm64",
         "linux-dev",
-        "linux-rel",
         "linux-dev-yaml",
-        "mac-rel-wpt",
+        "linux-rel",
         "mac-dev-unit",
         "mac-rel-css",
-        "android",
-        "arm32",
-        "arm64",
-        "android-nightly",
+        "mac-rel-wpt",
         "windows",
     ],
 ))
 c['schedulers'].append(schedulers.Nightly(
     name="Nightly",
     branch="master",
-    builderNames=["android-nightly"],
+    builderNames=[
+        "android-nightly",
+        "linux-nightly",
+        "mac-nightly",
+    ],
     hour=1,
     minute=0,
 ))
@@ -156,12 +160,14 @@ class DynamicServoYAMLBuilder(util.BuilderConfig):
 c['builders'] = [
     DynamicServoBuilder("linux-dev", LINUX_SLAVES, envs.build_linux),
     DynamicServoBuilder("linux-rel", LINUX_SLAVES, envs.build_linux),
+    DynamicServoBuilder("linux-nightly", LINUX_SLAVES, envs.build_linux),
     DynamicServoBuilder("android", CROSS_SLAVES, envs.build_android),
     DynamicServoBuilder("arm32", CROSS_SLAVES, envs.build_arm32),
     DynamicServoBuilder("arm64", CROSS_SLAVES, envs.build_arm64),
     DynamicServoBuilder("mac-dev-unit", MAC_SLAVES, envs.build_mac),
     DynamicServoBuilder("mac-rel-wpt", MAC_SLAVES, envs.build_mac),
     DynamicServoBuilder("mac-rel-css", MAC_SLAVES, envs.build_mac),
+    DynamicServoBuilder("mac-nightly", MAC_SLAVES, envs.build_mac),
     DynamicServoBuilder("android-nightly", CROSS_SLAVES, envs.build_android),
     # The below builders are not dynamic but rather have hard-coded factories
     util.BuilderConfig(

buildbot/master/files/config/passwords.py

@@ -1,6 +1,18 @@
-HTTP_USERNAME = "{{ pillar['buildbot']['credentials']['http-user'] }}"
-HTTP_PASSWORD = "{{ pillar['buildbot']['credentials']['http-pass'] }}"
-SLAVE_PASSWORD = "{{ pillar['buildbot']['credentials']['slave-pass'] }}"
-CHANGE_PASSWORD = "{{ pillar['buildbot']['credentials']['change-pass'] }}"
-GITHUB_DOC_TOKEN = "{{pillar['buildbot']['credentials']['gh-doc-token'] }}"
-HOMU_BUILDBOT_SECRET = "{{pillar['buildbot']['credentials']['homu-secret'] }}"
+import json
+
+# Jinja will replace the inside with double-quote-using JSON,
+# so use single quotes to delimit the string.
+# Use double quotes inside to keep the expression as a single string.
+credentials = json.loads('{{ pillar["buildbot"]["credentials"]|json }}')
+# json.loads creates unicode strings but Buildbot requires bytestrings.
+# Python 2's Unicode situation makes me sad.
+credentials = {k: v.encode('utf-8') for k, v in credentials.items()}
+
+HTTP_USERNAME = credentials['http-user']
+HTTP_PASSWORD = credentials['http-pass']
+SLAVE_PASSWORD = credentials['slave-pass']
+CHANGE_PASSWORD = credentials['change-pass']
+GITHUB_DOC_TOKEN = credentials['gh-doc-token']
+HOMU_BUILDBOT_SECRET = credentials['homu-secret']
+S3_UPLOAD_ACCESS_KEY_ID = credentials['s3-upload-access-key-id']
+S3_UPLOAD_SECRET_ACCESS_KEY = credentials['s3-upload-secret-access-key']

buildbot/master/files/config/steps.yml

@@ -22,6 +22,11 @@ mac-rel-css:
   - bash ./etc/ci/lockfile_changed.sh
   - bash ./etc/ci/manifest_changed.sh
 
+mac-nightly:
+  - ./mach build --release
+  - ./mach package --release
+  - ./etc/ci/upload_nightly.sh mac
+
 linux-dev:
   - ./mach test-tidy --no-progress --all
   - ./mach test-tidy --no-progress --self-test
@@ -45,6 +50,11 @@ linux-rel:
   - bash ./etc/ci/lockfile_changed.sh
   - bash ./etc/ci/manifest_changed.sh
 
+linux-nightly:
+  - ./mach build --release
+  - ./mach package --release
+  - ./etc/ci/upload_nightly.sh linux
+
 android:
   - ./mach build --android --dev
   - ./mach package --android --dev
@@ -55,7 +65,7 @@ android:
 android-nightly:
   - ./mach build --android --release
   - ./mach package --android --release
-  - s3cmd put target/arm-linux-androideabi/release/servo.apk s3://servo-rust/nightly/servo.apk
+  - ./etc/ci/upload_nightly.sh android
 
 arm32:
   - ./mach build --rel --target=arm-unknown-linux-gnueabihf