Skip to content
Permalink
Browse files

Move nightly upload script into Mach

Now that MinGW and MSYS have been removed from the Windows builders,
bash is not available to run the previous upload_nightlies.sh script.
Convert the script to Python 2 for cross-platform support.
Additionally, switch to the `boto3` library for easy uploading
without needing to install `s3cmd`,
and move the code into mach for easy `boto3` installation
as the new `./mach upload-nightly` command.

Also, hard-code the paths to the packages instead of using
globs to look for them, as the paths are static.
(The paths used to contain timestamps,
but we now insert timestamps when uploading to S3
to improve reproducibility.)
  • Loading branch information
aneeshusa committed May 20, 2017
1 parent 97c14f0 commit 043f78db53818431ef20d24107e445128e856822
Showing with 129 additions and 119 deletions.
  1. +5 −5 etc/ci/buildbot_steps.yml
  2. +0 −113 etc/ci/upload_nightly.sh
  3. +3 −0 python/requirements.txt
  4. +121 −1 python/servo/package_commands.py
@@ -38,8 +38,8 @@ mac-nightly:
- ./mach clean-nightlies --keep 3 --force
- ./mach build --release
- ./mach package --release
- ./etc/ci/upload_nightly.sh mac
- ./etc/ci/upload_nightly.sh macbrew
- ./mach upload-nightly mac
- ./mach upload-nightly macbrew

linux-rel-intermittent:
- ./mach clean-nightlies --keep 3 --force
@@ -89,7 +89,7 @@ linux-nightly:
- ./mach clean-nightlies --keep 3 --force
- ./mach build --release
- ./mach package --release
- ./etc/ci/upload_nightly.sh linux
- ./mach upload-nightly linux

android:
- ./mach clean-nightlies --keep 3 --force
@@ -103,7 +103,7 @@ android-nightly:
- ./mach clean-nightlies --keep 3 --force
- env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach build --android --release
- env SERVO_RUSTC_LLVM_ASSERTIONS=1 ANDROID_SDK=/home/servo/android/sdk/r25.2.3 ./mach package --android --release
- ./etc/ci/upload_nightly.sh android
- ./mach upload-nightly android

arm32:
- ./mach clean-nightlies --keep 3 --force
@@ -128,4 +128,4 @@ windows-msvc-nightly:
- mach.bat clean-nightlies --keep 3 --force
- mach.bat build --release
- mach.bat package --release
- bash -l ./etc/ci/upload_nightly.sh windows-msvc
- mach.bat upload-nightly windows-msvc

This file was deleted.

@@ -24,4 +24,7 @@ ply == 3.8
# For Cross-platform colored terminal text
colorama == 0.3.7

# For package uploading
boto3 == 1.4.4

-e python/tidy
@@ -7,13 +7,17 @@
# option. This file may not be copied, modified, or distributed
# except according to those terms.

from __future__ import print_function, unicode_literals
from __future__ import absolute_import, print_function, unicode_literals

from datetime import datetime
import hashlib
import json
import os
import os.path as path
import shutil
import subprocess
import sys
import tempfile

from mach.decorators import (
CommandArgument,
@@ -35,6 +39,26 @@
from servo.util import delete


PACKAGES = {
'android': [
'target/arm-linux-androideabi/release/servo.apk',
],
'linux': [
'target/release/servo-tech-demo.tar.gz',
],
'mac': [
'target/release/servo-tech-demo.dmg',
],
'macbrew': [
'target/release/brew/servo.tar.gz',
],
'windows-msvc': [
r'target\release\msi\Servo.msi',
r'target\release\msi\Servo.zip',
],
}


def otool(s):
o = subprocess.Popen(['/usr/bin/otool', '-L', s], stdout=subprocess.PIPE)
for l in o.stdout:
@@ -364,3 +388,99 @@ def install(self, release=False, dev=False, android=False):

print(" ".join(exec_command))
return subprocess.call(exec_command, env=self.build_env())

@Command('upload-nightly',
description='Upload Servo nightly to S3',
category='package')
@CommandArgument('platform',
choices=PACKAGES.keys(),
help='Package platform type to upload')
def upload_nightly(self, platform):
import boto3

def nightly_filename(package, timestamp):
return '{}-{}'.format(
timestamp.isoformat() + 'Z', # The `Z` denotes UTC
path.basename(package)
)

def upload_to_s3(platform, package, timestamp):
s3 = boto3.client('s3')
BUCKET = 'servo-builds'

nightly_dir = 'nightly/{}'.format(platform)
filename = nightly_filename(package, timestamp)
package_upload_key = '{}/{}'.format(nightly_dir, filename)
extension = path.basename(package).partition('.')[2]
latest_upload_key = '{}/servo-latest.{}'.format(nightly_dir, extension)

s3.upload_file(package, BUCKET, package_upload_key)
copy_source = {
'Bucket': BUCKET,
'Key': package_upload_key,
}
s3.copy(copy_source, BUCKET, latest_upload_key)

def update_brew(package, timestamp):
print("Updating brew formula")

package_url = 'https://download.servo.org/nightly/macbrew/{}'.format(
nightly_filename(package, timestamp)
)
with open(package) as p:
digest = hashlib.sha256(p.read()).hexdigest()

brew_version = timestamp.strftime('%Y.%m.%d')

with tempfile.TemporaryDirectory(prefix='homebrew-servo') as tmp_dir:
def call_git(cmd, **kwargs):
subprocess.check_call(
['git', '-C', tmp_dir] + cmd,
**kwargs
)

call_git([
'clone',
'https://github.com/servo/homebrew-servo.git',
'.',
])

script_dir = path.dirname(path.realpath(__file__))
with open(path.join(script_dir, 'servo-binary-formula.rb.in')) as f:
formula = f.read()
formula = formula.replace('PACKAGEURL', package_url)
formula = formula.replace('SHA', digest)
formula = formula.replace('VERSION', brew_version)
with open(path.join(tmp_dir, 'Formula', 'servo-bin.rb')) as f:
f.write(formula)

call_git(['add', path.join('.', 'Formula', 'servo-bin.rb')])
call_git([
'-c', 'user.name=Tom Servo',
'-c', 'user.email=servo@servo.org',
'commit',
'--message=Version Bump: {}'.format(brew_version),
])

token = os.environ['GITHUB_HOMEBREW_TOKEN']
call_git([
'push',
'-qf',
'https://{}@github.com/servo/homebrew-servo.git'.format(token),
'master',
], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)

timestamp = datetime.utcnow().replace(microsecond=0)
for package in PACKAGES[platform]:
if not path.isfile(package):
print("Could not find package for {} at {}".format(
platform,
package
), file=sys.stderr)
return 1
upload_to_s3(platform, package, timestamp)

if platform == 'macbrew':
update_brew(package, timestamp)

return 0

0 comments on commit 043f78d

Please sign in to comment.
You can’t perform that action at this time.