Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Bug 777530: improvements to build scripts for b2g panda builds. r=aki

--HG--
extra : rebase_source : 61dfc92bb9d975ca61df73e242d03bf9de55f2f9
  • Loading branch information...
commit d5863cad69e14e8139f8b8a8e2e564fcdd7020a7 1 parent c59e886
@catlee catlee authored
View
31 configs/b2g/releng-try.py
@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+import os
+config = {
+ "default_actions": [
+ 'checkout-gecko',
+ 'download-gonk',
+ 'unpack-gonk',
+ 'build',
+ 'make-updates',
+ 'prep-upload',
+ 'upload',
+ ],
+ "ssh_key": os.path.expanduser("~/.ssh/b2gtry_dsa"),
+ "ssh_user": "b2gtry",
+ "upload_remote_host": "pvtbuilds2.dmz.scl3.mozilla.com",
+ #"upload_remote_host": "dev-stage01.srv.releng.scl3.mozilla.com",
+ "upload_remote_basepath": "/pub/mozilla.org/b2g/try-builds",
+ "enable_try_uploads": True,
+ "tooltool_servers": ["http://runtime-binaries.pvt.build.mozilla.org/tooltool/"],
+ "vcs_share_base": "/builds/hg-shared",
+ "vcs_base_mirror_urls": ["http://hg-internal.dmz.scl3.mozilla.com"],
+ "vcs_base_bundle_urls": ["http://ftp.mozilla.org/pub/mozilla.org/firefox/bundles"],
+ "exes": {
+ "tooltool.py": "/tools/tooltool.py",
+ },
+ "env": {
+ "CCACHE_DIR": "/builds/ccache",
+ "CCACHE_COMPRESS": "1",
+ "CCACHE_UMASK": "002",
+ },
+}
View
14 configs/b2g/releng.py
@@ -1,12 +1,24 @@
#!/usr/bin/env python
import os
config = {
+ "default_actions": [
+ 'checkout-gecko',
+ 'download-gonk',
+ 'unpack-gonk',
+ 'build',
+ 'make-updates',
+ 'prep-upload',
+ 'upload',
+ ],
"ssh_key": os.path.expanduser("~/.ssh/b2gbld_dsa"),
"ssh_user": "b2gbld",
- "upload_remote_host": "stage.mozilla.org",
+ "upload_remote_host": "pvtbuilds2.dmz.scl3.mozilla.com",
+ #"upload_remote_host": "dev-stage01.srv.releng.scl3.mozilla.com",
"upload_remote_basepath": "/pub/mozilla.org/b2g/tinderbox-builds",
"tooltool_servers": ["http://runtime-binaries.pvt.build.mozilla.org/tooltool/"],
"vcs_share_base": "/builds/hg-shared",
+ "vcs_base_mirror_urls": ["http://hg-internal.dmz.scl3.mozilla.com"],
+ "vcs_base_bundle_urls": ["http://ftp.mozilla.org/pub/mozilla.org/firefox/bundles"],
"exes": {
"tooltool.py": "/tools/tooltool.py",
},
View
5 mozharness/base/script.py
@@ -472,7 +472,10 @@ def run_command(self, command, cwd=None, error_list=None, parse_at_end=False,
if p.poll() is not None:
"""Avoid losing the final lines of the log?"""
loop = False
- for line in p.stdout:
+ while True:
+ line = p.stdout.readline()
+ if not line:
+ break
parser.add_lines(line)
return_level = INFO
if p.returncode not in success_codes:
View
88 mozharness/base/vcs/hgtool.py
@@ -0,0 +1,88 @@
+import os
+import re
+import urlparse
+
+from mozharness.base.script import ShellMixin, OSMixin
+from mozharness.base.log import LogMixin, OutputParser
+from mozharness.base.errors import HgErrorList, VCSException
+
+
+class HgtoolParser(OutputParser):
+ """
+ A class that extends OutputParser such that it can find the "Got revision"
+ string from hgtool.py output
+ """
+
+ got_revision_exp = re.compile(r'Got revision (\w+)')
+ got_revision = None
+
+ def parse_single_line(self, line):
+ m = self.got_revision_exp.match(line)
+ if m:
+ self.got_revision = m.group(1)
+ super(HgtoolParser, self).parse_single_line(line)
+
+
+class HgtoolVCS(ShellMixin, OSMixin, LogMixin):
+ def __init__(self, log_obj=None, config=None, vcs_config=None):
+ super(HgtoolVCS, self).__init__()
+
+ self.log_obj = log_obj
+ if config:
+ self.config = config
+ else:
+ self.config = {}
+ # vcs_config = {
+ # hg_host: hg_host,
+ # repo: repository,
+ # branch: branch,
+ # revision: revision,
+ # ssh_username: ssh_username,
+ # ssh_key: ssh_key,
+ # }
+ self.vcs_config = vcs_config
+ self.hgtool = self.query_exe('hgtool.py', return_type='list')
+
+ def ensure_repo_and_revision(self):
+ """Makes sure that `dest` is has `revision` or `branch` checked out
+ from `repo`.
+
+ Do what it takes to make that happen, including possibly clobbering
+ dest.
+ """
+ c = self.vcs_config
+ for conf_item in ('dest', 'repo'):
+ assert self.vcs_config[conf_item]
+ dest = os.path.abspath(c['dest'])
+ repo = c['repo']
+ revision = c.get('revision')
+ branch = c.get('branch')
+ share_base = c.get('vcs_share_base', os.environ.get("HG_SHARE_BASE_DIR", None))
+ env = {'PATH': os.environ.get('PATH')}
+ if share_base is not None:
+ env['HG_SHARE_BASE_DIR'] = share_base
+
+ cmd = self.hgtool[:]
+ if branch:
+ cmd.extend(['-b', branch])
+ if revision:
+ cmd.extend(['-r', revision])
+
+ for base_mirror_url in self.config.get('vcs_base_mirror_urls', []):
+ bits = urlparse.urlparse(repo)
+ mirror_url = urlparse.urljoin(base_mirror_url, bits.path)
+ cmd.extend(['--mirror', mirror_url])
+
+ for base_bundle_url in self.config.get('vcs_base_bundle_urls', []):
+ bundle_url = "%s/%s.hg" % (base_bundle_url, os.path.basename(repo))
+ cmd.extend(['--bundle', bundle_url])
+
+ cmd.extend([repo, dest])
+ parser = HgtoolParser(config=self.config, log_obj=self.log_obj,
+ error_list=HgErrorList)
+ retval = self.run_command(cmd, error_list=HgErrorList, env=env, output_parser=parser)
+
+ if retval != 0:
+ raise VCSException("Unable to checkout")
+
+ return parser.got_revision
View
2  mozharness/base/vcs/vcsbase.py
@@ -16,10 +16,12 @@
from mozharness.base.errors import VCSException
from mozharness.base.script import BaseScript
from mozharness.base.vcs.mercurial import MercurialVCS
+from mozharness.base.vcs.hgtool import HgtoolVCS
# Update this with supported VCS name : VCS object
VCS_DICT = {
'hg': MercurialVCS,
+ 'hgtool': HgtoolVCS,
}
# VCSMixin {{{1
View
86 scripts/b2g_build.py
@@ -60,6 +60,7 @@ def __init__(self, require_config_file=False):
'download-gonk',
'unpack-gonk',
'build',
+ 'make-updates',
'prep-upload',
'upload',
],
@@ -75,7 +76,7 @@ def __init__(self, require_config_file=False):
config={
'repo': 'http://hg.mozilla.org/mozilla-central', # from buildprops
'branch': 'mozilla-central', # from buildprops
- 'default_vcs': 'hg',
+ 'default_vcs': 'hgtool',
'vcs_share_base': os.environ.get('HG_SHARE_BASE_DIR'),
'ccache': True,
'buildbot_json_path': os.environ.get('PROPERTIES_FILE'),
@@ -84,6 +85,7 @@ def __init__(self, require_config_file=False):
'ssh_user': None,
'upload_remote_host': None,
'upload_remote_basepath': None,
+ 'enable_try_uploads': False,
},
)
@@ -147,16 +149,21 @@ def query_buildid(self):
if buildid:
return buildid.group(1)
+ def query_revision(self):
+ if 'revision' in self.buildbot_properties:
+ return self.buildbot_properties['revision']
+
+ if self.buildbot_config and 'sourcestamp' in self.buildbot_config:
+ return self.buildbot_config['sourcestamp']['revision']
+
+ return None
+
# Actions {{{2
def checkout_gecko(self):
dirs = self.query_abs_dirs()
repo = self.query_repo()
- if self.buildbot_config and 'sourcestamp' in self.buildbot_config:
- rev = self.vcs_checkout(repo=repo, dest=dirs['src'], revision=self.buildbot_config['sourcestamp']['revision'])
- else:
- rev = self.vcs_checkout(repo=repo, dest=dirs['src'])
-
+ rev = self.vcs_checkout(repo=repo, dest=dirs['src'], revision=self.query_revision())
self.set_buildbot_property('revision', rev, write_to_file=True)
def download_gonk(self):
@@ -189,7 +196,7 @@ def unpack_gonk(self):
def build(self):
dirs = self.query_abs_dirs()
gecko_config = self.load_gecko_config()
- cmd = ['./build.sh']
+ cmd = ['./build.sh', 'boottarball', 'systemtarball', 'userdatatarball']
env = self.query_env()
env.update(gecko_config.get('env', {}))
if self.config['ccache']:
@@ -222,6 +229,30 @@ def build(self):
if retval != 0:
self.fatal("failed to build", exit_code=2)
+ def make_updates(self):
+ dirs = self.query_abs_dirs()
+ gecko_config = self.load_gecko_config()
+ cmd = ['./build.sh', 'gecko-update-full']
+ env = self.query_env()
+ env.update(gecko_config.get('env', {}))
+
+ # Write .userconfig to point to the correct object directory for gecko
+ # Normally this is embedded inside the .config file included with the snapshot
+ # TODO: factor this out so it doesn't get run twice
+ user_config = open(os.path.join(dirs['work_dir'], '.userconfig'), 'w')
+ user_config.write("GECKO_OBJDIR=%s/objdir-gecko\n" % dirs['work_dir'])
+ user_config.close()
+
+ if 'mock_target' in gecko_config:
+ # initialize mock
+ self.setup_mock(gecko_config['mock_target'], gecko_config['mock_packages'])
+ retval = self.run_mock_command(gecko_config['mock_target'], cmd, cwd=dirs['work_dir'], env=env, error_list=MakefileErrorList)
+ else:
+ retval = self.run_command(cmd, cwd=dirs['work_dir'], env=env, error_list=MakefileErrorList)
+
+ if retval != 0:
+ self.fatal("failed to create complete update", exit_code=2)
+
def prep_upload(self):
# Copy stuff into build/upload directory
dirs = self.query_abs_dirs()
@@ -235,23 +266,46 @@ def prep_upload(self):
self.info("copying %s to upload directory" % f)
self.copy_to_upload_dir(f)
- for f in glob.glob("%s/*.img" % output_dir):
+ for f in glob.glob("%s/*.tar.bz2" % output_dir):
+ self.info("copying %s to upload directory" % f)
+ self.copy_to_upload_dir(f)
+ f = os.path.join(dirs['abs_upload_dir'], os.path.basename(f))
+
+ objdir = os.path.join(dirs['work_dir'], 'objdir-gecko')
+ for f in glob.glob("%s/dist/b2g-update/*.mar" % objdir):
+ self.info("copying %s to upload directory" % f)
+ self.copy_to_upload_dir(f)
+ f = os.path.join(dirs['abs_upload_dir'], os.path.basename(f))
+
+ for f in glob.glob("%s/dist/b2g-*.tar.gz" % objdir):
self.info("copying %s to upload directory" % f)
self.copy_to_upload_dir(f)
f = os.path.join(dirs['abs_upload_dir'], os.path.basename(f))
- self.info("compressing %s" % f)
- self.run_command(['gzip', '-f', f])
self.copy_logs_to_upload_dir()
def upload(self):
dirs = self.query_abs_dirs()
- upload_path = "%(basepath)s/%(branch)s-%(target)s/%(buildid)s" % dict(
- basepath=self.config['upload_remote_basepath'],
- branch=self.query_branch(),
- buildid=self.query_buildid(),
- target=self.config['target'],
- )
+ c = self.config
+ if c['enable_try_uploads']:
+ try:
+ user = self.buildbot_config['sourcestamp']['changes'][0]['who']
+ except KeyError:
+ user = "unknown"
+ upload_path = "%(basepath)s/%(user)s-%(rev)s/%(branch)s-%(target)s" % dict(
+ basepath=self.config['upload_remote_basepath'],
+ branch=self.query_branch(),
+ target=self.config['target'],
+ user=user,
+ rev=self.query_revision(),
+ )
+ else:
+ upload_path = "%(basepath)s/%(branch)s-%(target)s/%(buildid)s" % dict(
+ basepath=self.config['upload_remote_basepath'],
+ branch=self.query_branch(),
+ buildid=self.query_buildid(),
+ target=self.config['target'],
+ )
retval = self.rsync_upload_directory(
dirs['abs_upload_dir'],
View
1  scripts/hgtool.py
Please sign in to comment.
Something went wrong with that request. Please try again.