Permalink
Browse files

Bug 821954 - allow specification of repo, build types, r=mcote.

  • Loading branch information...
1 parent 5d632fe commit d687f599e7214609f51c61983b0fd9dd02983bad @bclary bclary committed Dec 20, 2012
Showing with 107 additions and 42 deletions.
  1. +31 −7 autophone.py
  2. +45 −28 builds.py
  3. +31 −7 trigger_runs.py
View
@@ -78,7 +78,8 @@ def handle(self):
def __init__(self, clear_cache, reboot_phones, test_path, cachefile,
ipaddr, port, logfile, loglevel, emailcfg, enable_pulse,
- enable_unittests, override_build_dir):
+ enable_unittests, override_build_dir,
+ repos, buildtypes):
self._test_path = test_path
self._cache = cachefile
if ipaddr:
@@ -92,7 +93,8 @@ def __init__(self, clear_cache, reboot_phones, test_path, cachefile,
self.logfile = logfile
self.loglevel = loglevel
self.mailer = Mailer(emailcfg, '[autophone] ')
- self.build_cache = builds.BuildCache(override_build_dir=override_build_dir,
+ self.build_cache = builds.BuildCache(repos, buildtypes,
+ override_build_dir=override_build_dir,
enable_unittests=enable_unittests)
self._stop = False
self._next_worker_num = 0
@@ -125,9 +127,9 @@ def __init__(self, clear_cache, reboot_phones, test_path, cachefile,
if enable_pulse:
self.pulsemonitor = start_pulse_monitor(buildCallback=self.on_build,
- trees=['mozilla-central'],
+ trees=repos,
platforms=['android'],
- buildtypes=['opt'],
+ buildtypes=buildtypes,
logger=logging.getLogger())
else:
self.pulsemonitor = None
@@ -487,7 +489,7 @@ def stop(self):
def main(clear_cache, reboot_phones, test_path, cachefile, ipaddr, port,
logfile, loglevel_name, emailcfg, enable_pulse, enable_unittests,
- override_build_dir):
+ override_build_dir, repos, buildtypes):
def sigterm_handler(signum, frame):
autophone.stop()
@@ -513,7 +515,8 @@ def sigterm_handler(signum, frame):
autophone = AutoPhone(clear_cache, reboot_phones, test_path, cachefile,
ipaddr, port, logfile, loglevel, emailcfg,
enable_pulse, enable_unittests,
- override_build_dir)
+ override_build_dir,
+ repos, buildtypes)
except builds.BuildCacheException, e:
print '''%s
@@ -585,12 +588,33 @@ def sigterm_handler(signum, frame):
help='Use the specified directory as the current build '
'cache directory without attempting to download a build '
'or test package.')
+ parser.add_option('--repo',
+ dest='repos',
+ action='append',
+ help='The repos to test. '
+ 'One of mozilla-central, mozilla-inbound, mozilla-aurora, '
+ 'mozilla-beta. To specify multiple repos, specify them '
+ 'with additional --repo options. Defaults to mozilla-central.')
+ parser.add_option('--buildtype',
+ dest='buildtypes',
+ action='append',
+ help='The build types to test. '
+ 'One of opt or debug. To specify multiple build types, '
+ 'specify them with additional --buildtype options. '
+ 'Defaults to opt.')
(options, args) = parser.parse_args()
+ if not options.repos:
+ options.repos = ['mozilla-central']
+
+ if not options.buildtypes:
+ options.buildtypes = ['opt']
+
exit_code = main(options.clear_cache, options.reboot_phones,
options.test_path, options.cachefile, options.ipaddr,
options.port, options.logfile, options.loglevel,
options.emailcfg, options.enable_pulse,
- options.enable_unittests, options.override_build_dir)
+ options.enable_unittests, options.override_build_dir,
+ options.repos, options.buildtypes)
sys.exit(exit_code)
View
@@ -17,9 +17,13 @@
import traceback
-class NightlyBranch(object):
+class Nightly(object):
- nightly_dirnames = [re.compile('(.*)-mozilla-central-android$')]
+ def __init__(self, repos, buildtypes):
+ self.repos = repos
+ self.buildtypes = buildtypes
+ self.nightly_dirnames = [(re.compile('(.*)-%s-android$'
+ % repo)) for repo in repos]
def nightly_ftpdir(self, year, month):
return 'ftp://ftp.mozilla.org/pub/mobile/nightly/%d/%02d/' % (year,
@@ -57,23 +61,28 @@ def build_info_from_ftp(self, ftpline):
def build_date_from_url(self, url):
# nightly urls are of the form
# ftp://ftp.mozilla.org/pub/mobile/nightly/<year>/<month>/<year>-
- # <month>-<day>-<hour>-<minute>-<second>-<branch>-android/
+ # <month>-<day>-<hour>-<minute>-<second>-<repo>-android(-armv6)?/
# <buildfile>
m = re.search('nightly\/[\d]{4}\/[\d]{2}\/([\d]{4}-[\d]{2}-[\d]{2}-[\d]{2}-[\d]{2}-[\d]{2})-', url)
if not m:
return None
return datetime.datetime.strptime(m.group(1), '%Y-%m-%d-%H-%M-%S')
-class TinderboxBranch(object):
+class Tinderbox(object):
main_ftp_url = 'ftp://ftp.mozilla.org/pub/mozilla.org/mobile/tinderbox-builds/'
+ def __init__(self, repos, buildtypes):
+ self.repos = repos
+ self.buildtypes = buildtypes
+
def ftpdirs(self, start_time, end_time):
# FIXME: Can we be certain that there's only one buildID (unique
- # timestamp) regardless of branch (at least m-i vs m-c)?
- return [self.main_ftp_url + 'mozilla-inbound-android/',
- self.main_ftp_url + 'mozilla-central-android/']
+ # timestamp) regardless of repo (at least m-i vs m-c)?
+ dirnames = [('%s%s-android/' % (self.main_ftp_url, repo)) for repo in self.repos]
+
+ return dirnames
def build_info_from_ftp(self, ftpline):
srcdir = ftpline.split()[8].strip()
@@ -83,29 +92,37 @@ def build_info_from_ftp(self, ftpline):
def build_date_from_url(self, url):
# tinderbox urls are of the form
# ftp://ftp.mozilla.org/pub/mozilla.org/mobile/tinderbox-builds/
- # <branch>-android/<build timestamp>/<buildfile>
- m = re.search('tinderbox-builds\/.*-android\/[\d]+\/', url)
+ # <repo>-android/<build timestamp>/<buildfile>
+ m = re.search('tinderbox-builds\/.*-android\/([\d]+)\/', url)
+ logging.debug('build_date_from_url: url: %s, match: %s' % (url, m))
if not m:
return None
+ logging.debug('build_date_from_url: match.group(1): %s' % m.group(1))
return datetime.datetime.fromtimestamp(int(m.group(1)),
pytz.timezone('US/Pacific'))
class BuildCacheException(Exception):
pass
+
class BuildCache(object):
MAX_NUM_BUILDS = 20
- EXPIRE_AFTER_SECONDS = 60*60*24
+ EXPIRE_AFTER_SECONDS = 60 * 60 * 24
class FtpLineCache(object):
def __init__(self):
self.lines = []
+
def __call__(self, line):
self.lines.append(line)
- def __init__(self, cache_dir='builds', override_build_dir = None, enable_unittests = False):
+ def __init__(self, repos, buildtypes,
+ cache_dir='builds', override_build_dir=None,
+ enable_unittests=False):
+ self.repos = repos
+ self.buildtypes = buildtypes
self.cache_dir = cache_dir
self.enable_unittests = enable_unittests
self.override_build_dir = override_build_dir
@@ -125,31 +142,30 @@ def __init__(self, cache_dir='builds', override_build_dir = None, enable_unittes
if not os.path.exists(self.cache_dir):
os.mkdir(self.cache_dir)
- @classmethod
- def branch(cls, s):
+ def build_location(self, s):
if 'nightly' in s:
- return NightlyBranch()
+ return Nightly(self.repos, self.buildtypes)
if 'tinderbox' in s:
- return TinderboxBranch()
+ return Tinderbox(self.repos, self.buildtypes)
return None
- def find_latest_build(self, branch_name='nightly'):
+ def find_latest_build(self, build_location_name='nightly'):
window = datetime.timedelta(days=3)
now = datetime.datetime.now()
- builds = self.find_builds(now - window, now, branch_name)
+ builds = self.find_builds(now - window, now, build_location_name)
if not builds:
logging.error('Could not find any nightly builds in the last '
'%d days!' % window.days)
return None
builds.sort()
return builds[-1]
- def find_builds(self, start_time, end_time, branch_name='nightly'):
+ def find_builds(self, start_time, end_time, build_location_name='nightly'):
logging.debug('Finding most recent build between %s and %s...' %
(start_time, end_time))
- branch = self.branch(branch_name)
- if not branch:
- logging.error('unsupported branch "%s"' % branch_name)
+ build_location = self.build_location(build_location_name)
+ if not build_location:
+ logging.error('unsupported build_location "%s"' % build_location_name)
return []
if not start_time.tzinfo:
@@ -160,7 +176,7 @@ def find_builds(self, start_time, end_time, branch_name='nightly'):
builds = []
fennecregex = re.compile("fennec.*\.android-arm\.apk")
- for d in branch.ftpdirs(start_time, end_time):
+ for d in build_location.ftpdirs(start_time, end_time):
url = urlparse.urlparse(d)
logging.debug('Logging into %s...' % url.netloc)
f = ftplib.FTP(url.netloc)
@@ -170,7 +186,7 @@ def find_builds(self, start_time, end_time, branch_name='nightly'):
f.dir(url.path, lines)
file('lines.out', 'w').write('\n'.join(lines.lines))
for line in lines.lines:
- srcdir, build_time = branch.build_info_from_ftp(line)
+ srcdir, build_time = build_location.build_info_from_ftp(line)
if not build_time:
continue
@@ -192,10 +208,10 @@ def find_builds(self, start_time, end_time, branch_name='nightly'):
return builds
def build_date(self, url):
- branch = self.branch(url)
+ build_location = self.build_location(url)
builddate = None
- if branch:
- builddate = branch.build_date_from_url(url)
+ if build_location:
+ builddate = build_location.build_date_from_url(url)
if not builddate:
logging.error('bad URL "%s"' % url)
return builddate
@@ -299,6 +315,7 @@ def get(self, buildurl, enable_unittests, force=False):
def clean_cache(self, preserve=[]):
def lastused_path(d):
return os.path.join(self.cache_dir, d, 'lastused')
+
def keep_build(d):
if preserve and d in preserve:
# specifically keep this build
@@ -308,11 +325,11 @@ def keep_build(d):
return True
if ((datetime.datetime.now() -
datetime.datetime.fromtimestamp(os.stat(lastused_path(d)).st_mtime) <=
- datetime.timedelta(microseconds=1000*1000*self.EXPIRE_AFTER_SECONDS))):
+ datetime.timedelta(microseconds=1000 * 1000 * self.EXPIRE_AFTER_SECONDS))):
# too new
return True
return False
-
+
builds = [(x, os.stat(lastused_path(x)).st_mtime) for x in
os.listdir(self.cache_dir) if not keep_build(x)]
builds.sort(key=lambda x: x[1])
View
@@ -23,7 +23,9 @@ def from_iso_date_or_datetime(s):
def main(args, options):
logging.info('Looking for builds...')
if args[0] == 'latest':
- cache_build_dir = builds.BuildCache().find_latest_build(options.branch)
+ cache_build_dir = (builds.BuildCache(options.repos,
+ options.buildtypes).
+ find_latest_build(options.build_location))
if not cache_build_dir:
return 1
commands = ['triggerjobs %s' % cache_build_dir]
@@ -43,8 +45,10 @@ def main(args, options):
start_time = start_time.replace(tzinfo=pytz.timezone('US/Pacific'))
if not end_time.tzinfo:
end_time = end_time.replace(tzinfo=pytz.timezone('US/Pacific'))
- cache_build_dir_list = builds.BuildCache().find_builds(start_time, end_time,
- options.branch)
+ cache_build_dir_list = (builds.BuildCache(options.repos,
+ options.buildtypes).
+ find_builds(start_time, end_time,
+ options.build_location))
if not cache_build_dir_list:
return 1
commands = ['triggerjobs %s' % cache_build_dir for cache_build_dir in
@@ -92,17 +96,37 @@ def main(args, options):
parser.add_option('-p', '--port', action='store', type='int', dest='port',
default=28001,
help='port of autophone controller; defaults to 28001')
- parser.add_option('-b', '--branch', action='store', type='string',
- dest='branch', default='nightly',
- help='branch to search for builds, defaults to nightly;'
+ parser.add_option('-b', '--build-location', action='store', type='string',
+ dest='build_location', default='nightly',
+ help='build location to search for builds, defaults to nightly;'
' can be "tinderbox" for both m-c and m-i')
parser.add_option('-v', '--verbose', action='store_true', dest='verbose',
default=False, help='verbose output')
+ parser.add_option('--repo',
+ dest='repos',
+ action='append',
+ help='The repos to test. '
+ 'One of mozilla-central, mozilla-inbound, mozilla-aurora, '
+ 'mozilla-beta. To specify multiple repos, specify them '
+ 'with additional --repo options. Defaults to mozilla-central.')
+ parser.add_option('--buildtype',
+ dest='buildtypes',
+ action='append',
+ help='The build types to test. '
+ 'One of opt or debug. To specify multiple build types, '
+ 'specify them with additional --buildtype options. '
+ 'Defaults to opt.')
(options, args) = parser.parse_args()
if len(args) > 2:
parser.print_help()
sys.exit(errno.EINVAL)
-
+
+ if not options.repos:
+ options.repos = ['mozilla-central']
+
+ if not options.buildtypes:
+ options.buildtypes = ['opt']
+
if options.verbose:
log_level = logging.DEBUG
else:

0 comments on commit d687f59

Please sign in to comment.