From d687f599e7214609f51c61983b0fd9dd02983bad Mon Sep 17 00:00:00 2001 From: Bob Clary Date: Thu, 20 Dec 2012 11:48:49 -0800 Subject: [PATCH] Bug 821954 - allow specification of repo, build types, r=mcote. --- autophone.py | 38 ++++++++++++++++++++----- builds.py | 73 ++++++++++++++++++++++++++++++------------------- trigger_runs.py | 38 ++++++++++++++++++++----- 3 files changed, 107 insertions(+), 42 deletions(-) diff --git a/autophone.py b/autophone.py index 925c950..f1edcfc 100644 --- a/autophone.py +++ b/autophone.py @@ -78,7 +78,8 @@ def handle(self): def __init__(self, clear_cache, reboot_phones, test_path, cachefile, ipaddr, port, logfile, loglevel, emailcfg, enable_pulse, - enable_unittests, override_build_dir): + enable_unittests, override_build_dir, + repos, buildtypes): self._test_path = test_path self._cache = cachefile if ipaddr: @@ -92,7 +93,8 @@ def __init__(self, clear_cache, reboot_phones, test_path, cachefile, self.logfile = logfile self.loglevel = loglevel self.mailer = Mailer(emailcfg, '[autophone] ') - self.build_cache = builds.BuildCache(override_build_dir=override_build_dir, + self.build_cache = builds.BuildCache(repos, buildtypes, + override_build_dir=override_build_dir, enable_unittests=enable_unittests) self._stop = False self._next_worker_num = 0 @@ -125,9 +127,9 @@ def __init__(self, clear_cache, reboot_phones, test_path, cachefile, if enable_pulse: self.pulsemonitor = start_pulse_monitor(buildCallback=self.on_build, - trees=['mozilla-central'], + trees=repos, platforms=['android'], - buildtypes=['opt'], + buildtypes=buildtypes, logger=logging.getLogger()) else: self.pulsemonitor = None @@ -487,7 +489,7 @@ def stop(self): def main(clear_cache, reboot_phones, test_path, cachefile, ipaddr, port, logfile, loglevel_name, emailcfg, enable_pulse, enable_unittests, - override_build_dir): + override_build_dir, repos, buildtypes): def sigterm_handler(signum, frame): autophone.stop() @@ -513,7 +515,8 @@ def sigterm_handler(signum, frame): autophone = AutoPhone(clear_cache, reboot_phones, test_path, cachefile, ipaddr, port, logfile, loglevel, emailcfg, enable_pulse, enable_unittests, - override_build_dir) + override_build_dir, + repos, buildtypes) except builds.BuildCacheException, e: print '''%s @@ -585,12 +588,33 @@ def sigterm_handler(signum, frame): help='Use the specified directory as the current build ' 'cache directory without attempting to download a build ' 'or test package.') + parser.add_option('--repo', + dest='repos', + action='append', + help='The repos to test. ' + 'One of mozilla-central, mozilla-inbound, mozilla-aurora, ' + 'mozilla-beta. To specify multiple repos, specify them ' + 'with additional --repo options. Defaults to mozilla-central.') + parser.add_option('--buildtype', + dest='buildtypes', + action='append', + help='The build types to test. ' + 'One of opt or debug. To specify multiple build types, ' + 'specify them with additional --buildtype options. ' + 'Defaults to opt.') (options, args) = parser.parse_args() + if not options.repos: + options.repos = ['mozilla-central'] + + if not options.buildtypes: + options.buildtypes = ['opt'] + exit_code = main(options.clear_cache, options.reboot_phones, options.test_path, options.cachefile, options.ipaddr, options.port, options.logfile, options.loglevel, options.emailcfg, options.enable_pulse, - options.enable_unittests, options.override_build_dir) + options.enable_unittests, options.override_build_dir, + options.repos, options.buildtypes) sys.exit(exit_code) diff --git a/builds.py b/builds.py index fe70fdf..b968e06 100644 --- a/builds.py +++ b/builds.py @@ -17,9 +17,13 @@ import traceback -class NightlyBranch(object): +class Nightly(object): - nightly_dirnames = [re.compile('(.*)-mozilla-central-android$')] + def __init__(self, repos, buildtypes): + self.repos = repos + self.buildtypes = buildtypes + self.nightly_dirnames = [(re.compile('(.*)-%s-android$' + % repo)) for repo in repos] def nightly_ftpdir(self, year, month): return 'ftp://ftp.mozilla.org/pub/mobile/nightly/%d/%02d/' % (year, @@ -57,7 +61,7 @@ def build_info_from_ftp(self, ftpline): def build_date_from_url(self, url): # nightly urls are of the form # ftp://ftp.mozilla.org/pub/mobile/nightly///- - # ------android/ + # ------android(-armv6)?/ # m = re.search('nightly\/[\d]{4}\/[\d]{2}\/([\d]{4}-[\d]{2}-[\d]{2}-[\d]{2}-[\d]{2}-[\d]{2})-', url) if not m: @@ -65,15 +69,20 @@ def build_date_from_url(self, url): return datetime.datetime.strptime(m.group(1), '%Y-%m-%d-%H-%M-%S') -class TinderboxBranch(object): +class Tinderbox(object): main_ftp_url = 'ftp://ftp.mozilla.org/pub/mozilla.org/mobile/tinderbox-builds/' + def __init__(self, repos, buildtypes): + self.repos = repos + self.buildtypes = buildtypes + def ftpdirs(self, start_time, end_time): # FIXME: Can we be certain that there's only one buildID (unique - # timestamp) regardless of branch (at least m-i vs m-c)? - return [self.main_ftp_url + 'mozilla-inbound-android/', - self.main_ftp_url + 'mozilla-central-android/'] + # timestamp) regardless of repo (at least m-i vs m-c)? + dirnames = [('%s%s-android/' % (self.main_ftp_url, repo)) for repo in self.repos] + + return dirnames def build_info_from_ftp(self, ftpline): srcdir = ftpline.split()[8].strip() @@ -83,10 +92,12 @@ def build_info_from_ftp(self, ftpline): def build_date_from_url(self, url): # tinderbox urls are of the form # ftp://ftp.mozilla.org/pub/mozilla.org/mobile/tinderbox-builds/ - # -android// - m = re.search('tinderbox-builds\/.*-android\/[\d]+\/', url) + # -android// + m = re.search('tinderbox-builds\/.*-android\/([\d]+)\/', url) + logging.debug('build_date_from_url: url: %s, match: %s' % (url, m)) if not m: return None + logging.debug('build_date_from_url: match.group(1): %s' % m.group(1)) return datetime.datetime.fromtimestamp(int(m.group(1)), pytz.timezone('US/Pacific')) @@ -94,18 +105,24 @@ def build_date_from_url(self, url): class BuildCacheException(Exception): pass + class BuildCache(object): MAX_NUM_BUILDS = 20 - EXPIRE_AFTER_SECONDS = 60*60*24 + EXPIRE_AFTER_SECONDS = 60 * 60 * 24 class FtpLineCache(object): def __init__(self): self.lines = [] + def __call__(self, line): self.lines.append(line) - def __init__(self, cache_dir='builds', override_build_dir = None, enable_unittests = False): + def __init__(self, repos, buildtypes, + cache_dir='builds', override_build_dir=None, + enable_unittests=False): + self.repos = repos + self.buildtypes = buildtypes self.cache_dir = cache_dir self.enable_unittests = enable_unittests self.override_build_dir = override_build_dir @@ -125,18 +142,17 @@ def __init__(self, cache_dir='builds', override_build_dir = None, enable_unittes if not os.path.exists(self.cache_dir): os.mkdir(self.cache_dir) - @classmethod - def branch(cls, s): + def build_location(self, s): if 'nightly' in s: - return NightlyBranch() + return Nightly(self.repos, self.buildtypes) if 'tinderbox' in s: - return TinderboxBranch() + return Tinderbox(self.repos, self.buildtypes) return None - def find_latest_build(self, branch_name='nightly'): + def find_latest_build(self, build_location_name='nightly'): window = datetime.timedelta(days=3) now = datetime.datetime.now() - builds = self.find_builds(now - window, now, branch_name) + builds = self.find_builds(now - window, now, build_location_name) if not builds: logging.error('Could not find any nightly builds in the last ' '%d days!' % window.days) @@ -144,12 +160,12 @@ def find_latest_build(self, branch_name='nightly'): builds.sort() return builds[-1] - def find_builds(self, start_time, end_time, branch_name='nightly'): + def find_builds(self, start_time, end_time, build_location_name='nightly'): logging.debug('Finding most recent build between %s and %s...' % (start_time, end_time)) - branch = self.branch(branch_name) - if not branch: - logging.error('unsupported branch "%s"' % branch_name) + build_location = self.build_location(build_location_name) + if not build_location: + logging.error('unsupported build_location "%s"' % build_location_name) return [] if not start_time.tzinfo: @@ -160,7 +176,7 @@ def find_builds(self, start_time, end_time, branch_name='nightly'): builds = [] fennecregex = re.compile("fennec.*\.android-arm\.apk") - for d in branch.ftpdirs(start_time, end_time): + for d in build_location.ftpdirs(start_time, end_time): url = urlparse.urlparse(d) logging.debug('Logging into %s...' % url.netloc) f = ftplib.FTP(url.netloc) @@ -170,7 +186,7 @@ def find_builds(self, start_time, end_time, branch_name='nightly'): f.dir(url.path, lines) file('lines.out', 'w').write('\n'.join(lines.lines)) for line in lines.lines: - srcdir, build_time = branch.build_info_from_ftp(line) + srcdir, build_time = build_location.build_info_from_ftp(line) if not build_time: continue @@ -192,10 +208,10 @@ def find_builds(self, start_time, end_time, branch_name='nightly'): return builds def build_date(self, url): - branch = self.branch(url) + build_location = self.build_location(url) builddate = None - if branch: - builddate = branch.build_date_from_url(url) + if build_location: + builddate = build_location.build_date_from_url(url) if not builddate: logging.error('bad URL "%s"' % url) return builddate @@ -299,6 +315,7 @@ def get(self, buildurl, enable_unittests, force=False): def clean_cache(self, preserve=[]): def lastused_path(d): return os.path.join(self.cache_dir, d, 'lastused') + def keep_build(d): if preserve and d in preserve: # specifically keep this build @@ -308,11 +325,11 @@ def keep_build(d): return True if ((datetime.datetime.now() - datetime.datetime.fromtimestamp(os.stat(lastused_path(d)).st_mtime) <= - datetime.timedelta(microseconds=1000*1000*self.EXPIRE_AFTER_SECONDS))): + datetime.timedelta(microseconds=1000 * 1000 * self.EXPIRE_AFTER_SECONDS))): # too new return True return False - + builds = [(x, os.stat(lastused_path(x)).st_mtime) for x in os.listdir(self.cache_dir) if not keep_build(x)] builds.sort(key=lambda x: x[1]) diff --git a/trigger_runs.py b/trigger_runs.py index 65a3338..2e15c8b 100644 --- a/trigger_runs.py +++ b/trigger_runs.py @@ -23,7 +23,9 @@ def from_iso_date_or_datetime(s): def main(args, options): logging.info('Looking for builds...') if args[0] == 'latest': - cache_build_dir = builds.BuildCache().find_latest_build(options.branch) + cache_build_dir = (builds.BuildCache(options.repos, + options.buildtypes). + find_latest_build(options.build_location)) if not cache_build_dir: return 1 commands = ['triggerjobs %s' % cache_build_dir] @@ -43,8 +45,10 @@ def main(args, options): start_time = start_time.replace(tzinfo=pytz.timezone('US/Pacific')) if not end_time.tzinfo: end_time = end_time.replace(tzinfo=pytz.timezone('US/Pacific')) - cache_build_dir_list = builds.BuildCache().find_builds(start_time, end_time, - options.branch) + cache_build_dir_list = (builds.BuildCache(options.repos, + options.buildtypes). + find_builds(start_time, end_time, + options.build_location)) if not cache_build_dir_list: return 1 commands = ['triggerjobs %s' % cache_build_dir for cache_build_dir in @@ -92,17 +96,37 @@ def main(args, options): parser.add_option('-p', '--port', action='store', type='int', dest='port', default=28001, help='port of autophone controller; defaults to 28001') - parser.add_option('-b', '--branch', action='store', type='string', - dest='branch', default='nightly', - help='branch to search for builds, defaults to nightly;' + parser.add_option('-b', '--build-location', action='store', type='string', + dest='build_location', default='nightly', + help='build location to search for builds, defaults to nightly;' ' can be "tinderbox" for both m-c and m-i') parser.add_option('-v', '--verbose', action='store_true', dest='verbose', default=False, help='verbose output') + parser.add_option('--repo', + dest='repos', + action='append', + help='The repos to test. ' + 'One of mozilla-central, mozilla-inbound, mozilla-aurora, ' + 'mozilla-beta. To specify multiple repos, specify them ' + 'with additional --repo options. Defaults to mozilla-central.') + parser.add_option('--buildtype', + dest='buildtypes', + action='append', + help='The build types to test. ' + 'One of opt or debug. To specify multiple build types, ' + 'specify them with additional --buildtype options. ' + 'Defaults to opt.') (options, args) = parser.parse_args() if len(args) > 2: parser.print_help() sys.exit(errno.EINVAL) - + + if not options.repos: + options.repos = ['mozilla-central'] + + if not options.buildtypes: + options.buildtypes = ['opt'] + if options.verbose: log_level = logging.DEBUG else: