Skip to content
This repository has been archived by the owner on Aug 14, 2020. It is now read-only.

Commit

Permalink
Bug 821954 - allow specification of repo, build types, r=mcote.
Browse files Browse the repository at this point in the history
  • Loading branch information
@bclary
bclary committed Dec 20, 2012
1 parent 5d632fe commit d687f59
Show file tree
Hide file tree
Showing 3 changed files with 107 additions and 42 deletions.
38 changes: 31 additions & 7 deletions autophone.py
View file
Expand Up @@ -78,7 +78,8 @@ def handle(self):

def __init__(self, clear_cache, reboot_phones, test_path, cachefile,
ipaddr, port, logfile, loglevel, emailcfg, enable_pulse,
enable_unittests, override_build_dir):
enable_unittests, override_build_dir,
repos, buildtypes):
self._test_path = test_path
self._cache = cachefile
if ipaddr:
Expand All @@ -92,7 +93,8 @@ def __init__(self, clear_cache, reboot_phones, test_path, cachefile,
self.logfile = logfile
self.loglevel = loglevel
self.mailer = Mailer(emailcfg, '[autophone] ')
self.build_cache = builds.BuildCache(override_build_dir=override_build_dir,
self.build_cache = builds.BuildCache(repos, buildtypes,
override_build_dir=override_build_dir,
enable_unittests=enable_unittests)
self._stop = False
self._next_worker_num = 0
Expand Down Expand Up @@ -125,9 +127,9 @@ def __init__(self, clear_cache, reboot_phones, test_path, cachefile,

if enable_pulse:
self.pulsemonitor = start_pulse_monitor(buildCallback=self.on_build,
trees=['mozilla-central'],
trees=repos,
platforms=['android'],
buildtypes=['opt'],
buildtypes=buildtypes,
logger=logging.getLogger())
else:
self.pulsemonitor = None
Expand Down Expand Up @@ -487,7 +489,7 @@ def stop(self):

def main(clear_cache, reboot_phones, test_path, cachefile, ipaddr, port,
logfile, loglevel_name, emailcfg, enable_pulse, enable_unittests,
override_build_dir):
override_build_dir, repos, buildtypes):

def sigterm_handler(signum, frame):
autophone.stop()
Expand All @@ -513,7 +515,8 @@ def sigterm_handler(signum, frame):
autophone = AutoPhone(clear_cache, reboot_phones, test_path, cachefile,
ipaddr, port, logfile, loglevel, emailcfg,
enable_pulse, enable_unittests,
override_build_dir)
override_build_dir,
repos, buildtypes)
except builds.BuildCacheException, e:
print '''%s
Expand Down Expand Up @@ -585,12 +588,33 @@ def sigterm_handler(signum, frame):
help='Use the specified directory as the current build '
'cache directory without attempting to download a build '
'or test package.')
parser.add_option('--repo',
dest='repos',
action='append',
help='The repos to test. '
'One of mozilla-central, mozilla-inbound, mozilla-aurora, '
'mozilla-beta. To specify multiple repos, specify them '
'with additional --repo options. Defaults to mozilla-central.')
parser.add_option('--buildtype',
dest='buildtypes',
action='append',
help='The build types to test. '
'One of opt or debug. To specify multiple build types, '
'specify them with additional --buildtype options. '
'Defaults to opt.')

(options, args) = parser.parse_args()
if not options.repos:
options.repos = ['mozilla-central']

if not options.buildtypes:
options.buildtypes = ['opt']

exit_code = main(options.clear_cache, options.reboot_phones,
options.test_path, options.cachefile, options.ipaddr,
options.port, options.logfile, options.loglevel,
options.emailcfg, options.enable_pulse,
options.enable_unittests, options.override_build_dir)
options.enable_unittests, options.override_build_dir,
options.repos, options.buildtypes)

sys.exit(exit_code)
73 changes: 45 additions & 28 deletions builds.py
View file
Expand Up @@ -17,9 +17,13 @@
import traceback


class NightlyBranch(object):
class Nightly(object):

nightly_dirnames = [re.compile('(.*)-mozilla-central-android$')]
def __init__(self, repos, buildtypes):
self.repos = repos
self.buildtypes = buildtypes
self.nightly_dirnames = [(re.compile('(.*)-%s-android$'
% repo)) for repo in repos]

def nightly_ftpdir(self, year, month):
return 'ftp://ftp.mozilla.org/pub/mobile/nightly/%d/%02d/' % (year,
Expand Down Expand Up @@ -57,23 +61,28 @@ def build_info_from_ftp(self, ftpline):
def build_date_from_url(self, url):
# nightly urls are of the form
# ftp://ftp.mozilla.org/pub/mobile/nightly/<year>/<month>/<year>-
# <month>-<day>-<hour>-<minute>-<second>-<branch>-android/
# <month>-<day>-<hour>-<minute>-<second>-<repo>-android(-armv6)?/
# <buildfile>
m = re.search('nightly\/[\d]{4}\/[\d]{2}\/([\d]{4}-[\d]{2}-[\d]{2}-[\d]{2}-[\d]{2}-[\d]{2})-', url)
if not m:
return None
return datetime.datetime.strptime(m.group(1), '%Y-%m-%d-%H-%M-%S')


class TinderboxBranch(object):
class Tinderbox(object):

main_ftp_url = 'ftp://ftp.mozilla.org/pub/mozilla.org/mobile/tinderbox-builds/'

def __init__(self, repos, buildtypes):
self.repos = repos
self.buildtypes = buildtypes

def ftpdirs(self, start_time, end_time):
# FIXME: Can we be certain that there's only one buildID (unique
# timestamp) regardless of branch (at least m-i vs m-c)?
return [self.main_ftp_url + 'mozilla-inbound-android/',
self.main_ftp_url + 'mozilla-central-android/']
# timestamp) regardless of repo (at least m-i vs m-c)?
dirnames = [('%s%s-android/' % (self.main_ftp_url, repo)) for repo in self.repos]

return dirnames

def build_info_from_ftp(self, ftpline):
srcdir = ftpline.split()[8].strip()
Expand All @@ -83,29 +92,37 @@ def build_info_from_ftp(self, ftpline):
def build_date_from_url(self, url):
# tinderbox urls are of the form
# ftp://ftp.mozilla.org/pub/mozilla.org/mobile/tinderbox-builds/
# <branch>-android/<build timestamp>/<buildfile>
m = re.search('tinderbox-builds\/.*-android\/[\d]+\/', url)
# <repo>-android/<build timestamp>/<buildfile>
m = re.search('tinderbox-builds\/.*-android\/([\d]+)\/', url)
logging.debug('build_date_from_url: url: %s, match: %s' % (url, m))
if not m:
return None
logging.debug('build_date_from_url: match.group(1): %s' % m.group(1))
return datetime.datetime.fromtimestamp(int(m.group(1)),
pytz.timezone('US/Pacific'))


class BuildCacheException(Exception):
pass


class BuildCache(object):

MAX_NUM_BUILDS = 20
EXPIRE_AFTER_SECONDS = 60*60*24
EXPIRE_AFTER_SECONDS = 60 * 60 * 24

class FtpLineCache(object):
def __init__(self):
self.lines = []

def __call__(self, line):
self.lines.append(line)

def __init__(self, cache_dir='builds', override_build_dir = None, enable_unittests = False):
def __init__(self, repos, buildtypes,
cache_dir='builds', override_build_dir=None,
enable_unittests=False):
self.repos = repos
self.buildtypes = buildtypes
self.cache_dir = cache_dir
self.enable_unittests = enable_unittests
self.override_build_dir = override_build_dir
Expand All @@ -125,31 +142,30 @@ def __init__(self, cache_dir='builds', override_build_dir = None, enable_unittes
if not os.path.exists(self.cache_dir):
os.mkdir(self.cache_dir)

@classmethod
def branch(cls, s):
def build_location(self, s):
if 'nightly' in s:
return NightlyBranch()
return Nightly(self.repos, self.buildtypes)
if 'tinderbox' in s:
return TinderboxBranch()
return Tinderbox(self.repos, self.buildtypes)
return None

def find_latest_build(self, branch_name='nightly'):
def find_latest_build(self, build_location_name='nightly'):
window = datetime.timedelta(days=3)
now = datetime.datetime.now()
builds = self.find_builds(now - window, now, branch_name)
builds = self.find_builds(now - window, now, build_location_name)
if not builds:
logging.error('Could not find any nightly builds in the last '
'%d days!' % window.days)
return None
builds.sort()
return builds[-1]

def find_builds(self, start_time, end_time, branch_name='nightly'):
def find_builds(self, start_time, end_time, build_location_name='nightly'):
logging.debug('Finding most recent build between %s and %s...' %
(start_time, end_time))
branch = self.branch(branch_name)
if not branch:
logging.error('unsupported branch "%s"' % branch_name)
build_location = self.build_location(build_location_name)
if not build_location:
logging.error('unsupported build_location "%s"' % build_location_name)
return []

if not start_time.tzinfo:
Expand All @@ -160,7 +176,7 @@ def find_builds(self, start_time, end_time, branch_name='nightly'):
builds = []
fennecregex = re.compile("fennec.*\.android-arm\.apk")

for d in branch.ftpdirs(start_time, end_time):
for d in build_location.ftpdirs(start_time, end_time):
url = urlparse.urlparse(d)
logging.debug('Logging into %s...' % url.netloc)
f = ftplib.FTP(url.netloc)
Expand All @@ -170,7 +186,7 @@ def find_builds(self, start_time, end_time, branch_name='nightly'):
f.dir(url.path, lines)
file('lines.out', 'w').write('\n'.join(lines.lines))
for line in lines.lines:
srcdir, build_time = branch.build_info_from_ftp(line)
srcdir, build_time = build_location.build_info_from_ftp(line)

if not build_time:
continue
Expand All @@ -192,10 +208,10 @@ def find_builds(self, start_time, end_time, branch_name='nightly'):
return builds

def build_date(self, url):
branch = self.branch(url)
build_location = self.build_location(url)
builddate = None
if branch:
builddate = branch.build_date_from_url(url)
if build_location:
builddate = build_location.build_date_from_url(url)
if not builddate:
logging.error('bad URL "%s"' % url)
return builddate
Expand Down Expand Up @@ -299,6 +315,7 @@ def get(self, buildurl, enable_unittests, force=False):
def clean_cache(self, preserve=[]):
def lastused_path(d):
return os.path.join(self.cache_dir, d, 'lastused')

def keep_build(d):
if preserve and d in preserve:
# specifically keep this build
Expand All @@ -308,11 +325,11 @@ def keep_build(d):
return True
if ((datetime.datetime.now() -
datetime.datetime.fromtimestamp(os.stat(lastused_path(d)).st_mtime) <=
datetime.timedelta(microseconds=1000*1000*self.EXPIRE_AFTER_SECONDS))):
datetime.timedelta(microseconds=1000 * 1000 * self.EXPIRE_AFTER_SECONDS))):
# too new
return True
return False

builds = [(x, os.stat(lastused_path(x)).st_mtime) for x in
os.listdir(self.cache_dir) if not keep_build(x)]
builds.sort(key=lambda x: x[1])
Expand Down
38 changes: 31 additions & 7 deletions trigger_runs.py
View file
Expand Up @@ -23,7 +23,9 @@ def from_iso_date_or_datetime(s):
def main(args, options):
logging.info('Looking for builds...')
if args[0] == 'latest':
cache_build_dir = builds.BuildCache().find_latest_build(options.branch)
cache_build_dir = (builds.BuildCache(options.repos,
options.buildtypes).
find_latest_build(options.build_location))
if not cache_build_dir:
return 1
commands = ['triggerjobs %s' % cache_build_dir]
Expand All @@ -43,8 +45,10 @@ def main(args, options):
start_time = start_time.replace(tzinfo=pytz.timezone('US/Pacific'))
if not end_time.tzinfo:
end_time = end_time.replace(tzinfo=pytz.timezone('US/Pacific'))
cache_build_dir_list = builds.BuildCache().find_builds(start_time, end_time,
options.branch)
cache_build_dir_list = (builds.BuildCache(options.repos,
options.buildtypes).
find_builds(start_time, end_time,
options.build_location))
if not cache_build_dir_list:
return 1
commands = ['triggerjobs %s' % cache_build_dir for cache_build_dir in
Expand Down Expand Up @@ -92,17 +96,37 @@ def main(args, options):
parser.add_option('-p', '--port', action='store', type='int', dest='port',
default=28001,
help='port of autophone controller; defaults to 28001')
parser.add_option('-b', '--branch', action='store', type='string',
dest='branch', default='nightly',
help='branch to search for builds, defaults to nightly;'
parser.add_option('-b', '--build-location', action='store', type='string',
dest='build_location', default='nightly',
help='build location to search for builds, defaults to nightly;'
' can be "tinderbox" for both m-c and m-i')
parser.add_option('-v', '--verbose', action='store_true', dest='verbose',
default=False, help='verbose output')
parser.add_option('--repo',
dest='repos',
action='append',
help='The repos to test. '
'One of mozilla-central, mozilla-inbound, mozilla-aurora, '
'mozilla-beta. To specify multiple repos, specify them '
'with additional --repo options. Defaults to mozilla-central.')
parser.add_option('--buildtype',
dest='buildtypes',
action='append',
help='The build types to test. '
'One of opt or debug. To specify multiple build types, '
'specify them with additional --buildtype options. '
'Defaults to opt.')
(options, args) = parser.parse_args()
if len(args) > 2:
parser.print_help()
sys.exit(errno.EINVAL)


if not options.repos:
options.repos = ['mozilla-central']

if not options.buildtypes:
options.buildtypes = ['opt']

if options.verbose:
log_level = logging.DEBUG
else:
Expand Down

0 comments on commit d687f59

Please sign in to comment.