Permalink
Browse files

trigger_runs.py can trigger jobs for tinderbox builds and from local …

…files.

Also log all stdout and sterr from adb commands, cached builds expire after
one day, phone worker does heartbeat every 15 minutes instead of every minute,
verify cached builds and redownload if bad zip, removed some unused imports,
and a couple tiny fixes.
  • Loading branch information...
1 parent 5a0dfa8 commit 6251b23d0d22b75fb840475cc12cdfe9351e7e5e @markrcote markrcote committed May 14, 2012
Showing with 136 additions and 74 deletions.
  1. +6 −1 androidutils.py
  2. +57 −27 autophone.py
  3. +51 −21 builds.py
  4. +2 −1 tests/s1s2test.py
  5. +20 −24 trigger_runs.py
View
@@ -95,7 +95,7 @@ def install_build_adb(phoneid=None, apkpath=None, blddate=None,
phoneid)
ret = False
- o = run_adb('install', [apkpath], serial, timeout=120)
+ o = run_adb('install', ['-r', apkpath], serial, timeout=120)
print o
if o.lower().find('success') == -1:
logging.error('Unable to install application on phoneID: %s' % phoneid)
@@ -179,6 +179,11 @@ def run_adb(adbcmd, cmd, serial=None, check_for_error=False, timeout=0):
logging.error('adb error: %s' % stderr)
raise AndroidError(stderr)
logging.debug('adb command finished')
+ logging.debug('stdout:')
+ logging.debug(stdout)
+ if stderr:
+ logging.debug('stderr:')
+ logging.debug(stderr)
return stdout
View
@@ -15,10 +15,10 @@
import shutil
import socket
import sys
+import tempfile
import threading
import time
import traceback
-import urllib
import urlparse
import zipfile
@@ -104,8 +104,9 @@ class PhoneWorker(object):
this back to the main AutoPhone process.
"""
- max_reboot_wait_seconds = 300
- max_reboot_attempts = 3
+ MAX_REBOOT_WAIT_SECONDS = 300
+ MAX_REBOOT_ATTEMPTS = 3
+ PING_SECONDS = 60*15
def __init__(self, worker_num, tests, phone_cfg, callback_ipaddr,
autophone_queue, main_logfile, loglevel, mailer):
@@ -211,14 +212,14 @@ def clear_test_base_paths(self):
def recover_phone(self):
reboots = 0
while not self.disabled:
- if reboots < self.max_reboot_attempts:
+ if reboots < self.MAX_REBOOT_ATTEMPTS:
logging.info('Rebooting phone...')
phone_is_up = False
reboots += 1
androidutils.reboot_adb(self.phone_cfg['serial'])
time.sleep(10)
max_time = datetime.datetime.now() + \
- datetime.timedelta(seconds=self.max_reboot_wait_seconds)
+ datetime.timedelta(seconds=self.MAX_REBOOT_WAIT_SECONDS)
while datetime.datetime.now() <= max_time:
dm = DeviceManagerSUT(self.phone_cfg['ip'],
self.phone_cfg['sutcmdport'])
@@ -232,7 +233,7 @@ def recover_phone(self):
return
else:
logging.info('Phone did not come back up within %d seconds.' %
- self.max_reboot_wait_seconds)
+ self.MAX_REBOOT_WAIT_SECONDS)
else:
logging.info('Phone has been rebooted %d times; giving up.' %
reboots)
@@ -297,22 +298,33 @@ def loop(self):
self.phone_cfg['phoneid'],
phonetest.PhoneTestMessage.IDLE))
+ last_ping = None
+
while True:
request = None
try:
request = self.job_queue.get(timeout=60)
except Queue.Empty:
- if not self.disabled:
- # verify that the phone is still responding
- response = androidutils.run_adb('shell', ['ps'],
- self.phone_cfg['serial'])
- if response:
- status = phonetest.PhoneTestMessage.IDLE
- else:
- # FIXME: recover phone here?
- status = phonetest.PhoneTestMessage.DISCONNECTED
- self.status_update(phonetest.PhoneTestMessage(
- self.phone_cfg['phoneid'], status, self.current_build))
+ if (not last_ping or
+ ((datetime.datetime.now() - last_ping) >
+ datetime.timedelta(
+ microseconds=1000*1000*self.PING_SECONDS))):
+ last_ping = datetime.datetime.now()
+ if not self.disabled:
+ logging.info('Pinging phone')
+ # verify that the phone is still responding
+ response = androidutils.run_adb('shell', ['ps'],
+ self.phone_cfg['serial'])
+ if response:
+ status = phonetest.PhoneTestMessage.IDLE
+ logging.info('Pong!')
+ else:
+ logging.info('No response!')
+ # FIXME: recover phone here?
+ status = phonetest.PhoneTestMessage.DISCONNECTED
+ self.status_update(phonetest.PhoneTestMessage(
+ self.phone_cfg['phoneid'], status,
+ self.current_build))
except KeyboardInterrupt:
return
if self.should_stop():
@@ -662,7 +674,7 @@ def read_tests(self):
self._tests.extend(tests)
def trigger_jobs(self, data):
- job = self.build_job(self.get_remote_build(data))
+ job = self.build_job(self.get_build(data))
logging.info('Adding user-specified job: %s' % job)
self.start_tests(job)
@@ -681,23 +693,42 @@ def on_build(self, msg):
# those, and only run the ones with real URLs
# We create jobs for all the phones and push them into the queue
if 'buildurl' in msg:
- self.start_tests(self.build_job(self.get_remote_build(msg['buildurl'])))
+ self.start_tests(self.build_job(self.get_build(msg['buildurl'])))
- def get_remote_build(self, buildurl):
- return self.build_cache.get(buildurl)
+ def get_build(self, url_or_path):
+ cmps = urlparse.urlparse(url_or_path)
+ if not cmps.scheme or cmps.scheme == 'file':
+ return cmps.path
+ apkpath = self.build_cache.get(url_or_path)
+ try:
+ z = zipfile.ZipFile(apkpath)
+ z.testzip()
+ except zipfile.BadZipfile:
+ logging.warn('%s is a bad apk; redownloading...' % apkpath)
+ apkpath = self.build_cache.get(url_or_path, force=True)
+ return apkpath
def build_job(self, apkpath):
- apkfile = zipfile.ZipFile(apkpath)
- apkfile.extract('application.ini', 'extdir')
+ tmpdir = tempfile.mkdtemp()
+ try:
+ apkfile = zipfile.ZipFile(apkpath)
+ apkfile.extract('application.ini', tmpdir)
+ except zipfile.BadZipfile:
+ # we should have already tried to redownload bad zips, so treat
+ # this as fatal.
+ logging.error('%s is a bad apk; aborting job.' % apkpath)
+ shutil.rmtree(tmpdir)
+ return None
cfg = ConfigParser.RawConfigParser()
- cfg.read('extdir/application.ini')
+ cfg.read(os.path.join(tmpdir, 'application.ini'))
rev = cfg.get('App', 'SourceStamp')
ver = cfg.get('App', 'Version')
repo = cfg.get('App', 'SourceRepository')
blddate = datetime.datetime.strptime(cfg.get('App', 'BuildID'),
'%Y%m%d%H%M%S')
procname = ''
- if repo == 'http://hg.mozilla.org/mozilla-central':
+ if (repo == 'http://hg.mozilla.org/mozilla-central' or
+ repo == 'http://hg.mozilla.org/integration/mozilla-inbound'):
procname = 'org.mozilla.fennec'
elif repo == 'http://hg.mozilla.org/releases/mozilla-aurora':
procname = 'org.mozilla.fennec_aurora'
@@ -710,8 +741,7 @@ def build_job(self, apkpath):
'androidprocname': procname,
'version': ver,
'bldtype': 'opt' }
- if os.path.exists('extdir'):
- shutil.rmtree('extdir')
+ shutil.rmtree(tmpdir)
return job
def stop(self):
View
@@ -9,15 +9,17 @@
import pytz
import re
import shutil
+import tempfile
import urllib
import urllib2
-dirnames = [re.compile('(.*)-mozilla-central-android$')]
+nightly_dirnames = [re.compile('(.*)-mozilla-central-android$')]
class BuildCache(object):
- MAX_NUM_BUILDS = 10
+ MAX_NUM_BUILDS = 20
+ EXPIRE_AFTER_SECONDS = 60*60*24
def __init__(self, cache_dir='builds'):
self.cache_dir = cache_dir
@@ -27,21 +29,26 @@ def __init__(self, cache_dir='builds'):
def nightly_ftpdir(self, year, month):
return 'ftp://ftp.mozilla.org/pub/mobile/nightly/%d/%02d/' % (year, month)
- def find_builds(self, time_range, type='nightly'):
+ def find_builds(self, start_time, end_time, branch='nightly'):
+ if not start_time.tzinfo:
+ start_time = start_time.replace(tzinfo=pytz.timezone('US/Pacific'))
+ if not end_time.tzinfo:
+ end_time = end_time.replace(tzinfo=pytz.timezone('US/Pacific'))
builds = []
fennecregex = re.compile("fennec.*\.android-arm\.apk")
ftpdirs = []
- if type == 'nightly':
- y = time_range[0].year
- m = time_range[0].month
- while y < time_range[1].year or m <= time_range[1].month:
+ # FIXME: refactor branch into objects
+ if branch == 'nightly':
+ y = start_time.year
+ m = start_time.month
+ while y < end_time.year or m <= end_time.month:
ftpdirs.append(self.nightly_ftpdir(y, m))
if m == 12:
y += 1
m = 1
else:
m += 1
- elif type == 'tinderbox':
+ elif branch == 'tinderbox':
# FIXME: Can we be certain that there's only one buildID (unique
# timestamp) regardless of branch (at least m-i vs m-c)?
ftpdirs.append('ftp://ftp.mozilla.org/pub/mozilla.org/mobile/tinderbox-builds/mozilla-inbound-android/')
@@ -51,10 +58,10 @@ def find_builds(self, time_range, type='nightly'):
f = urllib2.urlopen(d)
for line in f:
build_time = None
- if type == 'nightly':
+ if branch == 'nightly':
srcdir = line.split(' ')[-1].strip()
dirnamematch = None
- for r in dirnames:
+ for r in nightly_dirnames:
dirnamematch = r.match(srcdir)
if dirnamematch:
break
@@ -63,12 +70,12 @@ def find_builds(self, time_range, type='nightly'):
'%Y-%m-%d-%H-%M-%S')
else:
continue
- elif type == 'tinderbox':
+ elif branch == 'tinderbox':
srcdir = line.split()[8].strip()
build_time = datetime.datetime.fromtimestamp(int(srcdir), pytz.timezone('US/Pacific'))
- if build_time and (build_time < time_range[0] or
- build_time > time_range[1]):
+ if build_time and (build_time < start_time or
+ build_time > end_time):
continue
newurl = d + srcdir
@@ -104,21 +111,44 @@ def build_date(self, url):
pytz.timezone('US/Pacific'))
return builddate
- def get(self, url):
+ def get(self, url, force=False):
build_dir = base64.b64encode(url)
- self.clean_cache(build_dir)
+ self.clean_cache([build_dir])
dir = os.path.join(self.cache_dir, build_dir)
f = os.path.join(dir, 'build.apk')
if not os.path.exists(dir):
os.makedirs(dir)
- if not os.path.exists(f):
- urllib.urlretrieve(url, f)
+ if force or not os.path.exists(f):
+ # retrieve to temporary file then move over, so we don't end
+ # up with half a file if it aborts
+ tmpf = tempfile.NamedTemporaryFile(delete=False)
+ tmpf.close()
+ urllib.urlretrieve(url, tmpf.name)
+ os.rename(tmpf.name, f)
file(os.path.join(dir, 'lastused'), 'w')
return f
- def clean_cache(self, preserve=None):
- builds = [(x, os.stat(os.path.join(self.cache_dir, x, 'lastused')).st_mtime) for x in
- os.listdir(self.cache_dir) if preserve and x != preserve and os.path.exists(os.path.join(self.cache_dir, x, 'lastused'))]
+ def clean_cache(self, preserve=[]):
+ def lastused_path(d):
+ return os.path.join(self.cache_dir, d, 'lastused')
+ def keep_build(d):
+ if preserve and d in preserve:
+ # specifically keep this build
+ return True
+ if not os.path.exists(lastused_path(d)):
+ # probably not a build dir
+ return True
+ if ((datetime.datetime.now() -
+ datetime.datetime.fromtimestamp(os.stat(lastused_path(d)).st_mtime) <=
+ datetime.timedelta(microseconds=1000*1000*self.EXPIRE_AFTER_SECONDS))):
+ # too new
+ return True
+ return False
+
+ builds = [(x, os.stat(lastused_path(x)).st_mtime) for x in
+ os.listdir(self.cache_dir) if not keep_build(x)]
builds.sort(key=lambda x: x[1])
while len(builds) > self.MAX_NUM_BUILDS:
- shutil.rmtree(os.path.join(self.cache_dir, builds.pop(0)[0]))
+ b = builds.pop(0)[0]
+ logging.info('Expiring %s' % b)
+ shutil.rmtree(os.path.join(self.cache_dir, b))
View
@@ -139,7 +139,8 @@ def prepare_phone(self, job):
self._resulturl = cfg.get('settings', 'resulturl')
def analyze_logcat(self, job):
- buf = androidutils.run_adb('logcat', ['-d'], self.phone_cfg['serial'])
+ buf = androidutils.run_adb('logcat', ['-d'], self.phone_cfg['serial'],
+ timeout=60)
buf = buf.split('\r\n')
throbberstartRE = re.compile('.*Throbber start$')
throbberstopRE = re.compile('.*Throbber stop$')
Oops, something went wrong.

0 comments on commit 6251b23

Please sign in to comment.