Skip to content
Browse files

Bug 785086 - Add ability to download unit test prerequisites and to r…

…un from a specific cached build directory, r=mcote.
  • Loading branch information...
1 parent 7ed2d36 commit d1e02c520b4685c4432f6f660a64a2122a66032c @bclary bclary committed Sep 13, 2012
Showing with 132 additions and 47 deletions.
  1. +55 −21 autophone.py
  2. +61 −15 builds.py
  3. +6 −2 tests/s1s2test.py
  4. +8 −7 trigger_runs.py
  5. +2 −2 worker.py
View
76 autophone.py
@@ -77,7 +77,8 @@ def handle(self):
self.request.send(response + '\n')
def __init__(self, clear_cache, reboot_phones, test_path, cachefile,
- ipaddr, port, logfile, loglevel, emailcfg, enable_pulse):
+ ipaddr, port, logfile, loglevel, emailcfg, enable_pulse,
+ enable_unittests, override_build_dir):
self._test_path = test_path
self._cache = cachefile
if ipaddr:
@@ -91,7 +92,8 @@ def __init__(self, clear_cache, reboot_phones, test_path, cachefile,
self.logfile = logfile
self.loglevel = loglevel
self.mailer = Mailer(emailcfg)
- self.build_cache = builds.BuildCache()
+ self.build_cache = builds.BuildCache(override_build_dir=override_build_dir,
+ enable_unittests=enable_unittests)
self._stop = False
self.phone_workers = {} # indexed by mac address
self.worker_lock = threading.Lock()
@@ -129,6 +131,8 @@ def __init__(self, clear_cache, reboot_phones, test_path, cachefile,
else:
self.pulsemonitor = None
+ self.enable_unittests = enable_unittests
+
def run(self):
self.server = self.CmdTCPServer(('0.0.0.0', self.port),
self.CmdTCPHandler)
@@ -319,12 +323,13 @@ def read_tests(self):
self._tests.extend(tests)
def trigger_jobs(self, data):
+ logging.debug('trigger_jobs: data %s' % data)
job = self.build_job(self.get_build(data))
logging.info('Adding user-specified job: %s' % job)
self.start_tests(job)
def reset_phones(self):
- logging.info('Restting phones...')
+ logging.info('Resetting phones...')
for phoneid, phone in self.phone_workers.iteritems():
phone.reboot()
@@ -340,28 +345,30 @@ def on_build(self, msg):
if 'buildurl' in msg:
self.start_tests(self.build_job(self.get_build(msg['buildurl'])))
- def get_build(self, url_or_path):
- cmps = urlparse.urlparse(url_or_path)
- if not cmps.scheme or cmps.scheme == 'file':
- return cmps.path
- apkpath = self.build_cache.get(url_or_path)
+ def get_build(self, buildurl):
+ cache_build_dir = self.build_cache.get(buildurl,
+ self.enable_unittests)
try:
- z = zipfile.ZipFile(apkpath)
+ build_path = os.path.join(cache_build_dir, 'build.apk')
+ z = zipfile.ZipFile(build_path)
z.testzip()
except zipfile.BadZipfile:
- logging.warn('%s is a bad apk; redownloading...' % apkpath)
- apkpath = self.build_cache.get(url_or_path, force=True)
- return apkpath
+ logging.warn('%s is a bad apk; redownloading...' % build_path)
+ cache_build_dir = self.build_cache.get(buildurl,
+ self.enable_unittests,
+ force=True)
+ return cache_build_dir
- def build_job(self, apkpath):
+ def build_job(self, cache_build_dir):
tmpdir = tempfile.mkdtemp()
try:
- apkfile = zipfile.ZipFile(apkpath)
+ build_path = os.path.join(cache_build_dir, 'build.apk')
+ apkfile = zipfile.ZipFile(build_path)
apkfile.extract('application.ini', tmpdir)
except zipfile.BadZipfile:
# we should have already tried to redownload bad zips, so treat
# this as fatal.
- logging.error('%s is a bad apk; aborting job.' % apkpath)
+ logging.error('%s is a bad apk; aborting job.' % build_path)
shutil.rmtree(tmpdir)
return None
cfg = ConfigParser.RawConfigParser()
@@ -380,7 +387,7 @@ def build_job(self, apkpath):
elif repo == 'http://hg.mozilla.org/releases/mozilla-beta':
procname = 'org.mozilla.firefox'
- job = { 'apkpath': apkpath,
+ job = { 'cache_build_dir': cache_build_dir,
'blddate': math.trunc(time.mktime(blddate.timetuple())),
'revision': rev,
'androidprocname': procname,
@@ -398,7 +405,8 @@ def stop(self):
def main(clear_cache, reboot_phones, test_path, cachefile, ipaddr, port,
- logfile, loglevel_name, emailcfg, enable_pulse):
+ logfile, loglevel_name, emailcfg, enable_pulse, enable_unittests,
+ override_build_dir):
def sigterm_handler(signum, frame):
autophone.stop()
@@ -420,9 +428,25 @@ def sigterm_handler(signum, frame):
print '%s Starting server on port %d.' % \
(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), port)
- autophone = AutoPhone(clear_cache, reboot_phones, test_path, cachefile,
- ipaddr, port, logfile, loglevel, emailcfg,
- enable_pulse)
+ try:
+ autophone = AutoPhone(clear_cache, reboot_phones, test_path, cachefile,
+ ipaddr, port, logfile, loglevel, emailcfg,
+ enable_pulse, enable_unittests,
+ override_build_dir)
+ except builds.BuildCacheException, e:
+ print '''%s
+
+When specifying --override-build-dir, the directory must already exist
+and contain a build.apk package file to be tested.
+
+In addition, if you have specified --enable-unittests, the override
+build directory must also contain a tests directory containing the
+unpacked tests package for the build.
+
+ ''' % e
+ parser.print_help()
+ return 1
+
signal.signal(signal.SIGTERM, sigterm_handler)
autophone.run()
print '%s AutoPhone terminated.' % datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
@@ -471,11 +495,21 @@ def sigterm_handler(signum, frame):
parser.add_option('--disable-pulse', action='store_false',
dest="enable_pulse", default=True,
help="Disable connecting to pulse to look for new builds")
+ parser.add_option('--enable-unittests', action='store_true',
+ dest='enable_unittests', default=False,
+ help='Enable running unittests by downloading and installing '
+ 'the unittests package for each build')
+ parser.add_option('--override-build-dir', type='string',
+ dest='override_build_dir', default=None,
+ help='Use the specified directory as the current build '
+ 'cache directory without attempting to download a build '
+ 'or test package.')
(options, args) = parser.parse_args()
exit_code = main(options.clear_cache, options.reboot_phones,
options.test_path, options.cachefile, options.ipaddr,
options.port, options.logfile, options.loglevel,
- options.emailcfg, options.enable_pulse)
+ options.emailcfg, options.enable_pulse,
+ options.enable_unittests, options.override_build_dir)
sys.exit(exit_code)
View
76 builds.py
@@ -13,7 +13,7 @@
import tempfile
import urllib
import urlparse
-
+import zipfile
class NightlyBranch(object):
@@ -91,6 +91,9 @@ def build_date_from_url(self, url):
pytz.timezone('US/Pacific'))
+class BuildCacheException(Exception):
+ pass
+
class BuildCache(object):
MAX_NUM_BUILDS = 20
@@ -102,8 +105,23 @@ def __init__(self):
def __call__(self, line):
self.lines.append(line)
- def __init__(self, cache_dir='builds'):
+ def __init__(self, cache_dir='builds', override_build_dir = None, enable_unittests = False):
self.cache_dir = cache_dir
+ self.enable_unittests = enable_unittests
+ self.override_build_dir = override_build_dir
+ if override_build_dir:
+ if not os.path.exists(override_build_dir):
+ raise BuildCacheException('Override Build Directory does not exist')
+
+ build_path = os.path.join(override_build_dir, 'build.apk')
+ if not os.path.exists(build_path):
+ raise BuildCacheException('Override Build Directory %s does not contain a build.apk.' %
+ override_build_dir)
+
+ tests_path = os.path.join(override_build_dir, 'tests')
+ if self.enable_unittests and not os.path.exists(tests_path):
+ raise BuildCacheException('Override Build Directory %s does not contain a tests directory.' %
+ override_build_dir)
if not os.path.exists(self.cache_dir):
os.mkdir(self.cache_dir)
@@ -165,8 +183,8 @@ def find_builds(self, start_time, end_time, branch_name='nightly'):
for l2 in lines2.lines:
filename = l2.split(' ')[-1].strip()
if fennecregex.match(filename):
- fileurl = url.scheme + '://' + url.netloc + newpath + "/" + filename
- builds.append(fileurl)
+ buildurl = url.scheme + '://' + url.netloc + newpath + "/" + filename
+ builds.append(buildurl)
if not builds:
logging.error('No builds found.')
return builds
@@ -180,22 +198,50 @@ def build_date(self, url):
logging.error('bad URL "%s"' % url)
return builddate
- def get(self, url, force=False):
- build_dir = base64.b64encode(url)
+ def get(self, buildurl, enable_unittests, force=False):
+ if self.override_build_dir:
+ return self.override_build_dir
+ build_dir = base64.b64encode(buildurl)
self.clean_cache([build_dir])
- dir = os.path.join(self.cache_dir, build_dir)
- f = os.path.join(dir, 'build.apk')
- if not os.path.exists(dir):
- os.makedirs(dir)
- if force or not os.path.exists(f):
+ cache_build_dir = os.path.join(self.cache_dir, build_dir)
+ build_path = os.path.join(cache_build_dir, 'build.apk')
+ if not os.path.exists(cache_build_dir):
+ os.makedirs(cache_build_dir)
+ if force or not os.path.exists(build_path):
# retrieve to temporary file then move over, so we don't end
# up with half a file if it aborts
tmpf = tempfile.NamedTemporaryFile(delete=False)
tmpf.close()
- urllib.urlretrieve(url, tmpf.name)
- os.rename(tmpf.name, f)
- file(os.path.join(dir, 'lastused'), 'w')
- return f
+ urllib.urlretrieve(buildurl, tmpf.name)
+ os.rename(tmpf.name, build_path)
+ file(os.path.join(cache_build_dir, 'lastused'), 'w')
+ if enable_unittests:
+ tests_path = os.path.join(cache_build_dir, 'tests')
+ if (force or not os.path.exists(tests_path)) and enable_unittests:
+ tmpf = tempfile.NamedTemporaryFile(delete=False)
+ tmpf.close()
+ # XXX: assumes fixed buildurl-> tests_url mapping
+ tests_url = re.sub('.apk$', '.tests.zip', buildurl)
+ urllib.urlretrieve(tests_url, tmpf.name)
+ tests_zipfile = zipfile.ZipFile(tmpf.name)
+ tests_zipfile.extractall(tests_path)
+ tests_zipfile.close()
+ os.unlink(tmpf.name)
+ # XXX: assumes fixed buildurl-> robocop mapping
+ robocop_url = urlparse.urljoin(buildurl, 'robocop.apk')
+ robocop_path = os.path.join(cache_build_dir, 'robocop.apk')
+ tmpf = tempfile.NamedTemporaryFile(delete=False)
+ tmpf.close()
+ urllib.urlretrieve(robocop_url, tmpf.name)
+ os.rename(tmpf.name, robocop_path)
+ # XXX: assumes fixed buildurl-> fennec_ids.txt mapping
+ fennec_ids_url = urlparse.urljoin(buildurl, 'fennec_ids.txt')
+ fennec_ids_path = os.path.join(cache_build_dir, 'fennec_ids.txt')
+ tmpf = tempfile.NamedTemporaryFile(delete=False)
+ tmpf.close()
+ urllib.urlretrieve(fennec_ids_url, tmpf.name)
+ os.rename(tmpf.name, fennec_ids_path)
+ return cache_build_dir
def clean_cache(self, preserve=[]):
def lastused_path(d):
View
8 tests/s1s2test.py
@@ -187,8 +187,12 @@ def publish_results(self, starttime=0, tstrt=0, tstop=0, drawing=0, job=None, te
try:
f = urllib2.urlopen(req)
except urllib2.URLError, e:
- self.logger.error('Could not send results to server: %s' %
- e.reason.strerror)
+ try:
+ self.logger.error('Could not send results to server: %s' %
+ e.reason.strerror)
+ except:
+ self.logger.error('Could not send results to server: %s' %
+ e.reason)
else:
f.read()
f.close()
View
15 trigger_runs.py
@@ -23,10 +23,10 @@ def from_iso_date_or_datetime(s):
def main(args, options):
logging.info('Looking for builds...')
if args[0] == 'latest':
- build = builds.BuildCache().find_latest_build(options.branch)
- if not build:
+ cache_build_dir = builds.BuildCache().find_latest_build(options.branch)
+ if not cache_build_dir:
return 1
- commands = ['triggerjobs %s' % build]
+ commands = ['triggerjobs %s' % cache_build_dir]
else:
if re.match('\d{14}', args[0]):
# build id
@@ -43,11 +43,12 @@ def main(args, options):
start_time = start_time.replace(tzinfo=pytz.timezone('US/Pacific'))
if not end_time.tzinfo:
end_time = end_time.replace(tzinfo=pytz.timezone('US/Pacific'))
- build_list = builds.BuildCache().find_builds(start_time, end_time,
- options.branch)
- if not build_list:
+ cache_build_dir_list = builds.BuildCache().find_builds(start_time, end_time,
+ options.branch)
+ if not cache_build_dir_list:
return 1
- commands = ['triggerjobs %s' % url for url in build_list]
+ commands = ['triggerjobs %s' % cache_build_dir for cache_build_dir in
+ cache_build_dir_list]
logging.info('Connecting to autophone server...')
commands.append('exit')
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
View
4 worker.py
@@ -311,8 +311,8 @@ def loop(self):
logging.info('Installing build %s.' % datetime.datetime.fromtimestamp(float(job['blddate'])))
pathOnDevice = posixpath.join(self.dm.getDeviceRoot(),
- os.path.basename(job['apkpath']))
- self.dm.pushFile(job['apkpath'], pathOnDevice)
+ 'build.apk')
+ self.dm.pushFile(os.path.join(job['cache_build_dir'], 'build.apk'), pathOnDevice)
self.dm.installApp(pathOnDevice)
self.dm.removeFile(pathOnDevice)

0 comments on commit d1e02c5

Please sign in to comment.
Something went wrong with that request. Please try again.