Skip to content
Browse files

Smoke test, refactored builds.py.

Also added signal handler, support for getting latest build, switched builds.py
from using urllib2 to faster ftplib.
  • Loading branch information...
1 parent c66279b commit eee972407faf8cf5d2f5df801f7f93c4c00467e0 @markrcote markrcote committed Jul 19, 2012
Showing with 377 additions and 86 deletions.
  1. +8 −0 autophone.py
  2. +131 −67 builds.py
  3. +34 −0 selftest/buildcache.py
  4. +1 −0 selftest/manifest.ini
  5. +65 −0 smoketest.sh
  6. +6 −0 test.py
  7. +104 −0 tests/smoketest.py
  8. +2 −0 tests/smoketest_manifest.ini
  9. +26 −19 trigger_runs.py
View
8 autophone.py
@@ -13,6 +13,7 @@
import multiprocessing
import os
import shutil
+import signal
import socket
import sys
import tempfile
@@ -149,6 +150,9 @@ def worker_msg_loop(self):
msg = self.worker_msg_queue.get(timeout=5)
except Queue.Empty:
continue
+ except IOError, e:
+ if e.errno == errno.EINTR:
+ continue
self.phone_workers[msg.phoneid].process_msg(msg)
except KeyboardInterrupt:
self.stop()
@@ -399,6 +403,9 @@ def stop(self):
def main(is_restarting, reboot_phones, test_path, cachefile, ipaddr, port,
logfile, loglevel_name, emailcfg):
+ def sigterm_handler(signum, frame):
+ autophone.stop()
+
adb_check = androidutils.check_for_adb()
if adb_check != 0:
print 'Could not execute adb: %s.' % os.strerror(adb_check)
@@ -425,6 +432,7 @@ def main(is_restarting, reboot_phones, test_path, cachefile, ipaddr, port,
(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), port)
autophone = AutoPhone(is_restarting, reboot_phones, test_path, cachefile,
ipaddr, port, logfile, loglevel, emailcfg)
+ signal.signal(signal.SIGTERM, sigterm_handler)
autophone.run()
print '%s AutoPhone terminated.' % datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
return 0
View
198 builds.py
@@ -4,112 +4,176 @@
import base64
import datetime
+import ftplib
import logging
import os
import pytz
import re
import shutil
import tempfile
import urllib
-import urllib2
+import urlparse
-nightly_dirnames = [re.compile('(.*)-mozilla-central-android$')]
+
+class NightlyBranch(object):
+
+ nightly_dirnames = [re.compile('(.*)-mozilla-central-android$')]
+
+ def nightly_ftpdir(self, year, month):
+ return 'ftp://ftp.mozilla.org/pub/mobile/nightly/%d/%02d/' % (year,
+ month)
+
+ def ftpdirs(self, start_time, end_time):
+ dirs = []
+ y = start_time.year
+ m = start_time.month
+ while y < end_time.year or m <= end_time.month:
+ dirs.append(self.nightly_ftpdir(y, m))
+ if m == 12:
+ y += 1
+ m = 1
+ else:
+ m += 1
+ return dirs
+
+ def build_info_from_ftp(self, ftpline):
+ srcdir = ftpline.split(' ')[-1].strip()
+ build_time = None
+ dirnamematch = None
+ logging.debug('matching dir names')
+ for r in self.nightly_dirnames:
+ dirnamematch = r.match(srcdir)
+ if dirnamematch:
+ break
+ if dirnamematch:
+ logging.debug('build time')
+ build_time = datetime.datetime.strptime(dirnamematch.group(1),
+ '%Y-%m-%d-%H-%M-%S')
+ build_time = build_time.replace(tzinfo=pytz.timezone('US/Pacific'))
+ logging.debug('got build time')
+ logging.debug('got info')
+ return (srcdir, build_time)
+
+ def build_date_from_url(self, url):
+ # nightly urls are of the form
+ # ftp://ftp.mozilla.org/pub/mobile/nightly/<year>/<month>/<year>-
+ # <month>-<day>-<hour>-<minute>-<second>-<branch>-android/
+ # <buildfile>
+ m = re.search('nightly\/[\d]{4}\/[\d]{2}\/([\d]{4}-[\d]{2}-[\d]{2}-[\d]{2}-[\d]{2}-[\d]{2})-', url)
+ if not m:
+ return None
+ return datetime.datetime.strptime(m.group(1), '%Y-%m-%d-%H-%M-%S')
+
+
+class TinderboxBranch(object):
+
+ main_ftp_url = 'ftp://ftp.mozilla.org/pub/mozilla.org/mobile/tinderbox-builds/'
+
+ def ftpdirs(self, start_time, end_time):
+ # FIXME: Can we be certain that there's only one buildID (unique
+ # timestamp) regardless of branch (at least m-i vs m-c)?
+ return [self.main_ftp_url + 'mozilla-inbound-android/',
+ self.main_ftp_url + 'mozilla-central-android/']
+
+ def build_info_from_ftp(self, ftpline):
+ srcdir = ftpline.split()[8].strip()
+ build_time = datetime.datetime.fromtimestamp(int(srcdir), pytz.timezone('US/Pacific'))
+ return (srcdir, build_time)
+
+ def build_date_from_url(self, url):
+ # tinderbox urls are of the form
+ # ftp://ftp.mozilla.org/pub/mozilla.org/mobile/tinderbox-builds/
+ # <branch>-android/<build timestamp>/<buildfile>
+ m = re.search('tinderbox-builds\/.*-android\/[\d]+\/', url)
+ if not m:
+ return None
+ return datetime.datetime.fromtimestamp(int(m.group(1)),
+ pytz.timezone('US/Pacific'))
class BuildCache(object):
MAX_NUM_BUILDS = 20
EXPIRE_AFTER_SECONDS = 60*60*24
+ class FtpLineCache(object):
+ def __init__(self):
+ self.lines = []
+ def __call__(self, line):
+ self.lines.append(line)
+
def __init__(self, cache_dir='builds'):
self.cache_dir = cache_dir
if not os.path.exists(self.cache_dir):
os.mkdir(self.cache_dir)
- def nightly_ftpdir(self, year, month):
- return 'ftp://ftp.mozilla.org/pub/mobile/nightly/%d/%02d/' % (year, month)
+ @classmethod
+ def branch(cls, s):
+ if 'nightly' in s:
+ return NightlyBranch()
+ if 'tinderbox' in s:
+ return TinderboxBranch()
+ return None
+
+ def find_latest_build(self, branch_name='nightly'):
+ # This assumes at least one build has been created in the last 24
+ # hours.
+ now = datetime.datetime.now()
+ builds = self.find_builds(now - datetime.timedelta(days=1), now,
+ branch_name)
+ builds.sort()
+ return builds[-1]
+
+ def find_builds(self, start_time, end_time, branch_name='nightly'):
+ branch = self.branch(branch_name)
+ if not branch:
+ logging.error('unsupport branch "%s"' % branch_name)
+ return []
- def find_builds(self, start_time, end_time, branch='nightly'):
if not start_time.tzinfo:
start_time = start_time.replace(tzinfo=pytz.timezone('US/Pacific'))
if not end_time.tzinfo:
end_time = end_time.replace(tzinfo=pytz.timezone('US/Pacific'))
+
builds = []
fennecregex = re.compile("fennec.*\.android-arm\.apk")
- ftpdirs = []
- # FIXME: refactor branch into objects
- if branch == 'nightly':
- y = start_time.year
- m = start_time.month
- while y < end_time.year or m <= end_time.month:
- ftpdirs.append(self.nightly_ftpdir(y, m))
- if m == 12:
- y += 1
- m = 1
- else:
- m += 1
- elif branch == 'tinderbox':
- # FIXME: Can we be certain that there's only one buildID (unique
- # timestamp) regardless of branch (at least m-i vs m-c)?
- ftpdirs.append('ftp://ftp.mozilla.org/pub/mozilla.org/mobile/tinderbox-builds/mozilla-inbound-android/')
- ftpdirs.append('ftp://ftp.mozilla.org/pub/mozilla.org/mobile/tinderbox-builds/mozilla-central-android/')
-
- for d in ftpdirs:
- f = urllib2.urlopen(d)
- for line in f:
- build_time = None
- if branch == 'nightly':
- srcdir = line.split(' ')[-1].strip()
- dirnamematch = None
- for r in nightly_dirnames:
- dirnamematch = r.match(srcdir)
- if dirnamematch:
- break
- if dirnamematch:
- build_time = datetime.datetime.strptime(dirnamematch.group(1),
- '%Y-%m-%d-%H-%M-%S')
- build_time = build_time.replace(tzinfo=pytz.timezone('US/Pacific'))
- else:
- continue
- elif branch == 'tinderbox':
- srcdir = line.split()[8].strip()
- build_time = datetime.datetime.fromtimestamp(int(srcdir), pytz.timezone('US/Pacific'))
+
+ for d in branch.ftpdirs(start_time, end_time):
+ url = urlparse.urlparse(d)
+ logging.debug('logging into %s...' % url.netloc)
+ f = ftplib.FTP(url.netloc)
+ f.login()
+ logging.debug('looking for builds in %s...' % url.path)
+ lines = self.FtpLineCache()
+ f.dir(url.path, lines)
+ file('lines.out', 'w').write('\n'.join(lines.lines))
+ for line in lines.lines:
+ srcdir, build_time = branch.build_info_from_ftp(line)
+
+ if not build_time:
+ continue
if build_time and (build_time < start_time or
build_time > end_time):
continue
- newurl = d + srcdir
- f2 = urllib.urlopen(newurl)
- for l2 in f2:
+ newpath = url.path + srcdir
+ lines2 = self.FtpLineCache()
+ f.dir(newpath, lines2)
+ for l2 in lines2.lines:
filename = l2.split(' ')[-1].strip()
if fennecregex.match(filename):
- fileurl = newurl + "/" + filename
+ fileurl = url.scheme + '://' + url.netloc + newpath + "/" + filename
builds.append(fileurl)
return builds
def build_date(self, url):
- # nightly urls are of the form
- # ftp://ftp.mozilla.org/pub/mobile/nightly/<year>/<month>/<year>-
- # <month>-<day>-<hour>-<minute>-<second>-<branch>-android/
- # <buildfile>
- # tinderbox urls are of the form
- # ftp://ftp.mozilla.org/pub/mozilla.org/mobile/tinderbox-builds/
- # <branch>-android/<build timestamp>/<buildfile>
+ branch = self.branch(url)
builddate = None
- if 'nightly' in url:
- m = re.search('nightly\/[\d]{4}\/[\d]{2}\/([\d]{4}-[\d]{2}-[\d]{2}-[\d]{2}-[\d]{2}-[\d]{2})-', url)
- if not m:
- logging.error('bad URL "%s"' % url)
- return None
- builddate = datetime.datetime.strptime(m.group(1), '%Y-%m-%d-%H-%M-%S')
- elif 'tinderbox' in url:
- m = re.search('tinderbox-builds\/.*-android\/[\d]+\/', url)
- if not m:
- logging.error('bad URL "%s"' % url)
- return None
- builddate = datetime.datetime.fromtimestamp(int(m.group(1)),
- pytz.timezone('US/Pacific'))
+ if branch:
+ builddate = branch.build_date_from_url(url)
+ if not builddate:
+ logging.error('bad URL "%s"' % url)
return builddate
def get(self, url, force=False):
View
34 selftest/buildcache.py
@@ -0,0 +1,34 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import datetime
+import logging
+import shutil
+import tempfile
+import unittest
+
+import builds
+
+class BuildsTest(unittest.TestCase):
+
+ def setUp(self):
+ self.cache_dir = tempfile.mkdtemp()
+
+ def tearDown(self):
+ shutil.rmtree(self.cache_dir)
+
+ def test_find_builds(self):
+ """Basic test just to ensure that we can find any builds at all, with
+ no errors."""
+ bc = builds.BuildCache(self.cache_dir)
+ now = datetime.datetime.now()
+ yesterday = now - datetime.timedelta(days=2)
+ buildlist = bc.find_builds(yesterday, now, 'nightly')
+ for l in buildlist:
+ logging.info(l)
+ self.assertTrue(buildlist)
+ buildlist = bc.find_builds(yesterday, now, 'tinderbox')
+ for l in buildlist:
+ logging.info(l)
+ self.assertTrue(buildlist)
View
1 selftest/manifest.ini
@@ -1,2 +1,3 @@
[phoneworker.py]
+[buildcache.py]
View
65 smoketest.sh
@@ -0,0 +1,65 @@
+#!/bin/bash
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Simple smoketest. Launches autophone with a fresh cache file and waits
+# until the user restarts a connected phone. It then runs a simple test
+# that just verifies that fennec can be installed and launched, and reports
+# the result.
+
+cleanup() {
+ if [ $cache ] && [ -a $cache ]
+ then
+ rm $cache
+ fi
+ kill %1
+ sleep 1
+ exit
+}
+
+trap cleanup INT TERM EXIT
+cache=`mktemp`
+rm -f smoketest_pass smoketest_fail
+echo Launching autophone...
+python autophone.py --cache $cache -t tests/smoketest_manifest.ini &
+sleep 1
+echo Please restart the agent/phone for smoke test...
+i=0
+while [ $i -le 60 ]
+do
+ if [ -s $cache ]
+ then
+ break
+ fi
+ sleep 5
+done
+
+if [ -s $cache ]
+then
+ echo Detected phone. Proceeding with test...
+else
+ echo Failed to detect phone.
+ exit 1
+fi
+
+echo Triggering run...
+python trigger_runs.py latest
+echo Waiting for result...
+i=0
+while [ $i -le 60 ]
+do
+ i=$(($i+1))
+ if [ -a smoketest_pass ]
+ then
+ echo 'Smoke test passed!'
+ break
+ fi
+ if [ -a smoketest_fail ]
+ then
+ echo 'Smoke test failed!'
+ break
+ fi
+ sleep 5
+done
View
6 test.py
@@ -10,6 +10,7 @@
"""
import imp
+import logging
import manifestparser
import os
import sys
@@ -31,6 +32,11 @@ def unittests(path):
return unittests
def main(args=sys.argv[1:]):
+ logging.basicConfig(filename='test.log',
+ filemode='w',
+ level=logging.DEBUG,
+ format='%(asctime)s|%(levelname)s|%(message)s')
+
# read the manifest
if args:
manifests = args
View
104 tests/smoketest.py
@@ -0,0 +1,104 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import datetime
+import re
+import os
+import threading
+import androidutils
+import ConfigParser
+import json
+import urllib2
+from time import sleep
+from phonetest import PhoneTest
+from devicemanagerSUT import DeviceManagerSUT
+from mozprofile import FirefoxProfile
+
+class SmokeTest(PhoneTest):
+
+ def __init__(self, phone_cfg, config_file=None, status_cb=None):
+ PhoneTest.__init__(self, phone_cfg, config_file, status_cb)
+
+ def runjob(self, job):
+ try:
+ os.unlink('smoketest_pass')
+ except OSError, e:
+ pass
+ try:
+ os.unlink('smoketest_fail')
+ except OSError, e:
+ pass
+
+ if 'androidprocname' not in job or \
+ 'revision' not in job or 'blddate' not in job or \
+ 'bldtype' not in job or 'version' not in job:
+ self.logger.error('Invalid job configuration: %s' % job)
+ raise NameError('ERROR: Invalid job configuration: %s' % job)
+
+ # Read our config file which gives us our number of
+ # iterations and urls that we will be testing
+ self.prepare_phone(job)
+
+ self.dm = DeviceManagerSUT(self.phone_cfg['ip'],
+ self.phone_cfg['sutcmdport'])
+
+ intent = job['androidprocname'] + '/.App'
+
+ # Clear logcat
+ self.logger.debug('clearing logcat')
+ androidutils.run_adb('logcat', ['-c'], self.phone_cfg['serial'])
+ self.logger.debug('logcat cleared')
+
+ # Run test
+ self.logger.debug('running fennec')
+ self.run_fennec_with_profile(intent, 'about:fennec')
+
+ fennec_launched = self.analyze_logcat(job)
+ start = datetime.datetime.now()
+ while (not fennec_launched and (datetime.datetime.now() - start
+ <= datetime.timedelta(seconds=60))):
+ sleep(3)
+ fennec_launched = self.analyze_logcat(job)
+
+ if fennec_launched:
+ self.logger.info('fennec successfully launched')
+ file('smoketest_pass', 'w')
+ else:
+ self.logger.error('failed to launch fennec')
+ file('smoketest_fail', 'w')
+
+ self.logger.debug('killing fennec')
+ # Get rid of the browser and session store files
+ androidutils.kill_proc_sut(self.phone_cfg['ip'],
+ self.phone_cfg['sutcmdport'],
+ job['androidprocname'])
+
+ self.logger.debug('removing sessionstore files')
+ self.remove_sessionstore_files()
+
+ def prepare_phone(self, job):
+ prefs = { 'browser.firstrun.show.localepicker': False,
+ 'browser.sessionstore.resume_from_crash': False,
+ 'browser.firstrun.show.uidiscovery': False,
+ 'shell.checkDefaultClient': False,
+ 'browser.warnOnQuit': False,
+ 'browser.EULA.override': True,
+ 'toolkit.telemetry.prompted': 2 }
+ profile = FirefoxProfile(preferences=prefs)
+ self.install_profile(profile)
+
+ def analyze_logcat(self, job):
+ buf = androidutils.run_adb('logcat', ['-d'], self.phone_cfg['serial'],
+ timeout=60)
+ buf = [x.strip() for x in buf.split('\n')]
+ got_start = False
+ got_end = False
+
+ for line in buf:
+ if not got_start and 'Start proc org.mozilla.fennec' in line:
+ got_start = True
+ if not got_end and 'Throbber stop' in line:
+ got_end = True
+ return got_start and got_end
+
View
2 tests/smoketest_manifest.ini
@@ -0,0 +1,2 @@
+[smoketest.py]
+
View
45 trigger_runs.py
@@ -21,26 +21,30 @@ def from_iso_date_or_datetime(s):
def main(args, options):
- m = re.match('(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})', args[0])
- if re.match('\d{14}', args[0]):
- # build id
- build_time = datetime.datetime.strptime(args[0], '%Y%m%d%H%M%S')
- start_time = build_time
- end_time = build_time
+ logging.info('Looking for builds...')
+ if args[0] == 'latest':
+ commands = ['triggerjobs %s' %
+ builds.BuildCache().find_latest_build(options.branch)]
else:
- start_time = from_iso_date_or_datetime(args[0])
- if len(args) > 1:
- end_time = from_iso_date_or_datetime(args[1])
+ m = re.match('(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})', args[0])
+ if re.match('\d{14}', args[0]):
+ # build id
+ build_time = datetime.datetime.strptime(args[0], '%Y%m%d%H%M%S')
+ start_time = build_time
+ end_time = build_time
else:
- end_time = datetime.datetime.now()
- if not start_time.tzinfo:
- start_time = start_time.replace(tzinfo=pytz.timezone('US/Pacific'))
- if not end_time.tzinfo:
- end_time = end_time.replace(tzinfo=pytz.timezone('US/Pacific'))
- logging.info('Looking for builds...')
- commands = ['triggerjobs %s' % url for url in
- builds.BuildCache().find_builds(start_time, end_time,
- options.branch)]
+ start_time = from_iso_date_or_datetime(args[0])
+ if len(args) > 1:
+ end_time = from_iso_date_or_datetime(args[1])
+ else:
+ end_time = datetime.datetime.now()
+ if not start_time.tzinfo:
+ start_time = start_time.replace(tzinfo=pytz.timezone('US/Pacific'))
+ if not end_time.tzinfo:
+ end_time = end_time.replace(tzinfo=pytz.timezone('US/Pacific'))
+ commands = ['triggerjobs %s' % url for url in
+ builds.BuildCache().find_builds(start_time, end_time,
+ options.branch)]
logging.info('Connecting to autophone server...')
commands.append('exit')
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
@@ -67,6 +71,7 @@ def main(args, options):
- a build ID, e.g. 20120403063158
- a date/datetime, e.g. 2012-04-03 or 2012-04-03T06:31:58
- a date/datetime range, e.g. 2012-04-03T06:31:58 2012-04-05
+- the string "latest"
If a build ID is given, a test run is initiated for that, and only that,
particular build.
@@ -75,7 +80,9 @@ def main(args, options):
with build IDs between the given date/datetime and now.
If a date/datetime range is given, test runs are initiated for all builds
-with build IDs in the given range.'''
+with build IDs in the given range.
+
+If "latest" is given, test runs are initiated for the most recent build.'''
parser = OptionParser(usage=usage)
parser.add_option('-i', '--ip', action='store', type='string', dest='ip',
default='127.0.0.1',

0 comments on commit eee9724

Please sign in to comment.
Something went wrong with that request. Please try again.