Smoke test, refactored builds.py.

Also added signal handler, support for getting latest build, switched builds.py
from using urllib2 to faster ftplib.

autophone.py

@@ -13,6 +13,7 @@
 import multiprocessing
 import os
 import shutil
+import signal
 import socket
 import sys
 import tempfile
@@ -149,6 +150,9 @@ def worker_msg_loop(self):
                     msg = self.worker_msg_queue.get(timeout=5)
                 except Queue.Empty:
                     continue
+                except IOError, e:
+                    if e.errno == errno.EINTR:
+                        continue
                 self.phone_workers[msg.phoneid].process_msg(msg)
         except KeyboardInterrupt:
             self.stop()
@@ -399,6 +403,9 @@ def stop(self):
 def main(is_restarting, reboot_phones, test_path, cachefile, ipaddr, port,
          logfile, loglevel_name, emailcfg):
 
+    def sigterm_handler(signum, frame):
+        autophone.stop()
+
     adb_check = androidutils.check_for_adb()
     if adb_check != 0:
         print 'Could not execute adb: %s.' % os.strerror(adb_check)
@@ -425,6 +432,7 @@ def main(is_restarting, reboot_phones, test_path, cachefile, ipaddr, port,
         (datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), port)
     autophone = AutoPhone(is_restarting, reboot_phones, test_path, cachefile,
                           ipaddr, port, logfile, loglevel, emailcfg)
+    signal.signal(signal.SIGTERM, sigterm_handler)
     autophone.run()
     print '%s AutoPhone terminated.' % datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
     return 0

builds.py

@@ -4,112 +4,176 @@
 
 import base64
 import datetime
+import ftplib
 import logging
 import os
 import pytz
 import re
 import shutil
 import tempfile
 import urllib
-import urllib2
+import urlparse
 
-nightly_dirnames = [re.compile('(.*)-mozilla-central-android$')]
+
+class NightlyBranch(object):
+
+    nightly_dirnames = [re.compile('(.*)-mozilla-central-android$')]
+
+    def nightly_ftpdir(self, year, month):
+        return 'ftp://ftp.mozilla.org/pub/mobile/nightly/%d/%02d/' % (year,
+                                                                      month)
+
+    def ftpdirs(self, start_time, end_time):
+        dirs = []
+        y = start_time.year
+        m = start_time.month
+        while y < end_time.year or m <= end_time.month:
+            dirs.append(self.nightly_ftpdir(y, m))
+            if m == 12:
+                y += 1
+                m = 1
+            else:
+                m += 1
+        return dirs
+
+    def build_info_from_ftp(self, ftpline):
+        srcdir = ftpline.split(' ')[-1].strip()
+        build_time = None
+        dirnamematch = None
+        logging.debug('matching dir names')
+        for r in self.nightly_dirnames:
+            dirnamematch = r.match(srcdir)
+            if dirnamematch:
+                break
+        if dirnamematch:
+            logging.debug('build time')
+            build_time = datetime.datetime.strptime(dirnamematch.group(1),
+                                                    '%Y-%m-%d-%H-%M-%S')
+            build_time = build_time.replace(tzinfo=pytz.timezone('US/Pacific'))
+            logging.debug('got build time')
+        logging.debug('got info')
+        return (srcdir, build_time)
+
+    def build_date_from_url(self, url):
+        # nightly urls are of the form
+        #  ftp://ftp.mozilla.org/pub/mobile/nightly/<year>/<month>/<year>-
+        #    <month>-<day>-<hour>-<minute>-<second>-<branch>-android/
+        #    <buildfile>
+        m = re.search('nightly\/[\d]{4}\/[\d]{2}\/([\d]{4}-[\d]{2}-[\d]{2}-[\d]{2}-[\d]{2}-[\d]{2})-', url)
+        if not m:
+            return None
+        return datetime.datetime.strptime(m.group(1), '%Y-%m-%d-%H-%M-%S')
+
+
+class TinderboxBranch(object):
+
+    main_ftp_url = 'ftp://ftp.mozilla.org/pub/mozilla.org/mobile/tinderbox-builds/'
+
+    def ftpdirs(self, start_time, end_time):
+        # FIXME: Can we be certain that there's only one buildID (unique
+        # timestamp) regardless of branch (at least m-i vs m-c)?
+        return [self.main_ftp_url + 'mozilla-inbound-android/',
+                self.main_ftp_url + 'mozilla-central-android/']
+
+    def build_info_from_ftp(self, ftpline):
+        srcdir = ftpline.split()[8].strip()
+        build_time = datetime.datetime.fromtimestamp(int(srcdir), pytz.timezone('US/Pacific'))
+        return (srcdir, build_time)
+
+    def build_date_from_url(self, url):
+        # tinderbox urls are of the form
+        #   ftp://ftp.mozilla.org/pub/mozilla.org/mobile/tinderbox-builds/
+        #     <branch>-android/<build timestamp>/<buildfile>
+        m = re.search('tinderbox-builds\/.*-android\/[\d]+\/', url)
+        if not m:
+            return None
+        return datetime.datetime.fromtimestamp(int(m.group(1)),
+                                               pytz.timezone('US/Pacific'))
 
 
 class BuildCache(object):
 
     MAX_NUM_BUILDS = 20
     EXPIRE_AFTER_SECONDS = 60*60*24
 
+    class FtpLineCache(object):
+        def __init__(self):
+            self.lines = []
+        def __call__(self, line):
+            self.lines.append(line)
+
     def __init__(self, cache_dir='builds'):
         self.cache_dir = cache_dir
         if not os.path.exists(self.cache_dir):
             os.mkdir(self.cache_dir)
 
-    def nightly_ftpdir(self, year, month):
-        return 'ftp://ftp.mozilla.org/pub/mobile/nightly/%d/%02d/' % (year, month)
+    @classmethod
+    def branch(cls, s):
+        if 'nightly' in s:
+            return NightlyBranch()
+        if 'tinderbox' in s:
+            return TinderboxBranch()
+        return None
+
+    def find_latest_build(self, branch_name='nightly'):
+        # This assumes at least one build has been created in the last 24
+        # hours.
+        now = datetime.datetime.now()
+        builds = self.find_builds(now - datetime.timedelta(days=1), now,
+                                  branch_name)
+        builds.sort()
+        return builds[-1]
+
+    def find_builds(self, start_time, end_time, branch_name='nightly'):
+        branch = self.branch(branch_name)
+        if not branch:
+            logging.error('unsupport branch "%s"' % branch_name)
+            return []
 
-    def find_builds(self, start_time, end_time, branch='nightly'):
         if not start_time.tzinfo:
             start_time = start_time.replace(tzinfo=pytz.timezone('US/Pacific'))
         if not end_time.tzinfo:
             end_time = end_time.replace(tzinfo=pytz.timezone('US/Pacific'))
+
         builds = []
         fennecregex = re.compile("fennec.*\.android-arm\.apk")
-        ftpdirs = []
-        # FIXME: refactor branch into objects
-        if branch == 'nightly':
-            y = start_time.year
-            m = start_time.month
-            while y < end_time.year or m <= end_time.month:
-                ftpdirs.append(self.nightly_ftpdir(y, m))
-                if m == 12:
-                    y += 1
-                    m = 1
-                else:
-                    m += 1
-        elif branch == 'tinderbox':
-            # FIXME: Can we be certain that there's only one buildID (unique
-            # timestamp) regardless of branch (at least m-i vs m-c)?
-            ftpdirs.append('ftp://ftp.mozilla.org/pub/mozilla.org/mobile/tinderbox-builds/mozilla-inbound-android/')
-            ftpdirs.append('ftp://ftp.mozilla.org/pub/mozilla.org/mobile/tinderbox-builds/mozilla-central-android/')
-
-        for d in ftpdirs:
-            f = urllib2.urlopen(d)
-            for line in f:
-                build_time = None
-                if branch == 'nightly':
-                    srcdir = line.split(' ')[-1].strip()
-                    dirnamematch = None
-                    for r in nightly_dirnames:
-                        dirnamematch = r.match(srcdir)
-                        if dirnamematch:
-                            break
-                    if dirnamematch:
-                        build_time = datetime.datetime.strptime(dirnamematch.group(1),
-                                                                '%Y-%m-%d-%H-%M-%S')
-                        build_time = build_time.replace(tzinfo=pytz.timezone('US/Pacific'))
-                    else:
-                        continue
-                elif branch == 'tinderbox':
-                    srcdir = line.split()[8].strip()
-                    build_time = datetime.datetime.fromtimestamp(int(srcdir), pytz.timezone('US/Pacific'))
+
+        for d in branch.ftpdirs(start_time, end_time):
+            url = urlparse.urlparse(d)
+            logging.debug('logging into %s...' % url.netloc)
+            f = ftplib.FTP(url.netloc)
+            f.login()
+            logging.debug('looking for builds in %s...' % url.path)
+            lines = self.FtpLineCache()
+            f.dir(url.path, lines)
+            file('lines.out', 'w').write('\n'.join(lines.lines))
+            for line in lines.lines:
+                srcdir, build_time = branch.build_info_from_ftp(line)
+
+                if not build_time:
+                    continue
 
                 if build_time and (build_time < start_time or
                                    build_time > end_time):
                     continue
 
-                newurl = d + srcdir
-                f2 = urllib.urlopen(newurl)
-                for l2 in f2:
+                newpath = url.path + srcdir
+                lines2 = self.FtpLineCache()
+                f.dir(newpath, lines2)
+                for l2 in lines2.lines:
                     filename = l2.split(' ')[-1].strip()
                     if fennecregex.match(filename):
-                        fileurl = newurl + "/" + filename
+                        fileurl = url.scheme + '://' + url.netloc + newpath + "/" + filename
                         builds.append(fileurl)
         return builds
 
     def build_date(self, url):
-        # nightly urls are of the form
-        #  ftp://ftp.mozilla.org/pub/mobile/nightly/<year>/<month>/<year>-
-        #    <month>-<day>-<hour>-<minute>-<second>-<branch>-android/
-        #    <buildfile>
-        # tinderbox urls are of the form
-        #   ftp://ftp.mozilla.org/pub/mozilla.org/mobile/tinderbox-builds/
-        #     <branch>-android/<build timestamp>/<buildfile>
+        branch = self.branch(url)
         builddate = None
-        if 'nightly' in url:
-            m = re.search('nightly\/[\d]{4}\/[\d]{2}\/([\d]{4}-[\d]{2}-[\d]{2}-[\d]{2}-[\d]{2}-[\d]{2})-', url)
-            if not m:
-                logging.error('bad URL "%s"' % url)
-                return None
-            builddate = datetime.datetime.strptime(m.group(1), '%Y-%m-%d-%H-%M-%S')
-        elif 'tinderbox' in url:
-            m = re.search('tinderbox-builds\/.*-android\/[\d]+\/', url)
-            if not m:
-                logging.error('bad URL "%s"' % url)
-                return None
-            builddate = datetime.datetime.fromtimestamp(int(m.group(1)),
-                                                        pytz.timezone('US/Pacific'))
+        if branch:
+            builddate = branch.build_date_from_url(url)
+        if not builddate:
+            logging.error('bad URL "%s"' % url)
         return builddate
 
     def get(self, url, force=False):

selftest/buildcache.py

@@ -0,0 +1,34 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import datetime
+import logging
+import shutil
+import tempfile
+import unittest
+
+import builds
+
+class BuildsTest(unittest.TestCase):
+
+    def setUp(self):
+        self.cache_dir = tempfile.mkdtemp()
+
+    def tearDown(self):
+        shutil.rmtree(self.cache_dir)
+
+    def test_find_builds(self):
+        """Basic test just to ensure that we can find any builds at all, with
+        no errors."""
+        bc = builds.BuildCache(self.cache_dir)
+        now = datetime.datetime.now()
+        yesterday = now - datetime.timedelta(days=2)
+        buildlist = bc.find_builds(yesterday, now, 'nightly')
+        for l in buildlist:
+            logging.info(l)
+        self.assertTrue(buildlist)
+        buildlist = bc.find_builds(yesterday, now, 'tinderbox')
+        for l in buildlist:
+            logging.info(l)
+        self.assertTrue(buildlist)

selftest/manifest.ini

@@ -1,2 +1,3 @@
 [phoneworker.py]
+[buildcache.py]
 

smoketest.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Simple smoketest. Launches autophone with a fresh cache file and waits
+# until the user restarts a connected phone. It then runs a simple test
+# that just verifies that fennec can be installed and launched, and reports
+# the result.
+
+cleanup() {
+    if [ $cache ] && [ -a $cache ]
+    then
+        rm $cache
+    fi
+    kill %1
+    sleep 1
+    exit
+}
+
+trap cleanup INT TERM EXIT
+cache=`mktemp`
+rm -f smoketest_pass smoketest_fail
+echo Launching autophone...
+python autophone.py --cache $cache -t tests/smoketest_manifest.ini &
+sleep 1
+echo Please restart the agent/phone for smoke test...
+i=0
+while [ $i -le 60 ]
+do
+    if [ -s $cache ]
+    then
+        break
+    fi
+    sleep 5
+done
+
+if [ -s $cache ]
+then
+    echo Detected phone. Proceeding with test...
+else
+    echo Failed to detect phone.
+    exit 1
+fi
+
+echo Triggering run...
+python trigger_runs.py latest
+echo Waiting for result...
+i=0
+while [ $i -le 60 ]
+do
+    i=$(($i+1))
+    if [ -a smoketest_pass ]
+    then
+        echo 'Smoke test passed!'
+        break
+    fi
+    if [ -a smoketest_fail ]
+    then
+        echo 'Smoke test failed!'
+        break
+    fi
+    sleep 5
+done