Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Bug 849251 - Decouple build cache as TCP server in a separate thread.…

… r=bc
  • Loading branch information...
commit 88f839b989a7e43f4eafa3ba36e79057394c8b27 1 parent 33564b1
Mark Côté markrcote authored
183 autophone.py
@@ -2,38 +2,28 @@
2 2 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
3 3 # You can obtain one at http://mozilla.org/MPL/2.0/.
4 4
5   -import ConfigParser
6 5 import Queue
7 6 import SocketServer
8 7 import datetime
9 8 import errno
10 9 import inspect
  10 +import json
11 11 import logging
12   -import math
13 12 import multiprocessing
14 13 import os
15   -import shutil
16 14 import signal
17 15 import socket
18 16 import sys
19   -import tempfile
20 17 import threading
21   -import time
22 18 import traceback
23 19 import urlparse
24   -import zipfile
25   -
26   -try:
27   - import json
28   -except ImportError:
29   - # for python 2.5 compatibility
30   - import simplejson as json
31 20
32 21 from manifestparser import TestManifest
33 22 from mozdevice.devicemanager import NetworkTools
34 23 from pulsebuildmonitor import start_pulse_monitor
35 24
36 25 import builds
  26 +import buildserver
37 27 import phonetest
38 28
39 29 from mailer import Mailer
@@ -71,15 +61,13 @@ def handle(self):
71 61 if not line:
72 62 continue
73 63 if line == 'quit' or line == 'exit':
74   - self.request.close()
75 64 return
76 65 response = self.server.cmd_cb(line)
77 66 self.request.send(response + '\n')
78 67
79 68 def __init__(self, clear_cache, reboot_phones, test_path, cachefile,
80 69 ipaddr, port, logfile, loglevel, emailcfg, enable_pulse,
81   - enable_unittests, cache_dir, override_build_dir,
82   - repos, buildtypes):
  70 + repos, buildtypes, build_cache_port):
83 71 self._test_path = test_path
84 72 self._cache = cachefile
85 73 if ipaddr:
@@ -93,10 +81,8 @@ def __init__(self, clear_cache, reboot_phones, test_path, cachefile,
93 81 self.logfile = logfile
94 82 self.loglevel = loglevel
95 83 self.mailer = Mailer(emailcfg, '[autophone] ')
96   - self.build_cache = builds.BuildCache(repos, buildtypes,
97   - cache_dir=cache_dir,
98   - override_build_dir=override_build_dir,
99   - enable_unittests=enable_unittests)
  84 + self.build_cache_port = build_cache_port
  85 +
100 86 self._stop = False
101 87 self._next_worker_num = 0
102 88 self.phone_workers = {} # indexed by mac address
@@ -135,7 +121,6 @@ def __init__(self, clear_cache, reboot_phones, test_path, cachefile,
135 121 else:
136 122 self.pulsemonitor = None
137 123
138   - self.enable_unittests = enable_unittests
139 124 self.restart_workers = {}
140 125
141 126 @property
@@ -149,6 +134,7 @@ def run(self):
149 134 self.CmdTCPHandler)
150 135 self.server.cmd_cb = self.route_cmd
151 136 self.server_thread = threading.Thread(target=self.server.serve_forever)
  137 + self.server_thread.daemon = True
152 138 self.server_thread.start()
153 139 self.worker_msg_loop()
154 140
@@ -211,13 +197,12 @@ def worker_msg_loop(self):
211 197 self.stop()
212 198
213 199 # Start the phones for testing
214   - def start_tests(self, job):
215   - if not self.is_valid_job(job):
216   - return
  200 + def start_tests(self, build_url):
217 201 self.worker_lock.acquire()
218 202 for p in self.phone_workers.values():
219   - logging.info('Starting job on phone: %s' % p.phone_cfg['phoneid'])
220   - p.add_job(job)
  203 + logging.info('Notifying device %s of new build.' %
  204 + p.phone_cfg['phoneid'])
  205 + p.new_build(build_url)
221 206 self.worker_lock.release()
222 207
223 208 def route_cmd(self, data):
@@ -295,7 +280,7 @@ def create_worker(self, phone_cfg, user_cfg):
295 280 worker = PhoneWorker(self.next_worker_num, self.ipaddr,
296 281 tests, phone_cfg, user_cfg, self.worker_msg_queue,
297 282 '%s-%s' % (logfile_prefix, phone_cfg['phoneid']),
298   - self.loglevel, self.mailer)
  283 + self.loglevel, self.mailer, self.build_cache_port)
299 284 self.phone_workers[phone_cfg['phoneid']] = worker
300 285 worker.start()
301 286
@@ -386,10 +371,8 @@ def read_tests(self):
386 371 self._tests.extend(tests)
387 372
388 373 def trigger_jobs(self, data):
389   - logging.debug('trigger_jobs: data %s' % data)
390   - job = self.build_job(self.get_build(data))
391   - logging.info('Received user-specified job: %s' % job)
392   - self.start_tests(job)
  374 + logging.info('Received user-specified job: %s' % data)
  375 + self.start_tests(data)
393 376
394 377 def reset_phones(self):
395 378 logging.info('Resetting phones...')
@@ -406,100 +389,7 @@ def on_build(self, msg):
406 389 # those, and only run the ones with real URLs
407 390 # We create jobs for all the phones and push them into the queue
408 391 if 'buildurl' in msg:
409   - self.start_tests(self.build_job(self.get_build(msg['buildurl'])))
410   -
411   - def get_build(self, buildurl):
412   - cache_build_dir = self.build_cache.get(buildurl,
413   - self.enable_unittests)
414   - if not cache_build_dir:
415   - logging.warn('Errors occured getting build %s.' % buildurl)
416   - return None
417   - try:
418   - build_path = os.path.join(cache_build_dir, 'build.apk')
419   - z = zipfile.ZipFile(build_path)
420   - z.testzip()
421   - except zipfile.BadZipfile:
422   - logging.error('%s is a bad apk; redownloading...' % build_path)
423   - cache_build_dir = self.build_cache.get(buildurl,
424   - self.enable_unittests,
425   - force=True)
426   - return cache_build_dir
427   -
428   - def build_job(self, cache_build_dir):
429   - if not cache_build_dir:
430   - logging.warn('No build available. Aborting job.')
431   - return None
432   - tmpdir = tempfile.mkdtemp()
433   - try:
434   - build_path = os.path.join(cache_build_dir, 'build.apk')
435   - apkfile = zipfile.ZipFile(build_path)
436   - apkfile.extract('application.ini', tmpdir)
437   - except zipfile.BadZipfile:
438   - # we should have already tried to redownload bad zips, so treat
439   - # this as fatal.
440   - logging.error('%s is a bad apk; aborting job.' % build_path)
441   - shutil.rmtree(tmpdir)
442   - return None
443   - cfg = ConfigParser.RawConfigParser()
444   - cfg.read(os.path.join(tmpdir, 'application.ini'))
445   - rev = cfg.get('App', 'SourceStamp')
446   - ver = cfg.get('App', 'Version')
447   - repo = cfg.get('App', 'SourceRepository')
448   - buildid = cfg.get('App', 'BuildID')
449   - blddate = datetime.datetime.strptime(buildid,
450   - '%Y%m%d%H%M%S')
451   - procname = ''
452   - if repo == 'http://hg.mozilla.org/mozilla-central':
453   - tree = 'mozilla-central'
454   - procname = 'org.mozilla.fennec'
455   - elif repo == 'http://hg.mozilla.org/integration/mozilla-inbound':
456   - tree = 'mozilla-inbound'
457   - procname = 'org.mozilla.fennec'
458   - elif repo == 'http://hg.mozilla.org/releases/mozilla-aurora':
459   - tree = 'mozilla-aurora'
460   - procname = 'org.mozilla.fennec_aurora'
461   - elif repo == 'http://hg.mozilla.org/releases/mozilla-beta':
462   - tree = 'mozilla-beta'
463   - procname = 'org.mozilla.firefox'
464   -
465   - job = {'cache_build_dir': cache_build_dir,
466   - 'tree': tree,
467   - 'blddate': math.trunc(time.mktime(blddate.timetuple())),
468   - 'buildid': buildid,
469   - 'revision': rev,
470   - 'androidprocname': procname,
471   - 'version': ver,
472   - 'bldtype': 'opt'}
473   - shutil.rmtree(tmpdir)
474   - return job
475   -
476   - def is_valid_job(self, job):
477   - if job is None:
478   - return False
479   -
480   - error_list = []
481   -
482   - if 'androidprocname' not in job:
483   - error_list.append('missing androidprocname')
484   -
485   - if 'revision' not in job:
486   - error_list.append('missing revision')
487   -
488   - if 'blddate' not in job:
489   - error_list.append('missing blddate')
490   -
491   - if 'bldtype' not in job:
492   - error_list.append('missing bldtype')
493   -
494   - if 'version' not in job:
495   - error_list.append('missing version')
496   -
497   - if len(error_list) > 0:
498   - error_message = 'ERROR: Invalid job configuration: %s ' % job + ', '.join(error_list)
499   - self.logger.error(error_message)
500   - raise NameError(error_message)
501   -
502   - return True
  392 + self.start_tests(msg['buildurl'])
503 393
504 394 def stop(self):
505 395 self._stop = True
@@ -511,7 +401,7 @@ def stop(self):
511 401
512 402 def main(clear_cache, reboot_phones, test_path, cachefile, ipaddr, port,
513 403 logfile, loglevel_name, emailcfg, enable_pulse, enable_unittests,
514   - cache_dir, override_build_dir, repos, buildtypes):
  404 + cache_dir, override_build_dir, repos, buildtypes, build_cache_port):
515 405
516 406 def sigterm_handler(signum, frame):
517 407 autophone.stop()
@@ -531,14 +421,14 @@ def sigterm_handler(signum, frame):
531 421 level=loglevel,
532 422 format='%(asctime)s|%(levelname)s|%(message)s')
533 423
534   - print '%s Starting server on port %d.' % \
535   - (datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), port)
  424 + print '%s Starting build-cache server on port %d.' % (
  425 + datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
  426 + build_cache_port)
536 427 try:
537   - autophone = AutoPhone(clear_cache, reboot_phones, test_path, cachefile,
538   - ipaddr, port, logfile, loglevel, emailcfg,
539   - enable_pulse, enable_unittests,
540   - cache_dir, override_build_dir,
541   - repos, buildtypes)
  428 + build_cache = builds.BuildCache(
  429 + repos, buildtypes, cache_dir=cache_dir,
  430 + override_build_dir=override_build_dir,
  431 + enable_unittests=enable_unittests)
542 432 except builds.BuildCacheException, e:
543 433 print '''%s
544 434
@@ -553,9 +443,26 @@ def sigterm_handler(signum, frame):
553 443 parser.print_help()
554 444 return 1
555 445
  446 + build_cache_server = buildserver.BuildCacheServer(
  447 + ('127.0.0.1', build_cache_port), buildserver.BuildCacheHandler)
  448 + build_cache_server.build_cache = build_cache
  449 + build_cache_server_thread = threading.Thread(
  450 + target=build_cache_server.serve_forever)
  451 + build_cache_server_thread.daemon = True
  452 + build_cache_server_thread.start()
  453 +
  454 + print '%s Starting server on port %d.' % (
  455 + datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), port)
  456 + autophone = AutoPhone(clear_cache, reboot_phones, test_path, cachefile,
  457 + ipaddr, port, logfile, loglevel, emailcfg,
  458 + enable_pulse, repos, buildtypes, build_cache_port)
556 459 signal.signal(signal.SIGTERM, sigterm_handler)
557 460 autophone.run()
558 461 print '%s AutoPhone terminated.' % datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
  462 + print 'Shutting down build-cache server...'
  463 + build_cache_server.shutdown()
  464 + build_cache_server_thread.join()
  465 + print 'Done.'
559 466 return 0
560 467
561 468
@@ -628,6 +535,15 @@ def sigterm_handler(signum, frame):
628 535 'One of opt or debug. To specify multiple build types, '
629 536 'specify them with additional --buildtype options. '
630 537 'Defaults to opt.')
  538 + parser.add_option('--build-cache-port',
  539 + dest='build_cache_port',
  540 + action='store',
  541 + type='int',
  542 + default=buildserver.DEFAULT_PORT,
  543 + help='Port for build-cache server. If you are running '
  544 + 'multiple instances of autophone, this will have to be '
  545 + 'different in each. Defaults to %d.' %
  546 + buildserver.DEFAULT_PORT)
631 547
632 548 (options, args) = parser.parse_args()
633 549 if not options.repos:
@@ -642,6 +558,7 @@ def sigterm_handler(signum, frame):
642 558 options.emailcfg, options.enable_pulse,
643 559 options.enable_unittests,
644 560 options.cache_dir, options.override_build_dir,
645   - options.repos, options.buildtypes)
  561 + options.repos, options.buildtypes,
  562 + options.build_cache_port)
646 563
647 564 sys.exit(exit_code)
120 builds.py
@@ -2,15 +2,19 @@
2 2 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
3 3 # You can obtain one at http://mozilla.org/MPL/2.0/.
4 4
  5 +import ConfigParser
5 6 import base64
6 7 import datetime
7 8 import ftplib
  9 +import json
8 10 import logging
  11 +import math
9 12 import os
10 13 import pytz
11 14 import re
12 15 import shutil
13 16 import tempfile
  17 +import time
14 18 import urllib
15 19 import urlparse
16 20 import zipfile
@@ -216,16 +220,39 @@ def build_date(self, url):
216 220 logging.error('bad URL "%s"' % url)
217 221 return builddate
218 222
219   - def get(self, buildurl, enable_unittests, force=False):
  223 + def get(self, buildurl, force=False):
  224 + """Returns info on a cached build, fetching it if necessary.
  225 + Returns a dict with a boolean 'success' item.
  226 + If 'success' is False, the dict also contains an 'error' item holding a
  227 + descriptive string.
  228 + If 'success' is True, the dict also contains a 'metadata' item, which is
  229 + a dict of build metadata. The path to the build is the
  230 + 'cache_build_dir' item, which is a directory containing build.apk,
  231 + symbols/, and, if self.enable_unittests is true, robocop.apk and tests/.
  232 + If not found, fetches them, assuming a standard file structure.
  233 + Cleans the cache before getting started.
  234 + If self.override_build_dir is set, 'cache_build_dir' is set to
  235 + that value without verifying the contents nor fetching anything (though
  236 + it will still try to open build.apk to read in the metadata).
  237 + See BuildCache.build_metadata() for the other metadata items.
  238 + """
220 239 if self.override_build_dir:
221   - return self.override_build_dir
  240 + return {'success': True,
  241 + 'metadata': self.build_metadata(self.override_build_dir)}
222 242 build_dir = base64.b64encode(buildurl)
223 243 self.clean_cache([build_dir])
224 244 cache_build_dir = os.path.join(self.cache_dir, build_dir)
225 245 build_path = os.path.join(cache_build_dir, 'build.apk')
226 246 if not os.path.exists(cache_build_dir):
227 247 os.makedirs(cache_build_dir)
228   - if force or not os.path.exists(build_path):
  248 +
  249 + # build
  250 + try:
  251 + download_build = (force or not os.path.exists(build_path) or
  252 + zipfile.ZipFile(build_path).testzip() is not None)
  253 + except zipfile.BadZipFile:
  254 + download_build = True
  255 + if download_build:
229 256 # retrieve to temporary file then move over, so we don't end
230 257 # up with half a file if it aborts
231 258 tmpf = tempfile.NamedTemporaryFile(delete=False)
@@ -234,11 +261,14 @@ def get(self, buildurl, enable_unittests, force=False):
234 261 urllib.urlretrieve(buildurl, tmpf.name)
235 262 except IOError:
236 263 os.unlink(tmpf.name)
237   - logging.error('IO Error retrieving build: %s.' % buildurl)
  264 + err = 'IO Error retrieving build: %s.' % buildurl
  265 + logging.error(err)
238 266 logging.error(traceback.format_exc())
239   - return None
  267 + return {'success': False, 'error': err}
240 268 shutil.move(tmpf.name, build_path)
241 269 file(os.path.join(cache_build_dir, 'lastused'), 'w')
  270 +
  271 + # symbols
242 272 symbols_path = os.path.join(cache_build_dir, 'symbols')
243 273 if force or not os.path.exists(symbols_path):
244 274 tmpf = tempfile.NamedTemporaryFile(delete=False)
@@ -264,9 +294,11 @@ def get(self, buildurl, enable_unittests, force=False):
264 294 except:
265 295 pass
266 296 os.unlink(tmpf.name)
267   - if enable_unittests:
  297 +
  298 + # tests
  299 + if self.enable_unittests:
268 300 tests_path = os.path.join(cache_build_dir, 'tests')
269   - if (force or not os.path.exists(tests_path)) and enable_unittests:
  301 + if force or not os.path.exists(tests_path):
270 302 tmpf = tempfile.NamedTemporaryFile(delete=False)
271 303 tmpf.close()
272 304 # XXX: assumes fixed buildurl-> tests_url mapping
@@ -275,9 +307,10 @@ def get(self, buildurl, enable_unittests, force=False):
275 307 urllib.urlretrieve(tests_url, tmpf.name)
276 308 except IOError:
277 309 os.unlink(tmpf.name)
278   - logging.error('IO Error retrieving tests: %s.' % tests_url)
  310 + err = 'IO Error retrieving tests: %s.' % tests_url
  311 + logging.error(err)
279 312 logging.error(traceback.format_exc())
280   - return None
  313 + return {'success': False, 'error': err}
281 314 tests_zipfile = zipfile.ZipFile(tmpf.name)
282 315 tests_zipfile.extractall(tests_path)
283 316 tests_zipfile.close()
@@ -291,10 +324,10 @@ def get(self, buildurl, enable_unittests, force=False):
291 324 urllib.urlretrieve(robocop_url, tmpf.name)
292 325 except IOError:
293 326 os.unlink(tmpf.name)
294   - logging.error('IO Error retrieving robocop.apk: %s.' %
295   - robocop_url)
  327 + err = 'IO Error retrieving robocop.apk: %s.' % robocop_url
  328 + logging.error(err)
296 329 logging.error(traceback.format_exc())
297   - return None
  330 + return {'success': False, 'error': err}
298 331 shutil.move(tmpf.name, robocop_path)
299 332 # XXX: assumes fixed buildurl-> fennec_ids.txt mapping
300 333 fennec_ids_url = urlparse.urljoin(buildurl, 'fennec_ids.txt')
@@ -305,12 +338,15 @@ def get(self, buildurl, enable_unittests, force=False):
305 338 urllib.urlretrieve(fennec_ids_url, tmpf.name)
306 339 except IOError:
307 340 os.unlink(tmpf.name)
308   - logging.error('IO Error retrieving fennec_ids.txt: %s.' %
309   - fennec_ids_url)
  341 + err = 'IO Error retrieving fennec_ids.txt: %s.' % \
  342 + fennec_ids_url
  343 + logging.error(err)
310 344 logging.error(traceback.format_exc())
311   - return None
  345 + return {'success': False, 'error': err}
312 346 shutil.move(tmpf.name, fennec_ids_path)
313   - return cache_build_dir
  347 +
  348 + return {'success': True,
  349 + 'metadata': self.build_metadata(cache_build_dir)}
314 350
315 351 def clean_cache(self, preserve=[]):
316 352 def lastused_path(d):
@@ -337,3 +373,55 @@ def keep_build(d):
337 373 b = builds.pop(0)[0]
338 374 logging.info('Expiring %s' % b)
339 375 shutil.rmtree(os.path.join(self.cache_dir, b))
  376 +
  377 + def build_metadata(self, build_dir):
  378 + build_metadata_path = os.path.join(build_dir, 'metadata.json')
  379 + if os.path.exists(build_metadata_path):
  380 + try:
  381 + return json.loads(file(build_metadata_path).read())
  382 + except (ValueError, IOError):
  383 + pass
  384 + tmpdir = tempfile.mkdtemp()
  385 + try:
  386 + build_path = os.path.join(build_dir, 'build.apk')
  387 + apkfile = zipfile.ZipFile(build_path)
  388 + apkfile.extract('application.ini', tmpdir)
  389 + except zipfile.BadZipfile:
  390 + # we should have already tried to redownload bad zips, so treat
  391 + # this as fatal.
  392 + logging.error('%s is a bad apk; aborting job.' % build_path)
  393 + shutil.rmtree(tmpdir)
  394 + return None
  395 + cfg = ConfigParser.RawConfigParser()
  396 + cfg.read(os.path.join(tmpdir, 'application.ini'))
  397 + rev = cfg.get('App', 'SourceStamp')
  398 + ver = cfg.get('App', 'Version')
  399 + repo = cfg.get('App', 'SourceRepository')
  400 + buildid = cfg.get('App', 'BuildID')
  401 + blddate = datetime.datetime.strptime(buildid,
  402 + '%Y%m%d%H%M%S')
  403 + procname = ''
  404 + if repo == 'http://hg.mozilla.org/mozilla-central':
  405 + tree = 'mozilla-central'
  406 + procname = 'org.mozilla.fennec'
  407 + elif repo == 'http://hg.mozilla.org/integration/mozilla-inbound':
  408 + tree = 'mozilla-inbound'
  409 + procname = 'org.mozilla.fennec'
  410 + elif repo == 'http://hg.mozilla.org/releases/mozilla-aurora':
  411 + tree = 'mozilla-aurora'
  412 + procname = 'org.mozilla.fennec_aurora'
  413 + elif repo == 'http://hg.mozilla.org/releases/mozilla-beta':
  414 + tree = 'mozilla-beta'
  415 + procname = 'org.mozilla.firefox'
  416 +
  417 + metadata = {'cache_build_dir': build_dir,
  418 + 'tree': tree,
  419 + 'blddate': math.trunc(time.mktime(blddate.timetuple())),
  420 + 'buildid': buildid,
  421 + 'revision': rev,
  422 + 'androidprocname': procname,
  423 + 'version': ver,
  424 + 'bldtype': 'opt'}
  425 + shutil.rmtree(tmpdir)
  426 + file(build_metadata_path, 'w').write(json.dumps(metadata))
  427 + return metadata
84 buildserver.py
... ... @@ -0,0 +1,84 @@
  1 +# This Source Code Form is subject to the terms of the Mozilla Public
  2 +# License, v. 2.0. If a copy of the MPL was not distributed with this file,
  3 +# You can obtain one at http://mozilla.org/MPL/2.0/.
  4 +
  5 +import SocketServer
  6 +import errno
  7 +import json
  8 +import socket
  9 +import threading
  10 +
  11 +from builds import BuildCache
  12 +
  13 +DEFAULT_PORT = 28008
  14 +
  15 +class BuildCacheServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer):
  16 +
  17 + build_cache = None
  18 + cache_lock = threading.Lock()
  19 +
  20 +
  21 +class BuildCacheHandler(SocketServer.BaseRequestHandler):
  22 +
  23 + def handle(self):
  24 + buffer = ''
  25 + while True:
  26 + try:
  27 + data = self.request.recv(1024)
  28 + except socket.error, e:
  29 + if e.errno == errno.ECONNRESET:
  30 + return
  31 + raise e
  32 + if not data:
  33 + return
  34 + buffer += data
  35 + while buffer:
  36 + line, nl, rest = buffer.partition('\n')
  37 + if not nl:
  38 + break
  39 + buffer = rest
  40 + line = line.strip()
  41 + if not line:
  42 + continue
  43 + if line == 'quit' or line == 'exit':
  44 + return
  45 + cmd = line.split()
  46 + build = cmd[0]
  47 + force = (len(cmd) > 1 and cmd[1].lower() == 'force')
  48 + self.server.cache_lock.acquire()
  49 + results = self.server.build_cache.get(build, force)
  50 + self.server.cache_lock.release()
  51 + self.request.send(json.dumps(results) + '\n')
  52 +
  53 +
  54 +class BuildCacheClient(object):
  55 +
  56 + def __init__(self, host='127.0.0.1', port=DEFAULT_PORT):
  57 + self.host = host
  58 + self.port = port
  59 + self.sock = None
  60 +
  61 + def connect(self):
  62 + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
  63 + self.sock.connect((self.host, self.port))
  64 +
  65 + def close(self):
  66 + self.sock.close()
  67 + self.sock = None
  68 +
  69 + def get(self, url, force=False):
  70 + if not self.sock:
  71 + self.connect()
  72 + line = url
  73 + if force:
  74 + line += ' force'
  75 + self.sock.sendall(line + '\n')
  76 + buf = ''
  77 + while not '\n' in buf:
  78 + data = self.sock.recv(1024)
  79 + if not data:
  80 + print 'build server hung up!'
  81 + return None
  82 + buf += data
  83 + return json.loads(buf)
  84 +
2  phonetest.py
@@ -96,7 +96,7 @@ def base_device_path(self):
96 96 def profile_path(self):
97 97 return self.base_device_path + '/profile'
98 98
99   - def runjob(self, job):
  99 + def runjob(self, build_metadata, worker_subprocess):
100 100 raise NotImplementedError
101 101
102 102 def set_dm_debug(self, level):
12 tests/runtestsremote.py
@@ -28,7 +28,7 @@
28 28
29 29 class UnitTest(PhoneTest):
30 30
31   - def runjob(self, job, worker_subprocess):
  31 + def runjob(self, build_metadata, worker_subprocess):
32 32
33 33 self.logger.debug('runtestsremote.py runjob start')
34 34 self.set_status(msg='runtestsremote.py runjob start')
@@ -40,15 +40,15 @@ def runjob(self, job, worker_subprocess):
40 40 phone_ip_address = self.phone_cfg['ip']
41 41 device_port = self.phone_cfg['sutcmdport']
42 42
43   - cache_build_dir = os.path.abspath(job["cache_build_dir"])
  43 + cache_build_dir = os.path.abspath(build_metadata["cache_build_dir"])
44 44 symbols_path = os.path.join(cache_build_dir, 'symbols')
45 45 if not os.path.exists(symbols_path):
46 46 symbols_path = None
47 47
48   - androidprocname = job['androidprocname']
49   - revision = job['revision']
50   - buildid = job['buildid']
51   - tree = job['tree']
  48 + androidprocname = build_metadata['androidprocname']
  49 + revision = build_metadata['revision']
  50 + buildid = build_metadata['buildid']
  51 + tree = build_metadata['tree']
52 52
53 53 if self.logger.getEffectiveLevel() == logging.DEBUG:
54 54 for prop in self.phone_cfg:
78 tests/s1s2test.py
@@ -17,26 +17,28 @@
17 17
18 18 class S1S2Test(PhoneTest):
19 19
20   - def runjob(self, job, worker_subprocess):
  20 + def runjob(self, build_metadata, worker_subprocess):
21 21 # Read our config file which gives us our number of
22 22 # iterations and urls that we will be testing
23 23 for cache_enabled in (False, True):
24   - self.runtest(job, worker_subprocess, cache_enabled)
  24 + self.runtest(build_metadata, worker_subprocess, cache_enabled)
25 25
26   - def runtest(self, job, worker_subprocess, cache_enabled):
  26 + def runtest(self, build_metadata, worker_subprocess, cache_enabled):
27 27 # Read our config file which gives us our number of
28 28 # iterations and urls that we will be testing
29   - self.prepare_phone(job, cache_enabled)
  29 + self.prepare_phone(build_metadata, cache_enabled)
30 30
31   - intent = job['androidprocname'] + '/.App'
  31 + intent = build_metadata['androidprocname'] + '/.App'
32 32
33 33 # Initialize profile
34 34 self.logger.debug('initializing profile...')
35 35 self.run_fennec_with_profile(intent, self._initialize_url)
36   - if not self.wait_for_fennec(job):
  36 + if not self.wait_for_fennec(build_metadata):
37 37 self.logger.info('%s: Failed to initialize profile for build %s' %
38   - (self.phone_cfg['phoneid'], job['buildid']))
39   - self.set_status(msg='Failed to initialize profile for build %s' % job['buildid'])
  38 + (self.phone_cfg['phoneid'],
  39 + build_metadata['buildid']))
  40 + self.set_status(msg='Failed to initialize profile for build %s' %
  41 + build_metadata['buildid'])
40 42 return
41 43
42 44 for testnum,(testname,url) in enumerate(self._urls.iteritems(), 1):
@@ -70,9 +72,10 @@ def runtest(self, job, worker_subprocess, cache_enabled):
70 72
71 73 # Get results - do this now so we don't have as much to
72 74 # parse in logcat.
73   - throbberstart, throbberstop = self.analyze_logcat(job)
  75 + throbberstart, throbberstop = self.analyze_logcat(
  76 + build_metadata)
74 77
75   - self.wait_for_fennec(job)
  78 + self.wait_for_fennec(build_metadata)
76 79
77 80 # Get rid of the browser and session store files
78 81 self.logger.debug('removing sessionstore files')
@@ -86,17 +89,19 @@ def runtest(self, job, worker_subprocess, cache_enabled):
86 89
87 90 # Publish results
88 91 self.logger.debug('%s throbbers after %d attempts' %
89   - ('successfully got' if success else 'failed to get', attempt))
  92 + ('successfully got' if success else
  93 + 'failed to get', attempt))
90 94 if success:
91 95 self.logger.debug('publishing results')
92 96 self.publish_results(starttime=int(starttime),
93 97 tstrt=throbberstart,
94 98 tstop=throbberstop,
95   - job=job,
  99 + build_metadata=build_metadata,
96 100 testname=testname,
97 101 cache_enabled=cache_enabled)
98 102
99   - def wait_for_fennec(self, job, max_wait_time=60, wait_time=5, kill_wait_time=20):
  103 + def wait_for_fennec(self, build_metadata, max_wait_time=60, wait_time=5,
  104 + kill_wait_time=20):
100 105 # Wait for up to a max_wait_time seconds for fennec to close
101 106 # itself in response to the quitter request. Check that fennec
102 107 # is still running every wait_time seconds. If fennec doesn't
@@ -106,14 +111,14 @@ def wait_for_fennec(self, job, max_wait_time=60, wait_time=5, kill_wait_time=20)
106 111 # Re-raise the last exception if fennec can not be killed.
107 112 max_wait_attempts = max_wait_time / wait_time
108 113 for wait_attempt in range(max_wait_attempts):
109   - if not self.dm.processExist(job['androidprocname']):
  114 + if not self.dm.processExist(build_metadata['androidprocname']):
110 115 return True
111 116 sleep(wait_time)
112 117 self.logger.debug('killing fennec')
113 118 max_killattempts = 3
114 119 for kill_attempt in range(max_killattempts):
115 120 try:
116   - self.dm.killProcess(job['androidprocname'])
  121 + self.dm.killProcess(build_metadata['androidprocname'])
117 122 break
118 123 except DMError:
119 124 self.logger.info('Attempt %d to kill fennec failed' % kill_attempt)
@@ -122,9 +127,9 @@ def wait_for_fennec(self, job, max_wait_time=60, wait_time=5, kill_wait_time=20)
122 127 sleep(kill_wait_time)
123 128 return False
124 129
125   - def prepare_phone(self, job, cache_enabled):
  130 + def prepare_phone(self, build_metadata, cache_enabled):
126 131 telemetry_prompt = 999
127   - if job['blddate'] < '2013-01-03':
  132 + if build_metadata['blddate'] < '2013-01-03':
128 133 telemetry_prompt = 2
129 134 prefs = { 'browser.firstrun.show.localepicker': False,
130 135 'browser.sessionstore.resume_from_crash': False,
@@ -172,7 +177,7 @@ def prepare_phone(self, job, cache_enabled):
172 177 self._resulturl = cfg.get('settings', 'resulturl')
173 178 self._initialize_url = cfg.get('settings', 'initialize_url')
174 179
175   - def analyze_logcat(self, job):
  180 + def analyze_logcat(self, build_metadata):
176 181 self.logger.debug('analyzing logcat')
177 182 throbberstartRE = re.compile('.*Throbber start$')
178 183 throbberstopRE = re.compile('.*Throbber stop$')
@@ -189,7 +194,7 @@ def analyze_logcat(self, job):
189 194 fennec_still_running = True
190 195 while (fennec_still_running and
191 196 attempt < max_attempts and (throbstart == 0 or throbstop == 0)):
192   - if not self.dm.processExist(job['androidprocname']):
  197 + if not self.dm.processExist(build_metadata['androidprocname']):
193 198 fennec_still_running = False
194 199 buf = [x.strip() for x in self.dm.getLogcat()]
195 200 for line in buf:
@@ -213,14 +218,18 @@ def analyze_logcat(self, job):
213 218
214 219 return (int(throbstart), int(throbstop))
215 220
216   - def publish_results(self, starttime=0, tstrt=0, tstop=0, job=None, testname='', cache_enabled=True):
217   - msg = 'Start Time: %s Throbber Start: %s Throbber Stop: %s' % (starttime, tstrt, tstop)
  221 + def publish_results(self, starttime=0, tstrt=0, tstop=0,
  222 + build_metadata=None, testname='', cache_enabled=True):
  223 + msg = 'Start Time: %s Throbber Start: %s Throbber Stop: %s' % (
  224 + starttime, tstrt, tstop)
218 225 cache_msg = 'cached' if cache_enabled else 'not cached'
219   - print 'RESULTS (%s) %s %s:%s' % (cache_msg,
220   - self.phone_cfg['phoneid'],
221   - datetime.datetime.fromtimestamp(int(job['blddate'])),
222   - msg)
223   - self.logger.info('RESULTS (%s): %s:%s' % (cache_msg, self.phone_cfg['phoneid'], msg))
  226 + print 'RESULTS (%s) %s %s:%s' % (
  227 + cache_msg,
  228 + self.phone_cfg['phoneid'],
  229 + datetime.datetime.fromtimestamp(int(build_metadata['blddate'])),
  230 + msg)
  231 + self.logger.info('RESULTS (%s): %s:%s' %
  232 + (cache_msg, self.phone_cfg['phoneid'], msg))
224 233
225 234 # Create JSON to send to webserver
226 235 resultdata = {}
@@ -229,14 +238,14 @@ def publish_results(self, starttime=0, tstrt=0, tstop=0, job=None, testname='',
229 238 resultdata['starttime'] = starttime
230 239 resultdata['throbberstart'] = tstrt
231 240 resultdata['throbberstop'] = tstop
232   - resultdata['blddate'] = job['blddate']
  241 + resultdata['blddate'] = build_metadata['blddate']
233 242 resultdata['cached'] = cache_enabled
234 243
235   - resultdata['revision'] = job['revision']
236   - resultdata['productname'] = job['androidprocname']
237   - resultdata['productversion'] = job['version']
  244 + resultdata['revision'] = build_metadata['revision']
  245 + resultdata['productname'] = build_metadata['androidprocname']
  246 + resultdata['productversion'] = build_metadata['version']
238 247 resultdata['osver'] = self.phone_cfg['osver']
239   - resultdata['bldtype'] = job['bldtype']
  248 + resultdata['bldtype'] = build_metadata['bldtype']
240 249 resultdata['machineid'] = self.phone_cfg['machinetype']
241 250
242 251 # Upload
@@ -246,12 +255,7 @@ def publish_results(self, starttime=0, tstrt=0, tstop=0, job=None, testname='',
246 255 try:
247 256 f = urllib2.urlopen(req)
248 257 except urllib2.URLError, e:
249   - try:
250   - self.logger.error('Could not send results to server: %s' %
251   - e.reason.strerror)
252   - except:
253   - self.logger.error('Could not send results to server: %s' %
254   - e.reason)
  258 + self.logger.error('Could not send results to server: %s' % e)
255 259 else:
256 260 f.read()
257 261 f.close()
18 tests/smoketest.py
@@ -13,7 +13,7 @@
13 13
14 14 class SmokeTest(PhoneTest):
15 15
16   - def runjob(self, job, worker_subprocess):
  16 + def runjob(self, build_metadata, worker_subprocess):
17 17 try:
18 18 os.unlink('smoketest_pass')
19 19 except OSError:
@@ -25,9 +25,9 @@ def runjob(self, job, worker_subprocess):
25 25
26 26 # Read our config file which gives us our number of
27 27 # iterations and urls that we will be testing
28   - self.prepare_phone(job)
  28 + self.prepare_phone(build_metadata)
29 29
30   - intent = job['androidprocname'] + '/.App'
  30 + intent = build_metadata['androidprocname'] + '/.App'
31 31
32 32 # Clear logcat
33 33 self.dm.recordLogcat()
@@ -37,12 +37,12 @@ def runjob(self, job, worker_subprocess):
37 37 self.run_fennec_with_profile(intent, 'about:fennec')
38 38
39 39 self.logger.debug('analyzing logcat...')
40   - fennec_launched = self.analyze_logcat(job)
  40 + fennec_launched = self.analyze_logcat(build_metadata)
41 41 start = datetime.datetime.now()
42 42 while (not fennec_launched and (datetime.datetime.now() - start
43 43 <= datetime.timedelta(seconds=60))):
44 44 sleep(3)
45   - fennec_launched = self.analyze_logcat(job)
  45 + fennec_launched = self.analyze_logcat(build_metadata)
46 46
47 47 if fennec_launched:
48 48 self.logger.info('fennec successfully launched')
@@ -53,12 +53,12 @@ def runjob(self, job, worker_subprocess):
53 53
54 54 self.logger.debug('killing fennec')
55 55 # Get rid of the browser and session store files
56   - self.dm.killProcess(job['androidprocname'])
  56 + self.dm.killProcess(build_metadata['androidprocname'])
57 57
58 58 self.logger.debug('removing sessionstore files')
59 59 self.remove_sessionstore_files()
60 60
61   - def prepare_phone(self, job):
  61 + def prepare_phone(self, build_metadata):
62 62 prefs = { 'browser.firstrun.show.localepicker': False,
63 63 'browser.sessionstore.resume_from_crash': False,
64 64 'browser.firstrun.show.uidiscovery': False,
@@ -69,8 +69,8 @@ def prepare_phone(self, job):
69 69 'toolkit.telemetry.notifiedOptOut': 999 }
70 70 profile = FirefoxProfile(preferences=prefs)
71 71 self.install_profile(profile)
72   -
73   - def analyze_logcat(self, job):
  72 +
  73 + def analyze_logcat(self, build_metadata):
74 74 buf = self.dm.getLogcat()
75 75 got_start = False
76 76 got_end = False
80 worker.py
@@ -5,7 +5,6 @@
5 5 from __future__ import with_statement
6 6
7 7 import Queue
8   -import StringIO
9 8 import datetime
10 9 import logging
11 10 import multiprocessing
@@ -17,6 +16,7 @@
17 16 import time
18 17 import traceback
19 18
  19 +import buildserver
20 20 import phonetest
21 21 from mozdevice import DeviceManagerSUT, DMError
22 22
@@ -45,7 +45,8 @@ class PhoneWorker(object):
45 45 process."""
46 46
47 47 def __init__(self, worker_num, ipaddr, tests, phone_cfg, user_cfg,
48   - autophone_queue, logfile_prefix, loglevel, mailer):
  48 + autophone_queue, logfile_prefix, loglevel, mailer,
  49 + build_cache_port):
49 50 self.phone_cfg = phone_cfg
50 51 self.user_cfg = user_cfg
51 52 self.worker_num = worker_num
@@ -54,13 +55,14 @@ def __init__(self, worker_num, ipaddr, tests, phone_cfg, user_cfg,
54 55 self.first_status_of_type = None
55 56 self.last_status_of_previous_type = None
56 57 self.crashes = Crashes()
57   - self.job_queue = multiprocessing.Queue()
  58 + self.cmd_queue = multiprocessing.Queue()
58 59 self.lock = multiprocessing.Lock()
59 60 self.subprocess = PhoneWorkerSubProcess(self.worker_num, self.ipaddr,
60 61 tests, phone_cfg, user_cfg,
61 62 autophone_queue,
62   - self.job_queue, logfile_prefix,
63   - loglevel, mailer)
  63 + self.cmd_queue, logfile_prefix,
  64 + loglevel, mailer,
  65 + build_cache_port)
64 66
65 67 def is_alive(self):
66 68 return self.subprocess.is_alive()
@@ -71,17 +73,17 @@ def start(self, status=phonetest.PhoneTestMessage.IDLE):
71 73 def stop(self):
72 74 self.subprocess.stop()
73 75
74   - def add_job(self, job):
75   - self.job_queue.put_nowait(('job', job))
  76 + def new_build(self, build_url):
  77 + self.cmd_queue.put_nowait(('build', build_url))
76 78
77 79 def reboot(self):
78   - self.job_queue.put_nowait(('reboot', None))
  80 + self.cmd_queue.put_nowait(('reboot', None))
79 81
80 82 def disable(self):
81   - self.job_queue.put_nowait(('disable', None))
  83 + self.cmd_queue.put_nowait(('disable', None))
82 84
83 85 def enable(self):
84   - self.job_queue.put_nowait(('enable', None))
  86 + self.cmd_queue.put_nowait(('enable', None))
85 87
86 88 def debug(self, level):
87 89 try:
@@ -90,10 +92,10 @@ def debug(self, level):
90 92 logging.error('Invalid argument for debug: %s' % level)
91 93 else:
92 94 self.user_cfg['debug'] = level
93   - self.job_queue.put_nowait(('debug', level))
  95 + self.cmd_queue.put_nowait(('debug', level))
94 96
95 97 def ping(self):
96   - self.job_queue.put_nowait(('ping', None))
  98 + self.cmd_queue.put_nowait(('ping', None))
97 99
98 100 def process_msg(self, msg):
99 101 """These are status messages routed back from the autophone_queue
@@ -119,21 +121,23 @@ class PhoneWorkerSubProcess(object):
119 121 MAX_REBOOT_WAIT_SECONDS = 300
120 122 MAX_REBOOT_ATTEMPTS = 3
121 123 PING_SECONDS = 60*15
122   - JOB_QUEUE_TIMEOUT_SECONDS = 10
  124 + CMD_QUEUE_TIMEOUT_SECONDS = 10
123 125
124 126 def __init__(self, worker_num, ipaddr, tests, phone_cfg, user_cfg,
125   - autophone_queue, job_queue, logfile_prefix, loglevel, mailer):
  127 + autophone_queue, cmd_queue, logfile_prefix, loglevel, mailer,
  128 + build_cache_port):
126 129 self.worker_num = worker_num
127 130 self.ipaddr = ipaddr
128 131 self.tests = tests
129 132 self.phone_cfg = phone_cfg
130 133 self.user_cfg = user_cfg
131 134 self.autophone_queue = autophone_queue
132   - self.job_queue = job_queue
  135 + self.cmd_queue = cmd_queue
133 136 self.logfile = logfile_prefix + '.log'
134 137 self.outfile = logfile_prefix + '.out'
135 138 self.loglevel = loglevel
136 139 self.mailer = mailer
  140 + self.build_cache_port = build_cache_port
137 141 self.p = None
138 142 self.skipped_job_queue = []
139 143 self.current_build = None
@@ -171,8 +175,8 @@ def start(self, status):
171 175 def stop(self):
172 176 """Call from main process."""
173 177 if self.is_alive():
174   - self.job_queue.put_nowait(('stop', None))
175   - self.p.join(self.JOB_QUEUE_TIMEOUT_SECONDS*2)
  178 + self.cmd_queue.put_nowait(('stop', None))
  179 + self.p.join(self.CMD_QUEUE_TIMEOUT_SECONDS*2)
176 180
177 181 def has_error(self):
178 182 return (self.status == phonetest.PhoneTestMessage.DISABLED or
@@ -334,7 +338,7 @@ def ping(self):
334 338 logging.error('Got empty device root!')
335 339 return False
336 340
337   - def do_job(self, job):
  341 + def run_tests(self, build_metadata):
338 342 if not self.has_error():
339 343 logging.info('Rebooting...')
340 344 self.reboot()
@@ -351,14 +355,15 @@ def do_job(self, job):
351 355 self.status_update(phonetest.PhoneTestMessage(
352 356 self.phone_cfg['phoneid'],
353 357 phonetest.PhoneTestMessage.INSTALLING,
354   - job['blddate']))
355   - logging.info('Installing build %s.' %
356   - datetime.datetime.fromtimestamp(float(job['blddate'])))
  358 + build_metadata['blddate']))
  359 + logging.info(
  360 + 'Installing build %s.' % datetime.datetime.fromtimestamp(
  361 + float(build_metadata['blddate'])))
357 362
358 363 try:
359 364 pathOnDevice = posixpath.join(self.dm.getDeviceRoot(),
360 365 'build.apk')
361   - self.dm.pushFile(os.path.join(job['cache_build_dir'],
  366 + self.dm.pushFile(os.path.join(build_metadata['cache_build_dir'],
362 367 'build.apk'), pathOnDevice)
363 368 self.dm.installApp(pathOnDevice)
364 369 self.dm.removeFile(pathOnDevice)
@@ -367,18 +372,18 @@ def do_job(self, job):
367 372 logging.error(exc)
368 373 self.phone_disconnected(exc)
369 374 return False
370   - self.current_build = job['blddate']
  375 + self.current_build = build_metadata['blddate']
371 376
372 377 logging.info('Running tests...')
373 378 for t in self.tests:
374 379 if self.has_error():
375 380 break
376   - t.current_build = job['blddate']
  381 + t.current_build = build_metadata['blddate']
377 382 # TODO: Attempt to see if pausing between jobs helps with
378 383 # our reconnection issues
379 384 time.sleep(30)
380 385 try:
381   - t.runjob(job, self)
  386 + t.runjob(build_metadata, self)
382 387 except DMError:
383 388 exc = 'Uncaught device error while running test!\n\n%s' % \
384 389 traceback.format_exc()
@@ -423,8 +428,8 @@ def loop(self):
423 428 while True:
424 429 request = None
425 430 try:
426   - request = self.job_queue.get(
427   - timeout=self.JOB_QUEUE_TIMEOUT_SECONDS)
  431 + request = self.cmd_queue.get(
  432 + timeout=self.CMD_QUEUE_TIMEOUT_SECONDS)
428 433 except Queue.Empty:
429 434 if (self.status != phonetest.PhoneTestMessage.DISABLED and
430 435 (not last_ping or
@@ -453,12 +458,19 @@ def loop(self):
453 458 continue
454 459 if request[0] == 'stop':
455 460 return
456   - if request[0] == 'job':
457   - job = request[1]
458   - if not job:
  461 + if request[0] == 'build':
  462 + build_url = request[1]
  463 + logging.info('Got notification of build %s.' % build_url)
  464 + client = buildserver.BuildCacheClient(
  465 + port=self.build_cache_port)
  466 + logging.info('Fetching build...')
  467 + cache_response = client.get(build_url)
  468 + client.close()
  469 + if not cache_response['success']:
  470 + logging.warn('Errors occured getting build %s: %s' %
  471 + (build_url, cache_response['error']))
459 472 continue
460   - logging.info('Got job.')
461   - if self.do_job(job):
  473 + if self.run_tests(cache_response['metadata']):
462 474 logging.info('Job completed.')
463 475 self.status_update(phonetest.PhoneTestMessage(
464 476 self.phone_cfg['phoneid'],
@@ -466,7 +478,7 @@ def loop(self):
466 478 self.current_build))
467 479 else:
468 480 logging.error('Job failed; queuing it for later.')
469   - self.skipped_job_queue.append(job)
  481 + self.skipped_job_queue.append(cache_response['metadata'])
470 482 elif request[0] == 'reboot':
471 483 logging.info('Rebooting at user\'s request...')
472 484 self.reboot()
@@ -481,7 +493,7 @@ def loop(self):
481 493 self.current_build))
482 494 last_ping = None
483 495 for j in self.skipped_job_queue:
484   - self.job_queue.put(('job', j))
  496 + self.cmd_queue.put(('job', j))
485 497 elif request[0] == 'debug':
486 498 self.user_cfg['debug'] = request[1]
487 499 DeviceManagerSUT.debug = self.user_cfg['debug']

0 comments on commit 88f839b

Please sign in to comment.
Something went wrong with that request. Please try again.