Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Newer
Older
100644 1402 lines (1281 sloc) 62.18 kB
91b2c03 @markrcote Initial commit.
markrcote authored
1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
3 # You can obtain one at http://mozilla.org/MPL/2.0/.
4
6d57554 @bclary Bug 968905 - Autophone - parameterize control constants via config fi…
bclary authored
5 import ConfigParser
91b2c03 @markrcote Initial commit.
markrcote authored
6 import Queue
7 import SocketServer
3ae1b09 @bclary Bug 1155885 - Autophone - should automatically recover when devices a…
bclary authored
8 import datetime
91b2c03 @markrcote Initial commit.
markrcote authored
9 import errno
10 import inspect
5d020e1 @bclary Bug 1037077 - Report run status to treeherder, r=mcote.
bclary authored
11 import json
91b2c03 @markrcote Initial commit.
markrcote authored
12 import logging
1102e03 @bclary bug 853961 - revamp logging, r=mcote.
bclary authored
13 import logging.handlers
91b2c03 @markrcote Initial commit.
markrcote authored
14 import multiprocessing
15 import os
fadc14c @bclary Bug 1118758 - Autophone - add support for job specific tests, r=mcote.
bclary authored
16 import re
eee9724 @markrcote Smoke test, refactored builds.py.
markrcote authored
17 import signal
91b2c03 @markrcote Initial commit.
markrcote authored
18 import socket
3ae1b09 @bclary Bug 1155885 - Autophone - should automatically recover when devices a…
bclary authored
19 import subprocess
91b2c03 @markrcote Initial commit.
markrcote authored
20 import sys
21 import threading
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
22 import traceback
91b2c03 @markrcote Initial commit.
markrcote authored
23
24 from manifestparser import TestManifest
d82c92e @markrcote Fixed retry loop; reboot is tried 3 times, and each test tried twice.…
markrcote authored
25
c433f56 @markrcote Bug 776737 Replaced all adb usage with DeviceManagerSUT. r=wlach
markrcote authored
26 import builds
88f839b @markrcote Bug 849251 - Decouple build cache as TCP server in a separate thread.…
markrcote authored
27 import buildserver
63933d5 @markrcote Bug 795417 - Persistent job queues. r=bc
markrcote authored
28 import jobs
0bd108b @bclary Bug 1133580 - Autophone - hookup new pulsemonitor, consolidate compat…
bclary authored
29 import utils
30
3ae1b09 @bclary Bug 1155885 - Autophone - should automatically recover when devices a…
bclary authored
31 from adb import ADBHost
8292c84 @bclary Bug 1063886 - Autophone - paydown technical debt, r=mcote.
bclary authored
32 from adb_android import ADBAndroid as ADBDevice
0bd108b @bclary Bug 1133580 - Autophone - hookup new pulsemonitor, consolidate compat…
bclary authored
33 from autophonepulsemonitor import AutophonePulseMonitor
34 from autophonetreeherder import AutophoneTreeherder
10770b5 @markrcote Added unit tests.
markrcote authored
35 from mailer import Mailer
8292c84 @bclary Bug 1063886 - Autophone - paydown technical debt, r=mcote.
bclary authored
36 from options import AutophoneOptions
37 from phonestatus import PhoneStatus
38 from phonetest import PhoneTest
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
39 from process_states import ProcessStates
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
40 from sensitivedatafilter import SensitiveDataFilter
6e8806f @bclary Bug 1153992 - Autophone - allow test manifest to include multiple sec…
bclary authored
41 from worker import PhoneWorker
8292c84 @bclary Bug 1063886 - Autophone - paydown technical debt, r=mcote.
bclary authored
42
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
43 logger = None
44 console_logger = None
45
8292c84 @bclary Bug 1063886 - Autophone - paydown technical debt, r=mcote.
bclary authored
46 class PhoneData(object):
6098a69 @bclary Bug 1072423 - Autophone - support split APKs for 2.3/later, r=mcote.
bclary authored
47 def __init__(self, phoneid, serial, machinetype, osver, abi, sdk, ipaddr):
8292c84 @bclary Bug 1063886 - Autophone - paydown technical debt, r=mcote.
bclary authored
48 self.id = phoneid
49 self.serial = serial
50 self.machinetype = machinetype
51 self.osver = osver
52 self.abi = abi
6098a69 @bclary Bug 1072423 - Autophone - support split APKs for 2.3/later, r=mcote.
bclary authored
53 self.sdk = sdk
8292c84 @bclary Bug 1063886 - Autophone - paydown technical debt, r=mcote.
bclary authored
54 self.host_ip = ipaddr
55
5d020e1 @bclary Bug 1037077 - Report run status to treeherder, r=mcote.
bclary authored
56 @property
57 def architecture(self):
b8f2f26 @bclary Bug 1079923 - Autophone - update unittests to fix bitrot, r=mcote.
bclary authored
58 abi = self.abi
59 if 'armeabi-v7a' in abi:
60 abi = 'armv7'
61 return abi
5d020e1 @bclary Bug 1037077 - Report run status to treeherder, r=mcote.
bclary authored
62
63 @property
64 def os(self):
65 return 'android-%s' % '-'.join(self.osver.split('.')[:2])
66
b8f2f26 @bclary Bug 1079923 - Autophone - update unittests to fix bitrot, r=mcote.
bclary authored
67 @property
68 def platform(self):
69 if self.architecture == 'x86':
70 return '%s-x86' % self.os
71 return '%s-%s-%s' % (self.os,
72 self.architecture,
73 ''.join(self.sdk.split('-')))
74
8292c84 @bclary Bug 1063886 - Autophone - paydown technical debt, r=mcote.
bclary authored
75 def __str__(self):
76 return '%s' % self.__dict__
91b2c03 @markrcote Initial commit.
markrcote authored
77
78
8bddba3 @bclary Bug 990601 - Autophone - use adb instead of tcp/ip to control devices…
bclary authored
79 class AutoPhone(object):
14cdbd7 @bclary Bug 969518 - Autophone - allow use of usb networking on Linux, r=mcote.
bclary authored
80
524b0db @markrcote Bug 790380 - Handle worker crashes and misc. other improvements and f…
markrcote authored
81 class CmdTCPServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer):
1b4ba68 @markrcote Moved TCP classes into AutoPhone. Implemented status command.
markrcote authored
82
83 allow_reuse_address = True
84 daemon_threads = True
85 cmd_cb = None
86
87 class CmdTCPHandler(SocketServer.BaseRequestHandler):
88 def handle(self):
89 buffer = ''
90 self.request.send('Hello? Yes this is Autophone.\n')
91 while True:
92 try:
93 data = self.request.recv(1024)
94 except socket.error, e:
95 if e.errno == errno.ECONNRESET:
96 break
5f92265 @markrcote Added build cache. Fixed infinite loop in tcp request handler. Simpli…
markrcote authored
97 raise e
1b4ba68 @markrcote Moved TCP classes into AutoPhone. Implemented status command.
markrcote authored
98 if not data:
99 break
100 buffer += data
101 while buffer:
102 line, nl, rest = buffer.partition('\n')
103 if not nl:
104 break
105 buffer = rest
106 line = line.strip()
107 if not line:
108 continue
109 if line == 'quit' or line == 'exit':
5f92265 @markrcote Added build cache. Fixed infinite loop in tcp request handler. Simpli…
markrcote authored
110 return
1b4ba68 @markrcote Moved TCP classes into AutoPhone. Implemented status command.
markrcote authored
111 response = self.server.cmd_cb(line)
112 self.request.send(response + '\n')
113
6d57554 @bclary Bug 968905 - Autophone - parameterize control constants via config fi…
bclary authored
114 def __init__(self, loglevel, options):
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
115 self.state = ProcessStates.STARTING
3ae1b09 @bclary Bug 1155885 - Autophone - should automatically recover when devices a…
bclary authored
116 self.unrecoverable_error = False
6d57554 @bclary Bug 968905 - Autophone - parameterize control constants via config fi…
bclary authored
117 self.options = options
2e980e1 @markrcote Log adb-not-in-path warning only once per process. Workers log to the…
markrcote authored
118 self.loglevel = loglevel
8292c84 @bclary Bug 1063886 - Autophone - paydown technical debt, r=mcote.
bclary authored
119 self.mailer = Mailer(options.emailcfg, '[autophone] ')
88f839b @markrcote Bug 849251 - Decouple build cache as TCP server in a separate thread.…
markrcote authored
120
524b0db @markrcote Bug 790380 - Handle worker crashes and misc. other improvements and f…
markrcote authored
121 self._next_worker_num = 0
f33a3cc @bclary Bug 1165403 - Autophone - allow duplicate jobs, r=gbrown.
bclary authored
122 self.jobs = jobs.Jobs(self.mailer,
123 allow_duplicates=options.allow_duplicate_jobs)
0bd108b @bclary Bug 1133580 - Autophone - hookup new pulsemonitor, consolidate compat…
bclary authored
124 self.phone_workers = {} # indexed by phone id
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
125 self.lock = threading.RLock()
279303d @bclary Bug 1161631 - Autophone - Use a shared process lock to prevent Autoph…
bclary authored
126 self.shared_lock = multiprocessing.Lock()
91b2c03 @markrcote Initial commit.
markrcote authored
127 self._tests = []
7358c98 @bclary Bug 1161631 - Autophone - part 6 - reuse ADBDevice instances, r=gbrown.
bclary authored
128 self._devices = {} # dict indexed by device names found in devices ini file
8bddba3 @bclary Bug 990601 - Autophone - use adb instead of tcp/ip to control devices…
bclary authored
129 self.server = None
130 self.server_thread = None
0bd108b @bclary Bug 1133580 - Autophone - hookup new pulsemonitor, consolidate compat…
bclary authored
131 self.pulse_monitor = None
8bddba3 @bclary Bug 990601 - Autophone - use adb instead of tcp/ip to control devices…
bclary authored
132 self.restart_workers = {}
0bd108b @bclary Bug 1133580 - Autophone - hookup new pulsemonitor, consolidate compat…
bclary authored
133 self.treeherder = AutophoneTreeherder(None,
279303d @bclary Bug 1161631 - Autophone - Use a shared process lock to prevent Autoph…
bclary authored
134 self.options,
135 shared_lock=self.shared_lock)
8bddba3 @bclary Bug 990601 - Autophone - use adb instead of tcp/ip to control devices…
bclary authored
136
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
137 console_logger.info('Starting autophone.')
524b0db @markrcote Bug 790380 - Handle worker crashes and misc. other improvements and f…
markrcote authored
138
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
139 # Queue for listening to status updates from
140 # PhoneWorkerSubProcess workers.
141 self.queue = multiprocessing.Queue()
91b2c03 @markrcote Initial commit.
markrcote authored
142
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
143 console_logger.info('Loading tests.')
91b2c03 @markrcote Initial commit.
markrcote authored
144 self.read_tests()
92416ce @bclary Bug 1007220 - Add webapp startup test, r=mcote.
bclary authored
145
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
146 console_logger.info('Initializing devices.')
92416ce @bclary Bug 1007220 - Add webapp startup test, r=mcote.
bclary authored
147
8bddba3 @bclary Bug 990601 - Autophone - use adb instead of tcp/ip to control devices…
bclary authored
148 self.read_devices()
91b2c03 @markrcote Initial commit.
markrcote authored
149
279303d @bclary Bug 1161631 - Autophone - Use a shared process lock to prevent Autoph…
bclary authored
150 self.state = ProcessStates.RUNNING
151 for worker in self.phone_workers.values():
152 worker.start()
153
154 # We must wait to start the pulse monitor until after the
155 # workers have started in order to make certain that the
156 # shared_lock is passed to the worker subprocesses in an
157 # unlocked state.
8292c84 @bclary Bug 1063886 - Autophone - paydown technical debt, r=mcote.
bclary authored
158 if options.enable_pulse:
0bd108b @bclary Bug 1133580 - Autophone - hookup new pulsemonitor, consolidate compat…
bclary authored
159 self.pulse_monitor = AutophonePulseMonitor(
160 userid=options.pulse_user,
161 password=options.pulse_password,
162 jobaction_exchange_name=options.pulse_jobactions_exchange,
91b0bca @bclary Bug 1118758 - Autophone - use AutophonePulseBuildMonitor, r=mcote.
bclary authored
163 build_callback=self.on_build,
0bd108b @bclary Bug 1133580 - Autophone - hookup new pulsemonitor, consolidate compat…
bclary authored
164 jobaction_callback=self.on_jobaction,
165 treeherder_url=self.options.treeherder_url,
91b0bca @bclary Bug 1118758 - Autophone - use AutophonePulseBuildMonitor, r=mcote.
bclary authored
166 trees=options.repos,
167 platforms=['android',
168 'android-api-9',
169 'android-api-10',
170 'android-api-11',
171 'android-x86'],
172 buildtypes=options.buildtypes,
279303d @bclary Bug 1161631 - Autophone - Use a shared process lock to prevent Autoph…
bclary authored
173 durable_queues=self.options.pulse_durable_queue,
174 shared_lock=self.shared_lock,
175 verbose=options.verbose)
0bd108b @bclary Bug 1133580 - Autophone - hookup new pulsemonitor, consolidate compat…
bclary authored
176 self.pulse_monitor.start()
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
177
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
178 logger.debug('autophone_options: %s' % self.options)
6d57554 @bclary Bug 968905 - Autophone - parameterize control constants via config fi…
bclary authored
179
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
180 console_logger.info('Autophone started.')
3ae1b09 @bclary Bug 1155885 - Autophone - should automatically recover when devices a…
bclary authored
181 if self.options.reboot_on_error:
182 msg_subj = 'Starting'
183 msg_body = ('Hello, this is Autophone. '
184 'Just to let you know, I have started running. '
185 'Wish me luck and check on me from time to time. '
186 'I will send you emails if I have any problems.\n\n')
187 self.mailer.send(msg_subj, msg_body)
92416ce @bclary Bug 1007220 - Add webapp startup test, r=mcote.
bclary authored
188
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
189 def _get_frames(self):
190 """Return the stack to the current location"""
191 frames = traceback.format_list(traceback.extract_stack())
192 return ''.join(frames[:-2])
193
194 def lock_acquire(self, data=None):
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
195 if logger.getEffectiveLevel() == logging.DEBUG:
196 if self.options.verbose:
197 logger.debug('lock_acquire: %s\n%s' % (data, self._get_frames()))
198 else:
199 logger.debug('lock_acquire: %s' % data)
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
200 self.lock.acquire()
201
202 def lock_release(self, data=None):
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
203 if logger.getEffectiveLevel() == logging.DEBUG:
204 if self.options.verbose:
205 logger.debug('lock_release: %s\n%s' % (data, self._get_frames()))
206 else:
207 logger.debug('lock_release: %s' % data)
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
208 self.lock.release()
209
524b0db @markrcote Bug 790380 - Handle worker crashes and misc. other improvements and f…
markrcote authored
210 @property
211 def next_worker_num(self):
212 n = self._next_worker_num
213 self._next_worker_num += 1
214 return n
215
91b2c03 @markrcote Initial commit.
markrcote authored
216 def run(self):
8292c84 @bclary Bug 1063886 - Autophone - paydown technical debt, r=mcote.
bclary authored
217 self.server = self.CmdTCPServer(('0.0.0.0', self.options.port),
1b4ba68 @markrcote Moved TCP classes into AutoPhone. Implemented status command.
markrcote authored
218 self.CmdTCPHandler)
91b2c03 @markrcote Initial commit.
markrcote authored
219 self.server.cmd_cb = self.route_cmd
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
220 self.server_thread = threading.Thread(target=self.server.serve_forever,
221 name='CmdTCPThread')
88f839b @markrcote Bug 849251 - Decouple build cache as TCP server in a separate thread.…
markrcote authored
222 self.server_thread.daemon = True
91b2c03 @markrcote Initial commit.
markrcote authored
223 self.server_thread.start()
224 self.worker_msg_loop()
225
524b0db @markrcote Bug 790380 - Handle worker crashes and misc. other improvements and f…
markrcote authored
226 def check_for_dead_workers(self):
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
227 if self.state != ProcessStates.RUNNING:
228 return
a0bb699 @bclary Bug 1161631 - Autophone - part 3 -implement help and device commands,…
bclary authored
229 workers = self.phone_workers.values()
230 for worker in workers:
9860a54 @markrcote Bug 797371 - Update phone info and restart worker if reg msg changes.…
markrcote authored
231 if not worker.is_alive():
3ae1b09 @bclary Bug 1155885 - Autophone - should automatically recover when devices a…
bclary authored
232 phoneid = worker.phone.id
233 logger.debug('Worker %s %s is not alive' % (phoneid, worker.state))
234 if phoneid in self.restart_workers:
a0bb699 @bclary Bug 1161631 - Autophone - part 3 -implement help and device commands,…
bclary authored
235 initial_state = PhoneStatus.IDLE
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
236 logger.info('Worker %s exited; restarting with new '
3ae1b09 @bclary Bug 1155885 - Autophone - should automatically recover when devices a…
bclary authored
237 'values.' % phoneid)
a0bb699 @bclary Bug 1161631 - Autophone - part 3 -implement help and device commands,…
bclary authored
238 elif worker.state == ProcessStates.STOPPING:
86c921e @bclary Bug 1161631 - Autophone - part 4 -implement autophone-restart, autoph…
bclary authored
239 # The device will be removed and not restarted.
240 initial_state = None
241 elif worker.state == ProcessStates.RESTARTING:
a0bb699 @bclary Bug 1161631 - Autophone - part 3 -implement help and device commands,…
bclary authored
242 initial_state = PhoneStatus.IDLE
2b6d54e @markrcote Bug 795195 - Added DISCONNECTED state; updated for DM exceptions chan…
markrcote authored
243 else:
3ae1b09 @bclary Bug 1155885 - Autophone - should automatically recover when devices a…
bclary authored
244 console_logger.error('Worker %s died!' % phoneid)
245 msg_subj = 'Worker for phone %s died' % phoneid
246 msg_body = ('Hello, this is Autophone. '
247 'Just to let you know, '
248 'the worker process '
249 'for phone %s died.\n' %
250 phoneid)
a0bb699 @bclary Bug 1161631 - Autophone - part 3 -implement help and device commands,…
bclary authored
251 if worker.crashes.too_many_crashes():
252 initial_state = PhoneStatus.DISABLED
253 msg_subj += ' and was disabled'
3ae1b09 @bclary Bug 1155885 - Autophone - should automatically recover when devices a…
bclary authored
254 msg_body += (
255 'It looks really crashy, so I disabled it. '
256 'Sorry about that.\n\n')
a0bb699 @bclary Bug 1161631 - Autophone - part 3 -implement help and device commands,…
bclary authored
257 else:
258 initial_state = PhoneStatus.DISCONNECTED
259 logger.info('Sending notification...')
260 self.mailer.send(msg_subj, msg_body)
261
262 # Have to remove the tests for the worker prior to
86c921e @bclary Bug 1161631 - Autophone - part 4 -implement autophone-restart, autoph…
bclary authored
263 # removing or recreating it in order to remove it from
264 # the PhoneTest.instances.
a0bb699 @bclary Bug 1161631 - Autophone - part 3 -implement help and device commands,…
bclary authored
265 while worker.tests:
266 t = worker.tests.pop()
267 t.remove()
268
269 # Do we need to worry about a race between the pulse
270 # monitor locking the shared lock?
271
86c921e @bclary Bug 1161631 - Autophone - part 4 -implement autophone-restart, autoph…
bclary authored
272 if worker.state == ProcessStates.STOPPING:
3ae1b09 @bclary Bug 1155885 - Autophone - should automatically recover when devices a…
bclary authored
273 console_logger.info('Worker %s stopped' % phoneid)
274 del self.phone_workers[phoneid]
86c921e @bclary Bug 1161631 - Autophone - part 4 -implement autophone-restart, autoph…
bclary authored
275 else:
276 if worker.state == ProcessStates.RESTARTING:
277 # The device is being restarted with a
278 # potentially changed test manifest and
279 # changed test configurations. The changes to
280 # the test configuration files will be
281 # automatically picked up when the tests are
282 # recreated for the worker, but we must
283 # reparse the test manifest in order for the
284 # worker to pick up test manifest changes. We
285 # re-read the tests here, to update
286 # self._tests which will be incorporated into
287 # the new worker instance. If a worker dies
288 # and is restarted, it will automatically pick
289 # up these changes as well.
290 self.read_tests()
291
292 # We can not re-use the original worker instance
293 # since we need to recreate the
294 # multiprocessing.Process object before we can
295 # call start on it again.
3ae1b09 @bclary Bug 1155885 - Autophone - should automatically recover when devices a…
bclary authored
296 console_logger.info('Worker %s restarting' % phoneid)
86c921e @bclary Bug 1161631 - Autophone - part 4 -implement autophone-restart, autoph…
bclary authored
297 # Save the record of crashes before recreating the
298 # Worker, then restore it afterwards.
299 crashes = worker.crashes
3ae1b09 @bclary Bug 1155885 - Autophone - should automatically recover when devices a…
bclary authored
300 try:
301 new_worker = self.create_worker(worker.phone)
302 new_worker.crashes = crashes
303 new_worker.start(initial_state)
304 except Exception, e:
305 console_logger.info('Worker %s failed to restart' %
306 phoneid)
307 msg_subj = ('Worker for phone %s failed to restart' %
308 phoneid)
309 msg_body = ('Hello, this is Autophone. '
310 'Just to let you know, '
311 'the worker process '
312 'for phone %s '
313 'failed to restart due to %s.\n' %
314 (phoneid, e))
315 self.mailer.send(msg_subj, msg_body)
316 self.purge_worker(phoneid)
317
318 def check_for_unrecoverable_errors(self):
319 """Set the property unrecoverable_error to True if any devices have
320 lost usb connectivity or not updated their status within the
321 maximum allowed heartbeat time period. Forcefully stop any
322 workers which have exceeded the maximum heartbeat time.
323 """
324 for worker in self.phone_workers.values():
325 if not worker.last_status_msg:
326 continue
327
328 if worker.last_status_msg.phone_status == PhoneStatus.DISCONNECTED:
329 self.unrecoverable_error = True
330
331 # Do not check the last timestamp of a worker that
332 # is currently downloading a build due to the size
333 # of the downloads and the unknown network speed.
334 elapsed = datetime.datetime.now() - worker.last_status_msg.timestamp
335 if (worker.last_status_msg.phone_status != PhoneStatus.FETCHING and
336 elapsed > datetime.timedelta(seconds=self.options.maximum_heartbeat)):
337 self.unrecoverable_error = True
338 worker.stop()
524b0db @markrcote Bug 790380 - Handle worker crashes and misc. other improvements and f…
markrcote authored
339
91b2c03 @markrcote Initial commit.
markrcote authored
340 def worker_msg_loop(self):
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
341 self.lock_acquire()
91b2c03 @markrcote Initial commit.
markrcote authored
342 try:
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
343 while self.phone_workers and self.state != ProcessStates.STOPPING:
3ae1b09 @bclary Bug 1155885 - Autophone - should automatically recover when devices a…
bclary authored
344 if self.options.reboot_on_error:
345 self.check_for_unrecoverable_errors()
346 if (self.unrecoverable_error and
347 self.state != ProcessStates.SHUTTINGDOWN):
348 self.shutdown()
524b0db @markrcote Bug 790380 - Handle worker crashes and misc. other improvements and f…
markrcote authored
349 self.check_for_dead_workers()
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
350 if (self.state == ProcessStates.RUNNING and
351 self.pulse_monitor and not self.pulse_monitor.is_alive()):
0bd108b @bclary Bug 1133580 - Autophone - hookup new pulsemonitor, consolidate compat…
bclary authored
352 self.pulse_monitor.start()
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
353 # Temporarily release the lock while we are waiting
354 # for a message from the workers.
355 self.lock_release()
91b2c03 @markrcote Initial commit.
markrcote authored
356 try:
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
357 msg = self.queue.get(timeout=5)
91b2c03 @markrcote Initial commit.
markrcote authored
358 except Queue.Empty:
359 continue
eee9724 @markrcote Smoke test, refactored builds.py.
markrcote authored
360 except IOError, e:
361 if e.errno == errno.EINTR:
362 continue
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
363 finally:
364 # Reacquire the lock.
365 self.lock_acquire()
3ae1b09 @bclary Bug 1155885 - Autophone - should automatically recover when devices a…
bclary authored
366 if msg.phone.id not in self.phone_workers:
367 logger.warning('Received message %s '
368 'from Non-existent worker' % msg)
369 continue
8292c84 @bclary Bug 1063886 - Autophone - paydown technical debt, r=mcote.
bclary authored
370 self.phone_workers[msg.phone.id].process_msg(msg)
a0bb699 @bclary Bug 1161631 - Autophone - part 3 -implement help and device commands,…
bclary authored
371 if msg.phone_status == PhoneStatus.SHUTDOWN:
372 # Have to remove the tests for the worker prior to
373 # removing it in order to remove it from the
374 # PhoneTest.instances so that it will not appear
375 # in future PhoneTest.match results.
376 worker = self.phone_workers[msg.phone.id]
377 while worker.tests:
378 t = worker.tests.pop()
379 t.remove()
86c921e @bclary Bug 1161631 - Autophone - part 4 -implement autophone-restart, autoph…
bclary authored
380 if worker.state == ProcessStates.SHUTTINGDOWN:
381 # We are completely shutting down the device
382 # so we delete it from the phone_workers
383 # dictionary. Otherwise, the phone will be
384 # detected as dead and will be restarted.
385 del self.phone_workers[msg.phone.id]
386 console_logger.info('Worker %s shutdown' % msg.phone.id)
91b2c03 @markrcote Initial commit.
markrcote authored
387 except KeyboardInterrupt:
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
388 pass
389 finally:
390 if self.pulse_monitor:
391 self.pulse_monitor.stop()
392 self.pulse_monitor = None
393 if self.server:
394 self.server.shutdown()
395 if self.server_thread:
396 self.server_thread.join()
a0bb699 @bclary Bug 1161631 - Autophone - part 3 -implement help and device commands,…
bclary authored
397 for p in self.phone_workers.values():
398 p.stop()
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
399 self.lock_release()
91b2c03 @markrcote Initial commit.
markrcote authored
400
3ae1b09 @bclary Bug 1155885 - Autophone - should automatically recover when devices a…
bclary authored
401 if self.unrecoverable_error and self.options.reboot_on_error:
402 console_logger.info('Rebooting due to unrecoverable errors')
403 msg_subj = 'Rebooting host due to unrecoverable errors'
404 msg_body = ('Hello, this is Autophone. '
405 'Just to let you know, I have experienced '
406 'unrecoverable device errors and am rebooting in '
407 'the hope of resolving them.\n\n'
408 'Please check on me.\n')
409 self.mailer.send(msg_subj, msg_body)
410 subprocess.call('sudo reboot', shell=True)
411
86c921e @bclary Bug 1161631 - Autophone - part 4 -implement autophone-restart, autoph…
bclary authored
412 if self.state == ProcessStates.RESTARTING:
413 # Lifted from Sisyphus/Bughunter...
414 newargv = sys.argv
415 newargv.insert(0, sys.executable)
416
417 # Set all open file handlers to close on exec. Use 64K as
418 # the limit to check as that is the max on Windows XP. The
419 # performance issue of doing this is negligible since it
420 # is only run during a program reload.
421 from fcntl import fcntl, F_GETFD, F_SETFD, FD_CLOEXEC
422 for fd in xrange(0x3, 0x10000):
423 try:
424 fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC)
425 except KeyboardInterrupt:
426 raise
427 except:
428 pass
429
430 os.execvp(sys.executable, newargv)
431
91b2c03 @markrcote Initial commit.
markrcote authored
432 # Start the phones for testing
fadc14c @bclary Bug 1118758 - Autophone - add support for job specific tests, r=mcote.
bclary authored
433 def new_job(self, job_data):
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
434 logger.debug('new_job: %s' % job_data)
fadc14c @bclary Bug 1118758 - Autophone - add support for job specific tests, r=mcote.
bclary authored
435 build_url = job_data['build']
436 tests = job_data['tests']
0bd108b @bclary Bug 1133580 - Autophone - hookup new pulsemonitor, consolidate compat…
bclary authored
437
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
438 build_data = utils.get_build_data(build_url)
439 logger.debug('new_job: build_data %s' % build_data)
0bd108b @bclary Bug 1133580 - Autophone - hookup new pulsemonitor, consolidate compat…
bclary authored
440
441 if not build_data:
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
442 logger.warning('new_job: Could not find build_data for %s' %
0bd108b @bclary Bug 1133580 - Autophone - hookup new pulsemonitor, consolidate compat…
bclary authored
443 build_url)
444 return
445
446 revision_hash = utils.get_treeherder_revision_hash(
447 self.options.treeherder_url,
448 build_data['repo'],
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
449 build_data['revision'])
0bd108b @bclary Bug 1133580 - Autophone - hookup new pulsemonitor, consolidate compat…
bclary authored
450
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
451 logger.debug('new_job: revision_hash %s' % revision_hash)
0bd108b @bclary Bug 1133580 - Autophone - hookup new pulsemonitor, consolidate compat…
bclary authored
452
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
453 phoneids = set([test.phone.id for test in tests])
454 for phoneid in phoneids:
455 p = self.phone_workers[phoneid]
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
456 logger.debug('new_job: worker phoneid %s' % phoneid)
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
457 # Determine if we will test this build, which tests to run and if we
458 # need to enable unittests.
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
459 runnable_tests = PhoneTest.match(tests=tests, phoneid=phoneid)
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
460 if not runnable_tests:
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
461 logger.debug('new_job: Ignoring build %s for phone %s' % (build_url, phoneid))
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
462 continue
463 enable_unittests = False
464 for t in runnable_tests:
465 enable_unittests = enable_unittests or t.enable_unittests
466
467 new_tests = self.jobs.new_job(build_url,
468 build_id=build_data['id'],
469 changeset=build_data['changeset'],
470 tree=build_data['repo'],
471 revision=build_data['revision'],
472 revision_hash=revision_hash,
473 tests=runnable_tests,
474 enable_unittests=enable_unittests,
475 device=phoneid)
476 if new_tests:
477 self.treeherder.submit_pending(phoneid,
478 build_url,
479 build_data['repo'],
480 revision_hash,
481 tests=new_tests)
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
482 logger.info('new_job: Notifying device %s of new job '
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
483 '%s for tests %s, enable_unittests=%s.' %
484 (phoneid, build_url, runnable_tests,
485 enable_unittests))
486 p.new_job()
91b2c03 @markrcote Initial commit.
markrcote authored
487
488 def route_cmd(self, data):
645d88f @bclary bug 853961 - make sure to release locks in case of an exception, r=mc…
bclary authored
489 response = ''
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
490 self.lock_acquire(data=data)
645d88f @bclary bug 853961 - make sure to release locks in case of an exception, r=mc…
bclary authored
491 try:
492 response = self._route_cmd(data)
493 finally:
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
494 self.lock_release(data=data)
645d88f @bclary bug 853961 - make sure to release locks in case of an exception, r=mc…
bclary authored
495 return response
496
497 def _route_cmd(self, data):
524b0db @markrcote Bug 790380 - Handle worker crashes and misc. other improvements and f…
markrcote authored
498 # There is not currently any way to get proper responses for commands
499 # that interact with workers, since communication between the main
500 # process and the worker processes is asynchronous.
501 # It would be possible but nontrivial for the workers to put responses
502 # onto a queue and have them routed to the proper connection by using
503 # request IDs or something like that.
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
504 logger.debug('route_cmd: %s' % data)
91b2c03 @markrcote Initial commit.
markrcote authored
505 data = data.strip()
506 cmd, space, params = data.partition(' ')
507 cmd = cmd.lower()
508 response = 'ok'
509
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
510 if cmd.startswith('device-'):
511 # Device commands have prefix device- and are mapped into
512 # PhoneWorker methods by stripping the leading 'device-'
513 # from the command. The device id is the first parameter.
a0bb699 @bclary Bug 1161631 - Autophone - part 3 -implement help and device commands,…
bclary authored
514 valid_cmds = ('is_alive', 'stop', 'shutdown', 'reboot', 'disable',
e90a3a4 @bclary Bug 1161631 - Autophone - part 7 - remove device-debug, document com…
bclary authored
515 'enable', 'ping', 'status', 'restart')
a0bb699 @bclary Bug 1161631 - Autophone - part 3 -implement help and device commands,…
bclary authored
516 cmd = cmd.replace('device-', '').replace('-', '_')
517 if cmd not in valid_cmds:
518 response = 'Unknown command device-%s' % cmd
519 else:
520 phoneid, space, params = params.partition(' ')
521 response = 'error: phone not found'
522 for worker in self.phone_workers.values():
523 if (phoneid.lower() == 'all' or
524 worker.phone.serial == phoneid or
525 worker.phone.id == phoneid):
526 f = getattr(worker, cmd)
527 if params:
528 value = f(params)
529 else:
530 value = f()
531 if value is not None:
532 response = '%s\n' % value
533 else:
534 response = ''
535 response += 'ok'
86c921e @bclary Bug 1161631 - Autophone - part 4 -implement autophone-restart, autoph…
bclary authored
536 elif cmd == 'autophone-add-device':
537 phoneid, space, serialno = params.partition(' ')
3ae1b09 @bclary Bug 1155885 - Autophone - should automatically recover when devices a…
bclary authored
538 if phoneid in self.phone_workers:
539 response = 'device %s already exists' % phoneid
540 console_logger.warning(response)
541 else:
86c921e @bclary Bug 1161631 - Autophone - part 4 -implement autophone-restart, autoph…
bclary authored
542 try:
3ae1b09 @bclary Bug 1155885 - Autophone - should automatically recover when devices a…
bclary authored
543 dm = ADBDevice(
544 device=serialno,
545 device_ready_retry_wait=self.options.device_ready_retry_wait,
546 device_ready_retry_attempts=self.options.device_ready_retry_attempts,
547 verbose=self.options.verbose)
548
549 dm.power_on()
550 device = {"device_name": phoneid,
551 "serialno": serialno,
552 "dm" : dm}
553 device['osver'] = dm.get_prop('ro.build.version.release')
554 device['hardware'] = dm.get_prop('ro.product.model')
555 device['abi'] = dm.get_prop('ro.product.cpu.abi')
556 try:
557 sdk = int(dm.get_prop('ro.build.version.sdk'))
558 device['sdk'] = 'api-9' if sdk <= 10 else 'api-11'
559 except ValueError:
560 device['sdk'] = 'api-9'
561 self._devices[phoneid] = device
562 # We must reload the test manifest again to pick up the
563 # new device's test configuration.
564 console_logger.info('Adding device %s %s' % (phoneid, serialno))
565 self.read_tests()
566 self.register_cmd(device)
567 self.phone_workers[phoneid].start()
568 except Exception, e:
569 self.purge_worker(phoneid)
570 response = '%s: Unable to add device due to %s.' % (data, e)
571 console_logger.error(response)
86c921e @bclary Bug 1161631 - Autophone - part 4 -implement autophone-restart, autoph…
bclary authored
572 elif cmd == 'autophone-restart':
573 self.state = ProcessStates.RESTARTING
574 console_logger.info('Restarting Autophone...')
575 for worker in self.phone_workers.values():
576 worker.shutdown()
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
577 elif cmd == 'autophone-stop':
86c921e @bclary Bug 1161631 - Autophone - part 4 -implement autophone-restart, autoph…
bclary authored
578 console_logger.info('Stopping Autophone...')
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
579 self.stop()
580 elif cmd == 'autophone-shutdown':
86c921e @bclary Bug 1161631 - Autophone - part 4 -implement autophone-restart, autoph…
bclary authored
581 console_logger.info('Shutting down Autophone...')
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
582 self.shutdown()
583 elif cmd == 'autophone-log':
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
584 logger.info(params)
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
585 elif cmd == 'autophone-triggerjobs':
586 response = self.trigger_jobs(params)
587 elif cmd == 'autophone-status':
86c921e @bclary Bug 1161631 - Autophone - part 4 -implement autophone-restart, autoph…
bclary authored
588 response = 'state: %s\n' % self.state
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
589 phoneids = self.phone_workers.keys()
590 phoneids.sort()
591 for i in phoneids:
592 response += self.phone_workers[i].status()
593 response += 'ok'
a0bb699 @bclary Bug 1161631 - Autophone - part 3 -implement help and device commands,…
bclary authored
594 elif cmd == 'autophone-help':
595 response = '''
596 Autophone command help:
597
598 autophone-help
599 Generate this message.
600
e90a3a4 @bclary Bug 1161631 - Autophone - part 7 - remove device-debug, document com…
bclary authored
601 autophone-add-device <devicename> <serialno>
602 Adds a new device to the active workers. <devicename> refers to
603 the name given to the device in the devices.ini file while
604 <serialno> is its adb serial number.
86c921e @bclary Bug 1161631 - Autophone - part 4 -implement autophone-restart, autoph…
bclary authored
605
606 autophone-restart
607 Shutdown each worker after its current test, then restart
608 autophone.
a0bb699 @bclary Bug 1161631 - Autophone - part 3 -implement help and device commands,…
bclary authored
609
610 autophone-shutdown
86c921e @bclary Bug 1161631 - Autophone - part 4 -implement autophone-restart, autoph…
bclary authored
611 Shutdown each worker after its current test, then
a0bb699 @bclary Bug 1161631 - Autophone - part 3 -implement help and device commands,…
bclary authored
612 shutdown autophone.
613
86c921e @bclary Bug 1161631 - Autophone - part 4 -implement autophone-restart, autoph…
bclary authored
614 autophone-status
615 Generate a status report for each device.
616
a0bb699 @bclary Bug 1161631 - Autophone - part 3 -implement help and device commands,…
bclary authored
617 autophone-stop
618 Immediately stop autophone and all worker processes; may be
619 delayed by pending download.
620
e90a3a4 @bclary Bug 1161631 - Autophone - part 7 - remove device-debug, document com…
bclary authored
621 device-disable <devicename>
a0bb699 @bclary Bug 1161631 - Autophone - part 3 -implement help and device commands,…
bclary authored
622 Disable the device's worker and cancel its pending jobs.
623
e90a3a4 @bclary Bug 1161631 - Autophone - part 7 - remove device-debug, document com…
bclary authored
624 device-enable <devicename>
a0bb699 @bclary Bug 1161631 - Autophone - part 3 -implement help and device commands,…
bclary authored
625 Enable a disabled device's worker.
626
e90a3a4 @bclary Bug 1161631 - Autophone - part 7 - remove device-debug, document com…
bclary authored
627 device-is-alive <devicename>
86c921e @bclary Bug 1161631 - Autophone - part 4 -implement autophone-restart, autoph…
bclary authored
628 Check if the device's worker process is alive, report to log.
a0bb699 @bclary Bug 1161631 - Autophone - part 3 -implement help and device commands,…
bclary authored
629
e90a3a4 @bclary Bug 1161631 - Autophone - part 7 - remove device-debug, document com…
bclary authored
630 device-ping <devicename>
a0bb699 @bclary Bug 1161631 - Autophone - part 3 -implement help and device commands,…
bclary authored
631 Issue a ping command to the device's worker which checks the sdcard
632 availability.
633
e90a3a4 @bclary Bug 1161631 - Autophone - part 7 - remove device-debug, document com…
bclary authored
634 device-reboot <devicename>
86c921e @bclary Bug 1161631 - Autophone - part 4 -implement autophone-restart, autoph…
bclary authored
635 Reboot the device.
636
e90a3a4 @bclary Bug 1161631 - Autophone - part 7 - remove device-debug, document com…
bclary authored
637 device-restart <devicename>
86c921e @bclary Bug 1161631 - Autophone - part 4 -implement autophone-restart, autoph…
bclary authored
638 Shutdown the device's worker process after the current test, then
639 restart the worker picking up test manifest and test configuration
640 changes.
641
e90a3a4 @bclary Bug 1161631 - Autophone - part 7 - remove device-debug, document com…
bclary authored
642 device-status <devicename>
a0bb699 @bclary Bug 1161631 - Autophone - part 3 -implement help and device commands,…
bclary authored
643 Generate a status report for the device's worker.
86c921e @bclary Bug 1161631 - Autophone - part 4 -implement autophone-restart, autoph…
bclary authored
644
e90a3a4 @bclary Bug 1161631 - Autophone - part 7 - remove device-debug, document com…
bclary authored
645 device-shutdown <devicename>
86c921e @bclary Bug 1161631 - Autophone - part 4 -implement autophone-restart, autoph…
bclary authored
646 Shutdown the device's worker process after the current test. The
647 device's worker process will not be restarted and will be removed
648 from the active list of workers.
649
e90a3a4 @bclary Bug 1161631 - Autophone - part 7 - remove device-debug, document com…
bclary authored
650 device-stop <devicename>
86c921e @bclary Bug 1161631 - Autophone - part 4 -implement autophone-restart, autoph…
bclary authored
651 Immediately stop the device's worker process and remove it from the
652 list of active workers.
653
a0bb699 @bclary Bug 1161631 - Autophone - part 3 -implement help and device commands,…
bclary authored
654 ok
655 '''
91b2c03 @markrcote Initial commit.
markrcote authored
656 else:
657 response = 'Unknown command "%s"\n' % cmd
658 return response
659
8292c84 @bclary Bug 1063886 - Autophone - paydown technical debt, r=mcote.
bclary authored
660 def create_worker(self, phone):
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
661 logger.info('Creating worker for %s: %s.' % (phone, self.options))
7358c98 @bclary Bug 1161631 - Autophone - part 6 - reuse ADBDevice instances, r=gbrown.
bclary authored
662 dm = self._devices[phone.id]['dm']
6d57554 @bclary Bug 968905 - Autophone - parameterize control constants via config fi…
bclary authored
663 tests = []
0bd108b @bclary Bug 1133580 - Autophone - hookup new pulsemonitor, consolidate compat…
bclary authored
664 for test_class, config_file, test_devices_repos in self._tests:
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
665 logger.debug('create_worker: %s %s %s' % (
f306708 @bclary Bug 1162514 - Autophone - fix device repo mapping, r=gbrown.
bclary authored
666 test_class, config_file, test_devices_repos))
6d57554 @bclary Bug 968905 - Autophone - parameterize control constants via config fi…
bclary authored
667 skip_test = True
668 if not test_devices_repos:
669 # There is no restriction on this test being run by
670 # specific devices.
f306708 @bclary Bug 1162514 - Autophone - fix device repo mapping, r=gbrown.
bclary authored
671 repos = []
6d57554 @bclary Bug 968905 - Autophone - parameterize control constants via config fi…
bclary authored
672 skip_test = False
8292c84 @bclary Bug 1063886 - Autophone - paydown technical debt, r=mcote.
bclary authored
673 elif phone.id in test_devices_repos:
6e8806f @bclary Bug 1153992 - Autophone - allow test manifest to include multiple sec…
bclary authored
674 # This test is to be run by this device on
675 # the repos test_devices_repos[phone.id]
f306708 @bclary Bug 1162514 - Autophone - fix device repo mapping, r=gbrown.
bclary authored
676 repos = test_devices_repos[phone.id]
6d57554 @bclary Bug 968905 - Autophone - parameterize control constants via config fi…
bclary authored
677 skip_test = False
678 if not skip_test:
7358c98 @bclary Bug 1161631 - Autophone - part 6 - reuse ADBDevice instances, r=gbrown.
bclary authored
679 test = test_class(dm=dm,
680 phone=phone,
b8f2f26 @bclary Bug 1079923 - Autophone - update unittests to fix bitrot, r=mcote.
bclary authored
681 options=self.options,
f306708 @bclary Bug 1162514 - Autophone - fix device repo mapping, r=gbrown.
bclary authored
682 config_file=config_file,
683 repos=repos)
b8f2f26 @bclary Bug 1079923 - Autophone - update unittests to fix bitrot, r=mcote.
bclary authored
684 tests.append(test)
685 for chunk in range(2, test.chunks+1):
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
686 logger.debug('Creating chunk %d/%d' % (chunk, test.chunks))
7358c98 @bclary Bug 1161631 - Autophone - part 6 - reuse ADBDevice instances, r=gbrown.
bclary authored
687 tests.append(test_class(dm=dm,
688 phone=phone,
b8f2f26 @bclary Bug 1079923 - Autophone - update unittests to fix bitrot, r=mcote.
bclary authored
689 options=self.options,
690 config_file=config_file,
f306708 @bclary Bug 1162514 - Autophone - fix device repo mapping, r=gbrown.
bclary authored
691 chunk=chunk,
692 repos=repos))
6d57554 @bclary Bug 968905 - Autophone - parameterize control constants via config fi…
bclary authored
693 if not tests:
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
694 logger.warning('Not creating worker: No tests defined for '
0bd108b @bclary Bug 1133580 - Autophone - hookup new pulsemonitor, consolidate compat…
bclary authored
695 'worker for %s: %s.' %
696 (phone, self.options))
697 return
8292c84 @bclary Bug 1063886 - Autophone - paydown technical debt, r=mcote.
bclary authored
698 logfile_prefix = os.path.splitext(self.options.logfile)[0]
7358c98 @bclary Bug 1161631 - Autophone - part 6 - reuse ADBDevice instances, r=gbrown.
bclary authored
699 worker = PhoneWorker(dm, self.next_worker_num,
8292c84 @bclary Bug 1063886 - Autophone - paydown technical debt, r=mcote.
bclary authored
700 tests, phone, self.options,
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
701 self.queue,
8292c84 @bclary Bug 1063886 - Autophone - paydown technical debt, r=mcote.
bclary authored
702 '%s-%s' % (logfile_prefix, phone.id),
279303d @bclary Bug 1161631 - Autophone - Use a shared process lock to prevent Autoph…
bclary authored
703 self.loglevel, self.mailer, self.shared_lock)
8292c84 @bclary Bug 1063886 - Autophone - paydown technical debt, r=mcote.
bclary authored
704 self.phone_workers[phone.id] = worker
a0bb699 @bclary Bug 1161631 - Autophone - part 3 -implement help and device commands,…
bclary authored
705 return worker
91b2c03 @markrcote Initial commit.
markrcote authored
706
3ae1b09 @bclary Bug 1155885 - Autophone - should automatically recover when devices a…
bclary authored
707 def purge_worker(self, phoneid):
708 """Remove worker and its tests from cached locations."""
709 if phoneid in self.phone_workers:
710 del self.phone_workers[phoneid]
711 if phoneid in self.restart_workers:
712 del self.restart_workers[phoneid]
713 for t in PhoneTest.match(phoneid=phoneid):
714 t.remove()
715
91b2c03 @markrcote Initial commit.
markrcote authored
716 def register_cmd(self, data):
3ae1b09 @bclary Bug 1155885 - Autophone - should automatically recover when devices a…
bclary authored
717 # Map MAC Address to ip and user name for phone
718 # The configparser does odd things with the :'s so remove them.
719 phoneid = data['device_name']
720 phone = PhoneData(
721 phoneid,
722 data['serialno'],
723 data['hardware'],
724 data['osver'],
725 data['abi'],
726 data['sdk'],
727 self.options.ipaddr) # XXX IPADDR no longer needed?
728 if logger.getEffectiveLevel() == logging.DEBUG:
729 logger.debug('register_cmd: phone: %s' % phone)
730 if phoneid in self.phone_workers:
731 logger.debug('Received registration message for known phone '
732 '%s.' % phoneid)
733 worker = self.phone_workers[phoneid]
734 if worker.phone.__dict__ != phone.__dict__:
735 # This won't update the subprocess, but it will allow
736 # us to write out the updated values right away.
737 worker.phone = phone
738 logger.info('Registration info has changed; restarting '
739 'worker.')
740 if phoneid in self.restart_workers:
741 logger.info('Phone worker is already scheduled to be '
742 'restarted!')
743 else:
744 self.restart_workers[phoneid] = phone
745 worker.stop()
746 else:
747 try:
8292c84 @bclary Bug 1063886 - Autophone - paydown technical debt, r=mcote.
bclary authored
748 self.create_worker(phone)
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
749 logger.info('Registered phone %s.' % phone.id)
3ae1b09 @bclary Bug 1155885 - Autophone - should automatically recover when devices a…
bclary authored
750 except Exception:
751 console_logger.info('Worker %s failed to register' % phoneid)
752 self.purge_worker(phoneid)
753 raise
91b2c03 @markrcote Initial commit.
markrcote authored
754
8bddba3 @bclary Bug 990601 - Autophone - use adb instead of tcp/ip to control devices…
bclary authored
755 def read_devices(self):
756 cfg = ConfigParser.RawConfigParser()
8292c84 @bclary Bug 1063886 - Autophone - paydown technical debt, r=mcote.
bclary authored
757 cfg.read(self.options.devicescfg)
8bddba3 @bclary Bug 990601 - Autophone - use adb instead of tcp/ip to control devices…
bclary authored
758
759 for device_name in cfg.sections():
760 # failure for a device to have a serialno option is fatal.
761 serialno = cfg.get(device_name, 'serialno')
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
762 console_logger.info("Initializing device name=%s, serialno=%s" % (device_name, serialno))
6098a69 @bclary Bug 1072423 - Autophone - support split APKs for 2.3/later, r=mcote.
bclary authored
763 try:
86c921e @bclary Bug 1161631 - Autophone - part 4 -implement autophone-restart, autoph…
bclary authored
764 dm = ADBDevice(device=serialno,
7358c98 @bclary Bug 1161631 - Autophone - part 6 - reuse ADBDevice instances, r=gbrown.
bclary authored
765 device_ready_retry_wait=self.options.device_ready_retry_wait,
766 device_ready_retry_attempts=self.options.device_ready_retry_attempts,
767 verbose=self.options.verbose)
86c921e @bclary Bug 1161631 - Autophone - part 4 -implement autophone-restart, autoph…
bclary authored
768 dm.power_on()
769 device = {"device_name": device_name,
770 "serialno": serialno,
771 "dm" : dm}
772 device['osver'] = dm.get_prop('ro.build.version.release')
773 device['hardware'] = dm.get_prop('ro.product.model')
774 device['abi'] = dm.get_prop('ro.product.cpu.abi')
775 try:
776 sdk = int(dm.get_prop('ro.build.version.sdk'))
777 device['sdk'] = 'api-9' if sdk <= 10 else 'api-11'
778 except ValueError:
779 device['sdk'] = 'api-9'
780 self._devices[device_name] = device
781 self.register_cmd(device)
3ae1b09 @bclary Bug 1155885 - Autophone - should automatically recover when devices a…
bclary authored
782 except Exception, e:
783 console_logger.error('Unable to initialize device %s due to %s.' %
784 (device_name, e))
785 msg_subj = 'Unable to initialize device %s' % device_name
786 msg_body = ('Hello, this is Autophone. '
787 'Just to let you know, '
788 'phone %s '
789 'failed to initialize due to %s.\n' %
790 (device_name, e))
791 self.mailer.send(msg_subj, msg_body)
792 self.purge_worker(device_name)
91b2c03 @markrcote Initial commit.
markrcote authored
793
794 def read_tests(self):
795 self._tests = []
796 manifest = TestManifest()
8292c84 @bclary Bug 1063886 - Autophone - paydown technical debt, r=mcote.
bclary authored
797 manifest.read(self.options.test_path)
91b2c03 @markrcote Initial commit.
markrcote authored
798 tests_info = manifest.get()
799 for t in tests_info:
6e8806f @bclary Bug 1153992 - Autophone - allow test manifest to include multiple sec…
bclary authored
800 # Remove test section suffix.
801 t['name'] = t['name'].split()[0]
91b2c03 @markrcote Initial commit.
markrcote authored
802 if not t['here'] in sys.path:
803 sys.path.append(t['here'])
804 if t['name'].endswith('.py'):
805 t['name'] = t['name'][:-3]
806 # add all classes in module that are derived from PhoneTest to
6548982 @markrcote Allow specification of IP address as command-line option. Reboot phon…
markrcote authored
807 # the test list
60d7957 @bclary Bug 948512 - Autophone - specify device specific tests, r=mcote.
bclary authored
808 tests = []
6d57554 @bclary Bug 968905 - Autophone - parameterize control constants via config fi…
bclary authored
809 for member_name, member_value in inspect.getmembers(__import__(t['name']),
810 inspect.isclass):
811 if (member_name != 'PhoneTest' and
5d020e1 @bclary Bug 1037077 - Report run status to treeherder, r=mcote.
bclary authored
812 member_name != 'PerfTest' and
8292c84 @bclary Bug 1063886 - Autophone - paydown technical debt, r=mcote.
bclary authored
813 issubclass(member_value, PhoneTest)):
b8f2f26 @bclary Bug 1079923 - Autophone - update unittests to fix bitrot, r=mcote.
bclary authored
814 config = t.get('config', '')
815 # config is a space separated list of config
816 # files. The test will be instantiated for each
817 # of the config files allowing tests such as the
818 # runremotetests.py to handle more than one unit
819 # test at a time.
820 #
821 # Each config file can contain additional options
822 # for a test.
823 #
824 # Other options are:
825 #
826 # <device> = <repo-list>
827 #
828 # which determines the devices which should
829 # run the test. If no devices are listed, then
830 # all devices will run the test.
831
6d57554 @bclary Bug 968905 - Autophone - parameterize control constants via config fi…
bclary authored
832 devices = [device for device in t if device not in
833 ('name', 'here', 'manifest', 'path', 'config',
b8f2f26 @bclary Bug 1079923 - Autophone - update unittests to fix bitrot, r=mcote.
bclary authored
834 'relpath', 'unittests', 'subsuite')]
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
835 logger.debug('read_tests: test: %s, class: %s, '
0bd108b @bclary Bug 1133580 - Autophone - hookup new pulsemonitor, consolidate compat…
bclary authored
836 'config: %s, devices: %s' % (
837 member_name,
838 member_value,
839 config,
840 devices))
6d57554 @bclary Bug 968905 - Autophone - parameterize control constants via config fi…
bclary authored
841 test_devices_repos = {}
842 for device in devices:
843 test_devices_repos[device] = t[device].split()
b8f2f26 @bclary Bug 1079923 - Autophone - update unittests to fix bitrot, r=mcote.
bclary authored
844 configs = config.split()
845 for config_file in configs:
6e8806f @bclary Bug 1153992 - Autophone - allow test manifest to include multiple sec…
bclary authored
846 config_file = os.path.normpath(
847 os.path.join(t['here'], config_file))
848 tests.append((member_value,
849 config_file, test_devices_repos))
60d7957 @bclary Bug 948512 - Autophone - specify device specific tests, r=mcote.
bclary authored
850
6548982 @markrcote Allow specification of IP address as command-line option. Reboot phon…
markrcote authored
851 self._tests.extend(tests)
91b2c03 @markrcote Initial commit.
markrcote authored
852
6d57554 @bclary Bug 968905 - Autophone - parameterize control constants via config fi…
bclary authored
853
91b2c03 @markrcote Initial commit.
markrcote authored
854 def trigger_jobs(self, data):
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
855 logger.info('Received user-specified job: %s' % data)
6e8806f @bclary Bug 1153992 - Autophone - allow test manifest to include multiple sec…
bclary authored
856 trigger_data = json.loads(data)
857 if 'build' not in trigger_data:
cc8be50 @markrcote Bug 850395 - Allow device-specific jobs. r=bc
markrcote authored
858 return 'invalid args'
6e8806f @bclary Bug 1153992 - Autophone - allow test manifest to include multiple sec…
bclary authored
859 build_url = trigger_data['build']
860 tests = []
861 test_names = trigger_data['test_names']
862 if not test_names:
863 # No test names specified, force PhoneTest.match
864 # to return tests with any name.
865 test_names = [None]
866 devices = trigger_data['devices']
867 if not devices:
868 # No devices specified, force PhoneTest.match
869 # to return tests for any device.
870 devices = [None]
871 for test_name in test_names:
872 for device in devices:
873 tests.extend(PhoneTest.match(test_name=test_name,
874 phoneid=device,
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
875 build_url=build_url))
6e8806f @bclary Bug 1153992 - Autophone - allow test manifest to include multiple sec…
bclary authored
876 if tests:
877 job_data = {
878 'build': build_url,
879 'tests': tests,
880 }
881 self.new_job(job_data)
cc8be50 @markrcote Bug 850395 - Allow device-specific jobs. r=bc
markrcote authored
882 return 'ok'
91b2c03 @markrcote Initial commit.
markrcote authored
883
884 def reset_phones(self):
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
885 logger.info('Resetting phones...')
91b2c03 @markrcote Initial commit.
markrcote authored
886 for phoneid, phone in self.phone_workers.iteritems():
6548982 @markrcote Allow specification of IP address as command-line option. Reboot phon…
markrcote authored
887 phone.reboot()
91b2c03 @markrcote Initial commit.
markrcote authored
888
889 def on_build(self, msg):
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
890 self.lock_acquire()
891 try:
892 if self.state != ProcessStates.RUNNING:
893 return
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
894 logger.debug('PULSE BUILD FOUND %s' % msg)
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
895 build_url = msg['packageUrl']
896 if msg['branch'] != 'try':
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
897 tests = PhoneTest.match(build_url=build_url)
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
898 else:
899 # Autophone try builds will have a comment of the form:
900 # try: -b o -p android-api-9,android-api-11 -u autophone-smoke,autophone-s1s2 -t none
901 tests = []
902 reTests = re.compile('try:.* -u (.*) -t.*')
903 match = reTests.match(msg['comments'])
904 if match:
905 test_names = [t for t in match.group(1).split(',')
906 if t.startswith('autophone-')]
907 if 'autophone-tests' in test_names:
908 # Match all test names
909 test_names = [None]
910 for test_name in test_names:
911 tests.extend(PhoneTest.match(test_name=test_name,
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
912 build_url=build_url))
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
913 job_data = {'build': build_url, 'tests': tests}
914 self.new_job(job_data)
915 finally:
916 self.lock_release()
5d632fe @bclary Bug 785129 - Create Autophone tests to run unit tests, r=mcote.
bclary authored
917
0bd108b @bclary Bug 1133580 - Autophone - hookup new pulsemonitor, consolidate compat…
bclary authored
918 def on_jobaction(self, job_action):
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
919 self.lock_acquire()
0bd108b @bclary Bug 1133580 - Autophone - hookup new pulsemonitor, consolidate compat…
bclary authored
920 try:
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
921 if (self.state != ProcessStates.RUNNING or
922 job_action['job_group_name'] != 'Autophone'):
923 return
924 machine_name = job_action['machine_name']
925 if machine_name not in self.phone_workers:
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
926 logger.warning('on_jobaction: unknown device %s' % machine_name)
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
927 return
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
928 logger.debug('on_jobaction: found %s' % json.dumps(
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
929 job_action, sort_keys=True, indent=4))
930
f857458 @bclary Bug 1160158 - Autophone - validate device id when processing job acti…
bclary authored
931 p = self.phone_workers[machine_name]
0bd108b @bclary Bug 1133580 - Autophone - hookup new pulsemonitor, consolidate compat…
bclary authored
932 if job_action['action'] == 'cancel':
6e8806f @bclary Bug 1153992 - Autophone - allow test manifest to include multiple sec…
bclary authored
933 request = (job_action['job_guid'],)
934 p.cancel_test(request)
0bd108b @bclary Bug 1133580 - Autophone - hookup new pulsemonitor, consolidate compat…
bclary authored
935 elif job_action['action'] == 'retrigger':
6e8806f @bclary Bug 1153992 - Autophone - allow test manifest to include multiple sec…
bclary authored
936 test = PhoneTest.lookup(
f857458 @bclary Bug 1160158 - Autophone - validate device id when processing job acti…
bclary authored
937 machine_name,
6e8806f @bclary Bug 1153992 - Autophone - allow test manifest to include multiple sec…
bclary authored
938 job_action['config_file'],
939 job_action['chunk'])
940 if not test:
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
941 logger.warning(
f857458 @bclary Bug 1160158 - Autophone - validate device id when processing job acti…
bclary authored
942 'on_jobaction: No test found for %s' %
6e8806f @bclary Bug 1153992 - Autophone - allow test manifest to include multiple sec…
bclary authored
943 json.dumps(job_action, sort_keys=True, indent=4))
944 else:
945 job_data = {
946 'build': job_action['build_url'],
947 'tests': [test],
948 }
949 self.new_job(job_data)
0bd108b @bclary Bug 1133580 - Autophone - hookup new pulsemonitor, consolidate compat…
bclary authored
950 else:
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
951 logger.warning('on_jobaction: unknown action %s' %
0bd108b @bclary Bug 1133580 - Autophone - hookup new pulsemonitor, consolidate compat…
bclary authored
952 job_action['action'])
953 finally:
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
954 self.lock_release()
0bd108b @bclary Bug 1133580 - Autophone - hookup new pulsemonitor, consolidate compat…
bclary authored
955
91b2c03 @markrcote Initial commit.
markrcote authored
956 def stop(self):
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
957 self.state = ProcessStates.STOPPING
958
959 def shutdown(self):
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
960 logger.debug('AutoPhone.shutdown: enter')
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
961 self.state = ProcessStates.SHUTTINGDOWN
962 if self.pulse_monitor:
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
963 logger.debug('AutoPhone.shutdown: stopping pulse monitor')
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
964 self.pulse_monitor.stop()
965 self.pulse_monitor = None
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
966 logger.debug('AutoPhone.shutdown: shutting down workers')
91b2c03 @markrcote Initial commit.
markrcote authored
967 for p in self.phone_workers.values():
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
968 logger.debug('AutoPhone.shutdown: shutting down worker %s' % p.phone.id)
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
969 p.shutdown()
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
970 logger.debug('AutoPhone.shutdown: exit')
91b2c03 @markrcote Initial commit.
markrcote authored
971
6d57554 @bclary Bug 968905 - Autophone - parameterize control constants via config fi…
bclary authored
972 def load_autophone_options(cmd_options):
8292c84 @bclary Bug 1063886 - Autophone - paydown technical debt, r=mcote.
bclary authored
973 options = AutophoneOptions()
974 option_tuples = [(option_name, type(option_value))
975 for option_name, option_value in inspect.getmembers(options)
976 if not option_name.startswith('_')]
977 getter_map = {str: 'get', int: 'getint', bool: 'getboolean', list: 'get'}
6d57554 @bclary Bug 968905 - Autophone - parameterize control constants via config fi…
bclary authored
978
8292c84 @bclary Bug 1063886 - Autophone - paydown technical debt, r=mcote.
bclary authored
979 for option_name, option_type in option_tuples:
980 try:
981 value = getattr(cmd_options, option_name)
982 if value is not None:
983 value = option_type(value)
984 setattr(options, option_name, value)
985 except AttributeError:
986 pass
91b2c03 @markrcote Initial commit.
markrcote authored
987
b6dc95d @bclary Bug 1119654 - Autophone - separate installation specific settings, r=…
bclary authored
988 cfg = ConfigParser.RawConfigParser()
6d57554 @bclary Bug 968905 - Autophone - parameterize control constants via config fi…
bclary authored
989 if cmd_options.autophonecfg:
990 cfg.read(cmd_options.autophonecfg)
b6dc95d @bclary Bug 1119654 - Autophone - separate installation specific settings, r=…
bclary authored
991 if cfg.has_option('settings', 'credentials_file'):
992 cfg.read(cfg.get('settings', 'credentials_file'))
993 if cmd_options.credentials_file:
994 cfg.read(cmd_options.credentials_file)
6d57554 @bclary Bug 968905 - Autophone - parameterize control constants via config fi…
bclary authored
995
b6dc95d @bclary Bug 1119654 - Autophone - separate installation specific settings, r=…
bclary authored
996 if cmd_options.autophonecfg or cmd_options.credentials_file:
8292c84 @bclary Bug 1063886 - Autophone - paydown technical debt, r=mcote.
bclary authored
997 for option_name, option_type in option_tuples:
6d57554 @bclary Bug 968905 - Autophone - parameterize control constants via config fi…
bclary authored
998 try:
999 getter = getattr(ConfigParser.RawConfigParser,
8292c84 @bclary Bug 1063886 - Autophone - paydown technical debt, r=mcote.
bclary authored
1000 getter_map[option_type])
1001 value = getter(cfg, 'settings', option_name)
1002 if option_type == list:
6d57554 @bclary Bug 968905 - Autophone - parameterize control constants via config fi…
bclary authored
1003 value = value.split()
8292c84 @bclary Bug 1063886 - Autophone - paydown technical debt, r=mcote.
bclary authored
1004 setattr(options, option_name, option_type(value))
6d57554 @bclary Bug 968905 - Autophone - parameterize control constants via config fi…
bclary authored
1005 except ConfigParser.NoOptionError:
1006 pass
1007
5d020e1 @bclary Bug 1037077 - Report run status to treeherder, r=mcote.
bclary authored
1008 if options.treeherder_url and options.treeherder_credentials_path:
1009 with open(options.treeherder_credentials_path) as credentials_file:
1010 setattr(options, 'treeherder_credentials', json.loads(credentials_file.read()))
1011
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
1012 # record sensitive data that should be filtered from logs.
1013 options.sensitive_data = []
1014 options.sensitive_data.append(options.phonedash_password)
1015 options.sensitive_data.append(options.pulse_password)
1016 options.sensitive_data.append(options.aws_access_key_id)
1017 options.sensitive_data.append(options.aws_access_key)
2031330 @bclary Bug 1161631 - Autophone - part 2 - part duh - do not attempt to get t…
bclary authored
1018 if hasattr(options, 'treeherder_credentials'):
1019 for repo in options.repos:
1020 options.sensitive_data.append(options.treeherder_credentials[repo]['consumer_key'])
1021 options.sensitive_data.append(options.treeherder_credentials[repo]['consumer_secret'])
6d57554 @bclary Bug 968905 - Autophone - parameterize control constants via config fi…
bclary authored
1022 return options
1023
1024
1025 def main(options):
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
1026 global logger, console_logger
96834af @markrcote Can create, push, and use custom profiles. Some pulse support.
markrcote authored
1027
eee9724 @markrcote Smoke test, refactored builds.py.
markrcote authored
1028 def sigterm_handler(signum, frame):
1029 autophone.stop()
1030
91b2c03 @markrcote Initial commit.
markrcote authored
1031 loglevel = e = None
1032 try:
8292c84 @bclary Bug 1063886 - Autophone - paydown technical debt, r=mcote.
bclary authored
1033 loglevel = getattr(logging, options.loglevel)
91b2c03 @markrcote Initial commit.
markrcote authored
1034 except AttributeError, e:
1035 pass
1036 finally:
8292c84 @bclary Bug 1063886 - Autophone - paydown technical debt, r=mcote.
bclary authored
1037 if e or logging.getLevelName(loglevel) != options.loglevel:
1038 print 'Invalid log level %s' % options.loglevel
91b2c03 @markrcote Initial commit.
markrcote authored
1039 return errno.EINVAL
2e980e1 @markrcote Log adb-not-in-path warning only once per process. Workers log to the…
markrcote authored
1040
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
1041 sensitive_data_filter = SensitiveDataFilter(options.sensitive_data)
1042 logging.captureWarnings(True)
1043
1044 logger = logging.getLogger()
1045 logger.addFilter(sensitive_data_filter)
1102e03 @bclary bug 853961 - revamp logging, r=mcote.
bclary authored
1046 logger.setLevel(loglevel)
1047
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
1048 filehandler = logging.handlers.TimedRotatingFileHandler(options.logfile,
1049 when='midnight',
1050 backupCount=7)
1051 fileformatstring = ('%(asctime)s|%(process)d|%(threadName)s|%(name)s|'
1052 '%(levelname)s|%(message)s')
1102e03 @bclary bug 853961 - revamp logging, r=mcote.
bclary authored
1053 fileformatter = logging.Formatter(fileformatstring)
1054 filehandler.setFormatter(fileformatter)
1055 logger.addHandler(filehandler)
1056
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
1057 console_logger = logging.getLogger('console')
1058 console_logger.setLevel(loglevel)
1059 streamhandler = logging.StreamHandler(stream=sys.stderr)
1060 streamformatstring = ('%(asctime)s|%(process)d|%(threadName)s|%(name)s|'
1061 '%(levelname)s|%(message)s')
1102e03 @bclary bug 853961 - revamp logging, r=mcote.
bclary authored
1062 streamformatter = logging.Formatter(streamformatstring)
1063 streamhandler.setFormatter(streamformatter)
1064 console_logger.addHandler(streamhandler)
1065
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
1066 for other_logger_name, other_logger in logger.manager.loggerDict.iteritems():
1067 if ((other_logger_name == 'root' or other_logger_name == 'console')
1068 or not hasattr(other_logger, 'handlers')):
1069 continue
1070 other_logger.addFilter(sensitive_data_filter)
1071 for other_handler in other_logger.handlers:
1072 other_handler.flush()
1073 other_handler.close()
1074 other_logger.removeHandler(other_handler)
1075 other_logger.addHandler(logging.NullHandler())
1076 logger.debug('Library logger %s' % other_logger_name)
1077 if options.verbose:
1078 other_logger.setLevel(loglevel)
1079
8292c84 @bclary Bug 1063886 - Autophone - paydown technical debt, r=mcote.
bclary authored
1080 console_logger.info('Starting server on port %d.' % options.port)
1102e03 @bclary bug 853961 - revamp logging, r=mcote.
bclary authored
1081 console_logger.info('Starting build-cache server on port %d.' %
8292c84 @bclary Bug 1063886 - Autophone - paydown technical debt, r=mcote.
bclary authored
1082 options.build_cache_port)
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
1083
a0bb699 @bclary Bug 1161631 - Autophone - part 3 -implement help and device commands,…
bclary authored
1084 # By starting adb server before the build cache, we prevent adb
1085 # from listening to the build cache client port, thus preventing
1086 # restart without first killing adb.
1087 adbhost = ADBHost()
1088 adbhost.start_server()
1089
5d020e1 @bclary Bug 1037077 - Report run status to treeherder, r=mcote.
bclary authored
1090
c9ec58c @bclary Bug 945390 - Autophone - support build selection via revision, r=mcote.
bclary authored
1091 product = 'fennec'
6098a69 @bclary Bug 1072423 - Autophone - support split APKs for 2.3/later, r=mcote.
bclary authored
1092 build_platforms = ['android',
1093 'android-api-9',
1094 'android-api-10',
07892cf @kmoir Bug 1109144 - Autophone - add Android api 11, r=bc.
kmoir authored
1095 'android-api-11',
6098a69 @bclary Bug 1072423 - Autophone - support split APKs for 2.3/later, r=mcote.
bclary authored
1096 'android-x86']
c9ec58c @bclary Bug 945390 - Autophone - support build selection via revision, r=mcote.
bclary authored
1097 buildfile_ext = '.apk'
d1e02c5 @bclary Bug 785086 - Add ability to download unit test prerequisites and to r…
bclary authored
1098 try:
88f839b @markrcote Bug 849251 - Decouple build cache as TCP server in a separate thread.…
markrcote authored
1099 build_cache = builds.BuildCache(
8292c84 @bclary Bug 1063886 - Autophone - paydown technical debt, r=mcote.
bclary authored
1100 options.repos,
1101 options.buildtypes,
6d57554 @bclary Bug 968905 - Autophone - parameterize control constants via config fi…
bclary authored
1102 product,
1103 build_platforms,
1104 buildfile_ext,
8292c84 @bclary Bug 1063886 - Autophone - paydown technical debt, r=mcote.
bclary authored
1105 cache_dir=options.cache_dir,
1106 override_build_dir=options.override_build_dir,
1107 build_cache_size=options.build_cache_size,
5d020e1 @bclary Bug 1037077 - Report run status to treeherder, r=mcote.
bclary authored
1108 build_cache_expires=options.build_cache_expires,
1109 treeherder_url=options.treeherder_url)
d1e02c5 @bclary Bug 785086 - Add ability to download unit test prerequisites and to r…
bclary authored
1110 except builds.BuildCacheException, e:
524b0db @markrcote Bug 790380 - Handle worker crashes and misc. other improvements and f…
markrcote authored
1111 print '''%s
d1e02c5 @bclary Bug 785086 - Add ability to download unit test prerequisites and to r…
bclary authored
1112
1113 When specifying --override-build-dir, the directory must already exist
1114 and contain a build.apk package file to be tested.
1115
1116 In addition, if you have specified --enable-unittests, the override
1117 build directory must also contain a tests directory containing the
1118 unpacked tests package for the build.
1119
1120 ''' % e
6d57554 @bclary Bug 968905 - Autophone - parameterize control constants via config fi…
bclary authored
1121 raise
d1e02c5 @bclary Bug 785086 - Add ability to download unit test prerequisites and to r…
bclary authored
1122
88f839b @markrcote Bug 849251 - Decouple build cache as TCP server in a separate thread.…
markrcote authored
1123 build_cache_server = buildserver.BuildCacheServer(
8292c84 @bclary Bug 1063886 - Autophone - paydown technical debt, r=mcote.
bclary authored
1124 ('127.0.0.1', options.build_cache_port),
6d57554 @bclary Bug 968905 - Autophone - parameterize control constants via config fi…
bclary authored
1125 buildserver.BuildCacheHandler)
88f839b @markrcote Bug 849251 - Decouple build cache as TCP server in a separate thread.…
markrcote authored
1126 build_cache_server.build_cache = build_cache
1127 build_cache_server_thread = threading.Thread(
cd7c3c5 @bclary Bug 1161631 - Autophone - part 2 - revamp logging, r=gbrown.
bclary authored
1128 target=build_cache_server.serve_forever,
1129 name='BuildCacheThread')
88f839b @markrcote Bug 849251 - Decouple build cache as TCP server in a separate thread.…
markrcote authored
1130 build_cache_server_thread.daemon = True
1131 build_cache_server_thread.start()
1132
8bddba3 @bclary Bug 990601 - Autophone - use adb instead of tcp/ip to control devices…
bclary authored
1133 autophone = AutoPhone(loglevel, options)
1134
eee9724 @markrcote Smoke test, refactored builds.py.
markrcote authored
1135 signal.signal(signal.SIGTERM, sigterm_handler)
91b2c03 @markrcote Initial commit.
markrcote authored
1136 autophone.run()
1102e03 @bclary bug 853961 - revamp logging, r=mcote.
bclary authored
1137 # Drop pending messages and commands to prevent hangs on shutdown.
1138 while True:
1139 try:
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
1140 msg = autophone.queue.get_nowait()
1141 logger.debug('Dropping autphone.queue: %s' % msg)
1102e03 @bclary bug 853961 - revamp logging, r=mcote.
bclary authored
1142 except Queue.Empty:
1143 break
1144
1145 for phoneid in autophone.phone_workers:
1146 worker = autophone.phone_workers[phoneid]
1147 while True:
1148 try:
1414831 @bclary Bug 1161631 - Autophone - part 1 - implement shutdown command, r=gbrown.
bclary authored
1149 msg = worker.queue.get_nowait()
1150 logger.debug('Dropping phone %s worker.queue: %s' % (phoneid, msg))
1102e03 @bclary bug 853961 - revamp logging, r=mcote.
bclary authored
1151 except Queue.Empty:
1152 break
1153
1154 console_logger.info('AutoPhone terminated.')
1155 console_logger.info('Shutting down build-cache server...')
88f839b @markrcote Bug 849251 - Decouple build cache as TCP server in a separate thread.…
markrcote authored
1156 build_cache_server.shutdown()
1157 build_cache_server_thread.join()
1102e03 @bclary bug 853961 - revamp logging, r=mcote.
bclary authored
1158 console_logger.info('Done.')
91b2c03 @markrcote Initial commit.
markrcote authored
1159 return 0
1160
1161
1162 if __name__ == '__main__':
1163 from optparse import OptionParser
1164
1165 parser = OptionParser()
6548982 @markrcote Allow specification of IP address as command-line option. Reboot phon…
markrcote authored
1166 parser.add_option('--ipaddr', action='store', type='string', dest='ipaddr',
1167 default=None, help='IP address of interface to use for '
1168 'phone callbacks, e.g. after rebooting. If not given, '
1169 'it will be guessed.')
91b2c03 @markrcote Initial commit.
markrcote authored
1170 parser.add_option('--port', action='store', type='int', dest='port',
1171 default=28001,
1172 help='Port to listen for incoming connections, defaults '
1173 'to 28001')
1174 parser.add_option('--logfile', action='store', type='string',
1175 dest='logfile', default='autophone.log',
2e980e1 @markrcote Log adb-not-in-path warning only once per process. Workers log to the…
markrcote authored
1176 help='Log file to store logging from entire system. '
1177 'Individual phone worker logs will use '
1178 '<logfile>-<phoneid>[.<ext>]. Default: autophone.log')
91b2c03 @markrcote Initial commit.
markrcote authored
1179 parser.add_option('--loglevel', action='store', type='string',
df44529 @bclary Bug 1037067 - Improve initial setup and docs, r=gbrown, r=mcote.
bclary authored
1180 dest='loglevel', default='INFO',
91b2c03 @markrcote Initial commit.
markrcote authored
1181 help='Log level - ERROR, WARNING, DEBUG, or INFO, '
df44529 @bclary Bug 1037067 - Improve initial setup and docs, r=gbrown, r=mcote.
bclary authored
1182 'defaults to INFO')
91b2c03 @markrcote Initial commit.
markrcote authored
1183 parser.add_option('-t', '--test-path', action='store', type='string',
1184 dest='test_path', default='tests/manifest.ini',
1185 help='path to test manifest')
7fb94b3 @bclary bug 1080783 - Autophone - process crash dumps, r=mcote.
bclary authored
1186 parser.add_option('--minidump-stackwalk', action='store', type='string',
4c2c8fb @bclary Bug 1118793 - Autophone - fix leading space in default minidump_stack…
bclary authored
1187 dest='minidump_stackwalk', default='/usr/local/bin/minidump_stackwalk',
7fb94b3 @bclary bug 1080783 - Autophone - process crash dumps, r=mcote.
bclary authored
1188 help='Path to minidump_stackwalk executable; defaults to /usr/local/bin/minidump_stackwalk.')
d82c92e @markrcote Fixed retry loop; reboot is tried 3 times, and each test tried twice.…
markrcote authored
1189 parser.add_option('--emailcfg', action='store', type='string',
5d020e1 @bclary Bug 1037077 - Report run status to treeherder, r=mcote.
bclary authored
1190 dest='emailcfg', default='',
1191 help='config file for email settings; defaults to none')
b6dc95d @bclary Bug 1119654 - Autophone - separate installation specific settings, r=…
bclary authored
1192 parser.add_option('--phonedash-url', action='store', type='string',
1193 dest='phonedash_url', default='',
1194 help='Url to Phonedash server. If not set, results for '
1195 'each device will be written to comma delimited files in '
1196 'the form: autophone-results-<deviceid>.csv.')
1197 parser.add_option('--phonedash-user', action='store', type='string',
1198 dest='phonedash_user', default='',
1199 help='user id for connecting to Phonedash server')
1200 parser.add_option('--phonedash-password', action='store', type='string',
1201 dest='phonedash_password', default='',
1202 help='password for connecting to Phonedash server')
1203 parser.add_option('--webserver-url', action='store', type='string',
1204 dest='webserver_url', default='',
1205 help='Url to web server for remote tests.')
df44529 @bclary Bug 1037067 - Improve initial setup and docs, r=gbrown, r=mcote.
bclary authored
1206 parser.add_option('--enable-pulse', action='store_true',
1207 dest="enable_pulse", default=False,
1208 help='Enable connecting to Pulse to look for new builds. '
1209 'If specified, --pulse-user and --pulse-password must also '
1210 'be specified.')
91b0bca @bclary Bug 1118758 - Autophone - use AutophonePulseBuildMonitor, r=mcote.
bclary authored
1211 parser.add_option('--pulse-durable-queue', action='store_true',
1212 dest="pulse_durable_queue", default=False,
1213 help='Use a durable queue when connecting to Pulse.')
c0af37c @bclary Bug 1080261 - Autophone - upgrade to new pulsebuildmonitor with authe…
bclary authored
1214 parser.add_option('--pulse-user', action='store', type='string',
1215 dest='pulse_user', default='',
1216 help='user id for connecting to PulseGuardian')
1217 parser.add_option('--pulse-password', action='store', type='string',
1218 dest='pulse_password', default='',
1219 help='password for connecting to PulseGuardian')
0bd108b @bclary Bug 1133580 - Autophone - hookup new pulsemonitor, consolidate compat…
bclary authored
1220 parser.add_option('--pulse-jobactions-exchange', action='store', type='string',
1221 dest='pulse_jobactions_exchange',
1222 default='exchange/treeherder/v1/job-actions',
1223 help='Exchange for Pulse Job Actions queue; '
1224 'defaults to exchange/treeherder/v1/job-actions.')
cf68f74 @bclary bug 839167 - autophone - add option for build cache directory for mul…
bclary authored
1225 parser.add_option('--cache-dir', type='string',
1226 dest='cache_dir', default='builds',
1227 help='Use the specified directory as the build '
1228 'cache directory; defaults to builds.')
d1e02c5 @bclary Bug 785086 - Add ability to download unit test prerequisites and to r…
bclary authored
1229 parser.add_option('--override-build-dir', type='string',