/
pilot.py
executable file
·483 lines (371 loc) · 14.2 KB
/
pilot.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
#!/usr/bin/python2.7
# coding=utf-8
#
# The Doomsday Build Pilot
# (c) 2011-2012 Jaakko Keränen <jaakko.keranen@iki.fi>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not: http://www.opensource.org/
## @file pilot.py
##
## The build pilot runs on the autobuilder systems. On the host it manages
## tasks for the builder client systems. It listens on a TCP port for incoming
## queries and actions. On the clients the pilot is run periodically by cron,
## and any new tasks are carried out.
##
## The pilot's responsibility is distributed task management; the autobuild
## script carries out the actual tasks.
import sys
import os
import platform
import subprocess
import pickle
import struct
import time
import SocketServer
import builder.utils
def homeDir():
"""Determines the path of the pilot home directory."""
if 'Windows' in platform.platform():
return os.path.join(os.getenv('USERPROFILE'), '.pilot')
return os.path.join(os.getenv('HOME'), '.pilot')
# Get the configuration.
try:
sys.path.append(homeDir())
import pilotcfg
except ImportError:
print """Configuration needed: pilotcfg.py must be created in ~/.pilot/
pilotcfg.py contains information such as (global variables):
- HOST: pilot server address (for clients)
- PORT: pilot server port
- ID: identifier of the build system
- DISTRIB_DIR: distrib directory path
- EVENTS_DIR: events directory path
- APT_DIR: apt repository path (for Linux systems)
- IGNORED_TASKS: list of tasks to quietly ignore (marked as complete)
The function 'postTaskHook(task)' can be defined for actions to be carried out
after a successful execution of a task."""
sys.exit(1)
APP_NAME = 'Doomsday Build Pilot'
def main():
checkHome()
checkMasterActions()
startNewPilotInstance()
try:
if isServer():
listen()
else:
# Client mode. Check quietly for new tasks.
checkForTasks()
except Exception, x:
# Unexpected problem!
print APP_NAME + ':', x
finally:
endPilotInstance()
def msg(s):
print >> sys.stderr, s
def isServer():
return 'server' in sys.argv
def checkHome():
if not os.path.exists(homeDir()):
raise Exception(".pilot home directory does not exist.")
def branchFileName():
return os.path.join(homeDir(), 'branch')
def currentBranch():
if not os.path.exists(branchFileName()):
return 'master' # default branch
return file(branchFileName(), 'rt').read().strip()
def switchToBranch(branch):
f = file(branchFileName(), 'wt')
print >> f, branch
f.close()
def checkMasterActions():
"""Special master actions."""
if len(sys.argv) < 2: return
if sys.argv[1] == 'new':
assert pilotcfg.ID == 'master'
# Create a new task: new sysid[,sysid]* taskname
target = sys.argv[2]
taskName = sys.argv[3]
if target == 'ALL':
newTask(taskName, allClients=True)
else:
for tgt in target.split(','):
newTask(taskName, forClient=tgt)
sys.exit(0)
if sys.argv[1] == 'finish':
assert pilotcfg.ID == 'master'
# Check for completed tasks.
handleCompletedTasks()
sys.exit(0)
def pidFileName():
if isServer():
return 'server.pid'
else:
return 'client.pid'
def isStale(fn):
"""Files are considered stale after some time has passed."""
age = time.time() - os.stat(fn).st_ctime
if age > 4*60*60:
msg(fn + ' is stale, ignoring it.')
return True
return False
def startNewPilotInstance():
"""A new pilot instance can only be started if one is not already running
on the system. If an existing instance is detected, this one will quit
immediately."""
# Check for an existing pid file.
pid = os.path.join(homeDir(), pidFileName())
if os.path.exists(pid):
if not isStale(pid):
# Cannot start right now -- will be retried later.
sys.exit(0)
print >> file(pid, 'w'), str(os.getpid())
def endPilotInstance():
"""Ends this pilot instance."""
os.remove(os.path.join(homeDir(), pidFileName()))
def listTasks(clientId=None, includeCompleted=True, onlyCompleted=False,
allClients=False):
tasks = []
for name in os.listdir(homeDir()):
fn = os.path.join(homeDir(), name)
if fn == '.' or fn == '..': continue
# All tasks are specific to a client.
if os.path.isdir(fn) and (name == clientId or allClients):
for subname in os.listdir(fn):
# Every task begins with the task prefix.
if not subname.startswith('task_'): continue
subfn = os.path.join(fn, subname)
if not os.path.isdir(subfn):
tasks.append(subname[5:]) # Remove prefix.
if not includeCompleted:
tasks = filter(lambda n: not n.endswith('.done'), tasks)
if onlyCompleted:
tasks = filter(lambda n: isTaskComplete(n), tasks)
tasks.sort()
return tasks
def packs(s):
"""The length of the string is prefixed as a 32-bit integer in network
byte order."""
return struct.pack('!i', len(s)) + s
class ReqHandler(SocketServer.StreamRequestHandler):
"""Handler for requests from clients."""
def handle(self):
try:
bytes = struct.unpack('!i', self.rfile.read(4))[0]
self.request = pickle.loads(self.rfile.read(bytes))
if type(self.request) != dict:
raise Exception("Requests must be of type 'dict'")
self.doRequest()
except Exception, x:
msg('Request failed: ' + str(x))
response = { 'result': 'error', 'error': str(x) }
self.respond(response)
def respond(self, rsp):
self.wfile.write(packs(pickle.dumps(rsp, 2)))
def clientId(self):
if 'id' in self.request:
return self.request['id']
else:
return None
def doRequest(self):
if 'query' in self.request:
self.doQuery()
elif 'action' in self.request:
self.doAction()
else:
raise Exception("Unknown request")
def doQuery(self):
qry = self.request['query']
if qry == 'get_tasks':
# Returns the tasks that a client should work on next.
self.respond({ 'tasks': listTasks(self.clientId(),
includeCompleted=False), 'result': 'ok' })
else:
raise Exception("Unknown query: " + qry)
def doAction(self):
act = self.request['action']
if act == 'complete_task':
completeTask(self.request['task'], self.clientId())
self.respond({ 'result': 'ok', 'did_action': act })
else:
raise Exception("Unknown action: " + act)
def listen():
print APP_NAME + ' starting in server mode (port %i).' % pilotcfg.PORT
server = SocketServer.TCPServer(('0.0.0.0', pilotcfg.PORT), ReqHandler)
server.serve_forever()
def query(q):
"""Sends a query to the server and returns the result."""
import socket
response = None
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect((pilotcfg.HOST, pilotcfg.PORT))
sock.send(packs(pickle.dumps(q, 2)))
bytes = struct.unpack('!i', sock.recv(4))[0]
response = pickle.loads(sock.recv(bytes))
finally:
sock.close()
return response
def checkForTasks():
for task in query({'id': pilotcfg.ID, 'query': 'get_tasks'})['tasks']:
if not doTask(task):
# Ignore this task... (It will be done later.)
continue
# Check for a post-task hook.
if 'postTaskHook' in dir(pilotcfg):
pilotcfg.postTaskHook(task)
# No exception was thrown -- the task was successful.
query({'action': 'complete_task',
'task': task,
'id': pilotcfg.ID})
def doTask(task):
"""Throws an exception if the task fails."""
# Are we supposed to ignore this task?
if 'IGNORED_TASKS' in dir(pilotcfg) and \
task in pilotcfg.IGNORED_TASKS:
return True
if task.startswith('branch_'):
msg("SWITCH TO BRANCH: " + task[7:])
autobuild('pull')
switchToBranch(task[7:])
return autobuild('pull')
elif task.startswith('buildfrom_'):
msg("SWITCH TO BRANCH FOR BUILD: " + task[10:])
autobuild('pull')
switchToBranch(task[10:])
return autobuild('pull')
elif task == 'tag_build':
msg("TAG MASTER BRANCH")
return autobuild('create')
elif task == 'deb_changes':
msg("UPDATE .DEB CHANGELOG")
return autobuild('debchanges')
elif task == 'mac_meta':
msg("OS X METADATA UPDATE")
return autobuild('mac_meta')
elif task == 'build':
msg("BUILD RELEASE")
return autobuild('platform_release')
elif task == 'source':
msg("PACKAGE SOURCE")
return autobuild('source')
elif task == 'sign':
msg("SIGN PACKAGES")
return autobuild('sign')
elif task == 'publish':
msg("PUBLISH")
return autobuild("publish")
elif task == 'apt_refresh':
msg("APT REPOSITORY REFRESH")
return autobuild('apt')
elif task == 'update_feed':
msg("UPDATE FEED")
autobuild('feed')
autobuild('xmlfeed')
elif task == 'purge':
msg("PURGE")
return autobuild('purge')
elif task == 'generate_readme':
msg("GENERATE README")
return autobuild('readme')
elif task == 'generate_apidoc':
msg("GENERATE API DOCUMENTATION")
return autobuild('apidoc')
elif task == 'mirror':
msg("MIRROR")
systemCommand('mirror-buildmaster-to-idisk.sh')
return True
def handleCompletedTasks():
"""Check the completed tasks and see if we should start new tasks."""
while True:
tasks = listTasks(allClients=True, onlyCompleted=True)
if len(tasks) == 0: break
task = tasks[0][:-5] # Remove '.done'
assert isTaskComplete(task)
clearTask(task)
print "Task '%s' has been completed (noticed at %s)" % (task, time.asctime())
if task.startswith('buildfrom_'):
# Commence with a build when everyone is ready.
newTask('tag_build', forClient='master')
elif task == 'tag_build':
newTask('deb_changes', forClient='ubuntu')
newTask('mac_meta', forClient='master')
newTask('generate_readme', forClient='clikits')
elif task == 'deb_changes':
newTask('build', allClients=True)
newTask('generate_wiki', forClient='master')
elif task == 'build':
newTask('source', forClient='ubuntu')
newTask('source', forClient='saucy')
elif task == 'source':
newTask('sign', forClient='master')
elif task == 'sign':
newTask('publish', forClient='master')
newTask('apt_refresh', forClient='ubuntu')
# After the build we can switch to the master again.
newTask('branch_master', allClients=True)
elif task == 'publish':
newTask('update_feed', forClient='master')
elif task == 'update_feed':
newTask('mirror', forClient='master')
def autobuild(cmd):
cmdLine = "%s %s" % (os.path.join(pilotcfg.DISTRIB_DIR, 'autobuild.py'), cmd)
cmdLine += " --distrib %s" % pilotcfg.DISTRIB_DIR
if 'EVENTS_DIR' in dir(pilotcfg):
cmdLine += " --events %s" % pilotcfg.EVENTS_DIR
if 'APT_DIR' in dir(pilotcfg):
cmdLine += " --apt %s" % pilotcfg.APT_DIR
cmdLine += " --branch %s" % currentBranch()
builder.utils.run_python2(cmdLine)
return True
def systemCommand(cmd):
result = subprocess.call(cmd, shell=True)
if result != 0:
raise Exception("Error from " + cmd)
def newTask(name, forClient=None, allClients=False):
if allClients:
for fn in os.listdir(homeDir()):
if os.path.isdir(os.path.join(homeDir(), fn)):
newTask(name, fn)
return
path = os.path.join(homeDir(), forClient)
print "New task '%s' for client '%s'" % (name, forClient)
print >> file(os.path.join(path, 'task_' + name), 'wt'), time.asctime()
def completeTask(name, byClient):
path = os.path.join(homeDir(), byClient, 'task_' + name)
if not os.path.exists(path):
raise Exception("Cannot complete missing task '%s' (by client '%s')" % (name, byClient))
print "Task '%s' completed by '%s' at" % (name, byClient), time.asctime()
os.rename(path, path + '.done')
def clearTask(name, direc=None):
"""Delete all task files with this name."""
if not direc: direc = homeDir()
for fn in os.listdir(direc):
p = os.path.join(direc, fn)
if os.path.isdir(p):
clearTask(name, p)
if fn.startswith('task_') and (fn[5:] == name or \
fn[5:] == name + '.done'):
os.remove(p)
def isTaskComplete(name):
# Remote the possible '.done' suffix.
if name[-5:] == '.done': name = name[:-5]
# Check that everyone has completed it.
for task in listTasks(allClients=True):
if task.startswith(name) and not task.endswith('.done'):
# This one is not complete.
return False
return True
if __name__ == '__main__':
main()