Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Newer
Older
100644 279 lines (220 sloc) 9.362 kB
faf8846 @catlee Bug 628720: Fixes for Pulse plugin. r=dustin
catlee authored
1 """
2 pulse status plugin for buildbot
3 """
1072c41 @catlee Bug 614576: Add PulseStatus implementation to buildbotcustom. r=bhea…
catlee authored
4 import re
5 import os.path
6 import time
faf8846 @catlee Bug 628720: Fixes for Pulse plugin. r=dustin
catlee authored
7 import traceback
1072c41 @catlee Bug 614576: Add PulseStatus implementation to buildbotcustom. r=bhea…
catlee authored
8
9 from twisted.internet.threads import deferToThread
faf8846 @catlee Bug 628720: Fixes for Pulse plugin. r=dustin
catlee authored
10 from twisted.internet.defer import DeferredLock
11 from twisted.internet.task import LoopingCall
12 from twisted.internet import reactor
1072c41 @catlee Bug 614576: Add PulseStatus implementation to buildbotcustom. r=bhea…
catlee authored
13 from twisted.python import log
14
15 from buildbot.status.status_push import StatusPush
c33fd0a @catlee Bug 662885: out-of-process pulse and command handling. r=dustin,bhearsum
catlee authored
16 from buildbot.util import json
1072c41 @catlee Bug 614576: Add PulseStatus implementation to buildbotcustom. r=bhea…
catlee authored
17
62ff0b1 @catlee nobug: PEP8! r=bitrotallthethings
catlee authored
18
1072c41 @catlee Bug 614576: Add PulseStatus implementation to buildbotcustom. r=bhea…
catlee authored
19 def escape(name):
3cc9e50 @catlee Bug 614576: Escape spaces with underscores for pulse events. r=bhearsum
catlee authored
20 return name.replace(".", "_").replace(" ", "_")
1072c41 @catlee Bug 614576: Add PulseStatus implementation to buildbotcustom. r=bhea…
catlee authored
21
62ff0b1 @catlee nobug: PEP8! r=bitrotallthethings
catlee authored
22
faf8846 @catlee Bug 628720: Fixes for Pulse plugin. r=dustin
catlee authored
23 def hexid(obj):
24 return '<%s>' % hex(id(obj))
25
62ff0b1 @catlee nobug: PEP8! r=bitrotallthethings
catlee authored
26
1072c41 @catlee Bug 614576: Add PulseStatus implementation to buildbotcustom. r=bhea…
catlee authored
27 class PulseStatus(StatusPush):
faf8846 @catlee Bug 628720: Fixes for Pulse plugin. r=dustin
catlee authored
28 """
29 Status pusher for Mozilla Pulse (an AMQP broker).
30
c33fd0a @catlee Bug 662885: out-of-process pulse and command handling. r=dustin,bhearsum
catlee authored
31 Sends all events regarding activity on this master to the given queue,
32 which should be a buildbotcustom.status.queue.QueueDir instance.
faf8846 @catlee Bug 628720: Fixes for Pulse plugin. r=dustin
catlee authored
33
34 `ignoreBuilders`, if set, should be a list of strings or compiled regular
35 expressions of builders that should *NOT* be watched.
36
37 `send_logs`, if set, will enable sending log chunks to the message broker.
38
39 `heartbeat_time` (default 900) is how often we generate heartbeat events.
40 """
41
c33fd0a @catlee Bug 662885: out-of-process pulse and command handling. r=dustin,bhearsum
catlee authored
42 compare_attrs = StatusPush.compare_attrs + ['queuedir', 'ignoreBuilders',
62ff0b1 @catlee nobug: PEP8! r=bitrotallthethings
catlee authored
43 'send_logs']
1072c41 @catlee Bug 614576: Add PulseStatus implementation to buildbotcustom. r=bhea…
catlee authored
44
c33fd0a @catlee Bug 662885: out-of-process pulse and command handling. r=dustin,bhearsum
catlee authored
45 def __init__(self, queuedir, ignoreBuilders=None, send_logs=False,
62ff0b1 @catlee nobug: PEP8! r=bitrotallthethings
catlee authored
46 heartbeat_time=900):
c33fd0a @catlee Bug 662885: out-of-process pulse and command handling. r=dustin,bhearsum
catlee authored
47 self.queuedir = queuedir
1072c41 @catlee Bug 614576: Add PulseStatus implementation to buildbotcustom. r=bhea…
catlee authored
48 self.send_logs = send_logs
faf8846 @catlee Bug 628720: Fixes for Pulse plugin. r=dustin
catlee authored
49
1072c41 @catlee Bug 614576: Add PulseStatus implementation to buildbotcustom. r=bhea…
catlee authored
50 self.ignoreBuilders = []
51 if ignoreBuilders:
52 for i in ignoreBuilders:
53 if isinstance(i, basestring):
54 self.ignoreBuilders.append(re.compile(i))
55 else:
56 assert hasattr(i, 'match') and callable(i.match)
57 self.ignoreBuilders.append(i)
58 self.watched = []
faf8846 @catlee Bug 628720: Fixes for Pulse plugin. r=dustin
catlee authored
59
60 # Set up heartbeat
c33fd0a @catlee Bug 662885: out-of-process pulse and command handling. r=dustin,bhearsum
catlee authored
61 self.heartbeat_time = heartbeat_time
faf8846 @catlee Bug 628720: Fixes for Pulse plugin. r=dustin
catlee authored
62 self._heartbeat_loop = LoopingCall(self.heartbeat)
63
c33fd0a @catlee Bug 662885: out-of-process pulse and command handling. r=dustin,bhearsum
catlee authored
64 # Wait 10 seconds before sending our stuff
65 self.push_delay = 10
66 self.delayed_push = None
67
1072c41 @catlee Bug 614576: Add PulseStatus implementation to buildbotcustom. r=bhea…
catlee authored
68 StatusPush.__init__(self, PulseStatus.pushEvents, filter=False)
69
faf8846 @catlee Bug 628720: Fixes for Pulse plugin. r=dustin
catlee authored
70 def setServiceParent(self, parent):
71 StatusPush.setServiceParent(self, parent)
72
73 # Start heartbeat
74 # This should be done in startService, but our base class isn't
75 # behaving
76 # See http://trac.buildbot.net/ticket/1793
77 self._heartbeat_loop.start(self.heartbeat_time)
78
79 # Stubbed out for later
62ff0b1 @catlee nobug: PEP8! r=bitrotallthethings
catlee authored
80 # def startService(self):
81 # self._heartbeat_loop.start(self.heartbeat_time)
82 # return StatusPush.startService(self)
faf8846 @catlee Bug 628720: Fixes for Pulse plugin. r=dustin
catlee authored
83
84 def stopService(self):
85 log.msg("Pulse %s: shutting down" % (hexid(self),))
86 if self._heartbeat_loop.running:
87 self._heartbeat_loop.stop()
88
1072c41 @catlee Bug 614576: Add PulseStatus implementation to buildbotcustom. r=bhea…
catlee authored
89 self.status.unsubscribe(self)
90 for w in self.watched:
91 w.unsubscribe(self)
faf8846 @catlee Bug 628720: Fixes for Pulse plugin. r=dustin
catlee authored
92
c33fd0a @catlee Bug 662885: out-of-process pulse and command handling. r=dustin,bhearsum
catlee authored
93 # Write out any pending events
94 if self.delayed_push:
95 self.delayed_push.cancel()
96 self.delayed_push = None
faf8846 @catlee Bug 628720: Fixes for Pulse plugin. r=dustin
catlee authored
97
c33fd0a @catlee Bug 662885: out-of-process pulse and command handling. r=dustin,bhearsum
catlee authored
98 while self.queue.nbItems() > 0:
99 self._do_push()
100 return StatusPush.stopService(self)
1072c41 @catlee Bug 614576: Add PulseStatus implementation to buildbotcustom. r=bhea…
catlee authored
101
102 def pushEvents(self):
c33fd0a @catlee Bug 662885: out-of-process pulse and command handling. r=dustin,bhearsum
catlee authored
103 """Trigger a push"""
104 if not self.delayed_push:
62ff0b1 @catlee nobug: PEP8! r=bitrotallthethings
catlee authored
105 self.delayed_push = reactor.callLater(
106 self.push_delay, self._do_push)
c33fd0a @catlee Bug 662885: out-of-process pulse and command handling. r=dustin,bhearsum
catlee authored
107
108 def _do_push(self):
faf8846 @catlee Bug 628720: Fixes for Pulse plugin. r=dustin
catlee authored
109 """Push some events to pulse"""
c33fd0a @catlee Bug 662885: out-of-process pulse and command handling. r=dustin,bhearsum
catlee authored
110 self.delayed_push = None
faf8846 @catlee Bug 628720: Fixes for Pulse plugin. r=dustin
catlee authored
111
112 # Get the events
c33fd0a @catlee Bug 662885: out-of-process pulse and command handling. r=dustin,bhearsum
catlee authored
113 events = self.queue.popChunk()
114
115 # Nothing to do!
116 if not events:
117 return
118
119 # List of json-encodable messages to write to queuedir
120 to_write = []
121 start = time.time()
122 count = 0
123 heartbeats = 0
124 for e in events:
125 count += 1
126 if e['event'] == 'heartbeat':
127 heartbeats += 1
128
129 e['master_name'] = self.status.botmaster.master_name
130 e['master_incarnation'] = \
62ff0b1 @catlee nobug: PEP8! r=bitrotallthethings
catlee authored
131 self.status.botmaster.master_incarnation
c33fd0a @catlee Bug 662885: out-of-process pulse and command handling. r=dustin,bhearsum
catlee authored
132
133 # Transform time tuples to standard pulse time format
134 to_write.append(e)
135 end = time.time()
136 log.msg("Pulse %s: Processed %i events (%i heartbeats) "
62ff0b1 @catlee nobug: PEP8! r=bitrotallthethings
catlee authored
137 "in %.2f seconds" %
75e3f93 @edmorley Bug 1074826 - Clean up some pyflakes & PEP8 warnings in buildbotcusto…
edmorley authored
138 (hexid(self), count, heartbeats, (end - start)))
c33fd0a @catlee Bug 662885: out-of-process pulse and command handling. r=dustin,bhearsum
catlee authored
139 try:
140 self.queuedir.add(json.dumps(to_write))
141 except:
142 # Try again later?
143 self.queue.insertBackChunk(events)
144 log.err()
145
146 # If we still have more stuff, send it in a bit
147 if self.queue.nbItems() > 0:
148 self.pushEvents()
1072c41 @catlee Bug 614576: Add PulseStatus implementation to buildbotcustom. r=bhea…
catlee authored
149
150 def builderAdded(self, builderName, builder):
151 if self.stopped:
152 return None
153
154 for i in self.ignoreBuilders:
155 try:
156 if i.match(builderName):
157 return None
158 except:
159 log.msg("Exception matching builderName with %s" % str(i))
faf8846 @catlee Bug 628720: Fixes for Pulse plugin. r=dustin
catlee authored
160 log.msg("Shutting down PulseStatus - "
62ff0b1 @catlee nobug: PEP8! r=bitrotallthethings
catlee authored
161 "not sure if we're ignoring the right stuff!")
1072c41 @catlee Bug 614576: Add PulseStatus implementation to buildbotcustom. r=bhea…
catlee authored
162 log.err()
163 self.stopService()
164 if self.parent:
165 self.disownServiceParent()
166 self.watched.append(builder)
167 return self
168
169 def _translateBuilderName(self, builderName):
170 builder = self.status.getBuilder(builderName)
171 return os.path.basename(builder.basedir)
172
faf8846 @catlee Bug 628720: Fixes for Pulse plugin. r=dustin
catlee authored
173 def heartbeat(self):
174 """send a heartbeat event"""
175 # We're called from inside a LoopingCall, so make sure we never leak an
176 # exception up to it, otherwise we'll never be called again!!!
177 # OH NOES!
178 try:
179 log.msg("Pulse %s: heartbeat" % (hexid(self),))
180 self.push("heartbeat")
181 except:
182 log.msg("Pulse %s: failed to send heartbeat" % (hexid(self),))
183 log.err()
184
75e3f93 @edmorley Bug 1074826 - Clean up some pyflakes & PEP8 warnings in buildbotcusto…
edmorley authored
185 # Events we publish #
faf8846 @catlee Bug 628720: Fixes for Pulse plugin. r=dustin
catlee authored
186
1072c41 @catlee Bug 614576: Add PulseStatus implementation to buildbotcustom. r=bhea…
catlee authored
187 def buildStarted(self, builderName, build):
188 builderName = escape(self._translateBuilderName(builderName))
faf8846 @catlee Bug 628720: Fixes for Pulse plugin. r=dustin
catlee authored
189 self.push("build.%s.%i.started" % (builderName, build.number),
62ff0b1 @catlee nobug: PEP8! r=bitrotallthethings
catlee authored
190 build=build)
1072c41 @catlee Bug 614576: Add PulseStatus implementation to buildbotcustom. r=bhea…
catlee authored
191 return self
192
193 def buildFinished(self, builderName, build, results):
194 builderName = escape(self._translateBuilderName(builderName))
faf8846 @catlee Bug 628720: Fixes for Pulse plugin. r=dustin
catlee authored
195 self.push("build.%s.%i.finished" % (builderName, build.number),
62ff0b1 @catlee nobug: PEP8! r=bitrotallthethings
catlee authored
196 build=build, results=results)
1072c41 @catlee Bug 614576: Add PulseStatus implementation to buildbotcustom. r=bhea…
catlee authored
197
198 def slaveConnected(self, slavename):
faf8846 @catlee Bug 628720: Fixes for Pulse plugin. r=dustin
catlee authored
199 self.push("slave.%s.connected" % escape(slavename),
62ff0b1 @catlee nobug: PEP8! r=bitrotallthethings
catlee authored
200 slave=self.status.getSlave(slavename))
1072c41 @catlee Bug 614576: Add PulseStatus implementation to buildbotcustom. r=bhea…
catlee authored
201
202 def slaveDisconnected(self, slavename):
faf8846 @catlee Bug 628720: Fixes for Pulse plugin. r=dustin
catlee authored
203 self.push("slave.%s.disconnected" % escape(slavename),
62ff0b1 @catlee nobug: PEP8! r=bitrotallthethings
catlee authored
204 slavename=slavename)
1072c41 @catlee Bug 614576: Add PulseStatus implementation to buildbotcustom. r=bhea…
catlee authored
205
206 def changeAdded(self, change):
207 self.push("change.%i.added" % change.number, change=change)
208
209 def requestSubmitted(self, request):
faf8846 @catlee Bug 628720: Fixes for Pulse plugin. r=dustin
catlee authored
210 builderName = escape(self._translateBuilderName(
211 request.getBuilderName()))
1072c41 @catlee Bug 614576: Add PulseStatus implementation to buildbotcustom. r=bhea…
catlee authored
212 self.push("request.%s.submitted" % builderName, request=request)
213
214 def requestCancelled(self, builder, request):
215 builderName = escape(self._translateBuilderName(builder.name))
216 self.push("request.%s.cancelled" % builderName, request=request)
217
218 def stepStarted(self, build, step):
219 builderName = escape(self._translateBuilderName(build.builder.name))
faf8846 @catlee Bug 628720: Fixes for Pulse plugin. r=dustin
catlee authored
220 self.push("build.%s.%i.step.%s.started" %
75e3f93 @edmorley Bug 1074826 - Clean up some pyflakes & PEP8 warnings in buildbotcusto…
edmorley authored
221 (builderName, build.number, escape(step.name)),
62ff0b1 @catlee nobug: PEP8! r=bitrotallthethings
catlee authored
222 properties=build.getProperties().asList(),
223 step=step)
faf8846 @catlee Bug 628720: Fixes for Pulse plugin. r=dustin
catlee authored
224 # If logging is enabled, return ourself to subscribe to log events for
225 # this step
1072c41 @catlee Bug 614576: Add PulseStatus implementation to buildbotcustom. r=bhea…
catlee authored
226 if self.send_logs:
227 return self
228
229 def stepFinished(self, build, step, results):
230 builderName = escape(self._translateBuilderName(build.builder.name))
faf8846 @catlee Bug 628720: Fixes for Pulse plugin. r=dustin
catlee authored
231 self.push("build.%s.%i.step.%s.finished" %
75e3f93 @edmorley Bug 1074826 - Clean up some pyflakes & PEP8 warnings in buildbotcusto…
edmorley authored
232 (builderName, build.number, escape(step.name)),
62ff0b1 @catlee nobug: PEP8! r=bitrotallthethings
catlee authored
233 properties=build.getProperties().asList(),
234 step=step,
235 results=results)
1072c41 @catlee Bug 614576: Add PulseStatus implementation to buildbotcustom. r=bhea…
catlee authored
236
75e3f93 @edmorley Bug 1074826 - Clean up some pyflakes & PEP8 warnings in buildbotcusto…
edmorley authored
237 # Optional logging events #
faf8846 @catlee Bug 628720: Fixes for Pulse plugin. r=dustin
catlee authored
238
1072c41 @catlee Bug 614576: Add PulseStatus implementation to buildbotcustom. r=bhea…
catlee authored
239 def logStarted(self, build, step, log):
240 builderName = escape(self._translateBuilderName(build.builder.name))
faf8846 @catlee Bug 628720: Fixes for Pulse plugin. r=dustin
catlee authored
241 self.push("build.%s.%i.step.%s.log.%s.started" %
75e3f93 @edmorley Bug 1074826 - Clean up some pyflakes & PEP8 warnings in buildbotcusto…
edmorley authored
242 (builderName, build.number, escape(step.name), log.name))
1072c41 @catlee Bug 614576: Add PulseStatus implementation to buildbotcustom. r=bhea…
catlee authored
243 return self
244
245 def logChunk(self, build, step, log, channel, text):
246 # TODO: Strip out bad UTF-8 characters
247 builderName = escape(self._translateBuilderName(build.builder.name))
faf8846 @catlee Bug 628720: Fixes for Pulse plugin. r=dustin
catlee authored
248 self.push("build.%s.%i.step.%s.log.%s.chunk" %
75e3f93 @edmorley Bug 1074826 - Clean up some pyflakes & PEP8 warnings in buildbotcusto…
edmorley authored
249 (builderName, build.number, escape(step.name), log.name),
62ff0b1 @catlee nobug: PEP8! r=bitrotallthethings
catlee authored
250 channel=channel,
251 text=text,
252 )
1072c41 @catlee Bug 614576: Add PulseStatus implementation to buildbotcustom. r=bhea…
catlee authored
253
254 def logFinished(self, build, step, log):
255 builderName = escape(self._translateBuilderName(build.builder.name))
faf8846 @catlee Bug 628720: Fixes for Pulse plugin. r=dustin
catlee authored
256 self.push("build.%s.%i.step.%s.log.%s.finished" %
75e3f93 @edmorley Bug 1074826 - Clean up some pyflakes & PEP8 warnings in buildbotcusto…
edmorley authored
257 (builderName, build.number, escape(step.name), log.name))
1072c41 @catlee Bug 614576: Add PulseStatus implementation to buildbotcustom. r=bhea…
catlee authored
258 return self
259
75e3f93 @edmorley Bug 1074826 - Clean up some pyflakes & PEP8 warnings in buildbotcusto…
edmorley authored
260 # Events we ignore #
faf8846 @catlee Bug 628720: Fixes for Pulse plugin. r=dustin
catlee authored
261
1072c41 @catlee Bug 614576: Add PulseStatus implementation to buildbotcustom. r=bhea…
catlee authored
262 def buildsetSubmitted(self, buildset):
263 pass
264
265 def builderChangedState(self, builderName, state):
266 pass
267
268 def builderRemoved(self, builderName):
269 pass
270
271 def stepETAUpdate(self, build, step, ETA, expectations):
272 pass
273
274 def stepTextChanged(self, build, step, text):
275 pass
276
277 def stepText2Changed(self, build, step, text2):
278 pass
Something went wrong with that request. Please try again.