From 3c1652acf0199c16fdb9c43b49b79ed8f310e26b Mon Sep 17 00:00:00 2001 From: Pierre Tardy Date: Sun, 1 Dec 2013 14:09:51 +0100 Subject: [PATCH 1/2] sse: add possibility to control existing connection unit test, doc and coherency with websocket api Signed-off-by: Pierre Tardy --- .../buildbot/test/unit/test_data_changes.py | 53 +++---- master/buildbot/test/unit/test_www_sse.py | 127 +++++++++++++++++ master/buildbot/test/util/www.py | 13 +- master/buildbot/www/sse.py | 131 ++++++++++++++---- master/docs/developer/www.rst | 52 ++++++- 5 files changed, 315 insertions(+), 61 deletions(-) create mode 100644 master/buildbot/test/unit/test_www_sse.py diff --git a/master/buildbot/test/unit/test_data_changes.py b/master/buildbot/test/unit/test_data_changes.py index 21f0e007b55..3a34c163bc5 100644 --- a/master/buildbot/test/unit/test_data_changes.py +++ b/master/buildbot/test/unit/test_data_changes.py @@ -99,6 +99,32 @@ def test_startConsuming(self): class Change(interfaces.InterfaceTests, unittest.TestCase): + changeEvent = { + 'author': u'warner', + 'branch': u'warnerdb', + 'category': u'devel', + 'codebase': u'', + 'comments': u'fix whitespace', + 'changeid': 500, + 'files': [u'master/buildbot/__init__.py'], + 'project': u'Buildbot', + 'properties': {u'foo': (20, u'Change')}, + 'repository': u'git://warner', + 'revision': u'0e92a098b', + 'revlink': u'http://warner/0e92a098b', + 'when_timestamp': 256738404, + 'sourcestamp': { + 'branch': u'warnerdb', + 'codebase': u'', + 'patch': None, + 'project': u'Buildbot', + 'repository': u'git://warner', + 'revision': u'0e92a098b', + 'created_at': 10000000, + 'ssid': 100, + }, + # uid + } def setUp(self): self.master = fakemaster.make_master(wantMq=True, wantDb=True, @@ -144,32 +170,7 @@ def test_addChange(self): when_timestamp=256738404, properties={u'foo': 20}) expectedRoutingKey = ('change', '500', 'new') - expectedMessage = { - 'author': u'warner', - 'branch': u'warnerdb', - 'category': u'devel', - 'codebase': u'', - 'comments': u'fix whitespace', - 'changeid': 500, - 'files': [u'master/buildbot/__init__.py'], - 'project': u'Buildbot', - 'properties': {u'foo': (20, u'Change')}, - 'repository': u'git://warner', - 'revision': u'0e92a098b', - 'revlink': u'http://warner/0e92a098b', - 'when_timestamp': 256738404, - 'sourcestamp': { - 'branch': u'warnerdb', - 'codebase': u'', - 'patch': None, - 'project': u'Buildbot', - 'repository': u'git://warner', - 'revision': u'0e92a098b', - 'created_at': 10000000, - 'ssid': 100, - }, - # uid - } + expectedMessage = self.changeEvent expectedRow = fakedb.Change( changeid=500, author='warner', diff --git a/master/buildbot/test/unit/test_www_sse.py b/master/buildbot/test/unit/test_www_sse.py new file mode 100644 index 00000000000..62f96597142 --- /dev/null +++ b/master/buildbot/test/unit/test_www_sse.py @@ -0,0 +1,127 @@ +# This file is part of Buildbot. Buildbot is free software: you can +# redistribute it and/or modify it under the terms of the GNU General Public +# License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 51 +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright Buildbot Team Members + +from buildbot.test.unit import test_data_changes +from buildbot.test.util import www +from buildbot.util import json +from buildbot.www import sse +from twisted.trial import unittest + + +class EventResource(www.WwwTestMixin, unittest.TestCase): + + def setUp(self): + self.master = master = self.make_master(url='h:/a/b/') + self.sse = sse.EventResource(master) + + def test_oldapi(self): + self.render_resource(self.sse, '/change') + self.readUUID(self.request) + self.assertReceivesChangeNewMessage(self.request) + self.assertEqual(self.request.finished, False) + + def test_listen(self): + self.render_resource(self.sse, '/listen/change') + self.readUUID(self.request) + self.assertReceivesChangeNewMessage(self.request) + self.assertEqual(self.request.finished, False) + + def test_listen_add_then_close(self): + self.render_resource(self.sse, '/listen') + request = self.request + self.request = None + uuid = self.readUUID(request) + self.render_resource(self.sse, '/add/' + uuid + "/change") + self.assertReceivesChangeNewMessage(request) + self.assertEqual(self.request.finished, True) + self.assertEqual(request.finished, False) + request.finish() # fake close connection on client side + self.assertRaises(AssertionError, self.assertReceivesChangeNewMessage, request) + + def test_listen_add_then_remove(self): + self.render_resource(self.sse, '/listen') + request = self.request + uuid = self.readUUID(request) + self.render_resource(self.sse, '/add/' + uuid + "/change") + self.assertReceivesChangeNewMessage(request) + self.assertEqual(request.finished, False) + self.render_resource(self.sse, '/remove/' + uuid + "/change") + self.assertRaises(AssertionError, self.assertReceivesChangeNewMessage, request) + + def test_listen_add_nouuid(self): + self.render_resource(self.sse, '/listen') + request = self.request + self.readUUID(request) + self.render_resource(self.sse, '/add/') + self.assertEqual(self.request.finished, True) + self.assertEqual(self.request.responseCode, 400) + self.assertIn("need uuid", self.request.written) + + def test_listen_add_baduuid(self): + self.render_resource(self.sse, '/listen') + request = self.request + self.readUUID(request) + self.render_resource(self.sse, '/add/foo') + self.assertEqual(self.request.finished, True) + self.assertEqual(self.request.responseCode, 400) + self.assertIn("unknown uuid", self.request.written) + + def test_listen_add_badevent(self): + self.render_resource(self.sse, '/listen') + request = self.request + uuid = self.readUUID(request) + self.render_resource(self.sse, '/add/' + uuid + '/foo') + self.assertEqual(self.request.finished, True) + self.assertEqual(self.request.responseCode, 404) + self.assertIn("not implemented", self.request.written) + + def test_listen_add_then_remove_bad_event(self): + self.render_resource(self.sse, '/listen') + request = self.request + uuid = self.readUUID(request) + self.render_resource(self.sse, '/add/' + uuid + "/change") + self.assertReceivesChangeNewMessage(request) + self.assertEqual(request.finished, False) + self.render_resource(self.sse, '/remove/' + uuid + "/foo") + self.assertEqual(self.request.responseCode, 404) + self.assertIn("consumer is not listening to this event", self.request.written) + + def readEvent(self, request): + kw = {} + hasEmptyLine = False + for line in request.written.splitlines(): + if line.find(":") > 0: + k, v = line.split(": ", 1) + self.assertTrue(k not in kw, k + " in " + str(kw)) + kw[k] = v + else: + self.assertEqual(line, "") + hasEmptyLine = True + request.written = "" + self.assertTrue(hasEmptyLine) + return kw + + def readUUID(self, request): + kw = self.readEvent(request) + self.assertEqual(kw["event"], "handshake") + return kw["data"] + + def assertReceivesChangeNewMessage(self, request): + self.master.mq.callConsumer(("change", "500", "new"), test_data_changes.Change.changeEvent) + kw = self.readEvent(request) + self.assertEqual(kw["event"], "event") + msg = json.loads(kw["data"]) + self.assertEqual(msg["key"], [u'change', u'500', u'new']) + self.assertEqual(msg["message"], json.loads(json.dumps(test_data_changes.Change.changeEvent))) diff --git a/master/buildbot/test/util/www.py b/master/buildbot/test/util/www.py index cd2f01418c5..770241b45ae 100644 --- a/master/buildbot/test/util/www.py +++ b/master/buildbot/test/util/www.py @@ -75,6 +75,15 @@ def getHeader(self, key): def processingFailed(self, f): self.deferred.errback(f) + def notifyFinish(self): + d = defer.Deferred() + + @self.deferred.addBoth + def finished(res): + d.callback(res) + return res + return d + class RequiresWwwMixin(object): # mix this into a TestCase to skip if buildbot-www is not installed @@ -116,7 +125,9 @@ def render_resource(self, rsrc, path='/', accept=None, method='GET', rv = rsrc.render(request) if rv != server.NOT_DONE_YET: - return defer.succeed(rv) + if rv is not None: + request.write(rv) + request.finish() return request.deferred def render_control_resource(self, rsrc, path='/', params={}, diff --git a/master/buildbot/www/sse.py b/master/buildbot/www/sse.py index 65100b7850f..a912959270f 100644 --- a/master/buildbot/www/sse.py +++ b/master/buildbot/www/sse.py @@ -13,46 +13,121 @@ # # Copyright Team Members +import uuid + +from buildbot.data.exceptions import InvalidPathError from buildbot.util import json from twisted.web import resource from twisted.web import server +class Consumer(object): + + def __init__(self, request): + self.request = request + self.qrefs = {} + + def stopConsuming(self, key=None): + if key is not None: + self.qrefs[key].stopConsuming() + else: + for qref in self.qrefs.values(): + qref.stopConsuming() + self.qrefs = {} + + def onMessage(self, event, data): + request = self.request + msg = dict(key=event, message=data) + request.write("event: " + "event" + "\n") + request.write("data: " + json.dumps(msg) + "\n") + request.write("\n") + + def registerQref(self, path, qref): + self.qrefs[path] = qref + + class EventResource(resource.Resource): isLeaf = True def __init__(self, master): self.master = master + self.consumers = {} + + def decodePath(self, path): + for i, p in enumerate(path): + if p == '*': + path[i] = None + return path + + def finish(self, request, code, msg): + request.setResponseCode(code) + request.setHeader('content-type', 'text/plain; charset=utf-8') + request.write(msg) + return def render(self, request): + command = "listen" path = request.postpath if path and path[-1] == '': path = path[:-1] - for i, p in enumerate(path): - if p == '*': - path[i] = None - options = request.args - for k in options: - if len(options[k]) == 1: - options[k] = options[k][1] - try: - qref = self.master.data.startConsuming( - (lambda key, msg: self._sendEvent(request, key, msg)), - options, path) - except NotImplementedError: - request.setResponseCode(404) - request.setHeader('content-type', 'text/plain; charset=utf-8') - request.write("unimplemented") - return - request.setHeader("content-type", "text/event-stream") - request.write("") - d = request.notifyFinish() - d.addBoth(lambda _: qref.stopConsuming()) - return server.NOT_DONE_YET - - def _sendEvent(self, request, event, data): - # FIXME - #request.write("event: " + '.'.join(event) + "\n") - request.write("event: " + 'event' + "\n") - request.write("data: " + json.dumps(data) + "\n") - request.write("\n") + + if path and path[0] in ("listen", "add", "remove"): + command = path[0] + path = path[1:] + + if command == "listen": + cid = str(uuid.uuid4()) + consumer = Consumer(request) + + elif command == "add" or command == "remove": + if path: + cid = path[0] + path = path[1:] + if not cid in self.consumers: + return self.finish(request, 400, "unknown uuid") + consumer = self.consumers[cid] + else: + return self.finish(request, 400, "need uuid") + + pathref = "/".join(path) + path = self.decodePath(path) + + if command == "add" or (command == "listen" and path): + options = request.args + for k in options: + if len(options[k]) == 1: + options[k] = options[k][1] + + try: + qref = self.master.data.startConsuming( + consumer.onMessage, + options, tuple(path)) + consumer.registerQref(pathref, qref) + except NotImplementedError: + return self.finish(request, 404, "not implemented") + except InvalidPathError: + return self.finish(request, 404, "not implemented") + elif command == "remove": + try: + consumer.stopConsuming(pathref) + except KeyError: + return self.finish(request, 404, "consumer is not listening to this event") + + if command == "listen": + self.consumers[cid] = consumer + request.setHeader("content-type", "text/event-stream") + request.write("") + request.write("event: handshake\n") + request.write("data: " + cid + "\n") + request.write("\n") + d = request.notifyFinish() + + @d.addBoth + def onEndRequest(_): + consumer.stopConsuming() + del self.consumers[cid] + + return server.NOT_DONE_YET + + self.finish(request, 200, "ok") + return diff --git a/master/docs/developer/www.rst b/master/docs/developer/www.rst index 69b1bcb38c0..fb056acfca9 100644 --- a/master/docs/developer/www.rst +++ b/master/docs/developer/www.rst @@ -288,15 +288,55 @@ Message API ----------- Currently messages are implemented with two protocols: WebSockets and `server sent event `_. -This will likely change or be supplemented with other mechanisms before release. +This may be supplemented with other mechanisms before release. + +WebSocket +~~~~~~~~~ WebSocket is a protocol for arbitrary messaging to and from browser. -As an HTTP extension, the protocol is not yet well supported by all HTTP proxy technologies, and thus not well suited for enterprise. -Only one WebSocket connection is needed per browser. +As an HTTP extension, the protocol is not yet well supported by all HTTP proxy technologies. Although, it has been reported to work well used behind the https protocol. Only one WebSocket connection is needed per browser. + +Client can connect using url ``ws[s]:///ws`` + +The client can control which kind of messages he will receive using following message, encoded in json: + + * startConsuming: {'req': 'startConsuming', 'options': {}, 'path': ['change']} + startConsuming events that match ``path``. + + * stopConsuming: {'req': 'stopConsuming', 'path': ['change']} + stopConsuming events that match ``path`` + +Client will receive events as websocket frames encoded in json with following format: + + {'key':key, 'message':message} + +Server Sent Events +~~~~~~~~~~~~~~~~~~ + +SSE is a simpler protocol than WebSockets and is more REST compliant. It uses the chunk-encoding HTTP feature to stream the events. SSE also does not works well behind enterprise proxy, unless you use the https protocol + +Client can connect using following endpoints + + * ``http[s]:///sse/listen/``: Start listening to events on the http connection. Optionally setup a first event filter on ````. The first message send is a handshake, giving a uuid that can be used to add or remove event filters. + * ``http[s]:///sse/add//``: Configure a sse session to add an event filter + * ``http[s]:///sse/remote//``: Configure a sse session to remove an event filter + +Note that if a load balancer is setup as a front end to buildbot web masters, the load balancer must be configured to always use the same master given a client ip address for /sse endpoint. + +Client will receive events as sse events, encoded with following format: + +.. code-block:: none + + event: event + data: {'key': , 'message': } + +The first event received is a handshake, and is used to inform the client about uuid to use for configuring additional event filters + +.. code-block:: none + + event: handshake + data: -SSE is a simpler protocol than WebSockets and is more REST compliant. -It uses the chunk-encoding HTTP feature to stream the events. -It may use one connection to server per event type. JavaScript Application ---------------------- From 56381a38f081b19db0759009819b41ad35dd6a75 Mon Sep 17 00:00:00 2001 From: Pierre Tardy Date: Sun, 1 Dec 2013 14:10:00 +0100 Subject: [PATCH 2/2] fix validate on osx --- common/validate.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/common/validate.sh b/common/validate.sh index 7dabb13c391..5429a1ed743 100755 --- a/common/validate.sh +++ b/common/validate.sh @@ -76,7 +76,6 @@ check_relnotes() { return 0 fi } - run_tests() { if [ -n "${TRIALTMP}" ]; then TEMP_DIRECTORY_OPT="--temp-directory ${TRIALTMP}" @@ -96,8 +95,8 @@ fi # get a list of changed files, used below; this uses a tempfile to work around # shell behavior when piping to 'while' -tempfile=$(mktemp) -trap 'rm -f ${tempfile}' 1 2 3 15 +tempfile=$(mktemp -t bbvalidate) +trap "rm -f ${tempfile}" 1 2 3 15 git diff --name-only $REVRANGE | grep '\.py$' | grep -v '\(^master/\(contrib\|docs\)\|/setup\.py\)' > ${tempfile} py_files=() while read line; do @@ -109,9 +108,11 @@ git log "$REVRANGE" --pretty=oneline || exit 1 if $slow; then status "running 'setup.py develop' for www" - (cd www; python setup.py develop 2>&1 >/dev/null) || not_ok "www/setup.py failed" + (cd www; python setup.py develop 2>&1 >/dev/null ) || not_ok "www/setup.py failed" status "running 'grunt ci' for www" - (cd www; node_modules/.bin/grunt ci 2>&1 >/dev/null) || not_ok "grunt ci failed" + LOG=/dev/null + if [[ `uname` == "Darwin" ]] ;then LOG=/dev/stdout; fi # grunt >/dev/null hangs on osx ?! + (cd www; node_modules/.bin/grunt --no-color ci 2>&1 >$LOG ) || not_ok "grunt ci failed" fi if $slow; then status "running tests"