rest: implement http proxy to data.control api

We use either jsonrpc2 or POST urlencoded to pass params

Signed-off-by: Pierre Tardy <pierre.tardy@intel.com>

master/buildbot/data/exceptions.py

@@ -23,3 +23,6 @@ class InvalidPathError(DataException):
 class InvalidOptionException(DataException):
     "An option was invalid"
     pass
+class InvalidActionException(DataException):
+    "Action is not supported"
+    pass

master/buildbot/test/unit/test_www_rest.py

@@ -13,8 +13,10 @@
 #
 # Copyright Buildbot Team Members
 
+from cStringIO import StringIO
 from buildbot.www import rest
 from buildbot.test.util import www
+from buildbot.util import json
 from buildbot.data import exceptions
 from twisted.trial import unittest
 from twisted.internet import defer
@@ -61,6 +63,16 @@ def get(options, path):
                 rv['path'] = path
                 return defer.succeed(rv)
         self.master.data.get = get
+        def control(action, args, path):
+            if path == ('not', 'found'):
+                return defer.fail(exceptions.InvalidPathError())
+            elif action == "notfound":
+                return defer.fail(exceptions.InvalidActionException())
+            else:
+                rv = dict(orig_args=args.copy(),
+                          path = path)
+                return defer.succeed(rv)
+        self.master.data.control = control
         self.rsrc = rest.V2RootResource(self.master)
 
     def test_not_found(self):
@@ -128,3 +140,105 @@ def check(_):
             self.assertRequest(content='mycb({"path":["cb"]});',
                                responseCode=200)
         return d
+    def test_control_not_found(self):
+        d = self.render_control_resource(self.rsrc, ['not', 'found'],{"action":["test"]},
+                                         jsonRpc=False)
+        @d.addCallback
+        def check(_):
+            self.assertRequest(
+                contentJson=dict(error='invalid path'),
+                contentType='text/plain',
+                responseCode=404)
+        return d
+
+    def test_control_no_action(self):
+        d = self.render_control_resource(self.rsrc, ['not', 'found'], jsonRpc=False)
+        @d.addCallback
+        def check(_):
+            self.assertRequest(
+                contentJson=dict(error='need an action parameter for POST'),
+                contentType='text/plain',
+                responseCode=400)
+        return d
+
+    def test_control_urlencoded(self):
+        d = self.render_control_resource(self.rsrc, ['path'],{"action":["test"],"param1":["foo"]}, jsonRpc=False)
+        @d.addCallback
+        def check(_):
+            self.assertRequest(
+                contentJson={'orig_args': {'param1': 'foo'}, 'path': ['path']},
+                contentType='application/json',
+                responseCode=200)
+        return d
+
+    def test_controljs_not_found(self):
+        d = self.render_control_resource(self.rsrc, ['not', 'found'],action="test",
+                                         jsonRpc=True)
+        @d.addCallback
+        def check(_):
+            self.assertRequest(
+                errorJsonRPC={'code': -32603, 'message': 'invalid path'},
+                contentType='application/json',
+                responseCode=404)
+        return d
+
+    def test_controljs_badaction(self):
+        d = self.render_control_resource(self.rsrc, ['path'],{"param1":["foo"]},
+                                         jsonRpc=True)
+        @d.addCallback
+        def check(_):
+            self.assertRequest(
+                errorJsonRPC={'code': -32601, 'message': 'invalid method'},
+                contentType='application/json',
+                responseCode=501)
+        return d
+    def dotest_controljs_malformedjson(self, _json, error, noIdCheck=False, httpcode=400):
+        request = self.make_request(['path'])
+        request.content = StringIO(json.dumps(_json))
+        request.input_headers = {'content-type': 'application/json'}
+        d = self.render_control_resource(self.rsrc,
+                                         request = request,
+                                         jsonRpc=not noIdCheck)
+        @d.addCallback
+        def check(_):
+            self.assertRequest(
+                errorJsonRPC=error,
+                contentType='application/json',
+                responseCode=httpcode)
+        return d
+    def test_controljs_malformedjson1(self):
+        return self.dotest_controljs_malformedjson(
+            [ "list_not_supported"],
+            {'code': -32600, 'message': 'jsonrpc call batch is not supported'}
+            ,noIdCheck=True)
+
+    def test_controljs_malformedjson_no_dict(self):
+        return self.dotest_controljs_malformedjson(
+            "str_not_supported",
+            {'code': -32600, 'message': 'json root object must be a dictionary: "str_not_supported"'}
+            ,noIdCheck=True)
+    def test_controljs_malformedjson_nojsonrpc(self):
+        return self.dotest_controljs_malformedjson(
+            { "method": "action", "params": {"arg":"args"}, "id": "_id"},
+            {'code': -32600, 'message': "need 'jsonrpc' to be present and be a <type 'str'>"}
+            ,noIdCheck=True)
+    def test_controljs_malformedjson_no_method(self):
+        return self.dotest_controljs_malformedjson(
+            { "jsonrpc": "2.0", "params": {"arg":"args"}, "id": "_id"},
+            {'code': -32600, 'message': "need 'method' to be present and be a <type 'str'>"}
+            ,noIdCheck=True)
+    def test_controljs_malformedjson_no_param(self):
+        return self.dotest_controljs_malformedjson(
+            { "jsonrpc": "2.0", "method": "action",  "id": "_id"},
+            {'code': -32600, 'message': "need 'params' to be present and be a <type 'dict'>"}
+            ,noIdCheck=True)
+    def test_controljs_malformedjson_bad_param(self):
+        return self.dotest_controljs_malformedjson(
+            { "jsonrpc": "2.0", "method":"action", "params": ["args"], "id": "_id"},
+            {'code': -32600, 'message': "need 'params' to be present and be a <type 'dict'>"}
+            ,noIdCheck=True)
+    def test_controljs_malformedjson_no_id(self):
+        return self.dotest_controljs_malformedjson(
+            { "jsonrpc": "2.0", "method": "action",  "params": {"arg":"args"} },
+            {'code': -32600, 'message': "need 'id' to be present and be a <type 'str'>"}
+            ,noIdCheck=True)

master/buildbot/test/util/www.py

@@ -23,6 +23,8 @@
 from buildbot.www import service
 from buildbot.test.fake import fakemaster
 from twisted.python import failure
+from cStringIO import StringIO
+from uuid import uuid1
 
 class FakeRequest(object):
     written = ''
@@ -35,6 +37,7 @@ class FakeRequest(object):
 
     def __init__(self, postpath=None, args={}):
         self.headers = {}
+        self.input_headers = {}
         self.prepath = []
         self.postpath = postpath or []
         self.deferred = defer.Deferred()
@@ -55,7 +58,8 @@ def setResponseCode(self, code):
 
     def setHeader(self, hdr, value):
         self.headers.setdefault(hdr, []).append(value)
-
+    def getHeader(self, key, default=None):
+        return self.input_headers.get(key, default)
     def processingFailed(self, f):
         self.deferred.errback(f)
 
@@ -83,15 +87,47 @@ def render_resource(self, rsrc, postpath=None, args={}, request=None):
             return defer.succeed(rv)
         return request.deferred
 
+    def render_control_resource(self, rsrc, postpath=None, args={}, action="notfound",
+                                request=None, jsonRpc=True):
+        # pass *either* a request or postpath (and optionally args)
+        _id = str(uuid1())
+        if not request:
+            request = self.make_request(postpath=postpath, args=args)
+            request.method = "POST"
+            if jsonRpc:
+                request.content = StringIO(json.dumps(
+                    { "jsonrpc": "2.0", "method": action, "params": args, "id": _id}))
+                request.input_headers = {'content-type': 'application/json'}
+        rv = rsrc.render(request)
+        if rv != server.NOT_DONE_YET:
+            d = defer.succeed(rv)
+        else:
+            d = request.deferred
+        @d.addCallback
+        def check(_json):
+            if jsonRpc:
+                res = json.loads(_json)
+                self.assertIn("jsonrpc",res)
+                self.assertIn("id",res)
+                self.assertEqual(res["jsonrpc"], "2.0")
+                self.assertEqual(res["id"], _id)
+            return json
+        return d
+
     def assertRequest(self, content=None, contentJson=None, contentType=None,
-            responseCode=None, contentDisposition=None):
+            responseCode=None, contentDisposition=None, errorJsonRPC=None):
         got, exp = {}, {}
         if content is not None:
             got['content'] = self.request.written
             exp['content'] = content
         if contentJson is not None:
             got['contentJson'] = json.loads(self.request.written)
             exp['contentJson'] = contentJson
+        if errorJsonRPC is not None:
+            jsonrpc = json.loads(self.request.written)
+            self.assertIn("error", jsonrpc)
+            got['errorJsonRPC'] = jsonrpc["error"]
+            exp['errorJsonRPC'] = errorJsonRPC
         if contentType is not None:
             got['contentType'] =  self.request.headers['content-type']
             exp['contentType'] = [ contentType ]

master/buildbot/www/rest.py

@@ -57,42 +57,109 @@ def __init__(self, master):
         self.master = master
         JsonStatusResource.__init__(self,master.status)
 
+URL_ENCODED = "application/x-www-form-urlencoded"
+JSON_ENCODED = "application/json"
+JSONRPC_CODES = dict(parse_error= -32700,
+                     invalid_request= -32600,
+                     method_not_found= -32601,
+                     invalid_params= -32602,
+                     internal_error= -32603)
+
 class V2RootResource(resource.Resource):
     # rather than construct the entire possible hierarchy of Rest resources,
     # this is marked as a leaf node, and any remaining path items are parsed
     # during rendering
     isLeaf = True
 
     knownArgs = set(['as_text', 'filter', 'compact', 'callback'])
+    def decodeUrlEncoding(self, request):
+        # calculate the request options
+        reqOptions = {}
+        for option in set(request.args) - self.knownArgs:
+            reqOptions[option] = request.args[option][0]
+        return reqOptions
+    def decodeJsonRPC2(self, request):
+        """ In the case of json encoding, we choose jsonrpc2 as the encoding:
+        http://www.jsonrpc.org/specification
+        instead of just inventing our own. This allow easier re-use of client side code
+        This implementation is rather simple, and is not supporting all the features of jsonrpc:
+        -> params as list is not supported
+        -> rpc call batch is not supported
+        -> jsonrpc2 notifications are not supported (i.e. you always get an answer)
+       """
+        datastr = request.content.read()
+        data = json.loads(datastr)
+        if type(data) == list:
+            raise ValueError("jsonrpc call batch is not supported")
+        if type(data) != dict:
+            raise ValueError("json root object must be a dictionary: "+datastr)
 
+        def check(name, _type, _val=None):
+            if not name in data or type(data[name]) != _type:
+                raise ValueError("need '%s' to be present and be a %s"%(name, str(_type)))
+            if _val != None and data[name] != _val:
+                raise ValueError("need '%s' value to be '%s'"%(name, str(_val)))
+        check("jsonrpc", str, "2.0")
+        check("method", str)
+        check("id", str)
+        check("params", dict) # params can be a list in jsonrpc, but we dont support it.
+        return data["params"], data["method"], data["id"]
     def render(self, request):
         @defer.inlineCallbacks
         def render():
             reqPath = request.postpath
+            jsonRpcId = None
             # strip an empty string from the end (trailing slash)
             if reqPath and reqPath[-1] == '':
                 reqPath = reqPath[:-1]
 
-            # calculate the request options
-            reqOptions = {}
-            for option in set(request.args) - self.knownArgs:
-                reqOptions[option] = request.args[option][0]
-
-            def write_error(msg):
-                request.setResponseCode(404)
+            def write_error_default(msg, errcode=404, jsonrpccode=None):
+                request.setResponseCode(errcode)
                 # prefer text/plain here, since this is most likely user error
                 request.setHeader('content-type', 'text/plain')
                 request.write(json.dumps(dict(error=msg)))
 
+            def write_error_jsonrpc(msg, errcode=400, jsonrpccode=JSONRPC_CODES["internal_error"]):
+                request.setResponseCode(errcode)
+                request.setHeader('content-type', JSON_ENCODED)
+                request.write(json.dumps(dict(jsonrpc="2.0",
+                                              error=dict(code=jsonrpccode,
+                                                         message=msg),
+                                              id=jsonRpcId
+                                              )))
+            write_error = write_error_default
+            contenttype = request.getHeader('content-type', URL_ENCODED)
+
+            if contenttype.startswith(JSON_ENCODED):
+                write_error = write_error_jsonrpc
+                try:
+                    reqOptions, action, jsonRpcId = self.decodeJsonRPC2(request)
+                except ValueError,e:
+                    write_error(str(e), jsonrpccode=JSONRPC_CODES["invalid_request"])
+                    return
+            else:
+                reqOptions = self.decodeUrlEncoding(request)
+                if request.method == "POST":
+                    if not "action" in reqOptions:
+                        write_error("need an action parameter for POST", errcode=400)
+                        return
+                    action = reqOptions["action"]
+                    del reqOptions["action"]
             # get the value
             try:
-                data = yield self.master.data.get(reqOptions, tuple(reqPath))
+                if request.method == "POST":
+                    data = yield self.master.data.control(action, reqOptions, tuple(reqPath))
+                else:
+                    data = yield self.master.data.get(reqOptions, tuple(reqPath))
             except data_exceptions.InvalidPathError:
-                write_error("invalid path")
+                write_error("invalid path", errcode=404)
                 return
             except data_exceptions.InvalidOptionException:
                 write_error("invalid option")
                 return
+            except data_exceptions.InvalidActionException:
+                write_error("invalid method", errcode=501,jsonrpccode=JSONRPC_CODES["method_not_found"])
+                return
 
             if data is None:
                 write_error("no data")
@@ -108,7 +175,7 @@ def write_error(msg):
             if as_text:
                 request.setHeader("content-type", 'text/plain')
             else:
-                request.setHeader("content-type", 'application/json')
+                request.setHeader("content-type", JSON_ENCODED)
                 request.setHeader("content-disposition",
                             "attachment; filename=\"%s.json\"" % request.path)
 
@@ -125,6 +192,10 @@ def write_error(msg):
             if filter:
                 data = self._filterEmpty(data)
 
+            # if we are talking jsonrpc, we embed the result in standard encapsulation
+            if not jsonRpcId is None:
+                data = {"jsonrpc": "2.0", "result": data, "id": jsonRpcId}
+
             if compact:
                 data = json.dumps(data, default=self._render_links,
                                         sort_keys=True, separators=(',',':'))