Rewrite REST and JSONRPC support

Now with universal filtering, sorting, pagination, and field selection!

master/buildbot/config.py

@@ -531,6 +531,13 @@ def load_www(self, filename, config_dict):
         if 'www' not in config_dict:
             return
         www_cfg = config_dict['www']
+        allowed = set(['port', 'url', 'debug', 'json_cache_seconds',
+                       'rest_minimum_version', 'allowed_origins', 'jsonp'])
+        unknown = set(www_cfg.iterkeys()) - allowed
+        if unknown:
+            error("unknown www configuration parameter(s) %s" %
+                                            (', '.join(unknown),))
+
         self.www.update(www_cfg)
 
         # invent an appropriate URL given the port
@@ -540,6 +547,12 @@ def load_www(self, filename, config_dict):
         if not self.www['url'].endswith('/'):
             self.www['url'] += '/'
 
+        # convert allowed_origins to a lowercased set for faster membership
+        # checks
+        if 'allowed_origins' in www_cfg:
+            self.www['allowed_origins'] = set(o.lower()
+                                for o in self.www['allowed_origins'])
+
 
     def check_single_master(self):
         # check additional problems that are only valid in a single-master

master/buildbot/data/base.py

@@ -16,6 +16,7 @@
 import UserList
 import urllib
 from twisted.internet import defer
+from buildbot.data import exceptions
 
 class ResourceType(object):
     name = None
@@ -54,7 +55,7 @@ def get(self, resultSpec, kwargs):
         raise NotImplementedError
 
     def control(self, action, args, kwargs):
-        raise NotImplementedError
+        raise exceptions.InvalidControlException
 
     def startConsuming(self, callback, options, kwargs):
         raise NotImplementedError

master/buildbot/data/exceptions.py

@@ -19,8 +19,7 @@
 __all__ = [
     'SchedulerAlreadyClaimedError',
     'InvalidPathError',
-    'InvalidOptionException',
-    'InvalidActionException',
+    'InvalidControlException',
 ]
 
 class DataException(Exception):
@@ -30,10 +29,6 @@ class InvalidPathError(DataException):
     "A path argument was invalid or unknown"
     pass
 
-class InvalidOptionException(DataException):
-    "An option was invalid"
-    pass
-
-class InvalidActionException(DataException):
+class InvalidControlException(DataException):
     "Action is not supported"
     pass

master/buildbot/data/resultspec.py

@@ -95,18 +95,11 @@ def apply(self, data):
             return data
 
         if self.fields:
-            if '-' in [f[0] for f in self.fields]:
-                fields = set(f[1:] for f in self.fields)
-                def excludeFields(d):
-                    return dict((k,v) for k,v in d.iteritems()
-                            if k not in fields)
-                applyFields = excludeFields
-            else:
-                fields = set(self.fields)
-                def includeFields(d):
-                    return dict((k,v) for k,v in d.iteritems()
-                            if k in fields)
-                applyFields = includeFields
+            fields = set(self.fields)
+            def includeFields(d):
+                return dict((k,v) for k,v in d.iteritems()
+                        if k in fields)
+            applyFields = includeFields
         else:
             fields = None
 

master/buildbot/test/fake/endpoint.py

@@ -0,0 +1,74 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+# This is a static resource type and set of endpoints uesd as common data by
+# tests.
+
+from twisted.internet import defer
+from buildbot.data import base, types
+
+testData = {
+    13: {'id': 13, 'info': 'ok', 'success': True},
+    14: {'id': 14, 'info': 'failed', 'success': False},
+    15: {'id': 15, 'info': 'warned', 'success': True},
+    16: {'id': 16, 'info': 'skipped', 'success': True},
+    17: {'id': 17, 'info': 'ignored', 'success': True},
+    18: {'id': 18, 'info': 'unexp', 'success': False},
+    19: {'id': 19, 'info': 'todo', 'success': True},
+    20: {'id': 20, 'info': 'error', 'success': False},
+}
+
+class TestsEndpoint(base.Endpoint):
+    isCollection = True
+    pathPatterns = "/test"
+
+    def get(self, resultSpec, kwargs):
+        # results are sorted by ID for test stability
+        return defer.succeed(sorted(testData.values(), key=lambda v : v['id']))
+
+
+class FailEndpoint(base.Endpoint):
+    isCollection = False
+    pathPatterns = "/test/fail"
+
+    def get(self, resultSpec, kwargs):
+        return defer.fail(RuntimeError('oh noes'))
+
+
+class TestEndpoint(base.Endpoint):
+    isCollection = False
+    pathPatterns = "/test/n:testid"
+
+    def get(self, resultSpec, kwargs):
+        if kwargs['testid'] == 0:
+            return None
+        return defer.succeed(testData[kwargs['testid']])
+
+    def control(self, action, args, kwargs):
+        if action == "fail":
+            return defer.fail(RuntimeError("oh noes"))
+        return defer.succeed({'action': action, 'args': args, 'kwargs': kwargs})
+
+
+class Test(base.ResourceType):
+    name = "test"
+    plural = "tests"
+    endpoints = [ TestsEndpoint, TestEndpoint, FailEndpoint ]
+
+    class EntityType(types.Entity):
+        id = types.Integer()
+        info = types.String()
+        success = types.Boolean()
+    entityType = EntityType(name)

master/buildbot/test/fake/fakedata.py

@@ -273,6 +273,9 @@ def __init__(self, master, testcase):
         self.realConnector = connector.DataConnector(master)
         self.rtypes = self.realConnector.rtypes
 
+    def _scanModule(self, mod):
+        return self.realConnector._scanModule(mod)
+
     def getEndpoint(self, path):
         if not isinstance(path, tuple):
             raise TypeError('path must be a tuple')

master/buildbot/test/integration/test_www.py

@@ -0,0 +1,157 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+from twisted.internet import defer, reactor, protocol
+from twisted.trial import unittest
+from twisted.web import client
+from buildbot.util import json
+from buildbot.test.util import db
+from buildbot.test.fake import fakemaster, fakedb
+from buildbot.db import connector as dbconnector
+from buildbot.mq import connector as mqconnector
+from buildbot.data import connector as dataconnector
+from buildbot.www import service as wwwservice
+
+SOMETIME = 1348971992
+OTHERTIME = 1008971992
+
+
+class BodyReader(protocol.Protocol):
+    # an IProtocol that reads the entire HTTP body and then calls back
+    # with it
+
+    def __init__(self, finishedDeferred):
+        self.body = []
+        self.finishedDeferred = finishedDeferred
+
+    def dataReceived(self, bytes):
+        self.body.append(bytes)
+
+    def connectionLost(self, reason):
+        if reason.check(client.ResponseDone):
+            self.finishedDeferred.callback(''.join(self.body))
+        else:
+            self.finishedDeferred.errback(reason)
+
+
+class Www(db.RealDatabaseMixin, unittest.TestCase):
+
+    master = None
+
+    @defer.inlineCallbacks
+    def setUp(self):
+        # set up a full master serving HTTP
+        yield self.setUpRealDatabase(table_names=['masters'],
+                sqlite_memory=False)
+
+        master = fakemaster.FakeMaster()
+
+        master.config.db = dict(db_url=self.db_url)
+        master.db = dbconnector.DBConnector(master, 'basedir')
+        yield master.db.setup(check_version=False)
+
+        master.config.mq = dict(type='simple')
+        master.mq = mqconnector.MQConnector(master)
+        master.mq.setup()
+
+        master.data = dataconnector.DataConnector(master)
+
+        master.config.www = dict(
+                port='tcp:0:interface=127.0.0.1',
+                debug=True)
+        master.www = wwwservice.WWWService(master)
+        yield master.www.startService()
+        yield master.www.reconfigService(master.config)
+
+        # now that we have a port, construct the real URL and insert it into
+        # the config.  The second reconfig isn't really required, but doesn't
+        # hurt.
+        self.url = 'http://127.0.0.1:%d/' % master.www.getPortnum()
+        master.config.www['url'] = self.url
+        yield master.www.reconfigService(master.config)
+
+        self.master = master
+
+        # build an HTTP agent
+        self.pool = client.HTTPConnectionPool(reactor)
+        self.agent = client.Agent(reactor, pool=self.pool)
+
+    @defer.inlineCallbacks
+    def tearDown(self):
+        yield self.pool.closeCachedConnections()
+        if self.master:
+            yield self.master.www.stopService()
+
+    @defer.inlineCallbacks
+    def apiGet(self, url, expect200=True):
+        pg = yield self.agent.request('GET', url)
+
+        # this is kind of obscene, but protocols are like that
+        d = defer.Deferred()
+        bodyReader = BodyReader(d)
+        pg.deliverBody(bodyReader)
+        body = yield d
+
+        # check this *after* reading the body, otherwise Trial will
+        # complain tha the response is half-read
+        if expect200 and pg.code != 200:
+            self.fail("did not get 200 response for '%s'" % (url,))
+
+        defer.returnValue(json.loads(body))
+
+    def link(self, suffix):
+        return self.url + 'api/v2/' + suffix
+
+    # tests
+
+    # There's no need to be exhaustive here.  The intent is to test that data
+    # can get all the way from the DB to a real HTTP client, and a few
+    # resources will be sufficient to demonstrate that.
+
+    @defer.inlineCallbacks
+    def test_masters(self):
+        yield self.insertTestData([
+            fakedb.Master(id=7, name='some:master',
+                        active=0, last_active=SOMETIME),
+            fakedb.Master(id=8, name='other:master',
+                        active=1, last_active=OTHERTIME),
+        ])
+
+        res = yield self.apiGet(self.link('master'))
+        self.assertEqual(res, {
+            'masters': [
+                {'active': False, 'masterid': 7, 'name': 'some:master',
+                 'last_active': SOMETIME, 'link': self.link('master/7')},
+                {'active': True, 'masterid': 8, 'name': 'other:master',
+                 'last_active': OTHERTIME, 'link': self.link('master/8')},
+            ],
+            'meta': {
+                'total': 2,
+                'links': [
+                    {'href': self.link('master'), 'rel': 'self'}
+                ],
+            }})
+
+        res = yield self.apiGet(self.link('master/7'))
+        self.assertEqual(res, {
+            'masters': [
+                {'active': False, 'masterid': 7, 'name': 'some:master',
+                 'last_active': SOMETIME, 'link': self.link('master/7')},
+            ],
+            'meta': {
+                'links': [
+                    {'href': self.link('master/7'), 'rel': 'self'}
+                ],
+            }})

master/buildbot/test/unit/test_config.py

@@ -767,6 +767,12 @@ def test_load_www_url_no_slash(self):
                 dict(www=dict(url='http://foo', port=20)))
         self.assertResults(www=dict(port=20, url='http://foo/'))
 
+    def test_load_www_unknown(self):
+        self.cfg.load_www(self.filename,
+                dict(www=dict(foo="bar")))
+        self.assertConfigError(self.errors,
+            "unknown www configuration parameter(s) foo")
+
 
 class MasterConfig_checkers(ConfigErrorsMixin, unittest.TestCase):
 

master/buildbot/test/unit/test_data_resultspec.py

@@ -83,9 +83,6 @@ def test_apply_details_fields(self):
         self.assertEqual(
                 resultspec.ResultSpec(fields=['name', 'id']).apply(data),
                 dict(name="clyde", id=14))
-        self.assertEqual(
-                resultspec.ResultSpec(fields=['-name', '-id']).apply(data),
-                dict(favcolor="red"))
 
     def test_apply_collection_fields(self):
         data = mklist(('a', 'b', 'c'),
@@ -97,12 +94,6 @@ def test_apply_collection_fields(self):
         self.assertEqual(
                 resultspec.ResultSpec(fields=['a', 'c']).apply(data),
                 mklist(('a', 'c'), (1, 111), (2, 222)))
-        self.assertEqual(
-                resultspec.ResultSpec(fields=['-b']).apply(data),
-                mklist(('a', 'c'), (1, 111), (2, 222)))
-        self.assertEqual(
-                resultspec.ResultSpec(fields=['-b', '-a']).apply(data),
-                mklist('c', 111, 222))
 
     def test_apply_ordering(self):
         data = mklist('name', 'albert', 'bruce', 'cedric', 'dwayne')

master/buildbot/test/unit/test_www_resource.py

@@ -22,5 +22,5 @@ class Test(www.WwwTestMixin, unittest.TestCase):
     def test_RedirectResource(self):
         master = self.make_master(url='h:/a/b/')
         rsrc = resource.RedirectResource(master, 'foo')
-        self.render_resource(rsrc, [''])
+        self.render_resource(rsrc, '/')
         self.assertEqual(self.request.redirected_to, 'h:/a/b/foo')