dataTypes: add endpoint and cmdline to dump the application spec

All our data api is already described in python, we need to export
that to the coffeescript testsuite in order to make sure the mocked api
is the same as the python one.

As we are at it, we export it to the data api as well, so that we can
have a UI showing the API to the potential developer

master/buildbot/data/builds.py

@@ -105,7 +105,7 @@ class EntityType(types.Entity):
         master_link = types.Link()
         started_at = types.Integer()
         complete = types.Boolean()
-        complete_at = types.NoneOk(types.Integer)
+        complete_at = types.NoneOk(types.Integer())
         results = types.NoneOk(types.Integer())
         state_strings = types.List(of=types.String())
         link = types.Link()

master/buildbot/data/connector.py

@@ -118,3 +118,13 @@ def startConsuming(self, callback, options, path):
     def control(self, action, args, path):
         endpoint, kwargs = self.getEndpoint(path)
         return endpoint.control(action, args, kwargs)
+
+    def asList(self):
+        """return the full spec of the connector as a list of dicts
+        """
+        paths = []
+        for k, v in sorted(self.matcher._patterns.items()):
+            paths.append(dict(path="/".join(k),
+                              type=v.rtype.entityType.name,
+                              type_spec=v.rtype.entityType.asDict()))
+        return paths

master/buildbot/data/root.py

@@ -16,6 +16,7 @@
 from twisted.internet import defer
 from buildbot.data import base, types
 
+
 class RootEndpoint(base.Endpoint):
     isCollection = True
     pathPatterns = "/"
@@ -24,10 +25,18 @@ def get(self, resultSpec, kwargs):
         return defer.succeed(self.master.data.rootLinks)
 
 
+class SpecEndpoint(base.Endpoint):
+    isCollection = True
+    pathPatterns = "/application.spec"
+
+    def get(self, resultSpec, kwargs):
+        return defer.succeed(self.master.data.asList())
+
+
 class Root(base.ResourceType):
     name = "rootlink"
     plural = "rootlinks"
-    endpoints = [ RootEndpoint ]
+    endpoints = [RootEndpoint, SpecEndpoint]
 
     class EntityType(types.Entity):
         name = types.String()

master/buildbot/data/types.py

@@ -38,10 +38,15 @@ def cmp(self, val, arg):
     def validate(self, name, object):
         raise NotImplementedError
 
+    def asDict(self):
+        r = dict(name=self.name)
+        if self.doc is not None:
+            r["doc"] = self.doc
+        return r
 
 class NoneOk(Type):
-
     def __init__(self, nestedType):
+        assert isinstance(nestedType, Type)
         self.nestedType = nestedType
         self.name = self.nestedType.name + " or None"
 
@@ -56,6 +61,10 @@ def validate(self, name, object):
             return
         for msg in self.nestedType.validate(name, object):
             yield msg
+    def asDict(self):
+        r = self.nestedType.asDict()
+        r["can_be_null"] = True
+        return r
 
 
 class Instance(Type):
@@ -153,6 +162,9 @@ def validate(self, name, object):
             for msg in self.of.validate("%s[%d]" % (name, idx), elt):
                 yield msg
 
+    def asDict(self):
+        return dict(type=self.name,
+                    of=self.of.asDict())
 
 class SourcedProperties(Type):
 
@@ -178,6 +190,7 @@ def validate(self, name, object):
 
 
 class Dict(Type):
+    name = "dict"
 
     def __init__(self, **contents):
         self.contents = contents
@@ -206,9 +219,17 @@ def validate(self, name, object):
             for msg in f.validate("%s[%r]" % (name, k), object[k]):
                 yield msg
 
+    def asDict(self):
+        return dict(type=self.name,
+                    fields=[dict(name=k,
+                                 type=v.name,
+                                 type_spec=v.asDict())
+                            for k, v in self.contents.items()
+                            ])
 
-class JsonObject(Type):
 
+class JsonObject(Type):
+    name = "jsonobject"
     def validate(self, name, object):
         if type(object) != dict:
             yield "%s (%r) is not a dictionary (got type %s)" \
@@ -266,3 +287,11 @@ def validate(self, name, object):
             f = self.fields[k]
             for msg in f.validate("%s[%r]" % (name, k), object[k]):
                 yield msg
+
+    def asDict(self):
+        return dict(type=self.name,
+                    fields=[dict(name=k,
+                                 type=v.name,
+                                 type_spec=v.asDict())
+                            for k, v in self.fields.items()
+                            ])

master/buildbot/scripts/uitestserver.py → master/buildbot/scripts/dataspec.py

@@ -14,19 +14,25 @@
 # Copyright Buildbot Team Members
 
 
-import webbrowser,os
-from twisted.internet import reactor, defer
+from twisted.internet import defer
+from buildbot.util import in_reactor, json
+from buildbot.data import connector
 from buildbot.test.fake import fakemaster
+import os
 
+@in_reactor
 @defer.inlineCallbacks
-def _uitestserver(config):
-    master = yield fakemaster.make_master_for_uitest(int(config['port']))
-    if "DISPLAY" in os.environ:
-        webbrowser.open(master.config.www['url']+"app/base/bb/tests/runner.html")
+def dataspec(config):
+    master = yield fakemaster.make_master()
+    data = connector.DataConnector(master)
 
-def uitestserver(config):
-    def async():
-        return _uitestserver(config)
-    reactor.callWhenRunning(async)
-    # unlike in_reactor, we dont stop until CTRL-C
-    reactor.run()
+    dirs = os.path.dirname(config['out'])
+    if dirs and not os.path.exists(dirs):
+        os.makedirs(dirs)
+
+    with open(config['out'], "w") as f:
+        if config['global'] is not None:
+            f.write("window." + config['global'] + '=')
+        f.write(json.dumps(data.asList(), indent=2))
+    print "written", config['out']
+    defer.returnValue(0)

master/buildbot/scripts/runner.py

@@ -682,13 +682,15 @@ def postOptions(self):
             if info:
                 raise usage.UsageError("cannot use --info with 'remove' "
                                        "or 'get'")
-class UiTestOption(base.BasedirMixin, base.SubcommandOptions):
-    subcommandFunction = "buildbot.scripts.uitestserver.uitestserver"
+
+class DataSpecOption(base.BasedirMixin, base.SubcommandOptions):
+    subcommandFunction = "buildbot.scripts.dataspec.dataspec"
     optParameters  = [
-        ['port', 'p', "0", "force port number"],
+        ['out', 'o', "dataspec.json", "output to specified path"],
+        ['global', 'g', None, "output a js script, that sets a global, for inclusion in tessuite"],
         ]
     def getSynopsis(self):
-        return "Usage:   buildbot ui-test-server [options]"
+        return "Usage:   buildbot dataspec [options]"
 
 
 
@@ -726,8 +728,8 @@ class Options(usage.Options):
          "test the validity of a master.cfg config file"],
         ['user', None, UserOptions,
          "Manage users in buildbot's database"],
-        ['ui-test-server', None, UiTestOption,
-         "Start a fake master to test the www UI"]
+        ['dataspec', None, DataSpecOption,
+         "Output data api spec"]
         ]
 
     def opt_version(self):

master/buildbot/test/unit/test_data_root.py

@@ -15,7 +15,7 @@
 
 from twisted.trial import unittest
 from twisted.internet import defer
-from buildbot.data import root, base
+from buildbot.data import root, base, connector
 from buildbot.test.util import endpoint
 
 class RootEndpoint(endpoint.EndpointMixin, unittest.TestCase):
@@ -32,11 +32,49 @@ def setUp(self):
     def tearDown(self):
         self.tearDownEndpoint()
 
-
     @defer.inlineCallbacks
     def test_get(self):
         rootlinks = yield self.callGet(('',))
         [ self.validateData(root) for root in rootlinks ]
         self.assertEqual(rootlinks, [
             {'name': u'abc', 'link': base.Link(('abc',))},
         ])
+
+class SpecEndpoint(endpoint.EndpointMixin, unittest.TestCase):
+
+    endpointClass = root.SpecEndpoint
+    resourceTypeClass = root.Root
+
+
+    def setUp(self):
+        self.setUpEndpoint()
+        # replace fakeConnector with real DataConnector
+        self.master.data = connector.DataConnector(self.master)
+
+    def tearDown(self):
+        self.tearDownEndpoint()
+
+    @defer.inlineCallbacks
+    def test_get(self):
+        spec = yield self.callGet(('application.spec',))
+        for s in spec:
+            # only test an endpoint that is reasonably stable
+            if s['path'] == "master":
+                self.assertEqual(s, {'path': 'master',
+                                     'type': 'master',
+                                     'type_spec': {'fields': [{'name': 'active',
+                                                               'type': 'boolean',
+                                                               'type_spec': {'name': 'boolean'}},
+                                                              {'name': 'masterid',
+                                                               'type': 'integer',
+                                                               'type_spec': {'name': 'integer'}},
+                                                              {'name': 'link',
+                                                               'type': 'link',
+                                                               'type_spec': {'name': 'link'}},
+                                                              {'name': 'name',
+                                                               'type': 'string',
+                                                               'type_spec': {'name': 'string'}},
+                                                              {'name': 'last_active',
+                                                               'type': 'integer',
+                                                               'type_spec': {'name': 'integer'}}],
+                                                   'type': 'master'}})