nascent data API

This only implements changes, and implements them in a
not-very-efficient and not-very-scalable way.

master/buildbot/data/base.py

@@ -0,0 +1,67 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+import re
+
+class Endpoint(object):
+
+    # set the pathPattern to the pattern that should trigger this endpoint
+    pathPattern = None
+
+    # the mq topic corresponding to this path, with %(..)s safely substituted
+    # from the path kwargs; for more complex cases, override
+    # getSubscriptionTopic.
+
+    pathTopicTemplate = None
+
+    def __init__(self, master):
+        self.master = master
+
+    def get(self, options, kwargs):
+        raise NotImplementedError
+
+    def control(self, action, args, kwargs):
+        raise NotImplementedError
+
+    def getSubscriptionTopic(self, options, kwargs):
+        if self.pathTopicTemplate:
+            if '%'  not in self.pathTopicTemplate:
+                return self.pathTopicTemplate
+            safekwargs = SafeDict(kwargs)
+            return self.pathTopicTemplate % safekwargs
+
+
+class SafeDict(object):
+    # utility class to allow %-substitution with the results not containing
+    # topic metacharacters (.*#)
+
+    def __init__(self, dict):
+        self.dict = dict
+
+    metacharacters_re = re.compile('[.*#]')
+    def __getitem__(self, k):
+        return self.metacharacters_re.sub('_', self.dict[k])
+
+
+class Link(object):
+
+    __slots__ = [ 'path' ]
+
+    # a link to another resource, specified as a path
+    def __init__(self, path):
+        self.path = path
+
+    def __repr__(self):
+        return "Link(%r)" % (self.path,)

master/buildbot/data/changes.py

@@ -0,0 +1,56 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+from twisted.internet import defer
+from buildbot.data import base, exceptions
+from buildbot.util import datetime2epoch
+
+def _fixChange(change):
+    # TODO: make these mods in the DB API
+    if change:
+        change = change.copy()
+        del change['is_dir']
+        change['when_timestamp'] = datetime2epoch(change['when_timestamp'])
+        change['link'] = base.Link(('change', str(change['changeid'])))
+    return change
+
+
+class Change(base.Endpoint):
+
+    pathPattern = ( 'change', 'i:changeid' )
+
+    def get(self, options, kwargs):
+        d = self.master.db.changes.getChange(kwargs['changeid'])
+        d.addCallback(_fixChange)
+        return d
+
+
+class Changes(base.Endpoint):
+
+    pathPattern = ( 'change', )
+    pathTopicTemplate = 'change.#' # TODO: test
+
+    def get(self, options, kwargs):
+        try:
+            count = min(int(options.get('count', '50')), 50)
+        except:
+            return defer.fail(
+                    exceptions.InvalidOptionException('invalid count option'))
+        d = self.master.db.changes.getRecentChanges(count)
+        @d.addCallback
+        def sort(changes):
+            changes.sort(key=lambda chdict : chdict['changeid'])
+            return map(_fixChange, changes)
+        return d

master/buildbot/data/connector.py

@@ -0,0 +1,63 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+import inspect
+from twisted.application import service
+from buildbot.util import pathmatch
+from buildbot.data import update, exceptions, base
+
+class DataConnector(service.Service):
+
+    def __init__(self, master):
+        self.setName('data')
+        self.master = master
+        self.update = update.UpdateComponent(master)
+
+        self.matcher = pathmatch.Matcher()
+        self._setup()
+
+    def _setup(self):
+        def _scanModule(mod):
+            for sym in dir(mod):
+                obj = getattr(mod, sym)
+                if inspect.isclass(obj) and issubclass(obj, base.Endpoint):
+                    self.matcher[obj.pathPattern] = obj(self.master)
+
+        # scan all of the endpoint modules
+        from buildbot.data import changes
+        _scanModule(changes)
+
+    def _lookup(self, path):
+        try:
+            return self.matcher[path]
+        except KeyError:
+            raise exceptions.InvalidPathError
+
+    def get(self, options, path):
+        endpoint, kwargs = self._lookup(path)
+        return endpoint.get(options, kwargs)
+
+    def startConsuming(self, callback, options, path):
+        endpoint, kwargs = self._lookup(path)
+        topic = endpoint.getSubscriptionTopic(options, kwargs)
+        if not topic:
+            raise exceptions.InvalidPathError
+        # TODO: aggregate consumers of the same topics
+        # TODO: double this up with get() somehow
+        return self.master.mq.startConsuming(callback, topic)
+
+    def control(self, action, args, path):
+        endpoint, kwargs = self._lookup(path)
+        return endpoint.control(action, args, kwargs)

master/buildbot/data/exceptions.py

@@ -0,0 +1,25 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+class DataException(Exception):
+    pass
+
+class InvalidPathError(DataException):
+    "A path argument was invalid or unknown"
+    pass
+
+class InvalidOptionException(DataException):
+    "An option was invalid"
+    pass

master/buildbot/data/update.py

@@ -0,0 +1,24 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+from twisted.application import service
+
+class UpdateComponent(service.Service):
+
+    def __init__(self, master):
+        self.setName('data.update')
+        self.master = master
+
+    # TODO..

master/buildbot/master.py

@@ -33,6 +33,7 @@
 from buildbot.process.builder import BuilderControl
 from buildbot.db import connector as dbconnector, exceptions
 from buildbot.mq import connector as mqconnector
+from buildbot.data import connector as dataconnector
 from buildbot.schedulers.manager import SchedulerManager
 from buildbot.process.botmaster import BotMaster
 from buildbot.process import debug
@@ -127,6 +128,9 @@ def create_child_services(self):
         self.mq = mqconnector.MQConnector(self)
         self.mq.setServiceParent(self)
 
+        self.data = dataconnector.DataConnector(self)
+        self.data.setServiceParent(self)
+
         self.debug = debug.DebugServices(self)
         self.debug.setServiceParent(self)
 
@@ -427,7 +431,7 @@ def handle_deprec(oldname, old, newname, new, default=None,
                 codebase = self.config.codebaseGenerator(chdict)
             else:
                 codebase = ''
-            
+
         if src:
             # create user object, returning a corresponding uid
             wfd = defer.waitForDeferred(

master/buildbot/test/fake/fakedata.py

@@ -0,0 +1,32 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+class FakeUpdateComponent(object):
+    pass
+
+class FakeDataConnector(object):
+    def __init__(self, master):
+        self.master = master
+        self.update = FakeUpdateComponent()
+        self.update.master = master
+
+    def get(self, options, path):
+        pass
+
+    def startConsuming(self, callback, options, path):
+        return self.master.mq.startConsuming('x') # ugh..
+
+    def control(self, action, args, path):
+        pass

master/buildbot/test/fake/fakedb.py

@@ -19,6 +19,7 @@
 the real connector components.
 """
 
+import random
 import base64
 from buildbot.util import json, epoch2datetime, datetime2epoch
 from twisted.python import failure
@@ -112,17 +113,17 @@ class Change(Row):
 
     defaults = dict(
         changeid = None,
-        author = 'frank',
-        comments = 'test change',
+        author = u'frank',
+        comments = u'test change',
         is_dir = 0,
-        branch = 'master',
-        revision = 'abcd',
-        revlink = 'http://vc/abcd',
+        branch = u'master',
+        revision = u'abcd',
+        revlink = u'http://vc/abcd',
         when_timestamp = 1200000,
-        category = 'cat',
-        repository = 'repo',
-        codebase =  'cb',
-        project = 'proj',
+        category = u'cat',
+        repository = u'repo',
+        codebase =  u'cb',
+        project = u'proj',
     )
 
     lists = ('files',)
@@ -388,7 +389,23 @@ def getChange(self, changeid):
         except KeyError:
             return defer.succeed(None)
 
-        chdict = dict(
+        return defer.succeed(self._chdict(row))
+
+    def getChangeUids(self, changeid):
+        try:
+            ch_uids = [self.changes[changeid].uid]
+        except KeyError:
+            ch_uids = []
+        return defer.succeed(ch_uids)
+
+    def getRecentChanges(self, count):
+        ids = sorted(self.changes.keys())
+        chdicts = [ self._chdict(self.changes[id]) for id in ids[-count:] ]
+        random.shuffle(chdicts) # since order is not documented
+        return defer.succeed(chdicts)
+
+    def _chdict(self, row):
+        return dict(
                 changeid=row.changeid,
                 author=row.author,
                 files=row.files,
@@ -404,16 +421,6 @@ def getChange(self, changeid):
                 codebase=row.codebase,
                 project=row.project)
 
-        return defer.succeed(chdict)
-
-    def getChangeUids(self, changeid):
-        try:
-            ch_uids = [self.changes[changeid].uid]
-        except KeyError:
-            ch_uids = []
-        return defer.succeed(ch_uids)
-
-    # TODO: getRecentChanges
 
     # fake methods