Merge branch 'user_objects_git_commit' of git://github.com/dzhurley/b…

…uildbot

* 'user_objects_git_commit' of git://github.com/dzhurley/buildbot:
  added User Object creation on git changes

(Changes: removed unused import)

master/NEWS

@@ -5,6 +5,11 @@ Major User visible changes in Buildbot.             -*- outline -*-
 
 * Next Version
 
+***Any custom change_hook_dialects should now return a (changes, src)
+tuple from its getChange method, instead of just the changes. The src
+is used for noting what VCS the changes came from, and is just a string
+such as 'git'.
+
 ** EC2 instances are now terminated instead of stopped.
 
 This is really only relevant for EBS-backed instances, as Buildbot will now

master/buildbot/changes/gitpoller.py

@@ -283,7 +283,8 @@ def _process_changes(self, unused_output):
                    branch=self.branch,
                    category=self.category,
                    project=self.project,
-                   repository=self.repourl)
+                   repository=self.repourl,
+                   src='git')
             wfd = defer.waitForDeferred(d)
             yield wfd
             results = wfd.getResult()

master/buildbot/changes/pb.py

@@ -32,7 +32,7 @@ def attached(self, mind):
     def detached(self, mind):
         pass
 
-    def perspective_addChange(self, changedict):
+    def perspective_addChange(self, changedict, src=None):
         log.msg("perspective_addChange called")
 
         if 'revlink' in changedict and not changedict['revlink']:
@@ -89,7 +89,7 @@ def perspective_addChange(self, changedict):
 
         if not files:
             log.msg("No files listed in change... bit strange, but not fatal.")
-        return self.master.addChange(**changedict)
+        return self.master.addChange(src=src, **changedict)
 
 class PBChangeSource(base.ChangeSource):
     compare_attrs = ["user", "passwd", "port", "prefix", "port"]

master/buildbot/db/users.py

@@ -38,9 +38,6 @@ class UsersConnectorComponent(base.DBConnectorComponent):
     with conflicting names will be ignored.
     """
 
-    known_types = ['full_name', 'email']
-    known_info_types = ['authz_user', 'authz_pass', 'git']
-
     def addUser(self, identifier, attr_type, attr_data, _race_hook=None):
         """
         Get an existing user, or add a new one, based on the given attribute.

master/buildbot/master.py

@@ -44,6 +44,7 @@
 from buildbot.process import debug
 from buildbot.process import metrics
 from buildbot.process import cache
+from buildbot.process.users import users
 from buildbot.status.results import SUCCESS, WARNINGS, FAILURE
 from buildbot import monkeypatches
 
@@ -854,7 +855,7 @@ def keep(id):
     def addChange(self, who=None, files=None, comments=None, author=None,
             isdir=None, is_dir=None, links=None, revision=None, when=None,
             when_timestamp=None, branch=None, category=None, revlink='',
-            properties={}, repository='', project=''):
+            properties={}, repository='', project='', src=None):
         """
         Add a change to the buildmaster and act on it.
 
@@ -915,6 +916,9 @@ def addChange(self, who=None, files=None, comments=None, author=None,
         @param project: the project this change is a part of
         @type project: unicode string
 
+        @param src: source of the change (vcs or other)
+        @type src: string
+
         @returns: L{Change} instance via Deferred
         """
         metrics.MetricCountEvent.log("added_changes", 1)
@@ -944,11 +948,21 @@ def handle_deprec(oldname, old, newname, new, default=None,
         for n in properties:
             properties[n] = (properties[n], 'Change')
 
-        d = self.db.changes.addChange(author=author, files=files,
-                comments=comments, is_dir=is_dir, links=links,
-                revision=revision, when_timestamp=when_timestamp,
-                branch=branch, category=category, revlink=revlink,
-                properties=properties, repository=repository, project=project)
+        d = defer.succeed(None)
+        if src:
+            # create user object, returning a corresponding uid
+            d.addCallback(lambda _ : users.createUserObject(self, author, src))
+
+        # add the Change to the database
+        d.addCallback(lambda uid :
+                          self.db.changes.addChange(author=author, files=files,
+                                          comments=comments, is_dir=is_dir,
+                                          links=links, revision=revision,
+                                          when_timestamp=when_timestamp,
+                                          branch=branch, category=category,
+                                          revlink=revlink, properties=properties,
+                                          repository=repository, project=project,
+                                          uid=uid))
 
         # convert the changeid to a Change instance
         d.addCallback(lambda changeid :

master/buildbot/process/users/users.py

@@ -0,0 +1,54 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+from twisted.python import log
+from twisted.internet import defer
+
+@defer.deferredGenerator
+def createUserObject(master, author, src=None):
+    """
+    Take a Change author and source and translate them into a User Object,
+    storing the user in master.db, or returning None if the src is not
+    specified.
+
+    @param master: link to Buildmaster for database operations
+    @type master: master.Buildmaster instance
+
+    @param authors: Change author if string or Authz instance
+    @type authors: string or status.web.authz instance
+
+    @param src: source from which the User Object will be created
+    @type src: string
+    """
+
+    if not src:
+        log.msg("No vcs information found, unable to create User Object")
+        return
+
+    if src == 'git':
+        log.msg("checking for User Object from git Change for: %s" % author)
+        usdict = dict(identifier=author, attr_type='git', attr_data=author)
+    else:
+        log.msg("Unrecognized source argument: %s" % src)
+        return
+
+    d = master.db.users.addUser(identifier=usdict['identifier'],
+                                attr_type=usdict['attr_type'],
+                                attr_data=usdict['attr_data'])
+    wfd = defer.waitForDeferred(d)
+    yield wfd
+    uid = wfd.getResult()
+
+    yield uid

master/buildbot/status/web/change_hook.py

@@ -36,6 +36,7 @@ def __init__(self, dialects={}):
         configuration options to the dialect.
         """
         self.dialects = dialects
+        self.request_dialect = None
 
     def getChild(self, name, request):
         return self
@@ -58,7 +59,7 @@ def render_POST(self, request):
         """
 
         try:
-            changes = self.getChanges( request )
+            changes, src = self.getChanges( request )
         except ValueError, err:
             request.setResponseCode(400, err.args[0])
             return defer.succeed(err.args[0])
@@ -68,7 +69,7 @@ def render_POST(self, request):
         if not changes:
             log.msg("No changes found")
             return defer.succeed("no changes found")
-        d = self.submitChanges( changes, request )
+        d = self.submitChanges( changes, request, src )
         d.addCallback(lambda _ : "OK")
         return d
 
@@ -93,6 +94,7 @@ def getChanges(self, request):
             raise ValueError("URI doesn't match change_hook regex: %s" % request.uri)
 
         changes = []
+        src = None
 
         # Was there a dialect provided?
         if uriRE.group(1):
@@ -103,22 +105,22 @@ def getChanges(self, request):
         if dialect in self.dialects.keys():
             log.msg("Attempting to load module buildbot.status.web.hooks." + dialect)
             tempModule = namedModule('buildbot.status.web.hooks.' + dialect)
-            changes = tempModule.getChanges(request,self.dialects[dialect])
+            changes, src = tempModule.getChanges(request,self.dialects[dialect])
             log.msg("Got the following changes %s" % changes)
-
+            self.request_dialect = dialect
         else:
             m = "The dialect specified, '%s', wasn't whitelisted in change_hook" % dialect
             log.msg(m)
             log.msg("Note: if dialect is 'base' then it's possible your URL is malformed and we didn't regex it properly")
             raise ValueError(m)
 
-        return changes
+        return (changes, src)
 
     @defer.deferredGenerator
-    def submitChanges(self, changes, request):
+    def submitChanges(self, changes, request, src):
         master = request.site.buildbot_service.master
         for chdict in changes:
-            wfd = defer.waitForDeferred(master.addChange(**chdict))
+            wfd = defer.waitForDeferred(master.addChange(src=src, **chdict))
             yield wfd
             change = wfd.getResult()
             log.msg("injected change %s" % change)

master/buildbot/status/web/hooks/base.py

@@ -76,7 +76,7 @@ def firstOrNothing( value ):
                 branch=branch, category=category, revlink=revlink,
                 properties=properties, repository=repository,
                 project=project)
-        return [ chdict ]
+        return ([ chdict ], None)
 
 
 

master/buildbot/status/web/hooks/github.py

@@ -99,7 +99,7 @@ def getChanges(request, options = None):
             #private = payload['repository']['private']
             changes = process_change(payload, user, repo, repo_url, project)
             log.msg("Received %s changes from github" % len(changes))
-            return changes
+            return (changes, 'git')
         except Exception:
             logging.error("Encountered an exception:")
             for msg in traceback.format_exception(*sys.exc_info()):

master/buildbot/test/unit/test_changes_gitpoller.py

@@ -17,7 +17,7 @@
 from twisted.internet import defer
 from exceptions import Exception
 from buildbot.changes import gitpoller
-from buildbot.test.util import changesource, gpo
+from buildbot.test.util import changesource, gpo, users
 from buildbot.util import epoch2datetime
 
 class GitOutputParsing(gpo.GetProcessOutputMixin, unittest.TestCase):
@@ -101,14 +101,18 @@ def test_get_commit_timestamp(self):
 
 class TestGitPoller(gpo.GetProcessOutputMixin,
                     changesource.ChangeSourceMixin,
+                    users.UsersMixin,
                     unittest.TestCase):
 
     def setUp(self):
         self.setUpGetProcessOutput()
         d = self.setUpChangeSource()
+        d.addCallback(lambda _ : self.setUpUsers())
         def create_poller(_):
             self.poller = gitpoller.GitPoller('git@example.com:foo/baz.git')
             self.poller.master = self.master
+            self.poller.createUserObject = self.createUserObject
+            self.poller.master.db.users.addUser = self.addUser
         d.addCallback(create_poller)
         return d
 
@@ -153,19 +157,21 @@ def comments(rev):
         d = self.poller.poll()
 
         # check the results
-        def check(_):
+        def check_changes(_):
             self.assertEqual(len(self.changes_added), 2)
             self.assertEqual(self.changes_added[0]['author'], 'by:4423cdbc')
             self.assertEqual(self.changes_added[0]['when_timestamp'],
                                         epoch2datetime(1273258009))
             self.assertEqual(self.changes_added[0]['comments'], 'hello!')
             self.assertEqual(self.changes_added[0]['branch'], 'master')
             self.assertEqual(self.changes_added[0]['files'], [ '/etc/442' ])
+            self.assertEqual(self.changes_added[0]['src'], 'git')
             self.assertEqual(self.changes_added[1]['author'], 'by:64a5dc2a')
             self.assertEqual(self.changes_added[1]['when_timestamp'],
                                         epoch2datetime(1273258009))
             self.assertEqual(self.changes_added[1]['comments'], 'hello!')
             self.assertEqual(self.changes_added[1]['files'], [ '/etc/64a' ])
-        d.addCallback(check)
+            self.assertEqual(self.changes_added[1]['src'], 'git')
+        d.addCallback(check_changes)
 
         return d

master/buildbot/test/unit/test_changes_pb.py

@@ -85,17 +85,18 @@ class TestChangePerspective(unittest.TestCase):
     def setUp(self):
         self.added_changes = []
         self.master = mock.Mock()
+
         def addChange(**chdict):
             self.added_changes.append(chdict)
-            return defer.succeed(None)
+            return defer.succeed(mock.Mock())
         self.master.addChange = addChange
 
     def test_addChange_noprefix(self):
         cp = pb.ChangePerspective(self.master, None)
         d = cp.perspective_addChange(dict(who="bar", files=['a']))
         def check(_):
             self.assertEqual(self.added_changes,
-                    [ dict(author="bar", files=['a']) ])
+                    [ dict(author="bar", files=['a'], src=None) ])
         d.addCallback(check)
         return d
 
@@ -105,7 +106,7 @@ def test_addChange_prefix(self):
                 dict(who="bar", files=['xx/a', 'yy/b']))
         def check(_):
             self.assertEqual(self.added_changes,
-                    [ dict(author="bar", files=['a']) ])
+                    [ dict(author="bar", files=['a'], src=None) ])
         d.addCallback(check)
         return d
 
@@ -116,7 +117,8 @@ def test_addChange_sanitize_None(self):
                 )
         def check(_):
             self.assertEqual(self.added_changes,
-                    [ dict(project="", revlink="", repository="", files=[]) ])
+                    [ dict(project="", revlink="", repository="",
+                           files=[], src=None) ])
         d.addCallback(check)
         return d
 
@@ -127,7 +129,7 @@ def test_addChange_when_None(self):
                 )
         def check(_):
             self.assertEqual(self.added_changes,
-                    [ dict(when_timestamp=None, files=[]) ])
+                    [ dict(when_timestamp=None, files=[], src=None) ])
         d.addCallback(check)
         return d
 
@@ -138,7 +140,7 @@ def test_addChange_files_tuple(self):
                 )
         def check(_):
             self.assertEqual(self.added_changes,
-                    [ dict(files=['a', 'b']) ])
+                    [ dict(files=['a', 'b'], src=None) ])
         d.addCallback(check)
         return d
 
@@ -153,7 +155,8 @@ def check(_):
                     [ dict(author=u"\N{SNOWMAN}",
                       comments=u"\N{SNOWMAN}",
                       links=[u'\N{HEAVY BLACK HEART}'],
-                      files=[u'\N{VERY MUCH GREATER-THAN}']) ])
+                      files=[u'\N{VERY MUCH GREATER-THAN}'],
+                      src=None) ])
         d.addCallback(check)
         return d
 
@@ -168,7 +171,8 @@ def check(_):
                     [ dict(author=u"\N{SNOWMAN}",
                       comments=u"\N{SNOWMAN}",
                       links=[u'\N{HEAVY BLACK HEART}'],
-                      files=[u'\N{VERY MUCH GREATER-THAN}']) ])
+                      files=[u'\N{VERY MUCH GREATER-THAN}'],
+                      src=None) ])
         d.addCallback(check)
         return d
 
@@ -179,7 +183,7 @@ def test_addChange_non_utf8_bytestring(self):
         d = cp.perspective_addChange(dict(author=bogus_utf8, files=['a']))
         def check(_):
             self.assertEqual(self.added_changes,
-                    [ dict(author=replacement, files=['a']) ])
+                    [ dict(author=replacement, files=['a'], src=None) ])
         d.addCallback(check)
         return d
 
@@ -190,7 +194,16 @@ def test_addChange_old_param_names(self):
         def check(_):
             self.assertEqual(self.added_changes,
                     [ dict(is_dir=1, author='me', files=[],
-                        when_timestamp=epoch2datetime(1234)) ])
+                        when_timestamp=epoch2datetime(1234), src=None) ])
         d.addCallback(check)
         return d
 
+    def test_createUserObject_git_src(self):
+        cp = pb.ChangePerspective(self.master, None)
+        d = cp.perspective_addChange(dict(who="c <h@c>"), src='git')
+        def check_change(_):
+            self.assertEqual(self.added_changes, [ dict(author="c <h@c>",
+                                                        files=[],
+                                                        src='git') ])
+        d.addCallback(check_change)
+        return d