Add generic hooks interface for HTTP change notifications

Includes a github implementation, with room to add others

* 'master' of git://github.com/PerilousApricot/buildbot: (69 commits)
  removing enable_change_hook
  making some changes from djmitche
  removing dummy folder
  removing .project
  removing debug prints
  more typos
  fix
  more timestamp munging
  force UTC
  unbomb github change hook. looks like the timestamp conversion is correct
  more debugging
  match parens
  get types right on times
  getting some better when info
  times are integers
  debug
  more logging
  strip out correct time
  raise after we run the fun part
  incresing verbosity
  ...

master/buildbot/status/web/baseweb.py

@@ -27,6 +27,7 @@
 from buildbot.status.web.authz import Authz
 from buildbot.status.web.auth import AuthFailResource
 from buildbot.status.web.root import RootPage
+from buildbot.status.web.change_hook import ChangeHookResource
 
 # this class contains the WebStatus class.  Basic utilities are in base.py,
 # and specific pages are each in their own module.
@@ -81,6 +82,9 @@ class WebStatus(service.MultiService):
      /one_line_per_build/BUILDERNAME : same, but only for a single builder
      /about : describe this buildmaster (Buildbot and support library versions)
      /xmlrpc : (not yet implemented) an XMLRPC server with build status
+     /change_hook[/DIALECT] : accepts changes from external sources, optionally
+                              choosing the dialect that will be permitted
+                              (i.e. github format, etc..)
 
      and more!  see the manual.
 
@@ -134,7 +138,8 @@ def __init__(self, http_port=None, distrib_port=None, allowForce=None,
                  num_events=200, num_events_max=None, auth=None,
                  order_console_by_time=False, changecommentlink=None,
                  revlink=None, projects=None, repositories=None,
-                 authz=None, logRotateLength=None, maxRotatedFiles=None):
+                 authz=None, logRotateLength=None, maxRotatedFiles=None,
+                 change_hook_dialects = {}):
         """Run a web server that provides Buildbot status.
 
         @type  http_port: int or L{twisted.application.strports} string
@@ -233,7 +238,19 @@ def __init__(self, http_port=None, distrib_port=None, allowForce=None,
         @type maxRotatedFiles: None or int
         @param maxRotatedFiles: number of old http.log files to keep during log rotation.
             If not set, the value set in the buildbot.tac will be used, 
-             falling back to the BuildMaster's default value (10 files).        
+             falling back to the BuildMaster's default value (10 files).       
+        
+        @type  change_hook_dialects: None or dict
+        @param change_hook_dialects: If empty, disables change_hook support, otherwise      
+                                     whitelists valid dialects. In the format of
+                                     {"dialect1": "Option1", "dialect2", None}
+                                     Where the values are options that will be passed
+                                     to the dialect
+                                     
+                                     To enable the DEFAULT handler, use a key of DEFAULT
+                                     
+                                     
+        
     
         """
 
@@ -296,7 +313,11 @@ def __init__(self, http_port=None, distrib_port=None, allowForce=None,
         # down. See ticket #102 for more details.
         self.channels = weakref.WeakKeyDictionary()
 
-
+        # do we want to allow change_hook
+        self.change_hook_dialects = {}
+        if change_hook_dialects:
+            self.change_hook_dialects = change_hook_dialects
+            self.putChild("change_hook", ChangeHookResource(dialects = self.change_hook_dialects))
 
     def setupUsualPages(self, numbuilds, num_events, num_events_max):
         #self.putChild("", IndexOrWaterfallRedirection())
@@ -318,6 +339,7 @@ def setupUsualPages(self, numbuilds, num_events, num_events_max):
         self.putChild("about", AboutBuildbot())
         self.putChild("authfail", AuthFailResource())
 
+
     def __repr__(self):
         if self.http_port is None:
             return "<WebStatus on path %s at %s>" % (self.distrib_port,

master/buildbot/status/web/change_hook.py

@@ -0,0 +1,105 @@
+# code inspired/copied from contrib/github_buildbot
+#  and inspired from code from the Chromium project
+# otherwise, Andrew Melo <andrew.melo@gmail.com> wrote the rest
+
+# but "the rest" is pretty minimal
+from twisted.web import resource
+from buildbot.status.builder import FAILURE
+import re
+from buildbot import util, interfaces
+import traceback
+import sys
+from buildbot.process.properties import Properties
+from buildbot.changes.changes import Change
+from twisted.python.reflect import namedModule
+from twisted.python.log import msg,err
+
+class ChangeHookResource(resource.Resource):
+     # this is a cheap sort of template thingy
+    contentType = "text/html; charset=utf-8"
+    children    = {}
+    def __init__(self, dialects={}):
+        self.dialects = dialects
+
+    def getChild(self, name, request):
+        return self
+
+    def render_GET(self, request):
+        """
+        Reponds to events and starts the build process
+          different implementations can decide on what methods they will accept
+        """
+        self.render_POST(request)
+
+    def render_POST(self, request):
+        """
+        Reponds to events and starts the build process
+          different implementations can decide on what methods they will accept
+        
+        :arguments:
+            request
+                the http request object
+        """
+
+        changes = self.getChanges( request )
+        msg("Payload: " + str(request.args))
+
+        if not changes:
+            msg("No changes found")
+            return
+        self.submitChanges( changes, request )
+        return "changes %s" % changes
+
+
+    def getChanges(self, request):
+        """
+        Take the logic from the change hook, and then delegate it
+        to the proper handler
+        http://localhost/change_hook/DIALECT will load up
+        buildmaster/status/web/hooks/DIALECT.py
+        
+        and call getChanges()
+        
+        the return value is a list of changes
+        
+        if DIALECT is unspecified, a sample implementation is provided
+        """
+        uriRE = re.search(r'^/change_hook/?([a-zA-Z0-9_]*)', request.uri)
+
+        if not uriRE:
+            msg("URI doesn't match change_hook regex: %s" % request.uri)
+            return
+
+        changes = []
+
+        # Was there a dialect provided?
+        if uriRE.group(1):
+            dialect = uriRE.group(1)
+        else:
+            dialect = 'base'
+
+        if dialect in self.dialects.keys():
+#            try:
+                # note, this should be safe, only alphanumerics and _ are
+                # allowed in the dialect name
+            msg("Attempting to load module buildbot.status.web.hooks" + dialect)
+            tempModule = namedModule('buildbot.status.web.hooks.' + dialect)
+            changes = tempModule.getChanges(request,self.dialects[dialect])
+            msg("Got the following changes %s" % changes)
+#            except:
+#                err("Encountered an exception in change_hook:")
+        else:
+            msg("The dialect specified %s wasn't whitelisted in change_hook" % dialect)
+            msg("Note: if dialect is 'base' then it's possible your URL is malformed and we didn't regex it properly")
+
+        return changes        
+
+    def submitChanges(self, changes, request):
+        # get a control object
+        changeMaster = request.site.buildbot_service.master.change_svc
+        for onechange in changes:
+            msg("injecting change %s" % onechange)
+            changeMaster.addChange( onechange )
+
+
+

master/buildbot/status/web/hooks/__init__.py

@@ -0,0 +1 @@
+# test

master/buildbot/status/web/hooks/base.py

@@ -0,0 +1,71 @@
+# code inspired/copied from contrib/github_buildbot
+#  and inspired from code from the Chromium project
+# otherwise, Andrew Melo <andrew.melo@gmail.com> wrote the rest
+
+# but "the rest" is pretty minimal
+from twisted.web import resource
+from buildbot.status.builder import FAILURE
+import re
+from buildbot import util, interfaces
+import logging
+import traceback
+import sys
+from buildbot.process.properties import Properties
+from buildbot.changes.changes import Change
+from twisted.python.reflect import namedModule
+from buildbot.util import json
+
+def getChanges(self, request):
+        """
+        Consumes a naive build notification (the default for now)
+        basically, set POST variables to match commit object parameters:
+        revision, revlink, comments, branch, who, files, links
+        
+        files and links will be de-json'd, the rest are interpreted as strings
+        """
+
+        def firstOrNothing( value ):
+            """
+            Small helper function to return the first value (if value is a list)
+            or return the whole thing otherwise
+            """
+            if ( type(value) == type([])):
+                return value[0]
+            else:
+                return value
+
+        args = request.args
+
+        # first, convert files and links
+        files = None
+        if args.get('files'):
+            files = json.loads( args.get('files')[0] )
+        else:
+            files = []
+
+        links = None
+        if args.get('links'):
+            links = json.loads( args.get('links')[0] )
+        else:
+            links = []
+
+        revision = firstOrNothing(args.get('revision'))
+        when     = firstOrNothing(args.get('when'))
+        who = firstOrNothing(args.get('who'))
+        comments = firstOrNothing(args.get('comments'))
+        isdir = firstOrNothing(args.get('isdir',0))
+        branch = firstOrNothing(args.get('branch'))
+        category = firstOrNothing(args.get('category'))
+        revlink = firstOrNothing(args.get('revlink'))
+        properties = Properties()
+        # properties.update(properties, "Change")
+        repository = firstOrNothing(args.get('repository'))
+        project = firstOrNothing(args.get('project'))
+
+        ourchange = Change(who = who, files = files, comments = comments, isdir = isdir, links = links,
+                        revision=revision, when = when, branch = branch, category = category,
+                        revlink = revlink, repository = repository, project = project)  
+        return [ourchange]
+
+
+