Skip to content

Commit

Permalink
Merge branch 'poll-hook'
Browse files Browse the repository at this point in the history
  • Loading branch information
tomprince committed Jun 5, 2012
2 parents b9c00ac + 4c12f65 commit 44d9b40
Show file tree
Hide file tree
Showing 8 changed files with 256 additions and 22 deletions.
53 changes: 39 additions & 14 deletions master/buildbot/changes/base.py
Expand Up @@ -35,36 +35,61 @@ class PollingChangeSource(ChangeSource):
Utility subclass for ChangeSources that use some kind of periodic polling
operation. Subclasses should define C{poll} and set C{self.pollInterval}.
The rest is taken care of.
Any subclass will be available via the "poller" webhook.
"""

pollInterval = 60
"time (in seconds) between calls to C{poll}"

_loop = None

def __init__(self, name=None, pollInterval=60*10):
if name:
self.setName(name)
self.pollInterval = pollInterval

self.doPoll = util.misc.SerializedInvocation(self.doPoll)

def doPoll(self):
"""
This is the method that is called by LoopingCall to actually poll.
It may also be called by change hooks to request a poll.
It is serialiazed - if you call it while a poll is in progress
then the 2nd invocation won't start until the 1st has finished.
"""
d = defer.maybeDeferred(self.poll)
d.addErrback(log.err, 'while polling for changes')
return d

def poll(self):
"""
Perform the polling operation, and return a deferred that will fire
when the operation is complete. Failures will be logged, but the
method will be called again after C{pollInterval} seconds.
"""

def startLoop(self):
self._loop = task.LoopingCall(self.doPoll)
self._loop.start(self.pollInterval, now=False)

def stopLoop(self):
if self._loop and self._loop.running:
self._loop.stop()
self._loop = None

def startService(self):
ChangeSource.startService(self)
def do_poll():
d = defer.maybeDeferred(self.poll)
d.addErrback(log.err, 'while polling for changes')
return d

# delay starting the loop until the reactor is running, and do not
# run it immediately - if services are still starting up, they may
# miss an initial flood of changes
def start_loop():
self._loop = task.LoopingCall(do_poll)
self._loop.start(self.pollInterval, now=False)
reactor.callWhenRunning(start_loop)

# delay starting doing anything until the reactor is running - if
# services are still starting up, they may miss an initial flood of
# changes
if self.pollInterval:
reactor.callWhenRunning(self.startLoop)
else:
reactor.callWhenRunning(self.doPoll)

def stopService(self):
if self._loop and self._loop.running:
self._loop.stop()
self.stopLoop()
return ChangeSource.stopService(self)

6 changes: 4 additions & 2 deletions master/buildbot/changes/bonsaipoller.py
Expand Up @@ -207,14 +207,16 @@ class BonsaiPoller(base.PollingChangeSource):
"module", "branch", "cvsroot"]

def __init__(self, bonsaiURL, module, branch, tree="default",
cvsroot="/cvsroot", pollInterval=30, project=''):
cvsroot="/cvsroot", pollInterval=30, project='', name=None):

base.PollingChangeSource.__init__(self, name=name, pollInterval=pollInterval)

self.bonsaiURL = bonsaiURL
self.module = module
self.branch = branch
self.tree = tree
self.cvsroot = cvsroot
self.repository = module != 'all' and module or ''
self.pollInterval = pollInterval
self.lastChange = time.time()
self.lastPoll = time.time()

Expand Down
7 changes: 5 additions & 2 deletions master/buildbot/changes/gitpoller.py
Expand Up @@ -36,15 +36,18 @@ def __init__(self, repourl, branch='master',
gitbin='git', usetimestamps=True,
category=None, project=None,
pollinterval=-2, fetch_refspec=None,
encoding='utf-8'):
encoding='utf-8', name=None):

# for backward compatibility; the parameter used to be spelled with 'i'
if pollinterval != -2:
pollInterval = pollinterval

base.PollingChangeSource.__init__(self, name=name, pollInterval=pollInterval)

if project is None: project = ''

self.repourl = repourl
self.branch = branch
self.pollInterval = pollInterval
self.fetch_refspec = fetch_refspec
self.encoding = encoding
self.lastChange = time.time()
Expand Down
6 changes: 4 additions & 2 deletions master/buildbot/changes/p4poller.py
Expand Up @@ -66,11 +66,14 @@ def __init__(self, p4port=None, p4user=None, p4passwd=None,
p4base='//', p4bin='p4',
split_file=lambda branchfile: (None, branchfile),
pollInterval=60 * 10, histmax=None, pollinterval=-2,
encoding='utf8', project=None):
encoding='utf8', project=None, name=None):

# for backward compatibility; the parameter used to be spelled with 'i'
if pollinterval != -2:
pollInterval = pollinterval

base.PollingChangeSource.__init__(self, name=name, pollInterval=pollInterval)

if project is None:
project = ''

Expand All @@ -80,7 +83,6 @@ def __init__(self, p4port=None, p4user=None, p4passwd=None,
self.p4base = p4base
self.p4bin = p4bin
self.split_file = split_file
self.pollInterval = pollInterval
self.encoding = encoding
self.project = project

Expand Down
6 changes: 4 additions & 2 deletions master/buildbot/changes/svnpoller.py
Expand Up @@ -66,11 +66,14 @@ def __init__(self, svnurl, split_file=None,
pollInterval=10*60, histmax=100,
svnbin='svn', revlinktmpl='', category=None,
project='', cachepath=None, pollinterval=-2,
extra_args=None):
extra_args=None, name=None):

# for backward compatibility; the parameter used to be spelled with 'i'
if pollinterval != -2:
pollInterval = pollinterval

base.PollingChangeSource.__init__(self, name=name, pollInterval=pollInterval)

if svnurl.endswith("/"):
svnurl = svnurl[:-1] # strip the trailing slash
self.svnurl = svnurl
Expand All @@ -85,7 +88,6 @@ def __init__(self, svnurl, split_file=None,
# required for ssh-agent auth

self.svnbin = svnbin
self.pollInterval = pollInterval
self.histmax = histmax
self._prefix = None
self.category = category
Expand Down
54 changes: 54 additions & 0 deletions master/buildbot/status/web/hooks/poller.py
@@ -0,0 +1,54 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members

# This change hook allows GitHub or a hand crafted curl inovcation to "knock on
# the door" and trigger a change source to poll.

from buildbot.changes.base import PollingChangeSource


def getChanges(req, options=None):
change_svc = req.site.buildbot_service.master.change_svc
poll_all = not "poller" in req.args

allow_all = True
allowed = []
if isinstance(options, dict) and "allowed" in options:
allow_all = False
allowed = options["allowed"]

pollers = []

for source in change_svc:
if not isinstance(source, PollingChangeSource):
continue
if not hasattr(source, "name"):
continue
if not poll_all and not source.name in req.args['poller']:
continue
if not allow_all and not source.name in allowed:
continue
pollers.append(source)

if not poll_all:
missing = set(req.args['poller']) - set(s.name for s in pollers)
if missing:
raise ValueError("Could not find pollers: %s" % ",".join(missing))

for p in pollers:
p.doPoll()

return [], None

107 changes: 107 additions & 0 deletions master/buildbot/test/unit/test_status_web_change_hooks_poller.py
@@ -0,0 +1,107 @@
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members

from twisted.trial import unittest
from twisted.internet import defer
from buildbot.changes import base
import buildbot.status.web.change_hook as change_hook
from buildbot.test.fake.web import FakeRequest
from buildbot.changes.manager import ChangeManager


class TestPollingChangeHook(unittest.TestCase):
class Subclass(base.PollingChangeSource):
pollInterval = None
called = False

def poll(self):
self.called = True

def setUpRequest(self, args, options=True):
self.changeHook = change_hook.ChangeHookResource(dialects={'poller' : options})

self.request = FakeRequest(args=args)
self.request.uri = "/change_hook/poller"
self.request.method = "GET"

master = self.request.site.buildbot_service.master
master.change_svc = ChangeManager(master)

self.changesrc = self.Subclass("example", None)
self.changesrc.setServiceParent(master.change_svc)

self.disabledChangesrc = self.Subclass("disabled", None)
self.disabledChangesrc.setServiceParent(master.change_svc)

anotherchangesrc = base.ChangeSource()
anotherchangesrc.setName("notapoller")
anotherchangesrc.setServiceParent(master.change_svc)

return self.request.test_render(self.changeHook)

@defer.inlineCallbacks
def test_no_args(self):
yield self.setUpRequest({})
self.assertEqual(self.request.written, "no changes found")
self.assertEqual(self.changesrc.called, True)
self.assertEqual(self.disabledChangesrc.called, True)

@defer.inlineCallbacks
def test_no_poller(self):
yield self.setUpRequest({"poller": ["nosuchpoller"]})
expected = "Could not find pollers: nosuchpoller"
self.assertEqual(self.request.written, expected)
self.request.setResponseCode.assert_called_with(400, expected)
self.assertEqual(self.changesrc.called, False)
self.assertEqual(self.disabledChangesrc.called, False)

@defer.inlineCallbacks
def test_invalid_poller(self):
yield self.setUpRequest({"poller": ["notapoller"]})
expected = "Could not find pollers: notapoller"
self.assertEqual(self.request.written, expected)
self.request.setResponseCode.assert_called_with(400, expected)
self.assertEqual(self.changesrc.called, False)
self.assertEqual(self.disabledChangesrc.called, False)

@defer.inlineCallbacks
def test_trigger_poll(self):
yield self.setUpRequest({"poller": ["example"]})
self.assertEqual(self.request.written, "no changes found")
self.assertEqual(self.changesrc.called, True)
self.assertEqual(self.disabledChangesrc.called, False)

@defer.inlineCallbacks
def test_allowlist_deny(self):
yield self.setUpRequest({"poller": ["disabled"]}, options={"allowed": ["example"]})
expected = "Could not find pollers: disabled"
self.assertEqual(self.request.written, expected)
self.request.setResponseCode.assert_called_with(400, expected)
self.assertEqual(self.changesrc.called, False)
self.assertEqual(self.disabledChangesrc.called, False)

@defer.inlineCallbacks
def test_allowlist_allow(self):
yield self.setUpRequest({"poller": ["example"]}, options={"allowed": ["example"]})
self.assertEqual(self.request.written, "no changes found")
self.assertEqual(self.changesrc.called, True)
self.assertEqual(self.disabledChangesrc.called, False)

@defer.inlineCallbacks
def test_allowlist_all(self):
yield self.setUpRequest({}, options={"allowed": ["example"]})
self.assertEqual(self.request.written, "no changes found")
self.assertEqual(self.changesrc.called, True)
self.assertEqual(self.disabledChangesrc.called, False)
39 changes: 39 additions & 0 deletions master/docs/manual/cfg-statustargets.rst
Expand Up @@ -709,6 +709,45 @@ that periodically poll the Google Code commit feed for changes.

change_hook_dialects={'googlecode': {'secret_key': 'FSP3p-Ghdn4T0oqX', 'branch': 'master'}}

Poller hook
###########

The poller hook allows you to use GET requests to trigger polling. One
advantage of this is your buildbot instance can (at start up) poll to get
changes that happened while it was down, but then you can still use a commit
hook to get fast notification of new changes.

Suppose you have a poller configured like this::

c['change_source'] = SVNPoller(
name="amanda",
svnurl="https://amanda.svn.sourceforge.net/svnroot/amanda/amanda",
split_file=split_file_branches)

And you configure your WebStatus to enable this hook::

c['status'].append(html.WebStatus(
…,
change_hook_dialects={'poller': True}
))

Then you will be able to trigger a poll of the SVN repository by poking the
``/change_hook/poller`` URL from a commit hook like this::

curl http://yourbuildbot/change_hook/poller?poller=amanda

If no ``poller`` argument is provided then the hook will trigger polling of all
polling change sources.

You can restrict which pollers the webhook has access to using the ``allowed``
option::

c['status'].append(html.WebStatus(
…,
change_hook_dialects={'poller': {'allowed': ['amanda']}}
))


.. bb:status:: MailNotifier
.. index:: single: email; MailNotifier
Expand Down

0 comments on commit 44d9b40

Please sign in to comment.