Skip to content
Permalink
Browse files

ChangeSources: implement the resource type in terms of the new data i…

…nterfaces

This is modelled fairly closely to the Schedulers changes.

I've gone through all the CS classes and:

 - add a 'name' parameter, and generate one if it's not explicitly set. These will be used to 'claim' the CS for a particular master, so it needs to be meaningful in all cases.
 - use activate/deactivate to start and stop the CS. Usually this means just changing startService/stopService to activate/deactivate
  • Loading branch information...
Jared Grubb
Jared Grubb committed Mar 18, 2013
1 parent b03f805 commit 8bab6038342d562b5fa57c36e8f9c164dfab9509
Showing with 1,488 additions and 88 deletions.
  1. +0 −1 README.md
  2. +28 −15 master/buildbot/changes/base.py
  3. +3 −2 master/buildbot/changes/bonsaipoller.py
  4. +10 −5 master/buildbot/changes/gerritchangesource.py
  5. +12 −9 master/buildbot/changes/gitpoller.py
  6. +9 −5 master/buildbot/changes/hgpoller.py
  7. +6 −2 master/buildbot/changes/p4poller.py
  8. +18 −7 master/buildbot/changes/pb.py
  9. +8 −4 master/buildbot/changes/svnpoller.py
  10. +114 −0 master/buildbot/data/changesources.py
  11. +2 −0 master/buildbot/data/masters.py
  12. +98 −0 master/buildbot/db/changesources.py
  13. +47 −0 master/buildbot/db/migrate/versions/029_add_changesources_table.py
  14. +30 −0 master/buildbot/db/model.py
  15. +19 −0 master/buildbot/test/fake/fakedata.py
  16. +110 −1 master/buildbot/test/fake/fakedb.py
  17. +73 −1 master/buildbot/test/unit/test_changes_base.py
  18. +4 −0 master/buildbot/test/unit/test_changes_bonsaipoller.py
  19. +9 −2 master/buildbot/test/unit/test_changes_gerritchangesource.py
  20. +7 −0 master/buildbot/test/unit/test_changes_gitpoller.py
  21. +7 −0 master/buildbot/test/unit/test_changes_hgpoller.py
  22. +13 −0 master/buildbot/test/unit/test_changes_p4poller.py
  23. +69 −2 master/buildbot/test/unit/test_changes_pb.py
  24. +7 −0 master/buildbot/test/unit/test_changes_svnpoller.py
  25. +2 −0 master/buildbot/test/unit/test_config.py
  26. +224 −0 master/buildbot/test/unit/test_data_changesources.py
  27. +7 −0 master/buildbot/test/unit/test_data_masters.py
  28. +3 −3 master/buildbot/test/unit/test_data_schedulers.py
  29. +285 −0 master/buildbot/test/unit/test_db_changesources.py
  30. +54 −0 master/buildbot/test/unit/test_db_migrate_versions_029_add_changesources_table.py
  31. +10 −0 master/buildbot/test/unit/test_db_schedulers.py
  32. +1 −2 master/buildbot/test/unit/test_status_web_change_hooks_poller.py
  33. +23 −1 master/buildbot/test/util/changesource.py
  34. +3 −0 master/buildbot/test/util/pbmanager.py
  35. +44 −25 master/buildbot/test/util/validation.py
  36. +1 −0 master/docs/developer/data.rst
  37. +69 −0 master/docs/developer/db.rst
  38. +59 −0 master/docs/developer/rtype-changesource.rst
  39. +0 −1 master/docs/developer/rtype-scheduler.rst
@@ -91,7 +91,6 @@ The outstanding resource types are:

* buildrequest :runner: (in progress by Maria Marcano)
* buildslave :runner: (in progress by ewong)
* changesource :runner: (in progress by Jared Grubb)

For each resource type, we'll need the following (based on "Adding Resource Types" in ``master/docs/developer/data.rst``). use this list as a template in the list of types below when you begin a new type.

@@ -14,22 +14,40 @@
# Copyright Buildbot Team Members

from zope.interface import implements
from twisted.application import service
from twisted.internet import defer, task, reactor
from twisted.python import log

from buildbot.interfaces import IChangeSource
from buildbot import util
from buildbot.util import service, misc

class ChangeSource(service.Service, util.ComparableMixin):
class ChangeSource(service.ClusteredService):
implements(IChangeSource)

master = None
"if C{self.running} is true, then C{cs.master} points to the buildmaster."

def describe(self):
pass

## activity handling

def activate(self):
return defer.succeed(None)

def deactivate(self):
return defer.succeed(None)

## service handling

def _getServiceId(self):
return self.master.data.updates.findChangeSourceId(self.name)

def _claimService(self):
return self.master.data.updates.trySetChangeSourceMaster(self.serviceid,
self.master.masterid)

def _unclaimService(self):
return self.master.data.updates.trySetChangeSourceMaster(self.serviceid,
None)


class PollingChangeSource(ChangeSource):
"""
Utility subclass for ChangeSources that use some kind of periodic polling
@@ -45,11 +63,10 @@ class PollingChangeSource(ChangeSource):
_loop = None

def __init__(self, name=None, pollInterval=60*10):
if name:
self.setName(name)
ChangeSource.__init__(self, name)
self.pollInterval = pollInterval

self.doPoll = util.misc.SerializedInvocation(self.doPoll)
self.doPoll = misc.SerializedInvocation(self.doPoll)

def doPoll(self):
"""
@@ -78,9 +95,7 @@ def stopLoop(self):
self._loop.stop()
self._loop = None

def startService(self):
ChangeSource.startService(self)

def activate(self):
# delay starting doing anything until the reactor is running - if
# services are still starting up, they may miss an initial flood of
# changes
@@ -89,7 +104,5 @@ def startService(self):
else:
reactor.callWhenRunning(self.doPoll)

def stopService(self):
def deactivate(self):
self.stopLoop()
return ChangeSource.stopService(self)

@@ -202,8 +202,9 @@ def _getRevision(self):


class BonsaiPoller(base.PollingChangeSource):
compare_attrs = ["bonsaiURL", "pollInterval", "tree",
"module", "branch", "cvsroot"]
compare_attrs = base.PollingChangeSource.compare_attrs + \
("bonsaiURL", "pollInterval", "tree",
"module", "branch", "cvsroot")

def __init__(self, bonsaiURL, module, branch, tree="default",
cvsroot="/cvsroot", pollInterval=30, project=''):
@@ -26,7 +26,8 @@ class GerritChangeSource(base.ChangeSource):
"""This source will maintain a connection to gerrit ssh server
that will provide us gerrit events in json format."""

compare_attrs = ["gerritserver", "gerritport"]
compare_attrs = base.ChangeSource.compare_attrs + \
("gerritserver", "gerritport")

STREAM_GOOD_CONNECTION_TIME = 120
"(seconds) connections longer than this are considered good, and reset the backoff timer"
@@ -40,7 +41,7 @@ class GerritChangeSource(base.ChangeSource):
STREAM_BACKOFF_MAX = 60
"(seconds) maximum time to wait before retrying a failed connection"

def __init__(self, gerritserver, username, gerritport=29418, identity_file=None):
def __init__(self, gerritserver, username, gerritport=29418, identity_file=None, name=None):
"""
@type gerritserver: string
@param gerritserver: the dns or ip that host the gerrit ssh server,
@@ -57,6 +58,11 @@ def __init__(self, gerritserver, username, gerritport=29418, identity_file=None)
"""
# TODO: delete API comment when documented

if not name:
name = "GerritChangeSource:%s@%s:%d" % (username, gerritserver, gerritport)

base.ChangeSource.__init__(self, name=name)

self.gerritserver = gerritserver
self.gerritport = gerritport
self.username = username
@@ -182,17 +188,16 @@ def startStreamProcess(self):
self.process = reactor.spawnProcess(self.LocalPP(self), "ssh",
[ "ssh" ] + args + [ "gerrit", "stream-events" ])

def startService(self):
def activate(self):
self.wantProcess = True
self.startStreamProcess()

def stopService(self):
def deactivate(self):
self.wantProcess = False
if self.process:
self.process.signalProcess("KILL")
# TODO: if this occurs while the process is restarting, some exceptions may
# be logged, although things will settle down normally
return base.ChangeSource.stopService(self)

def describe(self):
status = ""
@@ -27,25 +27,30 @@ class GitPoller(base.PollingChangeSource, StateMixin):
"""This source will poll a remote git repo for changes and submit
them to the change master."""

compare_attrs = ["repourl", "branches", "workdir",
compare_attrs = base.PollingChangeSource.compare_attrs + \
("repourl", "branches", "workdir",
"pollInterval", "gitbin", "usetimestamps",
"category", "project"]
"category", "project")

def __init__(self, repourl, branches=None, branch=None,
workdir=None, pollInterval=10*60,
gitbin='git', usetimestamps=True,
category=None, project=None,
pollinterval=-2, fetch_refspec=None,
encoding='utf-8'):
encoding='utf-8', name=None):

# for backward compatibility; the parameter used to be spelled with 'i'
if pollinterval != -2:
pollInterval = pollinterval

base.PollingChangeSource.__init__(self, name=repourl,
if name is None:
name = repourl

base.PollingChangeSource.__init__(self, name=name,
pollInterval=pollInterval)

if project is None: project = ''
if project is None:
project = ''

if branch and branches:
config.error("GitPoller: can't specify both branch and branches")
@@ -69,10 +74,10 @@ def __init__(self, repourl, branches=None, branch=None,
config.error("GitPoller: fetch_refspec is no longer supported. "
"Instead, only the given branches are downloaded.")

if self.workdir == None:
if self.workdir is None:
self.workdir = 'gitpoller-work'

def startService(self):
def activate(self):
# make our workdir absolute, relative to the master's basedir
if not os.path.isabs(self.workdir):
self.workdir = os.path.join(self.master.basedir, self.workdir)
@@ -83,8 +88,6 @@ def setLastRev(lastRev):
self.lastRev = lastRev
d.addCallback(setLastRev)

d.addCallback(lambda _:
base.PollingChangeSource.startService(self))
d.addErrback(log.err, 'while initializing GitPoller repository')

return d
@@ -26,26 +26,30 @@ class HgPoller(base.PollingChangeSource):
"""This source will poll a remote hg repo for changes and submit
them to the change master."""

compare_attrs = ["repourl", "branch", "workdir",
compare_attrs = base.PollingChangeSource.compare_attrs + \
("repourl", "branch", "workdir",
"pollInterval", "hgpoller", "usetimestamps",
"category", "project"]
"category", "project")

db_class_name = 'HgPoller'

def __init__(self, repourl, branch='default',
workdir=None, pollInterval=10*60,
hgbin='hg', usetimestamps=True,
category=None, project='', pollinterval=-2,
encoding='utf-8'):
encoding='utf-8', name=None):

# for backward compatibility; the parameter used to be spelled with 'i'
if pollinterval != -2:
pollInterval = pollinterval

if name is None:
name = repourl

self.repourl = repourl
self.branch = branch
base.PollingChangeSource.__init__(
self, name=repourl, pollInterval=pollInterval)
self, name=name, pollInterval=pollInterval)
self.encoding = encoding
self.lastChange = time.time()
self.lastPoll = time.time()
@@ -57,7 +61,7 @@ def __init__(self, repourl, branch='default',
self.commitInfo = {}
self.initLock = defer.DeferredLock()

if self.workdir == None:
if self.workdir is None:
config.error("workdir is mandatory for now in HgPoller")

def describe(self):
@@ -45,8 +45,9 @@ class P4Source(base.PollingChangeSource, util.ComparableMixin):
"""This source will poll a perforce repository for changes and submit
them to the change master."""

compare_attrs = ["p4port", "p4user", "p4passwd", "p4base",
"p4bin", "pollInterval"]
compare_attrs = base.PollingChangeSource.compare_attrs + \
("p4port", "p4user", "p4passwd", "p4base",
"p4bin", "pollInterval")

env_vars = ["P4CLIENT", "P4PORT", "P4PASSWD", "P4USER",
"P4CHARSET" , "PATH"]
@@ -72,6 +73,9 @@ def __init__(self, p4port=None, p4user=None, p4passwd=None,
if pollinterval != -2:
pollInterval = pollinterval

if name is None:
name = "P4Source:%s:%s" % (p4port, p4base)

base.PollingChangeSource.__init__(self, name=name, pollInterval=pollInterval)

if project is None:
@@ -98,10 +98,19 @@ def perspective_addChange(self, changedict):
return d

class PBChangeSource(config.ReconfigurableServiceMixin, base.ChangeSource):
compare_attrs = ["user", "passwd", "port", "prefix", "port"]
compare_attrs = base.ChangeSource.compare_attrs + \
("user", "passwd", "port", "prefix", "port")

def __init__(self, user="change", passwd="changepw", port=None,
prefix=None):
prefix=None, name=None):

if name is None:
if prefix:
name = "PBChangeSource:%s:%s" % (prefix, port)
else:
name = "PBChangeSource:%s" % (port,)

base.ChangeSource.__init__(self, name)

self.user = user
self.passwd = passwd
@@ -125,17 +134,19 @@ def reconfigService(self, new_config):
port = new_config.slavePortnum

# and, if it's changed, re-register
if port != self.registered_port:
if port != self.registered_port and self.isActive():
yield self._unregister()
self._register(port)

yield config.ReconfigurableServiceMixin.reconfigService(
self, new_config)

def stopService(self):
d = defer.maybeDeferred(base.ChangeSource.stopService, self)
d.addCallback(lambda _ : self._unregister())
return d
def activate(self):
self._register(self.port)
return defer.succeed(None)

def deactivate(self):
return self._unregister()

def _register(self, port):
if not port:
@@ -68,10 +68,11 @@ class SVNPoller(base.PollingChangeSource, util.ComparableMixin):
master.
"""

compare_attrs = ["svnurl", "split_file",
compare_attrs = base.PollingChangeSource.compare_attrs + \
("svnurl", "split_file",
"svnuser", "svnpasswd", "project",
"pollInterval", "histmax",
"svnbin", "category", "cachepath"]
"svnbin", "category", "cachepath")

parent = None # filled in when we're added
last_change = None
@@ -82,13 +83,16 @@ def __init__(self, svnurl, split_file=None,
pollInterval=10*60, histmax=100,
svnbin='svn', revlinktmpl='', category=None,
project='', cachepath=None, pollinterval=-2,
extra_args=None):
extra_args=None, name=None):

# for backward compatibility; the parameter used to be spelled with 'i'
if pollinterval != -2:
pollInterval = pollinterval

base.PollingChangeSource.__init__(self, name=svnurl, pollInterval=pollInterval)
if name is None:
name = svnurl

base.PollingChangeSource.__init__(self, name=name, pollInterval=pollInterval)

if svnurl.endswith("/"):
svnurl = svnurl[:-1] # strip the trailing slash

0 comments on commit 8bab603

Please sign in to comment.
You can’t perform that action at this time.