Permalink
Cannot retrieve contributors at this time
Fetching contributors…
Cannot retrieve contributors at this time
| # Copyright (C) 2008-2009 Canonical | |
| # | |
| # This program is free software: you can redistribute it and/or modify | |
| # it under the terms of the GNU General Public License as published by | |
| # the Free Software Foundation, either version 3 of the License, or | |
| # (at your option) any later version. | |
| # | |
| # This program is distributed in the hope that it will be useful, | |
| # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
| # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
| # GNU General Public License for more details. | |
| # | |
| # You should have received a copy of the GNU General Public License | |
| # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
| """\ | |
| bzr buildbot integration | |
| ======================== | |
| This file contains both bzr commit/change hooks and a bzr poller. | |
| ------------ | |
| Requirements | |
| ------------ | |
| This has been tested with buildbot 0.7.9, bzr 1.10, and Twisted 8.1.0. It | |
| should work in subsequent releases. | |
| For the hook to work, Twisted must be installed in the same Python that bzr | |
| uses. | |
| ----- | |
| Hooks | |
| ----- | |
| To install, put this file in a bzr plugins directory (e.g., | |
| ~/.bazaar/plugins). Then, in one of your bazaar conf files (e.g., | |
| ~/.bazaar/locations.conf), set the location you want to connect with buildbot | |
| with these keys: | |
| - buildbot_on: one of 'commit', 'push, or 'change'. Turns the plugin on to | |
| report changes via commit, changes via push, or any changes to the trunk. | |
| 'change' is recommended. | |
| - buildbot_server: (required to send to a buildbot master) the URL of the | |
| buildbot master to which you will connect (as of this writing, the same | |
| server and port to which slaves connect). | |
| - buildbot_port: (optional, defaults to 9989) the port of the buildbot master | |
| to which you will connect (as of this writing, the same server and port to | |
| which slaves connect) | |
| - buildbot_pqm: (optional, defaults to not pqm) Normally, the user that | |
| commits the revision is the user that is responsible for the change. When | |
| run in a pqm (Patch Queue Manager, see https://launchpad.net/pqm) | |
| environment, the user that commits is the Patch Queue Manager, and the user | |
| that committed the *parent* revision is responsible for the change. To turn | |
| on the pqm mode, set this value to any of (case-insensitive) "Yes", "Y", | |
| "True", or "T". | |
| - buildbot_dry_run: (optional, defaults to not a dry run) Normally, the | |
| post-commit hook will attempt to communicate with the configured buildbot | |
| server and port. If this parameter is included and any of (case-insensitive) | |
| "Yes", "Y", "True", or "T", then the hook will simply print what it would | |
| have sent, but not attempt to contact the buildbot master. | |
| - buildbot_send_branch_name: (optional, defaults to not sending the branch | |
| name) If your buildbot's bzr source build step uses a repourl, do | |
| *not* turn this on. If your buildbot's bzr build step uses a baseURL, then | |
| you may set this value to any of (case-insensitive) "Yes", "Y", "True", or | |
| "T" to have the buildbot master append the branch name to the baseURL. | |
| When buildbot no longer has a hardcoded password, it will be a configuration | |
| option here as well. | |
| ------ | |
| Poller | |
| ------ | |
| Put this file somewhere that your buildbot configuration can import it. Even | |
| in the same directory as the master.cfg should work. Install the poller in | |
| the buildbot configuration as with any other change source. Minimally, | |
| provide a URL that you want to poll (bzr://, bzr+ssh://, or lp:), though make | |
| sure the buildbot user has necessary privileges. You may also want to specify | |
| these optional values. | |
| poll_interval: the number of seconds to wait between polls. Defaults to 10 | |
| minutes. | |
| branch_name: any value to be used as the branch name. Defaults to None, or | |
| specify a string, or specify the constants from this file SHORT | |
| or FULL to get the short branch name or full branch address. | |
| blame_merge_author: normally, the user that commits the revision is the user | |
| that is responsible for the change. When run in a pqm | |
| (Patch Queue Manager, see https://launchpad.net/pqm) | |
| environment, the user that commits is the Patch Queue | |
| Manager, and the user that committed the merged, *parent* | |
| revision is responsible for the change. set this value to | |
| True if this is pointed against a PQM-managed branch. | |
| ------------------- | |
| Contact Information | |
| ------------------- | |
| Maintainer/author: gary.poster@canonical.com | |
| """ | |
| try: | |
| import buildbot.util | |
| import buildbot.changes.base | |
| import buildbot.changes.changes | |
| except ImportError: | |
| DEFINE_POLLER = False | |
| else: | |
| DEFINE_POLLER = True | |
| import bzrlib.branch | |
| import bzrlib.errors | |
| import bzrlib.trace | |
| import twisted.cred.credentials | |
| import twisted.internet.base | |
| import twisted.internet.defer | |
| import twisted.internet.reactor | |
| import twisted.internet.selectreactor | |
| import twisted.internet.task | |
| import twisted.internet.threads | |
| import twisted.python.log | |
| import twisted.spread.pb | |
| ############################################################################# | |
| # This is the code that the poller and the hooks share. | |
| def generate_change(branch, | |
| old_revno=None, old_revid=None, | |
| new_revno=None, new_revid=None, | |
| blame_merge_author=False): | |
| """Return a dict of information about a change to the branch. | |
| Dict has keys of "files", "who", "comments", and "revision", as used by | |
| the buildbot Change (and the PBChangeSource). | |
| If only the branch is given, the most recent change is returned. | |
| If only the new_revno is given, the comparison is expected to be between | |
| it and the previous revno (new_revno -1) in the branch. | |
| Passing old_revid and new_revid is only an optimization, included because | |
| bzr hooks usually provide this information. | |
| blame_merge_author means that the author of the merged branch is | |
| identified as the "who", not the person who committed the branch itself. | |
| This is typically used for PQM. | |
| """ | |
| change = {} # files, who, comments, revision; NOT branch (= branch.nick) | |
| if new_revno is None: | |
| new_revno = branch.revno() | |
| if new_revid is None: | |
| new_revid = branch.get_rev_id(new_revno) | |
| # TODO: This falls over if this is the very first revision | |
| if old_revno is None: | |
| old_revno = new_revno -1 | |
| if old_revid is None: | |
| old_revid = branch.get_rev_id(old_revno) | |
| repository = branch.repository | |
| new_rev = repository.get_revision(new_revid) | |
| if blame_merge_author: | |
| # this is a pqm commit or something like it | |
| change['who'] = repository.get_revision( | |
| new_rev.parent_ids[-1]).get_apparent_author() | |
| else: | |
| change['who'] = new_rev.get_apparent_author() | |
| # maybe useful to know: | |
| # name, email = bzrtools.config.parse_username(change['who']) | |
| change['comments'] = new_rev.message | |
| change['revision'] = new_revno | |
| files = change['files'] = [] | |
| changes = repository.revision_tree(new_revid).changes_from( | |
| repository.revision_tree(old_revid)) | |
| for (collection, name) in ((changes.added, 'ADDED'), | |
| (changes.removed, 'REMOVED'), | |
| (changes.modified, 'MODIFIED')): | |
| for info in collection: | |
| path = info[0] | |
| kind = info[2] | |
| files.append(' '.join([path, kind, name])) | |
| for info in changes.renamed: | |
| oldpath, newpath, id, kind, text_modified, meta_modified = info | |
| elements = [oldpath, kind,'RENAMED', newpath] | |
| if text_modified or meta_modified: | |
| elements.append('MODIFIED') | |
| files.append(' '.join(elements)) | |
| return change | |
| ############################################################################# | |
| # poller | |
| # We don't want to make the hooks unnecessarily depend on buildbot being | |
| # installed locally, so we conditionally create the BzrPoller class. | |
| if DEFINE_POLLER: | |
| FULL = object() | |
| SHORT = object() | |
| class BzrPoller(buildbot.changes.base.ChangeSource, | |
| buildbot.util.ComparableMixin): | |
| compare_attrs = ['url'] | |
| def __init__(self, url, poll_interval=10*60, blame_merge_author=False, | |
| branch_name=None): | |
| # poll_interval is in seconds, so default poll_interval is 10 | |
| # minutes. | |
| # bzr+ssh://bazaar.launchpad.net/~launchpad-pqm/launchpad/devel/ | |
| # works, lp:~launchpad-pqm/launchpad/devel/ doesn't without help. | |
| if url.startswith('lp:'): | |
| url = 'bzr+ssh://bazaar.launchpad.net/' + url[3:] | |
| self.url = url | |
| self.poll_interval = poll_interval | |
| self.loop = twisted.internet.task.LoopingCall(self.poll) | |
| self.blame_merge_author = blame_merge_author | |
| self.branch_name = branch_name | |
| def startService(self): | |
| twisted.python.log.msg("BzrPoller(%s) starting" % self.url) | |
| buildbot.changes.base.ChangeSource.startService(self) | |
| twisted.internet.reactor.callWhenRunning( | |
| self.loop.start, self.poll_interval) | |
| for change in reversed(self.parent.changes): | |
| if change.branch == self.url: | |
| self.last_revision = change.revision | |
| break | |
| else: | |
| self.last_revision = None | |
| self.polling = False | |
| def stopService(self): | |
| twisted.python.log.msg("BzrPoller(%s) shutting down" % self.url) | |
| self.loop.stop() | |
| return buildbot.changes.base.ChangeSource.stopService(self) | |
| def describe(self): | |
| return "BzrPoller watching %s" % self.url | |
| @twisted.internet.defer.inlineCallbacks | |
| def poll(self): | |
| if self.polling: # this is called in a loop, and the loop might | |
| # conceivably overlap. | |
| return | |
| self.polling = True | |
| try: | |
| # On a big tree, even individual elements of the bzr commands | |
| # can take awhile. So we just push the bzr work off to a | |
| # thread. | |
| try: | |
| changes = yield twisted.internet.threads.deferToThread( | |
| self.getRawChanges) | |
| except (SystemExit, KeyboardInterrupt): | |
| raise | |
| except: | |
| # we'll try again next poll. Meanwhile, let's report. | |
| twisted.python.log.err() | |
| else: | |
| for change in changes: | |
| yield self.addChange( | |
| buildbot.changes.changes.Change(**change)) | |
| self.last_revision = change['revision'] | |
| finally: | |
| self.polling = False | |
| def getRawChanges(self): | |
| branch = bzrlib.branch.Branch.open_containing(self.url)[0] | |
| if self.branch_name is FULL: | |
| branch_name = self.url | |
| elif self.branch_name is SHORT: | |
| branch_name = branch.nick | |
| else: # presumably a string or maybe None | |
| branch_name = self.branch_name | |
| changes = [] | |
| change = generate_change( | |
| branch, blame_merge_author=self.blame_merge_author) | |
| if (self.last_revision is None or | |
| change['revision'] > self.last_revision): | |
| changes.append(change) | |
| if self.last_revision is not None: | |
| while self.last_revision + 1 < change['revision']: | |
| change = generate_change( | |
| branch, new_revno=change['revision']-1, | |
| blame_merge_author=self.blame_merge_author) | |
| change['branch'] = branch_name | |
| changes.append(change) | |
| changes.reverse() | |
| return changes | |
| def addChange(self, change): | |
| d = twisted.internet.defer.Deferred() | |
| def _add_change(): | |
| d.callback( | |
| self.parent.addChange(change)) | |
| twisted.internet.reactor.callLater(0, _add_change) | |
| return d | |
| ############################################################################# | |
| # hooks | |
| HOOK_KEY = 'buildbot_on' | |
| SERVER_KEY = 'buildbot_server' | |
| PORT_KEY = 'buildbot_port' | |
| DRYRUN_KEY = 'buildbot_dry_run' | |
| PQM_KEY = 'buildbot_pqm' | |
| SEND_BRANCHNAME_KEY = 'buildbot_send_branch_name' | |
| PUSH_VALUE = 'push' | |
| COMMIT_VALUE = 'commit' | |
| CHANGE_VALUE = 'change' | |
| def _is_true(config, key): | |
| val = config.get_user_option(key) | |
| return val is not None and val.lower().strip() in ( | |
| 'y', 'yes', 't', 'true') | |
| def _installed_hook(branch): | |
| value = branch.get_config().get_user_option(HOOK_KEY) | |
| if value is not None: | |
| value = value.strip().lower() | |
| if value not in (PUSH_VALUE, COMMIT_VALUE, CHANGE_VALUE): | |
| raise bzrlib.errors.BzrError( | |
| '%s, if set, must be one of %s, %s, or %s' % ( | |
| HOOK_KEY, PUSH_VALUE, COMMIT_VALUE, CHANGE_VALUE)) | |
| return value | |
| ########################## | |
| # Work around Twisted bug. | |
| # See http://twistedmatrix.com/trac/ticket/3591 | |
| import operator | |
| import socket | |
| from twisted.internet import defer | |
| from twisted.python import failure | |
| # replaces twisted.internet.thread equivalent | |
| def _putResultInDeferred(reactor, deferred, f, args, kwargs): | |
| """ | |
| Run a function and give results to a Deferred. | |
| """ | |
| try: | |
| result = f(*args, **kwargs) | |
| except: | |
| f = failure.Failure() | |
| reactor.callFromThread(deferred.errback, f) | |
| else: | |
| reactor.callFromThread(deferred.callback, result) | |
| # would be a proposed addition. deferToThread could use it | |
| def deferToThreadInReactor(reactor, f, *args, **kwargs): | |
| """ | |
| Run function in thread and return result as Deferred. | |
| """ | |
| d = defer.Deferred() | |
| reactor.callInThread(_putResultInDeferred, reactor, d, f, args, kwargs) | |
| return d | |
| # uses its own reactor for the threaded calls, unlike Twisted's | |
| class ThreadedResolver(twisted.internet.base.ThreadedResolver): | |
| def getHostByName(self, name, timeout = (1, 3, 11, 45)): | |
| if timeout: | |
| timeoutDelay = reduce(operator.add, timeout) | |
| else: | |
| timeoutDelay = 60 | |
| userDeferred = defer.Deferred() | |
| lookupDeferred = deferToThreadInReactor( | |
| self.reactor, socket.gethostbyname, name) | |
| cancelCall = self.reactor.callLater( | |
| timeoutDelay, self._cleanup, name, lookupDeferred) | |
| self._runningQueries[lookupDeferred] = (userDeferred, cancelCall) | |
| lookupDeferred.addBoth(self._checkTimeout, name, lookupDeferred) | |
| return userDeferred | |
| ########################## | |
| def send_change(branch, old_revno, old_revid, new_revno, new_revid, hook): | |
| config = branch.get_config() | |
| server = config.get_user_option(SERVER_KEY) | |
| if not server: | |
| bzrlib.trace.warning( | |
| 'bzr_buildbot: ERROR. If %s is set, %s must be set', | |
| HOOK_KEY, SERVER_KEY) | |
| return | |
| change = generate_change( | |
| branch, old_revno, old_revid, new_revno, new_revid, | |
| blame_merge_author=_is_true(config, PQM_KEY)) | |
| if _is_true(config, SEND_BRANCHNAME_KEY): | |
| change['branch'] = branch.nick | |
| # as of this writing (in Buildbot 0.7.9), 9989 is the default port when | |
| # you make a buildbot master. | |
| port = int(config.get_user_option(PORT_KEY) or 9989) | |
| # if dry run, stop. | |
| if _is_true(config, DRYRUN_KEY): | |
| bzrlib.trace.note("bzr_buildbot DRY RUN " | |
| "(*not* sending changes to %s:%d on %s)", | |
| server, port, hook) | |
| keys = change.keys() | |
| keys.sort() | |
| for k in keys: | |
| bzrlib.trace.note("[%10s]: %s", k, change[k]) | |
| return | |
| # We instantiate our own reactor so that this can run within a server. | |
| reactor = twisted.internet.selectreactor.SelectReactor() | |
| # See other reference to http://twistedmatrix.com/trac/ticket/3591 | |
| # above. This line can go away with a release of Twisted that addresses | |
| # this issue. | |
| reactor.resolver = ThreadedResolver(reactor) | |
| pbcf = twisted.spread.pb.PBClientFactory() | |
| reactor.connectTCP(server, port, pbcf) | |
| deferred = pbcf.login( | |
| twisted.cred.credentials.UsernamePassword('change', 'changepw')) | |
| def sendChanges(remote): | |
| """Send changes to buildbot.""" | |
| bzrlib.trace.mutter("bzrbuildout sending changes: %s", change) | |
| return remote.callRemote('addChange', change) | |
| deferred.addCallback(sendChanges) | |
| def quit(ignore, msg): | |
| bzrlib.trace.note("bzrbuildout: %s", msg) | |
| reactor.stop() | |
| def failed(failure): | |
| bzrlib.trace.warning("bzrbuildout: FAILURE\n %s", failure) | |
| reactor.stop() | |
| deferred.addCallback(quit, "SUCCESS") | |
| deferred.addErrback(failed) | |
| reactor.callLater(60, quit, None, "TIMEOUT") | |
| bzrlib.trace.note( | |
| "bzr_buildbot: SENDING CHANGES to buildbot master %s:%d on %s", | |
| server, port, hook) | |
| reactor.run(installSignalHandlers=False) # run in a thread when in server | |
| def post_commit(local_branch, master_branch, # branch is the master_branch | |
| old_revno, old_revid, new_revno, new_revid): | |
| if _installed_hook(master_branch) == COMMIT_VALUE: | |
| send_change(master_branch, | |
| old_revid, old_revid, new_revno, new_revid, COMMIT_VALUE) | |
| def post_push(result): | |
| if _installed_hook(result.target_branch) == PUSH_VALUE: | |
| send_change(result.target_branch, | |
| result.old_revid, result.old_revid, | |
| result.new_revno, result.new_revid, PUSH_VALUE) | |
| def post_change_branch_tip(result): | |
| if _installed_hook(result.branch) == CHANGE_VALUE: | |
| send_change(result.branch, | |
| result.old_revid, result.old_revid, | |
| result.new_revno, result.new_revid, CHANGE_VALUE) | |
| bzrlib.branch.Branch.hooks.install_named_hook( | |
| 'post_commit', post_commit, | |
| 'send change to buildbot master') | |
| bzrlib.branch.Branch.hooks.install_named_hook( | |
| 'post_push', post_push, | |
| 'send change to buildbot master') | |
| bzrlib.branch.Branch.hooks.install_named_hook( | |
| 'post_change_branch_tip', post_change_branch_tip, | |
| 'send change to buildbot master') |