split

setup.py → master/setup.py

@@ -192,8 +192,6 @@ def finalize_options(self):
               "buildbot.steps.package.rpm",
               "buildbot.process",
               "buildbot.clients",
-              "buildbot.slave",
-              "buildbot.slave.commands",
               "buildbot.schedulers",
               "buildbot.scripts",
               "buildbot.db",

slave/bbslave/__init__.py

@@ -0,0 +1 @@
+version = "latest"

buildbot/slave/bot.py → slave/bbslave/bot.py

@@ -1,22 +1,20 @@
-
 import os.path
 import sys
 
-import buildbot
-
 from twisted.spread import pb
 from twisted.python import log
 from twisted.internet import reactor, defer
 from twisted.application import service, internet
 from twisted.cred import credentials
 
-from buildbot.util import now
-from buildbot.pbutil import ReconnectingPBClientFactory
-from buildbot.slave.commands import registry
+import bbslave
+from bbslave.util import now
+from bbslave.pbutil import ReconnectingPBClientFactory
+from bbslave.commands import registry
 
 # make sure the standard commands get registered. This import is performed
 # for its side-effects.
-from buildbot.slave.commands import base, transfer, vcs
+from bbslave.commands import base, transfer, vcs
 
 class NoCommandRunning(pb.Error):
     pass
@@ -343,7 +341,7 @@ def remote_getSlaveInfo(self):
 
     def remote_getVersion(self):
         """Send our version back to the Master"""
-        return buildbot.version
+        return bbslave.version
 
 
 
@@ -480,7 +478,7 @@ class BuildSlave(service.MultiService):
     def __init__(self, buildmaster_host, port, name, passwd, basedir,
                  keepalive, usePTY, keepaliveTimeout=30, umask=None,
                  maxdelay=300, debugOpts={}, unicode_encoding=None):
-        log.msg("Creating BuildSlave -- buildbot.version: %s" % buildbot.version)
+        log.msg("Creating BuildSlave -- version: %s" % bbslave.version)
         service.MultiService.__init__(self)
         self.debugOpts = debugOpts.copy()
         bot = self.botClass(basedir, usePTY, unicode_encoding=unicode_encoding)

buildbot/slave/commands/base.py → slave/bbslave/commands/base.py

@@ -1,4 +1,3 @@
-# -*- test-case-name: buildbot.test.test_slavecommand -*-
 
 import os, signal, types, re, traceback
 from stat import ST_CTIME, ST_MTIME, ST_SIZE
@@ -9,9 +8,9 @@
 from twisted.internet import reactor, defer, task
 from twisted.python import log, runtime
 
-from buildbot.slave.interfaces import ISlaveCommand
-from buildbot.slave.commands.registry import registerSlaveCommand
-from buildbot import util
+from bbslave.interfaces import ISlaveCommand
+from bbslave.commands.registry import registerSlaveCommand
+from bbslave import util
 
 # this used to be a CVS $-style "Revision" auto-updated keyword, but since I
 # moved to Darcs as the primary repository, this is updated manually each

buildbot/slave/commands/registry.py → slave/bbslave/commands/registry.py

@@ -7,9 +7,9 @@ def registerSlaveCommand(name, factory, version):
 
     @type  name:    string
     @param name:    name under which the slave command will be registered; used
-                    for L{buildbot.slave.bot.SlaveBuilder.remote_startCommand}
+                    for L{bbslave.bot.SlaveBuilder.remote_startCommand}
                     
-    @type  factory: L{buildbot.slave.commands.Command}
+    @type  factory: L{bbslave.commands.Command}
     @type  version: string
     @param version: version string of the factory code
     """

buildbot/slave/commands/transfer.py → slave/bbslave/commands/transfer.py

@@ -3,8 +3,8 @@
 from twisted.python import log
 from twisted.internet import defer
 
-from buildbot.slave.commands.base import Command, command_version
-from buildbot.slave.commands.registry import registerSlaveCommand
+from bbslave.commands.base import Command, command_version
+from bbslave.commands.registry import registerSlaveCommand
 
 class SlaveFileUploadCommand(Command):
     """

buildbot/slave/commands/vcs.py → slave/bbslave/commands/vcs.py

@@ -5,10 +5,10 @@
 from twisted.python import log, failure, runtime
 from twisted.internet import defer
 
-from buildbot.slave.commands.base import Command, ShellCommand, AbandonChain, command_version, Obfuscated
-from buildbot.slave.commands.registry import registerSlaveCommand
-from buildbot.slave.commands.utils import getCommand, rmdirRecursive
-from buildbot.util import remove_userpassword
+from bbslave.commands.base import Command, ShellCommand, AbandonChain, command_version, Obfuscated
+from bbslave.commands.registry import registerSlaveCommand
+from bbslave.commands.utils import getCommand, rmdirRecursive
+from bbslave.util import remove_userpassword
 
 class SourceBase(Command):
     """Abstract base class for Version Control System operations (checkout

slave/bbslave/pbutil.py

@@ -0,0 +1,140 @@
+
+"""Base classes handy for use with PB clients.
+"""
+
+from twisted.spread import pb
+
+from twisted.spread.pb import PBClientFactory
+from twisted.internet import protocol
+from twisted.python import log
+
+class NewCredPerspective(pb.Avatar):
+    def attached(self, mind):
+        return self
+    def detached(self, mind):
+        pass
+
+class ReconnectingPBClientFactory(PBClientFactory,
+                                  protocol.ReconnectingClientFactory):
+    """Reconnecting client factory for PB brokers.
+
+    Like PBClientFactory, but if the connection fails or is lost, the factory
+    will attempt to reconnect.
+
+    Instead of using f.getRootObject (which gives a Deferred that can only
+    be fired once), override the gotRootObject method.
+
+    Instead of using the newcred f.login (which is also one-shot), call
+    f.startLogin() with the credentials and client, and override the
+    gotPerspective method.
+
+    Instead of using the oldcred f.getPerspective (also one-shot), call
+    f.startGettingPerspective() with the same arguments, and override
+    gotPerspective.
+
+    gotRootObject and gotPerspective will be called each time the object is
+    received (once per successful connection attempt). You will probably want
+    to use obj.notifyOnDisconnect to find out when the connection is lost.
+
+    If an authorization error occurs, failedToGetPerspective() will be
+    invoked.
+
+    To use me, subclass, then hand an instance to a connector (like
+    TCPClient).
+    """
+
+    def __init__(self):
+        PBClientFactory.__init__(self)
+        self._doingLogin = False
+        self._doingGetPerspective = False
+
+    def clientConnectionFailed(self, connector, reason):
+        PBClientFactory.clientConnectionFailed(self, connector, reason)
+        # Twisted-1.3 erroneously abandons the connection on non-UserErrors.
+        # To avoid this bug, don't upcall, and implement the correct version
+        # of the method here.
+        if self.continueTrying:
+            self.connector = connector
+            self.retry()
+
+    def clientConnectionLost(self, connector, reason):
+        PBClientFactory.clientConnectionLost(self, connector, reason,
+                                             reconnecting=True)
+        RCF = protocol.ReconnectingClientFactory
+        RCF.clientConnectionLost(self, connector, reason)
+
+    def clientConnectionMade(self, broker):
+        self.resetDelay()
+        PBClientFactory.clientConnectionMade(self, broker)
+        if self._doingLogin:
+            self.doLogin(self._root)
+        if self._doingGetPerspective:
+            self.doGetPerspective(self._root)
+        self.gotRootObject(self._root)
+
+    # oldcred methods
+
+    def getPerspective(self, *args):
+        raise RuntimeError, "getPerspective is one-shot: use startGettingPerspective instead"
+
+    def startGettingPerspective(self, username, password, serviceName,
+                                perspectiveName=None, client=None):
+        self._doingGetPerspective = True
+        if perspectiveName == None:
+            perspectiveName = username
+        self._oldcredArgs = (username, password, serviceName,
+                             perspectiveName, client)
+
+    def doGetPerspective(self, root):
+        # oldcred getPerspective()
+        (username, password,
+         serviceName, perspectiveName, client) = self._oldcredArgs
+        d = self._cbAuthIdentity(root, username, password)
+        d.addCallback(self._cbGetPerspective,
+                      serviceName, perspectiveName, client)
+        d.addCallbacks(self.gotPerspective, self.failedToGetPerspective)
+
+
+    # newcred methods
+
+    def login(self, *args):
+        raise RuntimeError, "login is one-shot: use startLogin instead"
+
+    def startLogin(self, credentials, client=None):
+        self._credentials = credentials
+        self._client = client
+        self._doingLogin = True
+
+    def doLogin(self, root):
+        # newcred login()
+        d = self._cbSendUsername(root, self._credentials.username,
+                                 self._credentials.password, self._client)
+        d.addCallbacks(self.gotPerspective, self.failedToGetPerspective)
+
+
+    # methods to override
+
+    def gotPerspective(self, perspective):
+        """The remote avatar or perspective (obtained each time this factory
+        connects) is now available."""
+        pass
+
+    def gotRootObject(self, root):
+        """The remote root object (obtained each time this factory connects)
+        is now available. This method will be called each time the connection
+        is established and the object reference is retrieved."""
+        pass
+
+    def failedToGetPerspective(self, why):
+        """The login process failed, most likely because of an authorization
+        failure (bad password), but it is also possible that we lost the new
+        connection before we managed to send our credentials.
+        """
+        log.msg("ReconnectingPBClientFactory.failedToGetPerspective")
+        if why.check(pb.PBConnectionLost):
+            log.msg("we lost the brand-new connection")
+            # retrying might help here, let clientConnectionLost decide
+            return
+        # probably authorization
+        self.stopTrying() # logging in harder won't help
+        log.err(why)

slave/bbslave/scripts/checkconfig.py

@@ -0,0 +1,38 @@
+import sys
+import os
+from shutil import copy, rmtree
+from tempfile import mkdtemp
+from os.path import isfile
+
+from buildbot import master
+
+class ConfigLoader(master.BuildMaster):
+    def __init__(self, basedir=os.getcwd(), configFileName="master.cfg"):
+        master.BuildMaster.__init__(self, basedir, configFileName)
+        configFileName = os.path.join(basedir, configFileName)
+        dir = os.getcwd()
+        # Use a temporary directory since loadConfig() creates a bunch of
+        # directories and compiles .py files
+        tempdir = mkdtemp()
+        try:
+            copy(configFileName, tempdir)
+            for entry in os.listdir("."):
+                # Any code in a subdirectory will _not_ be copied! This is a bug
+                if isfile(entry) and not entry.startswith("twistd.log"):
+                    copy(entry, tempdir)
+        except:
+            raise
+
+        try:
+            os.chdir(tempdir)
+            # Add the temp directory to the library path so local modules work
+            sys.path.append(tempdir)
+            configFile = open(configFileName, "r")
+            self.loadConfig(configFile, check_synchronously_only=True)
+        except:
+            os.chdir(dir)
+            configFile.close()
+            rmtree(tempdir)
+            raise
+        os.chdir(dir)
+        rmtree(tempdir)

slave/bbslave/scripts/logwatcher.py

@@ -0,0 +1,101 @@
+
+import os
+from twisted.python.failure import Failure
+from twisted.internet import defer, reactor, protocol, error
+from twisted.protocols.basic import LineOnlyReceiver
+
+class FakeTransport:
+    disconnecting = False
+
+class BuildmasterTimeoutError(Exception):
+    pass
+class BuildslaveTimeoutError(Exception):
+    pass
+class ReconfigError(Exception):
+    pass
+class BuildSlaveDetectedError(Exception):
+    pass
+
+class TailProcess(protocol.ProcessProtocol):
+    def outReceived(self, data):
+        self.lw.dataReceived(data)
+    def errReceived(self, data):
+        print "ERR: '%s'" % (data,)
+
+
+class LogWatcher(LineOnlyReceiver):
+    POLL_INTERVAL = 0.1
+    TIMEOUT_DELAY = 10.0
+    delimiter = os.linesep
+
+    def __init__(self, logfile):
+        self.logfile = logfile
+        self.in_reconfig = False
+        self.transport = FakeTransport()
+        self.pp = TailProcess()
+        self.pp.lw = self
+        self.processtype = "buildmaster"
+        self.timer = None
+
+    def start(self):
+        # If the log file doesn't exist, create it now.
+        if not os.path.exists(self.logfile):
+            open(self.logfile, 'a').close()
+
+        # return a Deferred that fires when the reconfig process has
+        # finished. It errbacks with TimeoutError if the finish line has not
+        # been seen within 10 seconds, and with ReconfigError if the error
+        # line was seen. If the logfile could not be opened, it errbacks with
+        # an IOError.
+        self.p = reactor.spawnProcess(self.pp, "/usr/bin/tail",
+                                      ("tail", "-f", "-n", "0", self.logfile),
+                                      env=os.environ,
+                                      )
+        self.running = True
+        d = defer.maybeDeferred(self._start)
+        return d
+
+    def _start(self):
+        self.d = defer.Deferred()
+        self.timer = reactor.callLater(self.TIMEOUT_DELAY, self.timeout)
+        return self.d
+
+    def timeout(self):
+        self.timer = None
+        if self.processtype == "buildmaster":
+            e = BuildmasterTimeoutError()
+        else:
+            e = BuildslaveTimeoutError()
+        self.finished(Failure(e))
+
+    def finished(self, results):
+        try:
+            self.p.signalProcess("KILL")
+        except error.ProcessExitedAlready:
+            pass
+        if self.timer:
+            self.timer.cancel()
+            self.timer = None
+        self.running = False
+        self.in_reconfig = False
+        self.d.callback(results)
+
+    def lineReceived(self, line):
+        if not self.running:
+            return
+        if "Log opened." in line:
+            self.in_reconfig = True
+        if "loading configuration from" in line:
+            self.in_reconfig = True
+        if "Creating BuildSlave" in line:
+            self.processtype = "buildslave"
+
+        if self.in_reconfig:
+            print line
+
+        if "message from master: attached" in line:
+            return self.finished("buildslave")
+        if "I will keep using the previous config file" in line:
+            return self.finished(Failure(ReconfigError()))
+        if "configuration update complete" in line:
+            return self.finished("buildmaster")