Merge branch 'server-source' of git://github.com/tomprince/buildbot

* 'server-source' of git://github.com/tomprince/buildbot:
  Handle with properties for the string in buildbot.steps.transfer.StringDownload.
  Add slave commands to make and remove directories.
  Add SetPropertiesFromEnv command, and slave side support. (closes #814)

(with whitespace cleanup)

master/buildbot/buildslave.py

@@ -249,6 +249,7 @@ def _got_info(info):
                 state["admin"] = info.get("admin")
                 state["host"] = info.get("host")
                 state["access_uri"] = info.get("access_uri", None)
+                state["slave_environ"] = info.get("environ", {})
             def _info_unavailable(why):
                 # maybe an old slave, doesn't implement remote_getSlaveInfo
                 log.msg("BuildSlave.info_unavailable")
@@ -289,6 +290,7 @@ def _accept_slave(res):
             self.slave_status.setVersion(state.get("version"))
             self.slave_status.setConnected(True)
             self.slave_commands = state.get("slave_commands")
+            self.slave_environ = state.get("slave_environ")
             self.slave = bot
             log.msg("bot attached")
             self.messageReceivedFromSlave()

master/buildbot/steps/master.py

@@ -102,3 +102,31 @@ def processEnded(self, status_object):
         else:
             self.step_status.setText(list(self.descriptionDone))
             self.finished(SUCCESS)
+
+class SetPropertiesFromEnv(BuildStep):
+    """
+    Sets properties from envirionment variables on the slave.
+
+    Note this is transfered when the slave first connects
+    """
+    name='SetPropertiesFromEnv'
+    description='Setting'
+    descriptionDone='Set'
+
+    def __init__(self, variables, source="SlaveEnvironment", **kwargs):
+        BuildStep.__init__(self, **kwargs)
+        self.addFactoryArguments(variables = variables,
+                                 source = source)
+        self.variables = variables
+        self.source = source
+
+    def start(self):
+        properties = self.build.getProperties()
+        environ = self.buildslave.slave_environ
+        if isinstance(self.variables, str):
+            self.variables = [self.variables]
+        for variable in self.variables:
+            value = environ.get(variable, None)
+            if value:
+                properties.setProperty(variable, value, self.source, runtime=True)
+        self.finished(SUCCESS)

master/buildbot/steps/transfer.py

@@ -548,7 +548,7 @@ def start(self):
                                   os.path.basename(slavedest)])
 
         # setup structures for reading the file
-        fp = StringIO(self.s)
+        fp = StringIO(properties.render(self.s))
         fileReader = _FileReader(fp)
 
         # default arguments

master/buildbot/test/unit/test_steps_master.py

@@ -0,0 +1,33 @@
+from twisted.trial import unittest
+
+from mock import Mock
+
+from buildbot.process.properties import Properties
+from buildbot.steps.master import SetPropertiesFromEnv
+
+class TestSetPropertiesFromEnv(unittest.TestCase):
+    def testBasic(self):
+        s = SetPropertiesFromEnv(variables = ["one", "two", "three", "five", "six"], source = "me")
+        s.build = Mock()
+        s.build.getProperties.return_value = props = Properties()
+        s.buildslave = Mock()
+        s.buildslave.slave_environ = { "one": 1, "two": None, "six": 6 }
+        props.setProperty("four", 4, "them")
+        props.setProperty("five", 5, "them")
+        props.setProperty("six", 99, "them")
+
+        s.step_status = Mock()
+        s.deferred = Mock()
+
+        s.start()
+
+        self.failUnlessEqual(props.getProperty('one'), 1)
+        self.failUnlessEqual(props.getPropertySource('one'), 'me')
+        self.failUnlessEqual(props.getProperty('two'), None)
+        self.failUnlessEqual(props.getProperty('three'), None)
+        self.failUnlessEqual(props.getProperty('four'), 4)
+        self.failUnlessEqual(props.getPropertySource('four'), 'them')
+        self.failUnlessEqual(props.getProperty('five'), 5)
+        self.failUnlessEqual(props.getPropertySource('five'), 'them')
+        self.failUnlessEqual(props.getProperty('six'), 6)
+        self.failUnlessEqual(props.getPropertySource('six'), 'me')

master/docs/cfg-buildsteps.texinfo

@@ -1535,6 +1535,8 @@ f.addStep(SetProperty(
 Then @code{my_extract} will see @code{stdout="output1\noutput2\n"}
 and @code{stderr="error\n"}.
 
+See also @ref{SetProperiesFromEnv}.
+
 @node SubunitShellCommand
 @subsubsection SubunitShellCommand
 
@@ -1896,6 +1898,16 @@ Note that, by default, this step passes a copy of the buildmaster's environment
 variables to the subprocess.  To pass an explicit environment instead, add an
 @code{env=@{..@}} argument.
 
+@bsindex buildbot.steps.master.SetPropertiesFromEnv
+
+@example
+from buildbot.steps.master import SetProperiesFromEnv
+from buildbot.steps.shell import Compile
+
+f.addStep(SetPropertiesFromEnv(variables=["SOME_JAVA_LIB_HOME", "JAVAC"]))
+f.addStep(Compile(commands=[WithProperties("%s","JAVAC"), "-cp", WithProperties("%s", "SOME_JAVA_LIB_HOME")))
+@end example
+
 @node Triggering Schedulers
 @subsection Triggering Schedulers
 @bsindex buildbot.steps.trigger.Trigger

slave/buildslave/bot.py

@@ -285,18 +285,20 @@ def remote_getSlaveInfo(self):
 
         files = {}
         basedir = os.path.join(self.basedir, "info")
-        if not os.path.isdir(basedir):
-            return files
-        for f in os.listdir(basedir):
-            filename = os.path.join(basedir, f)
-            if os.path.isfile(filename):
-                files[f] = open(filename, "r").read()
+        if os.path.isdir(basedir):
+            for f in os.listdir(basedir):
+                filename = os.path.join(basedir, f)
+                if os.path.isfile(filename):
+                    files[f] = open(filename, "r").read()
+        files['environ'] = os.environ.copy()
         return files
 
     def remote_getVersion(self):
         """Send our version back to the Master"""
         return buildslave.version
 
+    def remote_getEnviron(self):
+        return os.environ.copy()
 
 
 class BotFactory(ReconnectingPBClientFactory):

slave/buildslave/commands/fs.py

@@ -0,0 +1,174 @@
+import os
+
+from twisted.internet import defer
+from twisted.python import runtime
+
+from buildslave import runprocess
+from buildslave.commands import base, utils
+
+class MakeDirectory(base.Command):
+    """This is a Command which creates a directory. The args dict contains
+    the following keys:
+
+        - ['dir'] (required): subdirectory which the command will create,
+                                  relative to the builder dir
+
+    MakeDirectory creates the following status messages:
+        - {'rc': rc} : when the process has terminated
+    """
+
+    header = "mkdir"
+
+    def start(self):
+        args = self.args
+        # args['dir'] is relative to Builder directory, and is required.
+        assert args['dir'] is not None
+        dirname = os.path.join(self.builder.basedir, args['dir'])
+
+        try:
+            if not os.path.isdir(dirname):
+                os.makedirs(dirname)
+            self.sendStatus({'rc': 0})
+        except:
+            self.sendStatus({'rc': 1})
+
+class RemoveDirectory(base.Command):
+    """This is a Command which removes a directory. The args dict contains
+    the following keys:
+
+        - ['dir'] (required): subdirectory which the command will create,
+                                  relative to the builder dir
+
+        - ['timeout']:  seconds of silence tolerated before we kill off the
+                        command
+
+        - ['maxTime']:  seconds before we kill off the command
+
+
+    RemoveDirectory creates the following status messages:
+        - {'rc': rc} : when the process has terminated
+    """
+
+    header = "rmdir"
+
+    def start(self):
+        args = self.args
+        # args['dir'] is relative to Builder directory, and is required.
+        assert args['dir'] is not None
+        dirname = args['dir']
+
+        self.timeout = args.get('timeout', 120)
+        self.maxTime = args.get('maxTime', None)
+
+        # TODO: remove the old tree in the background
+        self.dir = os.path.join(self.builder.basedir, dirname)
+        if runtime.platformType != "posix":
+            # if we're running on w32, use rmtree instead. It will block,
+            # but hopefully it won't take too long.
+            utils.rmdirRecursive(self.dir)
+            return defer.succeed(0)
+
+	d = self._clobber(None)
+	d.addCallback(self._sendRC)
+	return d
+
+    def _clobber(self, dummy, chmodDone = False):
+        command = ["rm", "-rf", self.dir]
+        c = runprocess.RunProcess(self.builder, command, self.builder.basedir,
+                         sendRC=0, timeout=self.timeout, maxTime=self.maxTime,
+                         usePTY=False)
+
+        self.command = c
+        # sendRC=0 means the rm command will send stdout/stderr to the
+        # master, but not the rc=0 when it finishes. That job is left to
+        # _sendRC
+        d = c.start()
+        # The rm -rf may fail if there is a left-over subdir with chmod 000
+        # permissions. So if we get a failure, we attempt to chmod suitable
+        # permissions and re-try the rm -rf.
+        if chmodDone:
+            d.addCallback(self._abandonOnFailure)
+        else:
+            d.addCallback(self._tryChmod)
+        return d
+
+    def _tryChmod(self, rc):
+        assert isinstance(rc, int)
+        if rc == 0:
+            return defer.succeed(0)
+        # Attempt a recursive chmod and re-try the rm -rf after.
+
+        command = ["chmod", "-Rf", "u+rwx", os.path.join(self.builder.basedir, self.dir)]
+        if sys.platform.startswith('freebsd'):
+            # Work around a broken 'chmod -R' on FreeBSD (it tries to recurse into a
+            # directory for which it doesn't have permission, before changing that
+            # permission) by running 'find' instead
+            command = ["find", os.path.join(self.builder.basedir, self.dir),
+                                '-exec', 'chmod', 'u+rwx', '{}', ';' ]
+        c = runprocess.RunProcess(self.builder, command, self.builder.basedir,
+                         sendRC=0, timeout=self.timeout, maxTime=self.maxTime,
+                         usePTY=False)
+
+        self.command = c
+        d = c.start()
+        d.addCallback(self._abandonOnFailure)
+        d.addCallback(lambda dummy: self._clobber(dummy, True))
+        return d
+
+class CopyDirectory(base.Command):
+    """This is a Command which copies a directory. The args dict contains
+    the following keys:
+
+        - ['fromdir'] (required): subdirectory which the command will copy,
+                                  relative to the builder dir
+        - ['todir'] (required): subdirectory which the command will create,
+                                  relative to the builder dir
+
+        - ['timeout']:  seconds of silence tolerated before we kill off the
+                        command
+
+        - ['maxTime']:  seconds before we kill off the command
+
+
+    RemoveDirectory creates the following status messages:
+        - {'rc': rc} : when the process has terminated
+    """
+
+    header = "rmdir"
+
+    def start(self):
+        args = self.args
+        # args['todir'] is relative to Builder directory, and is required.
+        # args['fromdir'] is relative to Builder directory, and is required.
+        assert args['todir'] is not None
+        assert args['fromdir'] is not None
+
+        fromdir = os.path.join(self.builder.basedir, args['fromdir'])
+        todir = os.path.join(self.builder.basedir, args['todir'])
+
+        self.timeout = args.get('timeout', 120)
+        self.maxTime = args.get('maxTime', None)
+
+        if runtime.platformType != "posix":
+            self.sendStatus({'header': "Since we're on a non-POSIX platform, "
+            "we're not going to try to execute cp in a subprocess, but instead "
+            "use shutil.copytree(), which will block until it is complete.  "
+            "fromdir: %s, todir: %s\n" % (fromdir, todir)})
+            shutil.copytree(fromdir, todir)
+            return defer.succeed(0)
+
+        if not os.path.exists(os.path.dirname(todir)):
+            os.makedirs(os.path.dirname(todir))
+        if os.path.exists(todir):
+            # I don't think this happens, but just in case..
+            log.msg("cp target '%s' already exists -- cp will not do what you think!" % todir)
+
+        command = ['cp', '-R', '-P', '-p', fromdir, todir]
+        c = runprocess.RunProcess(self.builder, command, self.builder.basedir,
+                         sendRC=False, timeout=self.timeout, maxTime=self.maxTime,
+                         usePTY=False)
+        self.command = c
+        d = c.start()
+        d.addCallback(self._abandonOnFailure)
+	d.addCallback(self._sendRC)
+        return d

slave/buildslave/commands/registry.py

@@ -15,6 +15,9 @@
     "hg" : "buildslave.commands.hg.Mercurial",
     "p4" : "buildslave.commands.p4.P4",
     "p4sync" : "buildslave.commands.p4.P4Sync",
+    "mkdir" : "buildslave.commands.fs.MakeDirectory",
+    "rmdir" : "buildslave.commands.fs.RemoveDirectory",
+    "cpdir" : "buildslave.commands.fs.CopyDirectory",
 }
 
 def getFactory(command):

slave/buildslave/test/unit/test_bot.py

@@ -54,17 +54,18 @@ def test_getSlaveInfo(self):
         os.makedirs(infodir)
         open(os.path.join(infodir, "admin"), "w").write("testy!")
         open(os.path.join(infodir, "foo"), "w").write("bar")
+        open(os.path.join(infodir, "environ"), "w").write("something else")
 
         d = self.bot.callRemote("getSlaveInfo")
         def check(info):
-            self.assertEqual(info, dict(admin='testy!', foo='bar'))
+            self.assertEqual(info, dict(admin='testy!', foo='bar', environ=os.environ))
         d.addCallback(check)
         return d
 
     def test_getSlaveInfo_nodir(self):
         d = self.bot.callRemote("getSlaveInfo")
         def check(info):
-            self.assertEqual(info, {})
+            self.assertEqual(info.keys(), ['environ'])
         d.addCallback(check)
         return d