diff --git a/master/buildbot/buildslave.py b/master/buildbot/buildslave.py index 0c23bb115f7..fc9dbf9a67e 100644 --- a/master/buildbot/buildslave.py +++ b/master/buildbot/buildslave.py @@ -249,6 +249,7 @@ def _got_info(info): state["admin"] = info.get("admin") state["host"] = info.get("host") state["access_uri"] = info.get("access_uri", None) + state["slave_environ"] = info.get("environ", {}) def _info_unavailable(why): # maybe an old slave, doesn't implement remote_getSlaveInfo log.msg("BuildSlave.info_unavailable") @@ -289,6 +290,7 @@ def _accept_slave(res): self.slave_status.setVersion(state.get("version")) self.slave_status.setConnected(True) self.slave_commands = state.get("slave_commands") + self.slave_environ = state.get("slave_environ") self.slave = bot log.msg("bot attached") self.messageReceivedFromSlave() diff --git a/master/buildbot/steps/master.py b/master/buildbot/steps/master.py index 22beefe0990..793dd2a52ee 100644 --- a/master/buildbot/steps/master.py +++ b/master/buildbot/steps/master.py @@ -102,3 +102,31 @@ def processEnded(self, status_object): else: self.step_status.setText(list(self.descriptionDone)) self.finished(SUCCESS) + +class SetPropertiesFromEnv(BuildStep): + """ + Sets properties from envirionment variables on the slave. + + Note this is transfered when the slave first connects + """ + name='SetPropertiesFromEnv' + description='Setting' + descriptionDone='Set' + + def __init__(self, variables, source="SlaveEnvironment", **kwargs): + BuildStep.__init__(self, **kwargs) + self.addFactoryArguments(variables = variables, + source = source) + self.variables = variables + self.source = source + + def start(self): + properties = self.build.getProperties() + environ = self.buildslave.slave_environ + if isinstance(self.variables, str): + self.variables = [self.variables] + for variable in self.variables: + value = environ.get(variable, None) + if value: + properties.setProperty(variable, value, self.source, runtime=True) + self.finished(SUCCESS) diff --git a/master/buildbot/steps/transfer.py b/master/buildbot/steps/transfer.py index be3b2e53b9f..d103cca423c 100644 --- a/master/buildbot/steps/transfer.py +++ b/master/buildbot/steps/transfer.py @@ -548,7 +548,7 @@ def start(self): os.path.basename(slavedest)]) # setup structures for reading the file - fp = StringIO(self.s) + fp = StringIO(properties.render(self.s)) fileReader = _FileReader(fp) # default arguments diff --git a/master/buildbot/test/unit/test_steps_master.py b/master/buildbot/test/unit/test_steps_master.py new file mode 100644 index 00000000000..6b2c8307c64 --- /dev/null +++ b/master/buildbot/test/unit/test_steps_master.py @@ -0,0 +1,33 @@ +from twisted.trial import unittest + +from mock import Mock + +from buildbot.process.properties import Properties +from buildbot.steps.master import SetPropertiesFromEnv + +class TestSetPropertiesFromEnv(unittest.TestCase): + def testBasic(self): + s = SetPropertiesFromEnv(variables = ["one", "two", "three", "five", "six"], source = "me") + s.build = Mock() + s.build.getProperties.return_value = props = Properties() + s.buildslave = Mock() + s.buildslave.slave_environ = { "one": 1, "two": None, "six": 6 } + props.setProperty("four", 4, "them") + props.setProperty("five", 5, "them") + props.setProperty("six", 99, "them") + + s.step_status = Mock() + s.deferred = Mock() + + s.start() + + self.failUnlessEqual(props.getProperty('one'), 1) + self.failUnlessEqual(props.getPropertySource('one'), 'me') + self.failUnlessEqual(props.getProperty('two'), None) + self.failUnlessEqual(props.getProperty('three'), None) + self.failUnlessEqual(props.getProperty('four'), 4) + self.failUnlessEqual(props.getPropertySource('four'), 'them') + self.failUnlessEqual(props.getProperty('five'), 5) + self.failUnlessEqual(props.getPropertySource('five'), 'them') + self.failUnlessEqual(props.getProperty('six'), 6) + self.failUnlessEqual(props.getPropertySource('six'), 'me') diff --git a/master/docs/cfg-buildsteps.texinfo b/master/docs/cfg-buildsteps.texinfo index ca4dcca3401..97749599399 100644 --- a/master/docs/cfg-buildsteps.texinfo +++ b/master/docs/cfg-buildsteps.texinfo @@ -1535,6 +1535,8 @@ f.addStep(SetProperty( Then @code{my_extract} will see @code{stdout="output1\noutput2\n"} and @code{stderr="error\n"}. +See also @ref{SetProperiesFromEnv}. + @node SubunitShellCommand @subsubsection SubunitShellCommand @@ -1896,6 +1898,16 @@ Note that, by default, this step passes a copy of the buildmaster's environment variables to the subprocess. To pass an explicit environment instead, add an @code{env=@{..@}} argument. +@bsindex buildbot.steps.master.SetPropertiesFromEnv + +@example +from buildbot.steps.master import SetProperiesFromEnv +from buildbot.steps.shell import Compile + +f.addStep(SetPropertiesFromEnv(variables=["SOME_JAVA_LIB_HOME", "JAVAC"])) +f.addStep(Compile(commands=[WithProperties("%s","JAVAC"), "-cp", WithProperties("%s", "SOME_JAVA_LIB_HOME"))) +@end example + @node Triggering Schedulers @subsection Triggering Schedulers @bsindex buildbot.steps.trigger.Trigger diff --git a/slave/buildslave/bot.py b/slave/buildslave/bot.py index dac6148ada9..49775bb016a 100644 --- a/slave/buildslave/bot.py +++ b/slave/buildslave/bot.py @@ -285,18 +285,20 @@ def remote_getSlaveInfo(self): files = {} basedir = os.path.join(self.basedir, "info") - if not os.path.isdir(basedir): - return files - for f in os.listdir(basedir): - filename = os.path.join(basedir, f) - if os.path.isfile(filename): - files[f] = open(filename, "r").read() + if os.path.isdir(basedir): + for f in os.listdir(basedir): + filename = os.path.join(basedir, f) + if os.path.isfile(filename): + files[f] = open(filename, "r").read() + files['environ'] = os.environ.copy() return files def remote_getVersion(self): """Send our version back to the Master""" return buildslave.version + def remote_getEnviron(self): + return os.environ.copy() class BotFactory(ReconnectingPBClientFactory): diff --git a/slave/buildslave/commands/fs.py b/slave/buildslave/commands/fs.py new file mode 100644 index 00000000000..98696c71142 --- /dev/null +++ b/slave/buildslave/commands/fs.py @@ -0,0 +1,174 @@ +import os + +from twisted.internet import defer +from twisted.python import runtime + +from buildslave import runprocess +from buildslave.commands import base, utils + +class MakeDirectory(base.Command): + """This is a Command which creates a directory. The args dict contains + the following keys: + + - ['dir'] (required): subdirectory which the command will create, + relative to the builder dir + + MakeDirectory creates the following status messages: + - {'rc': rc} : when the process has terminated + """ + + header = "mkdir" + + def start(self): + args = self.args + # args['dir'] is relative to Builder directory, and is required. + assert args['dir'] is not None + dirname = os.path.join(self.builder.basedir, args['dir']) + + try: + if not os.path.isdir(dirname): + os.makedirs(dirname) + self.sendStatus({'rc': 0}) + except: + self.sendStatus({'rc': 1}) + +class RemoveDirectory(base.Command): + """This is a Command which removes a directory. The args dict contains + the following keys: + + - ['dir'] (required): subdirectory which the command will create, + relative to the builder dir + + - ['timeout']: seconds of silence tolerated before we kill off the + command + + - ['maxTime']: seconds before we kill off the command + + + RemoveDirectory creates the following status messages: + - {'rc': rc} : when the process has terminated + """ + + header = "rmdir" + + def start(self): + args = self.args + # args['dir'] is relative to Builder directory, and is required. + assert args['dir'] is not None + dirname = args['dir'] + + self.timeout = args.get('timeout', 120) + self.maxTime = args.get('maxTime', None) + + # TODO: remove the old tree in the background + self.dir = os.path.join(self.builder.basedir, dirname) + if runtime.platformType != "posix": + # if we're running on w32, use rmtree instead. It will block, + # but hopefully it won't take too long. + utils.rmdirRecursive(self.dir) + return defer.succeed(0) + + d = self._clobber(None) + d.addCallback(self._sendRC) + return d + + def _clobber(self, dummy, chmodDone = False): + command = ["rm", "-rf", self.dir] + c = runprocess.RunProcess(self.builder, command, self.builder.basedir, + sendRC=0, timeout=self.timeout, maxTime=self.maxTime, + usePTY=False) + + self.command = c + # sendRC=0 means the rm command will send stdout/stderr to the + # master, but not the rc=0 when it finishes. That job is left to + # _sendRC + d = c.start() + # The rm -rf may fail if there is a left-over subdir with chmod 000 + # permissions. So if we get a failure, we attempt to chmod suitable + # permissions and re-try the rm -rf. + if chmodDone: + d.addCallback(self._abandonOnFailure) + else: + d.addCallback(self._tryChmod) + return d + + def _tryChmod(self, rc): + assert isinstance(rc, int) + if rc == 0: + return defer.succeed(0) + # Attempt a recursive chmod and re-try the rm -rf after. + + command = ["chmod", "-Rf", "u+rwx", os.path.join(self.builder.basedir, self.dir)] + if sys.platform.startswith('freebsd'): + # Work around a broken 'chmod -R' on FreeBSD (it tries to recurse into a + # directory for which it doesn't have permission, before changing that + # permission) by running 'find' instead + command = ["find", os.path.join(self.builder.basedir, self.dir), + '-exec', 'chmod', 'u+rwx', '{}', ';' ] + c = runprocess.RunProcess(self.builder, command, self.builder.basedir, + sendRC=0, timeout=self.timeout, maxTime=self.maxTime, + usePTY=False) + + self.command = c + d = c.start() + d.addCallback(self._abandonOnFailure) + d.addCallback(lambda dummy: self._clobber(dummy, True)) + return d + +class CopyDirectory(base.Command): + """This is a Command which copies a directory. The args dict contains + the following keys: + + - ['fromdir'] (required): subdirectory which the command will copy, + relative to the builder dir + - ['todir'] (required): subdirectory which the command will create, + relative to the builder dir + + - ['timeout']: seconds of silence tolerated before we kill off the + command + + - ['maxTime']: seconds before we kill off the command + + + RemoveDirectory creates the following status messages: + - {'rc': rc} : when the process has terminated + """ + + header = "rmdir" + + def start(self): + args = self.args + # args['todir'] is relative to Builder directory, and is required. + # args['fromdir'] is relative to Builder directory, and is required. + assert args['todir'] is not None + assert args['fromdir'] is not None + + fromdir = os.path.join(self.builder.basedir, args['fromdir']) + todir = os.path.join(self.builder.basedir, args['todir']) + + self.timeout = args.get('timeout', 120) + self.maxTime = args.get('maxTime', None) + + if runtime.platformType != "posix": + self.sendStatus({'header': "Since we're on a non-POSIX platform, " + "we're not going to try to execute cp in a subprocess, but instead " + "use shutil.copytree(), which will block until it is complete. " + "fromdir: %s, todir: %s\n" % (fromdir, todir)}) + shutil.copytree(fromdir, todir) + return defer.succeed(0) + + if not os.path.exists(os.path.dirname(todir)): + os.makedirs(os.path.dirname(todir)) + if os.path.exists(todir): + # I don't think this happens, but just in case.. + log.msg("cp target '%s' already exists -- cp will not do what you think!" % todir) + + command = ['cp', '-R', '-P', '-p', fromdir, todir] + c = runprocess.RunProcess(self.builder, command, self.builder.basedir, + sendRC=False, timeout=self.timeout, maxTime=self.maxTime, + usePTY=False) + self.command = c + d = c.start() + d.addCallback(self._abandonOnFailure) + d.addCallback(self._sendRC) + return d diff --git a/slave/buildslave/commands/registry.py b/slave/buildslave/commands/registry.py index a0eb5359e92..ef9c8ccd78e 100644 --- a/slave/buildslave/commands/registry.py +++ b/slave/buildslave/commands/registry.py @@ -15,6 +15,9 @@ "hg" : "buildslave.commands.hg.Mercurial", "p4" : "buildslave.commands.p4.P4", "p4sync" : "buildslave.commands.p4.P4Sync", + "mkdir" : "buildslave.commands.fs.MakeDirectory", + "rmdir" : "buildslave.commands.fs.RemoveDirectory", + "cpdir" : "buildslave.commands.fs.CopyDirectory", } def getFactory(command): diff --git a/slave/buildslave/test/unit/test_bot.py b/slave/buildslave/test/unit/test_bot.py index cac84fbebc1..3e85a1e347f 100644 --- a/slave/buildslave/test/unit/test_bot.py +++ b/slave/buildslave/test/unit/test_bot.py @@ -54,17 +54,18 @@ def test_getSlaveInfo(self): os.makedirs(infodir) open(os.path.join(infodir, "admin"), "w").write("testy!") open(os.path.join(infodir, "foo"), "w").write("bar") + open(os.path.join(infodir, "environ"), "w").write("something else") d = self.bot.callRemote("getSlaveInfo") def check(info): - self.assertEqual(info, dict(admin='testy!', foo='bar')) + self.assertEqual(info, dict(admin='testy!', foo='bar', environ=os.environ)) d.addCallback(check) return d def test_getSlaveInfo_nodir(self): d = self.bot.callRemote("getSlaveInfo") def check(info): - self.assertEqual(info, {}) + self.assertEqual(info.keys(), ['environ']) d.addCallback(check) return d diff --git a/slave/buildslave/test/unit/test_commands_fs.py b/slave/buildslave/test/unit/test_commands_fs.py new file mode 100644 index 00000000000..9ecffd630c2 --- /dev/null +++ b/slave/buildslave/test/unit/test_commands_fs.py @@ -0,0 +1,99 @@ +import os + +from twisted.trial import unittest +from twisted.python import runtime + +from buildslave.test.util.command import CommandTestMixin +from buildslave.commands import fs + +class TestRemoveDirectory(CommandTestMixin, unittest.TestCase): + + def setUp(self): + self.setUpCommand() + + def tearDown(self): + self.tearDownCommand() + + def test_simple(self): + self.make_command(fs.RemoveDirectory, dict( + dir='workdir', + ), True) + d = self.run_command() + + def check(_): + self.assertFalse(os.path.exists(os.path.abspath(os.path.join(self.basedir,'workdir')))) + self.assertIn({'rc': 0}, + self.get_updates(), + self.builder.show()) + d.addCallback(check) + return d + +class TestCopyDirectory(CommandTestMixin, unittest.TestCase): + + def setUp(self): + self.setUpCommand() + + def tearDown(self): + self.tearDownCommand() + + def test_simple(self): + self.make_command(fs.CopyDirectory, dict( + fromdir='workdir', + todir='copy', + ), True) + d = self.run_command() + + def check(_): + self.assertTrue(os.path.exists(os.path.abspath(os.path.join(self.basedir,'copy')))) + self.assertIn({'rc': 0}, + self.get_updates(), + self.builder.show()) + d.addCallback(check) + return d + +class TestMakeDirectory(CommandTestMixin, unittest.TestCase): + + def setUp(self): + self.setUpCommand() + + def tearDown(self): + self.tearDownCommand() + + def test_simple(self): + self.make_command(fs.MakeDirectory, dict( + dir='test-dir', + ), True) + d = self.run_command() + + def check(_): + self.assertTrue(os.path.exists(os.path.abspath(os.path.join(self.basedir,'test-dir')))) + self.assertEqual(self.get_updates(), + [{'rc': 0}], + self.builder.show()) + d.addCallback(check) + return d + + def test_already_exists(self): + self.make_command(fs.MakeDirectory, dict( + dir='workdir', + ), True) + d = self.run_command() + + def check(_): + self.assertEqual(self.get_updates(), + [{'rc': 0}], + self.builder.show()) + d.addCallback(check) + return d + + def test_existing_file(self): + self.make_command(fs.MakeDirectory, dict( + dir='test-file', + ), True) + open(os.path.join(self.basedir, 'test-file'), "w") + d = self.run_command() + + def check(_): + self.assertEqual(self.get_updates(), [{'rc': 1}], self.builder.show()) + d.addErrback(check) + return d