Merge branch 'bug_1767' of git://github.com/bdbaddog/buildbot

* 'bug_1767' of git://github.com/bdbaddog/buildbot:
  add logEnviron to fs build steps to fix broken tests
  fix typo in buildsteps documentation
  added a default value for logEnviron on the slave side
  added logEnviron argument to all source steps (fixes #1767)

master/buildbot/steps/source/oldsource.py

@@ -76,7 +76,7 @@ class Source(LoggingBuildStep):
     branch = None # the default branch, should be set in __init__
 
     def __init__(self, workdir=None, mode='update', alwaysUseLatest=False,
-                 timeout=20*60, retry=None, **kwargs):
+                 timeout=20*60, retry=None, logEnviron=True, **kwargs):
         """
         @type  workdir: string
         @param workdir: local directory (relative to the Builder's root)
@@ -157,6 +157,13 @@ def __init__(self, workdir=None, mode='update', alwaysUseLatest=False,
                       their build failures are due to transient network
                       failures that could be handled by simply retrying a
                       couple times.
+                      
+        @type logEnviron: boolean
+        @param logEnviron: If this option is true (the default), then the
+                           step's logfile will describe the environment
+                           variables on the slave. In situations where the
+                           environment is not relevant and is long, it may
+                           be easier to set logEnviron=False.
 
         """
 
@@ -183,6 +190,8 @@ def __init__(self, workdir=None, mode='update', alwaysUseLatest=False,
 
         self.alwaysUseLatest = alwaysUseLatest
 
+        self.logEnviron = logEnviron
+
         # Compute defaults for descriptions:
         description = ["updating"]
         descriptionDone = ["update"]
@@ -252,6 +261,8 @@ def start(self):
 
         if self.alwaysUseLatest:
             revision = None
+
+        self.args['logEnviron'] = self.logEnviron
         self.startVC(branch, revision, patch)
 
     def commandComplete(self, cmd):

master/docs/cfg-buildsteps.texinfo

@@ -491,6 +491,12 @@ Specifies the timeout for slave-side operations, in seconds.  If your
 repositories are particularly large, then you may need to increase this 
 value from its default of 1200 (20 minutes).
 
+@item logEnviron
+If this option is true (the default), then the step's logfile will
+describe the environment variables on the slave. In situations where the
+environment is not relevant and is long, it may be easier to set
+logEnviron=False.
+
 @end table
 
 Use of @ref{WithProperties} with string, dict and callable is supported.

slave/buildslave/commands/base.py

@@ -252,6 +252,7 @@ def setup(self, args):
         self.timeout = args.get('timeout', 120)
         self.maxTime = args.get('maxTime', None)
         self.retry = args.get('retry')
+        self.logEnviron = args.get('logEnviron',True)
         self._commandPaths = {}
         # VC-specific subclasses should override this to extract more args.
         # Make sure to upcall!
@@ -501,7 +502,7 @@ def doClobber(self, dummy, dirname, chmodDone=False):
         command = ["rm", "-rf", d]
         c = runprocess.RunProcess(self.builder, command, self.builder.basedir,
                          sendRC=0, timeout=self.timeout, maxTime=self.maxTime,
-                         usePTY=False)
+                         logEnviron=self.logEnviron, usePTY=False)
 
         self.command = c
         # sendRC=0 means the rm command will send stdout/stderr to the
@@ -532,7 +533,7 @@ def doClobberTryChmodIfFail(self, rc, dirname):
                                 '-exec', 'chmod', 'u+rwx', '{}', ';' ]
         c = runprocess.RunProcess(self.builder, command, self.builder.basedir,
                          sendRC=0, timeout=self.timeout, maxTime=self.maxTime,
-                         usePTY=False)
+                         logEnviron=self.logEnviron, usePTY=False)
 
         self.command = c
         d = c.start()
@@ -561,7 +562,7 @@ def doCopy(self, res):
         command = ['cp', '-R', '-P', '-p', fromdir, todir]
         c = runprocess.RunProcess(self.builder, command, self.builder.basedir,
                          sendRC=False, timeout=self.timeout, maxTime=self.maxTime,
-                         usePTY=False)
+                         logEnviron=self.logEnviron, usePTY=False)
         self.command = c
         d = c.start()
         d.addCallback(self._abandonOnFailure)
@@ -599,7 +600,8 @@ def doPatch(self, res):
         # now apply the patch
         c = runprocess.RunProcess(self.builder, command, dir,
                          sendRC=False, timeout=self.timeout,
-                         maxTime=self.maxTime, usePTY=False)
+                         maxTime=self.maxTime, logEnviron=self.logEnviron,
+                         usePTY=False)
         self.command = c
         d = c.start()
 

slave/buildslave/commands/bk.py

@@ -57,7 +57,8 @@ def doVCUpdate(self):
         command = [bk, 'pull']
         c = runprocess.RunProcess(self.builder, command, d,
                          sendRC=False, timeout=self.timeout,
-                         keepStdout=True, usePTY=False)
+                         keepStdout=True, logEnviron=self.logEnviron,
+                         usePTY=False)
         self.command = c
         return c.start()
 
@@ -74,7 +75,7 @@ def doVCFull(self):
                    [self.bkurl, self.srcdir]
         c = runprocess.RunProcess(self.builder, command, d,
                          sendRC=False, timeout=self.timeout,
-                         usePTY=False)
+                         logEnviron=self.logEnviron, usePTY=False)
         self.command = c
         return c.start()
 
@@ -94,7 +95,8 @@ def parseGotRevision(self):
                          os.path.join(self.builder.basedir, self.srcdir),
                          environ=self.env, timeout=self.timeout,
                          sendStdout=False, sendStderr=False, sendRC=False,
-                         keepStdout=True, usePTY=False)
+                         keepStdout=True, logEnviron=self.logEnviron,
+                         usePTY=False)
         d = c.start()
         def _parse(res):
             r_raw = c.stdout.strip()

slave/buildslave/commands/bzr.py

@@ -66,7 +66,8 @@ def doVCUpdate(self):
         command = [bzr, 'update']
         c = runprocess.RunProcess(self.builder, command, srcdir,
                          sendRC=False, timeout=self.timeout,
-                         maxTime=self.maxTime, usePTY=False)
+                         maxTime=self.maxTime, logEnviron=self.logEnviron,
+                         usePTY=False)
         self.command = c
         return c.start()
 
@@ -97,7 +98,8 @@ def doVCFull(self):
 
         c = runprocess.RunProcess(self.builder, command, d,
                          sendRC=False, timeout=self.timeout,
-                         maxTime=self.maxTime, usePTY=False)
+                         maxTime=self.maxTime, logEnviron=self.logEnviron,
+                         usePTY=False)
         self.command = c
         d = c.start()
         return d
@@ -114,14 +116,16 @@ def doVCExport(self):
         command.append(tmpdir)
         c = runprocess.RunProcess(self.builder, command, self.builder.basedir,
                          sendRC=False, timeout=self.timeout,
-                         maxTime=self.maxTime, usePTY=False)
+                         maxTime=self.maxTime, logEnviron=self.logEnviron,
+                         usePTY=False)
         self.command = c
         d = c.start()
         def _export(res):
             command = [bzr, 'export', srcdir]
             c = runprocess.RunProcess(self.builder, command, tmpdir,
                              sendRC=False, timeout=self.timeout,
-                             maxTime=self.maxTime, usePTY=False)
+                             maxTime=self.maxTime, logEnviron=self.logEnviron,
+                             usePTY=False)
             self.command = c
             return c.start()
         d.addCallback(_export)
@@ -135,15 +139,17 @@ def doForceSharedRepo(self):
         # repo is not an error, just an indication that we need to make one.
         c = runprocess.RunProcess(self.builder, [bzr, 'info', '.'],
                          self.builder.basedir,
-                         sendStderr=False, sendRC=False, usePTY=False)
+                         sendStderr=False, sendRC=False,
+                         logEnviron=self.logEnviron,usePTY=False)
         d = c.start()
         def afterCheckSharedRepo(res):
             if type(res) is int and res != 0:
                 log.msg("No shared repo found, creating it")
                 # bzr info fails, try to create shared repo.
                 c = runprocess.RunProcess(self.builder, [bzr, 'init-repo', '.'],
                                  self.builder.basedir,
-                                 sendRC=False, usePTY=False)
+                                 sendRC=False, logEnviron=self.logEnviron,
+                                 usePTY=False)
                 self.command = c
                 return c.start()
             else:
@@ -170,7 +176,8 @@ def parseGotRevision(self):
                          os.path.join(self.builder.basedir, self.srcdir),
                          environ=self.env,
                          sendStdout=False, sendStderr=False, sendRC=False,
-                         keepStdout=True, usePTY=False)
+                         keepStdout=True, logEnviron=self.logEnviron,
+                         usePTY=False)
         d = c.start()
         def _parse(res):
             try:

slave/buildslave/commands/cvs.py

@@ -66,7 +66,8 @@ def start(self):
             c = runprocess.RunProcess(self.builder, command, d,
                              sendRC=False, timeout=self.timeout,
                              maxTime=self.maxTime,
-                             initialStdin=self.login+"\n", usePTY=False)
+                             initialStdin=self.login+"\n",
+                             logEnviron=self.logEnviron,usePTY=False)
             self.command = c
             d = c.start()
             d.addCallback(self._abandonOnFailure)
@@ -89,7 +90,8 @@ def doVCUpdate(self):
             command += ['-D', self.revision]
         c = runprocess.RunProcess(self.builder, command, d,
                          sendRC=False, timeout=self.timeout,
-                         maxTime=self.maxTime, usePTY=False)
+                         maxTime=self.maxTime, logEnviron=self.logEnviron,
+                         usePTY=False)
         self.command = c
         return c.start()
 
@@ -118,7 +120,8 @@ def doVCFull(self):
 
         c = runprocess.RunProcess(self.builder, command, d,
                          sendRC=False, timeout=self.timeout,
-                         maxTime=self.maxTime, usePTY=False)
+                         maxTime=self.maxTime, logEnviron=self.logEnviron,
+                         usePTY=False)
         self.command = c
         return c.start()
 

slave/buildslave/commands/darcs.py

@@ -49,7 +49,8 @@ def doVCUpdate(self):
         command = [darcs, 'pull', '--all', '--verbose']
         c = runprocess.RunProcess(self.builder, command, d,
                          sendRC=False, timeout=self.timeout,
-                         maxTime=self.maxTime, usePTY=False)
+                         maxTime=self.maxTime, logEnviron=self.logEnviron,
+                         usePTY=False)
         self.command = c
         return c.start()
 
@@ -72,7 +73,8 @@ def doVCFull(self):
 
         c = runprocess.RunProcess(self.builder, command, d,
                          sendRC=False, timeout=self.timeout,
-                         maxTime=self.maxTime, usePTY=False)
+                         maxTime=self.maxTime, logEnviron=self.logEnviron,
+                         usePTY=False)
         self.command = c
         d = c.start()
         if self.revision:
@@ -92,7 +94,8 @@ def parseGotRevision(self):
                          os.path.join(self.builder.basedir, self.srcdir),
                          environ=self.env, timeout=self.timeout,
                          sendStdout=False, sendStderr=False, sendRC=False,
-                         keepStdout=True, usePTY=False)
+                         keepStdout=True, logEnviron=self.logEnviron,
+                         usePTY=False)
         d = c.start()
         d.addCallback(lambda res: c.stdout)
         return d

slave/buildslave/commands/fs.py

@@ -69,6 +69,10 @@ class RemoveDirectory(base.Command):
     """
 
     header = "rmdir"
+
+    def setup(self,args):
+        self.logEnviron = args.get('logEnviron',True)
+
 
     @defer.deferredGenerator
     def start(self):
@@ -115,7 +119,7 @@ def _clobber(self, dummy, chmodDone = False):
         command = ["rm", "-rf", self.dir]
         c = runprocess.RunProcess(self.builder, command, self.builder.basedir,
                          sendRC=0, timeout=self.timeout, maxTime=self.maxTime,
-                         usePTY=False)
+                         logEnviron=self.logEnviron, usePTY=False)
 
         self.command = c
         # sendRC=0 means the rm command will send stdout/stderr to the
@@ -144,7 +148,7 @@ def _tryChmod(self, rc):
                                 '-exec', 'chmod', 'u+rwx', '{}', ';' ]
         c = runprocess.RunProcess(self.builder, command, self.builder.basedir,
                          sendRC=0, timeout=self.timeout, maxTime=self.maxTime,
-                         usePTY=False)
+                         logEnviron=self.logEnviron, usePTY=False)
 
         self.command = c
         d = c.start()
@@ -172,6 +176,9 @@ class CopyDirectory(base.Command):
 
     header = "rmdir"
 
+    def setup(self,args):
+        self.logEnviron = args.get('logEnviron',True)
+
     def start(self):
         args = self.args
         # args['todir'] is relative to Builder directory, and is required.
@@ -202,7 +209,7 @@ def start(self):
             command = ['cp', '-R', '-P', '-p', fromdir, todir]
             c = runprocess.RunProcess(self.builder, command, self.builder.basedir,
                              sendRC=False, timeout=self.timeout, maxTime=self.maxTime,
-                             usePTY=False)
+                             logEnviron=self.logEnviron, usePTY=False)
             self.command = c
             d = c.start()
             d.addCallback(self._abandonOnFailure)

slave/buildslave/commands/git.py

@@ -67,7 +67,8 @@ def _dovccmd(self, command, cb=None, **kwargs):
         git = self.getCommand("git")
         c = runprocess.RunProcess(self.builder, [git] + command, self._fullSrcdir(),
                          sendRC=False, timeout=self.timeout,
-                         maxTime=self.maxTime, usePTY=False, **kwargs)
+                         maxTime=self.maxTime, logEnviron=self.logEnviron,
+                         usePTY=False, **kwargs)
         self.command = c
         d = c.start()
         if cb:
@@ -195,7 +196,8 @@ def doVCFull(self):
             cmd.extend([self.repourl, self._fullSrcdir()])
             c = runprocess.RunProcess(self.builder, cmd, self.builder.basedir,
                              sendRC=False, timeout=self.timeout,
-                             maxTime=self.maxTime, usePTY=False)
+                             maxTime=self.maxTime, logEnviron=self.logEnviron,
+                             usePTY=False)
             self.command = c
             cmdexec = c.start()
             cmdexec.addCallback(self._didInit)

slave/buildslave/commands/hg.py

@@ -52,7 +52,8 @@ def doVCUpdate(self):
         command = [hg, 'pull', '--verbose', self.repourl]
         c = runprocess.RunProcess(self.builder, command, d,
                          sendRC=False, timeout=self.timeout,
-                         maxTime=self.maxTime, keepStdout=True, usePTY=False)
+                         maxTime=self.maxTime, keepStdout=True,
+                         logEnviron=self.logEnviron, usePTY=False)
         self.command = c
         d = c.start()
         d.addCallback(self._handleEmptyUpdate)
@@ -81,7 +82,8 @@ def doVCFull(self):
 
         c = runprocess.RunProcess(self.builder, command, self.builder.basedir,
                          sendRC=False, timeout=self.timeout,
-                         maxTime=self.maxTime, usePTY=False)
+                         maxTime=self.maxTime, logEnviron=self.logEnviron,
+                         usePTY=False)
         self.command = c
         cmd1 = c.start()
         cmd1.addCallback(self._update)
@@ -106,7 +108,8 @@ def _purge(self, dummy, dirname):
         d = os.path.join(self.builder.basedir, self.srcdir)
         purge = [hg, 'purge', '--all']
         purgeCmd = runprocess.RunProcess(self.builder, purge, d,
-                                keepStdout=True, keepStderr=True, usePTY=False)
+                                keepStdout=True, keepStderr=True,
+                                logEnviron=self.logEnviron, usePTY=False)
 
         def _clobber(res):
             if res != 0:
@@ -136,7 +139,8 @@ def _update(self, res):
         parentscmd = [hg, 'identify', '--num', '--branch']
         cmd = runprocess.RunProcess(self.builder, parentscmd, d,
                            sendRC=False, timeout=self.timeout, keepStdout=True,
-                           keepStderr=True, usePTY=False)
+                           keepStderr=True, logEnviron=self.logEnviron,
+                           usePTY=False)
 
         self.clobber = None
 
@@ -191,7 +195,8 @@ def _checkRepoURL(res):
             parentscmd = [hg, 'paths', 'default']
             cmd2 = runprocess.RunProcess(self.builder, parentscmd, d,
                                keepStdout=True, keepStderr=True, usePTY=False,
-                               timeout=self.timeout, sendRC=False)
+                               timeout=self.timeout, sendRC=False,
+                               logEnviron=self.logEnviron)
 
             def _parseRepoURL(res):
                 if res == 1:
@@ -258,7 +263,8 @@ def _update2(self, res):
             updatecmd.extend(['--rev', self.args.get('branch',  'default')])
         self.command = runprocess.RunProcess(self.builder, updatecmd,
             self.builder.basedir, sendRC=False,
-            timeout=self.timeout, maxTime=self.maxTime, usePTY=False)
+            timeout=self.timeout, maxTime=self.maxTime,
+            logEnviron=self.logEnviron, usePTY=False)
         return self.command.start()
 
     def parseGotRevision(self):
@@ -269,7 +275,8 @@ def parseGotRevision(self):
                          os.path.join(self.builder.basedir, self.srcdir),
                          environ=self.env, timeout=self.timeout,
                          sendRC=False,
-                         keepStdout=True, usePTY=False)
+                         keepStdout=True, usePTY=False,
+                         logEnviron=self.logEnviron)
         d = c.start()
         def _parse(res):
             m = re.search(r'^(\w+)', c.stdout)