Merge remote-tracking branch 'refs/remotes/origin/katana' into integr…

…ationtest/triggerjob

master/buildbot/process/builder.py

@@ -433,13 +433,12 @@ def run_cleanups():
             ping_success = yield slavebuilder.ping(timeout=self.master.config.remoteCallTimeout)
         except:
             log.err(failure.Failure(), 'while pinging slave before build:')
-            ping_success = False
+            raise
 
         if not ping_success:
             log.msg("build %s slave %s ping failed; re-queueing the request" % (build, slavebuilder))
             run_cleanups()
-            defer.returnValue(False)
-            return
+            raise Exception("Ping failed")
 
         #check slave is still available
         ready = slavebuilder.isAvailable()
@@ -448,16 +447,15 @@ def run_cleanups():
                 ready = yield slavebuilder.prepare(self.builder_status, build)
             except:
                 log.err(failure.Failure(), 'while preparing slavebuilder:')
-                ready = False
+                raise
 
         # If prepare returns True then it is ready and we start a build
         # If it returns false then we don't start a new build.
         if not ready:
             log.msg("slave %s can't build %s after all; re-queueing the "
                     "request" % (build, slavebuilder))
             run_cleanups()
-            defer.returnValue(False)
-            return
+            raise Exception("Unknown")
 
         # The buildslave is ready to go. slavebuilder.buildStarted() sets its
         # state to BUILDING (so we won't try to use it for any other builds).
@@ -468,17 +466,7 @@ def run_cleanups():
             log.msg("slave %s can't build %s after all; re-queueing the "
                     "request" % (build, slavebuilder))
             run_cleanups()
-            defer.returnValue(False)
-            return
-
-        # tell the remote that it's starting a build, too
-        try:
-            yield slavebuilder.remote.callRemote("startBuild")
-        except:
-            log.err(failure.Failure(), 'while calling remote startBuild:')
-            run_cleanups()
-            defer.returnValue(False)
-            return
+            raise Exception("Unknown")
 
         # create the BuildStatus object that goes with the Build
         if build_status is None:
@@ -506,8 +494,7 @@ def run_cleanups():
         except:
             log.err(failure.Failure(), 'while adding rows to build table:')
             run_cleanups()
-            defer.returnValue(False)
-            return
+            raise
 
         # IMPORTANT: no yielding is allowed from here to the startBuild call!
 
@@ -518,8 +505,7 @@ def run_cleanups():
         if not slavebuilder.remote:
             log.msg("slave disappeared before build could start")
             run_cleanups()
-            defer.returnValue(False)
-            return
+            raise Exception("Slave seems to have disappered")
 
         # let status know
         self.master.status.build_started(main_br.id, self.name, bs)
@@ -541,7 +527,6 @@ def run_cleanups():
 
         # make sure the builder's status is represented correctly
         self.updateBigStatus()
-
         defer.returnValue(True)
 
     def setupProperties(self, props):

master/buildbot/process/buildrequestdistributor.py

@@ -31,10 +31,17 @@
 import random
 
 def timerLogFinished(msg, timer):
-    log.msg(msg + " started at %s finished at %s elapsed %s" %
-            (util.epoch2datetime(timer.started),
-             util.epoch2datetime(util.now(timer._reactor)),
-             util.formatInterval(util.now(timer._reactor) - timer.started)))
+    finished = util.now(timer._reactor)
+    elapsed = "{0:.2f} secs".format(finished - timer.started)
+
+    log.msg(
+        msg + " started at %s finished at %s elapsed %s" %
+        (
+            util.epoch2datetime(timer.started),
+            finished,
+            elapsed
+        )
+    )
     timer.stop()
 
 def timerLogStart(msg, function_name):
@@ -1231,20 +1238,22 @@ def _maybeResumeBuildsOnBuilder(self):
 
         brids = [br.id for br in breqs]
         yield self.master.db.buildrequests.updateBuildRequests(brids, results=BEGINNING)
+
+        buildDefered = self.katanaBuildChooser.bldr.maybeResumeBuild(slave, buildnumber, breqs)
+        buildDefered.addErrback(self._resume, breqs, self.katanaBuildChooser.bldr.name)
 
-        buildStarted = yield self.katanaBuildChooser.bldr.maybeResumeBuild(slave, buildnumber, breqs)
+        self.katanaBuildChooser.removeBuildRequests(breqs)
 
         msg = "_maybeResumeBuildOnBuilder is resuming build"
-
-        if not buildStarted:
-            yield self.master.db.buildrequests.updateBuildRequests(brids, results=RESUME)
-            self.botmaster.maybeStartBuildsForBuilder(self.katanaBuildChooser.bldr.name)
-            msg = "_maybeResumeBuildOnBuilder could not resume build"
-        else:
-            self.katanaBuildChooser.removeBuildRequests(breqs)
-
         self.logResumeOrStartBuildStatus(msg, slave, breqs)
-        defer.returnValue(buildStarted)
+        defer.returnValue(True)
+
+    @defer.inlineCallbacks
+    def _resume(self, e, breqs, builderName):
+        brids = [br.id for br in breqs]
+        log.msg("Could not resume builds {}. Exception message: {}. Requeueing.".format(brids, e.value))
+        yield self.master.db.buildrequests.updateBuildRequests(brids, results=RESUME)
+        self.botmaster.maybeStartBuildsForBuilder(builderName)
 
     @defer.inlineCallbacks
     def _maybeStartBuildsOnBuilder(self):
@@ -1255,24 +1264,23 @@ def _maybeStartBuildsOnBuilder(self):
             return
 
         # claim brid's
-        brids = [br.id for br in breqs]
         yield self.katanaBuildChooser.claimBuildRequests(breqs)
 
-        buildStarted = yield self.katanaBuildChooser.bldr.maybeStartBuild(slave, breqs)
+        buildDefered = self.katanaBuildChooser.bldr.maybeStartBuild(slave, breqs)
+        buildDefered.addErrback(self._requeue, breqs, self.katanaBuildChooser.bldr.name)
 
         msg = "_maybeStartNewBuildsOnBuilder is starting build"
-
-        if not buildStarted:
-            yield self.master.db.buildrequests.unclaimBuildRequests(brids)
-            # and try starting builds again.  If we still have a working slave,
-            # then this may re-claim the same buildrequests
-            self.botmaster.maybeStartBuildsForBuilder(self.katanaBuildChooser.bldr.name)
-            msg = "_maybeStartNewBuildsOnBuilder could not start build"
-        else:
-            self.katanaBuildChooser.removeBuildRequests(breqs)
+        self.katanaBuildChooser.removeBuildRequests(breqs)
 
         self.logResumeOrStartBuildStatus(msg, slave, breqs)
-        defer.returnValue(buildStarted)
+        defer.returnValue(True)
+
+    @defer.inlineCallbacks
+    def _requeue(self, e, breqs, builderName):
+        brids = [br.id for br in breqs]
+        log.msg("Could not start builds {}. Exception message: {}. Requeueing.".format(brids, e.value))
+        yield self.master.db.buildrequests.unclaimBuildRequests(brids)
+        self.botmaster.maybeStartBuildsForBuilder(builderName)
 
     def createBuildChooser(self, builders, master):
         # just instantiate the build chooser requested

master/buildbot/process/slavebuilder.py

@@ -189,9 +189,6 @@ def includeSlavename(self, msg):
             msg += " %s" % self.slavename
         return msg
 
-    def pingTimeout(self, value, timeout):
-        log.msg("ping %s timeout after %s secs" % (self.slavename, timeout))
-
     def ping(self, remote, slavename, timeout=5):
         assert not self.running
         if not remote:
@@ -205,7 +202,7 @@ def ping(self, remote, slavename, timeout=5):
         # for this purpose is kind of silly.
         try:
             rd = remote.callRemote("print", "ping")
-            rd.addTimeout(timeout, reactor, onTimeoutCancel=self.pingTimeout)
+            rd.addTimeout(timeout, reactor)
             rd.addCallbacks(self._pong, self._ping_failed, errbackArgs=(remote,))
 
 

master/buildbot/scripts/buildbot_tac.tmpl

@@ -9,8 +9,8 @@ basedir = '.'
 basedir = {{ basedir|repr }}
 {%- endif %}
 {% if not no_logrotate -%}
-rotateLength = {{ log_size|repr }}
-maxRotatedFiles = {{ log_count|repr }}
+rotateLength = {{ log_size }}
+maxRotatedFiles = {{ log_count }}
 {%- endif %}
 configfile = {{ config|repr }}
 

master/buildbot/scripts/runner.py

@@ -140,11 +140,17 @@ def getSynopsis(self):
 
     def postOptions(self):
         base.BasedirMixin.postOptions(self)
-        if not re.match('^\d+$', self['log-size']):
-            raise usage.UsageError("log-size parameter needs to be an int")
-        if not re.match('^\d+$', self['log-count']) and \
-                self['log-count'] != 'None':
-            raise usage.UsageError("log-count parameter needs to be an int "+
+        if re.match('^\d+$', self['log-size']):
+            self['log-size'] = int(self['log-size'])
+        else:
+            raise usage.UsageError("log-size parameter needs to be a number")
+
+        if re.match('^\d+$', self['log-count']):
+            self['log-count'] = int(self['log-count'])
+        elif self['log-count'] == 'None':
+            self['log-count'] = None
+        else:
+            raise usage.UsageError("log-count parameter needs to be a number"
                                    " or None")
 
 

master/buildbot/steps/artifact.py

@@ -7,6 +7,10 @@
 from buildbot.process.slavebuilder import IDLE, BUILDING
 from buildbot.steps.resumebuild import ResumeBuild, ShellCommandResumeBuild
 
+# Change artifact location in August
+# datetime.datetime(2017, 7, 31, 23, 59, 59, tzinfo=UTC)
+ARTIFACT_LOCATION_CHANGE_DATE = epoch2datetime(1501545599)
+
 def FormatDatetime(value):
     return value.strftime("%d_%m_%Y_%H_%M_%S_%z")
 
@@ -183,9 +187,15 @@ def start(self):
 
         if self.artifactBuildrequest:
             self.step_status.setText(["Artifact has been already generated."])
-            self.artifactPath = "%s_%s_%s" % (self.build.builder.config.builddir,
-                                              self.artifactBuildrequest['brid'],
-                                              FormatDatetime(self.artifactBuildrequest['submitted_at']))
+
+            if self.artifactBuildrequest["submitted_at"] > ARTIFACT_LOCATION_CHANGE_DATE:
+                self.artifactPath = "%s/%s_%s" % (self.build.builder.config.builddir,
+                                                  self.artifactBuildrequest['brid'],
+                                                  FormatDatetime(self.artifactBuildrequest['submitted_at']))
+            else:
+                self.artifactPath = "%s_%s_%s" % (self.build.builder.config.builddir,
+                                                  self.artifactBuildrequest['brid'],
+                                                  FormatDatetime(self.artifactBuildrequest['submitted_at']))
 
             if self.artifactDirectory:
                 self.artifactPath += "/%s" %  self.artifactDirectory
@@ -233,8 +243,13 @@ def __init__(self,  artifactDirectory=None, artifactServer=None, artifactServerD
 
     def start(self):
         br = self.build.requests[0]
-        artifactPath  = "%s_%s_%s" % (self.build.builder.config.builddir,
-                                      br.id, FormatDatetime(mkdt(br.submittedAt)))
+        if mkdt(br.submittedAt) > ARTIFACT_LOCATION_CHANGE_DATE:
+            artifactPath  = "%s/%s_%s" % (self.build.builder.config.builddir,
+                                          br.id, FormatDatetime(mkdt(br.submittedAt)))
+        else:
+            artifactPath  = "%s_%s_%s" % (self.build.builder.config.builddir,
+                                          br.id, FormatDatetime(mkdt(br.submittedAt)))
+
         if (self.artifactDirectory):
             artifactPath += "/%s" % self.artifactDirectory
 
@@ -313,7 +328,10 @@ def start(self):
             master = self.build.builder.botmaster.parent
             reuse = yield master.db.buildrequests.updateMergedBuildRequest(self.build.requests)
 
-        artifactPath  = "%s_%s_%s" % (self.build.builder.config.builddir, br.id, FormatDatetime(mkdt(br.submittedAt)))
+        if mkdt(br.submittedAt) > ARTIFACT_LOCATION_CHANGE_DATE:
+            artifactPath  = "%s/%s_%s" % (self.build.builder.config.builddir, br.id, FormatDatetime(mkdt(br.submittedAt)))
+        else:
+            artifactPath = "%s_%s_%s" % (self.build.builder.config.builddir, br.id, FormatDatetime(mkdt(br.submittedAt)))
 
         artifactServerPath = self.build.getProperty("artifactServerPath", None)
         if artifactServerPath is None:
@@ -367,8 +385,13 @@ def start(self):
         triggeredbybrid = self.build.requests[0].id
         br = yield self.master.db.buildrequests.getBuildRequestTriggered(triggeredbybrid, self.artifactBuilderName)
 
-        artifactPath  = "%s_%s_%s" % (safeTranslate(self.artifactBuilderName),
-                                      br['brid'], FormatDatetime(br["submitted_at"]))
+        if br["submitted_at"] > ARTIFACT_LOCATION_CHANGE_DATE:
+            artifactPath  = "%s/%s_%s" % (safeTranslate(self.artifactBuilderName),
+                                          br['brid'], FormatDatetime(br["submitted_at"]))
+        else:
+            artifactPath  = "%s_%s_%s" % (safeTranslate(self.artifactBuilderName),
+                                          br['brid'], FormatDatetime(br["submitted_at"]))
+
         if (self.artifactDirectory):
             artifactPath += "/%s" % self.artifactDirectory
 

master/buildbot/test/unit/test_scripts_runner.py

@@ -86,8 +86,8 @@ def parse(self, *args):
     def defaults_and(self, **kwargs):
         defaults = dict(force=False, relocatable=False, config='master.cfg',
                 db='sqlite:///state.sqlite', basedir=os.getcwd(), quiet=False,
-                **{'no-logrotate':False, 'log-size':'10000000',
-                   'log-count':'10'})
+                **{'no-logrotate':False, 'log-size':10000000,
+                   'log-count':10})
         unk_keys = set(kwargs.keys()) - set(defaults.keys())
         assert not unk_keys, "invalid keys %s" % (unk_keys,)
         opts = defaults.copy()
@@ -155,12 +155,12 @@ def test_config_long(self):
 
     def test_log_size(self):
         opts = self.parse('-s124')
-        exp = self.defaults_and(**{'log-size':'124'})
+        exp = self.defaults_and(**{'log-size':124})
         self.assertOptions(opts, exp)
 
     def test_log_size_long(self):
         opts = self.parse('--log-size=124')
-        exp = self.defaults_and(**{'log-size':'124'})
+        exp = self.defaults_and(**{'log-size':124})
         self.assertOptions(opts, exp)
 
     def test_log_size_noninteger(self):
@@ -169,12 +169,17 @@ def test_log_size_noninteger(self):
 
     def test_log_count(self):
         opts = self.parse('-l124')
-        exp = self.defaults_and(**{'log-count':'124'})
+        exp = self.defaults_and(**{'log-count':124})
         self.assertOptions(opts, exp)
 
     def test_log_count_long(self):
         opts = self.parse('--log-count=124')
-        exp = self.defaults_and(**{'log-count':'124'})
+        exp = self.defaults_and(**{'log-count':124})
+        self.assertOptions(opts, exp)
+
+    def test_log_count_none(self):
+        opts = self.parse('--log-count=None')
+        exp = self.defaults_and(**{'log-count':None})
         self.assertOptions(opts, exp)
 
     def test_log_count_noninteger(self):

master/buildbot/test/unit/test_scripts_start.py

@@ -90,18 +90,6 @@ def cb(res):
             print res
         return d
 
-    @compat.skipUnlessPlatformIs('posix')
-    def test_start(self):
-        d = self.runStart()
-        @d.addCallback
-        def cb((out, err, rc)):
-            self.assertEqual((rc, err), (0, ''))
-            self.assertSubstring('BuildMaster is running', out)
-        return d
-
-    if twisted.version <= versions.Version('twisted', 9, 0, 0):
-        test_start.skip = test_start_quiet.skip = "Skipping due to suprious PotentialZombieWarning."
-
     # the remainder of this script does obscene things:
     #  - forks
     #  - shells out to tail

slave/buildslave/bot.py

@@ -112,13 +112,6 @@ def lostRemoteStep(self, remotestep):
         if self.stopCommandOnShutdown:
             self.stopCommand()
 
-    # the following are Commands that can be invoked by the master-side
-    # Builder
-    def remote_startBuild(self):
-        """This is invoked before the first step of any new build is run.  It
-        doesn't do much, but masters call it so it's still here."""
-        pass
-
     def remote_startCommand(self, stepref, stepId, command, args):
         """
         This gets invoked by L{buildbot.process.step.RemoteCommand.start}, as

slave/buildslave/scripts/create_slave.py

@@ -23,7 +23,7 @@
 from buildslave.bot import BuildSlave
 
 basedir = r'%(basedir)s'
-rotateLength = %(log-size)s
+rotateLength = %(log-size)d
 maxRotatedFiles = %(log-count)s
 
 # if this is a relocatable tac file, get the directory containing the TAC