Permalink
Browse files

Kill unused code. r=catlee

  • Loading branch information...
1 parent 3a9a45e commit 5b62abca882b429ba9d66d607e73e73ea448a51e @mozbhearsum mozbhearsum committed Aug 2, 2012
Showing with 2 additions and 1,037 deletions.
  1. +0 −46 l10n.py
  2. +2 −79 process/factory.py
  3. +0 −157 steps/misc.py
  4. +0 −13 steps/release.py
  5. +0 −79 steps/test.py
  6. +0 −366 steps/transfer.py
  7. +0 −297 steps/unittest.py
View
46 l10n.py
@@ -193,49 +193,3 @@ def __init__(self, name, builderNames, **kwargs):
def trigger(self, ss, set_props=None):
reason = "This build was triggered by the successful completion of the en-US nightly."
self.createL10nBuilds(revision=ss.revision, reason=reason, set_props=set_props)
-
-
-class DependentL10n(Dependent, L10nMixin):
- """
- This scheduler runs some set of 'downstream' builds when the
- 'upstream' scheduler has completed successfully.
- """
-
- compare_attrs = Dependent.compare_attrs[:]
-
- def __init__(self, name, upstream, builderNames, **kwargs):
- Dependent.__init__(self, name, upstream, builderNames)
- L10nMixin.__init__(self, **kwargs)
-
- def run(self):
- d = self.parent.db.runInteraction(self._run)
-
- def _createBuilds(sses):
- # This gets called with a list of SourceStamps to do builds for
- l = []
- for ss in sses:
- d1 = self.createL10nBuilds(ss.revision)
- l.append(d1)
- return defer.gatherResults(l)
-
- d.addCallback(_createBuilds)
- # Return None here since we don't need to be woken up in a specific
- # amount of time.
- d.addCallback(lambda ign: None)
-
- return d
-
- def _run(self, t):
- db = self.parent.db
- res = db.scheduler_get_subscribed_buildsets(self.schedulerid, t)
- # this returns bsid,ssid,results for all of our active subscriptions.
- # We ignore the ones that aren't complete yet. This leaves the
- # subscription in place until the buildset is complete.
- sses = []
- for (bsid,ssid,complete,results) in res:
- if complete:
- if results in (SUCCESS, WARNINGS):
- ss = db.getSourceStampNumberedNow(ssid, t)
- sses.append(ss)
- db.scheduler_unsubscribe_buildset(self.schedulerid, bsid, t)
- return sses
View
81 process/factory.py
@@ -22,7 +22,6 @@
import buildbotcustom.steps.release
import buildbotcustom.steps.source
import buildbotcustom.steps.test
-import buildbotcustom.steps.transfer
import buildbotcustom.steps.updates
import buildbotcustom.steps.talos
import buildbotcustom.steps.unittest
@@ -39,7 +38,6 @@
reload(buildbotcustom.steps.release)
reload(buildbotcustom.steps.source)
reload(buildbotcustom.steps.test)
-reload(buildbotcustom.steps.transfer)
reload(buildbotcustom.steps.updates)
reload(buildbotcustom.steps.talos)
reload(buildbotcustom.steps.unittest)
@@ -54,11 +52,11 @@
from buildbotcustom.steps.base import ShellCommand, SetProperty, Mercurial, \
Trigger, RetryingShellCommand, RetryingSetProperty
from buildbotcustom.steps.misc import TinderboxShellCommand, SendChangeStep, \
- GetBuildID, MozillaClobberer, FindFile, DownloadFile, UnpackFile, \
+ MozillaClobberer, FindFile, DownloadFile, UnpackFile, \
SetBuildProperty, DisconnectStep, OutputStep, MockCommand, \
RepackPartners, UnpackTest, FunctionalStep, setBuildIDProps, \
RetryingMockProperty, MockInit, MockInstall
-from buildbotcustom.steps.release import UpdateVerify, SnippetComparison
+from buildbotcustom.steps.release import SnippetComparison
from buildbotcustom.steps.source import MercurialCloneCommand
from buildbotcustom.steps.test import AliveTest, \
CompareLeakLogs, Codesighs, GraphServerPost
@@ -4556,21 +4554,6 @@ def getSnippetDir(self):
self.buildNumber)
-class UpdateVerifyFactory(ReleaseFactory):
- def __init__(self, verifyConfig, buildSpace=.3, useOldUpdater=False,
- **kwargs):
- ReleaseFactory.__init__(self, repoPath='nothing',
- buildSpace=buildSpace, **kwargs)
- command=['bash', 'verify.sh', '-c', verifyConfig]
- if useOldUpdater:
- command.append('--old-updater')
- self.addStep(UpdateVerify(
- command=command,
- workdir='tools/release/updates',
- description=['./verify.sh', verifyConfig]
- ))
-
-
class ReleaseFinalVerification(ReleaseFactory):
def __init__(self, verifyConfigs, platforms=None, **kwargs):
# MozillaBuildFactory needs the 'repoPath' argument, but we don't
@@ -6194,66 +6177,6 @@ def addAddOnInstallSteps(self):
))
n += 1
-class TryTalosFactory(TalosFactory):
- def addDownloadBuildStep(self):
- def get_url(build):
- url = build.source.changes[-1].files[0]
- return url
- self.addStep(DownloadFile(
- url_fn=get_url,
- url_property="fileURL",
- filename_property="filename",
- workdir=self.workdirBase,
- name="Download build",
- ignore_certs=True,
- ))
-
- def make_tinderbox_header(build):
- identifier = build.getProperty("filename").rsplit('-', 1)[0]
- # Grab the submitter out of the dir name. CVS and Mercurial builds
- # are a little different, so we need to try fairly hard to find
- # the e-mail address.
- dir = os.path.basename(os.path.dirname(build.getProperty("fileURL")))
- who = ''
- for section in dir.split('-'):
- if '@' in section:
- who = section
- break
- msg = 'TinderboxPrint: %s\n' % who
- msg += 'TinderboxPrint: %s\n' % identifier
- return msg
- self.addStep(OutputStep(data=make_tinderbox_header, log='header', name='echo_id'))
-
- def addDownloadSymbolsStep(self):
- def get_symbols_url(build):
- suffixes = ('.tar.bz2', '.dmg', '.zip')
- buildURL = build.getProperty('fileURL')
-
- for suffix in suffixes:
- if buildURL.endswith(suffix):
- return buildURL[:-len(suffix)] + '.crashreporter-symbols.zip'
-
- self.addStep(DownloadFile(
- url_fn=get_symbols_url,
- filename_property="symbolsFile",
- workdir=self.workdirBase,
- name="Download symbols",
- ignore_certs=True,
- haltOnFailure=False,
- flunkOnFailure=False,
- ))
- self.addStep(ShellCommand(
- command=['mkdir', 'symbols'],
- workdir=self.workdirBase,
- ))
- self.addStep(UnpackFile(
- filename=WithProperties("../%(symbolsFile)s"),
- workdir="%s/symbols" % self.workdirBase,
- name="Unpack symbols",
- haltOnFailure=False,
- flunkOnFailure=False,
- ))
-
class PartnerRepackFactory(ReleaseFactory):
def getReleaseTag(self, product, version):
View
157 steps/misc.py
@@ -43,58 +43,6 @@
from buildbotcustom.common import genBuildID, genBuildUID
from buildbotcustom.try_parser import processMessage
-def errbackAfter(wrapped_d, timeout):
- # Thanks to Dustin!
- """Calls wrapped_d's errback after timeout seconds"""
- wrapper_d = Deferred()
- already_fired = [False]
- def cb(*args, **kwargs):
- if not already_fired[0]:
- already_fired[0] = True
- wrapper_d.callback(*args, **kwargs)
- else:
- log.msg("callback called again: %s %s" % (args, kwargs))
- def eb(*args, **kwargs):
- if not already_fired[0]:
- already_fired[0] = True
- wrapper_d.errback(*args, **kwargs)
- else:
- log.msg("errback called again: %s %s" % (args, kwargs))
- def to():
- if not already_fired[0]:
- already_fired[0] = True
- wrapper_d.errback(TimeoutError("More than %i seconds elapsed" % timeout))
- reactor.callLater(timeout, to)
- wrapped_d.addCallbacks(cb, eb)
- return wrapper_d
-
-class InterruptableDeferred(Deferred):
- def __init__(self, wrapped_d):
- Deferred.__init__(self)
-
- self.already_fired = False
-
- def callback(*args, **kwargs):
- if not self.already_fired:
- self.already_fired = True
- self.callback(*args, **kwargs)
- else:
- log.msg("callback called again: %s %s" % (args, kwargs))
-
- def errback(*args, **kwargs):
- if not self.already_fired:
- self.already_fired = True
- self.errback(*args, **kwargs)
- else:
- log.msg("errback called again: %s %s" % (args, kwargs))
-
- wrapped_d.addCallbacks(callback, errback)
-
- def interrupt(self, reason="Interrupted"):
- if not self.already_fired:
- self.already_fired = True
- self.errback(DefaultException(reason))
-
class MockInit(ShellCommand):
haltOnFailure=True
@@ -257,25 +205,6 @@ def getText(self, cmd, results):
RetryingMockProperty = addRetryEvaluateCommand(MockProperty)
-class CreateDir(ShellCommand):
- name = "create dir"
- haltOnFailure = False
- warnOnFailure = True
-
- def __init__(self, platform, dir=None, **kwargs):
- self.super_class = ShellCommand
- self.super_class.__init__(self, **kwargs)
- self.addFactoryArguments(platform=platform, dir=dir)
- self.platform = platform
- if dir:
- self.dir = dir
- else:
- if self.platform.startswith('win'):
- self.command = r'if not exist ' + self.dir + r' mkdir ' + \
- self.dir
- else:
- self.command = ['mkdir', '-p', self.dir]
-
class TinderboxShellCommand(ShellCommand):
haltOnFailure = False
@@ -302,92 +231,6 @@ def evaluateCommand(self, cmd):
else:
return FAILURE
-class GetBuildID(ShellCommand):
- """Retrieves the BuildID from a Mozilla tree (using platform.ini) and sets
- it as a build property ('buildid'). If defined, uses objdir as it's base.
- """
- description=['getting buildid']
- descriptionDone=['get buildid']
- haltOnFailure=True
-
- def __init__(self, objdir="", inifile="application.ini", section="App",
- **kwargs):
- self.super_class = ShellCommand
- self.super_class.__init__(self, **kwargs)
- self.addFactoryArguments(objdir=objdir,
- inifile=inifile,
- section=section)
-
- self.objdir = objdir
- self.command = ['python', 'config/printconfigsetting.py',
- '%s/dist/bin/%s' % (self.objdir, inifile),
- section, 'BuildID']
-
- def commandComplete(self, cmd):
- buildid = ""
- try:
- buildid = cmd.logs['stdio'].getText().strip().rstrip()
- self.setProperty('buildid', buildid)
- except:
- log.msg("Could not find BuildID or BuildID invalid")
- log.msg("Found: %s" % buildid)
- return FAILURE
- return SUCCESS
-
-
-class SetMozillaBuildProperties(LoggingBuildStep):
- """Gathers and sets build properties for the following data:
- buildid - BuildID of the build (from application.ini, falling back on
- platform.ini)
- appVersion - The version of the application (from application.ini, falling
- back on platform.ini)
- packageFilename - The filename of the application package
- packageSize - The size (in bytes) of the application package
- packageHash - The sha1 hash of the application package
- installerFilename - The filename of the installer (win32 only)
- installerSize - The size (in bytes) of the installer (win32 only)
- installerHash - The sha1 hash of the installer (win32 only)
- completeMarFilename - The filename of the complete update
- completeMarSize - The size (in bytes) of the complete update
- completeMarHash - The sha1 hash of the complete update
-
- All of these will be set as build properties -- even if no data is found
- for them. When no data is found, the value of the property will be None.
-
- This function requires an argument of 'objdir', which is the path to the
- objdir relative to the builddir. ie, 'mozilla/fx-objdir'.
- """
-
- def __init__(self, objdir="", **kwargs):
- LoggingBuildStep.__init__(self, **kwargs)
- self.addFactoryArguments(objdir=objdir)
- self.objdir = objdir
-
- def describe(self, done=False):
- if done:
- return ["gather", "build", "properties"]
- else:
- return ["gathering", "build", "properties"]
-
- def start(self):
- args = {'objdir': self.objdir, 'timeout': 60}
- cmd = LoggedRemoteCommand("setMozillaBuildProperties", args)
- self.startCommand(cmd)
-
- def evaluateCommand(self, cmd):
- # set all of the data as build properties
- # some of this may come in with the value 'UNKNOWN' - these will still
- # be set as build properties but 'UNKNOWN' will be substituted with None
- try:
- log = cmd.logs['stdio'].getText()
- for property in log.split("\n"):
- name, value = property.split(": ")
- if value == "UNKNOWN":
- value = None
- self.setProperty(name, value)
- except:
- return FAILURE
- return SUCCESS
class SendChangeStep(ShellCommand):
warnOnFailure = True
View
13 steps/release.py
@@ -6,19 +6,6 @@
from buildbotcustom.steps.base import ShellCommand
from buildbotcustom.steps.misc import TinderboxShellCommand
-class UpdateVerify(ShellCommand):
- def __init__(self, **kwargs):
- self.super_class = ShellCommand
- self.super_class.__init__(self, **kwargs)
-
- def evaluateCommand(self, cmd):
- worst = self.super_class.evaluateCommand(self, cmd)
- for line in cmd.logs['stdio'].getText().split("\n"):
- if line.startswith('FAIL'):
- worst = worst_status(worst, FAILURE)
- return worst
-
-
class SnippetComparison(ShellCommand):
# Alphas/Betas are WARN'ed about in a releasetest vs release comparison
# because they were shipped on the 'beta' channel, which means patcher
View
79 steps/test.py
@@ -117,85 +117,6 @@ def tinderboxPrint(testName,
output += unitsSuffix
return output
-class CompareBloatLogs(ShellCommand):
- warnOnWarnings = True
- warnOnFailure = True
-
- def __init__(self, bloatLog, testname="", testnameprefix="",
- bloatDiffPath="tools/rb/bloatdiff.pl",
- mozillaDir="", tbPrint=True, **kwargs):
- self.super_class = ShellCommand
- self.super_class.__init__(self, **kwargs)
- self.addFactoryArguments(bloatLog=bloatLog, testname=testname,
- testnameprefix=testnameprefix,
- bloatDiffPath=bloatDiffPath,
- mozillaDir=mozillaDir, tbPrint=tbPrint)
- self.bloatLog = bloatLog
- self.testname = testname
- self.testnameprefix = testnameprefix
- self.bloatDiffPath = "build%s/%s" % (mozillaDir, bloatDiffPath)
- self.tbPrint = tbPrint
-
- if len(self.testname) > 0:
- self.testname += " "
- if len(self.testnameprefix) > 0:
- self.testnameprefix += " "
-
- self.name = "compare " + self.testname + "bloat logs"
- self.description = ["compare " + self.testname, "bloat logs"]
- self.command = ["perl", self.bloatDiffPath, self.bloatLog + '.old',
- self.bloatLog]
-
- def evaluateCommand(self, cmd):
- superResult = self.super_class.evaluateCommand(self, cmd)
- try:
- leaks = self.getProperty('leaks')
- except:
- log.msg("Could not find build property: leaks")
- return worst_status(superResult, FAILURE)
- if leaks and int(leaks) > 0:
- return worst_status(superResult, WARNINGS)
- return superResult
-
- def createSummary(self, log):
- summary = "######################## BLOAT STATISTICS\n"
- totalLineList = []
- leaks = 0
- bloat = 0
- for line in log.readlines():
- summary += line
- if leaks == 0 and bloat == 0:
- if "TOTAL" in line:
- m = re.search('TOTAL\s+(\d+)\s+[\-\d\.]+\%\s+(\d+)',
- line)
- leaks = int(m.group(1))
- bloat = int(m.group(2))
- summary += "######################## END BLOAT STATISTICS\n\n"
-
- # Scrape for leak/bloat totals from TOTAL line
- # TOTAL 23 0% 876224
- summary += "leaks = %d\n" % leaks
- summary += "bloat = %d\n" % bloat
-
- leaksAbbr = "%sRLk" % self.testnameprefix
- leaksTestname = ("%srefcnt_leaks" % self.testnameprefix).replace(' ', '_')
- leaksTestnameLabel = "%srefcnt Leaks" % self.testnameprefix
-
- if self.tbPrint:
- tinderLink = tinderboxPrint(leaksTestname,
- leaksTestnameLabel,
- 0,
- 'bytes',
- leaksAbbr,
- formatBytes(leaks,3)
- )
- summary += tinderLink
-
- self.setProperty('leaks',leaks)
- self.setProperty('bloat',bloat)
- self.setProperty('testresults', [(leaksAbbr, leaksTestname, leaks, formatBytes(leaks,3))])
- self.addCompleteLog(leaksAbbr + ":" + formatBytes(leaks,3),
- summary)
class CompareLeakLogs(ShellCommand):
warnOnWarnings = True
View
366 steps/transfer.py
@@ -1,366 +0,0 @@
-from os import path
-from time import gmtime, strftime, strptime
-
-import buildbot
-from buildbotcustom.common import getSupportedPlatforms
-from buildbotcustom.steps.base import ShellCommand
-
-class MozillaStageUpload(ShellCommand):
- def __init__(self, objdir, username, milestone, platform, remoteHost,
- remoteBasePath, group=None, chmodMode=755, sshKey=None,
- releaseToDated=True, releaseToLatest=True,
- releaseToTinderboxBuilds=True, tinderboxBuildsDir=None,
- releaseToCandidates=False, remoteCandidatesPath=None,
- dependToDated=True, uploadCompleteMar=True,
- uploadLangPacks=False, packageGlob=None, **kwargs):
- """
- @type objdir: string
- @param objdir: The obj directory used for the build. This is needed to
- find the packages in the source tree.
-
- @type username: string
- @param username: The username used to login with on the remote host.
- The buildslaves should have passwordless logins to
- this account.
-
- @type milestone: string
- @param milestone: The milestone of the build (eg, trunk)
-
- @type platform: string
- @param platform: The platform we are uploading for. One of 'win32',
- 'linux', 'linux64', 'macosx' or 'macosx64'.
-
- @type remoteHost: string
- @param remoteHost: The server to upload the builds to.
-
- @type remoteBasePath: string
- @param remoteBasePath: The directory on the server used as a base path
- for these builds. eg:
- /home/ftp/pub/firefox
-
- @type group: string
- @param group: If group is set, any files uploaded will be chgrp'ed to
- it. Default: None
-
- @type chmodMode: int
- @param chmodMode: The mode used when fixing permissions on remoteHost.
- Default: 755
-
- @type sshKey: string
- @param sshKey: If defined, the filename of the ssh key to use. It
- should be relative to ${HOME}/.ssh/. Default: None
-
- @type releaseToDated: bool
- @param releaseToDated: If True, builds will be pushed to
- 'remoteBasePath'/nightly/yyyy/mm/yyyy-mm-dd-hh-milestone. This
- directory will also be symlinked in 'remoteBasePath'/nightly/.
- Generally, this should be True for nightlies. Default: True
-
- @type releaseToLatest: bool
- @param releaseToLatest: If True, builds will be pushed to
- 'remoteBasePath'/nightly/latest-milestone. If
- releaseToDated=True, builds will be copied from
- 'remoteBasePath'/nightly/yyyy/mm/yyyy-mm-dd-hh-milestone.
- Otherwise, builds will be uploaded from the slave. Generally,
- this should be True for nightlies. Default: True
-
- @type releaseToTinderboxBuilds: bool
- @param releaseToTinderboxBuilds: If True, builds will be pushed to
- 'remoteBasePath'/tinderbox-builds/$hostname. This should
- generally be set to True for all builds. Default: True
-
- @type tinderboxBuildsDir: string
- @param tinderboxBuildsDir: This option only has effect when
- releaseToTinderboxBuilds is True. If this
- option is None (default), builds will be
- uploaded to:
- tinderbox-builds/builderName
- If otherwise set builds will be uploaded to
- tinderbox-builds/tinderboxBuildsDir.
-
- @type releaseToCandidates: bool
- @param releaseToCandidates: If True, builds will be pushed to
- remoteCandidatesDir. This should only be set for releases.
-
- @type remoteCandidatesDir: string
- @param remoteCandidatesDir: This option only has effect, and is
- required, when releaseToCandidates is
- True.
-
- @type dependToDated: This option only has effect when
- releaseToTinderboxBuilds is True. When
- dependToDated is True builds will be placed in
- a subdirectory named for the build start time
- (in unix time) when being pushed to the
- tinderbox-builds dir. For example:
- tinderbox-builds/builder/1203094573. The option
- defaults to True.
-
- @type uploadCompleteMar: bool
- @param uploadCompleteMar: When True, the MozillaStageUpload will upload
- the complete mar file found in dist/update to
- the datedDir/latestDir. This option only
- applies when releaseToDated or
- releaseToLatest is True. Default: True
-
- @type uploadLangPacks: bool
- @param uploadLangPacks: When True, the MozillaStageUpload will upload
- language pack XPIs to the datedDir/latestDir.
- This option only applies when releaseToDated or
- releaseToLatest is True. Default: False
-
- @type packageGlob: string
- @param packageGlob: The shell wildcard pattern that expresses the build
- files we will be uploading. Default: each platform gets
- a sensible default in objdir/dist/*.{ext} with ext tailored
- for that platform (i.e. .zip, .dmg, .tar.gz)
-
- """
-
- ShellCommand.__init__(self, **kwargs)
- self.addFactoryArguments(objdir=objdir,
- username=username,
- milestone=milestone,
- platform=platform,
- remoteHost=remoteHost,
- remoteBasePath=remoteBasePath,
- packageGlob=packageGlob,
- group=group,
- chmodMode=chmodMode,
- sshKey=sshKey,
- releaseToDated=releaseToDated,
- releaseToLatest=releaseToLatest,
- releaseToTinderboxBuilds=releaseToTinderboxBuilds,
- tinderboxBuildsDir=tinderboxBuildsDir,
- releaseToCandidates=releaseToCandidates,
- remoteCandidatesPath=remoteCandidatesPath,
- dependToDated=dependToDated,
- uploadCompleteMar=uploadCompleteMar,
- uploadLangPacks=uploadLangPacks)
-
- assert platform in getSupportedPlatforms()
- if releaseToCandidates:
- assert remoteCandidatesPath
- self.objdir = objdir
- self.username = username
- self.milestone = milestone
- self.platform = platform
- self.remoteHost = remoteHost
- self.remoteBasePath = remoteBasePath
- self.packageGlob = packageGlob
- self.group = group
- self.chmodMode = chmodMode
- self.sshKey = sshKey
- self.releaseToDated = releaseToDated
- self.releaseToLatest = releaseToLatest
- self.releaseToCandidates = releaseToCandidates
- self.remoteCandidatesPath = remoteCandidatesPath
- self.releaseToTinderboxBuilds = releaseToTinderboxBuilds
- self.tinderboxBuildsDir = tinderboxBuildsDir
- self.dependToDated = dependToDated
- self.uploadCompleteMar = uploadCompleteMar
- self.uploadLangPacks = uploadLangPacks
-
- self.description = ["uploading package(s) to", remoteHost]
- self.descriptionDone = ["upload package(s) to", remoteHost]
-
- def _getBaseCommand(self, ssh=False, scp=False):
- assert not (ssh and scp)
- assert (ssh or scp)
-
- command = ""
- # scp cannot use the '-l' format
- if ssh:
- command += 'ssh'
- command += ' -l ' + self.username
- else:
- command += 'scp'
-
- if self.sshKey:
- # surprisingly, this works on Windows (probably because Buildbot)
- # gets started from MSYS
- command += ' -i ' + '~/.ssh/%s' % self.sshKey
- return command
-
- def getBuildID(self):
- # the build id is extracted in a previous step and set as a build
- # property
- buildid = self.getProperty("buildid")
- if len(buildid) == 14:
- return strftime("%Y-%m-%d-%H-%M-%S", strptime(buildid[0:14], "%Y%m%d%H%M%S"))
- else:
- return strftime("%Y-%m-%d-%H", strptime(buildid[0:10], "%Y%m%d%H"))
-
- def getBuildStartTime(self):
- return int(self.step_status.build.getTimes()[0])
-
- def getPackageDirectory(self):
- return '%s-%s' % (self.getBuildID(), self.milestone)
-
- def getPackageGlob(self):
- if self.packageGlob:
- # allow a WithProperties packageGlob
- if str(self.packageGlob) is not self.packageGlob:
- properties = self.build.getProperties()
- self.packageGlob = properties.render(self.packageGlob)
- return self.packageGlob
- # i can't find a better way to do this.
- if self.platform == "win32":
- return '%s/dist/*.zip %s/dist/install/sea/*.exe' % (self.objdir,
- self.objdir)
- if self.platform.startswith('macosx'):
- return '%s/dist/*.dmg' % self.objdir
- if self.platform.startswith('linux'):
- return '%s/dist/*.tar.bz2' % self.objdir
-
- def getLongDatedPath(self):
- buildid = self.getBuildID()
- fullRemotePath = path.join(self.remoteBasePath, 'nightly',
- buildid.split('-')[0], # the year
- buildid.split('-')[1], # the month
- self.getPackageDirectory()
- )
- return fullRemotePath
-
- def getLatestPath(self):
- return path.join(self.remoteBasePath, 'nightly',
- 'latest-%s' % self.milestone)
-
- def getCandidatesPath(self):
- return self.remoteCandidatesPath
-
- def getTinderboxBuildsPath(self):
- tboxBuildsPath = path.join(self.remoteBasePath, 'tinderbox-builds')
- if self.tinderboxBuildsDir:
- tboxBuildsPath = path.join(tboxBuildsPath, self.tinderboxBuildsDir)
- else:
- tboxBuildsPath = path.join(tboxBuildsPath,
- self.step_status.build.builder.getName())
- if self.dependToDated:
- tboxBuildsPath = path.join(tboxBuildsPath,
- str(self.getBuildStartTime()))
- return tboxBuildsPath
-
- def createDirCommand(self, dir):
- return self._getBaseCommand(ssh=True) + ' ' + self.remoteHost + \
- ' mkdir -p ' + dir
-
- def uploadCommand(self, dir):
- return self._getBaseCommand(scp=True) + ' ' + self.getPackageGlob() + \
- ' ' + self.username + '@' + self.remoteHost + ':' + \
- dir
-
- def chmodCommand(self, dir):
- return self._getBaseCommand(ssh=True) + ' ' + self.remoteHost + \
- ' chmod -R ' + str(self.chmodMode) + ' ' + dir
-
- def chgrpCommand(self, dir):
- return self._getBaseCommand(ssh=True) + ' ' + self.remoteHost + \
- ' chgrp -R ' + self.group + ' ' + dir
-
- def syncCommand(self, src, dst):
- # rsync needs trailing slashes
- src += '/'
- dst += '/'
- return self._getBaseCommand(ssh=True) + ' ' + self.remoteHost + \
- ' rsync -av ' + src + ' ' + dst
-
- def symlinkDateDirCommand(self, datedDir):
- # Make a relative symlink, absolute symlinks break ftp
- # unless you are careful to get the right root
- # eg ln -fs 2008/03/2008-03-01-01-mozilla-central
- # /home/ftp/pub/firefox/nightly/
- targetDir = path.join(self.remoteBasePath, 'nightly','')
- shortDatedDir = datedDir.replace(targetDir, '')
- return self._getBaseCommand(ssh=True) + ' ' + self.remoteHost + \
- ' ln -fs ' + shortDatedDir + ' ' + targetDir
-
- def uploadCompleteMarCommand(self, dir):
- packageGlob = '%s/dist/update/*.complete.mar' % self.objdir
- return self._getBaseCommand(scp=True) + ' ' + packageGlob + \
- ' ' + self.username + '@' + self.remoteHost + ':' + \
- dir
-
- def uploadLangPacksCommand(self, dir):
- packageGlob = '%s/dist/install/*.langpack.xpi' % self.objdir
- return self._getBaseCommand(scp=True) + ' ' + packageGlob + \
- ' ' + self.username + '@' + self.remoteHost + ':' + \
- dir
-
- def start(self):
- datedDir = self.getLongDatedPath()
- latestDir = self.getLatestPath()
- tinderboxBuildsDir = self.getTinderboxBuildsPath()
- candidatesDir = self.getCandidatesPath()
-
- commands = []
- if self.releaseToDated:
- # 1) Create the directory on the staging server.
- # 2) Upload the package(s).
- # 3) Fix the permissions on the package(s).
- # 4) Maybe adjust the group on the package(s).
- # 5) Symlink the longer dated directory to the shorter one.
- cmd = ""
- cmd += self.createDirCommand(datedDir) + " && " + \
- self.uploadCommand(datedDir)
- if self.uploadCompleteMar:
- cmd += " && " + self.uploadCompleteMarCommand(datedDir)
- if self.uploadLangPacks:
- cmd += " && " + self.uploadLangPacksCommand(datedDir)
- cmd += " && " + self.chmodCommand(datedDir)
- if self.group:
- cmd += " && " + self.chgrpCommand(datedDir)
- cmd += " && " + self.symlinkDateDirCommand(datedDir)
- commands.append(cmd)
-
- if self.releaseToLatest:
- # 1) Create the directory on the staging server.
- # 2) If there was a dated release, rsync those files to the
- # latest-(milestone) directory.
- # 3) If not, upload the package(s).
- # 4) Fix the permissions on the package(s).
- # 5) Maybe adjust the group on the package(s).
- cmd = ""
- cmd += self.createDirCommand(latestDir) + " && "
- if self.releaseToDated:
- cmd += self.syncCommand(datedDir, latestDir) + " && "
- else:
- cmd += self.uploadCommand(latestDir) + " && "
- if self.uploadCompleteMar:
- cmd += self.uploadCompleteMarCommand(latestDir) + " && "
- if self.uploadLangPacks:
- cmd += self.uploadLangPacksCommand(latestDir) + " && "
- cmd += self.chmodCommand(latestDir)
- if self.group:
- cmd += " && " + self.chgrpCommand(latestDir)
- commands.append(cmd)
-
- if self.releaseToTinderboxBuilds:
- # 1) Create the directory on the staging server.
- # 2) Upload the package(s).
- # 3) Fix the permissions on the package(s).
- # 4) Maybe adjust the group on the package(s).
- cmd = ""
- cmd += self.createDirCommand(tinderboxBuildsDir) + " && " + \
- self.uploadCommand(tinderboxBuildsDir) + " && " + \
- self.chmodCommand(tinderboxBuildsDir)
- if self.group:
- cmd += " && " + self.chgrpCommand(tinderboxBuildsDir)
- commands.append(cmd)
-
- if self.releaseToCandidates:
- # 1) Create the directory on the staging server.
- # 2) Upload the package(s).
- # 3) Fix the permissions on the package(s).
- # 4) Maybe adjust the group on the package(s).
- cmd = ""
- cmd += self.createDirCommand(candidatesDir) + " && " + \
- self.uploadCommand(candidatesDir) + " && " + \
- self.chmodCommand(candidatesDir)
- if self.group:
- cmd += " && " + self.chgrpCommand(candidatesDir)
- commands.append(cmd)
-
- finalCommand = ' && '.join(commands)
- self.setCommand(finalCommand)
- ShellCommand.start(self)
View
297 steps/unittest.py
@@ -216,26 +216,6 @@ def summarizeTUnit(name, log):
return "TinderboxPrint: %s<br/>%s\n" % (name,
summaryText(passCount, failCount, leaked = leaked))
-def summarizeMozmillLog(name, log):
- log = log.getText()
- summary = { 'Passed': None,
- 'Failed': None,
- 'Skipped': None }
- beginnings = dict([(i, 'INFO %s:' % i) for i in summary])
- for line in log.splitlines():
- for condition in summary:
- beginning = beginnings[condition]
- if line.startswith(beginning):
- n = line.split(beginning, 1)[-1].strip()
- try:
- summary[condition] = int(n)
- except ValueError:
- continue
-
- _summaryText = summaryText(summary.get('Passed', 0),
- summary.get('Failed', 0))
- return 'TinderboxPrint: %s<br/>%s\n' % (name, _summaryText)
-
def evaluateMochitest(name, log, superResult):
# When a unittest fails we mark it orange, indicating with the
# WARNINGS status. Therefore, FAILURE needs to become WARNINGS
@@ -408,194 +388,6 @@ def evaluateCommand(self, cmd):
return superResult
-class MozillaPackagedMozmillTests(ShellCommandReportTimeout):
- warnOnFailure=True
- warnOnWarnings=True
-
- def __init__(self, tests_dir, binary, platform, restart=False, **kwargs):
- """
- run mozmill tests;
- should be run from the mozmill virtualenv directory
- - tests_dir : path to the tests to run
- - binary : path to the binary to be used
- - platform : system platform
- - restart : whether to run the restart tests
- """
-
- self.super_class = ShellCommandReportTimeout
- ShellCommandReportTimeout.__init__(self, **kwargs)
-
- self.addFactoryArguments(tests_dir=tests_dir,
- binary=binary,
- platform=platform,
- restart=restart)
-
- # set platform-dependent python scripts
- # relative to the created virtualenv
- if platform.startswith("win"):
- python = 'Scripts/python'
- if restart:
- mozmill = 'Scripts/mozmill-restart-script.py'
- else:
- mozmill = 'Scripts/mozmill-script.py'
- else:
- python = 'bin/python'
- if restart:
- mozmill = 'bin/mozmill-restart'
- else:
- mozmill = 'bin/mozmill'
-
- # command to run
- script = [
- python,
- '-E',
- mozmill,
- '--show-all',
- '-b', binary,
- '-t', tests_dir,
- ]
- self.command = ['bash', '-c', WithProperties(' '.join(script))]
-
- def createSummary(self, log):
- self.addCompleteLog('summary', summarizeMozmillLog(self.name, log))
-
- def evaluateCommand(self, cmd):
- superResult = self.super_class.evaluateCommand(self, cmd)
- if superResult == FAILURE:
- superResult = WARNINGS
-
- # If we find "TEST-UNEXPECTED", then return WARNINGS
- if "TEST-UNEXPECTED" in cmd.logs['stdio'].getText():
- return worst_status(superResult, WARNINGS)
-
- return worst_status(superResult, SUCCESS)
-
-
-class MozillaCheckoutClientMk(ShellCommandReportTimeout):
- haltOnFailure = True
- cvsroot = ":pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot"
-
- def __init__(self,
- cvsroot=":pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot",
- **kwargs):
- self.cvsroot = cvsroot
- if not 'command' in kwargs:
- kwargs['command'] = ["cvs", "-d", self.cvsroot, "co", "mozilla/client.mk"]
- ShellCommandReportTimeout.__init__(self, **kwargs)
- self.addFactoryArguments(cvsroot=cvsroot)
-
- def describe(self, done=False):
- return ["client.mk update"]
-
-
-class MozillaClientMkPull(ShellCommandReportTimeout):
- haltOnFailure = True
- def __init__(self, project="browser", workdir="mozilla", env={},
- command=["make", "-f", "client.mk", "pull_all"], **kwargs):
- self.project = project
- self.workdir = workdir
- # MOZ_CO_PROJECT: "used in the try server cvs trunk builders,
- # not used by the unittests on tryserver though".
- env = env.copy()
- env["MOZ_CO_PROJECT"] = self.project
- ShellCommandReportTimeout.__init__(self, command=command, env=env,
- **kwargs)
- self.addFactoryArguments(project=project, workdir=workdir, env=env,
- command=command)
-
- def describe(self, done=False):
- if not done:
- return ["pulling (" + self.project + ")"]
- return ["pull (" + self.project + ")"]
-
-
-class MozillaPackage(ShellCommandReportTimeout):
- name = "package"
- warnOnFailure = True
- description = ["packaging"]
- descriptionDone = ["package"]
- command = ["make"]
-
-class UpdateClobberFiles(ShellCommandReportTimeout):
- name = "update clobber files"
- warnOnFailure = True
- description = "updating clobber files"
- descriptionDone = "clobber files updated"
- clobberFilePath = "clobber_files/"
- logDir = '../logs/'
-
- def __init__(self, **kwargs):
- if not 'platform' in kwargs:
- return FAILURE
- self.platform = kwargs['platform']
- if 'clobberFilePath' in kwargs:
- self.clobberFilePath = kwargs['clobberFilePath']
- if 'logDir' in kwargs:
- self.logDir = kwargs['logDir']
- if self.platform.startswith('win32'):
- self.tboxClobberModule = 'mozilla/tools/tinderbox-configs/firefox/win32'
- elif self.platform.startswith('win64'):
- self.tboxClobberModule = 'mozilla/tools/tinderbox-configs/firefox/win64'
- else:
- self.tboxClobberModule = 'mozilla/tools/tinderbox-configs/firefox/' + self.platform
- if 'cvsroot' in kwargs:
- self.cvsroot = kwargs['cvsroot']
- if 'branch' in kwargs:
- self.branchString = ' -r ' + kwargs['branch']
- self.buildbotClobberModule = 'mozilla/tools/buildbot-configs/testing/unittest/CLOBBER/firefox/' + kwargs['branch'] + '/' + self.platform
- else:
- self.branchString = ''
- self.buildbotClobberModule = 'mozilla/tools/buildbot-configs/testing/unittest/CLOBBER/firefox/TRUNK/' + self.platform
-
- if not 'command' in kwargs:
- self.command = r'cd ' + self.clobberFilePath + r' && cvs -d ' + self.cvsroot + r' checkout' + self.branchString + r' -d tinderbox-configs ' + self.tboxClobberModule + r'>' + self.logDir + tboxClobberCvsCoLog + r' && cvs -d ' + self.cvsroot + r' checkout -d buildbot-configs ' + self.buildbotClobberModule + r'>' + self.logDir + buildbotClobberCvsCoLog
- ShellCommandReportTimeout.__init__(self, **kwargs)
-
-class MozillaClobber(ShellCommandReportTimeout):
- name = "clobber"
- description = "checking clobber file"
- descriptionDone = "clobber checked"
- clobberFilePath = "clobber_files/"
- logDir = 'logs/'
-
- def __init__(self, **kwargs):
- if 'platform' in kwargs:
- self.platform = kwargs['platform']
- if 'logDir' in kwargs:
- self.logDir = kwargs['logDir']
- if 'clobberFilePath' in kwargs:
- self.clobberFilePath = kwargs['clobberFilePath']
- if not 'command' in kwargs:
- tboxGrepCommand = r"grep -q '^U tinderbox-configs.CLOBBER' " + self.logDir + tboxClobberCvsCoLog
- tboxPrintHeader = "echo Tinderbox clobber file updated"
- tboxCatCommand = "cat %s/tinderbox-configs/CLOBBER" % self.clobberFilePath
- buildbotGrepCommand = r"grep -q '^U buildbot-configs.CLOBBER' " + self.logDir + buildbotClobberCvsCoLog
- buildbotPrintHeader = "echo Buildbot clobber file updated"
- buildbotCatCommand = "cat %s/buildbot-configs/CLOBBER" % self.clobberFilePath
- rmCommand = "rm -rf mozilla"
- printExitStatus = "echo No clobber required"
- self.command = tboxGrepCommand + r' && ' + tboxPrintHeader + r' && ' + tboxCatCommand + r' && ' + rmCommand + r'; if [ $? -gt 0 ]; then ' + buildbotGrepCommand + r' && ' + buildbotPrintHeader + r' && ' + buildbotCatCommand + r' && ' + rmCommand + r'; fi; if [ $? -gt 0 ]; then ' + printExitStatus + r'; fi'
- ShellCommandReportTimeout.__init__(self, **kwargs)
-
-class MozillaClobberWin(ShellCommandReportTimeout):
- name = "clobber win"
- description = "checking clobber file"
- descriptionDone = "clobber finished"
-
- def __init__(self, **kwargs):
- platformFlag = ""
- slaveNameFlag = ""
- branchFlag = ""
- if 'platform' in kwargs:
- platformFlag = " --platform=" + kwargs['platform']
- if 'slaveName' in kwargs:
- slaveNameFlag = " --slaveName=" + kwargs['slaveName']
- if 'branch' in kwargs:
- branchFlag = " --branch=" + kwargs['branch']
- if not 'command' in kwargs:
- self.command = 'python C:\\Utilities\\killAndClobberWin.py' + platformFlag + slaveNameFlag + branchFlag
- ShellCommandReportTimeout.__init__(self, **kwargs)
-
class MozillaCheck(ShellCommandReportTimeout):
warnOnFailure = True
@@ -645,95 +437,6 @@ def evaluateCommand(self, cmd):
return worst_status(superResult, SUCCESS)
-class MozillaReftest(ShellCommandReportTimeout):
- warnOnFailure = True
-
- def __init__(self, test_name, leakThreshold=None, env={}, **kwargs):
- self.name = test_name
- self.command = ["make", test_name]
- self.description = [test_name + " test"]
- self.descriptionDone = [self.description[0] + " complete"]
- self.super_class = ShellCommandReportTimeout
-
- env = env.copy()
- if leakThreshold:
- env["EXTRA_TEST_ARGS"] = "--leak-threshold=%d" % leakThreshold
- ShellCommandReportTimeout.__init__(self, env=env, **kwargs)
- self.addFactoryArguments(test_name=test_name,
- leakThreshold=leakThreshold)
-
- def createSummary(self, log):
- self.addCompleteLog('summary', summarizeLogReftest(self.name, log))
-
- def evaluateCommand(self, cmd):
- superResult = self.super_class.evaluateCommand(self, cmd)
- # When a unittest fails we mark it orange, indicating with the
- # WARNINGS status. Therefore, FAILURE needs to become WARNINGS
- # However, we don't want to override EXCEPTION or RETRY, so we still
- # need to use worst_status in further status decisions.
- if superResult == FAILURE:
- superResult = WARNINGS
-
- if superResult != SUCCESS:
- return worst_status(superResult, WARNINGS)
-
- # Assume that having the "Unexpected: 0" line
- # means the tests run completed (successfully).
- # Also check for "^TEST-UNEXPECTED-" for harness errors.
- if not re.search(r"^REFTEST INFO \| Unexpected: 0 \(", cmd.logs["stdio"].getText(), re.MULTILINE) or \
- re.search("^TEST-UNEXPECTED-", cmd.logs["stdio"].getText(), re.MULTILINE):
- return worst_status(superResult, WARNINGS)
-
- return worst_status(superResult, SUCCESS)
-
-
-class MozillaMochitest(ShellCommandReportTimeout):
- warnOnFailure = True
-
- def __init__(self, test_name, leakThreshold=None, env={}, **kwargs):
- self.name = test_name
- self.command = ["make", test_name]
- self.description = [test_name + " test"]
- self.descriptionDone = [self.description[0] + " complete"]
- self.super_class = ShellCommandReportTimeout
-
- env = env.copy()
- if leakThreshold:
- env["EXTRA_TEST_ARGS"] = "--leak-threshold=%d" % leakThreshold
- ShellCommandReportTimeout.__init__(self, env=env, **kwargs)
- self.addFactoryArguments(test_name=test_name,
- leakThreshold=leakThreshold)
-
- def createSummary(self, log):
- self.addCompleteLog('summary', summarizeLogMochitest(self.name, log))
-
- def evaluateCommand(self, cmd):
- superResult = self.super_class.evaluateCommand(self, cmd)
- # When a unittest fails we mark it orange, indicating with the
- # WARNINGS status. Therefore, FAILURE needs to become WARNINGS
- # However, we don't want to override EXCEPTION or RETRY, so we still
- # need to use worst_status in further status decisions.
- if superResult == FAILURE:
- superResult = WARNINGS
-
- if superResult != SUCCESS:
- return worst_status(superResult, WARNINGS)
-
- failIdent = r"^\d+ INFO Failed: 0"
- # Support browser-chrome result summary format which differs
- # from MozillaMochitest's.
- if self.name == 'mochitest-browser-chrome':
- failIdent = r"^\tFailed: 0"
- # Assume that having the 'failIdent' line
- # means the tests run completed (successfully).
- # Also check for "^TEST-UNEXPECTED-" for harness errors.
- if not re.search(failIdent, cmd.logs["stdio"].getText(), re.MULTILINE) or \
- re.search("^TEST-UNEXPECTED-", cmd.logs["stdio"].getText(), re.MULTILINE):
- return worst_status(superResult, WARNINGS)
-
- return worst_status(superResult, SUCCESS)
-
-
class MozillaPackagedXPCShellTests(ShellCommandReportTimeout):
warnOnFailure = True
warnOnWarnings = True

0 comments on commit 5b62abc

Please sign in to comment.