Browse files

PEP8! r=bhearsum

  • Loading branch information...
1 parent 432e177 commit 42df7d645bf18b5e7420fc5fb6b0deb398fe3bf7 @catlee catlee committed Feb 1, 2013
Showing with 4,608 additions and 3,195 deletions.
  1. +16 −11 buildbot-helpers/check_queuedir.py
  2. +36 −20 buildbot-helpers/command_runner.py
  3. +2 −1 buildbot-helpers/force_build.py
  4. +19 −18 buildbot-helpers/force_release_l10n.py
  5. +31 −22 buildbot-helpers/pulse_publisher.py
  6. +29 −27 buildbot-helpers/release_sanity.py
  7. +23 −13 buildfarm/breakpad/cleanup-breakpad-symbols.py
  8. +1 −1 buildfarm/breakpad/post-symbol-upload.py
  9. +7 −2 buildfarm/maintenance/add-master.py
  10. +21 −11 buildfarm/maintenance/buildbot-wrangler.py
  11. +15 −10 buildfarm/maintenance/check-master-json.py
  12. +14 −11 buildfarm/maintenance/count_and_reboot.py
  13. +74 −68 buildfarm/maintenance/foopy_fabric.py
  14. +22 −17 buildfarm/maintenance/manage_foopies.py
  15. +14 −11 buildfarm/maintenance/manage_masters.py
  16. +5 −3 buildfarm/maintenance/master_cleanup.py
  17. +4 −2 buildfarm/maintenance/normalize-masters.py
  18. +28 −20 buildfarm/maintenance/print_waits.py
  19. +26 −17 buildfarm/maintenance/purge_builds.py
  20. +3 −1 buildfarm/maintenance/purge_events.py
  21. +4 −3 buildfarm/maintenance/test_purge_builds.py
  22. +79 −75 buildfarm/maintenance/try_sendchange.py
  23. +31 −20 buildfarm/maintenance/watch_twistd_log.py
  24. +33 −30 buildfarm/mobile/devices_per_foopy.py
  25. +19 −13 buildfarm/mobile/n900-imaging/extract_rootfs.py
  26. +5 −4 buildfarm/mobile/n900-imaging/flash.py
  27. +49 −45 buildfarm/mobile/n900-imaging/flasher_wrapper.py
  28. +11 −8 buildfarm/mobile/n900-imaging/generate_image.py
  29. +40 −30 buildfarm/mobile/n900-imaging/image_manip.py
  30. +9 −5 buildfarm/mobile/n900-imaging/rootfs/bin/run-ntp.py
  31. +9 −6 buildfarm/mobile/n900-imaging/rootfs/bin/uptime-check.py
  32. +5 −5 buildfarm/mobile/pushfile.py
  33. +13 −10 buildfarm/mobile/setup_foopy.py
  34. +13 −9 buildfarm/opsi/regenerate-hostkey.py
  35. +4 −2 buildfarm/release/release-runner.py
  36. +6 −2 buildfarm/utils/count_ctors.py
  37. +31 −23 buildfarm/utils/generate-tpcomponent.py
  38. +143 −114 buildfarm/utils/generator/places/builddb/generate.py
  39. +4 −3 buildfarm/utils/generator/places/builddb/http.py
  40. +10 −9 buildfarm/utils/generator/places/builddb/increment_dates.py
  41. +52 −28 buildfarm/utils/generator/places/builddb/models.py
  42. +1 −1 buildfarm/utils/generator/places/manage.py
  43. +1 −1 buildfarm/utils/generator/places/settings.py
  44. +8 −8 buildfarm/utils/generator/places/urls.py
  45. +98 −101 buildfarm/utils/generator/places/uuid.py
  46. +8 −4 buildfarm/utils/gittool.py
  47. +9 −7 buildfarm/utils/graph_server_post.py
  48. +24 −20 buildfarm/utils/hgtool.py
  49. +2 −2 buildfarm/utils/jsontool.py
  50. +44 −27 buildfarm/utils/mar.py
  51. +3 −2 buildfarm/utils/printbuildrev.py
  52. +26 −14 buildfarm/utils/retry.py
  53. +17 −9 buildfarm/utils/run_jetpack.py
  54. +4 −2 buildfarm/utils/unix_util.py
  55. +38 −33 buildfarm/utils/updateGenerated.py
  56. +5 −2 buildfarm/utils/win32_util.py
  57. +180 −165 clobberer/clobberer.py
  58. +79 −60 clobberer/test_clobberer.py
  59. +3 −0 graphserver_webapp/webapp/config.py
  60. +58 −31 graphserver_webapp/webapp/graphserver.py
  61. +4 −2 graphserver_webapp/webapp/test/test_graphserver.py
  62. +12 −6 lib/python/balrog/client/api.py
  63. +2 −1 lib/python/build/download.py
  64. +21 −11 lib/python/build/l10n.py
  65. +1 −0 lib/python/build/misc.py
  66. +21 −19 lib/python/build/upload.py
  67. +14 −7 lib/python/build/versions.py
  68. +4 −3 lib/python/kickoff/api.py
  69. +12 −7 lib/python/mozilla_buildtools/queuedir.py
  70. +9 −4 lib/python/mozilla_buildtools/test/test_build_checksums.py
  71. +7 −2 lib/python/mozilla_buildtools/test/test_build_versions.py
  72. +3 −1 lib/python/mozilla_buildtools/test/test_release_config.py
  73. +6 −2 lib/python/mozilla_buildtools/test/test_release_info.py
  74. +2 −1 lib/python/mozilla_buildtools/test/test_release_l10n.py
  75. +3 −1 lib/python/mozilla_buildtools/test/test_release_paths.py
  76. +61 −31 lib/python/mozilla_buildtools/test/test_release_updates_patcher.py
  77. +19 −9 lib/python/mozilla_buildtools/test/test_release_updates_snippets.py
  78. +20 −17 lib/python/mozilla_buildtools/test/test_release_updates_verify.py
  79. +2 −0 lib/python/mozilla_buildtools/test/test_release_versions.py
  80. +128 −112 lib/python/mozilla_buildtools/test/test_signing.py
  81. +12 −6 lib/python/mozilla_buildtools/test/test_signing_server.py
  82. +1 −1 lib/python/mozilla_buildtools/test/test_util_algorithms.py
  83. +4 −1 lib/python/mozilla_buildtools/test/test_util_archives.py
  84. +5 −2 lib/python/mozilla_buildtools/test/test_util_commands.py
  85. +8 −3 lib/python/mozilla_buildtools/test/test_util_file.py
  86. +38 −19 lib/python/mozilla_buildtools/test/test_util_git.py
  87. +108 −66 lib/python/mozilla_buildtools/test/test_util_hg.py
  88. +13 −8 lib/python/mozilla_buildtools/test/test_util_paths.py
  89. +8 −1 lib/python/mozilla_buildtools/test/test_util_retry.py
  90. +0 −1 lib/python/release/__init__.py
  91. +2 −1 lib/python/release/config.py
  92. +12 −6 lib/python/release/download.py
  93. +22 −7 lib/python/release/info.py
  94. +18 −9 lib/python/release/l10n.py
  95. +3 −1 lib/python/release/paths.py
  96. +16 −2 lib/python/release/platforms.py
  97. +1 −0 lib/python/release/sanity.py
  98. +5 −2 lib/python/release/signing.py
  99. +45 −24 lib/python/release/updates/patcher.py
  100. +8 −4 lib/python/release/updates/snippets.py
  101. +4 −2 lib/python/release/updates/verify.py
  102. +5 −1 lib/python/release/versions.py
  103. +35 −21 lib/python/signing/client.py
  104. +101 −64 lib/python/signing/server.py
  105. +45 −23 lib/python/signing/utils.py
  106. +8 −2 lib/python/slavealloc/client.py
  107. +5 −4 lib/python/slavealloc/daemon/application.py
  108. +19 −4 lib/python/slavealloc/daemon/http/api.py
  109. +5 −2 lib/python/slavealloc/daemon/http/gettac.py
  110. +3 −0 lib/python/slavealloc/daemon/http/site.py
  111. +3 −1 lib/python/slavealloc/daemon/http/ui.py
  112. +8 −5 lib/python/slavealloc/daemon/service.py
  113. +130 −96 lib/python/slavealloc/data/model.py
  114. +67 −61 lib/python/slavealloc/data/queries.py
  115. +1 −0 lib/python/slavealloc/data/setup.py
  116. +1 −0 lib/python/slavealloc/exceptions.py
  117. +14 −13 lib/python/slavealloc/logic/allocate.py
  118. +1 −0 lib/python/slavealloc/logic/buildbottac.py
  119. +9 −5 lib/python/slavealloc/scripts/dbdump.py
  120. +56 −50 lib/python/slavealloc/scripts/dbimport.py
  121. +7 −3 lib/python/slavealloc/scripts/dbinit.py
  122. +22 −15 lib/python/slavealloc/scripts/disable.py
  123. +8 −3 lib/python/slavealloc/scripts/enable.py
  124. +12 −6 lib/python/slavealloc/scripts/gettac.py
  125. +18 −13 lib/python/slavealloc/scripts/lock.py
  126. +6 −4 lib/python/slavealloc/scripts/main.py
  127. +11 −6 lib/python/slavealloc/scripts/notes.py
  128. +5 −2 lib/python/util/__init__.py
  129. +5 −3 lib/python/util/algorithms.py
  130. +34 −19 lib/python/util/archives.py
  131. +1 −1 lib/python/util/commands.py
  132. +6 −6 lib/python/util/fabric/actions.py
  133. +12 −2 lib/python/util/file.py
  134. +12 −6 lib/python/util/git.py
  135. +56 −24 lib/python/util/hg.py
  136. +7 −2 lib/python/util/paths.py
  137. +32 −25 lib/python/util/post_file.py
  138. +11 −6 lib/python/util/retry.py
  139. +1 −0 lib/python/util/transfer.py
  140. +6 −5 lib/python/util/tuxedo.py
  141. +26 −13 release/generate-candidate-build-updates.py
  142. +4 −2 release/signing/get_token.py
  143. +64 −38 release/signing/sign-release.py
  144. +28 −13 release/signing/signing-server.py
  145. +37 −28 release/signing/signscript.py
  146. +47 −36 release/signing/signtool.py
  147. +1 −0 release/signing/verify-gpg.py
  148. +47 −28 release/signing/verify-signature.py
  149. +26 −17 release/tuxedo-add.py
  150. +33 −24 scripts/android/android_snippet.py
  151. +38 −23 scripts/l10n/create-release-repacks.py
  152. +6 −6 scripts/preproduction/repo_setup.py
  153. +4 −4 scripts/preproduction/repo_setup_config.py
  154. +5 −2 scripts/release/generate-sums.py
  155. +34 −20 scripts/release/stage-tasks.py
  156. +34 −16 scripts/release/tag-release.py
  157. +11 −7 scripts/release/updates/chunked-verify.py
  158. +8 −4 scripts/staging/release_downloader.py
  159. +21 −13 scripts/support/mouse_and_screen_resolution.py
  160. +1 −1 scripts/updates/balrog-client.py
  161. +26 −13 scripts/updates/create-snippets.py
  162. +1 −1 scripts/updates/create-update-verify-configs.py
  163. +8 −8 setup.py
  164. +18 −8 stage/extract_and_run_command.py
  165. +79 −49 stage/post_upload.py
  166. +67 −58 sut_tools/check.py
  167. +30 −22 sut_tools/cleanup.py
  168. +98 −77 sut_tools/clientproxy.py
  169. +18 −13 sut_tools/config.py
  170. +96 −83 sut_tools/dashboard.py
  171. +34 −23 sut_tools/installApp.py
  172. +8 −7 sut_tools/installTests.py
  173. +0 −1 sut_tools/mozdevice/__init__.py
  174. +35 −23 sut_tools/mozdevice/devicemanager.py
  175. +72 −49 sut_tools/mozdevice/devicemanagerADB.py
  176. +103 −72 sut_tools/mozdevice/devicemanagerSUT.py
  177. +73 −70 sut_tools/mozdevice/droid.py
  178. +16 −11 sut_tools/reboot.py
  179. +15 −4 sut_tools/relay.py
  180. +5 −4 sut_tools/sentrypdu.py
  181. +21 −16 sut_tools/smoketest.py
  182. +24 −20 sut_tools/stop.py
  183. +110 −60 sut_tools/sut_lib.py
  184. +3 −3 sut_tools/tegra_checkstalled.py
  185. +2 −2 sut_tools/tegra_powercycle.py
  186. +29 −13 sut_tools/test_installApp.py
  187. +17 −11 sut_tools/u_apks.py
  188. +24 −14 sut_tools/updateSUT.py
  189. +61 −41 sut_tools/verify.py
View
27 buildbot-helpers/check_queuedir.py
@@ -2,16 +2,21 @@
"""%prog -w <warn_new> -c <crit_new> -t <max_age> queuedir [queuedir...]
nagios plugin to monitor a queuedir"""
-import os, sys, traceback, time
+import os
+import sys
+import traceback
+import time
OK, WARNING, CRITICAL, UNKNOWN = range(4)
+
def check_queuedir(d, options):
status = OK
msgs = []
# Check 'dead'
- num_dead = len([f for f in os.listdir(os.path.join(d, 'dead')) if not f.endswith(".log")])
+ num_dead = len([f for f in os.listdir(
+ os.path.join(d, 'dead')) if not f.endswith(".log")])
if num_dead > 0:
status = CRITICAL
if num_dead == 1:
@@ -24,7 +29,7 @@ def check_queuedir(d, options):
num_new = len(new_files)
if num_new > 0:
oldest_new = min(
- os.path.getmtime(os.path.join(d, 'new', f)) for f in new_files)
+ os.path.getmtime(os.path.join(d, 'new', f)) for f in new_files)
if num_new >= options.crit_new:
status = CRITICAL
msgs.append("%i new items" % num_new)
@@ -37,23 +42,23 @@ def check_queuedir(d, options):
status = max(status, WARNING)
msgs.append("oldest item is %is old" % age)
-
return status, msgs
+
def main():
from optparse import OptionParser
parser = OptionParser(__doc__)
parser.set_defaults(
- warn_new=5,
- crit_new=10,
- max_age=300,
- )
+ warn_new=5,
+ crit_new=10,
+ max_age=300,
+ )
parser.add_option("-w", dest="warn_new", type="int",
- help="warn when there are more than this number of items in new")
+ help="warn when there are more than this number of items in new")
parser.add_option("-c", dest="crit_new", type="int",
- help="critical when there are more than this number of items in new")
+ help="critical when there are more than this number of items in new")
parser.add_option("-t", dest="max_age", type="int",
- help="warn when oldest item in new is more than this many seconds old")
+ help="warn when oldest item in new is more than this many seconds old")
options, args = parser.parse_args()
View
56 buildbot-helpers/command_runner.py
@@ -2,13 +2,16 @@
"""
Runs commands from a queue!
"""
-import subprocess, os, signal
+import subprocess
+import os
+import signal
import time
from mozilla_buildtools.queuedir import QueueDir
from buildbot.util import json
import logging
log = logging.getLogger(__name__)
+
class Job(object):
def __init__(self, cmd, item_id, log_fp):
self.cmd = cmd
@@ -24,15 +27,17 @@ def start(self):
devnull = open(os.devnull, 'r')
self.log.write("Running %s\n" % self.cmd)
self.log.flush()
- self.proc = subprocess.Popen(self.cmd, close_fds=True, stdin=devnull, stdout=self.log, stderr=self.log)
+ self.proc = subprocess.Popen(self.cmd, close_fds=True, stdin=devnull,
+ stdout=self.log, stderr=self.log)
self.started = time.time()
def check(self):
now = time.time()
if now - self.started > self.max_time:
# Kill stuff off
if now - self.last_signal_time > 60:
- s = {None: signal.SIGINT, signal.SIGINT: signal.SIGTERM}.get(self.last_signal, signal.SIGKILL)
+ s = {None: signal.SIGINT, signal.SIGINT:
+ signal.SIGTERM}.get(self.last_signal, signal.SIGKILL)
log.info("Killing %i with %i", self.proc.pid, s)
try:
self.log.write("Killing with %s\n" % s)
@@ -50,6 +55,7 @@ def check(self):
self.log.close()
return result
+
class CommandRunner(object):
def __init__(self, options):
self.queuedir = options.queuedir
@@ -73,7 +79,8 @@ def run(self, job):
job.start()
self.active.append(job)
except OSError:
- job.log.write("\nFailed with OSError; requeuing in %i seconds\n" % self.retry_time)
+ job.log.write("\nFailed with OSError; requeuing in %i seconds\n" %
+ self.retry_time)
# Wait to requeue it
# If we die, then it's still in cur, and will be moved back into
# 'new' eventually
@@ -94,7 +101,8 @@ def monitor(self):
else:
log.warn("%s failed; requeuing", job.item_id)
# Requeue it!
- self.q.requeue(job.item_id, self.retry_time, self.max_retries)
+ self.q.requeue(
+ job.item_id, self.retry_time, self.max_retries)
def loop(self):
"""
@@ -132,25 +140,32 @@ def loop(self):
finally:
fp.close()
+
def main():
from optparse import OptionParser
import logging.handlers
parser = OptionParser()
parser.set_defaults(
- concurrency=1,
- max_retries=1,
- retry_time=0,
- verbosity=0,
- logfile=None,
- max_time=60,
- )
+ concurrency=1,
+ max_retries=1,
+ retry_time=0,
+ verbosity=0,
+ logfile=None,
+ max_time=60,
+ )
parser.add_option("-q", "--queuedir", dest="queuedir")
- parser.add_option("-j", "--jobs", dest="concurrency", type="int", help="number of commands to run at once")
- parser.add_option("-r", "--max_retries", dest="max_retries", type="int", help="number of times to retry commands")
- parser.add_option("-t", "--retry_time", dest="retry_time", type="int", help="seconds to wait between retries")
- parser.add_option("-v", "--verbose", dest="verbosity", action="count", help="increase verbosity")
- parser.add_option("-l", "--logfile", dest="logfile", help="where to send logs")
- parser.add_option("-m", "--max_time", dest="max_time", type="int", help="maximum time for a command to run")
+ parser.add_option("-j", "--jobs", dest="concurrency", type="int",
+ help="number of commands to run at once")
+ parser.add_option("-r", "--max_retries", dest="max_retries",
+ type="int", help="number of times to retry commands")
+ parser.add_option("-t", "--retry_time", dest="retry_time",
+ type="int", help="seconds to wait between retries")
+ parser.add_option("-v", "--verbose", dest="verbosity",
+ action="count", help="increase verbosity")
+ parser.add_option(
+ "-l", "--logfile", dest="logfile", help="where to send logs")
+ parser.add_option("-m", "--max_time", dest="max_time", type="int",
+ help="maximum time for a command to run")
options, args = parser.parse_args()
@@ -163,12 +178,13 @@ def main():
log_level = logging.DEBUG
if not options.logfile:
- logging.basicConfig(level=log_level, format="%(asctime)s - %(message)s")
+ logging.basicConfig(
+ level=log_level, format="%(asctime)s - %(message)s")
else:
logger = logging.getLogger()
logger.setLevel(log_level)
handler = logging.handlers.RotatingFileHandler(
- options.logfile, maxBytes=1024**2, backupCount=5)
+ options.logfile, maxBytes=1024 ** 2, backupCount=5)
formatter = logging.Formatter("%(asctime)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
View
3 buildbot-helpers/force_build.py
@@ -4,6 +4,7 @@
from urllib2 import urlopen
from urlparse import urljoin
+
class Forcer:
maxProperties = 3
"""A Forcer knows how to force a build through the Buildbot web interface
@@ -27,7 +28,7 @@ def getArgs(self, name, comments, branch, revision, properties):
if revision:
args['revision'] = revision
i = 1
- for key,value in properties.iteritems():
+ for key, value in properties.iteritems():
p = "property%d" % i
args['%sname' % p] = key
args['%svalue' % p] = value
View
37 buildbot-helpers/force_release_l10n.py
@@ -12,8 +12,9 @@
ALL_PLATFORMS = getSupportedPlatforms()
+
class L10nForcer:
- ignoredLocales = ('en-US') # locales that are _always_ to be skipped
+ ignoredLocales = ('en-US') # locales that are _always_ to be skipped
def __init__(self, masterUrl, toForce, releaseTag, name="Unknown",
comments="Unknown", delay=2, loud=True):
@@ -24,7 +25,7 @@ def __init__(self, masterUrl, toForce, releaseTag, name="Unknown",
assert isinstance(toForce, dict), "toForce must be a dict!"
for value in toForce.itervalues():
assert isinstance(value, (tuple, list)), \
- "toForce values must be a list or tuple"
+ "toForce values must be a list or tuple"
for platform in toForce.iterkeys():
forcer = Forcer(masterUrl, '%s_repack' % platform, loud)
self.forcers[platform] = forcer
@@ -42,7 +43,7 @@ def getProperties(self, locale):
def forceBuilds(self):
errors = []
- for platform,locales in self.toForce.iteritems():
+ for platform, locales in self.toForce.iteritems():
forcer = self.forcers[platform]
for locale in sorted(locales):
if locale in self.ignoredLocales:
@@ -60,14 +61,14 @@ def forceBuilds(self):
branch=locale, properties=properties)
sleep(self.delay)
except:
- errors.append((platform,locale))
+ errors.append((platform, locale))
if self.loud:
"*** Error when forcing %s %s" % (platform, locale)
# Don't raise the exception, keep going
if self.loud:
if len(errors) > 0:
print "ERROR SUMMARY:"
- for platform,locale in map(lambda x: (x[0], x[1]), errors):
+ for platform, locale in map(lambda x: (x[0], x[1]), errors):
print " %s - %s" % (platform, locale)
@@ -77,7 +78,7 @@ def getShippedLocales(shippedLocales):
properlyParsed = {}
for p in ALL_PLATFORMS:
properlyParsed[p] = []
- for locale,platforms in parsed.iteritems():
+ for locale, platforms in parsed.iteritems():
if platforms:
for p in platforms:
# map the bouncer/shipped-locales platforms to automation ones
@@ -131,27 +132,27 @@ def filterPlatforms(shippedLocales, includedPlatforms):
default="Unknown",
help="Name to use when submitting.")
parser.add_option("-m", "--master", action="store", dest="master",
- help="The base url of the master to submit to. " + \
+ help="The base url of the master to submit to. " +
"Eg, http://localhost:8010")
parser.add_option("-t", "--release-tag", action="store", dest="releaseTag",
help="The tag to build with. Eg, FIREFOX_3_5_3_RELEASE")
parser.add_option("-c", "--comments", action="store", dest="comments",
default="Unknown")
parser.add_option("-d", "--delay", action="store", dest="delay",
default=2,
- help="Amount of time (in seconds) to wait between each" +\
+ help="Amount of time (in seconds) to wait between each" +
"POST. Defaults to 2 seconds")
parser.add_option("-v", "--verbose", action="store_true", dest="loud",
default=False)
parser.add_option("-g", "--hg", action="store", dest="hg",
default="http://hg.mozilla.org",
- help="Root of the HG server. Defaults to " + \
- "http://hg.mozilla.org. Only used when -s " + \
+ help="Root of the HG server. Defaults to " +
+ "http://hg.mozilla.org. Only used when -s " +
"isn't specified.")
parser.add_option("-b", "--branch", action="store", dest="branch",
- help="The branch, relative to the HG server, to " + \
- "locate shipped locales on. " + \
- "Eg, releases/mozilla-1.9.1. Only used when " + \
+ help="The branch, relative to the HG server, to " +
+ "locate shipped locales on. " +
+ "Eg, releases/mozilla-1.9.1. Only used when " +
"-s isn't specified.")
parser.add_option("-s", "--shipped-locales-file", action="store",
dest="shippedLocales", default=None,
@@ -169,10 +170,10 @@ def filterPlatforms(shippedLocales, includedPlatforms):
help="When specified, only platforms passed with -p will "
"have builds forced. By default, builds will be "
"forced for all platforms %s for " %
- str(ALL_PLATFORMS) +
- "locales without exceptions. Platform exceptions "
- "will be obeyed regardless of -p options used. See "
- "example usage for further details.")
+ str(ALL_PLATFORMS) +
+ "locales without exceptions. Platform exceptions "
+ "will be obeyed regardless of -p options used. See "
+ "example usage for further details.")
(options, args) = parser.parse_args()
@@ -183,7 +184,7 @@ def filterPlatforms(shippedLocales, includedPlatforms):
f.close()
else:
file = '%s/raw-file/%s/browser/locales/shipped-locales' % \
- (options.branch, options.releaseTag)
+ (options.branch, options.releaseTag)
url = urljoin(options.hg, file)
shippedLocales = urlopen(url).read()
View
53 buildbot-helpers/pulse_publisher.py
@@ -25,9 +25,11 @@
skip_exps = [
# Skip step events, they cause too much load
re.compile("^build\.\S+\.\d+\.step\."),
- ]
+]
# A UTC class.
+
+
class UTC(tzinfo):
"""UTC"""
@@ -40,6 +42,7 @@ def tzname(self, dt):
def dst(self, dt):
return ZERO
+
def transform_time(t):
"""Transform an epoch time to a string representation of the form
YYYY-mm-ddTHH:MM:SS+0000"""
@@ -51,6 +54,7 @@ def transform_time(t):
dt = datetime.fromtimestamp(t, UTC())
return dt.strftime('%Y-%m-%dT%H:%M:%S%z')
+
def transform_times(event):
"""Replace epoch times in event with string representations of the time"""
if isinstance(event, dict):
@@ -64,6 +68,7 @@ def transform_times(event):
retval = event
return retval
+
class PulsePusher(object):
"""
Publish buildbot events via pulse.
@@ -85,7 +90,7 @@ class PulsePusher(object):
`max_retries` - how many times to retry
"""
def __init__(self, queuedir, publisher, max_idle_time=300,
- max_connect_time=600, retry_time=60, max_retries=5):
+ max_connect_time=600, retry_time=60, max_retries=5):
self.queuedir = QueueDir('pulse', queuedir)
self.publisher = publisher
self.max_idle_time = max_idle_time
@@ -111,7 +116,7 @@ def send(self, events):
log.debug("Sending %i messages", len(events))
start = time.time()
skipped = 0
- sent = 0
+ sent = 0
for e in events:
routing_key = e['event']
if any(exp.search(routing_key) for exp in skip_exps):
@@ -124,7 +129,8 @@ def send(self, events):
self.publisher.publish(msg)
sent += 1
end = time.time()
- log.info("Sent %i messages in %.2fs (skipped %i)", sent, end-start, skipped)
+ log.info("Sent %i messages in %.2fs (skipped %i)", sent,
+ end - start, skipped)
self._last_activity = time.time()
# Update our timers
@@ -194,7 +200,8 @@ def loop(self):
# Don't try again soon, something has gone horribly wrong!
come_back_soon = False
for item_id in item_ids:
- self.queuedir.requeue(item_id, self.retry_time, self.max_retries)
+ self.queuedir.requeue(
+ item_id, self.retry_time, self.max_retries)
if come_back_soon:
# Let's do more right now!
@@ -212,28 +219,29 @@ def loop(self):
log.info("Waiting for %s", to_wait)
self.queuedir.wait(to_wait)
+
def main():
from optparse import OptionParser
from mozillapulse.publishers import GenericPublisher
from mozillapulse.config import PulseConfiguration
import logging.handlers
parser = OptionParser()
parser.set_defaults(
- verbosity=0,
- logfile=None,
- max_retries=5,
- retry_time=60,
- )
+ verbosity=0,
+ logfile=None,
+ max_retries=5,
+ retry_time=60,
+ )
parser.add_option("--passwords", dest="passwords")
parser.add_option("-q", "--queuedir", dest="queuedir")
parser.add_option("-v", "--verbose", dest="verbosity", action="count",
- help="increase verbosity")
+ help="increase verbosity")
parser.add_option("-l", "--logfile", dest="logfile",
- help="where to send logs")
+ help="where to send logs")
parser.add_option("-r", "--max_retries", dest="max_retries", type="int",
- help="number of times to retry")
+ help="number of times to retry")
parser.add_option("-t", "--retry_time", dest="retry_time", type="int",
- help="seconds to wait between retries")
+ help="seconds to wait between retries")
options, args = parser.parse_args()
@@ -246,12 +254,13 @@ def main():
log_level = logging.DEBUG
if not options.logfile:
- logging.basicConfig(level=log_level, format="%(asctime)s - %(message)s")
+ logging.basicConfig(
+ level=log_level, format="%(asctime)s - %(message)s")
else:
logger = logging.getLogger()
logger.setLevel(log_level)
handler = logging.handlers.RotatingFileHandler(
- options.logfile, maxBytes=1024**2, backupCount=5)
+ options.logfile, maxBytes=1024 ** 2, backupCount=5)
formatter = logging.Formatter("%(asctime)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
@@ -265,14 +274,14 @@ def main():
execfile(options.passwords, passwords, passwords)
publisher = GenericPublisher(
- PulseConfiguration(
- user=passwords['PULSE_USERNAME'],
- password=passwords['PULSE_PASSWORD'],
- ),
- exchange=passwords['PULSE_EXCHANGE'])
+ PulseConfiguration(
+ user=passwords['PULSE_USERNAME'],
+ password=passwords['PULSE_PASSWORD'],
+ ),
+ exchange=passwords['PULSE_EXCHANGE'])
pusher = PulsePusher(options.queuedir, publisher,
- max_retries=options.max_retries, retry_time=options.retry_time)
+ max_retries=options.max_retries, retry_time=options.retry_time)
pusher.loop()
if __name__ == '__main__':
View
56 buildbot-helpers/release_sanity.py
@@ -110,12 +110,12 @@ def verify_mozconfigs(branch, revision, hghost, product, mozconfigs,
if line[0] == '-':
if platform in mozconfigWhitelist.get(branch_name, {}):
if clean_line in \
- mozconfigWhitelist[branch_name][platform]:
+ mozconfigWhitelist[branch_name][platform]:
continue
elif line[0] == '+':
if platform in mozconfigWhitelist.get('nightly', {}):
if clean_line in \
- mozconfigWhitelist['nightly'][platform]:
+ mozconfigWhitelist['nightly'][platform]:
continue
else:
log.warning("%s not in %s %s!" % (
@@ -270,7 +270,7 @@ def verify_l10n_dashboard(l10n_changesets, l10n_dashboard_version=None):
elif revision != dash_revision:
log.error("\tlocale %s revisions not matching: %s (config)"
" vs. %s (dashboard)"
- % (locale, revision, dash_revision))
+ % (locale, revision, dash_revision))
success = False
error_tally.add('verify_l10n_dashboard')
for locale in dash_changesets:
@@ -307,7 +307,7 @@ def verify_options(cmd_options, config):
success = False
error_tally.add('verify_options')
if cmd_options.buildNumber and \
- int(cmd_options.buildNumber) != int(config['buildNumber']):
+ int(cmd_options.buildNumber) != int(config['buildNumber']):
log.error("buildNumber passed in does not match release_configs")
success = False
error_tally.add('verify_options')
@@ -353,7 +353,7 @@ def verify_options(cmd_options, config):
"--bypass-l10n-dashboard-check", dest="checkL10nDashboard",
action="store_false", help="don't verify l10n changesets against the dashboard (implied when --bypass-l10n-check is passed)")
parser.add_option(
- "-m", "--bypass-mozconfig-check", dest="checkMozconfigs",
+ "-m", "--bypass-mozconfig-check", dest="checkMozconfigs",
action="store_false", help="don't verify mozconfigs")
parser.add_option(
"-d", "--dryrun", "--dry-run", dest="dryrun", action="store_true",
@@ -451,7 +451,7 @@ def verify_options(cmd_options, config):
branchConfig = BRANCHES[options.branch]
source_repo = 'comm'
- #Match command line options to defaults in release_configs
+ # Match command line options to defaults in release_configs
if not verify_options(options, releaseConfig):
test_success = False
log.error("Error verifying command-line options,"
@@ -463,13 +463,15 @@ def verify_options(cmd_options, config):
repo_path = \
releaseConfig['sourceRepositories'][source_repo]['path']
revision = \
- releaseConfig['sourceRepositories'][source_repo]['revision']
+ releaseConfig[
+ 'sourceRepositories'][source_repo]['revision']
except KeyError:
try:
repo_path = \
releaseConfig['sourceRepositories']['mobile']['path']
revision = \
- releaseConfig['sourceRepositories']['mobile']['revision']
+ releaseConfig[
+ 'sourceRepositories']['mobile']['revision']
except KeyError:
log.error("Can't determine sourceRepo for mozconfigs")
if options.checkMozconfigs and \
@@ -483,8 +485,8 @@ def verify_options(cmd_options, config):
test_success = False
log.error("Error verifying mozconfigs")
- #verify that the release_configs on-disk match the tagged
- #revisions in hg
+ # verify that the release_configs on-disk match the tagged
+ # revisions in hg
l10nRevisionFile = path.join(configs_dir, 'mozilla',
releaseConfig['l10nRevisionFile'])
if not options.skip_verify_configs and \
@@ -499,34 +501,34 @@ def verify_options(cmd_options, config):
log.error("Error verifying configs")
if options.checkL10n:
- #verify that l10n changesets exist
+ # verify that l10n changesets exist
if not verify_l10n_changesets(branchConfig['hghost'],
l10nRevisionFile):
test_success = False
log.error("Error verifying l10n changesets")
if options.checkL10nDashboard:
- #verify that l10n changesets match the dashboard
+ # verify that l10n changesets match the dashboard
if not verify_l10n_dashboard(
l10nRevisionFile,
- options.l10n_dashboard_version):
+ options.l10n_dashboard_version):
test_success = False
log.error("Error verifying l10n dashboard changesets")
- #verify that l10n changesets match the shipped locales
+ # verify that l10n changesets match the shipped locales
if releaseConfig.get('shippedLocalesPath'):
sr = releaseConfig['sourceRepositories'][source_repo]
sourceRepoPath = sr.get('clonePath', sr['path'])
shippedLocales = getLocaleListFromShippedLocales(
- getShippedLocales(
- releaseConfig['productName'],
- releaseConfig['appName'],
- releaseConfig['version'],
- releaseConfig['buildNumber'],
- sourceRepoPath,
- 'http://hg.mozilla.org',
- sr['revision'],
- ))
+ getShippedLocales(
+ releaseConfig['productName'],
+ releaseConfig['appName'],
+ releaseConfig['version'],
+ releaseConfig['buildNumber'],
+ sourceRepoPath,
+ 'http://hg.mozilla.org',
+ sr['revision'],
+ ))
# l10n_changesets do not have an entry for en-US
if 'en-US' in shippedLocales:
shippedLocales.remove('en-US')
@@ -536,17 +538,17 @@ def verify_options(cmd_options, config):
log.error("Error verifying l10n_changesets matches"
" shipped_locales")
- #verify that the relBranch + revision in the release_configs
- #exists in hg
+ # verify that the relBranch + revision in the release_configs
+ # exists in hg
for sr in releaseConfig['sourceRepositories'].values():
sourceRepoPath = sr.get('clonePath', sr['path'])
if not verify_repo(sourceRepoPath, sr['revision'],
branchConfig['hghost']):
test_success = False
log.error("Error verifying repos")
- #if this is a respin, verify that the version/milestone files
- #have been bumped
+ # if this is a respin, verify that the version/milestone files
+ # have been bumped
if buildNumber > 1:
for sr in releaseConfig['sourceRepositories'].values():
if not verify_build(sr, branchConfig['hghost']):
View
36 buildfarm/breakpad/cleanup-breakpad-symbols.py
@@ -10,9 +10,10 @@
# $BUILDID_EXTRA is an optional identifier for feature branches, etc.
#
# This script, given a path to a symbol store, removes symbols
-# for the oldest builds there.
+# for the oldest builds there.
-import os, os.path
+import os
+import os.path
import sys
import re
from datetime import datetime, timedelta
@@ -44,6 +45,8 @@
# multiple times.
atoms = []
atomdict = {}
+
+
def atomize(s):
if s in atomdict:
return atomdict[s]
@@ -52,13 +55,16 @@ def atomize(s):
atomdict[s] = a
return a
+
def sortByBuildID(x, y):
"Sort two symbol index filenames by the Build IDs contained within"
(a, b) = (os.path.basename(x).split('-')[3],
os.path.basename(y).split('-')[3])
- return cmp(a,b)
+ return cmp(a, b)
buildidRE = re.compile("(\d\d\d\d)(\d\d)(\d\d)(\d\d)")
+
+
def datetimefrombuildid(f):
"""Given a symbol index filename, return a datetime representing the
Build ID contained within it."""
@@ -68,11 +74,13 @@ def datetimefrombuildid(f):
# punt
return datetime.now()
+
def adddefault(d, key, default):
"If d[key] does not exist, set d[key] = default."
if key not in d:
d[key] = default
+
def addFiles(symbolindex, filesDict):
"""Return a list of atoms representing the symbols in this index file.
Also add 1 to filesDict[atom] for each symbol."""
@@ -89,11 +97,13 @@ def addFiles(symbolindex, filesDict):
pass
return l
+
def markDeleteSymbols(symbols, filesDict):
"Decrement reference count by one for each symbol in this symbol index."
for a in symbols:
filesDict[a] -= 1
+
def deletefile(f):
if options.dry_run:
print "rm ", f
@@ -107,15 +117,15 @@ def deletefile(f):
builds = {}
allfiles = {}
buildfiles = {}
-print "[1/4] Reading symbol index files..."
+print "[1/4] Reading symbol index files..."
# get symbol index files, there's one per build
for f in os.listdir(symbolPath):
if not (os.path.isfile(os.path.join(symbolPath, f)) and
f.endswith("-symbols.txt")):
continue
# increment reference count of all symbol files listed in the index
# and also keep a list of files from that index
- buildfiles[f] = addFiles(os.path.join(symbolPath,f), allfiles)
+ buildfiles[f] = addFiles(os.path.join(symbolPath, f), allfiles)
# drop -symbols.txt
parts = f.split("-")[:-1]
(product, version, osName, buildId) = parts[:4]
@@ -137,34 +147,34 @@ def deletefile(f):
branch = version
# group into bins by product-branch-os[-featurebranch]
identifier = "%s-%s-%s" % (product, branch, osName)
- if len(parts) > 4: # extra buildid, probably
+ if len(parts) > 4: # extra buildid, probably
identifier += "-" + "-".join(parts[4:])
adddefault(builds, identifier, [])
builds[identifier].append(f)
-print "[2/4] Looking for symbols to delete..."
+print "[2/4] Looking for symbols to delete..."
oldestdate = datetime.now() - maxNightlyAge
for bin in builds:
builds[bin].sort(sortByBuildID)
if len(builds[bin]) > nightliesPerBin:
# delete the oldest builds if there are too many
for f in builds[bin][:-nightliesPerBin]:
markDeleteSymbols(buildfiles[f], allfiles)
- deletefile(os.path.join(symbolPath,f))
+ deletefile(os.path.join(symbolPath, f))
builds[bin] = builds[bin][-nightliesPerBin:]
# now look for really old symbol files
for f in builds[bin]:
if datetimefrombuildid(f) < oldestdate:
markDeleteSymbols(buildfiles[f], allfiles)
- deletefile(os.path.join(symbolPath,f))
+ deletefile(os.path.join(symbolPath, f))
-print "[3/4] Deleting symbols..."
+print "[3/4] Deleting symbols..."
# now delete all files marked for deletion
for a, refcnt in allfiles.iteritems():
if refcnt == 0:
- deletefile(os.path.join(symbolPath,atoms[a]))
+ deletefile(os.path.join(symbolPath, atoms[a]))
-print "[4/4] Pruning empty directories..."
+print "[4/4] Pruning empty directories..."
sys.exit(0)
# now delete empty directories.
for root, dirs, files in os.walk(symbolPath, topdown=False):
@@ -175,4 +185,4 @@ def deletefile(f):
print "rm -rf ", fullpath
else:
os.rmdir(fullpath)
-print "Done!"
+print "Done!"
View
2 buildfarm/breakpad/post-symbol-upload.py
@@ -7,4 +7,4 @@
# to convert symbols to a different format in the very near future.
if __name__ == '__main__':
- pass
+ pass
View
9 buildfarm/maintenance/add-master.py
@@ -1,10 +1,15 @@
#!/usr/bin/python
-import simplejson, sys, os, re
+import simplejson
+import sys
+import os
+import re
+
def new_master(hostname, type_, instance=1):
assert type_ in ('build', 'tests', 'try')
localname = "%s%i" % (type_, instance)
- hostname_data = re.match("buildbot-master(\d{2})\.build\.(\w+)\.mozilla\.com", hostname)
+ hostname_data = re.match(
+ "buildbot-master(\d{2})\.build\.(\w+)\.mozilla\.com", hostname)
assert hostname_data
host_abbrev = "bm%s" % hostname_data.group(1)
datacentre = hostname_data.group(2)
View
32 buildfarm/maintenance/buildbot-wrangler.py
@@ -5,7 +5,13 @@
Any errors generated will be printed to stderr.
"""
-import os, sys, time, signal, subprocess, urllib
+import os
+import sys
+import time
+import signal
+import subprocess
+import urllib
+
def graceful_stop(port):
url = "http://localhost:%s/shutdown" % port
@@ -15,22 +21,23 @@ def graceful_stop(port):
except IOError:
pass
+
class Watcher:
def __init__(self, fname):
self.fname = fname
self.started = False
if os.path.exists(fname):
- os.utime(fname, None)
+ os.utime(fname, None)
else:
- try:
- open(fname, 'w').close()
- except:
- print "Could not create %s." % fname
- sys.exit(1)
+ try:
+ open(fname, 'w').close()
+ except:
+ print "Could not create %s." % fname
+ sys.exit(1)
self.fp = open(fname)
- self.fp.seek(0, 2) # SEEK_END
+ self.fp.seek(0, 2) # SEEK_END
self.current_inode = os.fstat(self.fp.fileno()).st_ino
@@ -164,7 +171,8 @@ def watch_logfile(self):
w = ReconfigWatcher(twistd_log)
null = open(os.devnull, 'w')
- p = subprocess.Popen(['make', 'start', master_dir], stdout=null, stderr=null)
+ p = subprocess.Popen(
+ ['make', 'start', master_dir], stdout=null, stderr=null)
w.watch_logfile()
elif action == "stop":
@@ -191,7 +199,8 @@ def watch_logfile(self):
w = ReconfigWatcher(twistd_log)
null = open(os.devnull, 'w')
- p = subprocess.Popen(['make', 'start', master_dir], stdout=null, stderr=null)
+ p = subprocess.Popen(
+ ['make', 'start', master_dir], stdout=null, stderr=null)
w.watch_logfile()
elif action == "start":
@@ -200,5 +209,6 @@ def watch_logfile(self):
else:
w = ReconfigWatcher(twistd_log)
null = open(os.devnull, 'w')
- p = subprocess.Popen(['make', 'start', master_dir], stdout=null, stderr=null)
+ p = subprocess.Popen(
+ ['make', 'start', master_dir], stdout=null, stderr=null)
w.watch_logfile()
View
25 buildfarm/maintenance/check-master-json.py
@@ -9,28 +9,31 @@
log = logging.getLogger(__name__)
+
def check_masters(masters):
retval = True
for master in masters:
if not check_master(master):
retval = False
return retval
+
def load_masters(url):
if 'http' in url:
fp = urllib.urlopen(url)
else:
fp = open(url)
return json.load(fp)
+
def check_master(master):
# Check required keys
name = master['name']
required_keys = ('hostname', 'enabled', 'master_dir', 'name', 'role',
- 'basedir', 'bbconfigs_dir', 'db_name', 'bbcustom_dir',
- 'bbcustom_branch', 'bbconfigs_branch', 'tools_dir',
- 'tools_branch', 'datacentre', 'buildbot_bin', 'buildbot_branch',
- 'buildbot_python', 'buildbot_setup', 'environment')
+ 'basedir', 'bbconfigs_dir', 'db_name', 'bbcustom_dir',
+ 'bbcustom_branch', 'bbconfigs_branch', 'tools_dir',
+ 'tools_branch', 'datacentre', 'buildbot_bin', 'buildbot_branch',
+ 'buildbot_python', 'buildbot_setup', 'environment')
opt_keys = ('http_port', 'ssh_port', 'pb_port', 'buildbot_version',
'limit_fx_platforms', 'limit_tb_platforms', 'limit_b2g_platforms',
'release_branches', 'thunderbird_release_branches', 'mobile_release_branches')
@@ -95,20 +98,22 @@ def check_master(master):
return False
ports = {
- "ssh": 7000 + role_offset + instance_num,
- "http": 8000 + role_offset + instance_num,
- "pb": 9000 + role_offset + instance_num,
- }
+ "ssh": 7000 + role_offset + instance_num,
+ "http": 8000 + role_offset + instance_num,
+ "pb": 9000 + role_offset + instance_num,
+ }
for proto in required_ports:
master_port = master.get("%s_port" % proto)
if master_port != ports[proto]:
- log.error("%s - bad %s port (got %i, expected %i)", name, proto, master_port, ports[proto])
+ log.error("%s - bad %s port (got %i, expected %i)",
+ name, proto, master_port, ports[proto])
return False
# Check master_dir
if master['master_dir'] != "/builds/buildbot/%s/master" % instance:
# TODO this needs tweaking for tests
- log.error("%s - bad master_dir %s" % (name, master.get('master_dir', 'None')))
+ log.error("%s - bad master_dir %s" % (name, master.get(
+ 'master_dir', 'None')))
return False
# Check basedir
View
25 buildfarm/maintenance/count_and_reboot.py
@@ -6,7 +6,9 @@
Increments the value in countfile, and reboots the machine once the count
reaches or exceeds maxcount."""
-import os, sys, time
+import os
+import sys
+import time
if sys.platform in ('darwin', 'linux2'):
def reboot():
@@ -15,17 +17,18 @@ def reboot():
# This results in sudo not waiting forever for a password. If sudoers
# isn't set up properly, this will fail immediately
os.system("sudo -S reboot < /dev/null")
- # After starting the shutdown, we go to sleep since the system can
- # take a few minutes to shut everything down and reboot
- time.sleep(600)
+ # After starting the shutdown, we go to sleep since the system can
+ # take a few minutes to shut everything down and reboot
+ time.sleep(600)
elif sys.platform == "win32":
# Windows
def reboot():
os.system("shutdown -f -r -t 0")
- # After starting the shutdown, we go to sleep since the system can
- # take a few minutes to shut everything down and reboot
- time.sleep(600)
+ # After starting the shutdown, we go to sleep since the system can
+ # take a few minutes to shut everything down and reboot
+ time.sleep(600)
+
def increment_count(fname):
try:
@@ -41,14 +44,14 @@ def increment_count(fname):
parser = OptionParser(__doc__)
parser.add_option("-n", "--max-count", dest="maxcount", default=10,
- help="reboot after <maxcount> runs", type="int")
+ help="reboot after <maxcount> runs", type="int")
parser.add_option("-f", "--count-file", dest="countfile", default=None,
- help="file to record count in")
+ help="file to record count in")
parser.add_option("-z", "--zero-count", dest="zero", default=False,
- action="store_true", help="zero out the counter before rebooting")
+ action="store_true", help="zero out the counter before rebooting")
options, args = parser.parse_args()
-
+
if not options.countfile:
parser.error("countfile is required")
View
142 buildfarm/maintenance/foopy_fabric.py
@@ -1,68 +1,74 @@
-from fabric.api import run
-from fabric.context_managers import cd, hide, show
-from fabric.context_managers import settings
-from fabric.colors import green, yellow, red
-
-OK = green('[OK] ')
-FAIL = red('[FAIL]')
-INFO = yellow('[INFO]')
-
-def per_host(fn):
- fn.per_host = True
- return fn
-
-def per_device(fn):
- fn.per_device = True
- return fn
-
-@per_host
-def show_revision(foopy):
- with hide('stdout', 'stderr', 'running'):
- tools_rev = run('hg -R /builds/tools ident -i')
-
- print "%-14s %12s" % (foopy, tools_rev)
-
-@per_host
-def update(foopy):
- with show('running'):
- with cd('/builds/tools'):
- run('hg pull && hg update -C')
- with hide('stdout', 'stderr', 'running'):
- tools_rev = run('hg ident -i')
-
- print OK, "updated %s tools to %12s" % (foopy, tools_rev)
-
-@per_device
-def stop_cp(device):
- with show('running'):
- with cd('/builds'):
- run('./stop_cp.sh %s' % device)
- print OK, "Stopped clientproxy for %s" % (device)
-
-@per_device
-def what_master(device):
- import re
- with hide('stdout', 'stderr', 'running'):
- with cd('/builds'):
- tac_path = './%s/buildbot.tac' % device
- tac_exists = False
- with settings(hide('everything'), warn_only=True):
- tac_exists = not run('test -e "$(echo %s)"' % tac_path).failed
- if tac_exists:
- output = run('cat %s | grep "^buildmaster_host"' % tac_path)
- m = re.search('^buildmaster_host\s*=\s*([\'"])(.*)[\'"]', output, re.M)
- if not m:
- print FAIL, "Failed to parse buildbot.tac:", repr(output)
- master = "No Master"
- else:
- master = m.group(2)
- else:
- master = "No Master"
- print OK, "%s uses %s" % (device, master)
-
-actions = [
- 'what_master',
- 'show_revision',
- 'update',
- 'stop_cp',
- ]
+from fabric.api import run
+from fabric.context_managers import cd, hide, show
+from fabric.context_managers import settings
+from fabric.colors import green, yellow, red
+
+OK = green('[OK] ')
+FAIL = red('[FAIL]')
+INFO = yellow('[INFO]')
+
+
+def per_host(fn):
+ fn.per_host = True
+ return fn
+
+
+def per_device(fn):
+ fn.per_device = True
+ return fn
+
+
+@per_host
+def show_revision(foopy):
+ with hide('stdout', 'stderr', 'running'):
+ tools_rev = run('hg -R /builds/tools ident -i')
+
+ print "%-14s %12s" % (foopy, tools_rev)
+
+
+@per_host
+def update(foopy):
+ with show('running'):
+ with cd('/builds/tools'):
+ run('hg pull && hg update -C')
+ with hide('stdout', 'stderr', 'running'):
+ tools_rev = run('hg ident -i')
+
+ print OK, "updated %s tools to %12s" % (foopy, tools_rev)
+
+
+@per_device
+def stop_cp(device):
+ with show('running'):
+ with cd('/builds'):
+ run('./stop_cp.sh %s' % device)
+ print OK, "Stopped clientproxy for %s" % (device)
+
+
+@per_device
+def what_master(device):
+ import re
+ with hide('stdout', 'stderr', 'running'):
+ with cd('/builds'):
+ tac_path = './%s/buildbot.tac' % device
+ tac_exists = False
+ with settings(hide('everything'), warn_only=True):
+ tac_exists = not run('test -e "$(echo %s)"' % tac_path).failed
+ if tac_exists:
+ output = run('cat %s | grep "^buildmaster_host"' % tac_path)
+ m = re.search('^buildmaster_host\s*=\s*([\'"])(.*)[\'"]', output, re.M)
+ if not m:
+ print FAIL, "Failed to parse buildbot.tac:", repr(output)
+ master = "No Master"
+ else:
+ master = m.group(2)
+ else:
+ master = "No Master"
+ print OK, "%s uses %s" % (device, master)
+
+actions = [
+ 'what_master',
+ 'show_revision',
+ 'update',
+ 'stop_cp',
+]
View
39 buildfarm/maintenance/manage_foopies.py
@@ -9,23 +9,26 @@
_devices_map = dict()
_foopy_map = dict()
+
+
def _calc_maps(devices_file):
global _devices_map, _foopy_map
all_devices = json.load(urllib.urlopen(devices_file))
_devices_map = dict([[x, all_devices[x]] for x in all_devices
- if all_devices[x].has_key('foopy')
- and all_devices[x]['foopy'] is not None])
+ if 'foopy' in all_devices[x]
+ and all_devices[x]['foopy'] is not None])
# Extract foopies from devices
all_foopies = [all_devices[x]['foopy'] for x in all_devices
- if all_devices[x].has_key('foopy')]
+ if 'foopy' in all_devices[x]]
# Use set to trim dupes
all_foopies = set(all_foopies) - set(["None"])
# For sanity when using -H all, we revert to list and sort
all_foopies = [x for x in all_foopies]
all_foopies.sort()
for foopy in all_foopies:
_foopy_map[foopy] = dict([[x, _devices_map[x]] for x in _devices_map
- if _devices_map[x]['foopy'] == foopy])
+ if _devices_map[x]['foopy'] == foopy])
+
def _calc_selected(hosts, devices):
global _foopy_map, _devices_map
@@ -36,22 +39,23 @@ def _calc_selected(hosts, devices):
return selected
for f in hosts:
- if _foopy_map.has_key(f):
+ if f in _foopy_map:
selected[f] = deepcopy(_foopy_map[f])
for d in devices:
- if not _devices_map.has_key(d):
+ if d not in _devices_map:
continue
- if not _devices_map[d].has_key('foopy'):
+ if 'foopy' not in _devices_map[d]:
continue
- if not _foopy_map.has_key(_devices_map[d]['foopy']):
+ if _devices_map[d]['foopy'] not in _foopy_map:
continue
- if selected.has_key(_devices_map[d]['foopy']):
+ if _devices_map[d]['foopy'] in selected:
selected[_devices_map[d]['foopy']][d] = deepcopy(d)
- else: # No mapping for this foopy
+ else: # No mapping for this foopy
selected[_devices_map[d]['foopy']] = dict()
selected[_devices_map[d]['foopy']][d] = deepcopy(d)
return selected
+
def run_action_on_foopy(action, foopy):
atfork()
try:
@@ -68,6 +72,7 @@ def run_action_on_foopy(action, foopy):
print traceback.format_exc()
return False
+
def run_action_on_devices(action, foopy_dict):
atfork()
action_func = getattr(foopy_fabric, action)
@@ -94,14 +99,14 @@ def run_action_on_devices(action, foopy_dict):
import simplejson as json
except:
import json
-
+
parser = OptionParser("""%%prog [options] action [action ...]""")
-
+
parser.set_defaults(
hosts=[],
devices=[],
concurrency=1,
- )
+ )
parser.add_option("-f", "--devices-file", dest="devices_file", help="list/url of devices.json")
parser.add_option("-H", "--host", dest="hosts", action="append")
parser.add_option("-D", "--device", dest="devices", action="append")
@@ -124,9 +129,9 @@ def run_action_on_devices(action, foopy_dict):
if len(selected.keys()) == 0:
parser.error("You need to specify a foopy via -H or a device via -D")
-
+
env.user = 'cltbld'
-
+
for action in actions:
fn = run_action_on_foopy
genArg = lambda x: x
@@ -141,8 +146,8 @@ def run_action_on_devices(action, foopy_dict):
p = multiprocessing.Pool(processes=options.concurrency)
results = []
for foopy in sorted(selected.keys()):
- result = p.apply_async(fn, (action, genArg(foopy)) )
- results.append( (foopy, result) )
+ result = p.apply_async(fn, (action, genArg(foopy)))
+ results.append((foopy, result))
p.close()
failed = False
for foopy, result in results:
View
25 buildfarm/maintenance/manage_masters.py
@@ -7,10 +7,12 @@
from fabric.context_managers import settings
from Crypto.Random import atfork
-site.addsitedir(path.join(path.dirname(path.realpath(__file__)), "../../lib/python"))
+site.addsitedir(
+ path.join(path.dirname(path.realpath(__file__)), "../../lib/python"))
import util.fabric.actions
+
def print_status(remaining, failed_masters):
if remaining:
print "=" * 30, "Remaining masters", "=" * 30
@@ -55,12 +57,12 @@ def run_action_on_master(action, master):
%s""" % textwrap.fill(", ".join(util.fabric.actions.get_actions())))
parser.set_defaults(
- hosts=[],
- roles=[],
- concurrency=1,
- show_list=False,
- all_masters=False,
- )
+ hosts=[],
+ roles=[],
+ concurrency=1,
+ show_list=False,
+ all_masters=False,
+ )
parser.add_option("-f", "--master-file", dest="master_file",
help="list/url of masters")
parser.add_option("-H", "--host", dest="hosts", action="append")
@@ -116,7 +118,7 @@ def run_action_on_master(action, master):
fmt = "%(role)-9s %(name)-14s %(hostname)s:%(basedir)s"
print fmt % dict(role='role', name='name', hostname='hostname',
- basedir='basedir')
+ basedir='basedir')
for m in masters:
print fmt % m
sys.exit(0)
@@ -151,7 +153,7 @@ def run_action_on_master(action, master):
while True:
for master, result in list(results):
if result.ready():
- results.remove((master,result))
+ results.remove((master, result))
if not result.get():
failed_masters.append(master['name'])
print master['name'], "FAILED"
@@ -165,8 +167,9 @@ def run_action_on_master(action, master):
time.sleep(1)
p.join()
- # One final print before we exit, to be sure that results are not missed
+ # One final print before we exit, to be sure that results are not
+ # missed
print_status([m['name'] for (m, r) in results],
- failed_masters)
+ failed_masters)
if failed:
sys.exit(1)
View
8 buildfarm/maintenance/master_cleanup.py
@@ -3,6 +3,7 @@
import os
from cPickle import load
+
def maybe_delete(filename, timestamp):
"""Delete filename if it's older than timestamp"""
try:
@@ -13,6 +14,7 @@ def maybe_delete(filename, timestamp):
# We'll get it next time!
pass
+
def clean_dir(dirname, timestamp):
"""Delete old twisted log files, and old builder files from dirname"""
# Clean up files older than timestamp
@@ -52,13 +54,13 @@ def clean_dir(dirname, timestamp):
from optparse import OptionParser
parser = OptionParser()
parser.add_option("-t", "--time", dest="time", type="int",
- help="time, in days, for how old files have to be before being deleted",
- default=4)
+ help="time, in days, for how old files have to be before being deleted",
+ default=4)
options, args = parser.parse_args()
if len(args) == 0:
parser.error("Must specify at least one directory to clean up")
for d in args:
- clean_dir(d, time.time() - options.time * 24*3600)
+ clean_dir(d, time.time() - options.time * 24 * 3600)
View
6 buildfarm/maintenance/normalize-masters.py
@@ -1,10 +1,12 @@
#!/usr/bin/python
-import simplejson, sys, os
+import simplejson
+import sys
+import os
fn = sys.argv[1]
data = simplejson.load(open(sys.argv[1]))
-data.sort(key=lambda master:master['name'])
+data.sort(key=lambda master: master['name'])
os.rename(fn, '%s.bak' % fn)
View
48 buildfarm/maintenance/print_waits.py
@@ -1,4 +1,9 @@
-import cPickle, os, re, math, time
+import cPickle
+import os
+import re
+import math
+import time
+
def format_hist(h, units=1):
retval = []
@@ -9,17 +14,18 @@ def format_hist(h, units=1):
min_key = min(keys)
max_key = max(keys)
- for i in range(min_key, max_key+1):
+ for i in range(min_key, max_key + 1):
n = h.get(i, 0)
if total > 0:
- percentage = " %8.2f%%" % (n*100./total)
+ percentage = " %8.2f%%" % (n * 100. / total)
else:
percentage = ''
retval.append("%3i: %8i%s"
- % (i*units, n, percentage))
+ % (i * units, n, percentage))
return "\n".join(retval)
+
def scan_builder(builder, starttime, endtime, minutes_per_block, times, change_as_submittime=True):
"""Scans the build pickle files in the builder directory, and updates the dictionary `times`."""
if not os.path.exists(builder):
@@ -43,21 +49,22 @@ def scan_builder(builder, starttime, endtime, minutes_per_block, times, change_a
submittime = b.requests[0].submittedAt
if starttime < submittime < endtime:
- w = int(math.floor((b.started - submittime)/(minutes_per_block*60.0)))
+ w = int(math.floor(
+ (b.started - submittime) / (minutes_per_block * 60.0)))
times[w] = times.get(w, 0) + 1
if __name__ == "__main__":
from optparse import OptionParser
parser = OptionParser()
parser.set_defaults(
- minutes_per_block=15,
- change_as_submittime=True,
- name=os.uname()[1],
- builders={},
- directory=None,
- starttime=time.time()-24*3600,
- endtime=time.time(),
- )
+ minutes_per_block=15,
+ change_as_submittime=True,
+ name=os.uname()[1],
+ builders={},
+ directory=None,
+ starttime=time.time() - 24 * 3600,
+ endtime=time.time(),
+ )
def add_builder(option, opt_str, value, parser, *args, **kwargs):
if ":" in value:
@@ -73,7 +80,8 @@ def add_builder(option, opt_str, value, parser, *args, **kwargs):
parser.values.builders[platform].extend(builders)
parser.add_option("-m", "--minutes-per-block", type="int", help="How many minutes per block", dest="minutes_per_block")
- parser.add_option("-r", "--request-as-submittime", action="store_false", dest="change_as_submittime")
+ parser.add_option("-r", "--request-as-submittime",
+ action="store_false", dest="change_as_submittime")
parser.add_option("-n", "--name", dest="name")
parser.add_option("-b", "--builders", dest="builders", action="callback", nargs=1, type="string", callback=add_builder, help="platform:builder1,builder2,...")
parser.add_option("-d", "--directory", dest="directory")
@@ -91,21 +99,21 @@ def add_builder(option, opt_str, value, parser, *args, **kwargs):
if options.directory:
os.chdir(options.directory)
- text =[]
- text.append("Wait time report for %s for jobs submitted since %s\n"
+ text = []
+ text.append("Wait time report for %s for jobs submitted since %s\n"
% (options.name, time.ctime(options.starttime)))
hist = {}
for platform, builders in options.builders.items():
hist[platform] = {}
for builder in builders:
scan_builder(builder, options.starttime, options.endtime,
- options.minutes_per_block, hist[platform],
+ options.minutes_per_block, hist[platform],
options.change_as_submittime)
allhist = {}
for i in set([x for y in hist.keys() for x in hist[y]]):
- allhist[i] = sum([p.get(i,0) for p in hist.values()])
+ allhist[i] = sum([p.get(i, 0) for p in hist.values()])
total = sum(allhist.values())
text.append("Total Jobs: %i\n" % total)
text.append("Wait Times")
@@ -123,10 +131,10 @@ def add_builder(option, opt_str, value, parser, *args, **kwargs):
else:
import smtplib
import sys
-
+
zero_wait = 0
if total > 0:
- zero_wait = allhist.get(0,0)*100./total
+ zero_wait = allhist.get(0, 0) * 100. / total
subject = "Wait: %i/%.2f%% (%s)" % (total, zero_wait, options.name)
server = options.smtp or 'localhost'
View
43 buildfarm/maintenance/purge_builds.py
@@ -19,16 +19,19 @@
python %prog -s 6 /builds/moz2_slave /scratchbox/users/cltbld/home/cltbld/build
"""
-import os, shutil, sys
+import os
+import shutil
+import sys
from fnmatch import fnmatch
-DEFAULT_BASE_DIRS=["..", "/scratchbox/users/cltbld/home/cltbld/build"]
+DEFAULT_BASE_DIRS = ["..", "/scratchbox/users/cltbld/home/cltbld/build"]
-clobber_suffix='.deleteme'
+clobber_suffix = '.deleteme'
if sys.platform == 'win32':
# os.statvfs doesn't work on Windows
import win32file
+
def freespace(p):
secsPerClus, bytesPerSec, nFreeClus, totClus = win32file.GetDiskFreeSpace(p)
return secsPerClus * bytesPerSec * nFreeClus
@@ -38,10 +41,12 @@ def freespace(p):
r = os.statvfs(p)
return r.f_frsize * r.f_bavail
+
def mtime_sort(p1, p2):
"sorting function for sorting a list of paths by mtime"
return cmp(os.path.getmtime(p1), os.path.getmtime(p2))
+
def rmdirRecursive(dir):
"""This is a replacement for shutil.rmtree that works better under
windows. Thanks to Bear at the OSAF for the code.
@@ -79,6 +84,7 @@ def rmdirRecursive(dir):
os.remove(full_name)
os.rmdir(dir)
+