Permalink
Browse files

merge commit

  • Loading branch information...
2 parents 1bb4b77 + 3eb0b00 commit 00d6ce0025e4dd07c223b59a6123f73904c51a01 @gozer-mozillamessaging gozer-mozillamessaging committed Jan 11, 2010
View
@@ -32,3 +32,6 @@ aee8bf61808eca5b416bfbf905d6ec07f3b29d74 FIREFOX_3_6b4_RELEASE
c9999d50660f3b07003bda5c3cd621232830e6f4 FIREFOX_3_5_6_RELEASE
4ba469cb1bffcf55d35452a3a4fcc2fa7608d067 FIREFOX_3_0_16_RELEASE
90434a7d07c5d37e0bedc131224af079d62b9359 FIREFOX_3_6b5_RELEASE
+5e2d9cb03e90ee9b541ed5f642841ed367394af7 FIREFOX_3_5_7_RELEASE
+7451ab038fe72a2629afa323c0b3d6e5b74790a2 FIREFOX_3_0_17_RELEASE
+2075516e371ac6aabc6ae41e4dad85b52035532b FIREFOX_3_6rc1_RELEASE
@@ -1,6 +1,7 @@
#!/usr/bin/python
import re
+import os
DEFAULT_HEADER = """\
from twisted.application import service
@@ -37,7 +38,8 @@ def get_default_options(slavename):
d = {'slavename': quote_option(slavename)}
basedir = None
buildmaster_host = None
- if 'moz2' in slavename or 'xserve' in slavename or 'try-' in slavename:
+ if 'moz2' in slavename or 'xserve' in slavename or 'try-' in slavename or \
+ slavename.startswith('win32'):
if 'try-' in slavename:
buildmaster_host = TRY_BUILDMASTER
d['port'] = 9982
@@ -46,7 +48,7 @@ def get_default_options(slavename):
if 'linux' in slavename or 'darwin9' in slavename or \
'xserve' in slavename or 'mac' in slavename:
basedir = '/builds/slave'
- elif 'win32' in slavename:
+ elif 'win32' in slavename or 'w32' in slavename:
basedir = 'e:\\builds\\moz2_slave'
elif 'talos' in slavename or '-try' in slavename:
buildmaster_host = TALOS_BUILDMASTER
@@ -100,6 +102,7 @@ def __init__(self, tacOptions, header=DEFAULT_HEADER, footer=DEFAULT_FOOTER,
self.filename = filename
def save(self):
+ tmpfile = '%s.tmp' % self.filename
# Look for necessary, but missing options
missingOptions = []
for o in self.requiredOptions:
@@ -109,11 +112,13 @@ def save(self):
raise MissingOptionsError("Missing %s, cannot save %s" % \
(missingOptions, self.filename))
# If there wasn't any, save the file
- f = open(self.filename, "w")
+ f = open(tmpfile, "w")
f.write(self.header)
for key,value in self.tacOptions.iteritems():
f.write("%s = %s\n" % (key, value))
f.write(self.footer)
+ f.close()
+ os.rename(tmpfile, self.filename)
@@ -0,0 +1,64 @@
+#!/usr/bin/python
+"""Script to clean up buildbot master directories"""
+import os
+from cPickle import load
+
+def maybe_delete(filename, timestamp):
+ """Delete filename if it's older than timestamp"""
+ try:
+ if os.path.getmtime(filename) < timestamp:
+ os.unlink(filename)
+ except OSError:
+ # Ignore this error. The file may have already been moved.
+ # We'll get it next time!
+ pass
+
+def clean_dir(dirname, timestamp):
+ """Delete old twisted log files, and old builder files from dirname"""
+ # Clean up files older than timestamp
+ files = os.listdir(dirname)
+ # Look for twistd.log files
+ for f in files:
+ p = os.path.join(dirname, f)
+ if f.startswith("twistd.log"):
+ maybe_delete(p, timestamp)
+
+ elif os.path.isdir(p):
+ builder_file = os.path.join(p, "builder")
+ # Don't clean out non-builder directories
+ if not os.path.exists(builder_file):
+ continue
+
+ try:
+ builder = load(open(builder_file))
+ except:
+ continue
+
+ # Don't clean out release builders
+ if builder and builder.category and \
+ builder.category.endswith('release'):
+ continue
+
+ for build in os.listdir(p):
+ # Don't delete the 'builder' file
+ if build == "builder":
+ continue
+ build = os.path.join(p, build)
+ if os.path.isfile(build):
+ maybe_delete(build, timestamp)
+
+if __name__ == "__main__":
+ import time
+ from optparse import OptionParser
+ parser = OptionParser()
+ parser.add_option("-t", "--time", dest="time", type="int",
+ help="time, in days, for how old files have to be before being deleted",
+ default=4)
+
+ options, args = parser.parse_args()
+
+ if len(args) == 0:
+ parser.error("Must specify at least one directory to clean up")
+
+ for d in args:
+ clean_dir(d, time.time() - options.time * 24*3600)
@@ -0,0 +1,27 @@
+#!/usr/bin/python
+"""Run in a buildbot master directory once the master is shut down to
+purge old events from the builder files.
+
+NB: The master must be shut down for this to work!"""
+import cPickle, os, shutil
+
+for f in os.listdir("."):
+ builder_file = os.path.join(f, "builder")
+ if os.path.isdir(f) and os.path.exists(builder_file):
+ builder = cPickle.load(open(builder_file))
+ if builder.category == 'release':
+ print "Skipping", builder_file
+ continue
+ print "Backing up", builder_file
+ shutil.copyfile(builder_file, builder_file + ".bak")
+
+ # Set some dummy attributes that get deleted by __getstate__
+ builder.currentBigState = None
+ builder.basedir = None
+ builder.status = None
+ builder.nextBuildNumber = None
+
+ # Truncate to 500 events
+ builder.events = builder.events[-500:]
+ print "Writing", builder_file
+ cPickle.dump(builder, open(builder_file, "w"))
View
@@ -1,19 +1,18 @@
-import sys, shutil, urllib2, urllib, os
-from datetime import datetime, timedelta
+#!/usr/bin/python
+# vim:sts=2 sw=2
+import sys, shutil, urllib2, urllib, os, traceback, time
clobber_suffix='.deleteme'
-def str_to_datetime(s):
- return datetime.strptime(s, "%Y-%m-%d %H:%M:%S")
-
-def datetime_to_str(dt):
- return dt.strftime("%Y-%m-%d %H:%M:%S")
+def ts_to_str(ts):
+ if ts is None:
+ return None
+ return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(ts))
-def write_file(dt, fn):
- assert isinstance(dt, datetime)
- dt = datetime_to_str(dt)
+def write_file(ts, fn):
+ assert isinstance(ts, int)
f = open(fn, "w")
- f.write(dt)
+ f.write(str(ts))
f.close()
def read_file(fn):
@@ -22,7 +21,7 @@ def read_file(fn):
data = open(fn).read().strip()
try:
- return str_to_datetime(data)
+ return int(data)
except ValueError:
return None
@@ -89,88 +88,132 @@ def do_clobber(dir, dryrun=False, skip=None):
# Prevent repeated moving.
if f.endswith(clobber_suffix):
rmdirRecursive(f)
- else:
+ else:
shutil.move(f, clobber_path)
rmdirRecursive(clobber_path)
except:
print "Couldn't clobber properly, bailing out."
sys.exit(1)
-def getClobberDate(baseURL, branch, builder, slave):
- url = "%s?%s" % (baseURL,
- urllib.urlencode(dict(branch=branch, builder=builder, slave=slave)))
+def getClobberDates(clobberURL, branch, buildername, builddir, slave, master):
+ params = dict(branch=branch, buildername=buildername, builddir=builddir, slave=slave, master=master)
+ url = "%s?%s" % (clobberURL, urllib.urlencode(params))
print "Checking clobber URL: %s" % url
data = urllib2.urlopen(url).read().strip()
+
+ retval = {}
try:
- return str_to_datetime(data)
+ for line in data.split("\n"):
+ line = line.strip()
+ if not line:
+ continue
+ builddir, builder_time, who = line.split(":")
+ builder_time = int(builder_time)
+ retval[builddir] = (builder_time, who)
+ return retval
except ValueError:
- return None
+ print "Error parsing response from server"
+ print data
+ raise
if __name__ == "__main__":
from optparse import OptionParser
- parser = OptionParser()
+ parser = OptionParser("%prog [options] clobberURL branch buildername builddir slave master")
parser.add_option("-n", "--dry-run", dest="dryrun", action="store_true",
default=False, help="don't actually delete anything")
parser.add_option("-t", "--periodic", dest="period", type="float",
- default=24*7, help="hours between periodic clobbers")
- parser.add_option('-s', '--skip', help='do not delete this directory',
+ default=None, help="hours between periodic clobbers")
+ parser.add_option('-s', '--skip', help='do not delete this file/directory',
action='append', dest='skip', default=['last-clobber'])
parser.add_option('-d', '--dir', help='clobber this directory',
dest='dir', default='.', type='string')
+ parser.add_option('-v', '--verbose', help='be more verbose',
+ dest='verbose', action='store_true', default=False)
options, args = parser.parse_args()
- periodicClobberTime = timedelta(hours = options.period)
+ if len(args) != 6:
+ parser.error("Incorrect number of arguments")
- baseURL, branch, builder, slave = args
+ if options.period:
+ periodicClobberTime = options.period * 3600
+ else:
+ periodicClobberTime = None
+
+ clobberURL, branch, builder, my_builddir, slave, master = args
try:
- server_clobber_date = getClobberDate(baseURL, branch, builder, slave)
+ server_clobber_dates = getClobberDates(clobberURL, branch, builder, my_builddir, slave, master)
except:
+ if options.verbose:
+ traceback.print_exc()
print "Error contacting server"
sys.exit(1)
- our_clobber_date = read_file("last-clobber")
-
- clobber = False
-
- print "Our last clobber date: ", our_clobber_date
- print "Server clobber date: ", server_clobber_date
-
- # If we don't have a last clobber date, then this is probably a fresh build.
- # We should only do a forced server clobber if we know when our last clobber
- # was, and if the server date is more recent than that.
- if server_clobber_date is not None and our_clobber_date is not None:
- # If the server is giving us a clobber date, compare the server's idea of
- # the clobber date to our last clobber date
- if server_clobber_date > our_clobber_date:
- # If the server's clobber date is greater than our last clobber date,
- # then we should clobber.
- clobber = True
- # We should also update our clobber date to match the server's
- our_clobber_date = server_clobber_date
- print "Server is forcing a clobber"
-
- if not clobber:
- # Next, check if more than the periodicClobberTime period has passed since
- # our last clobber
- if our_clobber_date is None:
- # We've never been clobbered
- # Set our last clobber time to now, so that we'll clobber
- # properly after periodicClobberTime
- our_clobber_date = datetime.utcnow()
+ if options.verbose:
+ print "Server gave us", server_clobber_dates
+
+ now = int(time.time())
+
+ # Add ourself to the server_clobber_dates if it's not set
+ # This happens when this slave has never been clobbered
+ if my_builddir not in server_clobber_dates:
+ server_clobber_dates[my_builddir] = None, ""
+
+ root_dir = os.path.abspath(options.dir)
+
+ for builddir, (server_clobber_date, who) in server_clobber_dates.items():
+ builder_dir = os.path.join(root_dir, builddir)
+ if not os.path.isdir(builder_dir):
+ print "%s doesn't exist, skipping" % builder_dir
+ continue
+ os.chdir(builder_dir)
+
+ our_clobber_date = read_file("last-clobber")
+
+ clobber = False
+
+ print "%s:Our last clobber date: " % builddir, ts_to_str(our_clobber_date)
+ print "%s:Server clobber date: " % builddir, ts_to_str(server_clobber_date)
+
+ # If we don't have a last clobber date, then this is probably a fresh build.
+ # We should only do a forced server clobber if we know when our last clobber
+ # was, and if the server date is more recent than that.
+ if server_clobber_date is not None and our_clobber_date is not None:
+ # If the server is giving us a clobber date, compare the server's idea of
+ # the clobber date to our last clobber date
+ if server_clobber_date > our_clobber_date:
+ # If the server's clobber date is greater than our last clobber date,
+ # then we should clobber.
+ clobber = True
+ # We should also update our clobber date to match the server's
+ our_clobber_date = server_clobber_date
+ if who:
+ print "%s:Server is forcing a clobber, initiated by %s" % (builddir, who)
+ else:
+ print "%s:Server is forcing a clobber" % builddir
+
+ if not clobber:
+ # Disable periodic clobbers for builders that aren't my_builddir
+ if builddir != my_builddir:
+ continue
+
+ # Next, check if more than the periodicClobberTime period has passed since
+ # our last clobber
+ if our_clobber_date is None:
+ # We've never been clobbered
+ # Set our last clobber time to now, so that we'll clobber
+ # properly after periodicClobberTime
+ our_clobber_date = now
+ write_file(our_clobber_date, "last-clobber")
+ elif periodicClobberTime and now > our_clobber_date + periodicClobberTime:
+ # periodicClobberTime has passed since our last clobber
+ clobber = True
+ # Update our clobber date to now
+ our_clobber_date = now
+ print "%s:More than %s seconds have passed since our last clobber" % (builddir, periodicClobberTime)
+
+ if clobber:
+ # Finally, perform a clobber if we're supposed to
+ print "%s:Clobbering..." % builddir
+ do_clobber(builder_dir, options.dryrun, options.skip)
write_file(our_clobber_date, "last-clobber")
- elif datetime.utcnow() > our_clobber_date + periodicClobberTime:
- # periodicClobberTime has passed since our last clobber
- clobber = True
- # Update our clobber date to now
- our_clobber_date = datetime.utcnow()
- print "More than %s have passed since our last clobber" % periodicClobberTime
-
- if clobber:
- # Finally, perform a clobber if we're supposed to
- if os.path.exists(options.dir):
- print "Clobbering..."
- do_clobber(options.dir, options.dryrun, options.skip)
- else:
- print "Clobber failed because '%s' doesn't exist" % options.dir
- write_file(our_clobber_date, "last-clobber")
Oops, something went wrong.

0 comments on commit 00d6ce0

Please sign in to comment.