Permalink
Switch branches/tags
THUNDERBIRD_31_0b2_RELEASE THUNDERBIRD_31_0b2_BUILD1 THUNDERBIRD_31_0b1_RELEASE THUNDERBIRD_31_0b1_BUILD1 THUNDERBIRD_30_0b1_RELEASE THUNDERBIRD_30_0b1_BUILD2 THUNDERBIRD_30_0b1_BUILD1 THUNDERBIRD_29_0b1_RELEASE THUNDERBIRD_29_0b1_BUILD4 THUNDERBIRD_29_0b1_BUILD3 THUNDERBIRD_29_0b1_BUILD1 THUNDERBIRD_28_0b1_RELEASE THUNDERBIRD_28_0b1_BUILD1 THUNDERBIRD_27_0b1_RELEASE THUNDERBIRD_27_0b1_BUILD2 THUNDERBIRD_27_0b1_BUILD1 THUNDERBIRD_26_0b1_RELEASE THUNDERBIRD_26_0b1_BUILD1 THUNDERBIRD_25_0b1_RELEASE THUNDERBIRD_25_0b1_BUILD1 THUNDERBIRD_24_6_0_RELEASE THUNDERBIRD_24_6_0_BUILD3 THUNDERBIRD_24_6_0_BUILD2 THUNDERBIRD_24_6_0_BUILD1 THUNDERBIRD_24_5_0_RELEASE THUNDERBIRD_24_5_0_BUILD1 THUNDERBIRD_24_4_0_RELEASE THUNDERBIRD_24_4_0_BUILD1 THUNDERBIRD_24_3_0_RELEASE THUNDERBIRD_24_3_0_BUILD2 THUNDERBIRD_24_3_0_BUILD1 THUNDERBIRD_24_2_0_RELEASE THUNDERBIRD_24_2_0_BUILD1 THUNDERBIRD_24_1_1_RELEASE THUNDERBIRD_24_1_1_BUILD1 THUNDERBIRD_24_1_0_RELEASE THUNDERBIRD_24_1_0_BUILD1 THUNDERBIRD_24_0_1_RELEASE THUNDERBIRD_24_0_1_BUILD1 THUNDERBIRD_24_0b3_RELEASE THUNDERBIRD_24_0b3_BUILD1 THUNDERBIRD_24_0b2_RELEASE THUNDERBIRD_24_0b2_BUILD1 THUNDERBIRD_24_0b1_RELEASE THUNDERBIRD_24_0b1_BUILD4 THUNDERBIRD_24_0b1_BUILD3 THUNDERBIRD_24_0b1_BUILD2 THUNDERBIRD_24_0b1_BUILD1 THUNDERBIRD_24_0_RELEASE THUNDERBIRD_24_0_BUILD1 THUNDERBIRD_23_0b1_RELEASE THUNDERBIRD_23_0b1_BUILD1 THUNDERBIRD_22_0b1_RELEASE THUNDERBIRD_22_0b1_BUILD2 THUNDERBIRD_22_0b1_BUILD1 THUNDERBIRD_21_0b1_RELEASE THUNDERBIRD_21_0b1_BUILD1 THUNDERBIRD_20_0b1_RELEASE THUNDERBIRD_20_0b1_BUILD2 THUNDERBIRD_20_0b1_BUILD1 THUNDERBIRD_19_0b1_RELEASE THUNDERBIRD_19_0b1_BUILD1 THUNDERBIRD_18_0b1_RELEASE THUNDERBIRD_18_0b1_BUILD1 THUNDERBIRD_17_0_11esr_RELEASE THUNDERBIRD_17_0_11esr_BUILD1 THUNDERBIRD_17_0_10esr_RELEASE THUNDERBIRD_17_0_10esr_BUILD1 THUNDERBIRD_17_0_9esr_RELEASE THUNDERBIRD_17_0_9esr_BUILD1 THUNDERBIRD_17_0_8esr_RELEASE THUNDERBIRD_17_0_8esr_BUILD1 THUNDERBIRD_17_0_8_RELEASE THUNDERBIRD_17_0_8_BUILD1 THUNDERBIRD_17_0_7esr_RELEASE THUNDERBIRD_17_0_7esr_BUILD1 THUNDERBIRD_17_0_7_RELEASE THUNDERBIRD_17_0_7_BUILD1 THUNDERBIRD_17_0_6esr_RELEASE THUNDERBIRD_17_0_6esr_BUILD1 THUNDERBIRD_17_0_6_RELEASE THUNDERBIRD_17_0_6_BUILD1 THUNDERBIRD_17_0_5esr_RELEASE THUNDERBIRD_17_0_5esr_BUILD1 THUNDERBIRD_17_0_5_RELEASE THUNDERBIRD_17_0_5_BUILD1 THUNDERBIRD_17_0_4esr_RELEASE THUNDERBIRD_17_0_4esr_BUILD1 THUNDERBIRD_17_0_4_RELEASE THUNDERBIRD_17_0_4_BUILD1 THUNDERBIRD_17_0_3esr_RELEASE THUNDERBIRD_17_0_3esr_BUILD1 THUNDERBIRD_17_0_3_RELEASE THUNDERBIRD_17_0_3_BUILD1 THUNDERBIRD_17_0_2esr_RELEASE THUNDERBIRD_17_0_2esr_BUILD2 THUNDERBIRD_17_0_2esr_BUILD1 THUNDERBIRD_17_0_2_RELEASE THUNDERBIRD_17_0_2_BUILD2 THUNDERBIRD_17_0_2_BUILD1
Nothing to show
Find file
Fetching contributors…
Cannot retrieve contributors at this time
executable file 304 lines (245 sloc) 10.5 KB
#!/usr/bin/env python
# Written for Mozilla by Chris AtLee <catlee@mozilla.com> 2008
"""Delete old buildbot builds to make room for the current build.
%prog [options] base_dir1 [base_dir2 ...]
base_dir1 is the root of the directory tree you want to delete builds
from.
Sub-directories of base_dir1 will be deleted, in order from oldest to newest,
until the specified amount of space is free.
base_dir1 will always be used for space calculations, but if other base_dir#
are provided, subdirectories within those dirs will also be purged. This will
obviously only increase the available space if the other base_dirs are on the
same mountpoint, but this can be useful for, e.g., cleaning up scratchbox.
example:
python %prog -s 6 /builds/moz2_slave /scratchbox/users/cltbld/home/cltbld/build
"""
import os
import shutil
import sys
from fnmatch import fnmatch
import re
DEFAULT_BASE_DIRS = [".."]
clobber_suffix = '.deleteme'
if sys.platform == 'win32':
# os.statvfs doesn't work on Windows
from win32file import RemoveDirectory, DeleteFile, \
GetFileAttributesW, SetFileAttributesW, GetDiskFreeSpace, \
FILE_ATTRIBUTE_NORMAL, FILE_ATTRIBUTE_DIRECTORY
from win32api import FindFiles
def freespace(p):
secsPerClus, bytesPerSec, nFreeClus, totClus = GetDiskFreeSpace(p)
return secsPerClus * bytesPerSec * nFreeClus
else:
def freespace(p):
"Returns the number of bytes free under directory `p`"
r = os.statvfs(p)
return r.f_frsize * r.f_bavail
def mtime_sort(p1, p2):
"sorting function for sorting a list of paths by mtime"
return cmp(os.path.getmtime(p1), os.path.getmtime(p2))
def rmdirRecursiveWindows(dir):
"""Windows-specific version of rmdirRecursive that handles
path lengths longer than MAX_PATH.
"""
dir = os.path.realpath(dir)
# Make sure directory is writable
SetFileAttributesW('\\\\?\\' + dir, FILE_ATTRIBUTE_NORMAL)
for ffrec in FindFiles('\\\\?\\' + dir + '\*.*'):
file_attr = ffrec[0]
name = ffrec[8]
if name == '.' or name == '..':
continue
full_name = os.path.join(dir, name)
if file_attr & FILE_ATTRIBUTE_DIRECTORY:
rmdirRecursiveWindows(full_name)
else:
SetFileAttributesW('\\\\?\\' + full_name, FILE_ATTRIBUTE_NORMAL)
DeleteFile('\\\\?\\' + full_name)
RemoveDirectory('\\\\?\\' + dir)
def rmdirRecursive(dir):
"""This is a replacement for shutil.rmtree that works better under
windows. Thanks to Bear at the OSAF for the code.
(Borrowed from buildbot.slave.commands)"""
if os.name == 'nt':
rmdirRecursiveWindows(dir)
return
if not os.path.exists(dir):
# This handles broken links
if os.path.islink(dir):
os.remove(dir)
return
if os.path.islink(dir):
os.remove(dir)
return
# Verify the directory is read/write/execute for the current user
os.chmod(dir, 0700)
for name in os.listdir(dir):
full_name = os.path.join(dir, name)
# on Windows, if we don't have write permission we can't remove
# the file/directory either, so turn that on
if os.name == 'nt':
if not os.access(full_name, os.W_OK):
# I think this is now redundant, but I don't have an NT
# machine to test on, so I'm going to leave it in place
# -warner
os.chmod(full_name, 0600)
if os.path.isdir(full_name):
rmdirRecursive(full_name)
else:
# Don't try to chmod links
if not os.path.islink(full_name):
os.chmod(full_name, 0700)
os.remove(full_name)
os.rmdir(dir)
def str2seconds(s):
""" Accepts time intervals resembling:
30d (30 days)
10h (10 hours)
Returns the specified interval as a positive integer in seconds.
"""
m = re.match(r'^(\d+)([dh])$', s)
if (m):
mul = {'d': 24*60*60, 'h': 60*60}
n = int(m.group(1))
unit = m.group(2)
return n * mul[unit]
else:
raise ValueError("Unhandled time format '%s'" % s)
def purge(base_dirs, gigs, ignore, max_age, dry_run=False):
"""Delete directories under `base_dirs` until `gigs` GB are free.
Delete any directories older than max_age.
Will not delete directories listed in the ignore list except
those tagged with an expiry threshold. Example:
rel-*:40d
Will not delete rel-* directories until they are over 40 days old.
"""
gigs *= 1024 * 1024 * 1024
# convert 'ignore' to a dict resembling { directory: cutoff_time }
# where a cutoff time of -1 means 'never expire'.
ignore = dict(map(lambda x: x.split(':')[0:2] if len(x.split(':')) > 1 else [x, -1], ignore))
ignore = dict(map(lambda key: [key, time.time() - str2seconds(ignore[key])] if ignore[key] != -1 else [key, ignore[key]], ignore))
dirs = []
for base_dir in base_dirs:
if os.path.exists(base_dir):
for d in os.listdir(base_dir):
p = os.path.join(base_dir, d)
if not os.path.isdir(p):
continue
mtime = os.path.getmtime(p)
skip = False
for pattern, cutoff_time in ignore.iteritems():
if (fnmatch(d, pattern)):
if cutoff_time == -1 or mtime > cutoff_time:
skip = True
break
else:
print("Ignored directory '%s' exceeds cutoff time" % d)
if skip:
continue
dirs.append((mtime, p))
dirs.sort()
while dirs:
mtime, d = dirs.pop(0)
# If we're newer than max_age, and don't need any more free space,
# we're all done here
if (not max_age) or (mtime > max_age):
if freespace(base_dirs[0]) >= gigs:
break
print "Deleting", d
if not dry_run:
try:
clobber_path = d + clobber_suffix
if os.path.exists(clobber_path):
rmdirRecursive(clobber_path)
# Prevent repeated moving.
if d.endswith(clobber_suffix):
rmdirRecursive(d)
else:
shutil.move(d, clobber_path)
rmdirRecursive(clobber_path)
except:
print >>sys.stderr, "Couldn't purge %s properly. Skipping." % d
def purge_hg_shares(share_dir, gigs, max_age, dry_run=False):
"""Deletes old hg directories under share_dir"""
# Find hg directories
hg_dirs = []
for root, dirs, files in os.walk(share_dir):
for d in dirs[:]:
path = os.path.join(root, d, '.hg')
if os.path.exists(path) or os.path.exists(path + clobber_suffix):
hg_dirs.append(os.path.join(root, d))
# Remove d from the list so we don't go traversing down into it
dirs.remove(d)
# Now we have a list of hg directories, call purge on them
purge(hg_dirs, gigs, [], max_age, dry_run)
# Clean up empty directories
for d in hg_dirs:
if not os.path.exists(os.path.join(d, '.hg')):
print "Cleaning up", d
if not dry_run:
rmdirRecursive(d)
if __name__ == '__main__':
import time
from optparse import OptionParser
from ConfigParser import ConfigParser, NoOptionError
max_age = 14
config = ConfigParser()
config.read(os.path.expanduser('~/.purge_builds.cfg'))
try:
max_age = config.getint('DEFAULT', 'max_age')
except (NoOptionError, ValueError):
pass
cwd = os.path.basename(os.getcwd())
parser = OptionParser(usage=__doc__)
parser.set_defaults(size=5, share_size=1, skip=[cwd], dry_run=False, max_age=max_age)
parser.add_option('-s', '--size',
help='free space required (in GB, default 5)', dest='size',
type='float')
parser.add_option('--share-size',
help='free space required for vcs shares (in GB, default 1)', dest='share_size',
type='float')
parser.add_option('-n', '--not', help='do not delete this directory. Append :30d to skip for up to 30 days, or :30h to skip for up to 30 hours',
action='append', dest='skip')
parser.add_option('', '--dry-run', action='store_true',
dest='dry_run',
help='''do not delete anything, just print out what would be
deleted. note that since no directories are deleted, if the amount of free
disk space in base_dir(s) is less than the required size, then ALL directories
will be listed in the order in which they would be deleted.''')
parser.add_option('', '--max-age', dest='max_age', type='int',
help='''maximum age (in days) for directories. If any directory
has an mtime older than this, it will be deleted, regardless of how
much free space is required. Set to 0 to disable.''')
options, base_dirs = parser.parse_args()
if len(base_dirs) < 1:
for d in DEFAULT_BASE_DIRS:
if os.path.exists(d):
base_dirs.append(d)
if len(base_dirs) < 1:
parser.error("Must specify one or more base_dirs")
sys.exit(1)
# Figure out the mtime before which we'll start deleting old directories
if options.max_age:
cutoff_time = time.time() - 24 * 3600 * options.max_age
else:
cutoff_time = None
purge(base_dirs, options.size, options.skip, cutoff_time, options.dry_run)
# Try to cleanup shared hg repos. We run here even if we've freed enough
# space so we can be sure and delete repositories older than max_age
if 'HG_SHARE_BASE_DIR' in os.environ:
purge_hg_shares(os.environ['HG_SHARE_BASE_DIR'],
options.share_size, cutoff_time, options.dry_run)
after = freespace(base_dirs[0]) / (1024 * 1024 * 1024.0)
# Try to cleanup the current dir if we still need space and it will
# actually help.
if after < options.size:
# We skip the tools dir here because we've usually just cloned it.
purge(['.'], options.size, ['tools'], cutoff_time, options.dry_run)
after = freespace(base_dirs[0]) / (1024 * 1024 * 1024.0)
if after < options.size:
print "Error: unable to free %1.2f GB of space. " % options.size + \
"Free space only %1.2f GB" % after
sys.exit(1)
else:
print "%1.2f GB of space available" % after
sys.exit(0)