This repository has been archived by the owner on Sep 15, 2021. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 28
/
clobberer.py
executable file
·219 lines (189 loc) · 7.45 KB
/
clobberer.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
#!/usr/bin/env python
# vim:sts=2 sw=2
import sys, shutil, urllib2, urllib, os, traceback, time
clobber_suffix='.deleteme'
def ts_to_str(ts):
if ts is None:
return None
return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(ts))
def write_file(ts, fn):
assert isinstance(ts, int)
f = open(fn, "w")
f.write(str(ts))
f.close()
def read_file(fn):
if not os.path.exists(fn):
return None
data = open(fn).read().strip()
try:
return int(data)
except ValueError:
return None
def rmdirRecursive(dir):
"""This is a replacement for shutil.rmtree that works better under
windows. Thanks to Bear at the OSAF for the code.
(Borrowed from buildbot.slave.commands)"""
if not os.path.exists(dir):
# This handles broken links
if os.path.islink(dir):
os.remove(dir)
return
if os.path.islink(dir):
os.remove(dir)
return
# Verify the directory is read/write/execute for the current user
os.chmod(dir, 0700)
for name in os.listdir(dir):
full_name = os.path.join(dir, name)
# on Windows, if we don't have write permission we can't remove
# the file/directory either, so turn that on
if os.name == 'nt':
if not os.access(full_name, os.W_OK):
# I think this is now redundant, but I don't have an NT
# machine to test on, so I'm going to leave it in place
# -warner
os.chmod(full_name, 0600)
if os.path.isdir(full_name):
rmdirRecursive(full_name)
else:
# Don't try to chmod links
if not os.path.islink(full_name):
os.chmod(full_name, 0700)
os.remove(full_name)
os.rmdir(dir)
def do_clobber(dir, dryrun=False, skip=None):
try:
for f in os.listdir(dir):
if skip is not None and f in skip:
print "Skipping", f
continue
clobber_path=f+clobber_suffix
if os.path.isfile(f):
print "Removing", f
if not dryrun:
if os.path.exists(clobber_path):
os.unlink(clobber_path)
# Prevent repeated moving.
if f.endswith(clobber_suffix):
os.unlink(f)
else:
shutil.move(f, clobber_path)
os.unlink(clobber_path)
elif os.path.isdir(f):
print "Removing %s/" % f
if not dryrun:
if os.path.exists(clobber_path):
rmdirRecursive(clobber_path)
# Prevent repeated moving.
if f.endswith(clobber_suffix):
rmdirRecursive(f)
else:
shutil.move(f, clobber_path)
rmdirRecursive(clobber_path)
except:
print "Couldn't clobber properly, bailing out."
sys.exit(1)
def getClobberDates(clobberURL, branch, buildername, builddir, slave, master):
params = dict(branch=branch, buildername=buildername, builddir=builddir, slave=slave, master=master)
url = "%s?%s" % (clobberURL, urllib.urlencode(params))
print "Checking clobber URL: %s" % url
data = urllib2.urlopen(url).read().strip()
retval = {}
try:
for line in data.split("\n"):
line = line.strip()
if not line:
continue
builddir, builder_time, who = line.split(":")
builder_time = int(builder_time)
retval[builddir] = (builder_time, who)
return retval
except ValueError:
print "Error parsing response from server"
print data
raise
if __name__ == "__main__":
from optparse import OptionParser
parser = OptionParser("%prog [options] clobberURL branch buildername builddir slave master")
parser.add_option("-n", "--dry-run", dest="dryrun", action="store_true",
default=False, help="don't actually delete anything")
parser.add_option("-t", "--periodic", dest="period", type="float",
default=None, help="hours between periodic clobbers")
parser.add_option('-s', '--skip', help='do not delete this file/directory',
action='append', dest='skip', default=['last-clobber'])
parser.add_option('-d', '--dir', help='clobber this directory',
dest='dir', default='.', type='string')
parser.add_option('-v', '--verbose', help='be more verbose',
dest='verbose', action='store_true', default=False)
options, args = parser.parse_args()
if len(args) != 6:
parser.error("Incorrect number of arguments")
if options.period:
periodicClobberTime = options.period * 3600
else:
periodicClobberTime = None
clobberURL, branch, builder, my_builddir, slave, master = args
try:
server_clobber_dates = getClobberDates(clobberURL, branch, builder, my_builddir, slave, master)
except:
if options.verbose:
traceback.print_exc()
print "Error contacting server"
sys.exit(1)
if options.verbose:
print "Server gave us", server_clobber_dates
now = int(time.time())
# Add ourself to the server_clobber_dates if it's not set
# This happens when this slave has never been clobbered
if my_builddir not in server_clobber_dates:
server_clobber_dates[my_builddir] = None, ""
root_dir = os.path.abspath(options.dir)
for builddir, (server_clobber_date, who) in server_clobber_dates.items():
builder_dir = os.path.join(root_dir, builddir)
if not os.path.isdir(builder_dir):
print "%s doesn't exist, skipping" % builder_dir
continue
os.chdir(builder_dir)
our_clobber_date = read_file("last-clobber")
clobber = False
print "%s:Our last clobber date: " % builddir, ts_to_str(our_clobber_date)
print "%s:Server clobber date: " % builddir, ts_to_str(server_clobber_date)
# If we don't have a last clobber date, then this is probably a fresh build.
# We should only do a forced server clobber if we know when our last clobber
# was, and if the server date is more recent than that.
if server_clobber_date is not None and our_clobber_date is not None:
# If the server is giving us a clobber date, compare the server's idea of
# the clobber date to our last clobber date
if server_clobber_date > our_clobber_date:
# If the server's clobber date is greater than our last clobber date,
# then we should clobber.
clobber = True
# We should also update our clobber date to match the server's
our_clobber_date = server_clobber_date
if who:
print "%s:Server is forcing a clobber, initiated by %s" % (builddir, who)
else:
print "%s:Server is forcing a clobber" % builddir
if not clobber:
# Disable periodic clobbers for builders that aren't my_builddir
if builddir != my_builddir:
continue
# Next, check if more than the periodicClobberTime period has passed since
# our last clobber
if our_clobber_date is None:
# We've never been clobbered
# Set our last clobber time to now, so that we'll clobber
# properly after periodicClobberTime
our_clobber_date = now
write_file(our_clobber_date, "last-clobber")
elif periodicClobberTime and now > our_clobber_date + periodicClobberTime:
# periodicClobberTime has passed since our last clobber
clobber = True
# Update our clobber date to now
our_clobber_date = now
print "%s:More than %s seconds have passed since our last clobber" % (builddir, periodicClobberTime)
if clobber:
# Finally, perform a clobber if we're supposed to
print "%s:Clobbering..." % builddir
do_clobber(builder_dir, options.dryrun, options.skip)
write_file(our_clobber_date, "last-clobber")