Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100644 340 lines (310 sloc) 12.125 kb
e446d4d @apenwarr builder.py: don't import the 'random' module unless we need it.
authored
1 import sys, os, errno, stat
f7734c3 @apenwarr Split build functions into builder.py.
authored
2 import vars, jwack, state
22617d3 @apenwarr Half-support for using file checksums instead of stamps.
authored
3 from helpers import log, log_, debug, debug2, err, warn, unlink, close_on_exec
f7734c3 @apenwarr Split build functions into builder.py.
authored
4
5
6 def _possible_do_files(t):
6e6e453 @apenwarr Some speedups for doing redo-ifchange on a large number of static files.
authored
7 t = os.path.join(vars.BASE, t)
f7734c3 @apenwarr Split build functions into builder.py.
authored
8 yield "%s.do" % t, t, ''
9 dirname,filename = os.path.split(t)
10 l = filename.split('.')
11 l[0] = os.path.join(dirname, l[0])
12 for i in range(1,len(l)+1):
13 basename = '.'.join(l[:i])
14 ext = '.'.join(l[i:])
15 if ext: ext = '.' + ext
16 yield (os.path.join(dirname, "default%s.do" % ext),
17 os.path.join(dirname, basename), ext)
18
19
6e6e453 @apenwarr Some speedups for doing redo-ifchange on a large number of static files.
authored
20 def _find_do_file(f):
21 for dofile,basename,ext in _possible_do_files(f.name):
22 debug2('%s: %s ?\n' % (f.name, dofile))
f7734c3 @apenwarr Split build functions into builder.py.
authored
23 if os.path.exists(dofile):
6e6e453 @apenwarr Some speedups for doing redo-ifchange on a large number of static files.
authored
24 f.add_dep('m', dofile)
f7734c3 @apenwarr Split build functions into builder.py.
authored
25 return dofile,basename,ext
26 else:
6e6e453 @apenwarr Some speedups for doing redo-ifchange on a large number of static files.
authored
27 f.add_dep('c', dofile)
f7734c3 @apenwarr Split build functions into builder.py.
authored
28 return None,None,None
29
30
f644f3b @apenwarr Remove the need for relpath (and thus abspath) in builder.py.
authored
31 def _nice(t):
43b74f3 @apenwarr builder._nice(): show the right filename in the case of chdir().
authored
32 return state.relpath(t, vars.STARTDIR)
f644f3b @apenwarr Remove the need for relpath (and thus abspath) in builder.py.
authored
33
34
6d767e2 @apenwarr user-friendliness sanity checks: catch common mistakes regarding $1/$2/$...
authored
35 def _try_stat(filename):
36 try:
37 return os.stat(filename)
38 except OSError, e:
39 if e.errno == errno.ENOENT:
40 return None
41 else:
42 raise
43
44
f6d11d5 @apenwarr If a user manually changes a generated file, don't ever overwrite it.
authored
45 def warn_override(name):
46 warn('%s - you modified it; skipping\n' % name)
47
48
0126f6b @apenwarr Don't wipe the timestamp when a target fails to redo.
authored
49 class ImmediateReturn(Exception):
50 def __init__(self, rv):
51 Exception.__init__(self, "immediate return with exit code %d" % rv)
52 self.rv = rv
53
54
3209316 @apenwarr builder.py: now the only exported function is main().
authored
55 class BuildJob:
84169c5 @apenwarr Change locking stuff from fifos to fcntl.lockf().
authored
56 def __init__(self, t, sf, lock, shouldbuildfunc, donefunc):
57 self.t = t # original target name, not relative to vars.BASE
58 self.sf = sf
59201dd @apenwarr $3 and stdout no longer refer to the same file.
authored
59 self.tmpname1 = '%s.redo1.tmp' % t
60 self.tmpname2 = '%s.redo2.tmp' % t
3209316 @apenwarr builder.py: now the only exported function is main().
authored
61 self.lock = lock
62 self.shouldbuildfunc = shouldbuildfunc
63 self.donefunc = donefunc
6d767e2 @apenwarr user-friendliness sanity checks: catch common mistakes regarding $1/$2/$...
authored
64 self.before_t = _try_stat(self.t)
3209316 @apenwarr builder.py: now the only exported function is main().
authored
65
66 def start(self):
dcc2edb @apenwarr builder.py: further refactoring to run more stuff in the parent process
authored
67 assert(self.lock.owned)
0126f6b @apenwarr Don't wipe the timestamp when a target fails to redo.
authored
68 try:
f702417 @apenwarr The second half of redo-stamp: out-of-order building.
authored
69 dirty = self.shouldbuildfunc(self.t)
70 if not dirty:
0126f6b @apenwarr Don't wipe the timestamp when a target fails to redo.
authored
71 # target doesn't need to be built; skip the whole task
72 return self._after2(0)
73 except ImmediateReturn, e:
74 return self._after2(e.rv)
f702417 @apenwarr The second half of redo-stamp: out-of-order building.
authored
75
91630a8 @apenwarr Whoops, redo-oob was slightly wrong when used with -j.
authored
76 if vars.NO_OOB or dirty == True:
f702417 @apenwarr The second half of redo-stamp: out-of-order building.
authored
77 self._start_do()
78 else:
79 self._start_oob(dirty)
80
81 def _start_do(self):
82 assert(self.lock.owned)
83 t = self.t
84 sf = self.sf
f6d11d5 @apenwarr If a user manually changes a generated file, don't ever overwrite it.
authored
85 newstamp = sf.read_stamp()
86 if (sf.is_generated and
87 not sf.failed_runid and
88 newstamp != state.STAMP_MISSING and
89 (sf.stamp != newstamp or sf.is_override)):
90 warn_override(_nice(t))
91 sf.set_override()
92 sf.set_checked()
93 sf.save()
94 return self._after2(0)
66187e8 @apenwarr Slightly improve the "if target already existed" rule to ignore director...
authored
95 if (os.path.exists(t) and not os.path.exists(t + '/.')
84169c5 @apenwarr Change locking stuff from fifos to fcntl.lockf().
authored
96 and not sf.is_generated):
51bbdc6 @apenwarr If we can't find a .do file for a target, mark it as not is_generated.
authored
97 # an existing source file that was not generated by us.
98 # This step is mentioned by djb in his notes.
99 # For example, a rule called default.c.do could be used to try
100 # to produce hello.c, but we don't want that to happen if
101 # hello.c was created by the end user.
102 # FIXME: always refuse to redo any file that was modified outside
103 # of redo? That would make it easy for someone to override a
104 # file temporarily, and could be undone by deleting the file.
a62bd50 @apenwarr Switch state.py to use sqlite3 instead of filesystem-based stamps.
authored
105 debug2("-- static (%r)\n" % t)
84169c5 @apenwarr Change locking stuff from fifos to fcntl.lockf().
authored
106 sf.set_static()
107 sf.save()
dcc2edb @apenwarr builder.py: further refactoring to run more stuff in the parent process
authored
108 return self._after2(0)
84169c5 @apenwarr Change locking stuff from fifos to fcntl.lockf().
authored
109 sf.zap_deps()
110 (dofile, basename, ext) = _find_do_file(sf)
dcc2edb @apenwarr builder.py: further refactoring to run more stuff in the parent process
authored
111 if not dofile:
190dd65 @apenwarr It's okay if a file is marked as generated and doesn't have a .do.
authored
112 if os.path.exists(t):
84169c5 @apenwarr Change locking stuff from fifos to fcntl.lockf().
authored
113 sf.set_static()
114 sf.save()
190dd65 @apenwarr It's okay if a file is marked as generated and doesn't have a .do.
authored
115 return self._after2(0)
116 else:
117 err('no rule to make %r\n' % t)
118 return self._after2(1)
59201dd @apenwarr $3 and stdout no longer refer to the same file.
authored
119 unlink(self.tmpname1)
120 unlink(self.tmpname2)
121 ffd = os.open(self.tmpname1, os.O_CREAT|os.O_RDWR|os.O_EXCL, 0666)
2dbd471 @apenwarr state.py: reduce race condition between Lock.trylock() and unlock().
authored
122 close_on_exec(ffd, True)
135d1c1 @apenwarr builder.py: set FD_CLOEXEC flag on $3 when running a .do file.
authored
123 self.f = os.fdopen(ffd, 'w+')
dcc2edb @apenwarr builder.py: further refactoring to run more stuff in the parent process
authored
124 # this will run in the dofile's directory, so use only basenames here
125 argv = ['sh', '-e',
126 os.path.basename(dofile),
127 os.path.basename(basename), # target name (extension removed)
128 ext, # extension (if any), including leading dot
59201dd @apenwarr $3 and stdout no longer refer to the same file.
authored
129 os.path.basename(self.tmpname2) # randomized output file name
dcc2edb @apenwarr builder.py: further refactoring to run more stuff in the parent process
authored
130 ]
131 if vars.VERBOSE: argv[1] += 'v'
132 if vars.XTRACE: argv[1] += 'x'
133 if vars.VERBOSE or vars.XTRACE: log_('\n')
134 log('%s\n' % _nice(t))
135 self.argv = argv
84169c5 @apenwarr Change locking stuff from fifos to fcntl.lockf().
authored
136 sf.is_generated = True
137 sf.save()
a62bd50 @apenwarr Switch state.py to use sqlite3 instead of filesystem-based stamps.
authored
138 dof = state.File(name=dofile)
139 dof.set_static()
140 dof.save()
29d6c9a @apenwarr Don't db.commit() so frequently.
authored
141 state.commit()
dcc2edb @apenwarr builder.py: further refactoring to run more stuff in the parent process
authored
142 jwack.start_job(t, self._do_subproc, self._after)
143
f702417 @apenwarr The second half of redo-stamp: out-of-order building.
authored
144 def _start_oob(self, dirty):
145 # out-of-band redo of some sub-objects. This happens when we're not
146 # quite sure if t needs to be built or not (because some children look
147 # dirty, but might turn out to be clean thanks to checksums). We have
148 # to call redo-oob to figure it all out.
149 #
150 # Note: redo-oob will handle all the updating of sf, so we don't have
151 # to do it here, nor call _after1.
152 argv = ['redo-oob', self.sf.name] + [d.name for d in dirty]
153 log('(%s)\n' % _nice(self.t))
154 state.commit()
155 def run():
156 os.chdir(vars.BASE)
157 os.environ['REDO_DEPTH'] = vars.DEPTH + ' '
158 os.execvp(argv[0], argv)
159 assert(0)
160 # returns only if there's an exception
161 def after(t, rv):
162 return self._after2(rv)
163 jwack.start_job(self.t, run, after)
164
dcc2edb @apenwarr builder.py: further refactoring to run more stuff in the parent process
authored
165 def _do_subproc(self):
c29de89 @apenwarr Fix more trouble with .do scripts that cd to other directories.
authored
166 # careful: REDO_PWD was the PWD relative to the STARTPATH at the time
167 # we *started* building the current target; but that target ran
168 # redo-ifchange, and it might have done it from a different directory
a62bd50 @apenwarr Switch state.py to use sqlite3 instead of filesystem-based stamps.
authored
169 # than we started it in. So os.getcwd() might be != REDO_PWD right
170 # now.
8d0eba9 @apenwarr builder.py: use os.exec() instead of subprocess.call().
authored
171 dn = os.path.dirname(self.t)
c29de89 @apenwarr Fix more trouble with .do scripts that cd to other directories.
authored
172 newp = os.path.realpath(dn)
173 os.environ['REDO_PWD'] = state.relpath(newp, vars.STARTDIR)
8d0eba9 @apenwarr builder.py: use os.exec() instead of subprocess.call().
authored
174 os.environ['REDO_TARGET'] = os.path.basename(self.t)
175 os.environ['REDO_DEPTH'] = vars.DEPTH + ' '
176 if dn:
177 os.chdir(dn)
178 os.dup2(self.f.fileno(), 1)
179 os.close(self.f.fileno())
2dbd471 @apenwarr state.py: reduce race condition between Lock.trylock() and unlock().
authored
180 close_on_exec(1, False)
b3a14a2 @apenwarr When -x or -v is given, print the sh command we're executing.
authored
181 if vars.VERBOSE or vars.XTRACE: log_('* %s\n' % ' '.join(self.argv))
8d0eba9 @apenwarr builder.py: use os.exec() instead of subprocess.call().
authored
182 os.execvp(self.argv[0], self.argv)
135d1c1 @apenwarr builder.py: set FD_CLOEXEC flag on $3 when running a .do file.
authored
183 assert(0)
8d0eba9 @apenwarr builder.py: use os.exec() instead of subprocess.call().
authored
184 # returns only if there's an exception
dcc2edb @apenwarr builder.py: further refactoring to run more stuff in the parent process
authored
185
186 def _after(self, t, rv):
2dbd471 @apenwarr state.py: reduce race condition between Lock.trylock() and unlock().
authored
187 try:
3ef2bd7 @apenwarr Don't check as often whether the .redo directory exists.
authored
188 state.check_sane()
6d767e2 @apenwarr user-friendliness sanity checks: catch common mistakes regarding $1/$2/$...
authored
189 rv = self._after1(t, rv)
6e6e453 @apenwarr Some speedups for doing redo-ifchange on a large number of static files.
authored
190 state.commit()
2dbd471 @apenwarr state.py: reduce race condition between Lock.trylock() and unlock().
authored
191 finally:
192 self._after2(rv)
193
194 def _after1(self, t, rv):
dcc2edb @apenwarr builder.py: further refactoring to run more stuff in the parent process
authored
195 f = self.f
6d767e2 @apenwarr user-friendliness sanity checks: catch common mistakes regarding $1/$2/$...
authored
196 before_t = self.before_t
197 after_t = _try_stat(t)
59201dd @apenwarr $3 and stdout no longer refer to the same file.
authored
198 st1 = os.fstat(f.fileno())
199 st2 = _try_stat(self.tmpname2)
282bb04 @apenwarr If the created target is a directory, it's okay for the .do to create it...
authored
200 if after_t != before_t and not stat.S_ISDIR(after_t.st_mode):
59201dd @apenwarr $3 and stdout no longer refer to the same file.
authored
201 err('%s modified %s directly!\n' % (self.argv[2], t))
202 err('...you should update $3 (a temp file) or stdout, not $1.\n')
6d767e2 @apenwarr user-friendliness sanity checks: catch common mistakes regarding $1/$2/$...
authored
203 rv = 206
59201dd @apenwarr $3 and stdout no longer refer to the same file.
authored
204 elif st2 and st1.st_size > 0:
205 err('%s wrote to stdout *and* created $3.\n' % self.argv[2])
6d767e2 @apenwarr user-friendliness sanity checks: catch common mistakes regarding $1/$2/$...
authored
206 err('...you should write status messages to stderr, not stdout.\n')
207 rv = 207
dcc2edb @apenwarr builder.py: further refactoring to run more stuff in the parent process
authored
208 if rv==0:
59201dd @apenwarr $3 and stdout no longer refer to the same file.
authored
209 if st2:
210 os.rename(self.tmpname2, t)
211 os.unlink(self.tmpname1)
212 elif st1.st_size > 0:
213 os.rename(self.tmpname1, t)
214 if st2:
215 os.unlink(self.tmpname2)
216 else: # no output generated at all; that's ok
217 unlink(self.tmpname1)
218 unlink(t)
84169c5 @apenwarr Change locking stuff from fifos to fcntl.lockf().
authored
219 sf = self.sf
22617d3 @apenwarr Half-support for using file checksums instead of stamps.
authored
220 sf.refresh()
f6d11d5 @apenwarr If a user manually changes a generated file, don't ever overwrite it.
authored
221 sf.is_generated = True
22617d3 @apenwarr Half-support for using file checksums instead of stamps.
authored
222 sf.is_override = False
223 if sf.is_checked() or sf.is_changed():
224 # it got checked during the run; someone ran redo-stamp.
225 # update_stamp would call set_changed(); we don't want that
226 sf.stamp = sf.read_stamp()
227 else:
228 sf.csum = None
229 sf.update_stamp()
230 sf.set_changed()
a62bd50 @apenwarr Switch state.py to use sqlite3 instead of filesystem-based stamps.
authored
231 sf.save()
dcc2edb @apenwarr builder.py: further refactoring to run more stuff in the parent process
authored
232 else:
59201dd @apenwarr $3 and stdout no longer refer to the same file.
authored
233 unlink(self.tmpname1)
234 unlink(self.tmpname2)
84169c5 @apenwarr Change locking stuff from fifos to fcntl.lockf().
authored
235 sf = self.sf
0126f6b @apenwarr Don't wipe the timestamp when a target fails to redo.
authored
236 sf.set_failed()
a62bd50 @apenwarr Switch state.py to use sqlite3 instead of filesystem-based stamps.
authored
237 sf.save()
dcc2edb @apenwarr builder.py: further refactoring to run more stuff in the parent process
authored
238 f.close()
239 if rv != 0:
240 err('%s: exit code %d\n' % (_nice(t),rv))
241 else:
242 if vars.VERBOSE or vars.XTRACE:
243 log('%s (done)\n\n' % _nice(t))
6d767e2 @apenwarr user-friendliness sanity checks: catch common mistakes regarding $1/$2/$...
authored
244 return rv
3209316 @apenwarr builder.py: now the only exported function is main().
authored
245
dcc2edb @apenwarr builder.py: further refactoring to run more stuff in the parent process
authored
246 def _after2(self, rv):
2dbd471 @apenwarr state.py: reduce race condition between Lock.trylock() and unlock().
authored
247 try:
248 self.donefunc(self.t, rv)
249 assert(self.lock.owned)
250 finally:
251 self.lock.unlock()
3209316 @apenwarr builder.py: now the only exported function is main().
authored
252
253
254 def main(targets, shouldbuildfunc):
f7734c3 @apenwarr Split build functions into builder.py.
authored
255 retcode = [0] # a list so that it can be reassigned from done()
256 if vars.SHUFFLE:
e446d4d @apenwarr builder.py: don't import the 'random' module unless we need it.
authored
257 import random
f7734c3 @apenwarr Split build functions into builder.py.
authored
258 random.shuffle(targets)
259
260 locked = []
261
262 def done(t, rv):
263 if rv:
264 retcode[0] = 1
265
266 for i in range(len(targets)):
267 t = targets[i]
7aa7c41 @apenwarr builder,jwack: slight cleanup to token passing.
authored
268
269 # In the first cycle, we just build as much as we can without worrying
270 # about any lock contention. If someone else has it locked, we move on.
f7734c3 @apenwarr Split build functions into builder.py.
authored
271 for t in targets:
6e6e453 @apenwarr Some speedups for doing redo-ifchange on a large number of static files.
authored
272 if not jwack.has_token():
273 state.commit()
f7734c3 @apenwarr Split build functions into builder.py.
authored
274 jwack.get_token(t)
b937e62 @apenwarr Add a new -k (--keep-going) option, like make has.
authored
275 if retcode[0] and not vars.KEEP_GOING:
276 break
3ef2bd7 @apenwarr Don't check as often whether the .redo directory exists.
authored
277 if not state.check_sane():
278 err('.redo directory disappeared; cannot continue.\n')
dce0076 @apenwarr Print a useful message and exit when the .redo directory disappears.
authored
279 retcode[0] = 205
280 break
84169c5 @apenwarr Change locking stuff from fifos to fcntl.lockf().
authored
281 f = state.File(name=t)
282 lock = state.Lock(f.id)
f702417 @apenwarr The second half of redo-stamp: out-of-order building.
authored
283 if vars.UNLOCKED:
284 lock.owned = True
285 else:
286 lock.trylock()
f7734c3 @apenwarr Split build functions into builder.py.
authored
287 if not lock.owned:
03a054c @apenwarr Add a --debug-locks option.
authored
288 if vars.DEBUG_LOCKS:
289 log('%s (locked...)\n' % _nice(t))
84169c5 @apenwarr Change locking stuff from fifos to fcntl.lockf().
authored
290 locked.append((f.id,t))
f7734c3 @apenwarr Split build functions into builder.py.
authored
291 else:
84169c5 @apenwarr Change locking stuff from fifos to fcntl.lockf().
authored
292 BuildJob(t, f, lock, shouldbuildfunc, done).start()
7aa7c41 @apenwarr builder,jwack: slight cleanup to token passing.
authored
293
294 # Now we've built all the "easy" ones. Go back and just wait on the
f70c028 @apenwarr With --debug-locks, print a message when we stop to wait on a lock.
authored
295 # remaining ones one by one. There's no reason to do it any more
296 # efficiently, because if these targets were previously locked, that
297 # means someone else was building them; thus, we probably won't need to
298 # do anything. The only exception is if we're invoked as redo instead
299 # of redo-ifchange; then we have to redo it even if someone else already
300 # did. But that should be rare.
f7734c3 @apenwarr Split build functions into builder.py.
authored
301 while locked or jwack.running():
6e6e453 @apenwarr Some speedups for doing redo-ifchange on a large number of static files.
authored
302 state.commit()
f7734c3 @apenwarr Split build functions into builder.py.
authored
303 jwack.wait_all()
3209316 @apenwarr builder.py: now the only exported function is main().
authored
304 # at this point, we don't have any children holding any tokens, so
305 # it's okay to block below.
b937e62 @apenwarr Add a new -k (--keep-going) option, like make has.
authored
306 if retcode[0] and not vars.KEEP_GOING:
307 break
f7734c3 @apenwarr Split build functions into builder.py.
authored
308 if locked:
3ef2bd7 @apenwarr Don't check as often whether the .redo directory exists.
authored
309 if not state.check_sane():
310 err('.redo directory disappeared; cannot continue.\n')
dce0076 @apenwarr Print a useful message and exit when the .redo directory disappears.
authored
311 retcode[0] = 205
312 break
84169c5 @apenwarr Change locking stuff from fifos to fcntl.lockf().
authored
313 fid,t = locked.pop(0)
314 lock = state.Lock(fid)
f70c028 @apenwarr With --debug-locks, print a message when we stop to wait on a lock.
authored
315 lock.trylock()
c4be005 @apenwarr Release the jwack token when doing a synchronous lock wait.
authored
316 while not lock.owned:
16bebd2 @apenwarr builder: the (WAITING) message from --debug-locks didn't print every tim...
authored
317 if vars.DEBUG_LOCKS:
f70c028 @apenwarr With --debug-locks, print a message when we stop to wait on a lock.
authored
318 warn('%s (WAITING)\n' % _nice(t))
c4be005 @apenwarr Release the jwack token when doing a synchronous lock wait.
authored
319 # this sequence looks a little silly, but the idea is to
320 # give up our personal token while we wait for the lock to
321 # be released; but we should never run get_token() while
322 # holding a lock, or we could cause deadlocks.
323 jwack.release_mine()
f70c028 @apenwarr With --debug-locks, print a message when we stop to wait on a lock.
authored
324 lock.waitlock()
c4be005 @apenwarr Release the jwack token when doing a synchronous lock wait.
authored
325 lock.unlock()
326 jwack.get_token(t)
327 lock.trylock()
f7734c3 @apenwarr Split build functions into builder.py.
authored
328 assert(lock.owned)
03a054c @apenwarr Add a --debug-locks option.
authored
329 if vars.DEBUG_LOCKS:
330 log('%s (...unlocked!)\n' % _nice(t))
0126f6b @apenwarr Don't wipe the timestamp when a target fails to redo.
authored
331 if state.File(name=t).is_failed():
f644f3b @apenwarr Remove the need for relpath (and thus abspath) in builder.py.
authored
332 err('%s: failed in another thread\n' % _nice(t))
f7734c3 @apenwarr Split build functions into builder.py.
authored
333 retcode[0] = 2
334 lock.unlock()
335 else:
84169c5 @apenwarr Change locking stuff from fifos to fcntl.lockf().
authored
336 BuildJob(t, state.File(id=fid), lock,
337 shouldbuildfunc, done).start()
29d6c9a @apenwarr Don't db.commit() so frequently.
authored
338 state.commit()
f7734c3 @apenwarr Split build functions into builder.py.
authored
339 return retcode[0]
Something went wrong with that request. Please try again.