Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Newer
Older
100644 307 lines (280 sloc) 10.94 kB
e446d4d @apenwarr builder.py: don't import the 'random' module unless we need it.
authored
1 import sys, os, errno, stat
f7734c3 @apenwarr Split build functions into builder.py.
authored
2 import vars, jwack, state
22617d3 @apenwarr Half-support for using file checksums instead of stamps.
authored
3 from helpers import log, log_, debug, debug2, err, warn, unlink, close_on_exec
f7734c3 @apenwarr Split build functions into builder.py.
authored
4
5
6 def _possible_do_files(t):
6e6e453 @apenwarr Some speedups for doing redo-ifchange on a large number of static files.
authored
7 t = os.path.join(vars.BASE, t)
f7734c3 @apenwarr Split build functions into builder.py.
authored
8 yield "%s.do" % t, t, ''
9 dirname,filename = os.path.split(t)
10 l = filename.split('.')
11 l[0] = os.path.join(dirname, l[0])
12 for i in range(1,len(l)+1):
13 basename = '.'.join(l[:i])
14 ext = '.'.join(l[i:])
15 if ext: ext = '.' + ext
16 yield (os.path.join(dirname, "default%s.do" % ext),
17 os.path.join(dirname, basename), ext)
18
19
6e6e453 @apenwarr Some speedups for doing redo-ifchange on a large number of static files.
authored
20 def _find_do_file(f):
21 for dofile,basename,ext in _possible_do_files(f.name):
22 debug2('%s: %s ?\n' % (f.name, dofile))
f7734c3 @apenwarr Split build functions into builder.py.
authored
23 if os.path.exists(dofile):
6e6e453 @apenwarr Some speedups for doing redo-ifchange on a large number of static files.
authored
24 f.add_dep('m', dofile)
f7734c3 @apenwarr Split build functions into builder.py.
authored
25 return dofile,basename,ext
26 else:
6e6e453 @apenwarr Some speedups for doing redo-ifchange on a large number of static files.
authored
27 f.add_dep('c', dofile)
f7734c3 @apenwarr Split build functions into builder.py.
authored
28 return None,None,None
29
30
f644f3b @apenwarr Remove the need for relpath (and thus abspath) in builder.py.
authored
31 def _nice(t):
43b74f3 @apenwarr builder._nice(): show the right filename in the case of chdir().
authored
32 return state.relpath(t, vars.STARTDIR)
f644f3b @apenwarr Remove the need for relpath (and thus abspath) in builder.py.
authored
33
34
6d767e2 @apenwarr user-friendliness sanity checks: catch common mistakes regarding $1/$…
authored
35 def _try_stat(filename):
36 try:
37 return os.stat(filename)
38 except OSError, e:
39 if e.errno == errno.ENOENT:
40 return None
41 else:
42 raise
43
44
f6d11d5 @apenwarr If a user manually changes a generated file, don't ever overwrite it.
authored
45 def warn_override(name):
46 warn('%s - you modified it; skipping\n' % name)
47
48
0126f6b @apenwarr Don't wipe the timestamp when a target fails to redo.
authored
49 class ImmediateReturn(Exception):
50 def __init__(self, rv):
51 Exception.__init__(self, "immediate return with exit code %d" % rv)
52 self.rv = rv
53
54
3209316 @apenwarr builder.py: now the only exported function is main().
authored
55 class BuildJob:
84169c5 @apenwarr Change locking stuff from fifos to fcntl.lockf().
authored
56 def __init__(self, t, sf, lock, shouldbuildfunc, donefunc):
57 self.t = t # original target name, not relative to vars.BASE
58 self.sf = sf
59201dd @apenwarr $3 and stdout no longer refer to the same file.
authored
59 self.tmpname1 = '%s.redo1.tmp' % t
60 self.tmpname2 = '%s.redo2.tmp' % t
3209316 @apenwarr builder.py: now the only exported function is main().
authored
61 self.lock = lock
62 self.shouldbuildfunc = shouldbuildfunc
63 self.donefunc = donefunc
6d767e2 @apenwarr user-friendliness sanity checks: catch common mistakes regarding $1/$…
authored
64 self.before_t = _try_stat(self.t)
3209316 @apenwarr builder.py: now the only exported function is main().
authored
65
66 def start(self):
dcc2edb @apenwarr builder.py: further refactoring to run more stuff in the parent process
authored
67 assert(self.lock.owned)
68 t = self.t
84169c5 @apenwarr Change locking stuff from fifos to fcntl.lockf().
authored
69 sf = self.sf
0126f6b @apenwarr Don't wipe the timestamp when a target fails to redo.
authored
70 try:
71 if not self.shouldbuildfunc(t):
72 # target doesn't need to be built; skip the whole task
73 return self._after2(0)
74 except ImmediateReturn, e:
75 return self._after2(e.rv)
f6d11d5 @apenwarr If a user manually changes a generated file, don't ever overwrite it.
authored
76 newstamp = sf.read_stamp()
77 if (sf.is_generated and
78 not sf.failed_runid and
79 newstamp != state.STAMP_MISSING and
80 (sf.stamp != newstamp or sf.is_override)):
81 warn_override(_nice(t))
82 sf.set_override()
83 sf.set_checked()
84 sf.save()
85 return self._after2(0)
66187e8 @apenwarr Slightly improve the "if target already existed" rule to ignore direc…
authored
86 if (os.path.exists(t) and not os.path.exists(t + '/.')
84169c5 @apenwarr Change locking stuff from fifos to fcntl.lockf().
authored
87 and not sf.is_generated):
51bbdc6 @apenwarr If we can't find a .do file for a target, mark it as not is_generated.
authored
88 # an existing source file that was not generated by us.
89 # This step is mentioned by djb in his notes.
90 # For example, a rule called default.c.do could be used to try
91 # to produce hello.c, but we don't want that to happen if
92 # hello.c was created by the end user.
93 # FIXME: always refuse to redo any file that was modified outside
94 # of redo? That would make it easy for someone to override a
95 # file temporarily, and could be undone by deleting the file.
a62bd50 @apenwarr Switch state.py to use sqlite3 instead of filesystem-based stamps.
authored
96 debug2("-- static (%r)\n" % t)
84169c5 @apenwarr Change locking stuff from fifos to fcntl.lockf().
authored
97 sf.set_static()
98 sf.save()
dcc2edb @apenwarr builder.py: further refactoring to run more stuff in the parent process
authored
99 return self._after2(0)
84169c5 @apenwarr Change locking stuff from fifos to fcntl.lockf().
authored
100 sf.zap_deps()
101 (dofile, basename, ext) = _find_do_file(sf)
dcc2edb @apenwarr builder.py: further refactoring to run more stuff in the parent process
authored
102 if not dofile:
190dd65 @apenwarr It's okay if a file is marked as generated and doesn't have a .do.
authored
103 if os.path.exists(t):
84169c5 @apenwarr Change locking stuff from fifos to fcntl.lockf().
authored
104 sf.set_static()
105 sf.save()
190dd65 @apenwarr It's okay if a file is marked as generated and doesn't have a .do.
authored
106 return self._after2(0)
107 else:
108 err('no rule to make %r\n' % t)
109 return self._after2(1)
59201dd @apenwarr $3 and stdout no longer refer to the same file.
authored
110 unlink(self.tmpname1)
111 unlink(self.tmpname2)
112 ffd = os.open(self.tmpname1, os.O_CREAT|os.O_RDWR|os.O_EXCL, 0666)
2dbd471 @apenwarr state.py: reduce race condition between Lock.trylock() and unlock().
authored
113 close_on_exec(ffd, True)
135d1c1 @apenwarr builder.py: set FD_CLOEXEC flag on $3 when running a .do file.
authored
114 self.f = os.fdopen(ffd, 'w+')
dcc2edb @apenwarr builder.py: further refactoring to run more stuff in the parent process
authored
115 # this will run in the dofile's directory, so use only basenames here
116 argv = ['sh', '-e',
117 os.path.basename(dofile),
118 os.path.basename(basename), # target name (extension removed)
119 ext, # extension (if any), including leading dot
59201dd @apenwarr $3 and stdout no longer refer to the same file.
authored
120 os.path.basename(self.tmpname2) # randomized output file name
dcc2edb @apenwarr builder.py: further refactoring to run more stuff in the parent process
authored
121 ]
122 if vars.VERBOSE: argv[1] += 'v'
123 if vars.XTRACE: argv[1] += 'x'
124 if vars.VERBOSE or vars.XTRACE: log_('\n')
125 log('%s\n' % _nice(t))
126 self.argv = argv
84169c5 @apenwarr Change locking stuff from fifos to fcntl.lockf().
authored
127 sf.is_generated = True
128 sf.save()
a62bd50 @apenwarr Switch state.py to use sqlite3 instead of filesystem-based stamps.
authored
129 dof = state.File(name=dofile)
130 dof.set_static()
131 dof.save()
29d6c9a @apenwarr Don't db.commit() so frequently.
authored
132 state.commit()
dcc2edb @apenwarr builder.py: further refactoring to run more stuff in the parent process
authored
133 jwack.start_job(t, self._do_subproc, self._after)
134
135 def _do_subproc(self):
c29de89 @apenwarr Fix more trouble with .do scripts that cd to other directories.
authored
136 # careful: REDO_PWD was the PWD relative to the STARTPATH at the time
137 # we *started* building the current target; but that target ran
138 # redo-ifchange, and it might have done it from a different directory
a62bd50 @apenwarr Switch state.py to use sqlite3 instead of filesystem-based stamps.
authored
139 # than we started it in. So os.getcwd() might be != REDO_PWD right
140 # now.
8d0eba9 @apenwarr builder.py: use os.exec() instead of subprocess.call().
authored
141 dn = os.path.dirname(self.t)
c29de89 @apenwarr Fix more trouble with .do scripts that cd to other directories.
authored
142 newp = os.path.realpath(dn)
143 os.environ['REDO_PWD'] = state.relpath(newp, vars.STARTDIR)
8d0eba9 @apenwarr builder.py: use os.exec() instead of subprocess.call().
authored
144 os.environ['REDO_TARGET'] = os.path.basename(self.t)
145 os.environ['REDO_DEPTH'] = vars.DEPTH + ' '
146 if dn:
147 os.chdir(dn)
148 os.dup2(self.f.fileno(), 1)
149 os.close(self.f.fileno())
2dbd471 @apenwarr state.py: reduce race condition between Lock.trylock() and unlock().
authored
150 close_on_exec(1, False)
b3a14a2 @apenwarr When -x or -v is given, print the sh command we're executing.
authored
151 if vars.VERBOSE or vars.XTRACE: log_('* %s\n' % ' '.join(self.argv))
8d0eba9 @apenwarr builder.py: use os.exec() instead of subprocess.call().
authored
152 os.execvp(self.argv[0], self.argv)
135d1c1 @apenwarr builder.py: set FD_CLOEXEC flag on $3 when running a .do file.
authored
153 assert(0)
8d0eba9 @apenwarr builder.py: use os.exec() instead of subprocess.call().
authored
154 # returns only if there's an exception
dcc2edb @apenwarr builder.py: further refactoring to run more stuff in the parent process
authored
155
156 def _after(self, t, rv):
2dbd471 @apenwarr state.py: reduce race condition between Lock.trylock() and unlock().
authored
157 try:
3ef2bd7 @apenwarr Don't check as often whether the .redo directory exists.
authored
158 state.check_sane()
6d767e2 @apenwarr user-friendliness sanity checks: catch common mistakes regarding $1/$…
authored
159 rv = self._after1(t, rv)
6e6e453 @apenwarr Some speedups for doing redo-ifchange on a large number of static files.
authored
160 state.commit()
2dbd471 @apenwarr state.py: reduce race condition between Lock.trylock() and unlock().
authored
161 finally:
162 self._after2(rv)
163
164 def _after1(self, t, rv):
dcc2edb @apenwarr builder.py: further refactoring to run more stuff in the parent process
authored
165 f = self.f
6d767e2 @apenwarr user-friendliness sanity checks: catch common mistakes regarding $1/$…
authored
166 before_t = self.before_t
167 after_t = _try_stat(t)
59201dd @apenwarr $3 and stdout no longer refer to the same file.
authored
168 st1 = os.fstat(f.fileno())
169 st2 = _try_stat(self.tmpname2)
282bb04 @apenwarr If the created target is a directory, it's okay for the .do to create…
authored
170 if after_t != before_t and not stat.S_ISDIR(after_t.st_mode):
59201dd @apenwarr $3 and stdout no longer refer to the same file.
authored
171 err('%s modified %s directly!\n' % (self.argv[2], t))
172 err('...you should update $3 (a temp file) or stdout, not $1.\n')
6d767e2 @apenwarr user-friendliness sanity checks: catch common mistakes regarding $1/$…
authored
173 rv = 206
59201dd @apenwarr $3 and stdout no longer refer to the same file.
authored
174 elif st2 and st1.st_size > 0:
175 err('%s wrote to stdout *and* created $3.\n' % self.argv[2])
6d767e2 @apenwarr user-friendliness sanity checks: catch common mistakes regarding $1/$…
authored
176 err('...you should write status messages to stderr, not stdout.\n')
177 rv = 207
dcc2edb @apenwarr builder.py: further refactoring to run more stuff in the parent process
authored
178 if rv==0:
59201dd @apenwarr $3 and stdout no longer refer to the same file.
authored
179 if st2:
180 os.rename(self.tmpname2, t)
181 os.unlink(self.tmpname1)
182 elif st1.st_size > 0:
183 os.rename(self.tmpname1, t)
184 if st2:
185 os.unlink(self.tmpname2)
186 else: # no output generated at all; that's ok
187 unlink(self.tmpname1)
188 unlink(t)
84169c5 @apenwarr Change locking stuff from fifos to fcntl.lockf().
authored
189 sf = self.sf
22617d3 @apenwarr Half-support for using file checksums instead of stamps.
authored
190 sf.refresh()
f6d11d5 @apenwarr If a user manually changes a generated file, don't ever overwrite it.
authored
191 sf.is_generated = True
22617d3 @apenwarr Half-support for using file checksums instead of stamps.
authored
192 sf.is_override = False
193 if sf.is_checked() or sf.is_changed():
194 # it got checked during the run; someone ran redo-stamp.
195 # update_stamp would call set_changed(); we don't want that
196 sf.stamp = sf.read_stamp()
197 else:
198 sf.csum = None
199 sf.update_stamp()
200 sf.set_changed()
a62bd50 @apenwarr Switch state.py to use sqlite3 instead of filesystem-based stamps.
authored
201 sf.save()
dcc2edb @apenwarr builder.py: further refactoring to run more stuff in the parent process
authored
202 else:
59201dd @apenwarr $3 and stdout no longer refer to the same file.
authored
203 unlink(self.tmpname1)
204 unlink(self.tmpname2)
84169c5 @apenwarr Change locking stuff from fifos to fcntl.lockf().
authored
205 sf = self.sf
0126f6b @apenwarr Don't wipe the timestamp when a target fails to redo.
authored
206 sf.set_failed()
a62bd50 @apenwarr Switch state.py to use sqlite3 instead of filesystem-based stamps.
authored
207 sf.save()
dcc2edb @apenwarr builder.py: further refactoring to run more stuff in the parent process
authored
208 f.close()
209 if rv != 0:
210 err('%s: exit code %d\n' % (_nice(t),rv))
211 else:
212 if vars.VERBOSE or vars.XTRACE:
213 log('%s (done)\n\n' % _nice(t))
6d767e2 @apenwarr user-friendliness sanity checks: catch common mistakes regarding $1/$…
authored
214 return rv
3209316 @apenwarr builder.py: now the only exported function is main().
authored
215
dcc2edb @apenwarr builder.py: further refactoring to run more stuff in the parent process
authored
216 def _after2(self, rv):
2dbd471 @apenwarr state.py: reduce race condition between Lock.trylock() and unlock().
authored
217 try:
218 self.donefunc(self.t, rv)
219 assert(self.lock.owned)
220 finally:
221 self.lock.unlock()
3209316 @apenwarr builder.py: now the only exported function is main().
authored
222
223
224 def main(targets, shouldbuildfunc):
f7734c3 @apenwarr Split build functions into builder.py.
authored
225 retcode = [0] # a list so that it can be reassigned from done()
226 if vars.SHUFFLE:
e446d4d @apenwarr builder.py: don't import the 'random' module unless we need it.
authored
227 import random
f7734c3 @apenwarr Split build functions into builder.py.
authored
228 random.shuffle(targets)
229
230 locked = []
231
232 def done(t, rv):
233 if rv:
234 retcode[0] = 1
235
236 for i in range(len(targets)):
237 t = targets[i]
7aa7c41 @apenwarr builder,jwack: slight cleanup to token passing.
authored
238
239 # In the first cycle, we just build as much as we can without worrying
240 # about any lock contention. If someone else has it locked, we move on.
f7734c3 @apenwarr Split build functions into builder.py.
authored
241 for t in targets:
6e6e453 @apenwarr Some speedups for doing redo-ifchange on a large number of static files.
authored
242 if not jwack.has_token():
243 state.commit()
f7734c3 @apenwarr Split build functions into builder.py.
authored
244 jwack.get_token(t)
b937e62 @apenwarr Add a new -k (--keep-going) option, like make has.
authored
245 if retcode[0] and not vars.KEEP_GOING:
246 break
3ef2bd7 @apenwarr Don't check as often whether the .redo directory exists.
authored
247 if not state.check_sane():
248 err('.redo directory disappeared; cannot continue.\n')
dce0076 @apenwarr Print a useful message and exit when the .redo directory disappears.
authored
249 retcode[0] = 205
250 break
84169c5 @apenwarr Change locking stuff from fifos to fcntl.lockf().
authored
251 f = state.File(name=t)
252 lock = state.Lock(f.id)
f7734c3 @apenwarr Split build functions into builder.py.
authored
253 lock.trylock()
254 if not lock.owned:
03a054c @apenwarr Add a --debug-locks option.
authored
255 if vars.DEBUG_LOCKS:
256 log('%s (locked...)\n' % _nice(t))
84169c5 @apenwarr Change locking stuff from fifos to fcntl.lockf().
authored
257 locked.append((f.id,t))
f7734c3 @apenwarr Split build functions into builder.py.
authored
258 else:
84169c5 @apenwarr Change locking stuff from fifos to fcntl.lockf().
authored
259 BuildJob(t, f, lock, shouldbuildfunc, done).start()
7aa7c41 @apenwarr builder,jwack: slight cleanup to token passing.
authored
260
261 # Now we've built all the "easy" ones. Go back and just wait on the
f70c028 @apenwarr With --debug-locks, print a message when we stop to wait on a lock.
authored
262 # remaining ones one by one. There's no reason to do it any more
263 # efficiently, because if these targets were previously locked, that
264 # means someone else was building them; thus, we probably won't need to
265 # do anything. The only exception is if we're invoked as redo instead
266 # of redo-ifchange; then we have to redo it even if someone else already
267 # did. But that should be rare.
f7734c3 @apenwarr Split build functions into builder.py.
authored
268 while locked or jwack.running():
6e6e453 @apenwarr Some speedups for doing redo-ifchange on a large number of static files.
authored
269 state.commit()
f7734c3 @apenwarr Split build functions into builder.py.
authored
270 jwack.wait_all()
3209316 @apenwarr builder.py: now the only exported function is main().
authored
271 # at this point, we don't have any children holding any tokens, so
272 # it's okay to block below.
b937e62 @apenwarr Add a new -k (--keep-going) option, like make has.
authored
273 if retcode[0] and not vars.KEEP_GOING:
274 break
f7734c3 @apenwarr Split build functions into builder.py.
authored
275 if locked:
3ef2bd7 @apenwarr Don't check as often whether the .redo directory exists.
authored
276 if not state.check_sane():
277 err('.redo directory disappeared; cannot continue.\n')
dce0076 @apenwarr Print a useful message and exit when the .redo directory disappears.
authored
278 retcode[0] = 205
279 break
84169c5 @apenwarr Change locking stuff from fifos to fcntl.lockf().
authored
280 fid,t = locked.pop(0)
281 lock = state.Lock(fid)
f70c028 @apenwarr With --debug-locks, print a message when we stop to wait on a lock.
authored
282 lock.trylock()
c4be005 @apenwarr Release the jwack token when doing a synchronous lock wait.
authored
283 while not lock.owned:
16bebd2 @apenwarr builder: the (WAITING) message from --debug-locks didn't print every …
authored
284 if vars.DEBUG_LOCKS:
f70c028 @apenwarr With --debug-locks, print a message when we stop to wait on a lock.
authored
285 warn('%s (WAITING)\n' % _nice(t))
c4be005 @apenwarr Release the jwack token when doing a synchronous lock wait.
authored
286 # this sequence looks a little silly, but the idea is to
287 # give up our personal token while we wait for the lock to
288 # be released; but we should never run get_token() while
289 # holding a lock, or we could cause deadlocks.
290 jwack.release_mine()
f70c028 @apenwarr With --debug-locks, print a message when we stop to wait on a lock.
authored
291 lock.waitlock()
c4be005 @apenwarr Release the jwack token when doing a synchronous lock wait.
authored
292 lock.unlock()
293 jwack.get_token(t)
294 lock.trylock()
f7734c3 @apenwarr Split build functions into builder.py.
authored
295 assert(lock.owned)
03a054c @apenwarr Add a --debug-locks option.
authored
296 if vars.DEBUG_LOCKS:
297 log('%s (...unlocked!)\n' % _nice(t))
0126f6b @apenwarr Don't wipe the timestamp when a target fails to redo.
authored
298 if state.File(name=t).is_failed():
f644f3b @apenwarr Remove the need for relpath (and thus abspath) in builder.py.
authored
299 err('%s: failed in another thread\n' % _nice(t))
f7734c3 @apenwarr Split build functions into builder.py.
authored
300 retcode[0] = 2
301 lock.unlock()
302 else:
84169c5 @apenwarr Change locking stuff from fifos to fcntl.lockf().
authored
303 BuildJob(t, state.File(id=fid), lock,
304 shouldbuildfunc, done).start()
29d6c9a @apenwarr Don't db.commit() so frequently.
authored
305 state.commit()
f7734c3 @apenwarr Split build functions into builder.py.
authored
306 return retcode[0]
Something went wrong with that request. Please try again.