Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100644 190 lines (168 sloc) 6.568 kb
135d1c1 apenwarr builder.py: set FD_CLOEXEC flag on $3 when running a .do file.
authored
1 import sys, os, random, fcntl
f7734c3 apenwarr Split build functions into builder.py.
authored
2 import vars, jwack, state
f644f3b apenwarr Remove the need for relpath (and thus abspath) in builder.py.
authored
3 from helpers import log, log_, debug2, err, unlink
f7734c3 apenwarr Split build functions into builder.py.
authored
4
5
6 def _possible_do_files(t):
7 yield "%s.do" % t, t, ''
8 dirname,filename = os.path.split(t)
9 l = filename.split('.')
10 l[0] = os.path.join(dirname, l[0])
11 for i in range(1,len(l)+1):
12 basename = '.'.join(l[:i])
13 ext = '.'.join(l[i:])
14 if ext: ext = '.' + ext
15 yield (os.path.join(dirname, "default%s.do" % ext),
16 os.path.join(dirname, basename), ext)
17
18
19 def _find_do_file(t):
20 for dofile,basename,ext in _possible_do_files(t):
21 debug2('%s: %s ?\n' % (t, dofile))
22 if os.path.exists(dofile):
23 state.add_dep(t, 'm', dofile)
24 return dofile,basename,ext
25 else:
26 state.add_dep(t, 'c', dofile)
27 return None,None,None
28
29
f644f3b apenwarr Remove the need for relpath (and thus abspath) in builder.py.
authored
30 def _nice(t):
31 return os.path.normpath(os.path.join(vars.PWD, t))
32
33
135d1c1 apenwarr builder.py: set FD_CLOEXEC flag on $3 when running a .do file.
authored
34 def _close_on_exec(fd, yes):
35 fl = fcntl.fcntl(fd, fcntl.F_GETFD)
36 fl &= ~fcntl.FD_CLOEXEC
37 if yes:
38 fl |= fcntl.FD_CLOEXEC
39 fcntl.fcntl(fd, fcntl.F_SETFD, fl)
40
41
3209316 apenwarr builder.py: now the only exported function is main().
authored
42 class BuildJob:
43 def __init__(self, t, lock, shouldbuildfunc, donefunc):
44 self.t = t
dcc2edb apenwarr builder.py: further refactoring to run more stuff in the parent process
authored
45 self.tmpname = '%s.redo.tmp' % t
3209316 apenwarr builder.py: now the only exported function is main().
authored
46 self.lock = lock
47 self.shouldbuildfunc = shouldbuildfunc
48 self.donefunc = donefunc
49
50 def start(self):
dcc2edb apenwarr builder.py: further refactoring to run more stuff in the parent process
authored
51 assert(self.lock.owned)
52 t = self.t
53 tmpname = self.tmpname
54 if not self.shouldbuildfunc(t):
3209316 apenwarr builder.py: now the only exported function is main().
authored
55 # target doesn't need to be built; skip the whole task
dcc2edb apenwarr builder.py: further refactoring to run more stuff in the parent process
authored
56 return self._after2(0)
57 if (os.path.exists(t) and not state.is_generated(t)
58 and not os.path.exists('%s.do' % t)):
59 # an existing source file that is not marked as a generated file.
60 # This step is mentioned by djb in his notes. It turns out to be
61 # important to prevent infinite recursion. For example, a rule
62 # called default.c.do could be used to try to produce hello.c,
63 # which is undesirable since hello.c existed already.
64 state.stamp(t)
65 return self._after2(0)
66 state.start(t)
67 (dofile, basename, ext) = _find_do_file(t)
68 if not dofile:
69 err('no rule to make %r\n' % t)
70 return self._after2(1)
71 state.stamp(dofile)
72 unlink(tmpname)
135d1c1 apenwarr builder.py: set FD_CLOEXEC flag on $3 when running a .do file.
authored
73 ffd = os.open(tmpname, os.O_CREAT|os.O_RDWR|os.O_EXCL)
74 _close_on_exec(ffd, True)
75 self.f = os.fdopen(ffd, 'w+')
dcc2edb apenwarr builder.py: further refactoring to run more stuff in the parent process
authored
76 # this will run in the dofile's directory, so use only basenames here
77 argv = ['sh', '-e',
78 os.path.basename(dofile),
79 os.path.basename(basename), # target name (extension removed)
80 ext, # extension (if any), including leading dot
81 os.path.basename(tmpname) # randomized output file name
82 ]
83 if vars.VERBOSE: argv[1] += 'v'
84 if vars.XTRACE: argv[1] += 'x'
85 if vars.VERBOSE or vars.XTRACE: log_('\n')
86 log('%s\n' % _nice(t))
87 self.argv = argv
88 jwack.start_job(t, self._do_subproc, self._after)
89
90 def _do_subproc(self):
8d0eba9 apenwarr builder.py: use os.exec() instead of subprocess.call().
authored
91 td = os.environ.get('REDO_PWD', '')
92 dn = os.path.dirname(self.t)
93 os.environ['REDO_PWD'] = os.path.join(td, dn)
94 os.environ['REDO_TARGET'] = os.path.basename(self.t)
95 os.environ['REDO_DEPTH'] = vars.DEPTH + ' '
96 if dn:
97 os.chdir(dn)
98 os.dup2(self.f.fileno(), 1)
99 os.close(self.f.fileno())
135d1c1 apenwarr builder.py: set FD_CLOEXEC flag on $3 when running a .do file.
authored
100 _close_on_exec(1, False)
8d0eba9 apenwarr builder.py: use os.exec() instead of subprocess.call().
authored
101 os.execvp(self.argv[0], self.argv)
135d1c1 apenwarr builder.py: set FD_CLOEXEC flag on $3 when running a .do file.
authored
102 assert(0)
8d0eba9 apenwarr builder.py: use os.exec() instead of subprocess.call().
authored
103 # returns only if there's an exception
dcc2edb apenwarr builder.py: further refactoring to run more stuff in the parent process
authored
104
105 def _after(self, t, rv):
106 f = self.f
107 tmpname = self.tmpname
108 if rv==0:
109 if os.path.exists(tmpname) and os.stat(tmpname).st_size:
110 # there's a race condition here, but if the tmpfile disappears
111 # at *this* point you deserve to get an error, because you're
112 # doing something totally scary.
113 os.rename(tmpname, t)
114 else:
115 unlink(tmpname)
116 state.stamp(t)
117 else:
118 unlink(tmpname)
119 state.unstamp(t)
120 f.close()
121 if rv != 0:
122 err('%s: exit code %d\n' % (_nice(t),rv))
123 else:
124 if vars.VERBOSE or vars.XTRACE:
125 log('%s (done)\n\n' % _nice(t))
126 return self._after2(rv)
3209316 apenwarr builder.py: now the only exported function is main().
authored
127
dcc2edb apenwarr builder.py: further refactoring to run more stuff in the parent process
authored
128 def _after2(self, rv):
129 self.donefunc(self.t, rv)
130 assert(self.lock.owned)
131 self.lock.unlock()
3209316 apenwarr builder.py: now the only exported function is main().
authored
132
133
134 def main(targets, shouldbuildfunc):
f7734c3 apenwarr Split build functions into builder.py.
authored
135 retcode = [0] # a list so that it can be reassigned from done()
136 if vars.SHUFFLE:
137 random.shuffle(targets)
138
139 locked = []
140
141 def done(t, rv):
142 if rv:
143 retcode[0] = 1
144
145 for i in range(len(targets)):
146 t = targets[i]
147 if os.path.exists('%s/all.do' % t):
148 # t is a directory, but it has a default target
149 targets[i] = '%s/all' % t
7aa7c41 apenwarr builder,jwack: slight cleanup to token passing.
authored
150
151 # In the first cycle, we just build as much as we can without worrying
152 # about any lock contention. If someone else has it locked, we move on.
f7734c3 apenwarr Split build functions into builder.py.
authored
153 for t in targets:
154 jwack.get_token(t)
b937e62 apenwarr Add a new -k (--keep-going) option, like make has.
authored
155 if retcode[0] and not vars.KEEP_GOING:
156 break
f7734c3 apenwarr Split build functions into builder.py.
authored
157 lock = state.Lock(t)
158 lock.trylock()
159 if not lock.owned:
03a054c apenwarr Add a --debug-locks option.
authored
160 if vars.DEBUG_LOCKS:
161 log('%s (locked...)\n' % _nice(t))
f7734c3 apenwarr Split build functions into builder.py.
authored
162 locked.append(t)
163 else:
dcc2edb apenwarr builder.py: further refactoring to run more stuff in the parent process
authored
164 BuildJob(t, lock, shouldbuildfunc, done).start()
7aa7c41 apenwarr builder,jwack: slight cleanup to token passing.
authored
165
166 # Now we've built all the "easy" ones. Go back and just wait on the
167 # remaining ones one by one. This is technically non-optimal; we could
168 # use select.select() to wait on more than one at a time. But it should
169 # be rare enough that it doesn't matter, and the logic is easier this way.
f7734c3 apenwarr Split build functions into builder.py.
authored
170 while locked or jwack.running():
171 jwack.wait_all()
3209316 apenwarr builder.py: now the only exported function is main().
authored
172 # at this point, we don't have any children holding any tokens, so
173 # it's okay to block below.
b937e62 apenwarr Add a new -k (--keep-going) option, like make has.
authored
174 if retcode[0] and not vars.KEEP_GOING:
175 break
f7734c3 apenwarr Split build functions into builder.py.
authored
176 if locked:
177 t = locked.pop(0)
178 lock = state.Lock(t)
7aa7c41 apenwarr builder,jwack: slight cleanup to token passing.
authored
179 lock.waitlock()
f7734c3 apenwarr Split build functions into builder.py.
authored
180 assert(lock.owned)
03a054c apenwarr Add a --debug-locks option.
authored
181 if vars.DEBUG_LOCKS:
182 log('%s (...unlocked!)\n' % _nice(t))
f7734c3 apenwarr Split build functions into builder.py.
authored
183 if state.stamped(t) == None:
f644f3b apenwarr Remove the need for relpath (and thus abspath) in builder.py.
authored
184 err('%s: failed in another thread\n' % _nice(t))
f7734c3 apenwarr Split build functions into builder.py.
authored
185 retcode[0] = 2
186 lock.unlock()
187 else:
dcc2edb apenwarr builder.py: further refactoring to run more stuff in the parent process
authored
188 BuildJob(t, lock, shouldbuildfunc, done).start()
f7734c3 apenwarr Split build functions into builder.py.
authored
189 return retcode[0]
Something went wrong with that request. Please try again.