Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Newer
Older
100755 283 lines (239 sloc) 8.772 kB
6815943 @pv 3K: make setup.py call 2to3 automatically -- place sources to build/py3k
pv authored
1 #!/usr/bin/env python3
2 # -*- python -*-
3 """
4 %prog SUBMODULE...
5
6 Hack to pipe submodules of Numpy through 2to3 and build them in-place
7 one-by-one.
8
9 Example usage:
10
11 python3 tools/py3tool.py testing distutils core
12
13 This will copy files to _py3k/numpy, add a dummy __init__.py and
14 version.py on the top level, and copy and 2to3 the files of the three
15 submodules.
16
17 When running py3tool again, only changed files are re-processed, which
18 makes the test-bugfix cycle faster.
19
20 """
21 from optparse import OptionParser
22 import shutil
23 import os
24 import sys
25 import re
26 import subprocess
be838de @pv py3tool: customize 2to3 flags for some files
pv authored
27 import fnmatch
6815943 @pv 3K: make setup.py call 2to3 automatically -- place sources to build/py3k
pv authored
28
73b8b54 @pv ENH: 3K: optionally use lib2to3cache in 2to3 conversion
pv authored
29 if os.environ.get('USE_2TO3CACHE'):
30 import lib2to3cache
31
6815943 @pv 3K: make setup.py call 2to3 automatically -- place sources to build/py3k
pv authored
32 BASE = os.path.normpath(os.path.join(os.path.dirname(__file__), '..'))
33 TEMP = os.path.normpath(os.path.join(BASE, '_py3k'))
34
9f9098f @pv 3K: py3tool: use lib2to3 for the conversions
pv authored
35 SCRIPT_2TO3 = os.path.join(BASE, 'tools', '2to3.py')
36
be838de @pv py3tool: customize 2to3 flags for some files
pv authored
37 EXTRA_2TO3_FLAGS = {
38 '*/setup.py': '-x import',
39 'numpy/core/code_generators/generate_umath.py': '-x import',
40 'numpy/core/code_generators/generate_numpy_api.py': '-x import',
41 'numpy/core/code_generators/generate_ufunc_api.py': '-x import',
42 'numpy/core/defchararray.py': '-x unicode',
43 'numpy/compat/py3k.py': '-x unicode',
9f9098f @pv 3K: py3tool: use lib2to3 for the conversions
pv authored
44 'numpy/ma/timer_comparison.py': 'skip',
11b80e7 @pv 3K: BUG: work around bugs in Python 3.1.1 2to3 by not using fixes_reduce
pv authored
45 'numpy/distutils/system_info.py': '-x reduce',
46 'numpy/f2py/auxfuncs.py': '-x reduce',
47 'numpy/lib/arrayterator.py': '-x reduce',
48 'numpy/lib/tests/test_arrayterator.py': '-x reduce',
49 'numpy/ma/core.py': '-x reduce',
50 'numpy/ma/tests/test_core.py': '-x reduce',
51 'numpy/ma/tests/test_old_ma.py': '-x reduce',
52 'numpy/ma/timer_comparison.py': '-x reduce',
53 'numpy/oldnumeric/ma.py': '-x reduce',
be838de @pv py3tool: customize 2to3 flags for some files
pv authored
54 }
6815943 @pv 3K: make setup.py call 2to3 automatically -- place sources to build/py3k
pv authored
55
56 def main():
57 p = OptionParser(usage=__doc__.strip())
58 p.add_option("--clean", "-c", action="store_true",
59 help="clean source directory")
60 options, args = p.parse_args()
61
62 if not args:
63 p.error('no submodules given')
64 else:
65 dirs = ['numpy/%s' % x for x in map(os.path.basename, args)]
66
67 # Prepare
68 if not os.path.isdir(TEMP):
69 os.makedirs(TEMP)
70
71 # Set up dummy files (for building only submodules)
72 dummy_files = {
73 '__init__.py': 'from numpy.version import version as __version__',
74 'version.py': 'version = "1.4.0.dev"'
75 }
76
77 for fn, content in dummy_files.items():
78 fn = os.path.join(TEMP, 'numpy', fn)
79 if not os.path.isfile(fn):
80 try:
81 os.makedirs(os.path.dirname(fn))
82 except OSError:
83 pass
84 f = open(fn, 'wb+')
85 f.write(content.encode('ascii'))
86 f.close()
87
88 # Environment
89 pp = [os.path.abspath(TEMP)]
90 def getenv():
91 env = dict(os.environ)
92 env.update({'PYTHONPATH': ':'.join(pp)})
93 return env
94
95 # Copy
96 for d in dirs:
97 src = os.path.join(BASE, d)
98 dst = os.path.join(TEMP, d)
99
100 # Run 2to3
101 sync_2to3(dst=dst,
102 src=src,
103 patchfile=os.path.join(TEMP, os.path.basename(d) + '.patch'),
104 clean=options.clean)
105
106 # Run setup.py, falling back to Pdb post-mortem on exceptions
107 setup_py = os.path.join(dst, 'setup.py')
108 if os.path.isfile(setup_py):
109 code = """\
110 import pdb, sys, traceback
111 p = pdb.Pdb()
112 try:
113 import __main__
114 __main__.__dict__.update({
115 "__name__": "__main__", "__file__": "setup.py",
116 "__builtins__": __builtins__})
117 fp = open("setup.py", "rb")
118 try:
119 exec(compile(fp.read(), "setup.py", 'exec'))
120 finally:
121 fp.close()
122 except SystemExit:
123 raise
124 except:
125 traceback.print_exc()
126 t = sys.exc_info()[2]
127 p.interaction(None, t)
128 """
129 ret = subprocess.call([sys.executable, '-c', code,
130 'build_ext', '-i'],
131 cwd=dst,
132 env=getenv())
133 if ret != 0:
134 raise RuntimeError("Build failed.")
135
136 # Run nosetests
137 subprocess.call(['nosetests3', '-v', d], cwd=TEMP)
138
35523ea @pv py3tool: fix some relative imports for extension modules
pv authored
139 def custom_mangling(filename):
140 import_mangling = [
141 os.path.join('core', '__init__.py'),
142 os.path.join('core', 'numeric.py'),
143 os.path.join('core', '_internal.py'),
144 os.path.join('core', 'arrayprint.py'),
145 os.path.join('core', 'fromnumeric.py'),
146 os.path.join('numpy', '__init__.py'),
44118ae @charris Rename numpy/lib/io.py to numpy/lib/npyio.py. The py3tool can probabl…
charris authored
147 os.path.join('lib', 'npyio.py'),
35523ea @pv py3tool: fix some relative imports for extension modules
pv authored
148 os.path.join('lib', 'function_base.py'),
149 os.path.join('fft', 'fftpack.py'),
150 os.path.join('random', '__init__.py'),
151 ]
152
153 if any(filename.endswith(x) for x in import_mangling):
154 f = open(filename, 'r')
155 text = f.read()
156 f.close()
157 for mod in ['multiarray', 'scalarmath', 'umath', '_sort',
158 '_compiled_base', 'core', 'lib', 'testing', 'fft',
159 'polynomial', 'random', 'ma', 'linalg', 'compat',
6d49edc @pv BUG: Py3: ensure version.py imports are translated appropriately
pv authored
160 'mtrand', '_dotblas', 'version']:
35523ea @pv py3tool: fix some relative imports for extension modules
pv authored
161 text = re.sub(r'^(\s*)import %s' % mod,
162 r'\1from . import %s' % mod,
163 text, flags=re.M)
164 text = re.sub(r'^(\s*)from %s import' % mod,
165 r'\1from .%s import' % mod,
166 text, flags=re.M)
167 text = text.replace('from matrixlib', 'from .matrixlib')
168 f = open(filename, 'w')
169 f.write(text)
170 f.close()
171
6815943 @pv 3K: make setup.py call 2to3 automatically -- place sources to build/py3k
pv authored
172 def walk_sync(dir1, dir2, _seen=None):
173 if _seen is None:
174 seen = {}
175 else:
176 seen = _seen
177
178 if not dir1.endswith(os.path.sep):
179 dir1 = dir1 + os.path.sep
180
181 # Walk through stuff (which we haven't yet gone through) in dir1
182 for root, dirs, files in os.walk(dir1):
183 sub = root[len(dir1):]
184 if sub in seen:
185 dirs = [x for x in dirs if x not in seen[sub][0]]
186 files = [x for x in files if x not in seen[sub][1]]
187 seen[sub][0].extend(dirs)
188 seen[sub][1].extend(files)
189 else:
190 seen[sub] = (dirs, files)
191 if not dirs and not files:
192 continue
193 yield os.path.join(dir1, sub), os.path.join(dir2, sub), dirs, files
194
195 if _seen is None:
196 # Walk through stuff (which we haven't yet gone through) in dir2
197 for root2, root1, dirs, files in walk_sync(dir2, dir1, _seen=seen):
198 yield root1, root2, dirs, files
199
200 def sync_2to3(src, dst, patchfile=None, clean=False):
9f9098f @pv 3K: py3tool: use lib2to3 for the conversions
pv authored
201 import lib2to3.main
202 from io import StringIO
203
6815943 @pv 3K: make setup.py call 2to3 automatically -- place sources to build/py3k
pv authored
204 to_convert = []
205
206 for src_dir, dst_dir, dirs, files in walk_sync(src, dst):
207 for fn in dirs + files:
208 src_fn = os.path.join(src_dir, fn)
209 dst_fn = os.path.join(dst_dir, fn)
210
f5aefee @pv py3tool: skip temporary etc files
pv authored
211 # skip temporary etc. files
212 if fn.startswith('.#') or fn.endswith('~'):
213 continue
214
6815943 @pv 3K: make setup.py call 2to3 automatically -- place sources to build/py3k
pv authored
215 # remove non-existing
216 if os.path.exists(dst_fn) and not os.path.exists(src_fn):
217 if clean:
218 if os.path.isdir(dst_fn):
219 shutil.rmtree(dst_fn)
220 else:
221 os.unlink(dst_fn)
222 continue
223
224 # make directories
225 if os.path.isdir(src_fn):
226 if not os.path.isdir(dst_fn):
227 os.makedirs(dst_fn)
228 continue
229
230 dst_dir = os.path.dirname(dst_fn)
231 if os.path.isfile(dst_fn) and not os.path.isdir(dst_dir):
232 os.makedirs(dst_dir)
233
234 # don't replace up-to-date files
235 try:
236 if os.path.isfile(dst_fn) and \
237 os.stat(dst_fn).st_mtime >= os.stat(src_fn).st_mtime:
238 continue
239 except OSError:
240 pass
241
242 # copy file
243 shutil.copyfile(src_fn, dst_fn)
244
245 # add .py files to 2to3 list
246 if dst_fn.endswith('.py'):
be838de @pv py3tool: customize 2to3 flags for some files
pv authored
247 to_convert.append((src_fn, dst_fn))
6815943 @pv 3K: make setup.py call 2to3 automatically -- place sources to build/py3k
pv authored
248
249 # run 2to3
be838de @pv py3tool: customize 2to3 flags for some files
pv authored
250 flag_sets = {}
251 for fn, dst_fn in to_convert:
252 flag = ''
253 for pat, opt in EXTRA_2TO3_FLAGS.items():
254 if fnmatch.fnmatch(fn, pat):
255 flag = opt
256 break
257 flag_sets.setdefault(flag, []).append(dst_fn)
6815943 @pv 3K: make setup.py call 2to3 automatically -- place sources to build/py3k
pv authored
258
259 if patchfile:
260 p = open(patchfile, 'wb+')
261 else:
262 p = open(os.devnull, 'wb')
263
be838de @pv py3tool: customize 2to3 flags for some files
pv authored
264 for flags, filenames in flag_sets.items():
9f9098f @pv 3K: py3tool: use lib2to3 for the conversions
pv authored
265 if flags == 'skip':
266 continue
267
268 _old_stdout = sys.stdout
269 try:
270 sys.stdout = StringIO()
7ad8fef @rgommers ENH: do not make backup copies when running 2to3.
rgommers authored
271 lib2to3.main.main("lib2to3.fixes", ['-w', '-n'] + flags.split()+filenames)
9f9098f @pv 3K: py3tool: use lib2to3 for the conversions
pv authored
272 finally:
273 sys.stdout = _old_stdout
35523ea @pv py3tool: fix some relative imports for extension modules
pv authored
274
be838de @pv py3tool: customize 2to3 flags for some files
pv authored
275 for fn, dst_fn in to_convert:
35523ea @pv py3tool: fix some relative imports for extension modules
pv authored
276 # perform custom mangling
be838de @pv py3tool: customize 2to3 flags for some files
pv authored
277 custom_mangling(dst_fn)
35523ea @pv py3tool: fix some relative imports for extension modules
pv authored
278
6815943 @pv 3K: make setup.py call 2to3 automatically -- place sources to build/py3k
pv authored
279 p.close()
280
281 if __name__ == "__main__":
282 main()
Something went wrong with that request. Please try again.