/
do2to3.py
176 lines (164 loc) · 7.35 KB
/
do2to3.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
"""Helper script for building and installing Biopython on Python 3.
Note that we can't just use distutils.command.build_py function build_py_2to3
in setup.py since (as far as I can see) that does not allow us to alter the
2to3 options. In particular, we need to turn off the long fixer for some of
our files.
This code is intended to be called from setup.py automatically under Python 3,
and is not intended for end users. The basic idea follows the approach taken
by NumPy with their setup.py file calling tools/py3tool.py to do the 2to3
conversion automatically.
This calls the lib2to3 library functions to convert the Biopython source code
from Python 2 to Python 3, tracking changes to files so that unchanged files
need not be reconverted making development much easier (i.e. if you edit one
source file, doing 'python setup.py install' will only reconvert the one file).
This is done by the last modified date stamps (which will be updated by git if
you switch branches).
NOTE - This is intended to be run under Python 3 (not under Python 2), but
care has been taken to make it run under Python 2 enough to give a clear error
message. In particular, this meant avoiding with statements etc.
"""
import sys
if sys.version_info[0] < 3:
sys.stderr.write("Please run this under Python 3\n")
sys.exit(1)
import shutil
import os
import lib2to3.main
from io import StringIO
def run2to3(filenames):
stderr = sys.stderr
handle = StringIO()
try:
#Want to capture stderr (otherwise too noisy)
sys.stderr = handle
while filenames:
filename = filenames.pop(0)
print("Converting %s" % filename)
#TODO - Configurable options per file?
args = ["--nofix=long", "--no-diffs", "-n", "-w"]
e = lib2to3.main.main("lib2to3.fixes", args + [filename])
if e != 0:
sys.stderr = stderr
sys.stderr.write(handle.getvalue())
os.remove(filename) # Don't want a half edited file!
raise RuntimeError("Error %i from 2to3 on %s"
% (e, filename))
#And again for any doctests,
e = lib2to3.main.main("lib2to3.fixes", args + ["-d", filename])
if e != 0:
sys.stderr = stderr
sys.stderr.write(handle.getvalue())
os.remove(filename) # Don't want a half edited file!
raise RuntimeError("Error %i from 2to3 (doctests) on %s"
% (e, filename))
except KeyboardInterrupt:
sys.stderr = stderr
sys.stderr.write("Interrupted during %s\n" % filename)
os.remove(filename) # Don't want a half edited file!
for filename in filenames:
if os.path.isfile(filename):
#Don't want uncoverted files left behind:
os.remove(filename)
sys.exit(1)
finally:
#Restore stderr
sys.stderr = stderr
def do_update(py2folder, py3folder, verbose=False):
if not os.path.isdir(py2folder):
raise ValueError("Python 2 folder %r does not exist" % py2folder)
if not os.path.isdir(py3folder):
os.mkdir(py3folder)
#First remove any files from the 3to2 conversion which no
#longer existing the Python 2 origin (only expected to happen
#on a development machine).
for dirpath, dirnames, filenames in os.walk(py3folder):
relpath = os.path.relpath(dirpath, py3folder)
for d in dirnames:
new = os.path.join(py3folder, relpath, d)
old = os.path.join(py2folder, relpath, d)
if not os.path.isdir(old):
print("Removing %s" % new)
shutil.rmtree(new)
for f in filenames:
new = os.path.join(py3folder, relpath, f)
old = os.path.join(py2folder, relpath, f)
if not os.path.isfile(old):
print("Removing %s" % new)
os.remove(new)
#Check all the Python 2 original files have been copied/converted
#Note we need to do all the conversions *after* copying the files
#so that 2to3 can detect local imports successfully.
to_convert = []
for dirpath, dirnames, filenames in os.walk(py2folder):
if verbose:
print("Processing %s" % dirpath)
relpath = os.path.relpath(dirpath, py2folder)
#This is just to give cleaner filenames
if relpath[:2] == "/.":
relpath = relpath[2:]
elif relpath == ".":
relpath = ""
for d in dirnames:
new = os.path.join(py3folder, relpath, d)
if not os.path.isdir(new):
os.mkdir(new)
for f in filenames:
if f.startswith("."):
#Ignore hidden files
continue
elif f.endswith("~") or f.endswith(".bak") \
or f.endswith(".swp"):
#Ignore backup files
continue
elif f.endswith(".pyc") or f.endswith("$py.class"):
#Ignore compiled python
continue
old = os.path.join(py2folder, relpath, f)
new = os.path.join(py3folder, relpath, f)
#The filesystem can (in Linux) record nanoseconds, but
#when copying only microsecond accuracy is used.
#See http://bugs.python.org/issue10148
#Compare modified times down to milliseconds only. In theory
#might able to use times down to microseconds (10^-6), but
#that doesn't work on this Windows machine I'm testing on.
if os.path.isfile(new) and\
round(os.stat(new).st_mtime * 1000) >= \
round(os.stat(old).st_mtime * 1000):
if verbose:
print("Current: %s" % new)
continue
#Python, C code, data files, etc - copy with date stamp etc
shutil.copy2(old, new)
assert abs(os.stat(old).st_mtime - os.stat(new).st_mtime) < 0.0001, \
"Modified time not copied! %0.8f vs %0.8f, diff %f" \
% (os.stat(old).st_mtime, os.stat(new).st_mtime,
abs(os.stat(old).st_mtime - os.stat(new).st_mtime))
if f.endswith(".py"):
#Also run 2to3 on it
to_convert.append(new)
if verbose:
print("Will convert %s" % new)
else:
if verbose:
print("Updated %s" % new)
if to_convert:
print("Have %i python files to convert" % len(to_convert))
run2to3(to_convert)
def main(python2_source, python3_source,
children=["Bio", "BioSQL", "Tests", "Scripts", "Doc"]):
#Note want to use different folders for Python 3.1, 3.2, etc
#since the 2to3 libraries have changed so the conversion
#may differ slightly.
print("The 2to3 library will be called automatically now,")
print("and the converted files cached under %s" % python3_source)
if not os.path.isdir(python3_source):
os.mkdir(python3_source)
for child in children:
print("Processing %s" % child)
do_update(os.path.join(python2_source, child),
os.path.join(python3_source, child))
print("Python 2to3 processing done.")
if __name__ == "__main__":
python2_source = "."
python3_source = "build/py%i.%i" % sys.version_info[:2]
main(python2_source, python3_source)