forked from sympy/sympy
-
Notifications
You must be signed in to change notification settings - Fork 1
/
use2to3
executable file
·192 lines (155 loc) · 6.86 KB
/
use2to3
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
#!/usr/bin/env python3
"""
This script converts SymPy code to a Python 3-compatible version.
The script copies all files except the ones related to mpmath to a py3k-sympy
directory, runs 2to3 on them and then copies the vanilla mpmath files over. We
need this because running 2to3 on mpmath (which is already Python 3 compatible)
produces errors. You can then use SymPy normally from the py3k-sympy directory
(installing it or importing it directly).
Because copying and running 2to3 can take a lot of time, we try to do it only on
files that have been modified since the last run.
Note that the 2to3 shipped with Python 2.6 crashes when converting doctests. It
is recommended to use the Python 3.2 version (or newer) as it is much faster.
TODO: Add --destination argument (others?)
--destination # copy over the source to a user-specified destination
"""
import os
import re
import sys
import fnmatch
import shutil
import fileinput
from subprocess import check_output, CalledProcessError
destination = "py3k-sympy" # directory to copy to
np = os.path.normpath
modified_files = []
modified_rst_files = []
# we need to run 2to3 on .rst files; however, not all .rst files are doctests,
# so we need a list of files we care about
relevant_rst_files = []
skip_files = (
# this files produces doctest errors under py3k
# as we need it only for 2.5, just skip copying it
'sympy/parsing/ast_parser_python25.py',
'bin/use2to3',
)
try:
running_errors = (CalledProcessError, WindowsError)
except NameError:
# WindowsError is only defined in Windows
running_errors = (CalledProcessError,)
# generate the relevant rst files
# most of them should be in this directory:
for root, dirs, files in os.walk('doc/src/'):
# NOTE: this will consider mpmath-related files relevant, but it doesn't matter
for filename in fnmatch.filter(files, '*.rst'):
relevant_rst_files.append(os.path.join(root, filename))
# some files need 2to3, but don't have the .py suffix
relevant_no_extension = ('bin/doctest', 'bin/isympy', 'bin/test',
'bin/test_import', 'bin/test_isolated')
# Keep a record of destination files, used to remove any lingering files.
dst_files = set()
# ask git for a list of tracked files
try:
files = check_output(['git', 'ls-files'])
filenames = files.decode('utf-8').split('\n')
filenames.pop() # the last element is an empty string, delete it
for filename in skip_files:
filenames.remove(filename)
for filename in filenames:
src = filename
dst = os.path.normpath(os.path.join(destination, filename))
dst_files.add(dst)
dstdir = os.path.split(dst)[0]
if "mpmath" in dstdir:
continue
if not os.path.exists(dstdir):
os.makedirs(dstdir)
if os.path.isfile(dst):
if os.path.getmtime(src) - os.path.getmtime(dst) < 1:
# the file hasn't been modified since the last run, so skip it
# we check for one second of difference because Python can be
# imprecise (when copying) with smaller time periods
continue
shutil.copy2(src, dst)
# add to the list of files to pass to 2to3 if needed
if filename.endswith(".py"):
modified_files.append(dst)
elif filename.endswith(".rst"):
# we need to check the exact path here, not just the filename
# as there are eg. multiple index.rst files and not all are relevant
if np(src) in relevant_rst_files:
modified_rst_files.append(dst)
elif filename in relevant_no_extension:
modified_files.append(dst)
except running_errors:
print("git was not found: make sure it is in your PATH "
"environment or hardcode it into use2to3, changing\n"
" files = check_output(['git', 'ls-files'])\n"
"to something like the following (for Windows)\n"
" files = check_output(['C:/Program Files/Git/bin/git', 'ls-files'])\n"
"Exiting...")
exit(1)
# arguments to call 2to3 with
args_2to3 = [
"-w", # writes back the changes
"-n", # doesn't write a backup file
"--no-diffs", # don't show the diffs for individual files
]
args_2to3_doctests = args_2to3 + ["-d"] # convert doctests too
# extend the argument list with the list of files that need it
args_2to3.extend(modified_files)
args_2to3_doctests.extend(modified_files)
args_2to3_doctests.extend(modified_rst_files)
# call 2to3, once for regular files and once for doctests
from lib2to3.main import main as main2to3
if modified_files:
main2to3("lib2to3.fixes", args_2to3)
if modified_files or modified_rst_files:
main2to3("lib2to3.fixes", args_2to3_doctests)
# remove trailing spaced added to lines by 2to3
whitespace = re.compile(" +$", re.MULTILINE)
# remove multiple newlines at eof that were added by 2to3;
# \n\n is probably sufficient, but in case some spaces were
# slipped in before extra newlines, \s\s is used
eofnewlines = re.compile("\s\s+$")
for filepath in modified_files + modified_rst_files:
with open(filepath, 'r') as f:
text = f.read() or '\n' # all must end with newline
n = len(text)
text = whitespace.sub('', text)
text = eofnewlines.sub('\n', text)
if len(text) != n:
sys.stdout.write("Fixed trailing whitespace in %s\n" % filepath)
with open(filepath, 'w') as f:
f.write(text)
# Remove files that no longer exist
for root, dirs, files in os.walk(destination):
if "mpmath" in root:
continue
for file in files:
filepath = os.path.normpath(os.path.join(root, file))
if filepath not in dst_files:
sys.stdout.write("Removing lingering file %s\n" % filepath)
os.remove(filepath)
# once we are finished with everything, we should finally copy over the files
# provided by mpmath; these should all be in the following two directories
# to skip checking if something has been updated, just copy everything always
# the main bottleneck is running 2to3, not copying files
# TODO: only copy updated files; this would need a heavy modification to the
# above code, or copy-pasting the relevant part over
try:
shutil.rmtree(os.path.join(destination, "./sympy/mpmath"))
shutil.rmtree(os.path.join(destination, "./doc/src/modules/mpmath"))
except OSError: # directories don't exist
pass
shutil.copytree("sympy/mpmath", os.path.join(destination, "./sympy/mpmath"))
shutil.copytree("doc/src/modules/mpmath", os.path.join(destination, "./doc/src/modules/mpmath"))
if not modified_files:
exit() # fileinput doesn't work as expected when passed an empty list
# change the shebang lines to point to "python3"
for line in fileinput.input(modified_files, inplace=1):
if "#!/usr/bin/env python" in line:
line = line.replace("#!/usr/bin/env python", "#!/usr/bin/env python3")
# fileinput works by redirecting stdout to the file
sys.stdout.write(line)