Porting to Python 3, part 4 ("final")

.gitignore

@@ -8,6 +8,7 @@ MANIFEST
 my/
 dist/
 build/
+sympy-py3k/
 
 tox.ini
 .tox/

bin/use2to3

@@ -0,0 +1,130 @@
+#!/usr/bin/env python
+
+"""
+This script converts SymPy code to a Python 3-compatible version.
+
+The script copies all files except the ones related to mpmath to a sympy-py3k
+directory, runs 2to3 on them and then copies the vanilla mpmath files over. We
+need this because running 2to3 on mpmath (which is already Python 3 compatible)
+produces errors. You can then use SymPy normally from the sympy-py3k directory
+(installing it or importing it directly).
+
+Because copying and running 2to3 can take a lot of time, we try to do it only on
+files that have been modified since the last run.
+
+Note that the 2to3 shipped with Python 2.6 crashes when converting doctests. It
+is recommended to use the Python 3.2 version (or newer) as it is much faster.
+
+TODO: Add --destination argument (others?)
+ --destination # copy over the source to a user-specified destination
+"""
+import os
+import fnmatch
+import shutil
+
+destination = "sympy-py3k" # directory to copy to
+
+# TODO: build this from .gitignore
+skip_dirs = (
+    '.*',       # skip hidden dirs; .git and .tox in particular can be quite big
+    'mpmath',   # everything related to mpmath, both in doc/ and sympy/
+    '_build',   # files built by Sphinx
+    '__pycache__',
+    'covhtml',  # files produced by bin/test_coverage
+    'my',       # the user can have stuff here we don't want to copy
+    destination # this prevents infinite recursion if the dir already exists
+    )
+
+skip_files = (
+    '*.pyc',
+    '.*',
+    'ast_parser_python25.py', # this files produces doctest errors under py3k
+                              # as we need it only for 2.5, just skip copying it
+    )
+
+modified_files = []
+modified_txt_files = []
+
+# we need to run 2to3 on .txt files; however, not all .txt files are doctests,
+# so we need a list of files we care about
+relevant_txt_files = []
+
+# generate the relevant txt files
+# most of them should be in this directory:
+for root, dirs, files in os.walk('./doc/src/modules'):
+    # NOTE: this will consider mpmath-related files relevant, but it doesn't matter
+    for filename in fnmatch.filter(files, '*.txt'):
+        relevant_txt_files.append(os.path.join(root,filename))
+
+# some files aren't in /doc/src/modules, add them explicitly
+relevant_txt_files.append('./doc/src/tutorial.txt')
+relevant_txt_files.append('./doc/src/gotchas.txt')
+relevant_txt_files.append('./doc/src/guide.txt')
+relevant_txt_files.append('./doc/src/python-comparisons.txt')
+
+# walk the tree and copy over files as necessary
+for root, dirs, files in os.walk('.'):
+    for pattern in skip_dirs:
+        for directory in fnmatch.filter(dirs, pattern):
+            dirs.remove(directory)
+    for pattern in skip_files:
+        for filename in fnmatch.filter(files, pattern):
+            files.remove(filename)
+    for directory in dirs:
+        dstdir = os.path.join(destination, root, directory)
+        if not os.path.exists(dstdir):
+            os.makedirs(dstdir)
+    for filename in files:
+        src = os.path.join(root, filename)
+        dst = os.path.join(destination, root, filename)
+        if os.path.isfile(dst):
+            if os.path.getmtime(src) - os.path.getmtime(dst) < 1:
+                # the file hasn't been modified since the last run, so skip it
+                # we check for one second of difference because Python can be
+                # imprecise (when copying) with smaller time periods
+                continue
+        shutil.copy2(src, dst)
+        # add to the list of files to pass to 2to3 if needed
+        if filename.endswith(".py"):
+            modified_files.append(dst)
+        elif filename.endswith(".txt"):
+            # we need to check the exact path here, not just the filename
+            # as there are eg. multiple index.txt files and not all are relevant
+            if src in relevant_txt_files:
+                modified_txt_files.append(dst)
+
+
+# arguments to call 2to3 with
+args_2to3 = [
+    "-w",         # writes back the changes
+    "-n",         # doesn't write a backup file
+    "--no-diffs", # don't show the diffs for individual files
+]
+
+args_2to3_doctests = args_2to3 + ["-d"] # convert doctests too
+
+# extend the argument list with the list of files that need it
+args_2to3.extend(modified_files)
+args_2to3_doctests.extend(modified_files)
+args_2to3_doctests.extend(modified_txt_files)
+
+# call 2to3, once for regular files and once for doctests
+from lib2to3.main import main as main2to3
+main2to3("lib2to3.fixes", args_2to3)
+main2to3("lib2to3.fixes", args_2to3_doctests)
+
+# once we are finished with everything, we should finally copy over the files
+# provided by mpmath; these should all be in the following two directories
+
+# to skip checking if something has been updated, just copy everything always
+# the main bottleneck is running 2to3, not copying files
+# TODO: only copy updated files; this would need a heavy modification to the
+#       above code, or copy-pasting the relevant part over
+try:
+    shutil.rmtree(os.path.join(destination, "./sympy/mpmath"))
+    shutil.rmtree(os.path.join(destination, "./doc/src/modules/mpmath"))
+except OSError: # directories don't exist
+    pass
+
+shutil.copytree("sympy/mpmath", os.path.join(destination, "./sympy/mpmath"))
+shutil.copytree("doc/src/modules/mpmath", os.path.join(destination, "./doc/src/modules/mpmath"))

doc/src/gotchas.txt

@@ -279,7 +279,7 @@ to a Python expression.  Use the :func:`sympify` function, or just
     >>> 6.2 # Python float. Notice the floating point accuracy problems. #doctest: +SKIP
     6.2000000000000002
     >>> type(6.2)
-    <type 'float'>
+    <... 'float'>
     >>> S(6.2) # SymPy Float has no such problems because of arbitrary precision.
     6.20000000000000
     >>> type(S(6.2))

doc/src/modules/evalf.txt

@@ -46,10 +46,10 @@ numerically, calling ``.evalf()`` or ``N()`` returns the original expression, or
 You can also use the standard Python functions ``float()``, ``complex()`` to
 convert SymPy expressions to regular Python numbers:
 
-    >>> float(pi)
-    3.14159265359
-    >>> complex(pi+E*I)
-    (3.14159265359+2.71828182846j)
+    >>> float(pi) #doctest: +SKIP
+    3.1415926535...
+    >>> complex(pi+E*I) #doctest: +SKIP
+    (3.1415926535...+2.7182818284...j)
 
 
 If these functions are used, failure to evaluate the expression to an explicit
@@ -125,10 +125,10 @@ where φ is the golden ratio. With ordinary floating-point arithmetic,
 subtracting these numbers from each other erroneously results in a complete
 cancellation:
 
-    >>> float(GoldenRatio**1000/sqrt(5))
-    4.34665576869e+208
-    >>> float(fibonacci(1000))
-    4.34665576869e+208
+    >>> float(GoldenRatio**1000/sqrt(5)) #doctest: +SKIP
+    4.34665576869...e+208
+    >>> float(fibonacci(1000)) #doctest: +SKIP
+    4.34665576869...e+208
     >>> float(fibonacci(1000)) - float(GoldenRatio**1000/sqrt(5))
     0.0
 
@@ -255,9 +255,7 @@ Oscillatory quadrature requires an integrand containing a factor cos(ax+b) or
 sin(ax+b). Note that many other oscillatory integrals can be transformed to
 this form with a change of variables:
 
-    >>> from sympy import pprint
-    >>> import sys
-    >>> sys.displayhook = pprint
+    >>> init_printing(use_unicode=False, wrap_line=False, no_global=True)
     >>> intgrl = Integral(sin(1/x), (x, 0, 1)).transform(x, 1/x)
     >>> intgrl
      oo

doc/src/modules/integrals.txt

@@ -15,8 +15,7 @@ Examples
 SymPy can integrate a vast array of functions. It can integrate polynomial functions::
 
     >>> from sympy import *
-    >>> import sys
-    >>> sys.displayhook = pprint
+    >>> init_printing(use_unicode=False, wrap_line=False, no_global=True)
     >>> x = Symbol('x')
     >>> integrate(x**2 + x + 1, x)
      3    2

doc/src/modules/matrices.txt

@@ -9,9 +9,8 @@ Creating Matrices
 The linear algebra module is designed to be as simple as possible. First, we
 import and declare our first Matrix object:
 
-    >>> from sympy import pprint
-    >>> import sys
-    >>> sys.displayhook = pprint
+    >>> from sympy.interactive.printing import init_printing
+    >>> init_printing(use_unicode=False, wrap_line=False, no_global=True)
     >>> from sympy.matrices import *
     >>> Matrix([[1,0], [0,1]])
     [1  0]
@@ -217,13 +216,11 @@ All the standard arithmetic operations are supported:
 As well as some useful vector operations:
 
     >>> M.row_del(0)
-    None
     >>> M
     [4  5  6]
     [       ]
     [7  8  9]
     >>> M.col_del(1)
-    None
     >>> M
     [4  6]
     [    ]

doc/src/modules/plotting.txt

@@ -150,13 +150,12 @@ gray-scale because we have applied the default color gradient uniformly for
 each color component. When defining a color scheme in this way, you might want
 to supply a color gradient as well:
 
-    >>> p[1].color = (Fx**2 + Fy**2 + Fz**2)**(0.5),
-    ................ (0.1,0.1,0.9), (0.9,0.1,0.1)
+    >>> p[1].color = (Fx**2 + Fy**2 + Fz**2)**(0.5), (0.1,0.1,0.9), (0.9,0.1,0.1)
 
 Here's a color gradient with four steps:
 
     >>> gradient = [ 0.0, (0.1,0.1,0.9), 0.3, (0.1,0.9,0.1),
-    ................ 0.7, (0.9,0.9,0.1), 1.0, (1.0,0.0,0.0) ]
+    ...              0.7, (0.9,0.9,0.1), 1.0, (1.0,0.0,0.0) ]
     >>> p[1].color = (Fx**2 + Fy**2 + Fz**2)**(0.5), gradient
 
 The other way to specify a color scheme is to give a separate function for each

doc/src/modules/polys/basics.txt

@@ -12,9 +12,7 @@ polynomials within SymPy. All code examples assume::
 
     >>> from sympy import *
     >>> x, y, z = symbols('x,y,z')
-
-    >>> import sys
-    >>> sys.displayhook = pprint
+    >>> init_printing(use_unicode=False, wrap_line=False, no_global=True)
 
 Basic functionality
 ===================

doc/src/modules/statistics.txt

@@ -8,8 +8,7 @@ and related tools. Its contents can be imported with the following statement::
 
     >>> from sympy import *
     >>> from sympy.statistics import *
-    >>> import sys
-    >>> sys.displayhook = pprint
+    >>> init_printing(use_unicode=False, wrap_line=False, no_global=True)
 
 Normal distributions
 --------------------

doc/src/tutorial.txt

@@ -125,8 +125,8 @@ Using SymPy as a calculator
 
 Sympy has three built-in numeric types: Float, Rational and Integer.
 
-The Rational class represents a rational number as a pair of two Integers: the numerator
-and the denominator, so Rational(1,2) represents 1/2, Rational(5,2) 5/2 and so on.
+The Rational class represents a rational number as a pair of two Integers: the numerator and the denominator.
+So Rational(1,2) represents 1/2, Rational(5,2) represents 5/2, and so on.
 
 ::
 
@@ -143,23 +143,33 @@ and the denominator, so Rational(1,2) represents 1/2, Rational(5,2) 5/2 and so o
     1/88817841970012523233890533447265625
 
 
-proceed with caution while working with python int's since they truncate
-integer division, and that's why::
+Proceed with caution while working with Python int's and floating
+point numbers, especially in division, since you may create a
+Python number, not a SymPy number. A ratio of two Python ints may
+create a float -- the "true division" standard of Python 3
+and the default behavior of ``isympy`` which imports division
+from __future__::
 
-   >>> 1/2
-   0
-
-   >>> 1.0/2
-   0.5
+    >>> 1/2 #doctest: +SKIP
+    0.5
 
-You can however do::
+But in earlier Python versions where division has not been imported, a
+truncated int will result::
 
-  >>> from __future__ import division
+    >>> 1/2 #doctest: +SKIP
+    0
 
-  >>> 1/2 #doctest: +SKIP
-  0.5
+In both cases, however, you are not dealing with a SymPy Number because
+Python created its own number. Most of the time you will probably be
+working with Rational numbers, so make sure to use Rational to get
+the SymPy result. One might find it convenient to equate ``R`` and
+Rational:: 
 
-True division is going to be standard in python3k and ``isympy`` does that too.
+    >>> R = Rational
+    >>> R(1, 2)
+    1/2
+    >>> R(1)/2 # R(1) is a sympy Integer and Integer/int gives a Rational
+    1/2
 
 We also have some special constants, like e and pi, that are treated as symbols
 (1+pi won't evaluate to something numeric, rather it will remain as 1+pi), and
@@ -176,7 +186,7 @@ have arbitrary precision::
 
 as you see, evalf evaluates the expression to a floating-point number
 
-There is also a class representing mathematical infinity, called ``oo``::
+The symbol ``oo`` is used for a class defining mathematical infinity::
 
     >>> oo > 99999
     True
@@ -193,7 +203,9 @@ symbolic variables explicitly::
     >>> x = Symbol('x')
     >>> y = Symbol('y')
 
-Then you can play with them::
+On the left is the normal Python variable which has been assigned to the
+SymPy Symbol class. Instances of the Symbol class "play well together"
+and are the building blocks of expresions::
 
     >>> x+y+x-y
     2*x
@@ -204,23 +216,24 @@ Then you can play with them::
     >>> ((x+y)**2).expand()
     x**2 + 2*x*y + y**2
 
-And substitute them for other symbols or numbers using ``subs(old, new)``::
+They can be substituted with other numbers, symbols or expressions using ``subs(old, new)``::
 
     >>> ((x+y)**2).subs(x, 1)
     (y + 1)**2
 
     >>> ((x+y)**2).subs(x, y)
     4*y**2
 
+    >>> ((x+y)**2).subs(x, 1 - y)
+    1
+
 For the remainder of the tutorial, we assume that we have run::
 
-    >>> import sys
-    >>> oldhook = sys.displayhook
-    >>> sys.displayhook = pprint
+    >>> init_printing(use_unicode=False, wrap_line=False, no_global=True)
 
-So that things pretty print.  See the :ref:`printing-tutorial` section below.  If you have
-a unicode font installed, your output may look a little different (it will look slightly
-nicer).
+This will make things look better when printed. See the :ref:`printing-tutorial`
+section below. If you have a unicode font installed, you can pass
+use_unicode=True for a slightly nicer output.
 
 Algebra
 =======
@@ -729,7 +742,6 @@ This is what ``str(expression)`` returns and it looks like this:
 
 This is a nice ascii-art printing produced by a ``pprint`` function:
 
-    >>> sys.displayhook = oldhook
     >>> from sympy import Integral, pprint
     >>> from sympy.abc import x
     >>> pprint(x**2)
@@ -768,8 +780,7 @@ Tip: To make the pretty printing default in the python interpreter, use::
     [GCC 4.3.1] on linux2
     Type "help", "copyright", "credits" or "license" for more information.
     >>> from sympy import *
-    >>> import sys
-    >>> sys.displayhook = pprint
+    >>> init_printing(use_unicode=False, wrap_line=False, no_global=True)
     >>> var("x")
     x
     >>> x**3/3