Merge 3b70249 into baf9ef8

docs/modules/parsing.rst

@@ -10,31 +10,10 @@ Parsing Functions Reference
 
 .. autofunction:: sympy.parsing.sympy_parser.eval_expr
 
-.. autofunction:: sympy.parsing.sympy_tokenize.printtoken
-
-.. autofunction:: sympy.parsing.sympy_tokenize.tokenize
-
-.. autofunction:: sympy.parsing.sympy_tokenize.untokenize
-
-.. autofunction:: sympy.parsing.sympy_tokenize.generate_tokens
-
-.. autofunction:: sympy.parsing.sympy_tokenize.group
-
-.. autofunction:: sympy.parsing.sympy_tokenize.any
-
-.. autofunction:: sympy.parsing.sympy_tokenize.maybe
-
 .. autofunction:: sympy.parsing.maxima.parse_maxima
 
 .. autofunction:: sympy.parsing.mathematica.mathematica
 
-Parsing Exceptions Reference
-----------------------------
-
-.. autoclass:: sympy.parsing.sympy_tokenize.TokenError
-
-.. autoclass:: sympy.parsing.sympy_tokenize.StopTokenizing
-
 Parsing Transformations Reference
 ---------------------------------
 
@@ -66,8 +45,6 @@ These are included in
 :data:``sympy.parsing.sympy_parser.standard_transformations`` and generally
 don't need to be manually added by the user.
 
-.. autofunction:: sympy.parsing.sympy_parser.factorial_notation
-
 .. autofunction:: sympy.parsing.sympy_parser.auto_symbol
 
 .. autofunction:: sympy.parsing.sympy_parser.auto_number

sympy/core/compatibility.py

@@ -95,6 +95,8 @@ def u_decode(x):
     exec_=getattr(builtins, "exec")
 
     range=range
+
+    from tokenize import tokenize
 else:
     import codecs
     import types
@@ -141,6 +143,8 @@ def exec_(_code_, _globs_=None, _locs_=None):
         exec("exec _code_ in _globs_, _locs_")
     range=xrange
 
+    from tokenize import generate_tokens as tokenize
+
 
 def with_metaclass(meta, *bases):
     """

sympy/core/numbers.py

@@ -1099,8 +1099,6 @@ class Rational(Number):
     the sympify() function, and conversion of floats to expressions
     or simple fractions can be handled with nsimplify:
 
-    >>> S('.[3]')  # repeating digits in brackets
-    1/3
     >>> S('3**2/10')  # general expressions
     9/10
     >>> nsimplify(.3)  # numbers that have a simple form

sympy/core/tests/test_expr.py

@@ -1541,7 +1541,6 @@ def test_round():
     ans = S(d20).round(-2)
     assert ans.is_Float and ans == 12345678901234567900
     assert S('1/7').round(4) == 0.1429
-    assert S('.[12345]').round(4) == 0.1235
     assert S('.1349').round(2) == 0.13
     n = S(12345)
     ans = n.round()

sympy/core/tests/test_sympify.py

@@ -30,16 +30,7 @@ def test_sympify1():
     assert sympify('--.5') == n1
     assert sympify('-1/2') == -n1
     assert sympify('-+--.5') == -n1
-    assert sympify('-.[3]') == Rational(-1, 3)
-    assert sympify('.[3]') == Rational(1, 3)
-    assert sympify('+.[3]') == Rational(1, 3)
-    assert sympify('+0.[3]*10**-2') == Rational(1, 300)
-    assert sympify('.[052631578947368421]') == Rational(1, 19)
-    assert sympify('.0[526315789473684210]') == Rational(1, 19)
-    assert sympify('.034[56]') == Rational(1711, 49500)
     # options to make reals into rationals
-    assert sympify('1.22[345]', rational=True) == \
-        1 + Rational(22, 100) + Rational(345, 99900)
     assert sympify('2/2.6', rational=True) == Rational(10, 13)
     assert sympify('2.6/2', rational=True) == Rational(13, 10)
     assert sympify('2.6e2/17', rational=True) == Rational(260, 17)
@@ -59,8 +50,6 @@ def test_sympify1():
     # make sure longs in fractions work
     assert sympify('222222222222/11111111111') == \
         Rational(222222222222, 11111111111)
-    # ... even if they come from repetend notation
-    assert sympify('1/.2[123456789012]') == Rational(333333333333, 70781892967)
     # ... or from high precision reals
     assert sympify('.1234567890123456', rational=True) == \
         Rational(19290123283179, 156250000000000)
@@ -192,26 +181,6 @@ def test_sympify_poly():
     assert sympify(p) is p
 
 
-def test_sympify_factorial():
-    assert sympify('x!') == factorial(x)
-    assert sympify('(x+1)!') == factorial(x + 1)
-    assert sympify('(1 + y*(x + 1))!') == factorial(1 + y*(x + 1))
-    assert sympify('(1 + y*(x + 1)!)^2') == (1 + y*factorial(x + 1))**2
-    assert sympify('y*x!') == y*factorial(x)
-    assert sympify('x!!') == factorial2(x)
-    assert sympify('(x+1)!!') == factorial2(x + 1)
-    assert sympify('(1 + y*(x + 1))!!') == factorial2(1 + y*(x + 1))
-    assert sympify('(1 + y*(x + 1)!!)^2') == (1 + y*factorial2(x + 1))**2
-    assert sympify('y*x!!') == y*factorial2(x)
-    assert sympify('factorial2(x)!') == factorial(factorial2(x))
-
-    raises(SympifyError, lambda: sympify("+!!"))
-    raises(SympifyError, lambda: sympify(")!!"))
-    raises(SympifyError, lambda: sympify("!"))
-    raises(SympifyError, lambda: sympify("(!)"))
-    raises(SympifyError, lambda: sympify("x!!!"))
-
-
 def test_issue_3595():
     assert sympify("a_") == Symbol("a_")
     assert sympify("_a") == Symbol("_a")

sympy/parsing/sympy_parser.py

@@ -2,9 +2,9 @@
 
 from __future__ import print_function, division
 
-from .sympy_tokenize import \
-    generate_tokens, untokenize, TokenError, \
-    NUMBER, STRING, NAME, OP, ENDMARKER
+from sympy.core.compatibility import tokenize
+from tokenize import (untokenize, TokenError,
+                      NUMBER, STRING, NAME, OP, ENDMARKER)
 
 from keyword import iskeyword
 
@@ -13,11 +13,10 @@
 import unicodedata
 
 import sympy
+from io import BytesIO
 from sympy.core.compatibility import exec_, StringIO
 from sympy.core.basic import Basic
 
-_re_repeated = re.compile(r"^(\d*)\.(\d*)\[(\d+)\]$")
-
 
 def _token_splittable(token):
     """
@@ -52,34 +51,6 @@ def _token_callable(token, local_dict, global_dict, nextToken=None):
     return callable(func) and not isinstance(func, sympy.Symbol)
 
 
-def _add_factorial_tokens(name, result):
-    if result == [] or result[-1][1] == '(':
-        raise TokenError()
-
-    beginning = [(NAME, name), (OP, '(')]
-    end = [(OP, ')')]
-
-    diff = 0
-    length = len(result)
-
-    for index, token in enumerate(result[::-1]):
-        toknum, tokval = token
-        i = length - index - 1
-
-        if tokval == ')':
-            diff += 1
-        elif tokval == '(':
-            diff -= 1
-
-        if diff == 0:
-            if i - 1 >= 0 and result[i - 1][0] == NAME:
-                return result[:i - 1] + beginning + result[i - 1:] + end
-            else:
-                return result[:i] + beginning + result[i:] + end
-
-    return result
-
-
 class AppliedFunction(object):
     """
     A group of tokens representing a function and its arguments.
@@ -558,6 +529,9 @@ def lambda_notation(tokens, local_dict, global_dict):
     """
     result = []
     flag = False
+    if len(tokens) > 1:
+        if tokens[0][1] == 'utf-8' and tokens[1] == (NAME, 'lambda'):
+            tokens = tokens[1:]
     toknum, tokval = tokens[0]
     tokLen = len(tokens)
     if toknum == NAME and tokval == 'lambda':
@@ -585,32 +559,6 @@ def lambda_notation(tokens, local_dict, global_dict):
     return result
 
 
-def factorial_notation(tokens, local_dict, global_dict):
-    """Allows standard notation for factorial."""
-    result = []
-    prevtoken = ''
-    for toknum, tokval in tokens:
-        if toknum == OP:
-            op = tokval
-
-            if op == '!!':
-                if prevtoken == '!' or prevtoken == '!!':
-                    raise TokenError
-                result = _add_factorial_tokens('factorial2', result)
-            elif op == '!':
-                if prevtoken == '!' or prevtoken == '!!':
-                    raise TokenError
-                result = _add_factorial_tokens('factorial', result)
-            else:
-                result.append((OP, op))
-        else:
-            result.append((toknum, tokval))
-
-        prevtoken = tokval
-
-    return result
-
-
 def convert_xor(tokens, local_dict, global_dict):
     """Treats XOR, ``^``, as exponentiation, ``**``."""
     result = []
@@ -647,35 +595,8 @@ def auto_number(tokens, local_dict, global_dict):
 
             if '.' in number or (('e' in number or 'E' in number) and
                     not (number.startswith('0x') or number.startswith('0X'))):
-                match = _re_repeated.match(number)
-
-                if match is not None:
-                    # Clear repeating decimals, e.g. 3.4[31] -> (3 + 4/10 + 31/990)
-                    pre, post, repetend = match.groups()
-
-                    zeros = '0'*len(post)
-                    post, repetends = [w.lstrip('0') for w in [post, repetend]]
-                                                # or else interpreted as octal
-
-                    a = pre or '0'
-                    b, c = post or '0', '1' + zeros
-                    d, e = repetends, ('9'*len(repetend)) + zeros
-
-                    seq = [
-                        (OP, '('),
-                        (NAME,
-                         'Integer'), (OP, '('), (NUMBER, a), (OP, ')'),
-                        (OP, '+'),
-                        (NAME, 'Rational'), (OP, '('), (
-                            NUMBER, b), (OP, ','), (NUMBER, c), (OP, ')'),
-                        (OP, '+'),
-                        (NAME, 'Rational'), (OP, '('), (
-                            NUMBER, d), (OP, ','), (NUMBER, e), (OP, ')'),
-                        (OP, ')'),
-                    ]
-                else:
-                    seq = [(NAME, 'Float'), (OP, '('),
-                           (NUMBER, repr(str(number))), (OP, ')')]
+                seq = [(NAME, 'Float'), (OP, '('),
+                       (NUMBER, repr(str(number))), (OP, ')')]
             else:
                 seq = [(NAME, 'Integer'), (OP, '('), (
                     NUMBER, number), (OP, ')')]
@@ -707,9 +628,8 @@ def rationalize(tokens, local_dict, global_dict):
 
 
 #: Standard transformations for :func:`~sympy.parsing.sympy_parser.parse_expr`.
-#: Inserts calls to :class:`~sympy.core.symbol.Symbol`, :class:`~sympy.core.numbers.Integer`, and other SymPy
-#: datatypes and allows the use of standard factorial notation (e.g. ``x!``).
-standard_transformations = (lambda_notation, auto_symbol, auto_number, factorial_notation)
+#: Inserts calls to :class:`~sympy.core.symbol.Symbol`, :class:`~sympy.core.numbers.Integer`, and other SymPy datatypes.
+standard_transformations = (lambda_notation, auto_symbol, auto_number)
 
 
 def stringify_expr(s, local_dict, global_dict, transformations):
@@ -720,8 +640,8 @@ def stringify_expr(s, local_dict, global_dict, transformations):
     """
 
     tokens = []
-    input_code = StringIO(s.strip())
-    for toknum, tokval, _, _, _ in generate_tokens(input_code.readline):
+    input_code = BytesIO(s.encode('utf-8').strip())
+    for toknum, tokval, _, _, _ in tokenize(input_code.readline):
         tokens.append((toknum, tokval))
 
     for transform in transformations:
@@ -764,8 +684,7 @@ def parse_expr(s, local_dict=None, transformations=standard_transformations,
         A tuple of transformation functions used to modify the tokens of the
         parsed expression before evaluation. The default transformations
         convert numeric literals into their SymPy equivalents, convert
-        undefined variables into SymPy symbols, and allow the use of standard
-        mathematical factorial notation (e.g. ``x!``).
+        undefined variables into SymPy symbols.
 
     evaluate : bool, optional
         When False, the order of the arguments will remain as they were in the