Enable conservative basestring fixer from past.builtins (issues #127 …

…and #156)

docs/whatsnew.rst

@@ -37,6 +37,9 @@ Bug fixes:
 - Improve robustness of test suite against opening .pyc files as text on Py2
 - Update backports of ``Counter`` and ``OrderedDict`` to use the newer
   implementations from Py3.4. This fixes ``.copy()`` preserving subclasses etc.
+- ``futurize`` no longer breaks working Py2 code by changing ``basestring`` to
+  ``str``. Instead it imports the ``basestring`` forward-port from
+  ``past.builtins`` (issues #127 and #156)
 
 
 What's new in version 0.14.3 (2014-12-15)

src/libfuturize/fixes/__init__.py

@@ -40,7 +40,6 @@
 # The following fixers add a dependency on the ``future`` package on order to
 # support Python 2:
 lib2to3_fix_names_stage2 = set([
-    'lib2to3.fixes.fix_basestring',
     # 'lib2to3.fixes.fix_buffer',    # perhaps not safe. Test this.
     # 'lib2to3.fixes.fix_callable',  # not needed in Py3.2+
     'lib2to3.fixes.fix_dict',        # TODO: add support for utils.viewitems() etc. and move to stage2
@@ -79,6 +78,7 @@
 ])
 
 libfuturize_fix_names_stage2 = set([
+    'libfuturize.fixes.fix_basestring',
     # 'libfuturize.fixes.fix_add__future__imports_except_unicode_literals',  # just in case
     'libfuturize.fixes.fix_cmp',
     'libfuturize.fixes.fix_division_safe',

src/libfuturize/fixes/fix_basestring.py

@@ -0,0 +1,18 @@
+"""
+Fixer that adds ``from past.builtins import basestring`` if there is a
+reference to ``basestring``
+"""
+
+from lib2to3 import fixer_base
+
+from libfuturize.fixer_util import touch_import_top
+
+
+class FixBasestring(fixer_base.BaseFix):
+    BM_compatible = True
+
+    PATTERN = "'basestring'"
+
+    def transform(self, node, results):
+        touch_import_top(u'past.builtins', 'basestring', node)
+

tests/test_future/test_futurize.py

@@ -1144,14 +1144,10 @@ def test_range_necessary_list_calls(self):
         """
         self.convert_check(before, after)
 
-
-class TestConservativeFuturize(CodeHandler):
-    @unittest.expectedFailure
     def test_basestring(self):
         """
-        In conservative mode, futurize would not modify "basestring"
-        but merely import it, and the following code would still run on
-        both Py2 and Py3.
+        The 2to3 basestring fixer breaks working Py2 code that uses basestring.
+        This tests whether something sensible is done instead.
         """
         before = """
         assert isinstance('hello', basestring)
@@ -1164,41 +1160,7 @@ def test_basestring(self):
         assert isinstance(u'hello', basestring)
         assert isinstance(b'hello', basestring)
         """
-        self.convert_check(before, after, conservative=True)
-
-    @unittest.expectedFailure
-    def test_open(self):
-        """
-        In conservative mode, futurize would not import io.open because
-        this changes the default return type from bytes to text.
-        """
-        before = """
-        filename = 'temp_file_open.test'
-        contents = 'Temporary file contents. Delete me.'
-        with open(filename, 'w') as f:
-            f.write(contents)
-
-        with open(filename, 'r') as f:
-            data = f.read()
-        assert isinstance(data, str)
-        assert data == contents
-        """
-        after = """
-        from past.builtins import open, str as oldbytes, unicode
-        filename = oldbytes(b'temp_file_open.test')
-        contents = oldbytes(b'Temporary file contents. Delete me.')
-        with open(filename, oldbytes(b'w')) as f:
-            f.write(contents)
-
-        with open(filename, oldbytes(b'r')) as f:
-            data = f.read()
-        assert isinstance(data, oldbytes)
-        assert data == contents
-        assert isinstance(oldbytes(b'hello'), basestring)
-        assert isinstance(unicode(u'hello'), basestring)
-        assert isinstance(oldbytes(b'hello'), basestring)
-        """
-        self.convert_check(before, after, conservative=True)
+        self.convert_check(before, after)
 
     def test_safe_division(self):
         """
@@ -1255,6 +1217,80 @@ def __truediv__(self, other):
         """
         self.convert_check(before, after)
 
+    def test_basestring_issue_156(self):
+        before = """
+        x = str(3)
+        allowed_types = basestring, int
+        assert isinstance('', allowed_types)
+        assert isinstance(u'', allowed_types)
+        assert isinstance(u'foo', basestring)
+        """
+        after = """
+        from builtins import str
+        from past.builtins import basestring
+        x = str(3)
+        allowed_types = basestring, int
+        assert isinstance('', allowed_types)
+        assert isinstance(u'', allowed_types)
+        assert isinstance(u'foo', basestring)
+        """
+        self.convert_check(before, after)
+
+
+class TestConservativeFuturize(CodeHandler):
+    @unittest.expectedFailure
+    def test_basestring(self):
+        """
+        In conservative mode, futurize would not modify "basestring"
+        but merely import it from ``past``, and the following code would still
+        run on both Py2 and Py3.
+        """
+        before = """
+        assert isinstance('hello', basestring)
+        assert isinstance(u'hello', basestring)
+        assert isinstance(b'hello', basestring)
+        """
+        after = """
+        from past.builtins import basestring
+        assert isinstance('hello', basestring)
+        assert isinstance(u'hello', basestring)
+        assert isinstance(b'hello', basestring)
+        """
+        self.convert_check(before, after, conservative=True)
+
+    @unittest.expectedFailure
+    def test_open(self):
+        """
+        In conservative mode, futurize would not import io.open because
+        this changes the default return type from bytes to text.
+        """
+        before = """
+        filename = 'temp_file_open.test'
+        contents = 'Temporary file contents. Delete me.'
+        with open(filename, 'w') as f:
+            f.write(contents)
+
+        with open(filename, 'r') as f:
+            data = f.read()
+        assert isinstance(data, str)
+        assert data == contents
+        """
+        after = """
+        from past.builtins import open, str as oldbytes, unicode
+        filename = oldbytes(b'temp_file_open.test')
+        contents = oldbytes(b'Temporary file contents. Delete me.')
+        with open(filename, oldbytes(b'w')) as f:
+            f.write(contents)
+
+        with open(filename, oldbytes(b'r')) as f:
+            data = f.read()
+        assert isinstance(data, oldbytes)
+        assert data == contents
+        assert isinstance(oldbytes(b'hello'), basestring)
+        assert isinstance(unicode(u'hello'), basestring)
+        assert isinstance(oldbytes(b'hello'), basestring)
+        """
+        self.convert_check(before, after, conservative=True)
 
 class TestFuturizeAllImports(CodeHandler):
     """