Merge pull request #146 from rowillia/add_urllib_fixer

Add a fixer for urllib based off of 2to3

docs/fixers.rst

@@ -181,6 +181,18 @@ version of ``six`` is installed.
    Changes all reference of :func:`unicode <python2:unicode>` to
    :data:`six.text_type`.
 
+.. 2to3fixer:: urllib_six
+
+   Changes::
+
+       from urllib import quote_plus
+       quote_plus('hello world')
+
+   to::
+
+       from six.moves.urllib.parse import quote_plus
+       quote_plus('hello world')
+
 .. 2to3fixer:: xrange_six
 
    Changes::

libmodernize/fixes/__init__.py

@@ -39,6 +39,7 @@
     'libmodernize.fixes.fix_raise_six',
     'libmodernize.fixes.fix_unicode',
     'libmodernize.fixes.fix_unicode_type',
+    'libmodernize.fixes.fix_urllib_six',
     'libmodernize.fixes.fix_xrange_six',
     'libmodernize.fixes.fix_zip',
 ])

libmodernize/fixes/fix_urllib_six.py

@@ -0,0 +1,206 @@
+"""Fix changes imports of urllib which are now incompatible.
+   This is a copy of Lib/lib2to3/fixes/fix_urllib.py, but modified to point to the
+   six.moves locations for new libraries instead of the Python 3 locations.
+"""
+# This is a derived work of Lib/lib2to3/fixes/fix_urllib.py. That file
+# is under the copyright of the Python Software Foundation and licensed
+# under the Python Software Foundation License 2.
+#
+# Copyright notice:
+#
+#     Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
+#     2011, 2012, 2013 Python Software Foundation. All rights reserved.
+
+# Author: Nick Edds
+
+# Local imports
+from lib2to3.fixes.fix_imports import alternates, FixImports
+from lib2to3 import fixer_base
+from lib2to3.fixer_util import (Name, Comma, FromImport, Newline,
+                                find_indentation, Node, syms)
+
+MAPPING = {"urllib":  [
+                ("six.moves.urllib.request",
+                    ["URLopener", "FancyURLopener", "urlretrieve",
+                     "_urlopener", "urlopen", "urlcleanup",
+                     "pathname2url", "url2pathname"]),
+                ("six.moves.urllib.parse",
+                    ["quote", "quote_plus", "unquote", "unquote_plus",
+                     "urlencode", "splitattr", "splithost", "splitnport",
+                     "splitpasswd", "splitport", "splitquery", "splittag",
+                     "splittype", "splituser", "splitvalue", ]),
+                ("six.moves.urllib.error",
+                    ["ContentTooShortError"])],
+           "urllib2" : [
+                ("six.moves.urllib.request",
+                    ["urlopen", "install_opener", "build_opener",
+                     "Request", "OpenerDirector", "BaseHandler",
+                     "HTTPDefaultErrorHandler", "HTTPRedirectHandler",
+                     "HTTPCookieProcessor", "ProxyHandler",
+                     "HTTPPasswordMgr",
+                     "HTTPPasswordMgrWithDefaultRealm",
+                     "AbstractBasicAuthHandler",
+                     "HTTPBasicAuthHandler", "ProxyBasicAuthHandler",
+                     "AbstractDigestAuthHandler",
+                     "HTTPDigestAuthHandler", "ProxyDigestAuthHandler",
+                     "HTTPHandler", "HTTPSHandler", "FileHandler",
+                     "FTPHandler", "CacheFTPHandler",
+                     "UnknownHandler"]),
+                ("six.moves.urllib.error",
+                    ["URLError", "HTTPError"]),
+           ]
+}
+
+# Duplicate the url parsing functions for urllib2.
+MAPPING["urllib2"].append(MAPPING["urllib"][1])
+
+
+def build_pattern():
+    bare = set()
+    for old_module, changes in MAPPING.items():
+        for change in changes:
+            new_module, members = change
+            members = alternates(members)
+            yield """import_name< 'import' (module=%r
+                                  | dotted_as_names< any* module=%r any* >) >
+                  """ % (old_module, old_module)
+            yield """import_from< 'from' mod_member=%r 'import'
+                       ( member=%s | import_as_name< member=%s 'as' any > |
+                         import_as_names< members=any*  >) >
+                  """ % (old_module, members, members)
+            yield """import_from< 'from' module_star=%r 'import' star='*' >
+                  """ % old_module
+            yield """import_name< 'import'
+                                  dotted_as_name< module_as=%r 'as' any > >
+                  """ % old_module
+            # bare_with_attr has a special significance for FixImports.match().
+            yield """power< bare_with_attr=%r trailer< '.' member=%s > any* >
+                  """ % (old_module, members)
+
+
+class FixUrllibSix(FixImports):
+
+    def build_pattern(self):
+        return "|".join(build_pattern())
+
+    def transform_import(self, node, results):
+        """Transform for the basic import case. Replaces the old
+           import name with a comma separated list of its
+           replacements.
+        """
+        import_mod = results.get("module")
+        pref = import_mod.prefix
+
+        names = []
+
+        # create a Node list of the replacement modules
+        for name in MAPPING[import_mod.value][:-1]:
+            names.extend([Name(name[0], prefix=pref), Comma()])
+        names.append(Name(MAPPING[import_mod.value][-1][0], prefix=pref))
+        import_mod.replace(names)
+
+    def transform_member(self, node, results):
+        """Transform for imports of specific module elements. Replaces
+           the module to be imported from with the appropriate new
+           module.
+        """
+        mod_member = results.get("mod_member")
+        pref = mod_member.prefix
+        member = results.get("member")
+
+        # Simple case with only a single member being imported
+        if member:
+            # this may be a list of length one, or just a node
+            if isinstance(member, list):
+                member = member[0]
+            new_name = None
+            for change in MAPPING[mod_member.value]:
+                if member.value in change[1]:
+                    new_name = change[0]
+                    break
+            if new_name:
+                mod_member.replace(Name(new_name, prefix=pref))
+            else:
+                self.cannot_convert(node, "This is an invalid module element")
+
+        # Multiple members being imported
+        else:
+            # a dictionary for replacements, order matters
+            modules = []
+            mod_dict = {}
+            members = results["members"]
+            for member in members:
+                # we only care about the actual members
+                if member.type == syms.import_as_name:
+                    as_name = member.children[2].value
+                    member_name = member.children[0].value
+                else:
+                    member_name = member.value
+                    as_name = None
+                if member_name != u",":
+                    for change in MAPPING[mod_member.value]:
+                        if member_name in change[1]:
+                            if change[0] not in mod_dict:
+                                modules.append(change[0])
+                            mod_dict.setdefault(change[0], []).append(member)
+
+            new_nodes = []
+            indentation = find_indentation(node)
+            first = True
+            def handle_name(name, prefix):
+                if name.type == syms.import_as_name:
+                    kids = [Name(name.children[0].value, prefix=prefix),
+                            name.children[1].clone(),
+                            name.children[2].clone()]
+                    return [Node(syms.import_as_name, kids)]
+                return [Name(name.value, prefix=prefix)]
+            for module in modules:
+                elts = mod_dict[module]
+                names = []
+                for elt in elts[:-1]:
+                    names.extend(handle_name(elt, pref))
+                    names.append(Comma())
+                names.extend(handle_name(elts[-1], pref))
+                new = FromImport(module, names)
+                if not first or node.parent.prefix.endswith(indentation):
+                    new.prefix = indentation
+                new_nodes.append(new)
+                first = False
+            if new_nodes:
+                nodes = []
+                for new_node in new_nodes[:-1]:
+                    nodes.extend([new_node, Newline()])
+                nodes.append(new_nodes[-1])
+                node.replace(nodes)
+            else:
+                self.cannot_convert(node, "All module elements are invalid")
+
+    def transform_dot(self, node, results):
+        """Transform for calls to module members in code."""
+        module_dot = results.get("bare_with_attr")
+        member = results.get("member")
+        new_name = None
+        if isinstance(member, list):
+            member = member[0]
+        for change in MAPPING[module_dot.value]:
+            if member.value in change[1]:
+                new_name = change[0]
+                break
+        if new_name:
+            module_dot.replace(Name(new_name,
+                                    prefix=module_dot.prefix))
+        else:
+            self.cannot_convert(node, "This is an invalid module element")
+
+    def transform(self, node, results):
+        if results.get("module"):
+            self.transform_import(node, results)
+        elif results.get("mod_member"):
+            self.transform_member(node, results)
+        elif results.get("bare_with_attr"):
+            self.transform_dot(node, results)
+        # Renaming and star imports are not supported for these modules.
+        elif results.get("module_star"):
+            self.cannot_convert(node, "Cannot handle star imports.")
+        elif results.get("module_as"):
+            self.cannot_convert(node, "This module is now multiple modules")

tests/test_fix_urllib_six.py

@@ -0,0 +1,82 @@
+from __future__ import absolute_import
+
+from utils import check_on_input
+
+
+URLLIB_MODULE_REFERENCE = ("""\
+import urllib
+urllib.quote_plus('hello world')
+""", """\
+from __future__ import absolute_import
+import six.moves.urllib.request, six.moves.urllib.parse, six.moves.urllib.error
+six.moves.urllib.parse.quote_plus('hello world')
+""")
+
+
+URLLIB_FUNCTION_REFERENCE = ("""\
+from urllib2 import urlopen
+urlopen('https://www.python.org')
+""", """\
+from __future__ import absolute_import
+from six.moves.urllib.request import urlopen
+urlopen('https://www.python.org')
+""")
+
+
+URLLIB_MULTI_IMPORT_REFERENCE = ("""\
+from urllib2 import HTTPError, urlopen
+""", """\
+from __future__ import absolute_import
+from six.moves.urllib.error import HTTPError
+from six.moves.urllib.request import urlopen
+""")
+
+
+URLLIB_IMPORT_AS = ("""\
+from urllib2 import urlopen as urlo
+from urllib2 import HTTPError, URLError as urle
+""", """\
+from __future__ import absolute_import
+from six.moves.urllib.request import urlopen as urlo
+from six.moves.urllib.error import HTTPError, URLError as urle
+""")
+
+
+# Can't be converted; translation would emit a warning.
+URLIB_INVALID_CODE = ("""\
+from urllib2 import *
+from urllib2 import foobarraz
+from urllib2 import foo, bar as raz
+import urllib as urllib_py2
+import urllib
+urllib.foobarraz('hello world')
+""", """\
+from __future__ import absolute_import
+from urllib2 import *
+from urllib2 import foobarraz
+from urllib2 import foo, bar as raz
+import urllib as urllib_py2
+import six.moves.urllib.request, six.moves.urllib.parse, six.moves.urllib.error
+urllib.foobarraz('hello world')
+""")
+
+
+def test_urllib_module_reference():
+    check_on_input(*URLLIB_MODULE_REFERENCE)
+
+
+def test_urllib_function_reference():
+    check_on_input(*URLLIB_FUNCTION_REFERENCE)
+
+
+def test_urllib_multi_import():
+    check_on_input(*URLLIB_MULTI_IMPORT_REFERENCE)
+
+
+def test_urllib_import_as():
+    check_on_input(*URLLIB_IMPORT_AS)
+
+
+def test_urllib_invalid_imports():
+    check_on_input(*URLIB_INVALID_CODE)
+