address idna review comments, removing the idna package version cap and improve docs on implementation

Author: mahmoud

hyperlink/_url.py

@@ -579,6 +579,50 @@ def _percent_decode(text, normalize_case=False, subencoding='utf-8',
 
 
 def _decode_host(host):
+    """Decode a host from ASCII-encodable text to IDNA-decoded text. If
+    the host text is not ASCII, it is returned unchanged, as it is
+    presumed that it is already IDNA-decoded.
+
+    Some technical details: _decode_host is built on top of the "idna"
+    package, which has some quirks:
+
+    Capital letters are not valid IDNA2008. The idna package will
+    raise an exception like this on capital letters:
+
+    > idna.core.InvalidCodepoint: Codepoint U+004B at position 1 ... not allowed
+
+    However, if a segment of a host (i.e., something in
+    url.host.split('.')) is already ASCII, idna doesn't perform its
+    usual checks. In fact, for capital letters it automatically
+    lowercases them.
+
+    This check and some other functionality can be bypassed by passing
+    uts46=True to idna.encode/decode. This allows a more permissive and
+    convenient interface. So far it seems like the balanced approach.
+
+    Example output (from idna==2.6):
+
+    >> idna.encode(u'mahmöud.io')
+    'xn--mahmud-zxa.io'
+    >> idna.encode(u'Mahmöud.io')
+    Traceback (most recent call last):
+      File "<stdin>", line 1, in <module>
+      File "/home/mahmoud/virtualenvs/hyperlink/local/lib/python2.7/site-packages/idna/core.py", line 355, in encode
+        result.append(alabel(label))
+      File "/home/mahmoud/virtualenvs/hyperlink/local/lib/python2.7/site-packages/idna/core.py", line 276, in alabel
+        check_label(label)
+      File "/home/mahmoud/virtualenvs/hyperlink/local/lib/python2.7/site-packages/idna/core.py", line 253, in check_label
+        raise InvalidCodepoint('Codepoint {0} at position {1} of {2} not allowed'.format(_unot(cp_value), pos+1, repr(label)))
+    idna.core.InvalidCodepoint: Codepoint U+004D at position 1 of u'Mahm\xf6ud' not allowed
+    >> idna.encode(u'Mahmoud.io')
+    'Mahmoud.io'
+
+    # Similar behavior for decodes below
+    >> idna.decode(u'Mahmoud.io')
+    u'mahmoud.io
+    >> idna.decode(u'Méhmoud.io', uts46=True)
+    u'm\xe9hmoud.io'
+    """
     if not host:
         return u''
     try:
@@ -1802,38 +1846,3 @@ def parse(url, decoded=True, lazy=False):
         return enc_url
     dec_url = DecodedURL(enc_url, lazy=lazy)
     return dec_url
-
-"""idna package notes:
-
-* If a segment of a host (i.e., something in url.host.split('.')) is
-already ascii, idna doesn't perform its usual checks. For instance,
-capital letters are not valid idna2008. The package automatically lowercases.
-
-You'll get something like:
-
-> idna.core.InvalidCodepoint: Codepoint U+004B at position 1 ... not allowed
-
-This check and some other functionality can be bypassed by passing
-uts46=True to encode/decode. This allows a more permission and
-convenient interface. So far it seems like the balanced approach.
-
-However, all of this is bypassed if the string segment contains no
-unicode characters.
-
-Example output:
-
->>> idna.encode(u'mahmöud.io')
-'xn--mahmud-zxa.io'
->>> idna.encode(u'Mahmöud.io')
-Traceback (most recent call last):
-  File "<stdin>", line 1, in <module>
-  File "/home/mahmoud/virtualenvs/hyperlink/local/lib/python2.7/site-packages/idna/core.py", line 355, in encode
-    result.append(alabel(label))
-  File "/home/mahmoud/virtualenvs/hyperlink/local/lib/python2.7/site-packages/idna/core.py", line 276, in alabel
-    check_label(label)
-  File "/home/mahmoud/virtualenvs/hyperlink/local/lib/python2.7/site-packages/idna/core.py", line 253, in check_label
-    raise InvalidCodepoint('Codepoint {0} at position {1} of {2} not allowed'.format(_unot(cp_value), pos+1, repr(label)))
-idna.core.InvalidCodepoint: Codepoint U+004D at position 1 of u'Mahm\xf6ud' not allowed
->>> idna.encode(u'Mahmoud.io')
-'Mahmoud.io'
-"""

setup.py

@@ -11,15 +11,15 @@
 
 
 __author__ = 'Mahmoud Hashemi and Glyph Lefkowitz'
-__version__ = '17.3.2dev'
+__version__ = '18.0.0dev'
 __contact__ = 'mahmoud@hatnote.com'
 __url__ = 'https://github.com/python-hyper/hyperlink'
 __license__ = 'MIT'
 
 
 setup(name='hyperlink',
       version=__version__,
-      description="A featureful, correct URL for Python.",
+      description="A featureful, immutable, and correct URL for Python.",
       long_description=__doc__,
       author=__author__,
       author_email=__contact__,
@@ -29,7 +29,7 @@
       zip_safe=False,
       license=__license__,
       platforms='any',
-      install_requires=['idna>=2.5,<2.7'],
+      install_requires=['idna>=2.5'],
       classifiers=[
           'Topic :: Utilities',
           'Intended Audience :: Developers',

tox.ini

@@ -4,7 +4,7 @@ envlist = py26,py27,py34,py35,py36,pypy,coverage-report,packaging
 [testenv]
 changedir = .tox
 deps = -rrequirements-test.txt
-commands = coverage run --parallel --rcfile {toxinidir}/.tox-coveragerc -m pytest --doctest-modules {envsitepackagesdir}/hyperlink {posargs}
+commands = coverage run --parallel --omit 'flycheck__*' --rcfile {toxinidir}/.tox-coveragerc -m pytest --doctest-modules {envsitepackagesdir}/hyperlink {posargs}
 
 # Uses default basepython otherwise reporting doesn't work on Travis where
 # Python 3.6 is only available in 3.6 jobs.