Merge branch 'release-6.0.0' into master

.gitignore

@@ -8,3 +8,5 @@ fpdocs/
 venv/
 build/
 *.egg-info/
+.doit.*
+*.html

.travis.yml

@@ -1,23 +1,16 @@
+os: linux
+dist: focal
 language: python
-
-sudo: false
-
-matrix:
+jobs:
   fast_finish: true
   include:
-    - python: "3.5"
-      env: TOXENV=py35
     - python: "3.6"
       env: TOXENV=py36
     - python: "3.7"
       env: TOXENV=py37
-      dist: xenial
-      sudo: true
     - python: "3.8"
       env: TOXENV=py38
-      dist: xenial
-      sudo: true
-
-
+    - python: "3.9-dev"
+      env: TOXENV=py39
 install: pip install -U tox coveralls
 script: travis_wait 30 tox

NEWS

@@ -1,6 +1,8 @@
 coming in the next release:
-    * Support Python 3.5, 3.6, 3.7 and 3.8
-    * Drop support for Python 2.4 through 2.7, and Python 3.0 through 3.4 (#169)
+
+6.0.0 - 12 September 2020
+    * Support Python 3.6, 3.7, 3.8 and 3.9
+    * Drop support for Python 2.4 through 2.7, and Python 3.0 through 3.5 (#169)
     * Convert feedparser from a monolithic file to a package
     * ``feedparser.parse(sanitize_html=bool)`` argument replaces the ``feedparser.SANITIZE_HTML`` global
     * ``feedparser.parse(resolve_relative_uris=bool)`` replaces the ``feedparser.RESOLVE_RELATIVE_URIS`` global
@@ -13,6 +15,12 @@ coming in the next release:
     * Document that datetimes are returned in UTC (#51)
     * Remove cjkpython references in the documentation (#57)
     * Resolve ResourceWarnings thrown during unit tests (#170)
+    * Fix tox build failures (#213)
+    * Use ``base64.decodebytes()`` directly to support Python 3.9 (#201)
+    * Fix Python 3.8 ``urllib.parse.splittype()`` deprecation warning (#211)
+    * Support parsing colons in RFC822 timezones (#144)
+    * Add `chardet` as an optional tox environment dependency
+    * Fix the Big5 unit test that fails when chardet is installed (#184)
 
 5.2.1 - July 23, 2015
     * Fix #22 (pip package keeps upgrading all the time)

README.rst

@@ -45,8 +45,8 @@ Feedparser has an extensive test suite, powered by tox. To run it, type this:
 ..  code-block:: shell
 
     $ python -m venv venv
-    $ source venv/bin/activate  # or "venv\bin\activate.bat" on Windows
-    (venv) $ pip install tox
+    $ source venv/bin/activate  # or "venv\bin\activate.ps1" on Windows
+    (venv) $ pip install -r requirements-dev.txt
     (venv) $ tox
 
 This will spawn an HTTP server that will listen on port 8097. The tests will

docs/add_custom_css.py

@@ -1,3 +1,3 @@
 # Makes Sphinx create a <link> to feedparser.css in the HTML output
 def setup(app):
-    app.add_stylesheet('feedparser.css')
+    app.add_css_file('feedparser.css')

docs/conf.py

@@ -1,16 +1,18 @@
 import os
 import pathlib
+import re
 import sys
 
-sys.path.append(str(pathlib.Path(__file__).parent.parent))
-import feedparser
+
+content = (pathlib.Path(__file__).parent.parent / 'feedparser/__init__.py').read_text()
+match = re.search(r"""__version__ = ['"](?P<version>.+?['"])""", content)
+version = match.group('version')
+release = version
 
 # project information
-project = u'feedparser'
-copyright = u'2010-2020 Kurt McKee, 2004-2008 Mark Pilgrim'
-version = feedparser.__version__
-release = feedparser.__version__
-language = u'en'
+project = 'feedparser'
+copyright = '2010-2020 Kurt McKee, 2004-2008 Mark Pilgrim'
+language = 'en'
 
 # documentation options
 master_doc = 'index'

docs/introduction.rst

@@ -11,7 +11,7 @@ and :abbr:`CDF (Channel Definition Format)` feeds.  It also parses several
 popular extension modules, including Dublin Core and Apple's :program:`iTunes`
 extensions.
 
-To use :program:`Universal Feed Parser`, you will need :program:`Python` 3.5 or
+To use :program:`Universal Feed Parser`, you will need :program:`Python` 3.6 or
 later. :program:`Universal Feed Parser` is not meant
 to run standalone; it is a module for you to use as part of a larger
 :program:`Python` program.

dodo.py

@@ -0,0 +1,127 @@
+# This file is part of feedparser.
+# Copyright 2020 Kurt McKee <contactme@kurtmckee.org>
+# Released under the BSD 2-clause license.
+
+# The tasks defined in this file automates the entire
+# development-to-release process.
+
+import os
+import pathlib
+import random
+import subprocess
+import platform
+
+import colorama
+import docutils.core
+import doit.action
+
+
+# Initialize colorama so that tox can output ANSI escape codes.
+colorama.init()
+
+DOIT_CONFIG = {'default_tasks': ['build', 'test']}
+PROJECT = 'feedparser'
+
+root = pathlib.Path(__file__).parent
+
+
+def task_build():
+    """Build the documentation.
+
+    The documentation will be converted to HTML files to help double-check
+    syntax and formatting on PyPI and on GitHub. Note that the HTML files
+    will not be included in the distribution files.
+    """
+
+    def build_single_files():
+        docutils.core.publish_cmdline(writer_name='html', argv=['README.rst', 'README.html'])
+
+    return {
+        'actions': [
+            build_single_files,
+            'sphinx-build -b html docs/ fpdocs',
+        ],
+        'verbosity': 2,
+        'file_dep': [root / 'README.rst'] + list((root / 'docs').rglob('*.rst')),
+        'targets': [root / 'README.html'],
+    }
+
+
+def task_test():
+    """Run the unit tests."""
+
+    env = {k: v for k, v in os.environ.items()}
+    env.update({
+        'PY_COLORS': '1',
+    })
+
+    return {
+        'actions': [
+            doit.action.CmdAction('tox', env=env),
+        ],
+        'verbosity': 2,
+    }
+
+
+def remove_dist_files():
+    """Erase existing files in the ``dist`` directory."""
+
+    for file in (root / 'dist/').glob('*'):
+        file.unlink()
+
+
+def open_browser(url):
+    """Open *url* in the default browser on Windows or Linux."""
+
+    if platform.system() == 'Windows':
+        subprocess.check_output(['start', url], shell=True)
+    elif platform.system() == 'Linux':
+        subprocess.check_output(['xdg-open', url], shell=True)
+
+
+def task_test_release():
+    """Upload to test.pypi.org."""
+
+    # Generate random suffixes to help prevent name and version conflicts
+    # on PyPI. These environment variables are used in `setup.py`.
+    env = {k: v for k, v in os.environ.items()}
+    env.update({
+        'NAME_SUFFIX': ''.join(chr(i) for i in random.sample(range(0x61, 0x61+26), 10)),
+        'VERSION_SUFFIX': str(random.choice(range(1, 1000))),
+    })
+
+    return {
+        'actions': [
+            remove_dist_files,
+            doit.action.CmdAction('python setup.py sdist bdist_wheel', env=env),
+            f'twine upload --repository testpypi dist/*{env["NAME_SUFFIX"]}*',
+            (open_browser, [f'https://test.pypi.org/project/{PROJECT}_{env["NAME_SUFFIX"]}']),
+        ],
+        'verbosity': 2,
+    }
+
+
+def validate_in_git_master_branch():
+    """Validate that the repository is in the git master branch."""
+
+    branch = subprocess.check_output('git rev-parse --abbrev-ref HEAD', shell=True)
+    return branch.decode('utf8', errors='ignore').strip() == 'master'
+
+
+def task_release():
+    """Upload to pypi.org.
+
+    This step must *always* be taken while in the git master branch.
+    This is an enforced requirement.
+    """
+
+    return {
+        'actions': [
+            validate_in_git_master_branch,
+            remove_dist_files,
+            'python setup.py sdist bdist_wheel',
+            'twine upload dist/*',
+            (open_browser, [f'https://pypi.org/project/{PROJECT}']),
+        ],
+        'verbosity': 2,
+    }

feedparser/__init__.py

@@ -35,7 +35,7 @@
 
 __author__ = 'Kurt McKee <contactme@kurtmckee.org>'
 __license__ = 'BSD 2-clause'
-__version__ = '6.0.0b3'
+__version__ = '6.0.0'
 
 # HTTP "User-Agent" header to send to servers when downloading feeds.
 # If you are embedding feedparser in a larger application, you should

feedparser/datetimes/rfc822.py

@@ -123,8 +123,12 @@ def _parse_date_rfc822(date):
     # Handle timezones like '-0500', '+0500', and 'EST'
     if parts[4] and parts[4][0] in ('-', '+'):
         try:
-            timezone_hours = int(parts[4][1:3])
-            timezone_minutes = int(parts[4][3:])
+            if ':' in parts[4]:
+                timezone_hours = int(parts[4][1:3])
+                timezone_minutes = int(parts[4][4:])
+            else:
+                timezone_hours = int(parts[4][1:3])
+                timezone_minutes = int(parts[4][3:])
         except ValueError:
             return None
         if parts[4].startswith('-'):

feedparser/http.py

@@ -34,47 +34,14 @@
 import struct
 import zlib
 
-try:
-    import urllib.parse
-    import urllib.request
-except ImportError:
-    # Mock urllib sufficiently to work on Python 2.7
-    from urllib import splithost, splittype, splituser
-    from urllib2 import build_opener, HTTPDigestAuthHandler, HTTPRedirectHandler, HTTPDefaultErrorHandler, Request
-    from urlparse import urlparse
-
-    class urllib(object):
-        class parse(object):
-            splithost = staticmethod(splithost)
-            splittype = staticmethod(splittype)
-            splituser = staticmethod(splituser)
-            urlparse = staticmethod(urlparse)
-
-        class request(object):
-            build_opener = staticmethod(build_opener)
-            HTTPDigestAuthHandler = HTTPDigestAuthHandler
-            HTTPRedirectHandler = HTTPRedirectHandler
-            HTTPDefaultErrorHandler = HTTPDefaultErrorHandler
-            Request = Request
-
-try:
-    from io import BytesIO as _StringIO
-except ImportError:
-    # Python 2.7
-    try:
-        from cStringIO import StringIO as _StringIO
-    except ImportError:
-        from StringIO import StringIO as _StringIO
-
 import base64
+from io import BytesIO as _StringIO
+import urllib.parse
+import urllib.request
 
 from .datetimes import _parse_date
 from .urls import convert_to_idn
 
-# Python 3.1 deprecated decodestring in favor of decodebytes.
-# This can be removed after Python 2.7 support is dropped.
-_base64decode = getattr(base64, 'decodebytes', base64.decodestring)
-
 try:
     basestring
 except NameError:
@@ -121,7 +88,7 @@ def http_error_401(self, req, fp, code, msg, headers):
         host = urllib.parse.urlparse(req.get_full_url())[1]
         if 'Authorization' not in req.headers or 'WWW-Authenticate' not in headers:
             return self.http_error_default(req, fp, code, msg, headers)
-        auth = _base64decode(req.headers['Authorization'].split(' ')[1])
+        auth = base64.decodebytes(req.headers['Authorization'].split(' ')[1].encode('utf8'))
         user, passw = auth.split(':')
         realm = re.findall('realm="([^"]*)"', headers['WWW-Authenticate'])[0]
         self.add_password(realm, host, user, passw)
@@ -181,13 +148,14 @@ def get(url, etag=None, modified=None, agent=None, referrer=None, handlers=None,
     # Test for inline user:password credentials for HTTP basic auth
     auth = None
     if not url.startswith('ftp:'):
-        urltype, rest = urllib.parse.splittype(url)
-        realhost, rest = urllib.parse.splithost(rest)
-        if realhost:
-            user_passwd, realhost = urllib.parse.splituser(realhost)
-            if user_passwd:
-                url = '%s://%s%s' % (urltype, realhost, rest)
-                auth = base64.standard_b64encode(user_passwd).strip()
+        url_pieces = urllib.parse.urlparse(url)
+        if url_pieces.username:
+            new_pieces = list(url_pieces)
+            new_pieces[1] = url_pieces.hostname
+            if url_pieces.port:
+                new_pieces[1] = f'{url_pieces.hostname}:{url_pieces.port}'
+            url = urllib.parse.urlunparse(new_pieces)
+            auth = base64.standard_b64encode(f'{url_pieces.username}:{url_pieces.password}').strip()
 
     # iri support
     if not isinstance(url, bytes_):

feedparser/mixin.py

@@ -48,11 +48,6 @@
 from .urls import _urljoin, make_safe_absolute_uri, resolve_relative_uris
 
 
-# Python 2.7 only offers "decodestring()".
-# This name substitution can be removed when Python 2.7 support is dropped.
-_base64decode = getattr(base64, 'decodebytes', base64.decodestring)
-
-
 bytes_ = type(b'')
 try:
     # Python 2
@@ -528,15 +523,11 @@ def pop(self, element, strip_whitespace=1):
         # decode base64 content
         if base64 and self.contentparams.get('base64', 0):
             try:
-                output = _base64decode(output)
+                output = base64.decodebytes(output.encode('utf8')).decode('utf8')
             except binascii.Error:
                 pass
             except binascii.Incomplete:
                 pass
-            except TypeError:
-                # In Python 3, base64 takes and outputs bytes, not str
-                # This may not be the most correct way to accomplish this
-                output = _base64decode(output.encode('utf-8')).decode('utf-8')
 
         # resolve relative URIs
         if (element in self.can_be_relative_uri) and output:

feedparser/namespaces/itunes.py

@@ -102,7 +102,7 @@ def _start_itunes_image(self, attrs_d):
 
     def _end_itunes_block(self):
         value = self.pop('itunes_block', 0)
-        self._get_context()['itunes_block'] = (value == 'yes') and 1 or 0
+        self._get_context()['itunes_block'] = (value == 'yes' or value == 'Yes') and 1 or 0
 
     def _end_itunes_explicit(self):
         value = self.pop('itunes_explicit', 0)