Merge pull request #16 from dave-shawley/forwarded-header

Forwarded header

.travis.yml

@@ -1,8 +1,9 @@
 language: python
 python:
 - 2.7
-- 3.3
 - 3.4
+- 3.5
+- 3.6
 - pypy
 sudo: false
 install:
@@ -21,4 +22,4 @@ deploy:
     tags: true
     all_branches: true
     repo: dave-shawley/ietfparse
-    python: 3.4
+    python: 3.5

docs/changelog.rst

@@ -3,6 +3,19 @@ Changelog
 
 .. py:currentmodule:: ietfparse
 
+`Next Release`_
+---------------
+- Officially drop support for Python 2.6 and 3.3.
+- Change :func:`headers.parse_accept` to also prefer explicit highest
+  quality preferences over inferred highest quality preferences.
+- Rename the ``normalized_parameter_values`` keyword of
+  :func:`headers._parse_parameter_list`.  The current spelling is retained
+  with a deprecation warning.  This will be removed in 2.0.
+- Add ``normalize_parameter_names`` keyword to the
+  :func:`headers._parse_parameter_list` internal function.
+- Add support for parsing :rfc:`7239` ``Forwarded`` headers with
+  :func:`headers.parse_forwarded`.
+
 `1.4.3`_ (30-Oct-2017)
 ----------------------
 - Change parsing of qualified lists to retain the initial ordering whenever

docs/header-parsing.rst

@@ -160,6 +160,29 @@ Notice that the ``(someday)`` comment embedded in the ``version``
 parameter was discarded and the ``msgtype`` parameter name was
 normalized as well.
 
+Forwarded
+---------
+:func:`parse_forwarded` parses an HTTP :http:header:`Forwarded`  header as
+described in :rfc:`7239` into a sequence of :class:`dict` instances.
+
+>>> from ietfparse import headers
+>>> parsed = headers.parse_forwarded('For=93.184.216.34;proto=http;'
+...                                  'By="[2606:2800:220:1:248:1893:25c8:1946]";'
+...                                  'host=example.com')
+>>> len(parsed)
+1
+>>> parsed[0]['for']
+'93.184.216.34'
+>>> parsed[0]['proto']
+'http'
+>>> parsed[0]['by']
+'[2606:2800:220:1:248:1893:25c8:1946]'
+>>> parsed[0]['host']
+'example.com'
+
+The names of the parameters are case-folded to lower case per the
+recommendation in :rfc:`7239`.
+
 Link
 ----
 :func:`parse_link` parses an HTTP :http:header:`Link` header as

docs/rfcs.rst

@@ -41,6 +41,11 @@ Relevant RFCs
 - :func:`ietfparse.headers.parse_parameter_list` parses the ``key=value``
   portions common to many header values.
 
+`RFC-7239`_
+-----------
+- :func:`ietfparse.headers.parse_forwarded` parses a :http:header:`Forwarded`
+  HTTP header.
+
 
 .. _RFC-2045: https://tools.ietf.org/html/rfc2045
 .. _5.1: https://tools.ietf.org/html/rfc2045#section-5.1
@@ -56,3 +61,5 @@ Relevant RFCs
 .. _5.3: https://tools.ietf.org/html/rfc7231#section-5.3
 .. _5.3.2: https://tools.ietf.org/html/rfc7231#section-5.3.2
 .. _5.3.3: https://tools.ietf.org/html/rfc7231#section-5.3.3
+
+.. _RFC-2739: https://tools.ietf.org/html/rfc7239

ietfparse/errors.py

@@ -21,3 +21,22 @@ class NoMatch(RootException):
 class MalformedLinkValue(RootException):
     """Value specified is not a valid link header."""
     pass
+
+
+class StrictHeaderParsingFailure(RootException, ValueError):
+    """
+    Non-standard header value detected.
+
+    This is raised when "strict" conformance is enabled for a
+    header parsing function and a header value fails due to one
+    of the "strict" rules.
+
+    See :func:`ietfparse.headers.parse_forwarded` for an example.
+
+    """
+
+    def __init__(self, header_name, header_value):
+        super(StrictHeaderParsingFailure, self).__init__(header_name,
+                                                         header_value)
+        self.header_name = header_name
+        self.header_value = header_value

ietfparse/headers.py

@@ -1,10 +1,11 @@
 """
 Functions for parsing headers.
 
+- :func:`.parse_accept`: parse an ``Accept`` value
 - :func:`.parse_accept_charset`: parse a ``Accept-Charset`` value
 - :func:`.parse_cache_control`: parse a ``Cache-Control`` value
 - :func:`.parse_content_type`: parse a ``Content-Type`` value
-- :func:`.parse_accept`: parse an ``Accept`` value
+- :func:`.parse_forwarded`: parse a :rfc:`7239` ``Forwarded`` value
 - :func:`.parse_link`: parse a :rfc:`5988` ``Link`` value
 - :func:`.parse_list`: parse a comma-separated list that is
   present in so many headers
@@ -15,16 +16,19 @@
 
 """
 import functools
+import decimal
 import re
 import warnings
 
 from . import datastructures, errors, _helpers
 
+
 _CACHE_CONTROL_BOOL_DIRECTIVES = \
     ('must-revalidate', 'no-cache', 'no-store', 'no-transform',
      'only-if-cached', 'public', 'private', 'proxy-revalidate')
 _COMMENT_RE = re.compile(r'\(.*\)')
 _QUOTED_SEGMENT_RE = re.compile(r'"([^"]*)"')
+_DEF_PARAM_VALUE = object()
 
 
 def parse_accept(header_value):
@@ -50,10 +54,17 @@ def parse_accept(header_value):
     .. _Accept: http://tools.ietf.org/html/rfc7231#section-5.3.2
 
     """
+    next_explicit_q = decimal.ExtendedContext.next_plus(decimal.Decimal('5.0'))
     headers = [parse_content_type(header)
                for header in parse_list(header_value)]
     for header in headers:
-        header.quality = float(header.parameters.pop('q', 1.0))
+        q = header.parameters.pop('q', None)
+        if q is None:
+            q = '1.0'
+        elif float(q) == 1.0:
+            q = float(next_explicit_q)
+            next_explicit_q = next_explicit_q.next_minus()
+        header.quality = float(q)
 
     def ordering(left, right):
         """
@@ -200,12 +211,46 @@ def parse_content_type(content_type, normalize_parameter_values=True):
     """
     parts = _remove_comments(content_type).split(';')
     content_type, content_subtype = parts.pop(0).split('/')
-    parameters = _parse_parameter_list(parts, normalize_parameter_values)
+    parameters = _parse_parameter_list(
+        parts, normalize_parameter_values=normalize_parameter_values)
 
     return datastructures.ContentType(content_type, content_subtype,
                                       dict(parameters))
 
 
+def parse_forwarded(header_value, only_standard_parameters=False):
+    """
+    Parse RFC7239 Forwarded header.
+
+    :param str header_value: value to parse
+    :keyword bool only_standard_parameters: if this keyword is specified
+        and given a *truthy* value, then a non-standard parameter name
+        will result in :exc:`~ietfparse.errors.StrictHeaderParsingFailure`
+    :return: an ordered :class:`list` of :class:`dict` instances
+    :raises: :exc:`ietfparse.errors.StrictHeaderParsingFailure` is
+        raised if `only_standard_parameters` is enabled and a non-standard
+        parameter name is encountered
+
+    This function parses a :rfc:`7239` HTTP header into a :class:`list`
+    of :class:`dict` instances with each instance containing the param
+    values.  The list is ordered as received from left to right and the
+    parameter names are folded to lower case strings.
+
+    """
+    result = []
+    for entry in parse_list(header_value):
+        param_tuples = _parse_parameter_list(entry.split(';'),
+                                             normalize_parameter_names=True,
+                                             normalize_parameter_values=False)
+        if only_standard_parameters:
+            for name, _ in param_tuples:
+                if name not in ('for', 'proto', 'by', 'host'):
+                    raise errors.StrictHeaderParsingFailure('Forwarded',
+                                                            header_value)
+        result.append(dict(param_tuples))
+    return result
+
+
 def parse_link(header_value, strict=True):
     """
     Parse a HTTP Link header.
@@ -284,21 +329,42 @@ def parse_list(value):
             for x in value.split(',')]
 
 
-def _parse_parameter_list(parameter_list, normalized_parameter_values=True):
+def _parse_parameter_list(parameter_list,
+                          normalized_parameter_values=_DEF_PARAM_VALUE,
+                          normalize_parameter_names=False,
+                          normalize_parameter_values=True):
     """
     Parse a named parameter list in the "common" format.
 
-    :param parameter_list:
-    :param bool normalized_parameter_values:
+    :param parameter_list: sequence of string values to parse
+    :keyword bool normalize_parameter_names: if specified and *truthy*
+        then parameter names will be case-folded to lower case
+    :keyword bool normalize_parameter_values: if omitted or specified
+        as *truthy*, then parameter values are case-folded to lower case
+    :keyword bool normalized_parameter_values: alternate way to spell
+        ``normalize_parameter_values`` -- this one is deprecated
     :return: a sequence containing the name to value pairs
 
+    The parsed values are normalized according to the keyword parameters
+    and returned as :class:`tuple` of name to value pairs preserving the
+    ordering from `parameter_list`.  The values will have quotes removed
+    if they were present.
+
     """
+    if normalized_parameter_values is not _DEF_PARAM_VALUE:  # pragma: no cover
+        warnings.warn('normalized_parameter_values keyword to '
+                      '_parse_parameter_list is deprecated, use '
+                      'normalize_parameter_values instead',
+                      DeprecationWarning)
+        normalize_parameter_values = normalized_parameter_values
     parameters = []
     for param in parameter_list:
         param = param.strip()
         if param:
             name, value = param.split('=')
-            if normalized_parameter_values:
+            if normalize_parameter_names:
+                name = name.lower()
+            if normalize_parameter_values:
                 value = value.lower()
             parameters.append((name, _dequote(value.strip())))
     return parameters

setup.py

@@ -55,7 +55,11 @@ def read_requirements_file(name):
         'Operating System :: OS Independent',
         'Programming Language :: Python',
         'Programming Language :: Python :: 2',
+        'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.4',
+        'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
         'Development Status :: 5 - Production/Stable',
         'Environment :: Web Environment',
         'Topic :: Internet :: WWW/HTTP',

tests/algorithm_tests.py

@@ -1,6 +1,6 @@
 import unittest
 
-from ietfparse import algorithms, datastructures, errors, headers
+from ietfparse import algorithms, errors, headers
 
 
 class ContentNegotiationTestCase(unittest.TestCase):
@@ -11,16 +11,20 @@ def assertContentTypeMatchedAs(self, expected, *supported, **kwargs):
             self.requested,
             [headers.parse_content_type(value) for value in supported],
         )
-        self.assertEqual(selected, headers.parse_content_type(expected))
+        self.assertEqual(
+            selected, headers.parse_content_type(expected),
+            '\nExpected to select "{!s}", actual selection was "{!s}"'.format(
+                expected, selected,
+            ))
         if 'matching_pattern' in kwargs:
             self.assertEqual(str(matched), kwargs['matching_pattern'])
 
 
-class WhenUsingProactiveContentNegotiation(ContentNegotiationTestCase):
+class ProactiveContentNegotiationTests(ContentNegotiationTestCase):
 
     @classmethod
     def setUpClass(cls):
-        super(WhenUsingProactiveContentNegotiation, cls).setUpClass()
+        super(ProactiveContentNegotiationTests, cls).setUpClass()
         cls.requested.extend(headers.parse_accept(
             'application/vnd.example.com+json;version=2, '
             'application/vnd.example.com+json;version=1;q=0.9, '
@@ -73,11 +77,11 @@ def test_that_inappropriate_value_is_not_matched(self):
             )
 
 
-class WhenUsingRfc7231Examples(ContentNegotiationTestCase):
+class Rfc7231ExampleTests(ContentNegotiationTestCase):
 
     @classmethod
     def setUpClass(cls):
-        super(WhenUsingRfc7231Examples, cls).setUpClass()
+        super(Rfc7231ExampleTests, cls).setUpClass()
         cls.requested.extend(headers.parse_accept(
             'text/*;q=0.3, text/html;q=0.7, text/html;level=1, '
             'text/html;level=2;q=0.4, */*;q=0.5'
@@ -111,17 +115,34 @@ def test_that_text_html_level_3_matches_text_html(self):
         )
 
 
-class WhenSelectingWithRawContentTypes(unittest.TestCase):
+class PriorizationTests(unittest.TestCase):
 
-    def test_that_raw_content_type_has_highest_quality(self):
+    def test_that_explicit_priority_1_is_preferred(self):
         selected, matched = algorithms.select_content_type(
-            [
-                datastructures.ContentType('type', 'preferred')
-            ],
-            [
-                datastructures.ContentType('type', 'acceptable'),
-                datastructures.ContentType('type', 'almost-perfect'),
-                datastructures.ContentType('type', 'preferred'),
-            ],
+            headers.parse_accept(
+                'application/vnd.com.example+json, '
+                'application/vnd.com.example+json;version=1;q=1.0, '
+                'application/vnd.com.example+json;version=2'),
+            [headers.parse_content_type(value)
+             for value in ('application/vnd.com.example+json;version=1',
+                           'application/vnd.com.example+json;version=2',
+                           'application/vnd.com.example+json;version=3')],
         )
-        self.assertEqual(selected.content_subtype, 'preferred')
+        self.assertEqual(str(selected),
+                         'application/vnd.com.example+json; version=1')
+
+    def test_that_multiple_matches_result_in_any_appropriate_value(self):
+        # note that this also tests that duplicated values are acceptable
+        selected, matched = algorithms.select_content_type(
+            headers.parse_accept(
+                'application/vnd.com.example+json;version=1, '
+                'application/vnd.com.example+json;version=1, '
+                'application/vnd.com.example+json;version=1;q=0.9, '
+                'application/vnd.com.example+json;version=2;q=0.9'),
+            [headers.parse_content_type(value)
+             for value in ('application/vnd.com.example+json;version=1',
+                           'application/vnd.com.example+json;version=2',
+                           'application/vnd.com.example+json;version=3')],
+        )
+        self.assertEqual(str(selected),
+                         'application/vnd.com.example+json; version=1')

tests/headers_content_type_tests.py

@@ -3,10 +3,10 @@
 from ietfparse import datastructures, headers
 
 
-class WhenParsingSimpleContentType(unittest.TestCase):
+class SimpleContentTypeParsingTests(unittest.TestCase):
 
     def setUp(self):
-        super(WhenParsingSimpleContentType, self).setUp()
+        super(SimpleContentTypeParsingTests, self).setUp()
         self.parsed = headers.parse_content_type(
             'text/plain', normalize_parameter_values=False)
 
@@ -20,10 +20,10 @@ def test_that_no_parameters_are_found(self):
         self.assertEqual(self.parsed.parameters, {})
 
 
-class WhenParsingComplexContentType(unittest.TestCase):
+class ParsingComplexContentTypeTests(unittest.TestCase):
 
     def setUp(self):
-        super(WhenParsingComplexContentType, self).setUp()
+        super(ParsingComplexContentTypeTests, self).setUp()
         self.parsed = headers.parse_content_type(
             'message/HTTP; version=2.0 (someday); MsgType="Request"',
             normalize_parameter_values=False)
@@ -41,7 +41,7 @@ def test_that_message_type_parameter_is_parsed(self):
         self.assertEqual(self.parsed.parameters['msgtype'], 'Request')
 
 
-class WhenParsingMediaTypeExamples(unittest.TestCase):
+class Rfc7231ExampleTests(unittest.TestCase):
     """Test cases from RFC7231, Section 3.1.1.1"""
 
     def setUp(self):