Handle the Forwarded header

CHANGELOG.rst

@@ -4,8 +4,18 @@ History of changes
 
 Unreleased
 ~~~~~~~~~~
+Changed
+-------
 - Notice `1277`_ (obsolete 'X-' prefix) is now reported only once per message.
 
+Added
+-----
+- Checks for the `Forwarded`_ header (notices `1296`_, `1297`_).
+
+.. _Forwarded: https://tools.ietf.org/html/rfc7239
+.. _1296: http://pythonhosted.org/HTTPolice/notices.html#1296
+.. _1297: http://pythonhosted.org/HTTPolice/notices.html#1297
+
 
 0.5.2 - 2017-03-24
 ~~~~~~~~~~~~~~~~~~

httpolice/header.py

@@ -55,6 +55,8 @@ def __getitem__(self, key):
                 cls = PreferView
             elif key == h.preference_applied:
                 cls = PreferenceAppliedView
+            elif key == h.forwarded:
+                cls = ForwardedView
 
             # For the rest, we only need to know
             # a generic "rule" for combining multiple entries,
@@ -118,7 +120,7 @@ class HeaderView(object):
     def __init__(self, message, name):
         self.message = message
         self.name = name
-        self._entries = self._value = None
+        self._entries = self._value = self._value_breakdown = None
 
     def __repr__(self):
         return '<%s %s>' % (self.__class__.__name__, self.name)
@@ -163,6 +165,13 @@ def entries(self):
             self._parse()
         return self._entries
 
+    @property
+    def value_breakdown(self):
+        """List of (parsed) values for every entry of this header."""
+        if self._entries is None:   # pragma: no cover
+            self._parse()
+        return self._value_breakdown
+
     @property
     def value(self):
         if self._entries is None:
@@ -259,6 +268,7 @@ def _parse(self):
         entries, values = self._pre_parse()
         self._value = b','.join(values) if values else None
         self._entries = entries
+        self._value_breakdown = values
 
 
 class SingleHeaderView(HeaderView):
@@ -271,9 +281,11 @@ def _parse(self):
             if len(entries) > 1:
                 self.message.complain(1013, header=self.name, entries=entries)
             self._value = values[-1]
+            self._value_breakdown = [values[-1]]
             self._entries = [entries[-1]]
         else:
             self._value = None
+            self._value_breakdown = []
             self._entries = []
 
 
@@ -289,6 +301,7 @@ def _parse(self):
                 self._value.append(Unavailable)
             else:
                 self._value.extend(sub_values)
+        self._value_breakdown = values
         self._entries = entries
 
     def __iter__(self):
@@ -417,3 +430,18 @@ class PreferenceAppliedView(DirectivesView, MultiHeaderView):
     """Wraps a ``Preference-Applied`` header."""
 
     knowledge_module = httpolice.known.preference
+
+
+class ForwardedView(DirectivesView, MultiHeaderView):
+
+    """Wraps a ``Forwarded`` header."""
+
+    knowledge_module = httpolice.known.forwarded_param
+
+    def _process_parsed(self, entry, parsed):
+        # Work at the second level of nesting (forwarded-pairs,
+        # not forwarded-elements).
+        return [super(ForwardedView, self)._process_parsed(entry, pairs)
+                for pairs in parsed]
+
+    __getattr__ = None

httpolice/known/__init__.py

@@ -9,6 +9,7 @@
 from httpolice.known.base import KnownDict
 from httpolice.known.cache_directive import known as cache
 from httpolice.known.content_coding import known as cc
+from httpolice.known.forwarded_param import known as forwarded
 from httpolice.known.header import known as h
 from httpolice.known.hsts_directive import known as hsts
 from httpolice.known.media_type import known as media

httpolice/known/forwarded_param.py

@@ -0,0 +1,36 @@
+# -*- coding: utf-8; -*-
+
+from httpolice.citation import RFC
+from httpolice.known.base import KnownDict
+from httpolice.structure import ForwardedParam
+from httpolice.syntax import rfc3986, rfc7230, rfc7239
+
+
+def argument_required(_):       # pragma: no cover
+    return True
+
+def no_argument(_):
+    return False
+
+def parser_for(name):
+    return known.get_info(name).get('parser')
+
+
+known = KnownDict(ForwardedParam, [
+ {'_': ForwardedParam(u'by'),
+  '_citations': [RFC(7239, section=(5, 1))],
+  'description': u'IP-address of incoming interface of a proxy',
+  'parser': rfc7239.node},
+ {'_': ForwardedParam(u'for'),
+  '_citations': [RFC(7239, section=(5, 2))],
+  'description': u'IP-address of client making a request through a proxy',
+  'parser': rfc7239.node},
+ {'_': ForwardedParam(u'host'),
+  '_citations': [RFC(7239, section=(5, 3))],
+  'description': u'Host header field of the incoming request',
+  'parser': rfc7230.Host},
+ {'_': ForwardedParam(u'proto'),
+  '_citations': [RFC(7239, section=(5, 4))],
+  'description': u'Application protocol used for incoming request',
+  'parser': rfc3986.scheme},
+], extra_info=['description', 'parser'])

httpolice/known/header.py

@@ -5,7 +5,7 @@
 from httpolice.structure import FieldName
 from httpolice.syntax import (internal, rfc5789, rfc5988, rfc6266, rfc6797,
                               rfc7230, rfc7231, rfc7232, rfc7233, rfc7234,
-                              rfc7235, rfc7240, rfc7540, rfc7838)
+                              rfc7235, rfc7239, rfc7240, rfc7540, rfc7838)
 
 
 SINGLE = 1
@@ -406,8 +406,14 @@ def parser_for(name):
   'rule': SINGLE},
  {'_': FieldName(u'Ext'), '_citations': [RFC(4229)]},
  {'_': FieldName(u'Forwarded'),
-  '_citations': [RFC(7239)],
-  'iana_status': u'standard'},
+  '_citations': [RFC(7239, section=(4,))],
+  'bad_for_connection': True,
+  'for_request': True, 'for_response': False,
+  'iana_status': u'standard',
+  'parser': rfc7239.Forwarded,
+  'precondition': False,
+  'proactive_conneg': False,
+  'rule': MULTI},
  {'_': FieldName(u'From'),
   '_citations': [RFC(7231, section=(5, 5, 1))],
   'for_request': True,

httpolice/notices.xml

@@ -2167,4 +2167,17 @@ One non-obvious thing is how references work
     <rfc num="7231" sect="4.1">The method token is case-sensitive</rfc>
   </error>
 
+  <error id="1296">
+    <title>Duplicate ‘<var ref="param"/>’ parameter in <var ref="entry"/></title>
+    <rfc num="7239" sect="4">
+      Each parameter MUST NOT occur more than once per field-value.
+    </rfc>
+  </error>
+
+  <comment id="1297">
+    <title>Possibly bad syntax in <var ref="entry"/></title>
+    <explain>The <h ref="no">Forwarded</h> header here contains <var ref="n_elements"/> elements, describing <var ref="n_elements"/> proxy hops, with only one parameter for each proxy — yet all parameters have different names.</explain>
+    <explain>If this was intended to be <var ref="n_elements"/> parameters for a single proxy hop, then the pairs must be separated with semicolons, not commas.</explain>
+  </comment>
+
 </notices>

httpolice/request.py

@@ -413,6 +413,17 @@ def check_request(req):
        (pref.handling, u'lenient') in headers.prefer.without_params:
         complain(1290)
 
+    for (entry, elements) in zip(headers.forwarded.entries,
+                                 headers.forwarded.value_breakdown):
+        if not okay(elements):
+            continue
+        for elem in elements:
+            for duped in duplicates(param for (param, _) in elem):
+                complain(1296, entry=entry, param=duped)
+        if len(elements) > 1 and all(len(elem) == 1 for elem in elements):
+            if not duplicates(param for [(param, _)] in elements):
+                complain(1297, entry=entry, n_elements=len(elements))
+
 
 def _check_basic_auth(req, hdr, credentials):
     if isinstance(credentials, six.text_type):   # ``token68`` form

httpolice/structure.py

@@ -398,3 +398,10 @@ class Preference(CaseInsensitive):
     """An HTTP preference token (RFC 7240 Section 2)."""
 
     __slots__ = ()
+
+
+class ForwardedParam(CaseInsensitive):
+
+    """A parameter for the Forwarded header (RFC 7239)."""
+
+    __slots__ = ()

httpolice/syntax/rfc7239.py

@@ -0,0 +1,36 @@
+# -*- coding: utf-8; -*-
+
+from httpolice.citation import RFC
+from httpolice.parse import (fill_names, many, maybe, pivot, skip, string1,
+                             string_times)
+from httpolice.structure import ForwardedParam
+from httpolice.syntax.common import ALPHA, DIGIT
+from httpolice.syntax.rfc3986 import IPv4address, IPv6address
+from httpolice.syntax.rfc7230 import comma_list1, quoted_string, token
+
+
+def _remove_empty(xs):
+    return [x for x in xs if x is not None]
+
+
+obfnode = '_' + string1(ALPHA | DIGIT | '.' | '_' | '-')                > pivot
+nodename = (IPv4address |
+            skip('[') * IPv6address * skip(']') |
+            'unknown' | obfnode)                                        > pivot
+
+port = int << string_times(1, 5, DIGIT)                                 > pivot
+obfport = '_' + string1(ALPHA | DIGIT | '.' | '_' | '-')                > pivot
+node_port = port | obfport                                              > pivot
+
+node = nodename * maybe(skip(':') * node_port)                          > pivot
+
+value = token | quoted_string                                           > pivot
+forwarded_pair = (ForwardedParam << token) * skip('=') * value          > pivot
+
+forwarded_element = _remove_empty << (
+    maybe(forwarded_pair) % many(skip(';') * maybe(forwarded_pair)))    > pivot
+
+Forwarded = comma_list1(forwarded_element)                              > pivot
+
+
+fill_names(globals(), RFC(7239))

test/combined_data/1000_10

@@ -0,0 +1,16 @@
+1000 1000
+
+======== BEGIN INBOUND STREAM ========
+GET / HTTP/1.1
+Host: example.com
+User-Agent: demo
+Forwarded: for=123.45.67.89; proto=https
+Forwarded: for=fe80::6eb6:566f:a443:ae95
+
+======== BEGIN OUTBOUND STREAM ========
+HTTP/1.1 200 OK
+Date: Thu, 31 Dec 2015 18:26:56 GMT
+Content-Type: text/plain
+Content-Length: 14
+
+Hello world!

test/combined_data/1158_4

@@ -0,0 +1,15 @@
+1158 1158 1158
+
+======== BEGIN INBOUND STREAM ========
+GET / HTTP/1.1
+Host: example.com
+User-Agent: demo
+Forwarded: for="fe80::c5cc:dcfa:782a:a232";by="example.net";proto="";
+
+======== BEGIN OUTBOUND STREAM ========
+HTTP/1.1 200 OK
+Date: Thu, 31 Dec 2015 18:26:56 GMT
+Content-Type: text/plain
+Content-Length: 14
+
+Hello world!

test/combined_data/1296_1

@@ -0,0 +1,17 @@
+1296 1296 1296
+
+======== BEGIN INBOUND STREAM ========
+GET / HTTP/1.1
+Host: example.com
+User-Agent: demo
+Forwarded: for=10.0.4.19;proto=https;for=10.0.4.19
+Forwarded: for=10.129.60.21;proto=https;for=10.129.60.21
+Forwarded: for=unknown;by=_cf56213e;;by=_fb2932cb;
+
+======== BEGIN OUTBOUND STREAM ========
+HTTP/1.1 200 OK
+Date: Thu, 31 Dec 2015 18:26:56 GMT
+Content-Type: text/plain
+Content-Length: 14
+
+Hello world!

test/combined_data/1297_1

@@ -0,0 +1,18 @@
+1297
+
+======== BEGIN INBOUND STREAM ========
+GET / HTTP/1.1
+Host: example.com
+User-Agent: demo
+Forwarded: for=172.16.154.38;proto=https
+Forwarded: for=21.0.61.67,proto=https
+Via: 1.1 gateway04-eu1
+Forwarded: for="[2a01:ac8::10:1f]:8080";proto=http;by=_gateway04-eu1
+
+======== BEGIN OUTBOUND STREAM ========
+HTTP/1.1 200 OK
+Date: Thu, 31 Dec 2015 18:26:56 GMT
+Content-Type: text/plain
+Content-Length: 14
+
+Hello world!

test/combined_data/1297_2

@@ -0,0 +1,17 @@
+# Notice 1297 should only look at one header entry at a time.
+
+======== BEGIN INBOUND STREAM ========
+GET / HTTP/1.1
+Host: example.com
+User-Agent: demo
+Forwarded: for=127.0.0.1
+Forwarded: proto=https
+Forwarded: by=unknown
+
+======== BEGIN OUTBOUND STREAM ========
+HTTP/1.1 200 OK
+Date: Thu, 31 Dec 2015 18:26:56 GMT
+Content-Type: text/plain
+Content-Length: 14
+
+Hello world!

test/combined_data/1297_3

@@ -0,0 +1,15 @@
+
+
+======== BEGIN INBOUND STREAM ========
+GET / HTTP/1.1
+Host: example.com
+User-Agent: demo
+Forwarded: for=2.92.0.60, for=2.93.17.180, for=49.141.62.82
+
+======== BEGIN OUTBOUND STREAM ========
+HTTP/1.1 200 OK
+Date: Thu, 31 Dec 2015 18:26:56 GMT
+Content-Type: text/plain
+Content-Length: 14
+
+Hello world!

tools/iana.py

@@ -36,10 +36,10 @@
 from httpolice.citation import RFC, Citation
 import httpolice.known
 from httpolice.structure import (AltSvcParam, AuthScheme, CacheDirective,
-                                 ContentCoding, FieldName, MediaType, Method,
-                                 Preference, RangeUnit, RelationType,
-                                 StatusCode, TransferCoding, UpgradeToken,
-                                 WarnCode)
+                                 ContentCoding, FieldName, ForwardedParam,
+                                 MediaType, Method, Preference, RangeUnit,
+                                 RelationType, StatusCode, TransferCoding,
+                                 UpgradeToken, WarnCode)
 from httpolice.util.text import normalize_whitespace
 
 
@@ -154,6 +154,7 @@ def get_all(self):
         tree = self._get_xml('http-parameters/http-parameters.xml')
         return [
             (ContentCoding, list(self._content_codings(tree))),
+            (ForwardedParam, list(self._forwarded_parameters(tree))),
             (Preference, list(self._preferences(tree))),
             (RangeUnit, list(self._range_units(tree))),
             (TransferCoding, list(self._transfer_codings(tree))),
@@ -169,6 +170,18 @@ def _content_codings(self, tree):
                 '_citations': list(self.extract_citations(record)),
             }
 
+    def _forwarded_parameters(self, tree):
+        records = tree.findall(
+            '//iana:registry[@id="forwarded"]/iana:record', self.xmlns)
+        for record in records:
+            yield {
+                '_': ForwardedParam(
+                    record.find('iana:name', self.xmlns).text),
+                '_citations': list(self.extract_citations(record)),
+                'description':
+                    record.find('iana:description', self.xmlns).text,
+            }
+
     def _preferences(self, tree):
         records = tree.findall(
             '//iana:registry[@id="preferences"]/iana:record', self.xmlns)