Remember input filenames and show them in reports

CHANGELOG.rst

@@ -22,8 +22,13 @@ Added
   a ``--fail-on`` option to exit with a non-zero status
   if any notices with a given severity have been reported.
 - Work around various problems in HAR files exported by Firefox and `Fiddler`_.
+- The name of the input file is now remembered for every request and response,
+  and can be shown in HTML reports with the "Show message sources" option
+  from the "options" menu.
+  In the `Python API`_, this corresponds to the new ``source`` argument
+  to ``Request`` and ``Response`` constructors.
 - Notices about HTTP/1.x framing errors (such as `1006`_)
-  now include the input filename.
+  now include the input filename as well.
 - Check for missing scheme name in authorization headers (notice `1274`_).
 - Check for missing quality values in headers like Accept (notice `1276`_).
 - Check for obsolete 'X-' prefix in experimental headers (notice `1277`_).
@@ -32,6 +37,7 @@ Added
 .. _HTML reports: http://httpolice.readthedocs.io/en/stable/reports.html
 .. _Fiddler: http://www.telerik.com/fiddler
 .. _PyPy: http://pypy.org/
+.. _Python API: http://httpolice.readthedocs.io/en/stable/api.html
 .. _1006: http://pythonhosted.org/HTTPolice/notices.html#1006
 .. _1093: http://pythonhosted.org/HTTPolice/notices.html#1093
 .. _1274: http://pythonhosted.org/HTTPolice/notices.html#1274

httpolice/framing1.py

@@ -83,6 +83,7 @@ def _parse_request(stream, scheme=None):
 
 
 def _parse_request_heading(stream, scheme=None):
+    beginning = stream.point
     try:
         with stream:
             method_ = Method(stream.consume_regex(rfc7230.method))
@@ -97,7 +98,8 @@ def _parse_request_heading(stream, scheme=None):
         stream.complain(1006, error=e)
         return Unavailable
     else:
-        req = Request(scheme, method_, target, version_, entries, body=None)
+        req = Request(scheme, method_, target, version_, entries, body=None,
+                      source=u'%s, offset %d' % (stream.name, beginning))
         stream.dump_complaints(req.complain, place=u'request heading')
         return req
 
@@ -154,6 +156,7 @@ def _parse_responses(stream, req):
 
 
 def _parse_response_heading(req, stream):
+    beginning = stream.point
     try:
         with stream:
             version_ = HTTPVersion(stream.consume_regex(rfc7230.HTTP_version))
@@ -168,7 +171,8 @@ def _parse_response_heading(req, stream):
         stream.sane = False
         return Unavailable
     else:
-        resp = Response(version_, status, reason, entries, body=None)
+        resp = Response(version_, status, reason, entries, body=None,
+                        source=u'%s, offset %d' % (stream.name, beginning))
         resp.request = req
         stream.dump_complaints(resp.complain, place=u'response heading')
         return resp

httpolice/inputs/har.py

@@ -40,21 +40,21 @@ def har_input(paths):
             try:
                 creator = data['log']['creator']['name']
                 for entry in data['log']['entries']:
-                    yield _process_entry(entry, creator)
+                    yield _process_entry(entry, creator, path)
             except (TypeError, KeyError) as exc:
                 six.raise_from(
                     InputError('%s: cannot understand HAR file: %r' %
                                (path, exc)),
                     exc)
 
 
-def _process_entry(data, creator):
-    req = _process_request(data['request'], creator)
-    resp = _process_response(data['response'], req, creator)
+def _process_entry(data, creator, path):
+    req = _process_request(data['request'], creator, path)
+    resp = _process_response(data['response'], req, creator, path)
     return Exchange(req, [resp] if resp is not None else [])
 
 
-def _process_request(data, creator):
+def _process_request(data, creator, path):
     (version, header_entries, pseudo_headers) = _process_message(data, creator)
     if creator in CHROME and version == http11 and u':host' in pseudo_headers:
         # SPDY exported from Chrome.
@@ -129,14 +129,15 @@ def _process_request(data, creator):
             text = None
             body = b''
 
-    req = Request(scheme, method, target, version, header_entries, body)
+    req = Request(scheme, method, target, version, header_entries, body,
+                  source=path)
     if text is not None:
         req.unicode_body = text
     req.is_to_proxy = None                      # See above.
     return req
 
 
-def _process_response(data, req, creator):
+def _process_response(data, req, creator, path):
     if data['status'] == 0:          # Indicates error in Chrome.
         return None
     (version, header_entries, _) = _process_message(data, creator)
@@ -182,7 +183,8 @@ def _process_response(data, req, creator):
         # Helps with SPDY in Chrome.
         version = None
 
-    resp = Response(version, status, reason, header_entries, body=body)
+    resp = Response(version, status, reason, header_entries, body=body,
+                    source=path)
 
     if data['content'].get('text') and status != st.not_modified:
         if data['content'].get('encoding', u'').lower() == u'base64':

httpolice/message.py

@@ -40,7 +40,8 @@ class Message(Blackboard):
 
     self_name = u'msg'
 
-    def __init__(self, version, header_entries, body, trailer_entries=None):
+    def __init__(self, version, header_entries, body, trailer_entries=None,
+                 source=None):
         super(Message, self).__init__()
         self.version = (HTTPVersion(force_unicode(version))
                         if version is not None else None)
@@ -51,6 +52,7 @@ def __init__(self, version, header_entries, body, trailer_entries=None):
                                 for k, v in trailer_entries or []]
         self.rebuild_headers()
         self.annotations = {}
+        self.source = source
 
     @property
     def annotated_header_entries(self):

httpolice/reports/html.css

@@ -85,6 +85,12 @@ div.message-display {
     max-width: 55em;
 }
 
+p.message-source {
+    margin: 0 1em 0.5em 0;
+    color: gray;
+    overflow: auto;
+}
+
 .message-display h2 {
     font-weight: bolder;
     margin: 0;

httpolice/reports/html.js

@@ -70,13 +70,21 @@ function installHovers() {
     }
 }
 
+function toggleSources(show) {
+    var i, sources = document.querySelectorAll('.message-source');
+    for (i = 0; i < sources.length; i += 1) {
+        sources[i].hidden = !show;
+    }
+}
+
 function onOptionsSubmit(event) {
     var boringNotices =
             document.getElementById('boringNotices').value.split(/\s+/),
         hideBoringNotices =
             document.getElementById('hideBoringNotices').checked,
         hideBoringExchanges =
             document.getElementById('hideBoringExchanges').checked,
+        showSources = document.getElementById('showSources').checked,
         i, exchanges, exchange, isBoringExchange,
         j, notices, notice, severity, ident, isBoringNotice;
 
@@ -100,14 +108,14 @@ function onOptionsSubmit(event) {
         }
         exchange.hidden = (hideBoringExchanges && isBoringExchange);
     }
+
+    toggleSources(showSources);
 }
 
 function installOptions() {
     var div, button, form,
         p1, label1, input1,
-        p2, label2, input2,
-        p3, label3, input3,
-        p4, submit;
+        p2, submit;
 
     div = document.createElement('div');
     div.classList.add('options');
@@ -138,44 +146,42 @@ function installOptions() {
     input1.placeholder = 'example: 1089 1135 C';
     p1.appendChild(input1);
 
+    function addCheckboxRow(id, text, title) {
+        var p, input, label;
+
+        p = document.createElement('p');
+        form.appendChild(p);
+
+        input = document.createElement('input');
+        input.type = 'checkbox';
+        input.name = id;
+        input.id = id;
+        p.appendChild(input);
+
+        label = document.createElement('label');
+        label.htmlFor = id;
+        label.textContent = text;
+        p.appendChild(label);
+
+        if (title) {
+            input.title = title;
+            label.title = title;
+        }
+    }
+
+    addCheckboxRow('hideBoringNotices', 'Hide boring notices');
+    addCheckboxRow('hideBoringExchanges', 'Hide boring exchanges',
+        'Hide exchanges that have no notices, ' +
+        'or only debug and boring notices');
+    addCheckboxRow('showSources', 'Show message sources');
+
     p2 = document.createElement('p');
     form.appendChild(p2);
 
-    input2 = document.createElement('input');
-    input2.type = 'checkbox';
-    input2.name = 'hideBoringNotices';
-    input2.id = input2.name;
-    p2.appendChild(input2);
-
-    label2 = document.createElement('label');
-    label2.htmlFor = 'hideBoringNotices';
-    label2.textContent = 'Hide boring notices';
-    p2.appendChild(label2);
-
-    p3 = document.createElement('p');
-    form.appendChild(p3);
-
-    input3 = document.createElement('input');
-    input3.type = 'checkbox';
-    input3.name = 'hideBoringExchanges';
-    input3.id = input3.name;
-    input3.title = 'Hide exchanges that have no notices, ' +
-        'or only debug and boring notices';
-    p3.appendChild(input3);
-
-    label3 = document.createElement('label');
-    label3.htmlFor = 'hideBoringExchanges';
-    label3.textContent = 'Hide boring exchanges';
-    label3.title = input3.title;
-    p3.appendChild(label3);
-
-    p4 = document.createElement('p');
-    form.appendChild(p4);
-
     submit = document.createElement('input');
     submit.type = 'submit';
     submit.value = 'Apply';
-    p4.appendChild(submit);
+    p2.appendChild(submit);
 
     document.body.insertBefore(div, document.querySelector('h1'));
 }
@@ -184,5 +190,6 @@ document.addEventListener('DOMContentLoaded', function () {
     collapseAll();
     installButtons();
     installHovers();
+    toggleSources(false);
     installOptions();
 });

httpolice/reports/html.py

@@ -102,6 +102,8 @@ def _render_exchanges(exchanges):
 def _render_request(req):
     with H.section():
         with H.div(_class=u'message-display'):
+            if req.source:
+                H.p(u'from %s' % req.source, _class=u'message-source')
             with H.h2(), H.code():      # Request line
                 # We don't insert spaces here because,
                 # without ``__pretty=False``,
@@ -119,6 +121,8 @@ def _render_request(req):
 def _render_response(resp):
     with H.section():
         with H.div(_class=u'message-display'):
+            if resp.source:
+                H.p(u'from %s' % resp.source, _class=u'message-source')
             with H.h2(), H.code():      # Status line
                 # See above regarding spaces.
                 if resp.version:

httpolice/request.py

@@ -26,7 +26,7 @@
 class Request(message.Message):
 
     def __init__(self, scheme, method, target, version, header_entries,
-                 body, trailer_entries=None):
+                 body, trailer_entries=None, source=None):
         # pylint: disable=redefined-outer-name
         """
         :param scheme:
@@ -96,9 +96,15 @@ def __init__(self, scheme, method, target, version, header_entries,
 
             The format is the same as for `header_entries`.
 
+        :param source:
+            If not `None`, this should be a Unicode string
+            describing where this request was obtained from.
+            For example: ``u'somefile.txt, line 2, column 4'``.
+            It may be shown in reports.
+
         """
         super(Request, self).__init__(version, header_entries, body,
-                                      trailer_entries)
+                                      trailer_entries, source)
         self.scheme = force_unicode(scheme) if scheme is not None else None
         self.method = Method(force_unicode(method))
         self.target = force_unicode(target)

httpolice/response.py

@@ -25,7 +25,7 @@
 class Response(message.Message):
 
     def __init__(self, version, status, reason, header_entries,
-                 body, trailer_entries=None):
+                 body, trailer_entries=None, source=None):
         """
         :param version:
             The response's protocol version, as a Unicode string,
@@ -85,9 +85,15 @@ def __init__(self, version, status, reason, header_entries,
 
             The format is the same as for `header_entries`.
 
+        :param source:
+            If not `None`, this should be a Unicode string
+            describing where this response was obtained from.
+            For example: ``u'somefile.txt, line 2, column 4'``.
+            It may be shown in reports.
+
         """
         super(Response, self).__init__(version, header_entries,
-                                       body, trailer_entries)
+                                       body, trailer_entries, source)
         self.status = StatusCode(status)
         self.reason = force_unicode(reason) if reason is not None else None
         self.request = None