fix Web: JSON logfile yields unicode, but twisted.web requires bytes

Also improve the error messages when the server 500s when a renderer function
returns unicode, to track this sort of thing down more easily in the future

src/foolscap/logging/web.py

@@ -78,6 +78,9 @@ def web_format_time(t, mode="short-local"):
     extended = "Local=%s  Local=%s  UTC=%s" % (time_ctime, time_local, time_utc)
     return time_s, extended
 
+def web_escape(u):
+    return html.escape(u.encode("utf-8"))
+
 class Welcome(resource.Resource):
     def __init__(self, viewer, timestamps):
         self.viewer = viewer
@@ -107,14 +110,16 @@ def render(self, req):
                 ((first_number, first_time),
                  (last_number, last_time),
                  num_events, levels, pid, versions) = self.viewer.summaries[lf]
+                # remember: the logfile uses JSON, so all strings will be
+                # unicode, and twisted.web requires bytes
                 data += "  <li>PID %s</li>\n" % html.escape(str(pid))
                 if versions:
                     data += "  <li>Application Versions:\n"
                     data += "   <ul>\n"
                     for name in sorted(versions.keys()):
                         ver = versions[name]
-                        data += "    <li>%s: %s</li>\n" % (html.escape(name),
-                                                           html.escape(ver))
+                        data += "    <li>%s: %s</li>\n" % (web_escape(name),
+                                                           web_escape(ver))
                     data += "   </ul>\n"
                     data += "  </li>\n"
                 if first_time and last_time:
@@ -281,8 +286,8 @@ def __init__(self, e):
         self.incarnation = base32.encode(e['d']['incarnation'][0])
         if 'num' in e['d']:
             self.index = (e['from'], e['d']['num'])
-            self.anchor_index = "%s_%s_%d" % (urllib.quote(e['from']),
-                                              self.incarnation,
+            self.anchor_index = "%s_%s_%d" % (urllib.quote(e['from'].encode("utf-8")),
+                                              self.incarnation.encode("utf-8"),
                                               e['d']['num'])
         self.parent_index = None
         if 'parent' in e['d']:
@@ -305,21 +310,23 @@ def level_class(self):
         return self.LEVELMAP.get(level, "UNKNOWN")
 
     def to_html(self, href_base="", timestamps="short-local"):
+        # this must return bytes to satisfy twisted.web, but the logfile is
+        # JSON so we get unicode here
         d = self.e['d']
         time_short, time_extended = web_format_time(d['time'], timestamps)
-        msg = html.escape(log.format_message(d))
+        msg = web_escape(log.format_message(d))
         if 'failure' in d:
             lines = str(d['failure']).split("\n")
-            html_lines = [html.escape(line) for line in lines]
+            html_lines = [web_escape(line) for line in lines]
             f_html = "\n".join(html_lines)
             msg += " FAILURE:<pre>%s</pre>" % f_html
         level = d.get('level', log.OPERATIONAL)
         level_s = ""
         if level >= log.UNUSUAL:
             level_s = self.LEVELMAP.get(level, "") + " "
         details = "  ".join(["Event #%d" % d['num'],
-                             "TubID=%s" % self.e['from'],
-                             "Incarnation=%s" % self.incarnation,
+                             "TubID=%s" % web_escape(self.e['from']),
+                             "Incarnation=%s" % web_escape(self.incarnation),
                              time_extended])
         label = '<span title="%s">%s</span>' % (details, time_short)
         data = '%s [<span id="E%s"><a href="%s#E%s">%d</a></span>]: %s%s' \

src/foolscap/test/test_logging.py

@@ -2251,6 +2251,9 @@ def getPage(url):
         # with a transport that does not have an abortConnection method")
         # which seems to be https://twistedmatrix.com/trac/ticket/8227
         page = yield client.readBody(response)
+    if response.code != 200:
+        raise ValueError("request failed (%d), page contents were: %s" % (
+            response.code, page))
     returnValue(page)
 
 class Web(unittest.TestCase):