Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Loading…

Fix for problems with non-ascii exception messages #43

Open
wants to merge 2 commits into from

6 participants

@Cito

For instance, PostgreSQL creates error messages in latin-1 or utf-8 depending on how it is configured. The debug toolbar assumes that error messages are always ascii strings or unicode and crashes with a UnicodeDecodeError on such exceptions. This patch makes the debug toolbar handle and display such exceptions properly.

@oxyum

ping?

@mcdonc
Owner

Bleh sorry for the delay on this. I'm making one more release without this fix (1.0). After that I'll try to review what's going on here and come up with some suitable fix if I don't just go ahead and apply this one.

@uralbash

I have the same problem when I'm working with postgres. I have a Russian environment in the Linux Debian and Postgres gives an error in Russian lang. After that I give error: / pyramid_debugtoolbar / panels / templates / logger.mako ", line 16, in render_body

$ {record ['message']} </ td>
UnicodeDecodeError: 'ascii' codec can't decode byte 0xd0 in position 12: ordinal not in range (128)

As a workaround you can fix pyramid_debugtoolbar / panels / templates / logger.mako

change

$ {record ['message']} to
$ {record ['message']. decode ('utf-8')}
@jhnwsk

Same problem here with German umlauts.

I cloned pyramid_debugtoolbar and pulled Cito's fix in. No luck.

uralbash's workaround works for me in:

--- a/pyramid_debugtoolbar/panels/templates/renderings.dbtmako
+++ b/pyramid_debugtoolbar/panels/templates/renderings.dbtmako
@@ -18,7 +18,7 @@
        </thead>
        <tbody> 
                <tr class="pDebugOdd">
-                       <td colspan="2">${rendering['val']|h}</td>
+                       <td colspan="2">${rendering['val'].decode('utf-8')|h}</td>
                </tr>
        </tbody>
        <thead>

my stack trace is

Traceback (most recent call last):
  File "/virtualenv/local/lib/python2.7/site-packages/waitress-0.8.1-py2.7.egg/waitress/channel.py", line 329, in service
    task.service()
  File "/virtualenv/local/lib/python2.7/site-packages/waitress-0.8.1-py2.7.egg/waitress/task.py", line 173, in service
    self.execute()
  File "/virtualenv/local/lib/python2.7/site-packages/waitress-0.8.1-py2.7.egg/waitress/task.py", line 380, in execute
    app_iter = self.channel.server.application(env, start_response)
  File "/virtualenv/local/lib/python2.7/site-packages/pyramid/router.py", line 251, in __call__
    response = self.invoke_subrequest(request, use_tweens=True)
  File "/virtualenv/local/lib/python2.7/site-packages/pyramid/router.py", line 227, in invoke_subrequest
    response = handle_request(request)
  File "/virtualenv/lib/python2.7/site-packages/pyramid_debugtoolbar-1.0.3-py2.7.egg/pyramid_debugtoolbar/toolbar.py", line 160, in toolbar_tween
    toolbar.process_response(response)
  File "/virtualenv/lib/python2.7/site-packages/pyramid_debugtoolbar-1.0.3-py2.7.egg/pyramid_debugtoolbar/toolbar.py", line 56, in process_response
    vars, request=request)
  File "/virtualenv/local/lib/python2.7/site-packages/pyramid/renderers.py", line 88, in render
    return helper.render(value, None, request=request)
  File "/virtualenv/local/lib/python2.7/site-packages/pyramid/renderers.py", line 550, in render
    result = renderer(value, system_values)
  File "/virtualenv/local/lib/python2.7/site-packages/pyramid/mako_templating.py", line 219, in __call__
    reraise(MakoRenderingException(errtext), None, exc_info[2])
  File "/virtualenv/local/lib/python2.7/site-packages/pyramid/mako_templating.py", line 211, in __call__
    result = template.render_unicode(**system)
  File "/virtualenv/local/lib/python2.7/site-packages/mako/template.py", line 421, in render_unicode
    as_unicode=True)
  File "/virtualenv/local/lib/python2.7/site-packages/mako/runtime.py", line 766, in _render
    **_kwargs_for_callable(callable_, data))
  File "/virtualenv/local/lib/python2.7/site-packages/mako/runtime.py", line 798, in _render_context
    _exec_template(inherit, lclcontext, args=args, kwargs=kwargs)
  File "/virtualenv/local/lib/python2.7/site-packages/mako/runtime.py", line 824, in _exec_template
    callable_(context, *args, **kwargs)
  File "pyramid_debugtoolbar_templates_toolbar_dbtmako", line 111, in render_body
  File "/virtualenv/lib/python2.7/site-packages/pyramid_debugtoolbar-1.0.3-py2.7.egg/pyramid_debugtoolbar/panels/renderings.py", line 46, in content
    }, request=self.request)
  File "/virtualenv/lib/python2.7/site-packages/pyramid_debugtoolbar-1.0.3-py2.7.egg/pyramid_debugtoolbar/panels/__init__.py", line 24, in render
    return render(template_name, vars, request=request)
  File "/virtualenv/local/lib/python2.7/site-packages/pyramid/renderers.py", line 88, in render
    return helper.render(value, None, request=request)
  File "/virtualenv/local/lib/python2.7/site-packages/pyramid/renderers.py", line 550, in render
    result = renderer(value, system_values)
  File "/virtualenv/local/lib/python2.7/site-packages/pyramid/mako_templating.py", line 219, in __call__
    reraise(MakoRenderingException(errtext), None, exc_info[2])
  File "/virtualenv/local/lib/python2.7/site-packages/pyramid/mako_templating.py", line 211, in __call__
    result = template.render_unicode(**system)
  File "/virtualenv/local/lib/python2.7/site-packages/mako/template.py", line 421, in render_unicode
    as_unicode=True)
  File "/virtualenv/local/lib/python2.7/site-packages/mako/runtime.py", line 766, in _render
    **_kwargs_for_callable(callable_, data))
  File "/virtualenv/local/lib/python2.7/site-packages/mako/runtime.py", line 798, in _render_context
    _exec_template(inherit, lclcontext, args=args, kwargs=kwargs)
  File "/virtualenv/local/lib/python2.7/site-packages/mako/runtime.py", line 824, in _exec_template
    callable_(context, *args, **kwargs)
  File "pyramid_debugtoolbar_panels_templates_renderings_dbtmako", line 31, in render_body
MakoRenderingException: 

Traceback (most recent call last):
  File "/virtualenv/local/lib/python2.7/site-packages/pyramid/mako_templating.py", line 211, in __call__
    result = template.render_unicode(**system)
  File "/virtualenv/local/lib/python2.7/site-packages/mako/template.py", line 421, in render_unicode
    as_unicode=True)
  File "/virtualenv/local/lib/python2.7/site-packages/mako/runtime.py", line 766, in _render
    **_kwargs_for_callable(callable_, data))
  File "/virtualenv/local/lib/python2.7/site-packages/mako/runtime.py", line 798, in _render_context
    _exec_template(inherit, lclcontext, args=args, kwargs=kwargs)
  File "/virtualenv/local/lib/python2.7/site-packages/mako/runtime.py", line 824, in _exec_template
    callable_(context, *args, **kwargs)
  File "/virtualenv/lib/python2.7/site-packages/pyramid_debugtoolbar-1.0.3-py2.7.egg/pyramid_debugtoolbar/templates/toolbar.dbtmako", line 60, in render_body
    ${panel.content()|n}
  File "/virtualenv/lib/python2.7/site-packages/pyramid_debugtoolbar-1.0.3-py2.7.egg/pyramid_debugtoolbar/panels/renderings.py", line 46, in content
    }, request=self.request)
  File "/virtualenv/lib/python2.7/site-packages/pyramid_debugtoolbar-1.0.3-py2.7.egg/pyramid_debugtoolbar/panels/__init__.py", line 24, in render
    return render(template_name, vars, request=request)
  File "/virtualenv/local/lib/python2.7/site-packages/pyramid/renderers.py", line 88, in render
    return helper.render(value, None, request=request)
  File "/virtualenv/local/lib/python2.7/site-packages/pyramid/renderers.py", line 550, in render
    result = renderer(value, system_values)
  File "/virtualenv/local/lib/python2.7/site-packages/pyramid/mako_templating.py", line 219, in __call__
    reraise(MakoRenderingException(errtext), None, exc_info[2])
  File "/virtualenv/local/lib/python2.7/site-packages/pyramid/mako_templating.py", line 211, in __call__
    result = template.render_unicode(**system)
  File "/virtualenv/local/lib/python2.7/site-packages/mako/template.py", line 421, in render_unicode
    as_unicode=True)
  File "/virtualenv/local/lib/python2.7/site-packages/mako/runtime.py", line 766, in _render
    **_kwargs_for_callable(callable_, data))
  File "/virtualenv/local/lib/python2.7/site-packages/mako/runtime.py", line 798, in _render_context
    _exec_template(inherit, lclcontext, args=args, kwargs=kwargs)
  File "/virtualenv/local/lib/python2.7/site-packages/mako/runtime.py", line 824, in _exec_template
    callable_(context, *args, **kwargs)
  File "/virtualenv/lib/python2.7/site-packages/pyramid_debugtoolbar-1.0.3-py2.7.egg/pyramid_debugtoolbar/panels/templates/renderings.dbtmako", line 21, in render_body
    <td colspan="2">${rendering['val']|h}</td>
MakoRenderingException: 

Traceback (most recent call last):
  File "/virtualenv/local/lib/python2.7/site-packages/pyramid/mako_templating.py", line 211, in __call__
    result = template.render_unicode(**system)
  File "/virtualenv/local/lib/python2.7/site-packages/mako/template.py", line 421, in render_unicode
    as_unicode=True)
  File "/virtualenv/local/lib/python2.7/site-packages/mako/runtime.py", line 766, in _render
    **_kwargs_for_callable(callable_, data))
  File "/virtualenv/local/lib/python2.7/site-packages/mako/runtime.py", line 798, in _render_context
    _exec_template(inherit, lclcontext, args=args, kwargs=kwargs)
  File "/virtualenv/local/lib/python2.7/site-packages/mako/runtime.py", line 824, in _exec_template
    callable_(context, *args, **kwargs)
  File "/virtualenv/lib/python2.7/site-packages/pyramid_debugtoolbar-1.0.3-py2.7.egg/pyramid_debugtoolbar/panels/templates/renderings.dbtmako", line 21, in render_body
    <td colspan="2">${rendering['val']|h}</td>
UnicodeDecodeError: 'ascii' codec can't decode byte 0xc3 in position 76: ordinal not in range(128)
@brodul

Hi,

Please see this pull request Pylons/pyramid_debugtoolbar#108.
This should fix the problems with utf-8 encoded exception messages. But the ones with latin-1 persist.

I recommend we introduce a new setting to the debug toolbar that will handle the decoding of database errors separately. Since we can not catch all the supported charsets.

An example of supported charsets by PostgreSQL
http://www.postgresql.org/docs/8.2/static/multibyte.html#CHARSET-TABLE

I recommend we should use utf-8 as the default.

Feedback highly appreciated.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Oct 26, 2011
  1. @Cito
Commits on Oct 27, 2011
  1. @Cito
This page is out of date. Refresh to see the latest.
View
30 pyramid_debugtoolbar/compat.py
@@ -21,25 +21,45 @@
# TODO check if errors is ever used
-def text_(s, encoding='latin-1', errors='strict'):
+def text_(s, encoding=None, errors='strict'):
if isinstance(s, binary_type):
+ if not encoding:
+ try:
+ return s.decode('utf-8')
+ except UnicodeDecodeError:
+ encoding = 'latin-1'
return s.decode(encoding, errors)
return s # pragma: no cover
-def bytes_(s, encoding='latin-1', errors='strict'):
+def bytes_(s, encoding=None, errors='strict'):
if isinstance(s, text_type): # pragma: no cover
+ if not encoding:
+ try:
+ return s.encode('latin-1')
+ except UnicodeEncodeError:
+ encoding = 'utf-8'
return s.encode(encoding, errors)
return s
if PY3: # pragma: no cover
- def native_(s, encoding='latin-1', errors='strict'):
+ def native_(s, encoding=None, errors='strict'):
if isinstance(s, text_type):
return s
+ if not encoding:
+ try:
+ return str(s, 'utf-8')
+ except UnicodeDecodeError:
+ encoding = 'latin-1'
return str(s, encoding, errors)
else:
- def native_(s, encoding='latin-1', errors='strict'):
+ def native_(s, encoding=None, errors='strict'):
if isinstance(s, text_type): # pragma: no cover
return s.encode(encoding, errors)
+ if not encoding:
+ try:
+ return s.encode('latin-1')
+ except UnicodeEncodeError:
+ encoding = 'utf-8'
return str(s)
if PY3: # pragma: no cover
@@ -124,7 +144,7 @@ def exec_(code, globs=None, locs=None):
xrange_ = range
else:
xrange_ = xrange
-
+
if PY3: # pragma: no cover
def iteritems_(d):
View
10 pyramid_debugtoolbar/tbtools.py
@@ -218,15 +218,15 @@ def render_summary(self, include_title=True, request=None):
))
if self.is_syntax_error:
- description_wrapper = text_('<pre class=syntaxerror>%s</pre>')
+ description_wrapper = '<pre class=syntaxerror>%s</pre>'
else:
- description_wrapper = text_('<blockquote>%s</blockquote>')
+ description_wrapper = '<blockquote>%s</blockquote>'
vars = {
'classes': text_(' '.join(classes)),
'title': title and text_('<h3>%s</h3>' % title) or text_(''),
'frames': text_('\n'.join(frames)),
- 'description': description_wrapper % escape(self.exception),
+ 'description': text_(description_wrapper % escape(self.exception)),
}
return render('pyramid_debugtoolbar:templates/exception_summary.mako',
vars, request=request)
@@ -235,7 +235,7 @@ def render_full(self, request, lodgeit_url=None):
"""Render the Full HTML page with the traceback info."""
static_path = request.static_url(STATIC_PATH)
root_path = request.route_url(ROOT_ROUTE_NAME)
- exc = escape(self.exception)
+ exc = text_(escape(self.exception))
summary = self.render_summary(include_title=False, request=request)
qs = {'token':request.exc_history.token, 'tb':str(self.id)}
url = request.route_url(EXC_ROUTE_NAME, _query=qs)
@@ -269,7 +269,7 @@ def generate_plaintext_traceback(self):
frame.function_name
))
yield text_(' ' + frame.current_line.strip())
- yield self.exception
+ yield text_(self.exception)
@reify
def plaintext(self):
View
7 pyramid_debugtoolbar/templates/console.mako
@@ -2,13 +2,14 @@
"http://www.w3.org/TR/html4/loose.dtd">
<html>
<head>
+ <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
<title>${title} // Werkzeug Debugger</title>
- <link rel="stylesheet" href="${static_path}css/debugger.css"
+ <link rel="stylesheet" href="${static_path}css/debugger.css"
type="text/css">
- <script type="text/javascript"
+ <script type="text/javascript"
src="${static_path}js/jquery-1.6.4.min.js"></script>
<script type="text/javascript">var jq = jQuery.noConflict(true);</script>
- <script type="text/javascript"
+ <script type="text/javascript"
src="${static_path}js/debugger.js"></script>
<script type="text/javascript">
var TRACEBACK = ${traceback_id},
View
33 pyramid_debugtoolbar/templates/exception.mako
@@ -3,13 +3,14 @@
<html>
<head>
+ <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
<title>${title} // Werkzeug Debugger</title>
- <link rel="stylesheet" href="${static_path}css/debugger.css"
+ <link rel="stylesheet" href="${static_path}css/debugger.css"
type="text/css">
- <script type="text/javascript"
+ <script type="text/javascript"
src="${static_path}js/jquery-1.6.4.min.js"></script>
<script type="text/javascript">var jq = jQuery.noConflict(true);</script>
- <script type="text/javascript"
+ <script type="text/javascript"
src="${static_path}js/debugger.js"></script>
<script type="text/javascript">
var TRACEBACK = ${traceback_id},
@@ -34,10 +35,10 @@
<input type="hidden" name="language" value="pytb">
This is the Copy/Paste friendly version of the traceback.
</p>
- <textarea cols="50" rows="10" name="code"
+ <textarea cols="50" rows="10" name="code"
readonly>${plaintext}</textarea>
</div>
-
+
<div class="explanation">
<p>
<b>Warning: this feature should not be enabled on production
@@ -46,27 +47,27 @@
% if evalex:
<p>
-
+
Hover over any gray area in the traceback and click on the
<img src="${static_path}img/console.png"/> icon on the right hand
side of that gray area to show an interactive console for the
associated frame. Type arbitrary Python into the console; it will be
evaluated in the context of the associated frame. In the interactive
console there are helpers available for introspection:
-
+
<ul>
<li><code>dump()</code> shows all variables in the frame
<li><code>dump(obj)</code> dumps all that's known about the object
</ul>
</p>
% endif
-
+
<p>
- Hover over any gray area in the traceback and click on
- <img src="${static_path}img/source.png"/> on the right hand side
+ Hover over any gray area in the traceback and click on
+ <img src="${static_path}img/source.png"/> on the right hand side
of that gray area to show the source of the file associated with the frame.
</p>
-
+
<p>
Click on the traceback header to switch back and forth between the
rendered version of the traceback and a plaintext copy-paste-friendly
@@ -77,18 +78,18 @@
URL to recover this traceback page: <a href="${url}">${url}</a>
</p>
</div>
-
+
<div class="footer">
Brought to you by <strong class="arthur">DONT PANIC</strong>, your
friendly Werkzeug powered traceback interpreter.
</div>
</div>
<!--
-
+
${plaintext_cs}
-
+
-->
-
+
</body>
-
+
</html>
View
37 pyramid_debugtoolbar/tests/test_toolbar.py
@@ -12,7 +12,7 @@ def setUp(self):
def tearDown(self):
del self.config
-
+
def _makeOne(self, request, panel_classes):
from pyramid_debugtoolbar.toolbar import DebugToolbar
return DebugToolbar(request, panel_classes)
@@ -107,7 +107,7 @@ def setUp(self):
def tearDown(self):
testing.tearDown()
-
+
def _callFUT(self, handler, registry):
from pyramid_debugtoolbar.toolbar import toolbar_tween_factory
return toolbar_tween_factory(handler, registry)
@@ -116,7 +116,7 @@ def test_it_disabled(self):
def handler(): pass
result = self._callFUT(handler, self.config.registry)
self.assertTrue(result is handler)
-
+
def test_it_enabled(self):
self.config.registry.settings['debugtoolbar.enabled'] = True
def handler(): pass
@@ -143,7 +143,7 @@ def _makeHandler(self):
def handler(request):
return self.response
return handler
-
+
def _callFUT(self, request, handler=None):
registry = self.config.registry
if request.remote_addr is None:
@@ -197,6 +197,33 @@ def handler(request):
self.assertEqual(len(request.exc_history.tracebacks), 1)
self.assertFalse(hasattr(request, 'debug_toolbar'))
self.assertTrue(response.status_int, 500)
+ self.assertTrue('NotImplementedError' in response.body)
+
+ def test_intercept_exc_with_latin1_message(self):
+ request = Request.blank('/')
+ def handler(request):
+ raise NotImplementedError(u'K\xe4se!'.encode('latin-1'))
+ self.config.registry.settings['debugtoolbar.intercept_exc'] = True
+ self.config.registry.settings['debugtoolbar.secret'] = 'abc'
+ self.config.add_route('debugtoolbar.exception', '/exception')
+ request.registry = self.config.registry
+ response = self._callFUT(request, handler)
+ self.assertTrue(response.status_int, 500)
+ self.assertTrue('charset=UTF-8' in response.body and
+ u'NotImplementedError: K\xe4se!'.encode('utf-8') in response.body)
+
+ def test_intercept_exc_with_utf8_message(self):
+ request = Request.blank('/')
+ def handler(request):
+ raise NotImplementedError(u'K\xe4se!'.encode('utf-8'))
+ self.config.registry.settings['debugtoolbar.intercept_exc'] = True
+ self.config.registry.settings['debugtoolbar.secret'] = 'abc'
+ self.config.add_route('debugtoolbar.exception', '/exception')
+ request.registry = self.config.registry
+ response = self._callFUT(request, handler)
+ self.assertTrue(response.status_int, 500)
+ self.assertTrue('charset=UTF-8' in response.body and
+ u'NotImplementedError: K\xe4se!'.encode('utf-8') in response.body)
def test_it_intercept_redirect_nonredirect_code(self):
request = Request.blank('/')
@@ -232,7 +259,7 @@ def process_response(self, response):
def wrap_handler(self, handler):
handler.wrapped = True
return handler
-
+
def dom_id(self):
return 'id'
View
17 pyramid_debugtoolbar/tests/test_utils.py
@@ -33,18 +33,19 @@ def test_relpath(self):
def test_unknown(self):
val = '..' + os.path.sep + 'foo'
- self.assertEqual(self._callFUT(val), './../foo')
+ self.assertEqual(self._callFUT(val), os.path.join('.', '..', 'foo'))
def test_module_file_path(self):
+ root = os.path.sep
sys_path = [
- '/foo/',
- '/foo/bar',
- '/usr/local/python/site-packages/',
+ os.path.join(root, 'foo', ''),
+ os.path.join(root, 'foo', 'bar'),
+ os.path.join(root, 'usr', 'local', 'python', 'site-packages')
]
- modpath = self._callFUT(
- '/foo/bar/pyramid_debugtoolbar/tests/debugfoo.py', sys_path)
- self.assertEqual(modpath,
- '<pyramid_debugtoolbar/tests/debugfoo.py>')
+ modpath = self._callFUT(os.path.join(root, 'foo', 'bar',
+ 'pyramid_debugtoolbar', 'tests', 'debugfoo.py'), sys_path)
+ self.assertEqual(modpath, '<%s>' %
+ os.path.join('pyramid_debugtoolbar', 'tests', 'debugfoo.py'))
def test_no_matching_sys_path(self):
val = '/foo/bar/pyramid_debugtoolbar/foo.py'
View
4 pyramid_debugtoolbar/tests/test_views.py
@@ -50,7 +50,7 @@ def test_with_bad_token_in_request(self):
request = self._makeRequest()
request.params['token'] = 'wrong'
self.assertRaises(HTTPBadRequest, self._makeOne, request)
-
+
def test_source(self):
request = self._makeRequest()
request.params['frm'] = '0'
@@ -163,6 +163,6 @@ def eval(self, cmd):
class DummyFrame(object):
def __init__(self):
self.console = DummyConsole()
-
+
def render_source(self):
return 'source'
View
2  pyramid_debugtoolbar/utils.py
@@ -28,7 +28,7 @@
def format_fname(value, _sys_path=None):
if _sys_path is None:
_sys_path = sys.path # dependency injection
- # If the value is not an absolute path, the it is a builtin or
+ # If the value is not an absolute path, then it is a builtin or
# a relative file (thus a project file).
if not os.path.isabs(value):
if value.startswith(('{', '<')):
Something went wrong with that request. Please try again.