diff --git a/MANIFEST.in b/MANIFEST.in index dcd3459ffa..3614dac5c8 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,4 +5,5 @@ include tornado/test/README include tornado/test/test.crt include tornado/test/test.key include tornado/test/static/robots.txt +include tornado/test/templates/utf8.html global-exclude _auto2to3* \ No newline at end of file diff --git a/setup.py b/setup.py index 872c737c4c..dca062d5c6 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,8 @@ packages = ["tornado", "tornado.test", "tornado.platform"], package_data = { "tornado": ["ca-certificates.crt"], - "tornado.test": ["README", "test.crt", "test.key", "static/robots.txt"], + "tornado.test": ["README", "test.crt", "test.key", "static/robots.txt", + "templates/utf8.html"], }, ext_modules = extensions, author="Facebook", diff --git a/tornado/template.py b/tornado/template.py index a211f39602..86a668facd 100644 --- a/tornado/template.py +++ b/tornado/template.py @@ -352,7 +352,7 @@ def resolve_path(self, name, parent_path=None): def _create_template(self, name): path = os.path.join(self.root, name) - f = open(path, "r") + f = open(path, "rb") template = Template(f.read(), name=name, loader=self) f.close() return template diff --git a/tornado/test/template_test.py b/tornado/test/template_test.py index c0175deea3..02f07be70c 100644 --- a/tornado/test/template_test.py +++ b/tornado/test/template_test.py @@ -1,9 +1,10 @@ from __future__ import absolute_import, division, with_statement +import os import traceback -from tornado.escape import utf8, native_str -from tornado.template import Template, DictLoader, ParseError +from tornado.escape import utf8, native_str, to_unicode +from tornado.template import Template, DictLoader, ParseError, Loader from tornado.testing import LogTrapTestCase from tornado.util import b, bytes_type, ObjectDict @@ -310,3 +311,12 @@ def render(template, name): b("""s = "';sys.exit()"\n""")) self.assertEqual(render("foo.py", ["not a string"]), b("""s = "['not a string']"\n""")) + +class TemplateLoaderTest(LogTrapTestCase): + def setUp(self): + self.loader = Loader(os.path.join(os.path.dirname(__file__), "templates")) + + def test_utf8_in_file(self): + tmpl = self.loader.load("utf8.html") + result = tmpl.generate() + self.assertEqual(to_unicode(result).strip(), u"H\u00e9llo") diff --git a/tornado/test/templates/utf8.html b/tornado/test/templates/utf8.html new file mode 100644 index 0000000000..c5253dfa86 --- /dev/null +++ b/tornado/test/templates/utf8.html @@ -0,0 +1 @@ +Héllo diff --git a/tox.ini b/tox.ini index e4f8e45616..0ab3581ca6 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,7 @@ [tox] # "-full" variants include optional dependencies, to ensure # that things work both in a bare install and with all the extras. -envlist = py27-full, py27-curl, py25-full, py32, pypy, py25, py26, py26-full, py27, py33 +envlist = py27-full, py27-curl, py25-full, py32, pypy, py25, py26, py26-full, py27, py32-utf8, py33 [testenv] commands = python -m tornado.test.runtests {posargs:} @@ -69,6 +69,17 @@ commands = python -m tornado.test.runtests --httpclient=tornado.curl_httpclient. # twisted under pypy takes a *very* long time. MySQL-python builds with # pypy, but doesn't work. +# In python 3, opening files in text mode uses a system-dependent encoding by +# default. Run the tests with "C" (ascii) and "utf-8" locales to ensure +# we don't have hidden dependencies on this setting. +[testenv:py32] +basepython = python3.2 +setenv = LANG=C + +[testenv:py32-utf8] +basepython = python3.2 +setenv = LANG=en_US.utf-8 + # No py32-full yet: none of our dependencies currently work on python3. [testenv:py33]