Open template files in binary mode (to be decoded as utf8 later)

Text mode in python 3 uses an environment-dependent encoding, so
add a test and run it in both C and utf-8 locales.

MANIFEST.in

@@ -5,4 +5,5 @@ include tornado/test/README
 include tornado/test/test.crt
 include tornado/test/test.key
 include tornado/test/static/robots.txt
+include tornado/test/templates/utf8.html
 global-exclude _auto2to3*

setup.py

@@ -45,7 +45,8 @@
     packages = ["tornado", "tornado.test", "tornado.platform"],
     package_data = {
         "tornado": ["ca-certificates.crt"],
-        "tornado.test": ["README", "test.crt", "test.key", "static/robots.txt"],
+        "tornado.test": ["README", "test.crt", "test.key", "static/robots.txt",
+                         "templates/utf8.html"],
         },
     ext_modules = extensions,
     author="Facebook",

tornado/template.py

@@ -352,7 +352,7 @@ def resolve_path(self, name, parent_path=None):
 
     def _create_template(self, name):
         path = os.path.join(self.root, name)
-        f = open(path, "r")
+        f = open(path, "rb")
         template = Template(f.read(), name=name, loader=self)
         f.close()
         return template

tornado/test/template_test.py

@@ -1,9 +1,10 @@
 from __future__ import absolute_import, division, with_statement
 
+import os
 import traceback
 
-from tornado.escape import utf8, native_str
-from tornado.template import Template, DictLoader, ParseError
+from tornado.escape import utf8, native_str, to_unicode
+from tornado.template import Template, DictLoader, ParseError, Loader
 from tornado.testing import LogTrapTestCase
 from tornado.util import b, bytes_type, ObjectDict
 
@@ -310,3 +311,12 @@ def render(template, name):
                          b("""s = "';sys.exit()"\n"""))
         self.assertEqual(render("foo.py", ["not a string"]),
                          b("""s = "['not a string']"\n"""))
+
+class TemplateLoaderTest(LogTrapTestCase):
+    def setUp(self):
+        self.loader = Loader(os.path.join(os.path.dirname(__file__), "templates"))
+
+    def test_utf8_in_file(self):
+        tmpl = self.loader.load("utf8.html")
+        result = tmpl.generate()
+        self.assertEqual(to_unicode(result).strip(), u"H\u00e9llo")

tornado/test/templates/utf8.html

@@ -0,0 +1 @@
+Héllo

tox.ini

@@ -13,7 +13,7 @@
 [tox]
 # "-full" variants include optional dependencies, to ensure
 # that things work both in a bare install and with all the extras.
-envlist = py27-full, py27-curl, py25-full, py32, pypy, py25, py26, py26-full, py27, py33
+envlist = py27-full, py27-curl, py25-full, py32, pypy, py25, py26, py26-full, py27, py32-utf8, py33
 [testenv]
 commands = python -m tornado.test.runtests {posargs:}
 
@@ -69,6 +69,17 @@ commands = python -m tornado.test.runtests --httpclient=tornado.curl_httpclient.
 # twisted under pypy takes a *very* long time.  MySQL-python builds with
 # pypy, but doesn't work.
 
+# In python 3, opening files in text mode uses a system-dependent encoding by
+# default.  Run the tests with "C" (ascii) and "utf-8" locales to ensure
+# we don't have hidden dependencies on this setting.
+[testenv:py32]
+basepython = python3.2
+setenv = LANG=C
+
+[testenv:py32-utf8]
+basepython = python3.2
+setenv = LANG=en_US.utf-8
+
 # No py32-full yet: none of our dependencies currently work on python3.
 
 [testenv:py33]