Index: test/template.py
===================================================================
--- test/template.py (revision 336)
+++ test/template.py (working copy)
@@ -159,8 +159,23 @@
lookup = TemplateLookup(directories=['./test_htdocs'], filesystem_checks=True, output_encoding='utf-8')
template = lookup.get_template('/read_unicode.html')
data = template.render(path=os.path.join('./test_htdocs', 'internationalization.html'))
-
+ def test_unicode_template(self):
+ lookup = TemplateLookup(directories=['./test_htdocs'], disable_unicode=True, default_filters=[])
+ template = lookup.get_template('/chs_utf8.html')
+ assert flatten_result(template.render(name='毛泽东')) == '毛泽东 是 新中国的主席
Welcome 你 to 北京.'
+
+ lookup = TemplateLookup(directories=['./test_htdocs'], output_encoding='utf-8', disable_unicode=False, default_filters=['decode.utf8'])
+ template = lookup.get_template('/chs_unicode.html')
+ assert flatten_result(template.render(name='毛泽东')) == '毛泽东 是 新中国的主席
Welcome 你 to 北京.'
+
+ def test_unicode_off(self):
+ template = Template("""Alors vous imaginez ma surprise, au lever du jour, quand une drôle de petit voix m’a réveillé. Elle disait: « S’il vous plaît… dessine-moi un mouton! »""", disable_unicode=True)
+ assert template.render() == """Alors vous imaginez ma surprise, au lever du jour, quand une drôle de petit voix m’a réveillé. Elle disait: « S’il vous plaît… dessine-moi un mouton! »"""
+ template = Template("""Alors vous imaginez ma surprise, au lever du jour, quand une drôle de petit voix m’a réveillé. Elle disait: « S’il vous plaît… dessine-moi un mouton! »""", input_encoding='utf8', output_encoding='utf8', disable_unicode=False)
+ assert template.render() == """Alors vous imaginez ma surprise, au lever du jour, quand une drôle de petit voix m’a réveillé. Elle disait: « S’il vous plaît… dessine-moi un mouton! »"""
+
+
class PageArgsTest(unittest.TestCase):
def test_basic(self):
template = Template("""
Index: lib/mako/filters.py
===================================================================
--- lib/mako/filters.py (revision 336)
+++ lib/mako/filters.py (working copy)
@@ -26,7 +26,8 @@
def url_escape(string):
# convert into a list of octets
- string = string.encode("utf8")
+ if isinstance(string, unicode):
+ string = string.encode("utf8")
return urllib.quote_plus(string)
def url_unescape(string):
Index: lib/mako/lookup.py
===================================================================
--- lib/mako/lookup.py (revision 336)
+++ lib/mako/lookup.py (working copy)
@@ -38,8 +38,8 @@
class TemplateLookup(TemplateCollection):
def __init__(self, directories=None, module_directory=None, filesystem_checks=True, collection_size=-1, format_exceptions=False,
- error_handler=None, output_encoding=None, encoding_errors='strict', cache_type=None, cache_dir=None, cache_url=None,
- modulename_callable=None, default_filters=['unicode'], buffer_filters=[], imports=None, input_encoding=None, preprocessor=None):
+ error_handler=None, disable_unicode=True, output_encoding=None, encoding_errors='strict', cache_type=None, cache_dir=None, cache_url=None,
+ modulename_callable=None, default_filters=None, buffer_filters=[], imports=None, input_encoding=None, preprocessor=None):
if isinstance(directories, basestring):
directories = [directories]
self.directories = [posixpath.normpath(d) for d in directories or []]
@@ -47,7 +47,7 @@
self.modulename_callable = modulename_callable
self.filesystem_checks = filesystem_checks
self.collection_size = collection_size
- self.template_args = {'format_exceptions':format_exceptions, 'error_handler':error_handler, 'output_encoding':output_encoding, 'encoding_errors':encoding_errors, 'input_encoding':input_encoding, 'module_directory':module_directory, 'cache_type':cache_type, 'cache_dir':cache_dir or module_directory, 'cache_url':cache_url, 'default_filters':default_filters, 'buffer_filters':buffer_filters, 'imports':imports, 'preprocessor':preprocessor}
+ self.template_args = {'format_exceptions':format_exceptions, 'error_handler':error_handler, 'disable_unicode':disable_unicode, 'output_encoding':output_encoding, 'encoding_errors':encoding_errors, 'input_encoding':input_encoding, 'module_directory':module_directory, 'cache_type':cache_type, 'cache_dir':cache_dir or module_directory, 'cache_url':cache_url, 'default_filters':default_filters, 'buffer_filters':buffer_filters, 'imports':imports, 'preprocessor':preprocessor}
if collection_size == -1:
self.__collection = {}
self._uri_cache = {}
Index: lib/mako/template.py
===================================================================
--- lib/mako/template.py (revision 336)
+++ lib/mako/template.py (working copy)
@@ -18,7 +18,7 @@
"""a compiled template"""
def __init__(self, text=None, filename=None, uri=None, format_exceptions=False, error_handler=None,
lookup=None, output_encoding=None, encoding_errors='strict', module_directory=None, cache_type=None,
- cache_dir=None, cache_url=None, module_filename=None, input_encoding=None, default_filters=['unicode'],
+ cache_dir=None, cache_url=None, module_filename=None, input_encoding=None, disable_unicode=False, default_filters=None,
buffer_filters=[], imports=None, preprocessor=None):
"""construct a new Template instance using either literal template text, or a previously loaded template module
@@ -42,9 +42,22 @@
self.module_id = "memory:" + hex(id(self))
self.uri = self.module_id
- self.default_filters = default_filters
+ self.input_encoding = input_encoding
+ self.output_encoding = output_encoding
+ self.encoding_errors = encoding_errors
+ if output_encoding and output_encoding != input_encoding:
+ self.disable_unicode = False
+ else:
+ self.disable_unicode = disable_unicode
+ if default_filters is None:
+ if self.disable_unicode:
+ self.default_filters = ['str']
+ else:
+ self.default_filters = ['unicode']
+ else:
+ self.default_filters = default_filters
self.buffer_filters = buffer_filters
- self.input_encoding = input_encoding
+
self.imports = imports
self.preprocessor = preprocessor
@@ -94,8 +107,6 @@
self.format_exceptions = format_exceptions
self.error_handler = error_handler
self.lookup = lookup
- self.output_encoding = output_encoding
- self.encoding_errors = encoding_errors
self.cache_type = cache_type
self.cache_dir = cache_dir
self.cache_url = cache_url
@@ -141,6 +152,7 @@
self.buffer_filters = parent.buffer_filters
self.input_encoding = parent.input_encoding
self.imports = parent.imports
+ self.disable_unicode = parent.disable_unicode
self.output_encoding = parent.output_encoding
self.encoding_errors = parent.encoding_errors
self.format_exceptions = parent.format_exceptions
@@ -191,7 +203,7 @@
def _compile_text(template, text, filename):
identifier = template.module_id
- lexer = Lexer(text, filename, input_encoding=template.input_encoding, preprocessor=template.preprocessor)
+ lexer = Lexer(text, filename, disable_unicode=template.disable_unicode, input_encoding=template.input_encoding, preprocessor=template.preprocessor)
node = lexer.parse()
source = codegen.compile(node, template.uri, filename, default_filters=template.default_filters, buffer_filters=template.buffer_filters, imports=template.imports, source_encoding=lexer.encoding)
#print source
@@ -203,7 +215,7 @@
def _compile_module_file(template, text, filename, outputpath):
identifier = template.module_id
- lexer = Lexer(text, filename, input_encoding=template.input_encoding, preprocessor=template.preprocessor)
+ lexer = Lexer(text, filename, disable_unicode=template.disable_unicode, input_encoding=template.input_encoding, preprocessor=template.preprocessor)
node = lexer.parse()
source = codegen.compile(node, template.uri, filename, default_filters=template.default_filters, buffer_filters=template.buffer_filters, imports=template.imports, source_encoding=lexer.encoding)
(dest, name) = tempfile.mkstemp()
Index: lib/mako/util.py
===================================================================
--- lib/mako/util.py (revision 336)
+++ lib/mako/util.py (working copy)
@@ -62,7 +62,7 @@
if self.encoding:
return u''.join(self.data).encode(self.encoding, self.errors)
else:
- return u''.join(self.data)
+ return ''.join(self.data)
class LRUCache(dict):
"""A dictionary-like object that stores a limited number of items, discarding
Index: lib/mako/lexer.py
===================================================================
--- lib/mako/lexer.py (revision 336)
+++ lib/mako/lexer.py (working copy)
@@ -13,7 +13,7 @@
_regexp_cache = {}
class Lexer(object):
- def __init__(self, text, filename=None, input_encoding=None, preprocessor=None):
+ def __init__(self, text, filename=None, disable_unicode=False, input_encoding=None, preprocessor=None):
self.text = text
self.filename = filename
self.template = parsetree.TemplateNode(self.filename)
@@ -23,6 +23,7 @@
self.match_position = 0
self.tag = []
self.control_line = []
+ self.disable_unicode = disable_unicode
self.encoding = input_encoding
if preprocessor is None:
self.preprocessor = []
@@ -108,7 +109,7 @@
raise exceptions.SyntaxException("Keyword '%s' not a legal ternary for keyword '%s'" % (node.keyword, self.control_line[-1].keyword), **self.exception_kwargs)
def escape_code(self, text):
- if self.encoding:
+ if isinstance(text,unicode) and self.encoding:
return text.encode('ascii', 'backslashreplace')
else:
return text
@@ -126,8 +127,7 @@
parsed_encoding = self.match_encoding()
if parsed_encoding:
self.encoding = parsed_encoding
- if not isinstance(self.text, unicode):
-
+ if not self.disable_unicode and not isinstance(self.text, unicode):
if self.encoding:
try:
self.text = self.text.decode(self.encoding)
Index: lib/mako/runtime.py
===================================================================
--- lib/mako/runtime.py (revision 336)
+++ lib/mako/runtime.py (working copy)
@@ -289,7 +289,7 @@
"""create a Context and return the string output of the given template and template callable."""
if as_unicode:
buf = util.FastEncodingBuffer()
- elif template.output_encoding:
+ elif not template.disable_unicode and template.output_encoding:
buf = util.FastEncodingBuffer(template.output_encoding, template.encoding_errors)
else:
buf = util.StringIO()
Index: lib/mako/codegen.py
===================================================================
--- lib/mako/codegen.py (revision 336)
+++ lib/mako/codegen.py (working copy)
@@ -16,7 +16,7 @@
def compile(node, uri, filename=None, default_filters=None, buffer_filters=None, imports=None, source_encoding=None):
"""generate module source code given a parsetree node, uri, and optional source filename"""
- buf = util.FastEncodingBuffer()
+ buf = util.StringIO()
printer = PythonPrinter(buf)
_GenerateRenderMethod(printer, _CompileContext(uri, filename, default_filters, buffer_filters, imports, source_encoding), node)
return buf.getvalue()
@@ -110,6 +110,8 @@
module_identifiers.declared = module_ident
# module-level names, python code
+ if self.compiler.source_encoding:
+ self.printer.writeline("# -*- encoding:%s -*-" % self.compiler.source_encoding)
self.printer.writeline("from mako import runtime, filters, cache")
self.printer.writeline("UNDEFINED = runtime.UNDEFINED")
self.printer.writeline("_magic_number = %s" % repr(MAGIC_NUMBER))
@@ -159,7 +161,7 @@
)
if buffered or filtered or cached:
self.printer.writeline("context.push_buffer()")
-
+
self.identifier_stack.append(self.compiler.identifiers.branch(self.node))
if not self.in_def and '**pageargs' in args:
self.identifier_stack[-1].argument_declared.add('pageargs')
@@ -544,6 +546,7 @@
)
self.write_variable_declares(body_identifiers)
self.identifier_stack.append(body_identifiers)
+
for n in node.nodes:
n.accept_visitor(self)
self.identifier_stack.pop()
@@ -563,7 +566,7 @@
"try:")
self.write_source_comment(node)
self.printer.writelines(
- "context.write(unicode(%s))" % node.attributes['expr'],
+ "context.write(%s)" % self.create_filter_callable([], node.attributes['expr'], True),
"finally:",
"context.caller_stack.nextcaller = None",
None