added first code for parser extensions and moved some code in speedup…

…s around

--HG--
branch : trunk

examples/translate.py

@@ -1,7 +1,6 @@
 from jinja2 import Environment
 
-print Environment().from_string("""\
-{{ foo.bar }}
+print Environment(parser_extensions=['jinja2.i18n.trans']).from_string("""\
 {% trans %}Hello {{ user }}!{% endtrans %}
 {% trans count=users|count %}{{ count }} user{% pluralize %}{{ count }} users{% endtrans %}
 """).render(user="someone")

jinja2/__init__.py

@@ -60,4 +60,5 @@
 from jinja2.loaders import BaseLoader, FileSystemLoader, PackageLoader, \
      DictLoader
 from jinja2.runtime import Undefined, DebugUndefined, StrictUndefined
+from jinja2.filters import environmentfilter, contextfilter
 from jinja2.utils import Markup, escape

jinja2/_speedups.c

@@ -46,19 +46,19 @@ escape_unicode(PyUnicodeObject *in)
 		switch (*inp++) {
 		case '&':
 			len += 5;
-			erepl++;
+			++erepl;
 			break;
 		case '"':
 			len += 6;
-			erepl++;
+			++erepl;
 			break;
 		case '<':
 		case '>':
 			len += 4;
-			erepl++;
+			++erepl;
 			break;
 		default:
-			len++;
+			++len;
 		}
 
 	/* Do we need to escape anything at all? */
@@ -84,22 +84,22 @@ escape_unicode(PyUnicodeObject *in)
 		case '&':
 			Py_UNICODE_COPY(outp, amp, 5);
 			outp += 5;
-			repl++;
+			++repl;
 			break;
 		case '"':
 			Py_UNICODE_COPY(outp, qt, 6);
 			outp += 6;
-			repl++;
+			++repl;
 			break;
 		case '<':
 			Py_UNICODE_COPY(outp, lt, 4);
 			outp += 4;
-			repl++;
+			++repl;
 			break;
 		case '>':
 			Py_UNICODE_COPY(outp, gt, 4);
 			outp += 4;
-			repl++;
+			++repl;
 			break;
 		default:
 			*outp++ = *inp;
@@ -112,17 +112,25 @@ escape_unicode(PyUnicodeObject *in)
 
 
 static PyObject*
-escape(PyObject *self, PyObject *args)
+soft_unicode(PyObject *self, PyObject *s)
 {
-	PyObject *text = NULL, *s = NULL, *rv = NULL;
-	if (!PyArg_UnpackTuple(args, "escape", 1, 1, &text))
-		return NULL;
+	if (!PyUnicode_Check(s))
+		return PyObject_Unicode(s);
+	Py_INCREF(s);
+	return s;
+}
+
+
+static PyObject*
+escape(PyObject *self, PyObject *text)
+{
+	PyObject *s = NULL, *rv = NULL;
 
 	/* we don't have to escape integers, bools or floats */
 	if (PyInt_CheckExact(text) || PyLong_CheckExact(text) ||
 	    PyFloat_CheckExact(text) || PyBool_Check(text) ||
 	    text == Py_None) {
-		args = PyTuple_New(1);
+		PyObject *args = PyTuple_New(1);
 		if (!args) {
 			Py_DECREF(s);
 			return NULL;
@@ -152,15 +160,7 @@ escape(PyObject *self, PyObject *args)
 		s = escape_unicode((PyUnicodeObject*)text);
 
 	/* convert the unicode string into a markup object. */
-	args = PyTuple_New(1);
-	if (!args) {
-		Py_DECREF(s);
-		return NULL;
-	}
-	PyTuple_SET_ITEM(args, 0, (PyObject*)s);
-	rv = PyObject_CallObject(markup, args);
-	Py_DECREF(args);
-	return rv;
+	return PyObject_CallFunctionObjArgs(markup, (PyObject*)s, NULL);
 }
 
 
@@ -192,11 +192,15 @@ tb_set_next(PyObject *self, PyObject *args)
 
 
 static PyMethodDef module_methods[] = {
-	{"escape", (PyCFunction)escape, METH_VARARGS,
+	{"escape", (PyCFunction)escape, METH_O,
 	 "escape(s) -> string\n\n"
 	 "Convert the characters &, <, >, and \" in string s to HTML-safe\n"
 	 "sequences. Use this if you need to display text that might contain\n"
 	 "such characters in HTML."},
+	{"soft_unicode", (PyCFunction)soft_unicode, METH_O,
+	 "soft_unicode(object) -> string\n\n"
+         "Make a string unicode if it isn't already.  That way a markup\n"
+         "string is not converted back to unicode."},
 	{"tb_set_next", (PyCFunction)tb_set_next, METH_VARARGS,
 	 "Set the tb_next member of a traceback object."},
 	{NULL, NULL, 0, NULL}		/* Sentinel */

jinja2/compiler.py

@@ -280,6 +280,12 @@ def __init__(self, environment, name, filename, stream=None):
         # the current indentation
         self._indentation = 0
 
+    def get_visitor(self, node):
+        """Custom nodes have their own compilation method."""
+        if isinstance(node, nodes.CustomStmt):
+            return node.compile
+        return NodeVisitor.get_visitor(self, node)
+
     def temporary_identifier(self):
         """Get a new unique identifier."""
         self._last_identifier += 1

jinja2/datastructure.py

@@ -13,9 +13,6 @@
 from jinja2.exceptions import TemplateSyntaxError, TemplateRuntimeError
 
 
-_missing = object()
-
-
 class Token(tuple):
     """
     Token class.
@@ -32,7 +29,27 @@ def __str__(self):
             return self.type
         elif self.type in reverse_operators:
             return reverse_operators[self.type]
-        return self.value
+        return '%s:%s' % (self.type, self.value)
+
+    def test(self, expr):
+        """Test a token against a token expression.  This can either be a
+        token type or 'token_type:token_value'.  This can only test against
+        string values!
+        """
+        # here we do a regular string equality check as test_many is usually
+        # passed an iterable of not interned strings.
+        if self.type == expr:
+            return True
+        elif ':' in expr:
+            return expr.split(':', 1) == [self.type, self.value]
+        return False
+
+    def test_many(self, iterable):
+        """Test against multiple token expressions."""
+        for expr in iterable:
+            if self.test(expr):
+                return True
+        return False
 
     def __repr__(self):
         return 'Token(%r, %r, %r)' % (
@@ -127,17 +144,11 @@ def close(self):
         self.current = Token(self.current.lineno, 'eof', '')
         self._next = None
 
-    def expect(self, token_type, token_value=_missing):
+    def expect(self, expr):
         """Expect a given token type and return it"""
-        if self.current.type is not token_type:
+        if not self.current.test(expr):
             raise TemplateSyntaxError("expected token %r, got %r" %
-                                      (token_type, self.current.type),
-                                      self.current.lineno,
-                                      self.filename)
-        elif token_value is not _missing and \
-             self.current.value != token_value:
-            raise TemplateSyntaxError("expected %r, got %r" %
-                                      (token_value, self.current.value),
+                                      (expr, self.current),
                                       self.current.lineno,
                                       self.filename)
         try:

jinja2/environment.py

@@ -10,11 +10,12 @@
 """
 import sys
 from jinja2.lexer import Lexer
-from jinja2.parser import Parser
+from jinja2.parser import Parser, ParserExtension
 from jinja2.optimizer import optimize
 from jinja2.compiler import generate
 from jinja2.runtime import Undefined
 from jinja2.debug import translate_exception
+from jinja2.utils import import_string
 from jinja2.defaults import DEFAULT_FILTERS, DEFAULT_TESTS, DEFAULT_NAMESPACE
 
 
@@ -43,6 +44,7 @@ def __init__(self,
                  optimized=True,
                  undefined=Undefined,
                  loader=None,
+                 parser_extensions=(),
                  finalize=unicode):
         """Here the possible initialization parameters:
 
@@ -68,6 +70,10 @@ def __init__(self,
         `undefined`               a subclass of `Undefined` that is used to
                                   represent undefined variables.
         `loader`                  the loader which should be used.
+        `parser_extensions`       List of parser extensions to use.
+        `finalize`                A callable that finalizes the variable.  Per
+                                  default this is `unicode`, other useful
+                                  builtin finalizers are `escape`.
         ========================= ============================================
         """
 
@@ -87,6 +93,14 @@ def __init__(self,
         self.comment_end_string = comment_end_string
         self.line_statement_prefix = line_statement_prefix
         self.trim_blocks = trim_blocks
+        self.parser_extensions = {}
+        for extension in parser_extensions:
+            if isinstance(extension, basestring):
+                extension = import_string(extension)
+            self.parser_extensions[extension.tag] = extension
+
+
+        # runtime information
         self.undefined = undefined
         self.optimized = optimized
         self.finalize = finalize

jinja2/filters.py

@@ -44,6 +44,13 @@ def environmentfilter(f):
     return f
 
 
+def do_forceescape(value):
+    """Enforce HTML escaping.  This will probably double escape variables."""
+    if hasattr(value, '__html__'):
+        value = value.__html__()
+    return escape(unicode(value))
+
+
 def do_replace(s, old, new, count=None):
     """Return a copy of the value with all occurrences of a substring
     replaced with a new one. The first argument is the substring
@@ -383,6 +390,7 @@ def do_int(value, default=0):
     try:
         return int(value)
     except (TypeError, ValueError):
+        # this quirk is necessary so that "42.23"|int gives 42.
         try:
             return int(float(value))
         except (TypeError, ValueError):
@@ -423,7 +431,9 @@ def do_trim(value):
 def do_striptags(value):
     """Strip SGML/XML tags and replace adjacent whitespace by one space.
     """
-    return ' '.join(_striptags_re.sub('', value).split())
+    if hasattr(value, '__html__'):
+        value = value.__html__()
+    return u' '.join(_striptags_re.sub('', value).split())
 
 
 def do_slice(value, slices, fill_with=None):
@@ -571,7 +581,7 @@ def do_groupby(environment, value, attribute):
     'lower':                do_lower,
     'escape':               escape,
     'e':                    escape,
-    'xmlattr':              do_xmlattr,
+    'forceescape':          do_forceescape,
     'capitalize':           do_capitalize,
     'title':                do_title,
     'default':              do_default,