Improvement on filters #66

Closed
wants to merge 9 commits into
from
View
17 jinja2/environment.py
@@ -347,34 +347,29 @@ def iter_extensions(self):
return iter(sorted(self.extensions.values(),
key=lambda x: x.priority))
- def getitem(self, obj, argument):
+ def getitem(self, obj, argument, strict=False):
"""Get an item or attribute of an object but prefer the item."""
try:
return obj[argument]
except (TypeError, LookupError):
- if isinstance(argument, basestring):
+ if not strict and isinstance(argument, basestring):
try:
attr = str(argument)
except Exception:
pass
else:
- try:
- return getattr(obj, attr)
- except AttributeError:
- pass
+ return self.getattr(obj, attr, strict=True)
return self.undefined(obj=obj, name=argument)
- def getattr(self, obj, attribute):
+ def getattr(self, obj, attribute, strict=False):
"""Get an item or attribute of an object but prefer the attribute.
Unlike :meth:`getitem` the attribute *must* be a bytestring.
"""
try:
return getattr(obj, attribute)
except AttributeError:
- pass
- try:
- return obj[attribute]
- except (TypeError, LookupError, AttributeError):
+ if not strict:
+ return self.getitem(obj, attribute, strict=True)
return self.undefined(obj=obj, name=attribute)
@internalcode
View
302 jinja2/filters.py
@@ -11,9 +11,9 @@
import re
import math
from random import choice
-from operator import itemgetter
+from operator import itemgetter, not_
from itertools import imap, groupby
-from jinja2.utils import Markup, escape, pformat, urlize, soft_unicode
+from jinja2.utils import Markup, escape, pformat, urlize, soft_unicode, min, max
from jinja2.runtime import Undefined
from jinja2.exceptions import FilterArgumentError
@@ -62,6 +62,55 @@ def attrgetter(item):
return item
return attrgetter
+def make_sort_func(func, case_sensitive):
+ """Returns a callable that is usable as key function for the sorted(),
+ built-in function, respecting the case sensitivity.
+ """
+ if case_sensitive:
+ return func
+
+ def sort_func(item):
+ if func is not None:
+ item = func(item)
+
+ if isinstance(item, basestring):
+ return item.lower()
+ return item
+
+ return sort_func
+
+def make_map_and_filter_func(environment, filters_or_tests, *args, **kwargs):
+ """Parses arguments of filters like `map` and `filter` and returns a
+ callable usable for Python's map() and filter() built-in. Those filters
+ can be called in following ways:
+
+ * With the name of a filter (in the case of `map`) or a test (in the case
+ of `filter` and `filterfalse`) as first positional argument.
+ All additional arguments are passed to the filter/test.
+ * With the keyword argument `attribute`. So an attribute lookup is performed
+ on the items in the sequence, instead of a filter or test.
+ * With no arguments at all. In that case this function returns None, so when
+ passed to the filter() built-in for exmpale, only items that evaluate to
+ True are returned.
+ """
+ if args:
+ name, args = args[0], args[1:]
+
+ try:
+ func = filters_or_tests[name]
+ except KeyError:
+ raise FilterArgumentError("there is no filter/test '%s'" % name)
+
+ return lambda x: func(x, *args, **kwargs)
+
+ attribute = kwargs.pop('attribute', None)
+
+ if kwargs:
+ raise FilterArgumentError('got an unexpected keyword argument')
+
+ if attribute is not None:
+ return make_attrgetter(environment, attribute)
+
def do_forceescape(value):
"""Enforce HTML escaping. This will probably double escape variables."""
@@ -182,13 +231,8 @@ def do_dictsort(value, case_sensitive=False, by='key'):
else:
raise FilterArgumentError('You can only sort by either '
'"key" or "value"')
- def sort_func(item):
- value = item[pos]
- if isinstance(value, basestring) and not case_sensitive:
- value = value.lower()
- return value
- return sorted(value.items(), key=sort_func)
+ return sorted(value.items(), key=make_sort_func(itemgetter(pos), case_sensitive))
@environmentfilter
@@ -219,18 +263,55 @@ def do_sort(environment, value, reverse=False, case_sensitive=False,
.. versionchanged:: 2.6
The `attribute` parameter was added.
"""
- if not case_sensitive:
- def sort_func(item):
- if isinstance(item, basestring):
- item = item.lower()
- return item
- else:
- sort_func = None
if attribute is not None:
getter = make_attrgetter(environment, attribute)
- def sort_func(item, processor=sort_func or (lambda x: x)):
- return processor(getter(item))
- return sorted(value, key=sort_func, reverse=reverse)
+ else:
+ getter = None
+ return sorted(value, key=make_sort_func(getter, case_sensitive), reverse=reverse)
+
+
+@environmentfilter
+def do_unique(environment, iterable, case_sensitive=False):
+ """Return a list of unique items from the the given iterable.
+
+ .. sourcecode:: jinja
+
+ {{ ['foo', 'bar', 'foobar', 'FooBar']|unique }}
+ -> ['foo', 'bar', 'foobar']
+
+ This filter complements the `groupby` filter, which sorts and groups an
+ iterable by a certain attribute. The `unique` filter groups the items
+ from the iterable by themself instead and always returns a flat list of
+ unique items. That can be useuful for example when you need to concatenate
+ that items:
+
+ .. sourcecode:: jinja
+
+ {{ ['foo', 'bar', 'foobar', 'FooBar']|unique|join(',') }}
+ -> foo,bar,foobar
+
+ Also note that the resulting list contains the items in the same order
+ as their first occurence in the iterable passed to the filter. If sorting
+ is needed you can still chain the `unique` and `sort` filter:
+
+ .. sourcecode:: jinja
+
+ {{ ['foo', 'bar', 'foobar', 'FooBar']|unique|sort }}
+ -> ['bar', 'foo', 'foobar']
+ """
+ sort_func = make_sort_func(lambda x: x, case_sensitive)
+
+ uniq_items = []
+ norm_items = []
+
+ for item in iterable:
+ norm_item = sort_func(item)
+
+ if norm_item not in norm_items:
+ norm_items.append(norm_item)
+ uniq_items.append(item)
+
+ return uniq_items
def do_default(value, default_value=u'', boolean=False):
@@ -323,7 +404,17 @@ def do_first(environment, seq):
def do_last(environment, seq):
"""Return the last item of a sequence."""
try:
- return iter(reversed(seq)).next()
+ try:
+ rv = reversed(seq).next()
+ except TypeError:
+ it = iter(seq)
+ rv = it.next()
+ while True:
+ try:
+ rv = it.next()
+ except StopIteration:
+ break
+ return rv
except StopIteration:
return environment.undefined('No last item, sequence was empty.')
@@ -337,6 +428,64 @@ def do_random(environment, seq):
return environment.undefined('No random item, sequence was empty.')
+@environmentfilter
+def do_min(environment, seq, attribute=None, case_sensitive=False):
+ """Return the smallest item from the sequence.
+
+ .. sourcecode:: jinja
+
+ {{ [1, 2, 3]|min }}
+ -> 1
+
+ It is also possible to get the item providing the smallest value for a
+ certain attribute:
+
+ .. sourcecode:: jinja
+
+ {{ users|min('last_login') }}
+ """
+ if attribute is not None:
+ getter = make_attrgetter(environment, attribute)
+ else:
+ getter = None
+ key = make_sort_func(getter, case_sensitive)
+ try:
+ if key is None:
+ return min(seq)
+ return min(seq, key=key)
+ except ValueError:
+ return environment.undefined('No smallest item, sequence was empty.')
+
+
+@environmentfilter
+def do_max(environment, seq, attribute=None, case_sensitive=False):
+ """Return the largest item from the sequence.
+
+ .. sourcecode:: jinja
+
+ {{ [1, 2, 3]|max }}
+ -> 3
+
+ It is also possible to get the item providing the largest value for a
+ certain attribute:
+
+ .. sourcecode:: jinja
+
+ {{ users|max('last_login') }}
+ """
+ if attribute is not None:
+ getter = make_attrgetter(environment, attribute)
+ else:
+ getter = None
+ key = make_sort_func(getter, case_sensitive)
+ try:
+ if key is None:
+ return max(seq)
+ return max(seq, key=key)
+ except ValueError:
+ return environment.undefined('No largest item, sequence was empty.')
+
+
def do_filesizeformat(value, binary=False):
"""Format the value like a 'human-readable' file size (i.e. 13 kB,
4.1 MB, 102 Bytes, etc). Per default decimal prefixes are used (Mega,
@@ -584,6 +733,83 @@ def do_batch(value, linecount, fill_with=None):
yield tmp
+@environmentfilter
+def do_map(environment, iterable, *args, **kwargs):
+ """Map an iterable using filters or attribute lookups, returning a list
+ that holds the values processed by the given filter or provided by the
+ given attribute.
+
+ .. sourcecode:: jinja
+
+ {{ ['foo', 'bar']|map('upper') }}
+ -> ['FOO','BAR']
+
+ It is also possible to map the items to a certain attribute:
+
+ .. sourcecode:: jinja
+
+ {{ users|map(attribute='full_name') }}
+ """
+ func = make_map_and_filter_func(environment, environment.filters, *args, **kwargs)
+ return map(func, iterable)
+
+
+@environmentfilter
+def do_filter(environment, iterable, *args, **kwargs):
+ """Filter an iterable using tests or attribute lookups, returning only
+ items where the tests passes or the given attribute evaluates to True.
+
+ .. sourcecode:: jinja
+
+ {{ range(10)|filter('even') }}
+ -> [0,2,4,6,8]
+
+ If you don't specify a ``'test'``, all items that evaluate to False
+ (e.g. none, false, '', 0) are excluded:
+
+ .. sourcecode:: jinja
+
+ {{ [none,true,false,'foo','',42,0]|filter }}
+ -> [true,'foo',42]
+
+ It is also possible to filter the items by a certain attribute:
+
+ .. sourcecode:: jinja
+
+ {{ users|filter(attribute='is_staff') }}
+ """
+ func = make_map_and_filter_func(environment, environment.tests, *args, **kwargs)
+ return filter(func, iterable)
+
+
+@environmentfilter
+def do_filterfalse(environment, iterable, *args, **kwargs):
+ """Filter an iterable using tests or attribute lookups, returning only
+ items where the tests fails or the given attribute evalutes to False.
+
+ .. sourcecode:: jinja
+
+ {{ range(10)|filterfalse('even') }}
+ -> [1,3,5,7,9]
+
+ If you don't specify a ``'test'``, only items that evaluate to False
+ (e.g. none, false, '', 0) are included:
+
+ .. sourcecode:: jinja
+
+ {{ [none,true,false,'foo','',42,0]|filterfalse }}
+ -> [none,false,'',0]
+
+ It is also possible to filter the items by a certain attribute:
+
+ .. sourcecode:: jinja
+
+ {{ users|filterfalse(attribute='is_staff') }}
+ """
+ func = make_map_and_filter_func(environment, environment.tests, *args, **kwargs)
+ return filter(func and (lambda x: not func(x)) or not_, iterable)
+
+
def do_round(value, precision=0, method='common'):
"""Round the number to a given precision. The first
parameter specifies the precision (default is ``0``), the
@@ -619,7 +845,7 @@ def do_round(value, precision=0, method='common'):
@environmentfilter
-def do_groupby(environment, value, attribute):
+def do_groupby(environment, value, attribute, case_sensitive=False):
"""Group a sequence of objects by a common attribute.
If you for example have a list of dicts or objects that represent persons
@@ -657,17 +883,23 @@ def do_groupby(environment, value, attribute):
It's now possible to use dotted notation to group by the child
attribute of another attribute.
"""
- expr = make_attrgetter(environment, attribute)
- return sorted(map(_GroupTuple, groupby(sorted(value, key=expr), expr)))
+ getter = make_attrgetter(environment, attribute)
+ expr = make_sort_func(getter, case_sensitive)
+
+ rv = []
+ for _, list_ in groupby(sorted(value, key=expr), expr):
+ list_ = list(list_)
+ rv.append(_GroupTuple(getter(list_[0]), list_))
+ return rv
class _GroupTuple(tuple):
__slots__ = ()
grouper = property(itemgetter(0))
list = property(itemgetter(1))
- def __new__(cls, (key, value)):
- return tuple.__new__(cls, (key, list(value)))
+ def __new__(cls, key, list_):
+ return tuple.__new__(cls, (key, list_))
@environmentfilter
@@ -735,21 +967,7 @@ def do_attr(environment, obj, name):
See :ref:`Notes on subscriptions <notes-on-subscriptions>` for more details.
"""
- try:
- name = str(name)
- except UnicodeError:
- pass
- else:
- try:
- value = getattr(obj, name)
- except AttributeError:
- pass
- else:
- if environment.sandboxed and not \
- environment.is_safe_attribute(obj, name, value):
- return environment.unsafe_undefined(obj, name)
- return value
- return environment.undefined(obj=obj, name=name)
+ return environment.getattr(obj, name, strict=True)
FILTERS = {
@@ -768,6 +986,7 @@ def do_attr(environment, obj, name):
'count': len,
'dictsort': do_dictsort,
'sort': do_sort,
+ 'unique': do_unique,
'length': len,
'reverse': do_reverse,
'center': do_center,
@@ -777,6 +996,8 @@ def do_attr(environment, obj, name):
'first': do_first,
'last': do_last,
'random': do_random,
+ 'min': do_min,
+ 'max': do_max,
'filesizeformat': do_filesizeformat,
'pprint': do_pprint,
'truncate': do_truncate,
@@ -792,6 +1013,9 @@ def do_attr(environment, obj, name):
'striptags': do_striptags,
'slice': do_slice,
'batch': do_batch,
+ 'map': do_map,
+ 'filter': do_filter,
+ 'filterfalse': do_filterfalse,
'sum': do_sum,
'abs': abs,
'round': do_round,
View
38 jinja2/sandbox.py
@@ -294,43 +294,19 @@ def call_unop(self, context, operator, arg):
"""
return self.unop_table[operator](arg)
- def getitem(self, obj, argument):
- """Subscribe an object from sandboxed code."""
- try:
- return obj[argument]
- except (TypeError, LookupError):
- if isinstance(argument, basestring):
- try:
- attr = str(argument)
- except Exception:
- pass
- else:
- try:
- value = getattr(obj, attr)
- except AttributeError:
- pass
- else:
- if self.is_safe_attribute(obj, argument, value):
- return value
- return self.unsafe_undefined(obj, argument)
- return self.undefined(obj=obj, name=argument)
-
- def getattr(self, obj, attribute):
+ def getattr(self, obj, attribute, strict=False):
"""Subscribe an object from sandboxed code and prefer the
attribute. The attribute passed *must* be a bytestring.
"""
try:
value = getattr(obj, attribute)
except AttributeError:
- try:
- return obj[attribute]
- except (TypeError, LookupError):
- pass
- else:
- if self.is_safe_attribute(obj, attribute, value):
- return value
- return self.unsafe_undefined(obj, attribute)
- return self.undefined(obj=obj, name=attribute)
+ if not strict:
+ return self.getitem(obj, attribute, strict=True)
+ return self.undefined(obj=obj, name=attribute)
+ if self.is_safe_attribute(obj, attribute, value):
+ return value
+ return self.unsafe_undefined(obj, attribute)
def unsafe_undefined(self, obj, attribute):
"""Return an undefined object for unsafe attributes."""
View
33 jinja2/utils.py
@@ -11,6 +11,7 @@
import re
import sys
import errno
+import operator
try:
from thread import allocate_lock
except ImportError:
@@ -72,6 +73,38 @@ def next(x):
return x.next()
+# for python 2.4 we wrap the min and max builtins in order to support
+# the 'key' argument.
+def _make_min_or_max(builtin, op):
+ try:
+ builtin([1], key=lambda x: x)
+ except TypeError:
+ def min_or_max(*seq, **kwargs):
+ if 'key' not in kwargs:
+ return builtin(*seq, **kwargs)
+
+ if len(seq) == 1:
+ seq = seq[0]
+ key = kwargs['key']
+
+ def func(x, y):
+ k = key(y)
+ if x is None or op(k, x[1]):
+ return (y, k)
+ return x
+
+ result = reduce(func, seq, None)
+ if result is None:
+ raise ValueError(builtin.__name__ + '() arg is an empty sequence')
+ return result[0]
+ else:
+ min_or_max = builtin
+ return min_or_max
+min = _make_min_or_max(min, operator.lt)
+max = _make_min_or_max(max, operator.gt)
+del _make_min_or_max
+
+
# if this python version is unable to deal with unicode filenames
# when passed to encode we let this function encode it properly.
# This is used in a couple of places. As far as Jinja is concerned