Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Loading…

Specify what whitespace is. #74

Closed
wants to merge 2 commits into from

3 participants

Bert Wesarg Orion Poplawski Armin Ronacher
Bert Wesarg

I had a hard time to get the desired result out of my template into a form which fits our codingstyle. I use jinja to generate C code. But in C empty lines should be preserved as much as possible to increase readebility. And while I'm not only generate C code but C header files which will be installed, these empty lines are of high value for our project. Unfortunately, the \s in pythons re engine also matches \n, thus any jinja token also eats up its leading empty lines.

This pull request includes my current solution to this problem. It lets the user specify what whitespace is. In my project I just use [ \t].

bertwesarg added some commits
Bert Wesarg bertwesarg Use kwargs when instanciating the Environment.
This protects for failures, when adding new keywords.
44d0d81
Bert Wesarg bertwesarg Specify what whitespace is.
Unfortunately \s in pythons re engine also matches newlines. This makes
it very hard to use jinja where empty lines are of value. Let the user
specify what whitespace is.
cfdde52
Orion Poplawski

@mitsuhiko Is there any chance of this being accepted? Looks like otf2 is now shipping a bundled version of jinja2 with this applied. It would be nice to not have forked versions out in the wild if it could be avoided.

Armin Ronacher
Owner

Is there any chance that this is not necessary with the new lstrip changes?

Armin Ronacher mitsuhiko closed this
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Nov 21, 2011
  1. Bert Wesarg

    Use kwargs when instanciating the Environment.

    bertwesarg authored
    This protects for failures, when adding new keywords.
  2. Bert Wesarg

    Specify what whitespace is.

    bertwesarg authored
    Unfortunately \s in pythons re engine also matches newlines. This makes
    it very hard to use jinja where empty lines are of value. Let the user
    specify what whitespace is.
This page is out of date. Refresh to see the latest.
1  jinja2/defaults.py
View
@@ -22,6 +22,7 @@
LINE_COMMENT_PREFIX = None
TRIM_BLOCKS = False
NEWLINE_SEQUENCE = '\n'
+WHITESPACE_RE = r'\s'
# default filters, tests and namespace
15 jinja2/environment.py
View
@@ -10,6 +10,7 @@
"""
import os
import sys
+import re
from jinja2 import nodes
from jinja2.defaults import *
from jinja2.lexer import get_lexer, TokenStream
@@ -87,6 +88,8 @@ def _environment_sanity_check(environment):
'start strings must be different'
assert environment.newline_sequence in ('\r', '\r\n', '\n'), \
'newline_sequence set to unknown line ending string.'
+ assert re.compile(environment.whitespace_re, re.U) is not None, \
+ 'whitespace_re is not a valid regular expression.'
return environment
@@ -140,6 +143,11 @@ class Environment(object):
useful default for Linux and OS X systems as well as web
applications.
+ `whitespace_re`
+ An regular expression to match whitespace. Defaults to ``'\s'``.
+ Because the python ``'\s'`` matches also ``'\n'``, you may choose a
+ different set of whitespace characters.
+
`extensions`
List of Jinja extensions to use. This can either be import paths
as strings or extension classes. For more information have a
@@ -225,6 +233,7 @@ def __init__(self,
line_comment_prefix=LINE_COMMENT_PREFIX,
trim_blocks=TRIM_BLOCKS,
newline_sequence=NEWLINE_SEQUENCE,
+ whitespace_re=WHITESPACE_RE,
extensions=(),
optimized=True,
undefined=Undefined,
@@ -256,6 +265,7 @@ def __init__(self,
self.line_comment_prefix = line_comment_prefix
self.trim_blocks = trim_blocks
self.newline_sequence = newline_sequence
+ self.whitespace_re = whitespace_re
# runtime information
self.undefined = undefined
@@ -817,6 +827,7 @@ def __new__(cls, source,
line_comment_prefix=LINE_COMMENT_PREFIX,
trim_blocks=TRIM_BLOCKS,
newline_sequence=NEWLINE_SEQUENCE,
+ whitespace_re=WHITESPACE_RE,
extensions=(),
optimized=True,
undefined=Undefined,
@@ -826,8 +837,8 @@ def __new__(cls, source,
block_start_string, block_end_string, variable_start_string,
variable_end_string, comment_start_string, comment_end_string,
line_statement_prefix, line_comment_prefix, trim_blocks,
- newline_sequence, frozenset(extensions), optimized, undefined,
- finalize, autoescape, None, 0, False, None)
+ newline_sequence, whitespace_re, frozenset(extensions), optimized,
+ undefined, finalize, autoescape, None, 0, False, None)
return env.from_string(source, template_class=cls)
@classmethod
21 jinja2/ext.py
View
@@ -574,16 +574,17 @@ def getbool(options, key, default=False):
options.get(key, str(default)).lower() in ('1', 'on', 'yes', 'true')
environment = Environment(
- options.get('block_start_string', BLOCK_START_STRING),
- options.get('block_end_string', BLOCK_END_STRING),
- options.get('variable_start_string', VARIABLE_START_STRING),
- options.get('variable_end_string', VARIABLE_END_STRING),
- options.get('comment_start_string', COMMENT_START_STRING),
- options.get('comment_end_string', COMMENT_END_STRING),
- options.get('line_statement_prefix') or LINE_STATEMENT_PREFIX,
- options.get('line_comment_prefix') or LINE_COMMENT_PREFIX,
- getbool(options, 'trim_blocks', TRIM_BLOCKS),
- NEWLINE_SEQUENCE, frozenset(extensions),
+ block_start_string=options.get('block_start_string', BLOCK_START_STRING),
+ block_end_string=options.get('block_end_string', BLOCK_END_STRING),
+ variable_start_string=options.get('variable_start_string', VARIABLE_START_STRING),
+ variable_end_string=options.get('variable_end_string', VARIABLE_END_STRING),
+ comment_start_string=options.get('comment_start_string', COMMENT_START_STRING),
+ comment_end_string=options.get('comment_end_string', COMMENT_END_STRING),
+ line_statement_prefix=options.get('line_statement_prefix') or LINE_STATEMENT_PREFIX,
+ line_comment_prefix=options.get('line_comment_prefix') or LINE_COMMENT_PREFIX,
+ trim_blocks=getbool(options, 'trim_blocks', TRIM_BLOCKS),
+ newline_sequence=NEWLINE_SEQUENCE, whitespace_re=WHITESPACE_RE,
+ extensions=frozenset(extensions),
cache_size=0,
auto_reload=False
)
31 jinja2/lexer.py
View
@@ -197,7 +197,8 @@ def compile_rules(environment):
if environment.line_statement_prefix is not None:
rules.append((len(environment.line_statement_prefix), 'linestatement',
- r'^\s*' + e(environment.line_statement_prefix)))
+ (r'^%s*' % (environment.whitespace_re,)) +
+ e(environment.line_statement_prefix)))
if environment.line_comment_prefix is not None:
rules.append((len(environment.line_comment_prefix), 'linecomment',
r'(?:^|(?<=\S))[^\S\r\n]*' +
@@ -383,7 +384,8 @@ def get_lexer(environment):
environment.line_statement_prefix,
environment.line_comment_prefix,
environment.trim_blocks,
- environment.newline_sequence)
+ environment.newline_sequence,
+ environment.whitespace_re)
lexer = _lexer_cache.get(key)
if lexer is None:
lexer = Lexer(environment)
@@ -432,13 +434,17 @@ def __init__(self, environment):
'root': [
# directives
(c('(.*?)(?:%s)' % '|'.join(
- [r'(?P<raw_begin>(?:\s*%s\-|%s)\s*raw\s*(?:\-%s\s*|%s))' % (
+ [r'(?P<raw_begin>(?:%s*%s\-|%s)%s*raw%s*(?:\-%s%s*|%s))' % (
+ environment.whitespace_re,
e(environment.block_start_string),
e(environment.block_start_string),
+ environment.whitespace_re, environment.whitespace_re,
e(environment.block_end_string),
+ environment.whitespace_re,
e(environment.block_end_string)
)] + [
- r'(?P<%s_begin>\s*%s\-|%s)' % (n, r, r)
+ r'(?P<%s_begin>%s*%s\-|%s)' %
+ (n, environment.whitespace_re, r, r)
for n, r in root_tag_rules
])), (TOKEN_DATA, '#bygroup'), '#bygroup'),
# data
@@ -446,8 +452,9 @@ def __init__(self, environment):
],
# comments
TOKEN_COMMENT_BEGIN: [
- (c(r'(.*?)((?:\-%s\s*|%s)%s)' % (
+ (c(r'(.*?)((?:\-%s%s*|%s)%s)' % (
e(environment.comment_end_string),
+ environment.whitespace_re,
e(environment.comment_end_string),
block_suffix_re
)), (TOKEN_COMMENT, TOKEN_COMMENT_END), '#pop'),
@@ -455,25 +462,30 @@ def __init__(self, environment):
],
# blocks
TOKEN_BLOCK_BEGIN: [
- (c('(?:\-%s\s*|%s)%s' % (
+ (c('(?:\-%s%s*|%s)%s' % (
e(environment.block_end_string),
+ environment.whitespace_re,
e(environment.block_end_string),
block_suffix_re
)), TOKEN_BLOCK_END, '#pop'),
] + tag_rules,
# variables
TOKEN_VARIABLE_BEGIN: [
- (c('\-%s\s*|%s' % (
+ (c('\-%s%s*|%s' % (
e(environment.variable_end_string),
+ environment.whitespace_re,
e(environment.variable_end_string)
)), TOKEN_VARIABLE_END, '#pop')
] + tag_rules,
# raw block
TOKEN_RAW_BEGIN: [
- (c('(.*?)((?:\s*%s\-|%s)\s*endraw\s*(?:\-%s\s*|%s%s))' % (
+ (c('(.*?)((?:%s*%s\-|%s)%s*endraw%s*(?:\-%s%s*|%s%s))' % (
+ environment.whitespace_re,
e(environment.block_start_string),
e(environment.block_start_string),
+ environment.whitespace_re, environment.whitespace_re,
e(environment.block_end_string),
+ environment.whitespace_re,
e(environment.block_end_string),
block_suffix_re
)), (TOKEN_DATA, TOKEN_RAW_END), '#pop'),
@@ -481,7 +493,8 @@ def __init__(self, environment):
],
# line statements
TOKEN_LINESTATEMENT_BEGIN: [
- (c(r'\s*(\n|$)'), TOKEN_LINESTATEMENT_END, '#pop')
+ (c(r'%s*(\n|$)' % (environment.whitespace_re,)),
+ TOKEN_LINESTATEMENT_END, '#pop')
] + tag_rules,
# line comments
TOKEN_LINECOMMENT_BEGIN: [
Something went wrong with that request. Please try again.