Skip to content
This repository

Specify what whitespace is. #74

Closed
wants to merge 2 commits into from

3 participants

Bert Wesarg Orion Poplawski Armin Ronacher
Bert Wesarg

I had a hard time to get the desired result out of my template into a form which fits our codingstyle. I use jinja to generate C code. But in C empty lines should be preserved as much as possible to increase readebility. And while I'm not only generate C code but C header files which will be installed, these empty lines are of high value for our project. Unfortunately, the \s in pythons re engine also matches \n, thus any jinja token also eats up its leading empty lines.

This pull request includes my current solution to this problem. It lets the user specify what whitespace is. In my project I just use [ \t].

bertwesarg added some commits
Bert Wesarg bertwesarg Use kwargs when instanciating the Environment.
This protects for failures, when adding new keywords.
44d0d81
Bert Wesarg bertwesarg Specify what whitespace is.
Unfortunately \s in pythons re engine also matches newlines. This makes
it very hard to use jinja where empty lines are of value. Let the user
specify what whitespace is.
cfdde52
Orion Poplawski

@mitsuhiko Is there any chance of this being accepted? Looks like otf2 is now shipping a bundled version of jinja2 with this applied. It would be nice to not have forked versions out in the wild if it could be avoided.

Armin Ronacher
Owner

Is there any chance that this is not necessary with the new lstrip changes?

Armin Ronacher mitsuhiko closed this
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Showing 2 unique commits by 1 author.

Nov 21, 2011
Bert Wesarg bertwesarg Use kwargs when instanciating the Environment.
This protects for failures, when adding new keywords.
44d0d81
Bert Wesarg bertwesarg Specify what whitespace is.
Unfortunately \s in pythons re engine also matches newlines. This makes
it very hard to use jinja where empty lines are of value. Let the user
specify what whitespace is.
cfdde52
This page is out of date. Refresh to see the latest.
1  jinja2/defaults.py
@@ -22,6 +22,7 @@
22 22 LINE_COMMENT_PREFIX = None
23 23 TRIM_BLOCKS = False
24 24 NEWLINE_SEQUENCE = '\n'
  25 +WHITESPACE_RE = r'\s'
25 26
26 27
27 28 # default filters, tests and namespace
15 jinja2/environment.py
@@ -10,6 +10,7 @@
10 10 """
11 11 import os
12 12 import sys
  13 +import re
13 14 from jinja2 import nodes
14 15 from jinja2.defaults import *
15 16 from jinja2.lexer import get_lexer, TokenStream
@@ -87,6 +88,8 @@ def _environment_sanity_check(environment):
87 88 'start strings must be different'
88 89 assert environment.newline_sequence in ('\r', '\r\n', '\n'), \
89 90 'newline_sequence set to unknown line ending string.'
  91 + assert re.compile(environment.whitespace_re, re.U) is not None, \
  92 + 'whitespace_re is not a valid regular expression.'
90 93 return environment
91 94
92 95
@@ -140,6 +143,11 @@ class Environment(object):
140 143 useful default for Linux and OS X systems as well as web
141 144 applications.
142 145
  146 + `whitespace_re`
  147 + An regular expression to match whitespace. Defaults to ``'\s'``.
  148 + Because the python ``'\s'`` matches also ``'\n'``, you may choose a
  149 + different set of whitespace characters.
  150 +
143 151 `extensions`
144 152 List of Jinja extensions to use. This can either be import paths
145 153 as strings or extension classes. For more information have a
@@ -225,6 +233,7 @@ def __init__(self,
225 233 line_comment_prefix=LINE_COMMENT_PREFIX,
226 234 trim_blocks=TRIM_BLOCKS,
227 235 newline_sequence=NEWLINE_SEQUENCE,
  236 + whitespace_re=WHITESPACE_RE,
228 237 extensions=(),
229 238 optimized=True,
230 239 undefined=Undefined,
@@ -256,6 +265,7 @@ def __init__(self,
256 265 self.line_comment_prefix = line_comment_prefix
257 266 self.trim_blocks = trim_blocks
258 267 self.newline_sequence = newline_sequence
  268 + self.whitespace_re = whitespace_re
259 269
260 270 # runtime information
261 271 self.undefined = undefined
@@ -817,6 +827,7 @@ def __new__(cls, source,
817 827 line_comment_prefix=LINE_COMMENT_PREFIX,
818 828 trim_blocks=TRIM_BLOCKS,
819 829 newline_sequence=NEWLINE_SEQUENCE,
  830 + whitespace_re=WHITESPACE_RE,
820 831 extensions=(),
821 832 optimized=True,
822 833 undefined=Undefined,
@@ -826,8 +837,8 @@ def __new__(cls, source,
826 837 block_start_string, block_end_string, variable_start_string,
827 838 variable_end_string, comment_start_string, comment_end_string,
828 839 line_statement_prefix, line_comment_prefix, trim_blocks,
829   - newline_sequence, frozenset(extensions), optimized, undefined,
830   - finalize, autoescape, None, 0, False, None)
  840 + newline_sequence, whitespace_re, frozenset(extensions), optimized,
  841 + undefined, finalize, autoescape, None, 0, False, None)
831 842 return env.from_string(source, template_class=cls)
832 843
833 844 @classmethod
21 jinja2/ext.py
@@ -574,16 +574,17 @@ def getbool(options, key, default=False):
574 574 options.get(key, str(default)).lower() in ('1', 'on', 'yes', 'true')
575 575
576 576 environment = Environment(
577   - options.get('block_start_string', BLOCK_START_STRING),
578   - options.get('block_end_string', BLOCK_END_STRING),
579   - options.get('variable_start_string', VARIABLE_START_STRING),
580   - options.get('variable_end_string', VARIABLE_END_STRING),
581   - options.get('comment_start_string', COMMENT_START_STRING),
582   - options.get('comment_end_string', COMMENT_END_STRING),
583   - options.get('line_statement_prefix') or LINE_STATEMENT_PREFIX,
584   - options.get('line_comment_prefix') or LINE_COMMENT_PREFIX,
585   - getbool(options, 'trim_blocks', TRIM_BLOCKS),
586   - NEWLINE_SEQUENCE, frozenset(extensions),
  577 + block_start_string=options.get('block_start_string', BLOCK_START_STRING),
  578 + block_end_string=options.get('block_end_string', BLOCK_END_STRING),
  579 + variable_start_string=options.get('variable_start_string', VARIABLE_START_STRING),
  580 + variable_end_string=options.get('variable_end_string', VARIABLE_END_STRING),
  581 + comment_start_string=options.get('comment_start_string', COMMENT_START_STRING),
  582 + comment_end_string=options.get('comment_end_string', COMMENT_END_STRING),
  583 + line_statement_prefix=options.get('line_statement_prefix') or LINE_STATEMENT_PREFIX,
  584 + line_comment_prefix=options.get('line_comment_prefix') or LINE_COMMENT_PREFIX,
  585 + trim_blocks=getbool(options, 'trim_blocks', TRIM_BLOCKS),
  586 + newline_sequence=NEWLINE_SEQUENCE, whitespace_re=WHITESPACE_RE,
  587 + extensions=frozenset(extensions),
587 588 cache_size=0,
588 589 auto_reload=False
589 590 )
31 jinja2/lexer.py
@@ -197,7 +197,8 @@ def compile_rules(environment):
197 197
198 198 if environment.line_statement_prefix is not None:
199 199 rules.append((len(environment.line_statement_prefix), 'linestatement',
200   - r'^\s*' + e(environment.line_statement_prefix)))
  200 + (r'^%s*' % (environment.whitespace_re,)) +
  201 + e(environment.line_statement_prefix)))
201 202 if environment.line_comment_prefix is not None:
202 203 rules.append((len(environment.line_comment_prefix), 'linecomment',
203 204 r'(?:^|(?<=\S))[^\S\r\n]*' +
@@ -383,7 +384,8 @@ def get_lexer(environment):
383 384 environment.line_statement_prefix,
384 385 environment.line_comment_prefix,
385 386 environment.trim_blocks,
386   - environment.newline_sequence)
  387 + environment.newline_sequence,
  388 + environment.whitespace_re)
387 389 lexer = _lexer_cache.get(key)
388 390 if lexer is None:
389 391 lexer = Lexer(environment)
@@ -432,13 +434,17 @@ def __init__(self, environment):
432 434 'root': [
433 435 # directives
434 436 (c('(.*?)(?:%s)' % '|'.join(
435   - [r'(?P<raw_begin>(?:\s*%s\-|%s)\s*raw\s*(?:\-%s\s*|%s))' % (
  437 + [r'(?P<raw_begin>(?:%s*%s\-|%s)%s*raw%s*(?:\-%s%s*|%s))' % (
  438 + environment.whitespace_re,
436 439 e(environment.block_start_string),
437 440 e(environment.block_start_string),
  441 + environment.whitespace_re, environment.whitespace_re,
438 442 e(environment.block_end_string),
  443 + environment.whitespace_re,
439 444 e(environment.block_end_string)
440 445 )] + [
441   - r'(?P<%s_begin>\s*%s\-|%s)' % (n, r, r)
  446 + r'(?P<%s_begin>%s*%s\-|%s)' %
  447 + (n, environment.whitespace_re, r, r)
442 448 for n, r in root_tag_rules
443 449 ])), (TOKEN_DATA, '#bygroup'), '#bygroup'),
444 450 # data
@@ -446,8 +452,9 @@ def __init__(self, environment):
446 452 ],
447 453 # comments
448 454 TOKEN_COMMENT_BEGIN: [
449   - (c(r'(.*?)((?:\-%s\s*|%s)%s)' % (
  455 + (c(r'(.*?)((?:\-%s%s*|%s)%s)' % (
450 456 e(environment.comment_end_string),
  457 + environment.whitespace_re,
451 458 e(environment.comment_end_string),
452 459 block_suffix_re
453 460 )), (TOKEN_COMMENT, TOKEN_COMMENT_END), '#pop'),
@@ -455,25 +462,30 @@ def __init__(self, environment):
455 462 ],
456 463 # blocks
457 464 TOKEN_BLOCK_BEGIN: [
458   - (c('(?:\-%s\s*|%s)%s' % (
  465 + (c('(?:\-%s%s*|%s)%s' % (
459 466 e(environment.block_end_string),
  467 + environment.whitespace_re,
460 468 e(environment.block_end_string),
461 469 block_suffix_re
462 470 )), TOKEN_BLOCK_END, '#pop'),
463 471 ] + tag_rules,
464 472 # variables
465 473 TOKEN_VARIABLE_BEGIN: [
466   - (c('\-%s\s*|%s' % (
  474 + (c('\-%s%s*|%s' % (
467 475 e(environment.variable_end_string),
  476 + environment.whitespace_re,
468 477 e(environment.variable_end_string)
469 478 )), TOKEN_VARIABLE_END, '#pop')
470 479 ] + tag_rules,
471 480 # raw block
472 481 TOKEN_RAW_BEGIN: [
473   - (c('(.*?)((?:\s*%s\-|%s)\s*endraw\s*(?:\-%s\s*|%s%s))' % (
  482 + (c('(.*?)((?:%s*%s\-|%s)%s*endraw%s*(?:\-%s%s*|%s%s))' % (
  483 + environment.whitespace_re,
474 484 e(environment.block_start_string),
475 485 e(environment.block_start_string),
  486 + environment.whitespace_re, environment.whitespace_re,
476 487 e(environment.block_end_string),
  488 + environment.whitespace_re,
477 489 e(environment.block_end_string),
478 490 block_suffix_re
479 491 )), (TOKEN_DATA, TOKEN_RAW_END), '#pop'),
@@ -481,7 +493,8 @@ def __init__(self, environment):
481 493 ],
482 494 # line statements
483 495 TOKEN_LINESTATEMENT_BEGIN: [
484   - (c(r'\s*(\n|$)'), TOKEN_LINESTATEMENT_END, '#pop')
  496 + (c(r'%s*(\n|$)' % (environment.whitespace_re,)),
  497 + TOKEN_LINESTATEMENT_END, '#pop')
485 498 ] + tag_rules,
486 499 # line comments
487 500 TOKEN_LINECOMMENT_BEGIN: [

Tip: You can add notes to lines in a file. Hover to the left of a line to make a note

Something went wrong with that request. Please try again.