Skip to content
This repository has been archived by the owner. It is now read-only.

Refactor parser code, add filename option #13

Open
wants to merge 8 commits into
base: master
Choose a base branch
from
@@ -5,7 +5,10 @@
from jinja2.ext import Extension, Markup

from pygments import highlight
from pygments.lexers import get_lexer_by_name, guess_lexer
from pygments.lexers import(
get_lexer_by_name, guess_lexer, guess_lexer_for_filename
)
from pygments.lexers.special import TextLexer
from pygments.formatters import HtmlFormatter
from pygments.util import ClassNotFound

@@ -23,6 +26,18 @@ class HighlightExtension(Extension):
{% endhighlight %}
If you are unable to provide the language to the highlight extension but can
provide the filename, use the filename option.
{% highlight filename="beer.py" %}
from fridge import Beer
pint_glass = Beer()
pint_glass.drink()
{% endhighlight %}
Line numbers can be turned on if it is assigned. Either 'inline' or 'table'
settings in Pygments can be used.
@@ -50,58 +65,38 @@ class HighlightExtension(Extension):
def parse(self, parser):
lineno = next(parser.stream).lineno

# extract the language and line numbering setting if available
if not parser.stream.current.test('block_end'):
# If first up we have an assignment, e.g. lineno='inline', work with that
if parser.stream.current.type == 'name':
# NOTE: The _highlight function parameter order must match the order of arg_list
arg_list = ['lang', 'lineno', 'filename']
parsed_args = {}

while not parser.stream.current.test('block_end'):
this_token_type = parser.stream.current.type

# If up we have an assignment, e.g. lineno='inline', work with that
if this_token_type == 'name':
name = parser.stream.expect('name')
# If the assign is lineno
if name.value == 'lineno':
if parser.stream.skip_if('assign'):
# Assume no language and then add the assigned line number setting
args = [nodes.Const(None)]
args.append(parser.parse_expression())
# If it's not a lineno assignment, ignore it
if name.value in arg_list and parser.stream.skip_if('assign'):
parsed_args[name.value] = parser.parse_expression()
else:
if parser.stream.skip_if('assign'):
next(parser.stream)
# Set our language and line number setting as None
args = [nodes.Const(None), nodes.Const(None)]
parser.fail('Unrecognized argument: %s' % name.value)
elif this_token_type == 'string':
# The only valid string literal argument is the language itself
parsed_args['lang'] = parser.parse_expression()
else:
# Otherwise if our first item is not a line numbering setting,
# assume it's the language setting
args = [parser.parse_expression()]

# If we have a comma next
if parser.stream.skip_if('comma'):
# Check to see if we have a lineno assignment
if parser.stream.current.type == 'name':
name = parser.stream.expect('name')
if name.value == 'lineno':
if parser.stream.skip_if('assign'):
args.append(parser.parse_expression())
# If the name of the variable being assigned is not lineno
# ignore it
else:
if parser.stream.skip_if('assign'):
next(parser.stream)
args.append(nodes.Const(None))
# Otherwise if there's nothing after the language, set the
# line number setting as None
else:
args.append(nodes.Const(None))
else:
# Otherwise if there are no additional arguments, set lang
# and line numbering to None
args = [nodes.Const(None), nodes.Const(None)]
parser.fail('Unexpected %s encountered' % this_token_type)

# Skip over optional commas
parser.stream.skip_if('comma')

args = [parsed_args.get(a, nodes.Const(None)) for a in arg_list]

# body of the block
# body of the block (the source code we want to highlight)
body = parser.parse_statements(['name:endhighlight'], drop_needle=True)

return nodes.CallBlock(self.call_method('_highlight', args),
[], [], body).set_lineno(lineno)

def _highlight(self, lang, linenos, caller=None):
def _highlight(self, lang, linenos, filename, caller=None):
# highlight code using Pygments
body = caller()

@@ -113,13 +108,15 @@ def _highlight(self, lang, linenos, caller=None):
cssclass = None

try:
if lang is None:
lexer = guess_lexer(body)
else:
if lang is not None:
lexer = get_lexer_by_name(lang, stripall=False)
elif filename is not None:
lexer = guess_lexer_for_filename(filename, body)
else:
lexer = guess_lexer(body)
except ClassNotFound as e:
print(e)
sys.exit(1)
# default to the plaintext lexer
lexer = TextLexer()

# Set the cssclass if we have one
# The linenos setting expects either 'inline' or 'table', as per Pygment's
@@ -6,6 +6,12 @@

class HighlightExtensionTestCase(unittest.TestCase):

def assertHtmlListEqual(self, a, b):
# Normalize the HTML lists so white space doesn't cause a failure.
html_a = "".join(a)
html_b = "".join(b)
self.assertEqual(html_a, html_b)

rendered = [
u'<div',
u'class="highlight"><pre>',
@@ -26,7 +32,17 @@ def test_python_tpl(self):
{% endhighlight %}
''')

assert tpl.render().split() == self.rendered
self.assertHtmlListEqual(tpl.render().split(), self.rendered)

def test_python_tpl_by_filename(self):
env = Environment(extensions=['jinja2_highlight.HighlightExtension'])
tpl = env.from_string('''
{% highlight filename="hello.py" %}
print("Hello world")
{% endhighlight %}
''')

self.assertHtmlListEqual(tpl.render().split(), self.rendered)

def test_python_tpl_with_autoescape(self):
# See: https://github.com/tlatsas/jinja2-highlight/pull/1
@@ -40,7 +56,7 @@ def test_python_tpl_with_autoescape(self):
{% endautoescape %}
''')

assert tpl.render().split() == self.rendered
self.assertHtmlListEqual(tpl.render().split(), self.rendered)

inline_rendered = [
u'<div',
@@ -64,7 +80,7 @@ def test_python_tpl_with_inline(self):
{% endhighlight %}
''')

assert tpl.render().split() == self.inline_rendered
self.assertHtmlListEqual(tpl.render().split(), self.inline_rendered)

table_rendered = [
u'<table',
@@ -91,57 +107,62 @@ def test_python_tpl_with_table(self):
{% endhighlight %}
''')

assert tpl.render().split() == self.table_rendered
self.assertHtmlListEqual(tpl.render().split(), self.table_rendered)

inline_no_lang_rendered = [
u'<div',
u'class="highlight"><pre><span',
u'class="lineno">1</span>',
u'<span',
u'class="n">print</span><span',
u'class="p">(</span><span',
u'class="s">&quot;Hello',
u'world&quot;</span><span',
u'class="p">)</span>',
u'</pre></div>'
]
u'<div',
u'class="highlight"><pre><span',
u'class="lineno">1',
u'</span>',
u'<span',
u'class="kn">from</span>',
u'<span',
u'class="nn">mypackage.mymodule</span>',
u'<span',
u'class="kn">import</span>',
u'<span',
u'class="n">myfn</span>',
u'</pre></div>'
]

def test_python_tpl_with_inline_no_lang(self):
env = Environment(extensions=['jinja2_highlight.HighlightExtension'])
tpl = env.from_string('''
{% highlight lineno="inline" %}
print("Hello world")
from mypackage.mymodule import myfn
{% endhighlight %}
''')

assert tpl.render().split() == self.inline_no_lang_rendered
self.assertHtmlListEqual(tpl.render().split(), self.inline_no_lang_rendered)

table_no_lang_rendered = [
u'<table',
u'class="highlighttable"><tr><td',
u'class="linenos"><div',
u'class="linenodiv"><pre>1</pre></div></td><td',
u'class="code"><div',
u'class="highlight"><pre>',
u'<span',
u'class="n">print</span><span',
u'class="p">(</span><span',
u'class="s">&quot;Hello',
u'world&quot;</span><span',
u'class="p">)</span>',
u'</pre></div>',
u'</td></tr></table>'
]
u'<table',
u'class="highlighttable"><tr><td',
u'class="linenos"><div',
u'class="linenodiv"><pre>1</pre></div></td><td',
u'class="code"><div',
u'class="highlight"><pre>',
u'<span',
u'class="kn">from</span>',
u'<span',
u'class="nn">mypackage.mymodule</span>',
u'<span',
u'class="kn">import</span>',
u'<span',
u'class="n">myfn</span>',
u'</pre></div>',
u'</td></tr></table>'
]

def test_python_tpl_with_table_no_lang(self):
env = Environment(extensions=['jinja2_highlight.HighlightExtension'])
tpl = env.from_string('''
{% highlight lineno="table" %}
print("Hello world")
from mypackage.mymodule import myfn
{% endhighlight %}
''')

assert tpl.render().split() == self.table_no_lang_rendered
self.assertHtmlListEqual(tpl.render().split(), self.table_no_lang_rendered)

cssclass_rendered = [
u'<div',
@@ -165,4 +186,23 @@ def test_python_tpl_with_cssclass(self):
{% endhighlight %}
''')

assert tpl.render().split() == self.cssclass_rendered
self.assertHtmlListEqual(tpl.render().split(), self.cssclass_rendered)

plaintext_rendered = [
u'<div',
u'class="highlight"><pre>',
u'abcdefg',
u'</pre></div>'
]

def test_unrecognized_language_defaults_to_plaintext(self):
env = Environment(extensions=['jinja2_highlight.HighlightExtension'])

# Unrecognized language
tpl = env.from_string('''
{% highlight %}
abcdefg
{% endhighlight %}
''')

self.assertHtmlListEqual(tpl.render().split(), self.plaintext_rendered)