Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Add GitLab formatter

  • Loading branch information...
commit d946a6067cbf9dd1be74e5d2ffcba1d8ef7f6c45 1 parent cbb2ee8
@riyad riyad authored
View
11 Rakefile
@@ -65,5 +65,14 @@ namespace :vendor do
FileUtils.cd(LEXERS_DIR) { sh "python _mapping.py" }
end
- task :update => [:clobber, 'vendor/pygments-main', :load_lexers]
+ # Load all the custom formatters in the `vendor/custom_formatters` folder
+ # and stick them in our custom Pygments vendor
+ task :load_formatters do
+ FORMATTERS_DIR = 'vendor/pygments-main/pygments/formatters'
+ formatters = FileList['vendor/custom_formatters/*.py']
+ formatters.each { |f| FileUtils.copy f, FORMATTERS_DIR }
+ FileUtils.cd(FORMATTERS_DIR) { sh "python _mapping.py" }
+ end
+
+ task :update => [:clobber, 'vendor/pygments-main', :load_lexers, :load_formatters]
end
View
171 vendor/custom_formatters/gitlab.py
@@ -0,0 +1,171 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.formatters.gitlab
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ GitLab specific formatter for HTML output.
+ Based on the standard HTML formatter.
+
+ :copyright: Copyright 2012 by the GitLab team (http://www.gitlab.org).
+ :license: BSD, see LICENSE for details.
+"""
+
+import os
+import sys
+import StringIO
+
+from pygments.formatter import Formatter
+from pygments.token import Token, Text, STANDARD_TYPES
+from pygments.util import get_bool_opt, get_int_opt
+
+
+__all__ = ['GitlabFormatter']
+
+
+_escape_html_table = {
+ ord('&'): u'&',
+ ord('<'): u'&lt;',
+ ord('>'): u'&gt;',
+ ord('"'): u'&quot;',
+ ord("'"): u'&#39;',
+}
+
+def escape_html(text, table=_escape_html_table):
+ """Escape &, <, > as well as single and double quotes for HTML."""
+ return text.translate(table)
+
+
+def _get_ttype_class(ttype):
+ fname = STANDARD_TYPES.get(ttype)
+ if fname:
+ return fname
+ aname = ''
+ while fname is None:
+ aname = '-' + ttype[-1] + aname
+ ttype = ttype.parent
+ fname = STANDARD_TYPES.get(ttype)
+ return fname + aname
+
+
+class GitlabFormatter(Formatter):
+ r"""
+ GitLab specific formatter for HTML output.
+
+ Additional options accepted:
+
+ `show_line_numbers`
+ Determines whether the line number column should be shown (default:
+ ``True``).
+
+ `first_line_number`
+ The line number for the first line (default: ``1``).
+ """
+
+ name = 'GitLab'
+ aliases = ['gitlab']
+ filenames = []
+
+ def __init__(self, **options):
+ Formatter.__init__(self, **options)
+ self.show_line_numbers = get_bool_opt(options, 'show_line_numbers', True)
+ self.first_line_number = abs(get_int_opt(options, 'first_line_number', 1))
+
+ def _wrap_table(self, inner):
+ """
+ Wrap the whole thing into a table and add line numbers
+ """
+ dummyoutfile = StringIO.StringIO()
+ lncount = 0
+ for line in inner:
+ lncount += 1
+ dummyoutfile.write(line)
+
+ sln = self.show_line_numbers
+ if sln:
+ fl = self.first_line_number
+ mw = len(str(lncount + fl - 1))
+
+ lines = []
+ for i in range(fl, fl+lncount):
+ lines.append('<a id="L%d" href="#L%d" rel="#L%d">' % (i, i, i) +
+ '<i class="icon-link"></i> %*d' % (mw, i) +
+ '</a>')
+ ls = '\n'.join(lines)
+
+ yield '<table class="lines"><tr>'
+ if sln:
+ yield '<td><pre class="line_numbers">' + ls + '</pre></td>'
+ yield '<td><div class="highlight"><pre>'
+ yield dummyoutfile.getvalue()
+ yield '</pre></div></td></tr></table>'
+
+ def _wrap_code_lines(self, inner):
+ """
+ Wrap each line in a <div class="line">
+ """
+ # subtract 1 since we have to increment i *before* yielding
+ i = self.first_line_number - 1
+
+ for line in inner:
+ i += 1
+ yield '<div id="LC%d" class="line">%s</div>' % (i, line)
+
+ def _format_code_lines(self, tokensource):
+ """
+ Just format the tokens, without any wrapping tags.
+ Yield individual lines.
+ """
+ # for <span style=""> lookup only
+ escape_table = _escape_html_table
+
+ lspan = ''
+ line = ''
+ for ttype, value in tokensource:
+ cls = _get_ttype_class(ttype)
+ cspan = cls and '<span class="%s">' % cls or ''
+
+ parts = escape_html(value).split('\n')
+
+ # for all but the last line
+ for part in parts[:-1]:
+ if line:
+ if lspan != cspan:
+ line += (lspan and '</span>') + cspan + part + \
+ (cspan and '</span>')
+ else: # both are the same
+ line += part + (lspan and '</span>')
+ yield line
+ line = ''
+ elif part:
+ yield cspan + part + (cspan and '</span>')
+ else:
+ yield '<br/>'
+ # for the last line
+ if line and parts[-1]:
+ if lspan != cspan:
+ line += (lspan and '</span>') + cspan + parts[-1]
+ lspan = cspan
+ else:
+ line += parts[-1]
+ elif parts[-1]:
+ line = cspan + parts[-1]
+ lspan = cspan
+ # else we neither have to open a new span nor set lspan
+
+ if line:
+ yield line + (lspan and '</span>')
+
+ def format_unencoded(self, tokensource, outfile):
+ """
+ The formatting process uses several nested generators; which of
+ them are used is determined by the user's options.
+
+ Each generator should take at least one argument, ``inner``,
+ and wrap the pieces of text generated by this.
+ """
+ source = self._format_code_lines(tokensource)
+ source = self._wrap_code_lines(source)
+ source = self._wrap_table(source)
+
+ for piece in source:
+ outfile.write(piece)
View
2  vendor/pygments-main/pygments/formatters/_mapping.py
@@ -17,6 +17,7 @@
# start
from pygments.formatters.bbcode import BBCodeFormatter
+from pygments.formatters.gitlab import GitlabFormatter
from pygments.formatters.html import HtmlFormatter
from pygments.formatters.img import BmpImageFormatter
from pygments.formatters.img import GifImageFormatter
@@ -34,6 +35,7 @@
BBCodeFormatter: ('BBCode', ('bbcode', 'bb'), (), 'Format tokens with BBcodes. These formatting codes are used by many bulletin boards, so you can highlight your sourcecode with pygments before posting it there.'),
BmpImageFormatter: ('img_bmp', ('bmp', 'bitmap'), ('*.bmp',), 'Create a bitmap image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'),
GifImageFormatter: ('img_gif', ('gif',), ('*.gif',), 'Create a GIF image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'),
+ GitlabFormatter: ('GitLab', ('gitlab',), (), 'GitLab specific formatter for HTML output.'),
HtmlFormatter: ('HTML', ('html',), ('*.html', '*.htm'), "Format tokens as HTML 4 ``<span>`` tags within a ``<pre>`` tag, wrapped in a ``<div>`` tag. The ``<div>``'s CSS class can be set by the `cssclass` option."),
ImageFormatter: ('img', ('img', 'IMG', 'png'), ('*.png',), 'Create a PNG image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'),
JpgImageFormatter: ('img_jpg', ('jpg', 'jpeg'), ('*.jpg',), 'Create a JPEG image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'),
View
171 vendor/pygments-main/pygments/formatters/gitlab.py
@@ -0,0 +1,171 @@
+# -*- coding: utf-8 -*-
+"""
+ pygments.formatters.gitlab
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ GitLab specific formatter for HTML output.
+ Based on the standard HTML formatter.
+
+ :copyright: Copyright 2012 by the GitLab team (http://www.gitlab.org).
+ :license: BSD, see LICENSE for details.
+"""
+
+import os
+import sys
+import StringIO
+
+from pygments.formatter import Formatter
+from pygments.token import Token, Text, STANDARD_TYPES
+from pygments.util import get_bool_opt, get_int_opt
+
+
+__all__ = ['GitlabFormatter']
+
+
+_escape_html_table = {
+ ord('&'): u'&amp;',
+ ord('<'): u'&lt;',
+ ord('>'): u'&gt;',
+ ord('"'): u'&quot;',
+ ord("'"): u'&#39;',
+}
+
+def escape_html(text, table=_escape_html_table):
+ """Escape &, <, > as well as single and double quotes for HTML."""
+ return text.translate(table)
+
+
+def _get_ttype_class(ttype):
+ fname = STANDARD_TYPES.get(ttype)
+ if fname:
+ return fname
+ aname = ''
+ while fname is None:
+ aname = '-' + ttype[-1] + aname
+ ttype = ttype.parent
+ fname = STANDARD_TYPES.get(ttype)
+ return fname + aname
+
+
+class GitlabFormatter(Formatter):
+ r"""
+ GitLab specific formatter for HTML output.
+
+ Additional options accepted:
+
+ `show_line_numbers`
+ Determines whether the line number column should be shown (default:
+ ``True``).
+
+ `first_line_number`
+ The line number for the first line (default: ``1``).
+ """
+
+ name = 'GitLab'
+ aliases = ['gitlab']
+ filenames = []
+
+ def __init__(self, **options):
+ Formatter.__init__(self, **options)
+ self.show_line_numbers = get_bool_opt(options, 'show_line_numbers', True)
+ self.first_line_number = abs(get_int_opt(options, 'first_line_number', 1))
+
+ def _wrap_table(self, inner):
+ """
+ Wrap the whole thing into a table and add line numbers
+ """
+ dummyoutfile = StringIO.StringIO()
+ lncount = 0
+ for line in inner:
+ lncount += 1
+ dummyoutfile.write(line)
+
+ sln = self.show_line_numbers
+ if sln:
+ fl = self.first_line_number
+ mw = len(str(lncount + fl - 1))
+
+ lines = []
+ for i in range(fl, fl+lncount):
+ lines.append('<a id="L%d" href="#L%d" rel="#L%d">' % (i, i, i) +
+ '<i class="icon-link"></i> %*d' % (mw, i) +
+ '</a>')
+ ls = '\n'.join(lines)
+
+ yield '<table class="lines"><tr>'
+ if sln:
+ yield '<td><pre class="line_numbers">' + ls + '</pre></td>'
+ yield '<td><div class="highlight"><pre>'
+ yield dummyoutfile.getvalue()
+ yield '</pre></div></td></tr></table>'
+
+ def _wrap_code_lines(self, inner):
+ """
+ Wrap each line in a <div class="line">
+ """
+ # subtract 1 since we have to increment i *before* yielding
+ i = self.first_line_number - 1
+
+ for line in inner:
+ i += 1
+ yield '<div id="LC%d" class="line">%s</div>' % (i, line)
+
+ def _format_code_lines(self, tokensource):
+ """
+ Just format the tokens, without any wrapping tags.
+ Yield individual lines.
+ """
+ # for <span style=""> lookup only
+ escape_table = _escape_html_table
+
+ lspan = ''
+ line = ''
+ for ttype, value in tokensource:
+ cls = _get_ttype_class(ttype)
+ cspan = cls and '<span class="%s">' % cls or ''
+
+ parts = escape_html(value).split('\n')
+
+ # for all but the last line
+ for part in parts[:-1]:
+ if line:
+ if lspan != cspan:
+ line += (lspan and '</span>') + cspan + part + \
+ (cspan and '</span>')
+ else: # both are the same
+ line += part + (lspan and '</span>')
+ yield line
+ line = ''
+ elif part:
+ yield cspan + part + (cspan and '</span>')
+ else:
+ yield '<br/>'
+ # for the last line
+ if line and parts[-1]:
+ if lspan != cspan:
+ line += (lspan and '</span>') + cspan + parts[-1]
+ lspan = cspan
+ else:
+ line += parts[-1]
+ elif parts[-1]:
+ line = cspan + parts[-1]
+ lspan = cspan
+ # else we neither have to open a new span nor set lspan
+
+ if line:
+ yield line + (lspan and '</span>')
+
+ def format_unencoded(self, tokensource, outfile):
+ """
+ The formatting process uses several nested generators; which of
+ them are used is determined by the user's options.
+
+ Each generator should take at least one argument, ``inner``,
+ and wrap the pieces of text generated by this.
+ """
+ source = self._format_code_lines(tokensource)
+ source = self._wrap_code_lines(source)
+ source = self._wrap_table(source)
+
+ for piece in source:
+ outfile.write(piece)

12 comments on commit d946a60

@avtobiff

At a glance this seems to be the only change from upstream (except lagging behind). Is it so?

Why not just make a pull request for this and remove the fork?

@avtobiff

Also, the reference in the file headers to a BSD LICENSE file is wrong. No such file exists.

Why would you want to change license in your fork from the MIT/Expat license to BSD?

@riyad

@avtobiff There are basically two deviations in this fork. The use of python2 for calling Python (see gitlabhq/gitlabhq#2214) and the custom GitLab formatter (see #1).

@avtobiff

@riyad

  1. Not seeing any consensus on gitlabhq/gitlabhq#2214 but ok.
  2. Why not integrate the GitLab formatter to upstream pygments.rb? It is a fairly small changeset anyways.
  3. References in file headers to BSD LICENSE are wrong. No such file exists.
  4. Why change from upstream (and still actually most of gitlab-pygments.rb) MIT/Expat to BSD?
@randx
Owner

@avtobiff what's so special with pygments upstream?

@avtobiff

@riyad

Well, generally a proliferation of a codebase should be avoided.

The Debian Ruby team is working with packaging GitLab and it's
dependencies. We don't want to package all custom forks of
different gems. This is why I suggest to pull request this change
to pygment.rb upstream.

Furthermore, Debian takes licensing seriously. It is a tad unclear
what the license of these additions are. (Code says BSD License
but references a file with the MIT License.)

@randx
Owner

Code says BSD License but references a file with the MIT License

can you point please? As I see all custom lexers have BSD LIcense but Pygments.rb itself has MIT

@riyad

tmm1/pygments.rb#74 seems to have made it upstream so we can drop the python2 hack. This only leaves the custom formatter.

@avtobiff

can you point please? As I see all custom lexers have BSD LIcense but Pygments.rb itself has MIT

Ah yes, I did not realize that vendor/pygments-main/ is actually a copy of upstream pygments itself.
So the changes should be pushed there and not to pygments.rb. (We rip pygments-main out in the
Debian package and use python-pygments.)

I only checked the pygments.rb root for the LICENSE file.

Do you want to push this to pygments bitbucket? Otherwise I volunteer!

@avtobiff

tmm1/pygments.rb#74 seems to have made it upstream so we can drop the python2 hack. This only leaves the custom formatter.

Sweet!

@riyad

@avtobiff go ahead :smile:

Please sign in to comment.
Something went wrong with that request. Please try again.