Enable GitHub custom lexers

Rakefile

@@ -46,5 +46,14 @@ namespace :vendor do
     rm_rf 'vendor/pygments-main'
   end
 
-  task :update => [:clobber, 'vendor/pygments-main']
+  # Load all the custom lexers in the `vendor/custom_lexers` folder
+  # and stick them in our custom Pygments vendor
+  task :load_lexers do
+    LEXERS_DIR = 'vendor/pygments-main/pygments/lexers'
+    lexers = FileList['vendor/custom_lexers/*.py']
+    lexers.each { |l| FileUtils.copy l, LEXERS_DIR }
+    FileUtils.cd(LEXERS_DIR) { sh "python _mapping.py" }
+  end
+
+  task :update => [:clobber, 'vendor/pygments-main', :load_lexers]
 end

vendor/custom_lexers/github.py

@@ -0,0 +1,330 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.github
+    ~~~~~~~~~~~~~~~~~~~
+
+    Custom lexers for GitHub.com
+
+    :copyright: Copyright 2012 by GitHub, Inc
+    :license: BSD, see LICENSE for details.
+"""
+import re
+
+from pygments.lexer import RegexLexer, include, bygroups, using, DelegatingLexer
+from pygments.token import Text, Name, Number, String, Comment, Punctuation, \
+     Other, Keyword, Operator, Literal
+
+__all__ = ['Dasm16Lexer', 'PuppetLexer', 'AugeasLexer']
+
+class Dasm16Lexer(RegexLexer):
+    """
+    Simple lexer for DCPU-16 Assembly
+
+    Check http://0x10c.com/doc/dcpu-16.txt
+    """
+    name = 'dasm16'
+    aliases = ['DASM16']
+    filenames = ['*.dasm16', '*.dasm']
+    mimetypes = ['text/x-dasm16']
+
+    INSTRUCTIONS = [
+        'SET', 'ADD', 'SUB', 'MUL', 'DIV', 'MOD', 'SHL',
+        'SHR', 'AND', 'BOR', 'XOR', 'IFE', 'IFN', 'IFG'
+        'IFB', 'JSR'
+    ]
+
+    REGISTERS = [
+        'A', 'B', 'C',
+        'X', 'Y', 'Z',
+        'I', 'J',
+        'SP', 'PC',
+        'POP', 'PEEK', 'PUSH'
+    ]
+
+    # Regexes yo
+    char = r'[a-zA-Z$._0-9@]'
+    identifier = r'(?:[a-zA-Z$_]' + char + '*|\.' + char + '+)'
+    number = r'(?:0[xX][a-zA-Z0-9]+|\d+)'
+    instruction = r'(?i)(' + '|'.join(INSTRUCTIONS) + ')'
+
+    def guess_identifier(lexer, match):
+        ident = match.group(0)
+        klass = Name.Variable if ident in lexer.REGISTERS else Name.Label
+        yield match.start(), klass, ident
+
+    tokens = {
+        'root': [
+            include('whitespace'),
+            (':' + identifier, Name.Label),
+            (instruction, Name.Function, 'instruction-args'),
+            (r'[\r\n]+', Text)
+        ],
+
+        'arg' : [
+            (identifier, guess_identifier),
+            (number, Number.Integer),
+        ],
+
+        'deref' : [
+            (r'\+', Punctuation),
+            (r'\]', Punctuation, '#pop'),
+            include('arg'),
+            include('whitespace')
+        ],
+
+        'instruction-args': [
+            (r',', Punctuation),
+            (r'\[', Punctuation, 'deref'),
+            include('arg'),
+
+            (r'[\r\n]+', Text, '#pop'),
+            (r';.*?$', Comment, '#pop'),
+            include('whitespace')
+        ],
+        'whitespace': [
+            (r'\n', Text),
+            (r'\s+', Text),
+            (r';.*?\n', Comment)
+        ],
+    }
+
+class PuppetLexer(RegexLexer):
+    name = 'Puppet'
+    aliases = ['puppet']
+    filenames = ['*.pp']
+
+    tokens = {
+        'root': [
+            include('puppet'),
+        ],
+        'puppet': [
+            include('comments'),
+            (r'(class|define)', Keyword.Declaration, ('block','class_name')),
+            (r'node', Keyword.Declaration, ('block', 'node_name')),
+            (r'elsif', Keyword.Reserved, ('block', 'conditional')),
+            (r'if', Keyword.Reserved, ('block', 'conditional')),
+            (r'unless', Keyword.Reserved, ('block', 'conditional')),
+            (r'(else)(\s*)(\{)', bygroups(Keyword.Reserved, Text, Punctuation), 'block'),
+            (r'case', Keyword.Reserved, ('case', 'conditional')),
+            (r'(::)?([A-Z][\w:]+)+(\s*)(<{1,2}\|)', bygroups(Name.Class, Name.Class, Text, Punctuation), 'spaceinvader'),
+            (r'(::)?([A-Z][\w:]+)+(\s*)(\{)', bygroups(Name.Class, Name.Class, Text, Punctuation), 'type'),
+            (r'(::)?([A-Z][\w:]+)+(\s*)(\[)', bygroups(Name.Class, Name.Class, Text, Punctuation), ('type', 'override_name')),
+            (r'(@{0,2}[\w:]+)(\s*)(\{)(\s*)', bygroups(Name.Class, Text, Punctuation, Text), ('type', 'namevar')),
+            (r'\$(::)?(\w+::)*\w+', Name.Variable, 'var_assign'),
+            (r'include', Keyword.Namespace, 'include'),
+            (r'import', Keyword.Namespace, 'import'),
+            (r'(\w+)(\()', bygroups(Name.Function, Punctuation), 'function'),
+            (r'\s', Text),
+        ],
+        'block': [
+            include('puppet'),
+            (r'\}', Text, '#pop'),
+        ],
+        'override_name': [
+            include('strings'),
+            include('variables'),
+            (r'\]', Punctuation),
+            (r'\s', Text),
+            (r'\{', Punctuation, '#pop'),
+        ],
+        'node_name': [
+            (r'inherits', Keyword.Declaration),
+            (r'[\w\.]+', String),
+            include('strings'),
+            include('variables'),
+            (r',', Punctuation),
+            (r'\s', Text),
+            (r'\{', Punctuation, '#pop'),
+        ],
+        'class_name': [
+            (r'inherits', Keyword.Declaration),
+            (r'[\w:]+', Name.Class),
+            (r'\s', Text),
+            (r'\{', Punctuation, '#pop'),
+            (r'\(', Punctuation, 'paramlist'),
+        ],
+        'include': [
+            (r'\n', Text, '#pop'),
+            (r'[\w:]+', Name.Class),
+            include('value'),
+            (r'\s', Text),
+        ],
+        'import': [
+            (r'\n', Text, '#pop'),
+            (r'[\/\w\.]+', String),
+            include('value'),
+            (r'\s', Text),
+        ],
+        'case': [
+            (r'(default)(:)(\s*)(\{)', bygroups(Keyword.Reserved, Punctuation, Text, Punctuation), 'block'),
+            include('case_values'),
+            (r'(:)(\s*)(\{)', bygroups(Punctuation, Text, Punctuation), 'block'),
+            (r'\s', Text),
+            (r'\}', Punctuation, '#pop'),
+        ],
+        'case_values': [
+            include('value'),
+            (r',', Punctuation),
+        ],
+        'comments': [
+            (r'\s*#.*\n', Comment.Singleline),
+        ],
+        'strings': [
+            (r"'.*?'", String.Single),
+            (r'\w+', String.Symbol),
+            (r'"', String.Double, 'dblstring'),
+            (r'\/.+?\/', String.Regex),
+        ],
+        'dblstring': [
+            (r'\$\{.+?\}', String.Interpol),
+            (r'(?:\\(?:[bdefnrstv\'"\$\\/]|[0-7][0-7]?[0-7]?|\^[a-zA-Z]))', String.Escape),
+            (r'[^"\\\$]+', String.Double),
+            (r'\$', String.Double),
+            (r'"', String.Double, '#pop'),
+        ],
+        'variables': [
+            (r'\$(::)?(\w+::)*\w+', Name.Variable),
+        ],
+        'var_assign': [
+            (r'\[', Punctuation, ('#pop', 'array')),
+            (r'\{', Punctuation, ('#pop', 'hash')),
+            (r'(\s*)(=)(\s*)', bygroups(Text, Operator, Text)),
+            (r'(\(|\))', Punctuation),
+            include('operators'),
+            include('value'),
+            (r'\s', Text, '#pop'),
+        ],
+        'booleans': [
+            (r'(true|false)', Literal),
+        ],
+        'operators': [
+            (r'(\s*)(==|=~|\*|-|\+|<<|>>|!=|!~|!|>=|<=|<|>|and|or|in)(\s*)', bygroups(Text, Operator, Text)),
+        ],
+        'conditional': [
+            include('operators'),
+            include('strings'),
+            include('variables'),
+            (r'\[', Punctuation, 'array'),
+            (r'\(', Punctuation, 'conditional'),
+            (r'\{', Punctuation, '#pop'),
+            (r'\)', Punctuation, '#pop'),
+            (r'\s', Text),
+        ],
+        'spaceinvader': [
+            include('operators'),
+            include('strings'),
+            include('variables'),
+            (r'\[', Punctuation, 'array'),
+            (r'\(', Punctuation, 'conditional'),
+            (r'\s', Text),
+            (r'\|>{1,2}', Punctuation, '#pop'),
+        ],
+        'namevar': [
+            include('value'),
+            (r'\[', Punctuation, 'array'),
+            (r'\s', Text),
+            (r':', Punctuation, '#pop'),
+            (r'\}', Punctuation, '#pop'),
+        ],
+        'function': [
+            (r'\[', Punctuation, 'array'),
+            include('value'),
+            (r',', Punctuation),
+            (r'\s', Text),
+            (r'\)', Punctuation, '#pop'),
+        ],
+        'paramlist': [
+            include('value'),
+            (r'=', Punctuation),
+            (r',', Punctuation),
+            (r'\s', Text),
+            (r'\[', Punctuation, 'array'),
+            (r'\)', Punctuation, '#pop'),
+        ],
+        'type': [
+            (r'(\w+)(\s*)(=>)(\s*)', bygroups(Name.Tag, Text, Punctuation, Text), 'param_value'),
+            (r'\}', Punctuation, '#pop'),
+            (r'\s', Text),
+            include('comments'),
+            (r'', Text, 'namevar'),
+        ],
+        'value': [
+            (r'[\d\.]', Number),
+            (r'([A-Z][\w:]+)+(\[)', bygroups(Name.Class, Punctuation), 'array'),
+            (r'(\w+)(\()', bygroups(Name.Function, Punctuation), 'function'),
+            include('strings'),
+            include('variables'),
+            include('comments'),
+            include('booleans'),
+            (r'(\s*)(\?)(\s*)(\{)', bygroups(Text, Punctuation, Text, Punctuation), 'selector'),
+            (r'\{', Punctuation, 'hash'),
+        ],
+        'selector': [
+            (r'default', Keyword.Reserved),
+            include('value'),
+            (r'=>', Punctuation),
+            (r',', Punctuation),
+            (r'\s', Text),
+            (r'\}', Punctuation, '#pop'),
+        ],
+        'param_value': [
+            include('value'),
+            (r'\[', Punctuation, 'array'),
+            (r',', Punctuation, '#pop'),
+            (r';', Punctuation, '#pop'),
+            (r'\s', Text, '#pop'),
+            (r'', Text, '#pop'),
+        ],
+        'array': [
+            include('value'),
+            (r'\[', Punctuation, 'array'),
+            (r',', Punctuation),
+            (r'\s', Text),
+            (r'\]', Punctuation, '#pop'),
+        ],
+        'hash': [
+            include('value'),
+            (r'\s', Text),
+            (r'=>', Punctuation),
+            (r',', Punctuation),
+            (r'\}', Punctuation, '#pop'),
+        ],
+    }
+
+class AugeasLexer(RegexLexer):
+    name = 'Augeas'
+    aliases = ['augeas']
+    filenames = ['*.aug']
+
+    tokens = {
+        'root': [
+            (r'(module)(\s*)([^\s=]+)', bygroups(Keyword.Namespace, Text, Name.Namespace)),
+            (r'(let)(\s*)([^\s=]+)', bygroups(Keyword.Declaration, Text, Name.Variable)),
+            (r'(del|store|value|counter|seq|key|label|autoload|incl|excl|transform|test|get|put)(\s+)', bygroups(Name.Builtin, Text)),
+            (r'(\()([^\:]+)(\:)(unit|string|regexp|lens|tree|filter)(\))', bygroups(Punctuation, Name.Variable, Punctuation, Keyword.Type, Punctuation)),
+            (r'\(\*', Comment.Multiline, 'comment'),
+            (r'[\+=\|\.\*\;\?-]', Operator),
+            (r'[\[\]\(\)\{\}]', Operator),
+            (r'"', String.Double, 'string'),
+            (r'\/', String.Regex, 'regex'),
+            (r'([A-Z]\w*)(\.)(\w+)', bygroups(Name.Namespace, Punctuation, Name.Variable)),
+            (r'.', Name.Variable),
+            (r'\s', Text),
+        ],
+        'string': [
+            (r'\\.', String.Escape),
+            (r'[^"]', String.Double),
+            (r'"', String.Double, '#pop'),
+        ],
+        'regex': [
+            (r'\\.', String.Escape),
+            (r'[^\/]', String.Regex),
+            (r'\/', String.Regex, '#pop'),
+        ],
+        'comment': [
+            (r'[^*\)]', Comment.Multiline),
+            (r'\(\*', Comment.Multiline, '#push'),
+            (r'\*\)', Comment.Multiline, '#pop'),
+            (r'[\*\)]', Comment.Multiline)
+        ],
+    }

vendor/pygments-main/pygments/lexers/_mapping.py

@@ -30,6 +30,7 @@
     'ApacheConfLexer': ('pygments.lexers.text', 'ApacheConf', ('apacheconf', 'aconf', 'apache'), ('.htaccess', 'apache.conf', 'apache2.conf'), ('text/x-apacheconf',)),
     'AppleScriptLexer': ('pygments.lexers.other', 'AppleScript', ('applescript',), ('*.applescript',), ()),
     'AsymptoteLexer': ('pygments.lexers.other', 'Asymptote', ('asy', 'asymptote'), ('*.asy',), ('text/x-asymptote',)),
+    'AugeasLexer': ('pygments.lexers.github', 'Augeas', ('augeas',), ('*.aug',), ()),
     'AutohotkeyLexer': ('pygments.lexers.other', 'autohotkey', ('ahk',), ('*.ahk', '*.ahkl'), ('text/x-autohotkey',)),
     'AwkLexer': ('pygments.lexers.other', 'Awk', ('awk', 'gawk', 'mawk', 'nawk'), ('*.awk',), ('application/x-awk',)),
     'BBCodeLexer': ('pygments.lexers.text', 'BBCode', ('bbcode',), (), ('text/x-bbcode',)),
@@ -71,6 +72,7 @@
     'DObjdumpLexer': ('pygments.lexers.asm', 'd-objdump', ('d-objdump',), ('*.d-objdump',), ('text/x-d-objdump',)),
     'DarcsPatchLexer': ('pygments.lexers.text', 'Darcs Patch', ('dpatch',), ('*.dpatch', '*.darcspatch'), ()),
     'DartLexer': ('pygments.lexers.web', 'Dart', ('dart',), ('*.dart',), ('text/x-dart',)),
+    'Dasm16Lexer': ('pygments.lexers.github', 'dasm16', ('DASM16',), ('*.dasm16', '*.dasm'), ('text/x-dasm16',)),
     'DebianControlLexer': ('pygments.lexers.text', 'Debian Control file', ('control',), ('control',), ()),
     'DelphiLexer': ('pygments.lexers.compiled', 'Delphi', ('delphi', 'pas', 'pascal', 'objectpascal'), ('*.pas',), ('text/x-pascal',)),
     'DiffLexer': ('pygments.lexers.text', 'Diff', ('diff', 'udiff'), ('*.diff', '*.patch'), ('text/x-diff', 'text/x-patch')),
@@ -131,9 +133,9 @@
     'JavascriptPhpLexer': ('pygments.lexers.templates', 'JavaScript+PHP', ('js+php', 'javascript+php'), (), ('application/x-javascript+php', 'text/x-javascript+php', 'text/javascript+php')),
     'JavascriptSmartyLexer': ('pygments.lexers.templates', 'JavaScript+Smarty', ('js+smarty', 'javascript+smarty'), (), ('application/x-javascript+smarty', 'text/x-javascript+smarty', 'text/javascript+smarty')),
     'JspLexer': ('pygments.lexers.templates', 'Java Server Page', ('jsp',), ('*.jsp',), ('application/x-jsp',)),
-    'KotlinLexer': ('pygments.lexers.jvm', 'Kotlin', ('kotlin',), ('*.kt',), ('text/x-kotlin',)),
-    'JuliaLexer': ('pygments.lexers.math', 'Julia', ('julia','jl'), ('*.jl',), ('text/x-julia','application/x-julia')),
     'JuliaConsoleLexer': ('pygments.lexers.math', 'Julia console', ('jlcon',), (), ()),
+    'JuliaLexer': ('pygments.lexers.math', 'Julia', ('julia', 'jl'), ('*.jl',), ('text/x-julia', 'application/x-julia')),
+    'KotlinLexer': ('pygments.lexers.jvm', 'Kotlin', ('kotlin',), ('*.kt',), ('text/x-kotlin',)),
     'LighttpdConfLexer': ('pygments.lexers.text', 'Lighttpd configuration file', ('lighty', 'lighttpd'), (), ('text/x-lighttpd-conf',)),
     'LiterateHaskellLexer': ('pygments.lexers.functional', 'Literate Haskell', ('lhs', 'literate-haskell'), ('*.lhs',), ('text/x-literate-haskell',)),
     'LlvmLexer': ('pygments.lexers.asm', 'LLVM', ('llvm',), ('*.ll',), ('text/x-llvm',)),
@@ -189,6 +191,7 @@
     'PrologLexer': ('pygments.lexers.compiled', 'Prolog', ('prolog',), ('*.prolog', '*.pro', '*.pl'), ('text/x-prolog',)),
     'PropertiesLexer': ('pygments.lexers.text', 'Properties', ('properties',), ('*.properties',), ('text/x-java-properties',)),
     'ProtoBufLexer': ('pygments.lexers.other', 'Protocol Buffer', ('protobuf',), ('*.proto',), ()),
+    'PuppetLexer': ('pygments.lexers.github', 'Puppet', ('puppet',), ('*.pp',), ()),
     'PyPyLogLexer': ('pygments.lexers.text', 'PyPy Log', ('pypylog', 'pypy'), ('*.pypylog',), ('application/x-pypylog',)),
     'Python3Lexer': ('pygments.lexers.agile', 'Python 3', ('python3', 'py3'), (), ('text/x-python3', 'application/x-python3')),
     'Python3TracebackLexer': ('pygments.lexers.agile', 'Python 3.0 Traceback', ('py3tb',), ('*.py3tb',), ('text/x-python3-traceback',)),