Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Update Pygments Upstream

  • Loading branch information...
commit 7b35292815a709d151add6ef8d2c2f7b20a5c749 1 parent 8f3173f
@vmg vmg authored
View
2  vendor/pygments-main/REVISION
@@ -1 +1 @@
-96e8b9cb83ef
+5cc94956e233
View
6 vendor/pygments-main/pygments/formatters/latex.py
@@ -32,6 +32,9 @@ def escape_tex(text, commandprefix):
replace('#', r'\%sZsh{}' % commandprefix). \
replace('%', r'\%sZpc{}' % commandprefix). \
replace('$', r'\%sZdl{}' % commandprefix). \
+ replace('-', r'\%sZhy{}' % commandprefix). \
+ replace("'", r'\%sZsq{}' % commandprefix). \
+ replace('"', r'\%sZdq{}' % commandprefix). \
replace('~', r'\%sZti{}' % commandprefix)
@@ -115,6 +118,9 @@ def escape_tex(text, commandprefix):
\def\%(cp)sZsh{\char`\#}
\def\%(cp)sZpc{\char`\%%}
\def\%(cp)sZdl{\char`\$}
+\def\%(cp)sZhy{\char`\-}
+\def\%(cp)sZsq{\char`\'}
+\def\%(cp)sZdq{\char`\"}
\def\%(cp)sZti{\char`\~}
%% for compatibility with earlier versions
\def\%(cp)sZat{@}
View
2  vendor/pygments-main/pygments/lexer.py
@@ -544,10 +544,10 @@ def get_tokens_unprocessed(self, text, stack=('root',)):
try:
if text[pos] == '\n':
# at EOL, reset state to "root"
- pos += 1
statestack = ['root']
statetokens = tokendefs['root']
yield pos, Text, u'\n'
+ pos += 1
continue
yield pos, Error, text[pos]
pos += 1
View
17 vendor/pygments-main/pygments/lexers/_postgres_builtins.py
@@ -204,17 +204,18 @@ def update_consts(filename, constname, content):
'bigint', 'bigserial', 'bit', 'bit varying', 'bool', 'boolean', 'box',
'bytea', 'char', 'character', 'character varying', 'cidr', 'circle',
'date', 'decimal', 'double precision', 'float4', 'float8', 'inet',
- 'int', 'int2', 'int4', 'int8', 'integer', 'interval', 'line', 'lseg',
- 'macaddr', 'money', 'numeric', 'path', 'point', 'polygon', 'real',
- 'serial', 'serial4', 'serial8', 'smallint', 'text', 'time', 'timestamp',
- 'timestamptz', 'timetz', 'tsquery', 'tsvector', 'txid_snapshot', 'uuid',
- 'varbit', 'varchar', 'with time zone', 'without time zone', 'xml',
+ 'int', 'int2', 'int4', 'int8', 'integer', 'interval', 'json', 'line',
+ 'lseg', 'macaddr', 'money', 'numeric', 'path', 'point', 'polygon',
+ 'real', 'serial', 'serial2', 'serial4', 'serial8', 'smallint',
+ 'smallserial', 'text', 'time', 'timestamp', 'timestamptz', 'timetz',
+ 'tsquery', 'tsvector', 'txid_snapshot', 'uuid', 'varbit', 'varchar',
+ 'with time zone', 'without time zone', 'xml',
]
PSEUDO_TYPES = [
- 'any', 'anyarray', 'anyelement', 'anyenum', 'anynonarray', 'cstring',
- 'internal', 'language_handler', 'fdw_handler', 'record', 'trigger',
- 'void', 'opaque',
+ 'any', 'anyelement', 'anyarray', 'anynonarray', 'anyenum', 'anyrange',
+ 'cstring', 'internal', 'language_handler', 'fdw_handler', 'record',
+ 'trigger', 'void', 'opaque',
]
# Remove 'trigger' from types
View
5 vendor/pygments-main/pygments/lexers/asm.py
@@ -31,7 +31,7 @@ class GasLexer(RegexLexer):
#: optional Comment or Whitespace
string = r'"(\\"|[^"])*"'
- char = r'[a-zA-Z$._0-9@]'
+ char = r'[a-zA-Z$._0-9@-]'
identifier = r'(?:[a-zA-Z$_]' + char + '*|\.' + char + '+)'
number = r'(?:0[xX][a-zA-Z0-9]+|\d+)'
@@ -304,7 +304,8 @@ class NasmLexer(RegexLexer):
floatn = decn + r'\.e?' + decn
string = r'"(\\"|[^"\n])*"|' + r"'(\\'|[^'\n])*'|" + r"`(\\`|[^`\n])*`"
declkw = r'(?:res|d)[bwdqt]|times'
- register = (r'[a-d][lh]|e?[a-d]x|e?[sb]p|e?[sd]i|[c-gs]s|st[0-7]|'
+ register = (r'r[0-9][0-5]?[bwd]|'
+ r'[a-d][lh]|[er]?[a-d]x|[er]?[sb]p|[er]?[sd]i|[c-gs]s|st[0-7]|'
r'mm[0-7]|cr[0-4]|dr[0-367]|tr[3-7]')
wordop = r'seg|wrt|strict'
type = r'byte|[dq]?word'
View
2  vendor/pygments-main/pygments/lexers/dotnet.py
@@ -102,7 +102,7 @@ class CSharpLexer(RegexLexer):
Comment.Preproc),
(r'\b(extern)(\s+)(alias)\b', bygroups(Keyword, Text,
Keyword)),
- (r'(abstract|as|base|break|case|catch|'
+ (r'(abstract|as|async|await|base|break|case|catch|'
r'checked|const|continue|default|delegate|'
r'do|else|enum|event|explicit|extern|false|finally|'
r'fixed|for|foreach|goto|if|implicit|in|interface|'
View
58 vendor/pygments-main/pygments/lexers/functional.py
@@ -1688,13 +1688,46 @@ class ElixirLexer(RegexLexer):
filenames = ['*.ex', '*.exs']
mimetypes = ['text/x-elixir']
+ def gen_elixir_sigil_rules():
+ states = {}
+
+ states['strings'] = [
+ (r'(%[A-Ba-z])?"""(?:.|\n)*?"""', String.Doc),
+ (r"'''(?:.|\n)*?'''", String.Doc),
+ (r'"', String.Double, 'dqs'),
+ (r"'.*'", String.Single),
+ (r'(?<!\w)\?(\\(x\d{1,2}|\h{1,2}(?!\h)\b|0[0-7]{0,2}(?![0-7])\b|'
+ r'[^x0MC])|(\\[MC]-)+\w|[^\s\\])', String.Other)
+ ]
+
+ for lbrace, rbrace, name, in ('\\{', '\\}', 'cb'), \
+ ('\\[', '\\]', 'sb'), \
+ ('\\(', '\\)', 'pa'), \
+ ('\\<', '\\>', 'lt'):
+
+ states['strings'] += [
+ (r'%[a-z]' + lbrace, String.Double, name + 'intp'),
+ (r'%[A-Z]' + lbrace, String.Double, name + 'no-intp')
+ ]
+
+ states[name +'intp'] = [
+ (r'' + rbrace + '[a-z]*', String.Double, "#pop"),
+ include('enddoublestr')
+ ]
+
+ states[name +'no-intp'] = [
+ (r'.*' + rbrace + '[a-z]*', String.Double , "#pop")
+ ]
+
+ return states
+
tokens = {
'root': [
(r'\s+', Text),
(r'#.*$', Comment.Single),
- (r'\b(case|end|bc|lc|if|unless|try|loop|receive|fn|defmodule|'
- r'defp|def|defprotocol|defimpl|defrecord|defmacro|defdelegate|'
- r'defexception|exit|raise|throw)\b(?![?!])|'
+ (r'\b(case|cond|end|bc|lc|if|unless|try|loop|receive|fn|defmodule|'
+ r'defp?|defprotocol|defimpl|defrecord|defmacrop?|defdelegate|'
+ r'defexception|exit|raise|throw|unless|after|rescue|catch|else)\b(?![?!])|'
r'(?<!\.)\b(do|\-\>)\b\s*', Keyword),
(r'\b(import|require|use|recur|quote|unquote|super|refer)\b(?![?!])',
Keyword.Namespace),
@@ -1708,26 +1741,18 @@ class ElixirLexer(RegexLexer):
r'\*\*?|=?~|<\-)|([a-zA-Z_]\w*([?!])?)(:)(?!:)', String.Symbol),
(r':"', String.Symbol, 'interpoling_symbol'),
(r'\b(nil|true|false)\b(?![?!])|\b[A-Z]\w*\b', Name.Constant),
- (r'\b(__(FILE|LINE|MODULE|LOCAL|MAIN|FUNCTION)__)\b(?![?!])', Name.Builtin.Pseudo),
+ (r'\b(__(FILE|LINE|MODULE|MAIN|FUNCTION)__)\b(?![?!])', Name.Builtin.Pseudo),
(r'[a-zA-Z_!][\w_]*[!\?]?', Name),
(r'[(){};,/\|:\\\[\]]', Punctuation),
(r'@[a-zA-Z_]\w*|&\d', Name.Variable),
(r'\b(0[xX][0-9A-Fa-f]+|\d(_?\d)*(\.(?![^\d\s])'
r'(_?\d)*)?([eE][-+]?\d(_?\d)*)?|0[bB][01]+)\b', Number),
+ (r'%r\/.*\/', String.Regex),
include('strings'),
],
- 'strings': [
- (r'"""(?:.|\n)*?"""', String.Doc),
- (r"'''(?:.|\n)*?'''", String.Doc),
- (r'"', String.Double, 'dqs'),
- (r"'.*'", String.Single),
- (r'(?<!\w)\?(\\(x\d{1,2}|\h{1,2}(?!\h)\b|0[0-7]{0,2}(?![0-7])\b|'
- r'[^x0MC])|(\\[MC]-)+\w|[^\s\\])', String.Other)
- ],
'dqs': [
(r'"', String.Double, "#pop"),
- include('interpoling'),
- (r'[^#"]+', String.Double),
+ include('enddoublestr')
],
'interpoling': [
(r'#{', String.Interpol, 'interpoling_string'),
@@ -1741,7 +1766,12 @@ class ElixirLexer(RegexLexer):
include('interpoling'),
(r'[^#"]+', String.Symbol),
],
+ 'enddoublestr' : [
+ include('interpoling'),
+ (r'[^#"]+', String.Double),
+ ]
}
+ tokens.update(gen_elixir_sigil_rules())
class ElixirConsoleLexer(Lexer):
View
17 vendor/pygments-main/pygments/lexers/github.py
@@ -28,16 +28,25 @@ class Dasm16Lexer(RegexLexer):
mimetypes = ['text/x-dasm16']
INSTRUCTIONS = [
- 'SET', 'ADD', 'SUB', 'MUL', 'DIV', 'MOD', 'SHL',
- 'SHR', 'AND', 'BOR', 'XOR', 'IFE', 'IFN', 'IFG', 'IFB',
- 'JSR'
+ 'SET',
+ 'ADD', 'SUB',
+ 'MUL', 'MLI',
+ 'DIV', 'DVI',
+ 'MOD', 'MDI',
+ 'AND', 'BOR', 'XOR',
+ 'SHR', 'ASR', 'SHL',
+ 'IFB', 'IFC', 'IFE', 'IFN', 'IFG', 'IFA', 'IFL', 'IFU',
+ 'ADX', 'SBX',
+ 'STI', 'STD',
+ 'JSR',
+ 'INT', 'IAG', 'IAS', 'RFI', 'IAQ', 'HWN', 'HWQ', 'HWI',
]
REGISTERS = [
'A', 'B', 'C',
'X', 'Y', 'Z',
'I', 'J',
- 'SP', 'PC',
+ 'SP', 'PC', 'EX',
'POP', 'PEEK', 'PUSH'
]
View
2  vendor/pygments-main/pygments/lexers/math.py
@@ -50,7 +50,7 @@ class JuliaLexer(RegexLexer):
# keywords
(r'(begin|while|for|in|return|break|continue|'
r'macro|quote|let|if|elseif|else|try|catch|end|'
- r'bitstype|ccall)\b', Keyword),
+ r'bitstype|ccall|do)\b', Keyword),
(r'(local|global|const)\b', Keyword.Declaration),
(r'(module|import|export)\b', Keyword.Reserved),
(r'(Bool|Int|Int8|Int16|Int32|Int64|Uint|Uint8|Uint16|Uint32|Uint64'
View
17 vendor/pygments-main/pygments/lexers/other.py
@@ -297,7 +297,6 @@ class SmalltalkLexer(RegexLexer):
(r'\^|\:=|\_', Operator),
# temporaries
(r'[\]({}.;!]', Text),
-
],
'method definition' : [
# Not perfect can't allow whitespaces at the beginning and the
@@ -316,7 +315,7 @@ class SmalltalkLexer(RegexLexer):
(r'', Text, '#pop'), # else pop
],
'literals' : [
- (r'\'[^\']*\'', String, 'afterobject'),
+ (r"'(''|[^'])*'", String, 'afterobject'),
(r'\$.', String.Char, 'afterobject'),
(r'#\(', String.Symbol, 'parenth'),
(r'\)', Text, 'afterobject'),
@@ -327,14 +326,14 @@ class SmalltalkLexer(RegexLexer):
(r'(\d+r)?-?\d+(\.\d+)?(e-?\d+)?', Number),
(r'[-+*/\\~<>=|&#!?,@%\w:]+', String.Symbol),
# literals
- (r'\'[^\']*\'', String),
+ (r"'(''|[^'])*'", String),
(r'\$.', String.Char),
(r'#*\(', String.Symbol, 'inner_parenth'),
],
'parenth' : [
# This state is a bit tricky since
# we can't just pop this state
- (r'\)', String.Symbol, ('root','afterobject')),
+ (r'\)', String.Symbol, ('root', 'afterobject')),
include('_parenth_helper'),
],
'inner_parenth': [
@@ -344,7 +343,7 @@ class SmalltalkLexer(RegexLexer):
'whitespaces' : [
# skip whitespace and comments
(r'\s+', Text),
- (r'"[^"]*"', Comment),
+ (r'"(""|[^"])*"', Comment),
],
'objects' : [
(r'\[', Text, 'blockvariables'),
@@ -353,7 +352,7 @@ class SmalltalkLexer(RegexLexer):
Name.Builtin.Pseudo, 'afterobject'),
(r'\b[A-Z]\w*(?!:)\b', Name.Class, 'afterobject'),
(r'\b[a-z]\w*(?!:)\b', Name.Variable, 'afterobject'),
- (r'#("[^"]*"|[-+*/\\~<>=|&!?,@%]+|[\w:]+)',
+ (r'#("(""|[^"])*"|[-+*/\\~<>=|&!?,@%]+|[\w:]+)',
String.Symbol, 'afterobject'),
include('literals'),
],
@@ -374,11 +373,11 @@ class SmalltalkLexer(RegexLexer):
],
'squeak fileout' : [
# Squeak fileout format (optional)
- (r'^"[^"]*"!', Keyword),
- (r"^'[^']*'!", Keyword),
+ (r'^"(""|[^"])*"!', Keyword),
+ (r"^'(''|[^'])*'!", Keyword),
(r'^(!)(\w+)( commentStamp: )(.*?)( prior: .*?!\n)(.*?)(!)',
bygroups(Keyword, Name.Class, Keyword, String, Keyword, Text, Keyword)),
- (r'^(!)(\w+(?: class)?)( methodsFor: )(\'[^\']*\')(.*?!)',
+ (r"^(!)(\w+(?: class)?)( methodsFor: )('(?:''|[^'])*')(.*?!)",
bygroups(Keyword, Name.Class, Keyword, String, Keyword)),
(r'^(\w+)( subclass: )(#\w+)'
r'(\s+instanceVariableNames: )(.*?)'
View
10 vendor/pygments-main/pygments/lexers/sql.py
@@ -39,7 +39,6 @@
"""
import re
-from copy import deepcopy
from pygments.lexer import Lexer, RegexLexer, do_insertions, bygroups
from pygments.token import Punctuation, \
@@ -61,9 +60,6 @@ def language_callback(lexer, match):
"""Parse the content of a $-string using a lexer
The lexer is chosen looking for a nearby LANGUAGE.
-
- Note: this function should have been a `PostgresBase` method, but the
- rules deepcopy fails in this case.
"""
l = None
m = language_re.match(lexer.text[match.end():match.end()+100])
@@ -93,8 +89,6 @@ class PostgresBase(object):
had, _tokens could be created on this ancestor and not updated for the
other classes, resulting e.g. in PL/pgSQL parsed as SQL. This shortcoming
seem to suggest that regexp lexers are not really subclassable.
-
- `language_callback` should really be our method, but this breaks deepcopy.
"""
def get_tokens_unprocessed(self, text, *args):
# Have a copy of the entire text to be used by `language_callback`.
@@ -182,7 +176,7 @@ class PlPgsqlLexer(PostgresBase, RegexLexer):
mimetypes = ['text/x-plpgsql']
flags = re.IGNORECASE
- tokens = deepcopy(PostgresLexer.tokens)
+ tokens = dict((k, l[:]) for (k, l) in PostgresLexer.tokens.iteritems())
# extend the keywords list
for i, pattern in enumerate(tokens['root']):
@@ -216,7 +210,7 @@ class PsqlRegexLexer(PostgresBase, RegexLexer):
aliases = [] # not public
flags = re.IGNORECASE
- tokens = deepcopy(PostgresLexer.tokens)
+ tokens = dict((k, l[:]) for (k, l) in PostgresLexer.tokens.iteritems())
tokens['root'].append(
(r'\\[^\s]+', Keyword.Pseudo, 'psql-command'))
View
1  vendor/pygments-main/pygments/lexers/templates.py
@@ -1504,6 +1504,7 @@ class ColdfusionLexer(RegexLexer):
],
}
+
class ColdfusionMarkupLexer(RegexLexer):
"""
Coldfusion markup only
View
2  vendor/pygments-main/pygments/lexers/text.py
@@ -528,7 +528,7 @@ class ApacheConfLexer(RegexLexer):
(r'(#.*?)$', Comment),
(r'(<[^\s>]+)(?:(\s+)(.*?))?(>)',
bygroups(Name.Tag, Text, String, Name.Tag)),
- (r'([a-zA-Z][a-zA-Z0-9]*)(\s+)',
+ (r'([a-zA-Z][a-zA-Z0-9_]*)(\s+)',
bygroups(Name.Builtin, Text), 'value'),
(r'\.+', Text),
],
View
17 vendor/pygments-main/pygments/lexers/web.py
@@ -101,6 +101,16 @@ class JSONLexer(RegexLexer):
filenames = ['*.json']
mimetypes = [ 'application/json', ]
+ # integer part of a number
+ int_part = r'-?(0|[1-9]\d*)'
+
+ # fractional part of a number
+ frac_part = r'\.\d+'
+
+ # exponential part of a number
+ exp_part = r'[eE](\+|-)?\d+'
+
+
flags = re.DOTALL
tokens = {
'whitespace': [
@@ -108,9 +118,12 @@ class JSONLexer(RegexLexer):
],
# represents a simple terminal value
- 'simplevalue':[
+ 'simplevalue': [
(r'(true|false|null)\b', Keyword.Constant),
- (r'-?[0-9]+', Number.Integer),
+ (('%(int_part)s(%(frac_part)s%(exp_part)s|'
+ '%(exp_part)s|%(frac_part)s)') % vars(),
+ Number.Float),
+ (int_part, Number.Integer),
(r'"(\\\\|\\"|[^"])*"', String.Double),
],
View
2  vendor/pygments-main/pygments/style.py
@@ -26,7 +26,7 @@ def colorformat(text):
if len(col) == 6:
return col
elif len(col) == 3:
- return col[0]+'0'+col[1]+'0'+col[2]+'0'
+ return col[0]*2 + col[1]*2 + col[2]*2
elif text == '':
return ''
assert False, "wrong color format %r" % text
View
8 vendor/pygments-main/tests/test_regexlexer.py
@@ -11,6 +11,7 @@
from pygments.token import Text
from pygments.lexer import RegexLexer
+from pygments.lexer import bygroups
class TestLexer(RegexLexer):
@@ -37,3 +38,10 @@ def test(self):
self.assertEqual(toks,
[(0, Text.Root, 'a'), (1, Text.Rag, 'b'), (2, Text.Rag, 'c'),
(3, Text.Beer, 'd'), (4, Text.Root, 'e')])
+
+ def test_multiline(self):
+ lx = TestLexer()
+ toks = list(lx.get_tokens_unprocessed('a\ne'))
+ self.assertEqual(toks,
+ [(0, Text.Root, 'a'), (1, Text, u'\n'),
+ (2, Text.Root, 'e')])
Please sign in to comment.
Something went wrong with that request. Please try again.