Skip to content

Commit

Permalink
yaml parse keys as NameTag rather than Keyword
Browse files Browse the repository at this point in the history
  • Loading branch information
@yaythomas @alecthomas
yaythomas authored and alecthomas committed May 11, 2020
1 parent ec5955f commit 0d4fb38
Show file tree
Hide file tree
Showing 2 changed files with 54 additions and 54 deletions.
100 changes: 50 additions & 50 deletions lexers/testdata/yaml.expected
View file
Expand Up @@ -15,12 +15,12 @@
{"type":"TextWhitespace","value":"\n"},
{"type":"Comment","value":"# which is equivalent to a dictionary, hash or object in other languages."},
{"type":"TextWhitespace","value":"\n"},
{"type":"Keyword","value":"key"},
{"type":"NameTag","value":"key"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"value"},
{"type":"TextWhitespace","value":"\n"},
{"type":"Keyword","value":"another_key"},
{"type":"NameTag","value":"another_key"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"Another"},
Expand All @@ -31,12 +31,12 @@
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"here."},
{"type":"TextWhitespace","value":"\n"},
{"type":"Keyword","value":"a_number_value"},
{"type":"NameTag","value":"a_number_value"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"LiteralNumber","value":"100"},
{"type":"TextWhitespace","value":"\n"},
{"type":"Keyword","value":"scientific_notation"},
{"type":"NameTag","value":"scientific_notation"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"LiteralNumber","value":"1e+12"},
Expand All @@ -45,46 +45,46 @@
{"type":"TextWhitespace","value":"\n"},
{"type":"Comment","value":"# it to be interpreted as a boolean, use true"},
{"type":"TextWhitespace","value":"\n"},
{"type":"Keyword","value":"boolean"},
{"type":"NameTag","value":"boolean"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"KeywordConstant","value":"true"},
{"type":"TextWhitespace","value":"\n"},
{"type":"Keyword","value":"null_value"},
{"type":"NameTag","value":"null_value"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"KeywordConstant","value":"null"},
{"type":"TextWhitespace","value":"\n"},
{"type":"Keyword","value":"key with spaces"},
{"type":"NameTag","value":"key with spaces"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"value"},
{"type":"TextWhitespace","value":"\n"},
{"type":"Comment","value":"# Notice that strings don't need to be quoted. However, they can be."},
{"type":"TextWhitespace","value":"\n"},
{"type":"Keyword","value":"however"},
{"type":"NameTag","value":"however"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"LiteralStringSingle","value":"'A string, enclosed in quotes.'"},
{"type":"TextWhitespace","value":"\n"},
{"type":"Keyword","value":"'Keys can be quoted too.'"},
{"type":"NameTag","value":"'Keys can be quoted too.'"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"LiteralStringDouble","value":"\"Useful if you want to put a ':' in your key.\""},
{"type":"TextWhitespace","value":"\n"},
{"type":"Keyword","value":"single quotes"},
{"type":"NameTag","value":"single quotes"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"LiteralStringSingle","value":"'have ''one'' escape pattern'"},
{"type":"TextWhitespace","value":"\n"},
{"type":"Keyword","value":"double quotes"},
{"type":"NameTag","value":"double quotes"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"LiteralStringDouble","value":"\"have many: \\\", \\0, \\t, \\u263A, \\x0d\\x0a == \\r\\n, and more.\""},
{"type":"TextWhitespace","value":"\n"},
{"type":"Comment","value":"# UTF-8/16/32 characters need to be encoded"},
{"type":"TextWhitespace","value":"\n"},
{"type":"Keyword","value":"Superscript two"},
{"type":"NameTag","value":"Superscript two"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"\\u00B2"},
Expand All @@ -93,7 +93,7 @@
{"type":"TextWhitespace","value":"\n"},
{"type":"Comment","value":"# or a 'folded block' (using '\u003e')."},
{"type":"TextWhitespace","value":"\n"},
{"type":"Keyword","value":"literal_block"},
{"type":"NameTag","value":"literal_block"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"LiteralStringDoc","value":"|\n This entire block of text will be the value of the 'literal_block' key,"},
Expand Down Expand Up @@ -155,7 +155,7 @@
{"type":"TextWhitespace","value":" "},
{"type":"LiteralStringDoc","value":"-\n these lines will be indented by 4 spaces."},
{"type":"TextWhitespace","value":"\n"},
{"type":"Keyword","value":"folded_style"},
{"type":"NameTag","value":"folded_style"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"LiteralStringDoc","value":"\u003e\n This entire block of text will be the value of 'folded_style', but this"},
Expand Down Expand Up @@ -226,32 +226,32 @@
{"type":"TextWhitespace","value":"\n\n"},
{"type":"Comment","value":"# Nesting uses indentation. 2 space indent is preferred (but not required)."},
{"type":"TextWhitespace","value":"\n"},
{"type":"Keyword","value":"a_nested_map"},
{"type":"NameTag","value":"a_nested_map"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Keyword","value":"key"},
{"type":"NameTag","value":"key"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"value"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Keyword","value":"another_key"},
{"type":"NameTag","value":"another_key"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"Another"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"Value"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Keyword","value":"another_nested_map"},
{"type":"NameTag","value":"another_nested_map"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Keyword","value":"hello"},
{"type":"NameTag","value":"hello"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"hello"},
{"type":"TextWhitespace","value":"\n\n"},
{"type":"Comment","value":"# Maps don't have to have string keys."},
{"type":"TextWhitespace","value":"\n"},
{"type":"Keyword","value":"0.25"},
{"type":"NameTag","value":"0.25"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"a"},
Expand Down Expand Up @@ -315,7 +315,7 @@
{"type":"TextWhitespace","value":"\n"},
{"type":"Comment","value":"# (note that the '-' counts as indentation):"},
{"type":"TextWhitespace","value":"\n"},
{"type":"Keyword","value":"a_sequence"},
{"type":"NameTag","value":"a_sequence"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Text","value":"- Item"},
Expand All @@ -336,12 +336,12 @@
{"type":"LiteralNumber","value":"4"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Text","value":"- "},
{"type":"Keyword","value":"key"},
{"type":"NameTag","value":"key"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"value"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Keyword","value":"another_key"},
{"type":"NameTag","value":"another_key"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"another_value"},
Expand Down Expand Up @@ -370,15 +370,15 @@
{"type":"TextWhitespace","value":"\n"},
{"type":"Comment","value":"# sequences:"},
{"type":"TextWhitespace","value":"\n"},
{"type":"Keyword","value":"json_map"},
{"type":"NameTag","value":"json_map"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"{"},
{"type":"Keyword","value":"\"key\": "},
{"type":"NameTag","value":"\"key\": "},
{"type":"LiteralStringDouble","value":"\"value\""},
{"type":"Text","value":"}"},
{"type":"TextWhitespace","value":"\n"},
{"type":"Keyword","value":"json_seq"},
{"type":"NameTag","value":"json_seq"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"["},
Expand All @@ -394,11 +394,11 @@
{"type":"LiteralStringDouble","value":"\"takeoff\""},
{"type":"Punctuation","value":"]"},
{"type":"TextWhitespace","value":"\n"},
{"type":"Keyword","value":"and quotes are optional"},
{"type":"NameTag","value":"and quotes are optional"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"{"},
{"type":"Keyword","value":"key"},
{"type":"NameTag","value":"key"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"["},
Expand All @@ -425,7 +425,7 @@
{"type":"TextWhitespace","value":"\n"},
{"type":"Comment","value":"# content across your document. Both of these keys will have the same value:"},
{"type":"TextWhitespace","value":"\n"},
{"type":"Keyword","value":"anchored_content"},
{"type":"NameTag","value":"anchored_content"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"CommentPreproc","value":"\u0026anchor_name"},
Expand All @@ -450,19 +450,19 @@
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"keys."},
{"type":"TextWhitespace","value":"\n"},
{"type":"Keyword","value":"other_anchor"},
{"type":"NameTag","value":"other_anchor"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"CommentPreproc","value":"*anchor_name"},
{"type":"TextWhitespace","value":"\n\n"},
{"type":"Comment","value":"# Anchors can be used to duplicate/inherit properties"},
{"type":"TextWhitespace","value":"\n"},
{"type":"Keyword","value":"base"},
{"type":"NameTag","value":"base"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"CommentPreproc","value":"\u0026base"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Keyword","value":"name"},
{"type":"NameTag","value":"name"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"Everyone"},
Expand All @@ -479,32 +479,32 @@
{"type":"TextWhitespace","value":"\n"},
{"type":"Comment","value":"# into the current map."},
{"type":"TextWhitespace","value":"\n\n"},
{"type":"Keyword","value":"foo"},
{"type":"NameTag","value":"foo"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"CommentPreproc","value":"\u0026foo"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Keyword","value":"\u003c\u003c"},
{"type":"NameTag","value":"\u003c\u003c"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"CommentPreproc","value":"*base"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Keyword","value":"age"},
{"type":"NameTag","value":"age"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"LiteralNumber","value":"10"},
{"type":"TextWhitespace","value":"\n\n"},
{"type":"Keyword","value":"bar"},
{"type":"NameTag","value":"bar"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"CommentPreproc","value":"\u0026bar"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Keyword","value":"\u003c\u003c"},
{"type":"NameTag","value":"\u003c\u003c"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"CommentPreproc","value":"*base"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Keyword","value":"age"},
{"type":"NameTag","value":"age"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"LiteralNumber","value":"20"},
Expand All @@ -513,7 +513,7 @@
{"type":"TextWhitespace","value":"\n\n"},
{"type":"Comment","value":"# YAML also has tags, which you can use to explicitly declare types."},
{"type":"TextWhitespace","value":"\n"},
{"type":"Keyword","value":"explicit_string"},
{"type":"NameTag","value":"explicit_string"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"CommentPreproc","value":"!!str"},
Expand All @@ -524,7 +524,7 @@
{"type":"TextWhitespace","value":"\n"},
{"type":"Comment","value":"# complex number type."},
{"type":"TextWhitespace","value":"\n"},
{"type":"Keyword","value":"python_complex_number"},
{"type":"NameTag","value":"python_complex_number"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"CommentPreproc","value":"!!python/complex"},
Expand Down Expand Up @@ -563,17 +563,17 @@
{"type":"TextWhitespace","value":"\n"},
{"type":"Comment","value":"# ISO-formatted date and datetime literals are also parsed."},
{"type":"TextWhitespace","value":"\n"},
{"type":"Keyword","value":"datetime"},
{"type":"NameTag","value":"datetime"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"LiteralDate","value":"2001-12-15T02:59:43.1Z"},
{"type":"TextWhitespace","value":"\n"},
{"type":"Keyword","value":"datetime_with_spaces"},
{"type":"NameTag","value":"datetime_with_spaces"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"LiteralDate","value":"2001-12-14 21:59:43.10 -5"},
{"type":"TextWhitespace","value":"\n"},
{"type":"Keyword","value":"date"},
{"type":"NameTag","value":"date"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"LiteralDate","value":"2002-12-14"},
Expand All @@ -582,7 +582,7 @@
{"type":"TextWhitespace","value":"\n"},
{"type":"Comment","value":"# representation of a binary blob."},
{"type":"TextWhitespace","value":"\n"},
{"type":"Keyword","value":"gif_file"},
{"type":"NameTag","value":"gif_file"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"CommentPreproc","value":"!!binary"},
Expand All @@ -596,7 +596,7 @@
{"type":"TextWhitespace","value":"\n\n"},
{"type":"Comment","value":"# YAML also has a set type, which looks like this:"},
{"type":"TextWhitespace","value":"\n"},
{"type":"Keyword","value":"set"},
{"type":"NameTag","value":"set"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Punctuation","value":"?"},
Expand All @@ -611,7 +611,7 @@
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"item3"},
{"type":"TextWhitespace","value":"\n"},
{"type":"Keyword","value":"or"},
{"type":"NameTag","value":"or"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"{item1"},
Expand All @@ -624,20 +624,20 @@
{"type":"TextWhitespace","value":"\n\n"},
{"type":"Comment","value":"# Sets are just maps with null values; the above is equivalent to:"},
{"type":"TextWhitespace","value":"\n"},
{"type":"Keyword","value":"set2"},
{"type":"NameTag","value":"set2"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Keyword","value":"item1"},
{"type":"NameTag","value":"item1"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"KeywordConstant","value":"null"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Keyword","value":"item2"},
{"type":"NameTag","value":"item2"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"KeywordConstant","value":"null"},
{"type":"TextWhitespace","value":"\n "},
{"type":"Keyword","value":"item3"},
{"type":"NameTag","value":"item3"},
{"type":"Punctuation","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"KeywordConstant","value":"null"},
Expand Down
8 changes: 4 additions & 4 deletions lexers/y/yaml.go
View file
Expand Up @@ -37,10 +37,10 @@ var YAML = internal.Register(MustNewLexer(
{`\b[\w]+\b`, Text, nil},
},
"key": {
{`"[^"\n].*": `, Keyword, nil},
{`(-)( )([^"\n{]*)(:)( )`, ByGroups(Punctuation, Whitespace, Keyword, Punctuation, Whitespace), nil},
{`([^"\n{]*)(:)( )`, ByGroups(Keyword, Punctuation, Whitespace), nil},
{`([^"\n{]*)(:)(\n)`, ByGroups(Keyword, Punctuation, Whitespace), nil},
{`"[^"\n].*": `, NameTag, nil},
{`(-)( )([^"\n{]*)(:)( )`, ByGroups(Punctuation, Whitespace, NameTag, Punctuation, Whitespace), nil},
{`([^"\n{]*)(:)( )`, ByGroups(NameTag, Punctuation, Whitespace), nil},
{`([^"\n{]*)(:)(\n)`, ByGroups(NameTag, Punctuation, Whitespace), nil},
},
"whitespace": {
{`\s+`, Whitespace, nil},
Expand Down

0 comments on commit 0d4fb38

Please sign in to comment.