Refinement of the C# lexer (alecthomas#563)

lexers/c/csharp.go

@@ -24,7 +24,8 @@ func cSharpRules() Rules {
 			{`^\s*\[.*?\]`, NameAttribute, nil},
 			{`[^\S\n]+`, Text, nil},
 			{`\\\n`, Text, nil},
-			{`//.*?\n`, CommentSingle, nil},
+			{`///[^\n\r]+`, CommentSpecial, nil},
+			{`//[^\n\r]+`, CommentSingle, nil},
 			{`/[*].*?[*]/`, CommentMultiline, nil},
 			{`\n`, Text, nil},
 			{`[~!%^&*()+=|\[\]:;,.<>/?-]`, Punctuation, nil},
@@ -34,12 +35,12 @@ func cSharpRules() Rules {
 			{`"(\\\\|\\"|[^"\n])*["\n]`, LiteralString, nil},
 			{`'\\.'|'[^\\]'`, LiteralStringChar, nil},
 			{`0[xX][0-9a-fA-F]+[Ll]?|[0-9_](\.[0-9]*)?([eE][+-]?[0-9]+)?[flFLdD]?`, LiteralNumber, nil},
-			{`#[ \t]*(if|endif|else|elif|define|undef|line|error|warning|region|endregion|pragma)\b.*?\n`, CommentPreproc, nil},
+			{`#[ \t]*(if|endif|else|elif|define|undef|line|error|warning|region|endregion|pragma|nullable)\b[^\n\r]+`, CommentPreproc, nil},
 			{`\b(extern)(\s+)(alias)\b`, ByGroups(Keyword, Text, Keyword), nil},
-			{`(abstract|as|async|await|base|break|by|case|catch|checked|const|continue|default|delegate|do|else|enum|event|explicit|extern|false|finally|fixed|for|foreach|goto|if|implicit|in|interface|internal|is|let|lock|new|null|on|operator|out|override|params|private|protected|public|readonly|ref|return|sealed|sizeof|stackalloc|static|switch|this|throw|true|try|typeof|unchecked|unsafe|virtual|void|while|get|set|new|partial|yield|add|remove|value|alias|ascending|descending|from|group|into|orderby|select|thenby|where|join|equals)\b`, Keyword, nil},
+			{`(abstract|as|async|await|base|break|by|case|catch|checked|const|continue|default|delegate|do|else|enum|event|explicit|extern|false|finally|fixed|for|foreach|goto|if|implicit|in|init|internal|is|let|lock|new|null|on|operator|out|override|params|private|protected|public|readonly|ref|return|sealed|sizeof|stackalloc|static|switch|this|throw|true|try|typeof|unchecked|unsafe|virtual|void|while|get|set|new|partial|yield|add|remove|value|alias|ascending|descending|from|group|into|orderby|select|thenby|where|join|equals)\b`, Keyword, nil},
 			{`(global)(::)`, ByGroups(Keyword, Punctuation), nil},
 			{`(bool|byte|char|decimal|double|dynamic|float|int|long|object|sbyte|short|string|uint|ulong|ushort|var)\b\??`, KeywordType, nil},
-			{`(class|struct)(\s+)`, ByGroups(Keyword, Text), Push("class")},
+			{`(class|struct|record|interface)(\s+)`, ByGroups(Keyword, Text), Push("class")},
 			{`(namespace|using)(\s+)`, ByGroups(Keyword, Text), Push("namespace")},
 			{`@?[_a-zA-Z]\w*`, Name, nil},
 		},

lexers/testdata/csharp/csharp_8_nullable.actual

@@ -0,0 +1,10 @@
+using System;
+
+#nullable enable
+
+public struct Student
+{
+    public string FirstName;
+    public string? MiddleName;
+    public string LastName;
+}

lexers/testdata/csharp/csharp_8_nullable.expected

@@ -0,0 +1,39 @@
+[
+  {"type":"Keyword","value":"using"},
+  {"type":"Text","value":" "},
+  {"type":"NameNamespace","value":"System"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"CommentPreproc","value":"#nullable enable"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"Keyword","value":"public"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"struct"},
+  {"type":"Text","value":" "},
+  {"type":"NameClass","value":"Student"},
+  {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":"{"},
+  {"type":"Text","value":"\n    "},
+  {"type":"Keyword","value":"public"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordType","value":"string"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"FirstName"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n    "},
+  {"type":"Keyword","value":"public"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordType","value":"string?"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"MiddleName"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n    "},
+  {"type":"Keyword","value":"public"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordType","value":"string"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"LastName"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":"}"}
+]

lexers/testdata/csharp/csharp_9_records_1.actual

@@ -0,0 +1 @@
+public record Person(string FirstName, string LastName);

lexers/testdata/csharp/csharp_9_records_1.expected

@@ -0,0 +1,17 @@
+[
+  {"type":"Keyword","value":"public"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"record"},
+  {"type":"Text","value":" "},
+  {"type":"NameClass","value":"Person"},
+  {"type":"Punctuation","value":"("},
+  {"type":"KeywordType","value":"string"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"FirstName"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"KeywordType","value":"string"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"LastName"},
+  {"type":"Punctuation","value":");"}
+]

lexers/testdata/csharp/csharp_9_records_2.actual

@@ -0,0 +1,5 @@
+public record Person
+{
+    public string FirstName { get; init; } = default!;
+    public string LastName { get; init; } = default!;
+};

lexers/testdata/csharp/csharp_9_records_2.expected

@@ -0,0 +1,53 @@
+[
+  {"type":"Keyword","value":"public"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"record"},
+  {"type":"Text","value":" "},
+  {"type":"NameClass","value":"Person"},
+  {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":"{"},
+  {"type":"Text","value":"\n    "},
+  {"type":"Keyword","value":"public"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordType","value":"string"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"FirstName"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"get"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"init"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"}"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"default"},
+  {"type":"Punctuation","value":"!;"},
+  {"type":"Text","value":"\n    "},
+  {"type":"Keyword","value":"public"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordType","value":"string"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"LastName"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"get"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"init"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"}"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"default"},
+  {"type":"Punctuation","value":"!;"},
+  {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":"};"}
+]

lexers/testdata/csharp/csharp_comment_multi.actual

@@ -0,0 +1,7 @@
+/*
+    Hello World
+*/
+
+/***
+*   Hello Developer
+***/

lexers/testdata/csharp/csharp_comment_multi.expected

@@ -0,0 +1,5 @@
+[
+  {"type":"CommentMultiline","value":"/*\n    Hello World\n*/"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"CommentMultiline","value":"/***\n*   Hello Developer\n***/"}
+]

lexers/testdata/csharp/csharp_comment_single.actual

@@ -0,0 +1,2 @@
+// TODO: Maybe later
+// TODO: Implement more features

lexers/testdata/csharp/csharp_comment_single.expected

@@ -0,0 +1,5 @@
+[
+  {"type":"CommentSingle","value":"// TODO: Maybe later"},
+  {"type":"Text","value":"\n"},
+  {"type":"CommentSingle","value":"// TODO: Implement more features"}
+]

lexers/testdata/csharp/csharp_comment_summary.actual

@@ -0,0 +1,3 @@
+/// <summary>
+/// Class <c>Point</c> models a point in a two-dimensional plane.
+/// </summary>

lexers/testdata/csharp/csharp_comment_summary.expected

@@ -0,0 +1,7 @@
+[
+  {"type":"CommentSpecial","value":"/// \u003csummary\u003e"},
+  {"type":"Text","value":"\n"},
+  {"type":"CommentSpecial","value":"/// Class \u003cc\u003ePoint\u003c/c\u003e models a point in a two-dimensional plane."},
+  {"type":"Text","value":"\n"},
+  {"type":"CommentSpecial","value":"/// \u003c/summary\u003e"}
+]

lexers/testdata/csharp/csharp_interface.actual

@@ -0,0 +1,5 @@
+public interface IOrder
+{
+    DateTime Purchased { get; }
+    decimal Cost { get; }
+}

lexers/testdata/csharp/csharp_interface.expected

@@ -0,0 +1,33 @@
+[
+  {"type":"Keyword","value":"public"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"interface"},
+  {"type":"Text","value":" "},
+  {"type":"NameClass","value":"IOrder"},
+  {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":"{"},
+  {"type":"Text","value":"\n    "},
+  {"type":"Name","value":"DateTime"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"Purchased"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"get"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"}"},
+  {"type":"Text","value":"\n    "},
+  {"type":"KeywordType","value":"decimal"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"Cost"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"get"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"}"},
+  {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":"}"}
+]