Major updates to Caddyfile lexer (#932)

* Major updates to Caddyfile lexer

* yaml editorconfig

* nolint false positive

.editorconfig

@@ -11,3 +11,7 @@ insert_final_newline = true
 indent_style = space
 indent_size = 2
 insert_final_newline = false
+
+[*.yml]
+indent_style = space
+indent_size = 2

lexers/caddyfile.go

@@ -4,52 +4,82 @@ import (
 	. "github.com/alecthomas/chroma/v2" // nolint
 )
 
+// Matcher token stub for docs, or
+// Named matcher: @name, or
+// Path matcher: /foo, or
+// Wildcard path matcher: *
+// nolint: gosec
+var caddyfileMatcherTokenRegexp = `(\[\<matcher\>\]|@[^\s]+|/[^\s]+|\*)`
+
+// Comment at start of line, or
+// Comment preceded by whitespace
+var caddyfileCommentRegexp = `(^|\s+)#.*\n`
+
 // caddyfileCommon are the rules common to both of the lexer variants
 func caddyfileCommonRules() Rules {
 	return Rules{
 		"site_block_common": {
+			Include("site_body"),
+			// Any other directive
+			{`[^\s#]+`, Keyword, Push("directive")},
+			Include("base"),
+		},
+		"site_body": {
 			// Import keyword
-			{`(import)(\s+)([^\s]+)`, ByGroups(Keyword, Text, NameVariableMagic), nil},
+			{`\b(import|invoke)\b( [^\s#]+)`, ByGroups(Keyword, Text), Push("subdirective")},
 			// Matcher definition
 			{`@[^\s]+(?=\s)`, NameDecorator, Push("matcher")},
 			// Matcher token stub for docs
 			{`\[\<matcher\>\]`, NameDecorator, Push("matcher")},
 			// These cannot have matchers but may have things that look like
 			// matchers in their arguments, so we just parse as a subdirective.
-			{`try_files`, Keyword, Push("subdirective")},
+			{`\b(try_files|tls|log|bind)\b`, Keyword, Push("subdirective")},
 			// These are special, they can nest more directives
-			{`handle_errors|handle|route|handle_path|not`, Keyword, Push("nested_directive")},
-			// Any other directive
-			{`[^\s#]+`, Keyword, Push("directive")},
-			Include("base"),
+			{`\b(handle_errors|handle_path|handle_response|replace_status|handle|route)\b`, Keyword, Push("nested_directive")},
+			// uri directive has special syntax
+			{`\b(uri)\b`, Keyword, Push("uri_directive")},
 		},
 		"matcher": {
 			{`\{`, Punctuation, Push("block")},
 			// Not can be one-liner
 			{`not`, Keyword, Push("deep_not_matcher")},
+			// Heredoc for CEL expression
+			Include("heredoc"),
+			// Backtick for CEL expression
+			{"`", StringBacktick, Push("backticks")},
 			// Any other same-line matcher
 			{`[^\s#]+`, Keyword, Push("arguments")},
 			// Terminators
-			{`\n`, Text, Pop(1)},
+			{`\s*\n`, Text, Pop(1)},
 			{`\}`, Punctuation, Pop(1)},
 			Include("base"),
 		},
 		"block": {
 			{`\}`, Punctuation, Pop(2)},
+			// Using double quotes doesn't stop at spaces
+			{`"`, StringDouble, Push("double_quotes")},
+			// Using backticks doesn't stop at spaces
+			{"`", StringBacktick, Push("backticks")},
 			// Not can be one-liner
 			{`not`, Keyword, Push("not_matcher")},
-			// Any other subdirective
+			// Directives & matcher definitions
+			Include("site_body"),
+			// Any directive
 			{`[^\s#]+`, Keyword, Push("subdirective")},
 			Include("base"),
 		},
 		"nested_block": {
 			{`\}`, Punctuation, Pop(2)},
-			// Matcher definition
-			{`@[^\s]+(?=\s)`, NameDecorator, Push("matcher")},
-			// Something that starts with literally < is probably a docs stub
-			{`\<[^#]+\>`, Keyword, Push("nested_directive")},
-			// Any other directive
-			{`[^\s#]+`, Keyword, Push("nested_directive")},
+			// Using double quotes doesn't stop at spaces
+			{`"`, StringDouble, Push("double_quotes")},
+			// Using backticks doesn't stop at spaces
+			{"`", StringBacktick, Push("backticks")},
+			// Not can be one-liner
+			{`not`, Keyword, Push("not_matcher")},
+			// Directives & matcher definitions
+			Include("site_body"),
+			// Any other subdirective
+			{`[^\s#]+`, Keyword, Push("directive")},
 			Include("base"),
 		},
 		"not_matcher": {
@@ -66,69 +96,97 @@ func caddyfileCommonRules() Rules {
 		},
 		"directive": {
 			{`\{(?=\s)`, Punctuation, Push("block")},
-			Include("matcher_token"),
-			Include("comments_pop_1"),
-			{`\n`, Text, Pop(1)},
+			{caddyfileMatcherTokenRegexp, NameDecorator, Push("arguments")},
+			{caddyfileCommentRegexp, CommentSingle, Pop(1)},
+			{`\s*\n`, Text, Pop(1)},
 			Include("base"),
 		},
 		"nested_directive": {
 			{`\{(?=\s)`, Punctuation, Push("nested_block")},
-			Include("matcher_token"),
-			Include("comments_pop_1"),
-			{`\n`, Text, Pop(1)},
+			{caddyfileMatcherTokenRegexp, NameDecorator, Push("nested_arguments")},
+			{caddyfileCommentRegexp, CommentSingle, Pop(1)},
+			{`\s*\n`, Text, Pop(1)},
 			Include("base"),
 		},
 		"subdirective": {
 			{`\{(?=\s)`, Punctuation, Push("block")},
-			Include("comments_pop_1"),
-			{`\n`, Text, Pop(1)},
+			{caddyfileCommentRegexp, CommentSingle, Pop(1)},
+			{`\s*\n`, Text, Pop(1)},
 			Include("base"),
 		},
 		"arguments": {
 			{`\{(?=\s)`, Punctuation, Push("block")},
-			Include("comments_pop_2"),
+			{caddyfileCommentRegexp, CommentSingle, Pop(2)},
 			{`\\\n`, Text, nil}, // Skip escaped newlines
-			{`\n`, Text, Pop(2)},
+			{`\s*\n`, Text, Pop(2)},
+			Include("base"),
+		},
+		"nested_arguments": {
+			{`\{(?=\s)`, Punctuation, Push("nested_block")},
+			{caddyfileCommentRegexp, CommentSingle, Pop(2)},
+			{`\\\n`, Text, nil}, // Skip escaped newlines
+			{`\s*\n`, Text, Pop(2)},
 			Include("base"),
 		},
 		"deep_subdirective": {
 			{`\{(?=\s)`, Punctuation, Push("block")},
-			Include("comments_pop_3"),
-			{`\n`, Text, Pop(3)},
+			{caddyfileCommentRegexp, CommentSingle, Pop(3)},
+			{`\s*\n`, Text, Pop(3)},
+			Include("base"),
+		},
+		"uri_directive": {
+			{`\{(?=\s)`, Punctuation, Push("block")},
+			{caddyfileMatcherTokenRegexp, NameDecorator, nil},
+			{`(strip_prefix|strip_suffix|replace|path_regexp)`, NameConstant, Push("arguments")},
+			{caddyfileCommentRegexp, CommentSingle, Pop(1)},
+			{`\s*\n`, Text, Pop(1)},
 			Include("base"),
 		},
-		"matcher_token": {
-			{`@[^\s]+`, NameDecorator, Push("arguments")},         // Named matcher
-			{`/[^\s]+`, NameDecorator, Push("arguments")},         // Path matcher
-			{`\*`, NameDecorator, Push("arguments")},              // Wildcard path matcher
-			{`\[\<matcher\>\]`, NameDecorator, Push("arguments")}, // Matcher token stub for docs
+		"double_quotes": {
+			Include("placeholder"),
+			{`\\"`, StringDouble, nil},
+			{`[^"]`, StringDouble, nil},
+			{`"`, StringDouble, Pop(1)},
 		},
-		"comments": {
-			{`^#.*\n`, CommentSingle, nil},   // Comment at start of line
-			{`\s+#.*\n`, CommentSingle, nil}, // Comment preceded by whitespace
+		"backticks": {
+			Include("placeholder"),
+			{"\\\\`", StringBacktick, nil},
+			{"[^`]", StringBacktick, nil},
+			{"`", StringBacktick, Pop(1)},
 		},
-		"comments_pop_1": {
-			{`^#.*\n`, CommentSingle, Pop(1)},   // Comment at start of line
-			{`\s+#.*\n`, CommentSingle, Pop(1)}, // Comment preceded by whitespace
+		"optional": {
+			// Docs syntax for showing optional parts with [ ]
+			{`\[`, Punctuation, Push("optional")},
+			Include("name_constants"),
+			{`\|`, Punctuation, nil},
+			{`[^\[\]\|]+`, String, nil},
+			{`\]`, Punctuation, Pop(1)},
 		},
-		"comments_pop_2": {
-			{`^#.*\n`, CommentSingle, Pop(2)},   // Comment at start of line
-			{`\s+#.*\n`, CommentSingle, Pop(2)}, // Comment preceded by whitespace
+		"heredoc": {
+			{`(<<([a-zA-Z0-9_-]+))(\n(.*|\n)*)(\s*)(\2)`, ByGroups(StringHeredoc, nil, String, String, String, StringHeredoc), nil},
 		},
-		"comments_pop_3": {
-			{`^#.*\n`, CommentSingle, Pop(3)},   // Comment at start of line
-			{`\s+#.*\n`, CommentSingle, Pop(3)}, // Comment preceded by whitespace
+		"name_constants": {
+			{`\b(most_recently_modified|largest_size|smallest_size|first_exist|internal|disable_redirects|ignore_loaded_certs|disable_certs|private_ranges|first|last|before|after|on|off)\b(\||(?=\]|\s|$))`, ByGroups(NameConstant, Punctuation), nil},
+		},
+		"placeholder": {
+			// Placeholder with dots, colon for default value, brackets for args[0:]
+			{`\{[\w+.\[\]\:\$-]+\}`, StringEscape, nil},
+			// Handle opening brackets with no matching closing one
+			{`\{[^\}\s]*\b`, String, nil},
 		},
 		"base": {
-			Include("comments"),
-			{`(on|off|first|last|before|after|internal|strip_prefix|strip_suffix|replace)\b`, NameConstant, nil},
-			{`(https?://)?([a-z0-9.-]+)(:)([0-9]+)`, ByGroups(Name, Name, Punctuation, LiteralNumberInteger), nil},
-			{`[a-z-]+/[a-z-+]+`, LiteralString, nil},
-			{`[0-9]+[km]?\b`, LiteralNumberInteger, nil},
-			{`\{[\w+.\$-]+\}`, LiteralStringEscape, nil}, // Placeholder
-			{`\[(?=[^#{}$]+\])`, Punctuation, nil},
-			{`\]|\|`, Punctuation, nil},
-			{`[^\s#{}$\]]+`, LiteralString, nil},
+			{caddyfileCommentRegexp, CommentSingle, nil},
+			{`\[\<matcher\>\]`, NameDecorator, nil},
+			Include("name_constants"),
+			Include("heredoc"),
+			{`(https?://)?([a-z0-9.-]+)(:)([0-9]+)([^\s]*)`, ByGroups(Name, Name, Punctuation, NumberInteger, Name), nil},
+			{`\[`, Punctuation, Push("optional")},
+			{"`", StringBacktick, Push("backticks")},
+			{`"`, StringDouble, Push("double_quotes")},
+			Include("placeholder"),
+			{`[a-z-]+/[a-z-+]+`, String, nil},
+			{`[0-9]+([smhdk]|ns|us|µs|ms)?\b`, NumberInteger, nil},
+			{`[^\s\n#\{]+`, String, nil},
 			{`/[^\s#]*`, Name, nil},
 			{`\s+`, Text, nil},
 		},
@@ -149,27 +207,29 @@ var Caddyfile = Register(MustNewLexer(
 func caddyfileRules() Rules {
 	return Rules{
 		"root": {
-			Include("comments"),
+			{caddyfileCommentRegexp, CommentSingle, nil},
 			// Global options block
 			{`^\s*(\{)\s*$`, ByGroups(Punctuation), Push("globals")},
+			// Top level import
+			{`(import)(\s+)([^\s]+)`, ByGroups(Keyword, Text, NameVariableMagic), nil},
 			// Snippets
-			{`(\([^\s#]+\))(\s*)(\{)`, ByGroups(NameVariableAnonymous, Text, Punctuation), Push("snippet")},
+			{`(&?\([^\s#]+\))(\s*)(\{)`, ByGroups(NameVariableAnonymous, Text, Punctuation), Push("snippet")},
 			// Site label
 			{`[^#{(\s,]+`, GenericHeading, Push("label")},
 			// Site label with placeholder
-			{`\{[\w+.\$-]+\}`, LiteralStringEscape, Push("label")},
+			{`\{[\w+.\[\]\:\$-]+\}`, StringEscape, Push("label")},
 			{`\s+`, Text, nil},
 		},
 		"globals": {
 			{`\}`, Punctuation, Pop(1)},
-			{`[^\s#]+`, Keyword, Push("directive")},
+			// Global options are parsed as subdirectives (no matcher)
+			{`[^\s#]+`, Keyword, Push("subdirective")},
 			Include("base"),
 		},
 		"snippet": {
 			{`\}`, Punctuation, Pop(1)},
-			// Matcher definition
-			{`@[^\s]+(?=\s)`, NameDecorator, Push("matcher")},
-			// Any directive
+			Include("site_body"),
+			// Any other directive
 			{`[^\s#]+`, Keyword, Push("directive")},
 			Include("base"),
 		},
@@ -179,7 +239,7 @@ func caddyfileRules() Rules {
 			{`,\s*\n?`, Text, nil},
 			{` `, Text, nil},
 			// Site label with placeholder
-			{`\{[\w+.\$-]+\}`, LiteralStringEscape, nil},
+			Include("placeholder"),
 			// Site label
 			{`[^#{(\s,]+`, GenericHeading, nil},
 			// Comment after non-block label (hack because comments end in \n)

lexers/testdata/caddyfile.actual

@@ -4,21 +4,42 @@
 	on_demand_tls {
 		ask https://example.com
 	}
+	log default {
+		output file /var/log/caddy/access.log
+		format json
+	}
+	auto_https disable_redirects
+	renew_interval 20m
+
+	# this is a comment
+	servers 192.168.1.2:8080 {
+		name public
+		trusted_proxies static private_ranges
+		log_credentials
+	}
 }
 
-(blocking) {  
+# top level comment
+
+(blocking) {
 	@blocked {
 		path *.txt *.md *.mdown /site/*
 	}
 	redir @blocked /
 }
 
+http://example.com {
+	respond "http"
+}
+
 example.com, fake.org, {$ENV_SITE} {
 	root * /srv
 
 	respond /get-env {$ENV_VAR}
+	respond /get-env {$ENV_VAR:default}
 
-	tls off
+	tls internal
+	tls /path/to/cert.pem /path/to/key.pem
 
 	route {
 		# Add trailing slash for directory requests
@@ -67,6 +88,55 @@ example.com, fake.org, {$ENV_SITE} {
 	respond @singleLine "Awesome."
 
 	import blocking
+	import blocking foo
+	import glob/*
 
 	file_server
-}
+
+	@named host example.com
+	handle @named {
+		handle /foo* {
+			handle /foo* {
+				respond "{path} foo"
+			}
+		}
+		respond "foo"
+	}
+
+	handle_path /foo* {
+		respond "foo"
+	}
+
+	reverse_proxy /api/* unix//var/run/api.sock {
+		@good status 200
+		handle_response @good {
+			rewrite * /foo{uri}
+			file_server
+		}
+	}
+
+	respond <<HTML
+		<!DOCTYPE html>
+		<html>
+			<head>
+				<title>Test</title>
+			</head>
+			<body>
+				<h1>Hello, world!</h1>
+			</body>
+		</html>
+		HTML 200
+
+	@file `file()`
+	@first `file({'try_files': [{path}, {path} + '/', 'index.html']})`
+	@smallest `file({'try_policy': 'smallest_size', 'try_files': ['a.txt', 'b.txt']})`
+
+	@without-both {
+		not {
+			path /api/*
+			method POST
+		}
+	}
+
+	path_regexp [<name>] <regexp>
+}