parser: replace eat_line_end_comments() with configurable eat_comments() (#8636)

Author: Lukas Neubert

vlib/v/parser/assign.v

@@ -99,7 +99,7 @@ fn (mut p Parser) partial_assign_stmt(left []ast.Expr, left_comments []ast.Comme
 	p.next()
 	mut comments := []ast.Comment{cap: 2 * left_comments.len + 1}
 	comments << left_comments
-	comments << p.eat_comments()
+	comments << p.eat_comments({})
 	mut right_comments := []ast.Comment{}
 	mut right := []ast.Expr{cap: left.len}
 	if p.tok.kind == .key_go {
@@ -113,7 +113,7 @@ fn (mut p Parser) partial_assign_stmt(left []ast.Expr, left_comments []ast.Comme
 		right, right_comments = p.expr_list()
 	}
 	comments << right_comments
-	end_comments := p.eat_line_end_comments()
+	end_comments := p.eat_comments(same_line: true)
 	mut has_cross_var := false
 	if op == .decl_assign {
 		// a, b := a + 1, b

vlib/v/parser/containers.v

@@ -41,7 +41,7 @@ fn (mut p Parser) array_init() ast.ArrayInit {
 		// [1,2,3] or [const]byte
 		for i := 0; p.tok.kind !in [.rsbr, .eof]; i++ {
 			exprs << p.expr(0)
-			ecmnts << p.eat_comments()
+			ecmnts << p.eat_comments({})
 			if p.tok.kind == .comma {
 				p.next()
 			}

vlib/v/parser/fn.v

@@ -87,7 +87,7 @@ pub fn (mut p Parser) call_expr(language table.Language, mod string) ast.CallExp
 	if fn_name in p.imported_symbols {
 		fn_name = p.imported_symbols[fn_name]
 	}
-	comments := p.eat_line_end_comments()
+	comments := p.eat_comments(same_line: true)
 	pos.update_last_line(p.prev_tok.line_nr)
 	return ast.CallExpr{
 		name: fn_name
@@ -121,7 +121,7 @@ pub fn (mut p Parser) call_args() []ast.CallArg {
 		if is_mut {
 			p.next()
 		}
-		mut comments := p.eat_comments()
+		mut comments := p.eat_comments({})
 		arg_start_pos := p.tok.position()
 		mut array_decompose := false
 		if p.tok.kind == .ellipsis {
@@ -146,7 +146,7 @@ pub fn (mut p Parser) call_args() []ast.CallArg {
 			comments = []ast.Comment{}
 		}
 		pos := arg_start_pos.extend(p.prev_tok.position())
-		comments << p.eat_comments()
+		comments << p.eat_comments({})
 		args << ast.CallArg{
 			is_mut: is_mut
 			share: table.sharetype_from_flags(is_shared, is_atomic)

vlib/v/parser/if_match.v

@@ -34,9 +34,9 @@ fn (mut p Parser) if_expr(is_comptime bool) ast.IfExpr {
 			p.tok.position()
 		}
 		if p.tok.kind == .key_else {
-			comments << p.eat_comments()
+			comments << p.eat_comments({})
 			p.check(.key_else)
-			comments << p.eat_comments()
+			comments << p.eat_comments({})
 			if p.tok.kind == .key_match {
 				p.error('cannot use `match` with `if` statements')
 				return ast.IfExpr{}
@@ -84,7 +84,7 @@ fn (mut p Parser) if_expr(is_comptime bool) ast.IfExpr {
 			p.error('cannot use `match` with `if` statements')
 			return ast.IfExpr{}
 		}
-		comments << p.eat_comments()
+		comments << p.eat_comments({})
 		mut cond := ast.Expr{}
 		mut is_guard := false
 		// `if x := opt() {`
@@ -93,9 +93,9 @@ fn (mut p Parser) if_expr(is_comptime bool) ast.IfExpr {
 			is_guard = true
 			var_pos := p.tok.position()
 			var_name := p.check_name()
-			comments << p.eat_comments()
+			comments << p.eat_comments({})
 			p.check(.decl_assign)
-			comments << p.eat_comments()
+			comments << p.eat_comments({})
 			expr := p.expr(0)
 			cond = ast.IfGuardExpr{
 				var_name: var_name
@@ -111,7 +111,7 @@ fn (mut p Parser) if_expr(is_comptime bool) ast.IfExpr {
 			prev_guard = false
 			cond = p.expr(0)
 		}
-		comments << p.eat_comments()
+		comments << p.eat_comments({})
 		end_pos := p.prev_tok.position()
 		body_pos := p.tok.position()
 		p.inside_if = false
@@ -129,7 +129,7 @@ fn (mut p Parser) if_expr(is_comptime bool) ast.IfExpr {
 		if is_guard {
 			p.close_scope()
 		}
-		comments = p.eat_comments()
+		comments = p.eat_comments({})
 		if is_comptime {
 			if p.tok.kind == .key_else {
 				p.error('use `\$else` instead of `else` in compile-time `if` branches')
@@ -168,7 +168,7 @@ fn (mut p Parser) match_expr() ast.MatchExpr {
 	if !no_lcbr {
 		p.check(.lcbr)
 	}
-	comments := p.eat_comments() // comments before the first branch
+	comments := p.eat_comments({}) // comments before the first branch
 	mut branches := []ast.MatchBranch{}
 	for p.tok.kind != .eof {
 		branch_first_pos := p.tok.position()
@@ -188,7 +188,7 @@ fn (mut p Parser) match_expr() ast.MatchExpr {
 			for {
 				// Sum type match
 				parsed_type := p.parse_type()
-				ecmnts << p.eat_comments()
+				ecmnts << p.eat_comments({})
 				types << parsed_type
 				exprs << ast.Type{
 					typ: parsed_type
@@ -205,7 +205,7 @@ fn (mut p Parser) match_expr() ast.MatchExpr {
 			for {
 				p.inside_match_case = true
 				expr := p.expr(0)
-				ecmnts << p.eat_comments()
+				ecmnts << p.eat_comments({})
 				p.inside_match_case = false
 				if p.tok.kind == .dotdot {
 					p.error_with_pos('match only supports inclusive (`...`) ranges, not exclusive (`..`)',
@@ -238,7 +238,7 @@ fn (mut p Parser) match_expr() ast.MatchExpr {
 		p.close_scope()
 		p.inside_match_body = false
 		pos := branch_first_pos.extend_with_last_line(branch_last_pos, p.prev_tok.line_nr)
-		post_comments := p.eat_comments()
+		post_comments := p.eat_comments({})
 		branches << ast.MatchBranch{
 			exprs: exprs
 			ecmnts: ecmnts
@@ -405,7 +405,7 @@ fn (mut p Parser) select_expr() ast.SelectExpr {
 			pos: branch_first_pos.pos
 			len: branch_last_pos.pos - branch_first_pos.pos + branch_last_pos.len
 		}
-		post_comments := p.eat_comments()
+		post_comments := p.eat_comments({})
 		pos.update_last_line(p.prev_tok.line_nr)
 		if post_comments.len > 0 {
 			pos.last_line = post_comments.last().pos.last_line

vlib/v/parser/parser.v

@@ -598,25 +598,23 @@ pub fn (mut p Parser) comment_stmt() ast.ExprStmt {
 	}
 }
 
-pub fn (mut p Parser) eat_comments() []ast.Comment {
-	mut comments := []ast.Comment{}
-	for {
-		if p.tok.kind != .comment {
-			break
-		}
-		comments << p.comment()
-	}
-	return comments
+struct EatCommentsConfig {
+	same_line bool // Only eat comments on the same line as the previous token
+	follow_up bool // Comments directly below the previous token as long as there is no empty line
 }
 
-pub fn (mut p Parser) eat_line_end_comments() []ast.Comment {
-	line := p.prev_tok.line_nr
+pub fn (mut p Parser) eat_comments(cfg EatCommentsConfig) []ast.Comment {
+	mut line := p.prev_tok.line_nr
 	mut comments := []ast.Comment{}
 	for {
-		if p.tok.kind != .comment || p.tok.line_nr > line {
+		if p.tok.kind != .comment || (cfg.same_line && p.tok.line_nr > line)
+			|| (cfg.follow_up && p.tok.line_nr > line + 1) {
 			break
 		}
 		comments << p.comment()
+		if cfg.follow_up {
+			line = p.prev_tok.line_nr
+		}
 	}
 	return comments
 }
@@ -1558,7 +1556,7 @@ fn (mut p Parser) dot_expr(left ast.Expr) ast.Expr {
 		//
 		end_pos := p.prev_tok.position()
 		pos := name_pos.extend(end_pos)
-		comments := p.eat_line_end_comments()
+		comments := p.eat_comments(same_line: true)
 		mcall_expr := ast.CallExpr{
 			left: left
 			name: field_name
@@ -1919,7 +1917,7 @@ fn (mut p Parser) import_stmt() ast.Import {
 			return import_node
 		}
 	}
-	import_node.comments = p.eat_line_end_comments()
+	import_node.comments = p.eat_comments(same_line: true)
 	p.imports[mod_alias] = mod_name
 	// if mod_name !in p.table.imports {
 	p.table.imports << mod_name
@@ -1992,7 +1990,7 @@ fn (mut p Parser) const_decl() ast.ConstDecl {
 			p.error_with_pos('const declaration is missing closing `)`', const_pos)
 			return ast.ConstDecl{}
 		}
-		comments = p.eat_comments()
+		comments = p.eat_comments({})
 		if p.tok.kind == .rpar {
 			break
 		}
@@ -2043,7 +2041,7 @@ fn (mut p Parser) return_stmt() ast.Return {
 	first_pos := p.tok.position()
 	p.next()
 	// no return
-	mut comments := p.eat_comments()
+	mut comments := p.eat_comments({})
 	if p.tok.kind == .rcbr {
 		return ast.Return{
 			comments: comments
@@ -2085,7 +2083,7 @@ fn (mut p Parser) global_decl() ast.GlobalDecl {
 	mut fields := []ast.GlobalField{}
 	mut comments := []ast.Comment{}
 	for {
-		comments = p.eat_comments()
+		comments = p.eat_comments({})
 		if p.tok.kind == .rpar {
 			break
 		}
@@ -2147,7 +2145,7 @@ fn (mut p Parser) enum_decl() ast.EnumDecl {
 	}
 	name := p.prepend_mod(enum_name)
 	p.check(.lcbr)
-	enum_decl_comments := p.eat_comments()
+	enum_decl_comments := p.eat_comments({})
 	mut vals := []string{}
 	// mut default_exprs := []ast.Expr{}
 	mut fields := []ast.EnumField{}
@@ -2168,8 +2166,8 @@ fn (mut p Parser) enum_decl() ast.EnumDecl {
 			pos: pos
 			expr: expr
 			has_expr: has_expr
-			comments: p.eat_line_end_comments()
-			next_comments: p.eat_comments()
+			comments: p.eat_comments(same_line: true)
+			next_comments: p.eat_comments({})
 		}
 	}
 	p.top_level_statement_end()
@@ -2248,7 +2246,7 @@ fn (mut p Parser) type_decl() ast.TypeDecl {
 		// function type: `type mycallback = fn(string, int)`
 		fn_name := p.prepend_mod(name)
 		fn_type := p.parse_fn_type(fn_name)
-		comments = p.eat_line_end_comments()
+		comments = p.eat_comments(same_line: true)
 		return ast.FnTypeDecl{
 			name: fn_name
 			is_pub: is_pub
@@ -2296,7 +2294,7 @@ fn (mut p Parser) type_decl() ast.TypeDecl {
 			}
 			is_public: is_pub
 		})
-		comments = p.eat_line_end_comments()
+		comments = p.eat_comments(same_line: true)
 		return ast.SumTypeDecl{
 			name: name
 			is_pub: is_pub
@@ -2332,7 +2330,7 @@ fn (mut p Parser) type_decl() ast.TypeDecl {
 		p.error_with_pos('a type alias can not refer to itself: $name', decl_pos.extend(type_alias_pos))
 		return ast.AliasTypeDecl{}
 	}
-	comments = p.eat_line_end_comments()
+	comments = p.eat_comments(same_line: true)
 	return ast.AliasTypeDecl{
 		name: name
 		is_pub: is_pub

vlib/v/parser/pratt.v

@@ -18,7 +18,7 @@ pub fn (mut p Parser) expr(precedence int) ast.Expr {
 	is_stmt_ident := p.is_stmt_ident
 	p.is_stmt_ident = false
 	if !p.pref.is_fmt {
-		p.eat_comments()
+		p.eat_comments({})
 	}
 	// Prefix
 	match p.tok.kind {

vlib/v/parser/struct.v

@@ -234,7 +234,7 @@ fn (mut p Parser) struct_decl() ast.StructDecl {
 				field_pos = field_start_pos.extend(type_pos)
 			}
 			// Comments after type (same line)
-			comments << p.eat_comments()
+			comments << p.eat_comments({})
 			if p.tok.kind == .lsbr {
 				// attrs are stored in `p.attrs`
 				p.attributes()
@@ -252,7 +252,7 @@ fn (mut p Parser) struct_decl() ast.StructDecl {
 						else {}
 					}
 					has_default_expr = true
-					comments << p.eat_comments()
+					comments << p.eat_comments({})
 				}
 				// TODO merge table and ast Fields?
 				ast_fields << ast.StructField{
@@ -343,7 +343,7 @@ fn (mut p Parser) struct_init(short_syntax bool) ast.StructInit {
 	if !short_syntax {
 		p.check(.lcbr)
 	}
-	pre_comments := p.eat_comments()
+	pre_comments := p.eat_comments({})
 	mut fields := []ast.StructInitField{}
 	mut i := 0
 	no_keys := p.peek_tok.kind != .colon && p.tok.kind != .rcbr && p.tok.kind != .ellipsis // `Vec{a,b,c}
@@ -364,19 +364,19 @@ fn (mut p Parser) struct_init(short_syntax bool) ast.StructInit {
 			// name will be set later in checker
 			expr = p.expr(0)
 			field_pos = expr.position()
-			comments = p.eat_line_end_comments()
+			comments = p.eat_comments(same_line: true)
 		} else if is_update_expr {
 			// struct updating syntax; f2 := Foo{ ...f, name: 'f2' }
 			p.check(.ellipsis)
 			update_expr = p.expr(0)
-			update_expr_comments << p.eat_line_end_comments()
+			update_expr_comments << p.eat_comments(same_line: true)
 			has_update_expr = true
 		} else {
 			first_field_pos := p.tok.position()
 			field_name = p.check_name()
 			p.check(.colon)
 			expr = p.expr(0)
-			comments = p.eat_line_end_comments()
+			comments = p.eat_comments(same_line: true)
 			last_field_pos := expr.position()
 			field_len := if last_field_pos.len > 0 {
 				last_field_pos.pos - first_field_pos.pos + last_field_pos.len
@@ -393,8 +393,8 @@ fn (mut p Parser) struct_init(short_syntax bool) ast.StructInit {
 		if p.tok.kind == .comma {
 			p.next()
 		}
-		comments << p.eat_line_end_comments()
-		nline_comments << p.eat_comments()
+		comments << p.eat_comments(same_line: true)
+		nline_comments << p.eat_comments({})
 		if !is_update_expr {
 			fields << ast.StructInitField{
 				name: field_name
@@ -435,7 +435,7 @@ fn (mut p Parser) interface_decl() ast.InterfaceDecl {
 	interface_name := p.prepend_mod(p.check_name()).clone()
 	// println('interface decl $interface_name')
 	p.check(.lcbr)
-	pre_comments := p.eat_comments()
+	pre_comments := p.eat_comments({})
 	// Declare the type
 	reg_idx := p.table.register_type_symbol(
 		is_public: is_pub
@@ -506,8 +506,8 @@ fn (mut p Parser) interface_decl() ast.InterfaceDecl {
 			if p.tok.kind.is_start_of_type() && p.tok.line_nr == line_nr {
 				method.return_type = p.parse_type()
 			}
-			mcomments := p.eat_line_end_comments()
-			mnext_comments := p.eat_comments()
+			mcomments := p.eat_comments(same_line: true)
+			mnext_comments := p.eat_comments({})
 			method.comments = mcomments
 			method.next_comments = mnext_comments
 			methods << method