scanner: preallocate space for tokens to reduce the number of allocations in the hot loop

spytheman · spytheman · commit 99e71d08686f · 2021-10-28T16:23:24.000+03:00
diff --git a/vlib/v/scanner/scanner.v b/vlib/v/scanner/scanner.v
@@ -560,6 +560,8 @@ pub fn (mut s Scanner) scan_all_tokens_in_buffer(mode CommentsMode) {
 	}
 	oldmode := s.comments_mode
 	s.comments_mode = mode
+	// preallocate space for tokens
+	s.all_tokens = []token.Token{cap: s.text.len / 3}
 	s.scan_remaining_text()
 	s.comments_mode = oldmode
 	s.tidx = 0

Original file line number	Diff line number	Diff line change
`@@ -560,6 +560,8 @@ pub fn (mut s Scanner) scan_all_tokens_in_buffer(mode CommentsMode) {`
`560`	`560`	`}`
`561`	`561`	`oldmode := s.comments_mode`
`562`	`562`	`s.comments_mode = mode`
	`563`	`+ // preallocate space for tokens`
	`564`	`+ s.all_tokens = []token.Token{cap: s.text.len / 3}`
`563`	`565`	`s.scan_remaining_text()`
`564`	`566`	`s.comments_mode = oldmode`
`565`	`567`	`s.tidx = 0`