scanner: change -d debugscanner to -d trace_scanner for uniformity with the other tracing options, described in CONTRIBUTING.md

Author: spytheman

CONTRIBUTING.md

@@ -273,17 +273,15 @@ a copy of the compiler rather than replacing it with `v self`.
 
 | Flag                              | Usage                                                                                                               |
 |-----------------------------------|---------------------------------------------------------------------------------------------------------------------|
-| `debugscanner`                    | Prints debug information during the scanning phase                                                                  |
 | `debug_codegen`                   | Prints automatically generated V code during the scanning phase                                                     |
 | `debug_interface_table`           | Prints generated interfaces during C generation                                                                     |
 | `debug_interface_type_implements` | Prints debug information when checking that a type implements in interface                                          |
 | `print_vweb_template_expansions`  | Prints vweb compiled HTML files                                                                                     |
 | `time_checking`                   | Prints the time spent checking files and other related information                                                  |
 | `time_parsing`                    | Prints the time spent parsing files and other related information                                                   |
 |                                   |                                                                                                                     |
+| `trace_scanner`                   | Prints details about the recognized tokens. *Very* verbose. Use `./vnew -no-builtin -check-syntax file.v` later.    |
 | `trace_parser`                    | Prints details about parsed statements and expressions. Very verbose. Use it for panics in the parser.              |
-| `trace_ccoptions`                 | Prints options passed down to the C compiler                                                                        |
-|                                   |                                                                                                                     |
 | `trace_checker`                   | Prints details about the statements being checked. Very verbose. Use it for panics in the checker.                  |
 |                                   |                                                                                                                     |
 | `trace_gen`                       | Prints all the strings written to the generated C file. Very verbose.                                               |
@@ -295,7 +293,10 @@ a copy of the compiler rather than replacing it with `v self`.
 | `trace_autofree`                  | Prints details about how/when -autofree puts free() calls                                                           |
 | `trace_autostr`                   | Prints details about `.str()` method auto-generated by the compiler during C generation                             |
 |                                   |                                                                                                                     |
+| `trace_ccoptions`                 | Prints options passed down to the C compiler                                                                        |
+|                                   |                                                                                                                     |
 | `trace_thirdparty_obj_files`      | Prints details about built thirdparty obj files                                                                     |
 | `trace_usecache`                  | Prints details when -usecache is used                                                                               |
 | `trace_embed_file`                | Prints details when $embed_file is used                                                                             |
 | `embed_only_metadata`             | Embed only the metadata for the file(s) with `$embed_file('somefile')`; faster; for development, *not* distribution |
+|-----------------------------------|---------------------------------------------------------------------------------------------------------------------|

vlib/v/parser/parser.v

@@ -4374,7 +4374,7 @@ fn (mut p Parser) top_level_statement_start() {
 	if p.comments_mode == .toplevel_comments {
 		p.scanner.set_is_inside_toplevel_statement(true)
 		p.rewind_scanner_to_current_token_in_new_mode()
-		$if debugscanner ? {
+		$if trace_scanner ? {
 			eprintln('>> p.top_level_statement_start | tidx:${p.tok.tidx:-5} | p.tok.kind: ${p.tok.kind:-10} | p.tok.lit: ${p.tok.lit} ${p.peek_tok.lit} ${p.peek_token(2).lit} ${p.peek_token(3).lit} ...')
 		}
 	}
@@ -4384,7 +4384,7 @@ fn (mut p Parser) top_level_statement_end() {
 	if p.comments_mode == .toplevel_comments {
 		p.scanner.set_is_inside_toplevel_statement(false)
 		p.rewind_scanner_to_current_token_in_new_mode()
-		$if debugscanner ? {
+		$if trace_scanner ? {
 			eprintln('>> p.top_level_statement_end   | tidx:${p.tok.tidx:-5} | p.tok.kind: ${p.tok.kind:-10} | p.tok.lit: ${p.tok.lit} ${p.peek_tok.lit} ${p.peek_token(2).lit} ${p.peek_token(3).lit} ...')
 		}
 	}

vlib/v/scanner/scanner.v

@@ -589,9 +589,9 @@ fn (mut s Scanner) scan_all_tokens_in_buffer() {
 	}
 	s.scan_remaining_text()
 	s.tidx = 0
-	$if debugscanner ? {
+	$if trace_scanner ? {
 		for t in s.all_tokens {
-			eprintln('> tidx:${t.tidx:-5} | kind: ${t.kind:-10} | lit: ${t.lit}')
+			eprintln('> tidx:${t.tidx:-5} | kind: ${t.kind:-10} | lit.len: ${t.lit.len:-5} | lit: `${t.lit}`')
 		}
 	}
 }