Skip to content

Commit

Permalink
XXX Don't capture contents of comment tokens.
Browse files Browse the repository at this point in the history
  • Loading branch information
@jasone
jasone committed Sep 17, 2020
1 parent b58e8c6 commit 2733236
Show file tree
Hide file tree
Showing 2 changed files with 34 additions and 46 deletions.
76 changes: 32 additions & 44 deletions bootstrap/src/hlc/scan.ml
View file
Expand Up @@ -92,8 +92,8 @@ module Token = struct
| LineDelim
| Dedent
| Whitespace
| HashComment of string
| ParenComment of string
| HashComment
| ParenComment
| Uident of string
| EndOfInput
| IndentError
Expand Down Expand Up @@ -172,8 +172,8 @@ module Token = struct
| LineDelim -> "<LineDelim>"
| Dedent -> "<Dedent>"
| Whitespace -> "<Whitespace>"
| HashComment comment -> "<HashComment=\"" ^ comment ^ "\">"
| ParenComment comment -> "<ParenComment=\"" ^ comment ^ "\">"
| HashComment -> "<HashComment>"
| ParenComment -> "<ParenComment>"
| Uident uident -> "<Uident=\"" ^ uident ^ "\">"
| EndOfInput -> "<EndOfInput>"
| IndentError -> "<IndentError>"
Expand Down Expand Up @@ -320,8 +320,7 @@ let whitespace _pcursor cursor t =

let hash_comment _pcursor cursor t =
let accept_hash_comment cursor t = begin
let comment_str = str_of_cursor cursor t in
accept_delim (Token.Variant.HashComment comment_str) cursor t
accept_delim Token.Variant.HashComment cursor t
end in
let rec fn cursor t = begin
match Text.Cursor.next_opt cursor with
Expand All @@ -336,8 +335,7 @@ let hash_comment _pcursor cursor t =

let paren_comment _pcursor cursor t =
let accept_paren_comment cursor t = begin
let comment_str = str_of_cursor cursor t in
accept (Token.Variant.ParenComment comment_str) cursor t
accept Token.Variant.ParenComment cursor t
end in
let rec fn nesting cursor t = begin
match Text.Cursor.next_opt cursor with
Expand Down Expand Up @@ -575,9 +573,9 @@ let dentation cursor t =
let t', token = start t in
match token.variant, t'.line_state with
| EndOfInput, _
| HashComment _, _
| HashComment, _
| Whitespace, LineDelim -> LineNoop
| ParenComment _, _
| ParenComment, _
| Whitespace, LineDentation -> fn t'
| Whitespace, LineBody -> not_reached ()
| _ -> LineExpr
Expand Down Expand Up @@ -665,36 +663,34 @@ let%expect_test "comment" =

[%expect{xxx|
{|#|}
1:0..1:1 : <HashComment="#">
1:0..1:1 : <HashComment>
1:1..1:1 : <LineDelim>
1:1..1:1 : <EndOfInput>
{|#...|}
1:0..1:4 : <HashComment="#...">
1:0..1:4 : <HashComment>
1:4..1:4 : <LineDelim>
1:4..1:4 : <EndOfInput>
{|#...
#...|}
1:0..2:0 : <HashComment="#...
">
2:0..2:4 : <HashComment="#...">
1:0..2:0 : <HashComment>
2:0..2:4 : <HashComment>
2:4..2:4 : <LineDelim>
2:4..2:4 : <EndOfInput>
{|#...
#...|}
1:0..2:0 : <HashComment="#...
">
1:0..2:0 : <HashComment>
2:0..2:4 : <Indent>
2:4..2:8 : <HashComment="#...">
2:4..2:8 : <HashComment>
2:8..2:8 : <Dedent>
2:8..2:8 : <EndOfInput>
{|(**)|}
1:0..1:4 : <ParenComment="(**)">
1:0..1:4 : <ParenComment>
1:4..1:4 : <EndOfInput>
{|(***)|}
1:0..1:5 : <ParenComment="(***)">
1:0..1:5 : <ParenComment>
1:5..1:5 : <EndOfInput>
{|(*(*(#*#)*)*)|}
1:0..1:13 : <ParenComment="(*(*(#*#)*)*)">
1:0..1:13 : <ParenComment>
1:13..1:13 : <EndOfInput>
{|(**|}
1:0..1:3 : <Error>
Expand Down Expand Up @@ -1029,19 +1025,16 @@ g
f
g
|}
1:0..2:0 : <HashComment="# a
">
1:0..2:0 : <HashComment>
2:0..2:0 : <LineDelim>
2:0..2:1 : <Uident="b">
2:1..3:0 : <Whitespace>
3:0..3:4 : <Indent>
3:4..3:5 : <Uident="c">
3:5..4:0 : <Whitespace>
4:0..4:4 : <LineDelim>
4:4..5:0 : <HashComment="# d
">
5:0..6:0 : <HashComment="# e
">
4:4..5:0 : <HashComment>
5:0..6:0 : <HashComment>
6:0..6:4 : <LineDelim>
6:4..6:5 : <Uident="f">
6:5..7:0 : <Whitespace>
Expand Down Expand Up @@ -1071,7 +1064,7 @@ g
2:0..2:4 : <Indent>
2:4..2:5 : <Uident="b">
2:5..3:0 : <Whitespace>
3:0..3:13 : <ParenComment="(* Ignore. *)">
3:0..3:13 : <ParenComment>
3:13..4:0 : <Whitespace>
4:0..4:4 : <LineDelim>
4:4..4:5 : <Uident="c">
Expand All @@ -1088,12 +1081,11 @@ g
2:0..2:4 : <Indent>
2:4..2:5 : <Uident="b">
2:5..3:0 : <Whitespace>
3:0..3:13 : <ParenComment="(* Ignore. *)">
3:0..3:13 : <ParenComment>
3:13..3:14 : <Whitespace>
3:14..3:23 : <ParenComment="(* ... *)">
3:14..3:23 : <ParenComment>
3:23..3:24 : <Whitespace>
3:24..4:0 : <HashComment="# ...
">
3:24..4:0 : <HashComment>
4:0..4:4 : <LineDelim>
4:4..4:5 : <Uident="c">
4:5..5:0 : <Whitespace>
Expand All @@ -1111,13 +1103,11 @@ g
2:0..2:4 : <Indent>
2:4..2:5 : <Uident="b">
2:5..3:0 : <Whitespace>
3:0..3:13 : <ParenComment="(* Ignore. *)">
3:0..3:13 : <ParenComment>
3:13..3:14 : <Whitespace>
3:14..4:7 : <ParenComment="(*
... *)">
3:14..4:7 : <ParenComment>
4:7..5:0 : <Whitespace>
5:0..6:0 : <HashComment="# ...
">
5:0..6:0 : <HashComment>
6:0..6:4 : <LineDelim>
6:4..6:5 : <Uident="c">
6:5..7:0 : <Whitespace>
Expand All @@ -1134,7 +1124,7 @@ g
2:4..2:5 : <Uident="b">
2:5..3:0 : <Whitespace>
3:0..3:0 : <Dedent>
3:0..3:19 : <ParenComment="(* Don't ignore. *)">
3:0..3:19 : <ParenComment>
3:19..3:20 : <Whitespace>
3:20..3:24 : <True>
3:24..4:0 : <Whitespace>
Expand All @@ -1156,10 +1146,9 @@ g
2:4..2:5 : <Uident="b">
2:5..3:0 : <Whitespace>
3:0..3:0 : <Dedent>
3:0..3:19 : <ParenComment="(* Don't ignore. *)">
3:0..3:19 : <ParenComment>
3:19..3:20 : <Whitespace>
3:20..4:3 : <ParenComment="(* ...
*)">
3:20..4:3 : <ParenComment>
4:3..5:1 : <Whitespace>
5:1..5:5 : <True>
5:5..6:0 : <Whitespace>
Expand All @@ -1180,10 +1169,9 @@ g
2:4..2:5 : <Uident="b">
2:5..3:0 : <Whitespace>
3:0..3:0 : <Dedent>
3:0..3:19 : <ParenComment="(* Don't ignore. *)">
3:0..3:19 : <ParenComment>
3:19..3:20 : <Whitespace>
3:20..4:7 : <ParenComment="(*
... *)">
3:20..4:7 : <ParenComment>
4:7..4:8 : <Whitespace>
4:8..4:12 : <True>
4:12..5:0 : <Whitespace>
Expand Down
4 changes: 2 additions & 2 deletions bootstrap/src/hlc/scan.mli
View file
Expand Up @@ -82,8 +82,8 @@ module Token : sig
| LineDelim
| Dedent
| Whitespace
| HashComment of string
| ParenComment of string
| HashComment
| ParenComment
| Uident of string
| EndOfInput
| IndentError
Expand Down

0 comments on commit 2733236

Please sign in to comment.