Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Remove the unused KEYWORD token

  • Loading branch information...
commit 26edd85adf9736d00a7a5b02156a38bd5ffd3722 1 parent 136ee2a
@np authored
View
7 README.md
@@ -17,10 +17,9 @@ This lexer has the following particularities:
Blanks, comments, newlines, lexical conventions for writing
literals, all of it is kept in the returned token stream.
Undesired information can easily be thrown out of the stream.
-* Keyword independent: while there is a token for keywords, it
- is not generated by this lexer. This is up to you to apply
- a keyword table to turn some LIDENTs and some SYMBOLs into
- KEYWORDs.
+* Keyword independent: there is no token for keywords. This
+ is up to you to cast some LIDENTs and some SYMBOLs into
+ proper keyword tokens of your own token type.
* Fault tolerant: errors take part of the token stream, allowing
to write fault tolerant translations.
* Flexible warnings: the lexer warn about some corner case of
View
6 camltoken.ml
@@ -25,7 +25,6 @@ type blanks = string
type comment = string
type caml_token =
- | KEYWORD of string
| LIDENT of string
| UIDENT of string
| SYMBOL of string
@@ -206,7 +205,6 @@ let line_directive_width x =
String.length x.l_blanks1 + x.l_zeros + int_width x.l_linenum + String.length x.l_blanks2 + fn_w + String.length x.l_comment + nl + 1
let string_of_token = function
- | KEYWORD s |
SYMBOL s |
LIDENT s |
UIDENT s |
@@ -241,7 +239,6 @@ the kind of code where the compiler could deduce the right code from
the spec.
*)
let token_width = function
- | KEYWORD s |
SYMBOL s |
LIDENT s |
UIDENT s |
@@ -265,7 +262,6 @@ let token_width = function
| LINE_DIRECTIVE ld -> line_directive_width ld
let strings_of_token = function
- | KEYWORD s -> ("KEYWORD", [s])
| SYMBOL s -> ("SYMBOL", [s])
| LIDENT s -> ("LIDENT", [s])
| UIDENT s -> ("UIDENT", [s])
@@ -430,7 +426,6 @@ let mkNATIVEINT s = try NATIVEINT(cvt_nativeint_literal s, s)
let mkFLOAT s = try FLOAT(float_of_string s, s)
with Failure _ -> literal_overflow s "float"
-let mkKEYWORD s = KEYWORD s
(* TODO add some assertions? *)
let mkLIDENT s = LIDENT s
let mkUIDENT s = UIDENT s
@@ -544,7 +539,6 @@ let warning_of_strings = function
| _ -> None
let token_of_strings = function
- | "KEYWORD", [s] -> Some (KEYWORD s)
| "SYMBOL", [s] -> Some (SYMBOL s)
| "LIDENT", [s] -> Some (LIDENT s)
| "UIDENT", [s] -> Some (UIDENT s)
View
2  camltoken.mli
@@ -20,7 +20,6 @@
both the parsed value and the original string representation. For instance
for int literals, `00000042' will represented by INT (42, "00000042"). *)
type caml_token = (*should we use private here?*)
- | KEYWORD of string (** Commonly used for words like `let', `open'... *)
| LIDENT of string (** Lower-case identifiers like `foo', `bar42'... *)
| UIDENT of string (** Upper-case identifiers like `Foo', `Bar42'... *)
| SYMBOL of string (** Symbol-based identifiers like `*', `++/--'... *)
@@ -112,7 +111,6 @@ val mkINT32 : string -> caml_token
val mkINT64 : string -> caml_token
val mkNATIVEINT : string -> caml_token
val mkFLOAT : string -> caml_token
-val mkKEYWORD : string -> caml_token
val mkLIDENT : string -> caml_token
val mkUIDENT : string -> caml_token
val mkSYMBOL : string -> caml_token
View
2  example.ml
@@ -19,5 +19,3 @@ caml $antiquo << quot $antiquo$ quot >> antiquo$ caml
(* This is not a quotation *)
caml $<< antiquo $caml$ antiquo >>$ caml
-
-(* NOTE that KEYWORD is never generated by this lexer *)
View
2  tests/example-no-quot.t/example.ml
@@ -19,5 +19,3 @@ caml $antiquo << quot $antiquo$ quot >> antiquo$ caml
(* This is not a quotation *)
caml $<< antiquo $caml$ antiquo >>$ caml
-
-(* NOTE that KEYWORD is never generated by this lexer *)
View
3  tests/example-no-quot.t/stdout
@@ -211,6 +211,3 @@ SYMBOL ">>$"
BLANKS " "
LIDENT "caml"
NEWLINE "\n"
-NEWLINE "\n"
-COMMENT "(* NOTE that KEYWORD is never generated by this lexer *)"
-NEWLINE "\n"
View
2  tests/example.t/example.ml
@@ -19,5 +19,3 @@ caml $antiquo << quot $antiquo$ quot >> antiquo$ caml
(* This is not a quotation *)
caml $<< antiquo $caml$ antiquo >>$ caml
-
-(* NOTE that KEYWORD is never generated by this lexer *)
View
3  tests/example.t/stdout
@@ -127,6 +127,3 @@ ANTIQUOT "" " antiquo >>"
BLANKS " "
LIDENT "caml"
NEWLINE "\n"
-NEWLINE "\n"
-COMMENT "(* NOTE that KEYWORD is never generated by this lexer *)"
-NEWLINE "\n"

0 comments on commit 26edd85

Please sign in to comment.
Something went wrong with that request. Please try again.