Remove the unused KEYWORD token

README.md

@@ -17,10 +17,9 @@ This lexer has the following particularities:
   Blanks, comments, newlines, lexical conventions for writing
   literals, all of it is kept in the returned token stream.
   Undesired information can easily be thrown out of the stream.
-* Keyword independent: while there is a token for keywords, it
-  is not generated by this lexer. This is up to you to apply
-  a keyword table to turn some LIDENTs and some SYMBOLs into
-  KEYWORDs.
+* Keyword independent: there is no token for keywords. This
+  is up to you to cast some LIDENTs and some SYMBOLs into
+  proper keyword tokens of your own token type.
 * Fault tolerant: errors take part of the token stream, allowing
   to write fault tolerant translations.
 * Flexible warnings: the lexer warn about some corner case of

camltoken.ml

@@ -25,7 +25,6 @@ type blanks = string
 type comment = string
 
 type caml_token =
-  | KEYWORD       of string
   | LIDENT        of string
   | UIDENT        of string
   | SYMBOL        of string
@@ -206,7 +205,6 @@ let line_directive_width x =
   String.length x.l_blanks1 + x.l_zeros + int_width x.l_linenum + String.length x.l_blanks2 + fn_w + String.length x.l_comment + nl + 1
 
 let string_of_token = function
-  | KEYWORD       s      |
     SYMBOL        s      |
     LIDENT        s      |
     UIDENT        s      |
@@ -241,7 +239,6 @@ the kind of code where the compiler could deduce the right code from
 the spec.
 *)
 let token_width = function
-  | KEYWORD       s      |
     SYMBOL        s      |
     LIDENT        s      |
     UIDENT        s      |
@@ -265,7 +262,6 @@ let token_width = function
   | LINE_DIRECTIVE ld -> line_directive_width ld
 
 let strings_of_token = function
-  | KEYWORD s        -> ("KEYWORD", [s])
   | SYMBOL s         -> ("SYMBOL", [s])
   | LIDENT s         -> ("LIDENT", [s])
   | UIDENT s         -> ("UIDENT", [s])
@@ -430,7 +426,6 @@ let mkNATIVEINT  s = try  NATIVEINT(cvt_nativeint_literal s, s)
 let mkFLOAT      s = try  FLOAT(float_of_string s, s)
                      with Failure _ -> literal_overflow s "float"
 
-let mkKEYWORD s = KEYWORD s
 (* TODO add some assertions? *)
 let mkLIDENT s = LIDENT s
 let mkUIDENT s = UIDENT s
@@ -544,7 +539,6 @@ let warning_of_strings = function
   | _ -> None
 
 let token_of_strings = function
-  | "KEYWORD", [s]           -> Some (KEYWORD s)
   | "SYMBOL", [s]            -> Some (SYMBOL s)
   | "LIDENT", [s]            -> Some (LIDENT s)
   | "UIDENT", [s]            -> Some (UIDENT s)

camltoken.mli

@@ -20,7 +20,6 @@
     both the parsed value and the original string representation. For instance
     for int literals, `00000042' will represented by INT (42, "00000042"). *)
 type caml_token = (*should we use private here?*)
-  | KEYWORD of string (** Commonly used for words like `let', `open'... *)
   | LIDENT of string  (** Lower-case identifiers like `foo', `bar42'... *)
   | UIDENT of string  (** Upper-case identifiers like `Foo', `Bar42'... *)
   | SYMBOL of string  (** Symbol-based identifiers like `*', `++/--'... *)
@@ -112,7 +111,6 @@ val mkINT32 : string -> caml_token
 val mkINT64 : string -> caml_token
 val mkNATIVEINT : string -> caml_token
 val mkFLOAT : string -> caml_token
-val mkKEYWORD : string -> caml_token
 val mkLIDENT : string -> caml_token
 val mkUIDENT : string -> caml_token
 val mkSYMBOL : string -> caml_token

example.ml

@@ -19,5 +19,3 @@ caml $antiquo << quot $antiquo$ quot >> antiquo$ caml
 
 (* This is not a quotation *)
 caml $<< antiquo $caml$ antiquo >>$ caml
-
-(* NOTE that KEYWORD is never generated by this lexer *)

tests/example-no-quot.t/example.ml

@@ -19,5 +19,3 @@ caml $antiquo << quot $antiquo$ quot >> antiquo$ caml
 
 (* This is not a quotation *)
 caml $<< antiquo $caml$ antiquo >>$ caml
-
-(* NOTE that KEYWORD is never generated by this lexer *)

tests/example-no-quot.t/stdout

@@ -211,6 +211,3 @@ SYMBOL ">>$"
 BLANKS " "
 LIDENT "caml"
 NEWLINE "\n"
-NEWLINE "\n"
-COMMENT "(* NOTE that KEYWORD is never generated by this lexer *)"
-NEWLINE "\n"

tests/example.t/example.ml

@@ -19,5 +19,3 @@ caml $antiquo << quot $antiquo$ quot >> antiquo$ caml
 
 (* This is not a quotation *)
 caml $<< antiquo $caml$ antiquo >>$ caml
-
-(* NOTE that KEYWORD is never generated by this lexer *)

tests/example.t/stdout

@@ -127,6 +127,3 @@ ANTIQUOT "" " antiquo >>"
 BLANKS " "
 LIDENT "caml"
 NEWLINE "\n"
-NEWLINE "\n"
-COMMENT "(* NOTE that KEYWORD is never generated by this lexer *)"
-NEWLINE "\n"