Lexer does not crash in case of unrecognized characters.

We added a special ERROR token to keep tokenizing the input even when
unrecognized characters exist.

Deve/README.md

@@ -43,4 +43,8 @@ Sample run:
 - : token list =
 [LET; NAME "x"; EQUALS; INT 5; IN; IF; NAME "x"; STAR; NAME "dummy"; THEN;
  INT 321; ELSE; EOF]
+# allTokens "x % abc Upper ??@ hey";;
+- : token list =
+[NAME "x"; ERROR '%'; NAME "abc"; ERROR 'U'; NAME "pper"; ERROR '?';
+ ERROR '?'; ERROR '@'; NAME "hey"; EOF]
 ```

Deve/lexer.ml

@@ -10,6 +10,7 @@ type token = INT of int
            | PLUS | STAR | MINUS | SLASH
            | LET | EQUALS | IN
            | IF | THEN | ELSE
+           | ERROR of char
            | EOF
 ;;
 
@@ -50,6 +51,7 @@ let rec tokenize chars =
      tokenizeInt rest (digitToInt c)
   | c::rest when isLowercaseLetter(c) ->
      tokenizeName rest (charToString c)
+  | c::rest -> (ERROR c)::(tokenize rest)
 
 and tokenizeInt chars n =
   match chars with