Simpler lexer

bin/test/errors/parsing-errors.t

@@ -5,15 +5,14 @@ Delimiter problems:
   $ echo "{{foo" > $PROBLEM
   $ mustache foo.json $PROBLEM
   Template parse error:
-  File "no-closing-mustache.mustache", line 2, character 0: syntax error.
+  File "no-closing-mustache.mustache", line 2, character 0: '}}' expected.
   [3]
 
   $ PROBLEM=one-closing-mustache.mustache
   $ echo "{{foo}" > $PROBLEM
   $ mustache foo.json $PROBLEM
   Template parse error:
-  File "one-closing-mustache.mustache", lines 1-2, characters 6-0:
-  syntax error.
+  File "one-closing-mustache.mustache", line 1, character 5: '}}' expected.
   [3]
 
   $ PROBLEM=eof-before-variable.mustache
@@ -80,14 +79,14 @@ Mismatches between opening and closing mustaches:
   $ echo "{{ foo }}}" > $PROBLEM
   $ mustache foo.json $PROBLEM
   Template parse error:
-  File "two-three.mustache", lines 1-2, characters 10-0: syntax error.
+  File "two-three.mustache", line 1, characters 7-10: '}}' expected.
   [3]
 
   $ PROBLEM=three-two.mustache
   $ echo "{{{ foo }}" > $PROBLEM
   $ mustache foo.json $PROBLEM
   Template parse error:
-  File "three-two.mustache", lines 1-2, characters 10-0: syntax error.
+  File "three-two.mustache", line 1, characters 8-10: '}}}' expected.
   [3]
 
 
@@ -115,3 +114,13 @@ Mismatch between section-start and section-end:
   File "wrong-nesting.mustache", lines 1-2, characters 41-0:
   Section mismatch: {{#foo}} is closed by {{/bar}}.
   [3]
+
+
+Weird cases that may confuse our lexer or parser:
+
+  $ PROBLEM=weird-tag-name.mustache
+  $ echo "{{.weird}} foo bar" > $PROBLEM
+  $ mustache foo.json $PROBLEM
+  Template parse error:
+  File "weird-tag-name.mustache", line 1, character 3: '}}' expected.
+  [3]

lib/mustache.ml

@@ -413,7 +413,7 @@ module Render = struct
     in
 
     let print_indented_string indent s =
-      let lines = Mustache_lexer.split_on_char '\n' s in
+      let lines = String.split_on_char '\n' s in
       align indent; Buffer.add_string buf (List.hd lines);
       List.iter (fun line ->
         Buffer.add_char buf '\n';

lib/mustache_lexer.mll

@@ -25,35 +25,28 @@
 
   exception Error of string
 
-  let tok_arg f lexbuf =
+  let tok_arg lexbuf f =
     let start_p = lexbuf.Lexing.lex_start_p in
     let x = f lexbuf in
     lexbuf.Lexing.lex_start_p <- start_p;
     x
 
-  let with_space space f =
-    tok_arg (fun lexbuf ->
+  let lex_tag lexbuf space ident tag_end =
+    tok_arg lexbuf (fun lexbuf ->
       let () = space lexbuf in
-      let x = f lexbuf in
+      let name = ident lexbuf in
       let () = space lexbuf in
-      x
+      let () = tag_end lexbuf in
+      name
     )
 
-  let split_on_char sep s =
-    let open String in
-    let r = ref [] in
-    let j = ref (length s) in
-    for i = length s - 1 downto 0 do
-      if unsafe_get s i = sep then begin
-        r := sub s (i + 1) (!j - i - 1) :: !r;
-        j := i
-      end
-    done;
-    sub s 0 !j :: !r
-
   let split_ident ident =
     if ident = "." then []
-    else split_on_char '.' ident
+    else String.split_on_char '.' ident
+
+  let check_mustaches ~expected ~lexed =
+    if expected <> lexed then
+      raise (Error (Printf.sprintf "'%s' expected" expected))
 }
 
 let blank = [' ' '\t']*
@@ -66,13 +59,12 @@ rule space = parse
   | blank newline { new_line lexbuf; space lexbuf }
   | blank { () }
 
-and id = parse
-  | id  { lexeme lexbuf }
-  | eof { raise (Error "id expected") }
-
 and ident = parse
   | ident { lexeme lexbuf }
-  | eof   { raise (Error "ident expected") }
+  | ""    { raise (Error "ident expected") }
+
+and end_on expected = parse
+  | ("}}" | "}}}" | "") as lexed { check_mustaches ~expected ~lexed }
 
 and comment acc = parse
   | "}}"        { String.concat "" (List.rev acc) }
@@ -82,16 +74,14 @@ and comment acc = parse
   | eof         { raise (Error "non-terminated comment") }
 
 and mustache = parse
-  | "{{{"        { UNESCAPE_START (with_space space ident lexbuf |> split_ident) }
-  | "{{&"        { UNESCAPE_START_AMPERSAND (with_space space ident lexbuf |> split_ident) }
-  | "{{#"        { SECTION_START (with_space space ident lexbuf |> split_ident) }
-  | "{{^"        { SECTION_INVERT_START (with_space space ident lexbuf |> split_ident) }
-  | "{{/"        { SECTION_END (with_space space ident lexbuf |> split_ident) }
-  | "{{>"        { PARTIAL_START (0, with_space space ident lexbuf) }
-  | "{{!"        { COMMENT (tok_arg (comment []) lexbuf) }
-  | "{{"         { ESCAPE_START (with_space space ident lexbuf |> split_ident) }
-  | "}}}"        { UNESCAPE_END }
-  | "}}"         { END }
+  | "{{"         { ESCAPE (lex_tag lexbuf space ident (end_on "}}") |> split_ident) }
+  | "{{{"        { UNESCAPE (lex_tag lexbuf space ident (end_on "}}}") |> split_ident) }
+  | "{{&"        { UNESCAPE (lex_tag lexbuf space ident (end_on "}}") |> split_ident) }
+  | "{{#"        { OPEN_SECTION (lex_tag lexbuf space ident (end_on "}}") |> split_ident) }
+  | "{{^"        { OPEN_INVERTED_SECTION (lex_tag lexbuf space ident (end_on "}}") |> split_ident) }
+  | "{{/"        { CLOSE_SECTION (lex_tag lexbuf space ident (end_on "}}") |> split_ident) }
+  | "{{>"        { PARTIAL (0, lex_tag lexbuf space ident (end_on "}}")) }
+  | "{{!"        { COMMENT (tok_arg lexbuf (comment [])) }
   | raw newline  { new_line lexbuf; RAW (lexeme lexbuf) }
   | raw          { RAW (lexeme lexbuf) }
   | ['{' '}']    { RAW (lexeme lexbuf) }
@@ -136,33 +126,24 @@ and mustache = parse
        in
        loop 0 l
      in
-     let segment_before tail l =
-       let rec loop acc = function
-         | [] -> List.rev acc
-         | l when l == tail -> List.rev acc
-         | y :: ys -> loop (y :: acc) ys
-       in
-       loop [] l
-     in
      let is_standalone toks =
        let (skipped, toks) = skip_blanks toks in
        match toks with
-       | (SECTION_START _, _, _) :: (END, _, _) :: toks'
-       | (SECTION_INVERT_START _, _, _) :: (END, _, _) :: toks'
-       | (SECTION_END _, _, _) :: (END, _, _) :: toks'
-       | (PARTIAL_START _, _, _) :: (END, _, _) :: toks'
-       | (COMMENT _, _, _) :: toks' ->
+       | ((OPEN_SECTION _
+          | OPEN_INVERTED_SECTION _
+          | CLOSE_SECTION _
+          | PARTIAL _
+          | COMMENT _), _, _) as tok :: toks' ->
          let (_, toks_rest) = skip_blanks toks' in
          begin match toks_rest with
          | [] | [(EOF, _, _)] ->
-           let toks_standalone =
-             segment_before toks' toks |>
-             function
-             | [(PARTIAL_START (_, p), loc1, loc2); tok_end] ->
-               [(PARTIAL_START (skipped, p), loc1, loc2); tok_end]
-             | toks -> toks
+           let tok =
+             match tok with
+             | (PARTIAL (_, p), loc1, loc2) ->
+               (PARTIAL (skipped, p), loc1, loc2)
+             | _ -> tok
            in
-           Some (toks_standalone, toks_rest)
+           Some (tok, toks_rest)
          | _ -> None
          end
        | _ -> None
@@ -176,9 +157,9 @@ and mustache = parse
        | [] ->
          let toks = slurp_line () in
          match is_standalone toks with
-         | Some (toks_standalone, toks_rest) ->
-           buffer := List.tl toks_standalone @ toks_rest;
-           List.hd toks_standalone
+         | Some (tok_standalone, toks_rest) ->
+           buffer := toks_rest;
+           tok_standalone
          | None ->
            buffer := List.tl toks; List.hd toks
 }

lib/mustache_parser.mly

@@ -37,16 +37,13 @@
 %}
 
 %token EOF
-%token END
-%token <string list> ESCAPE_START
-%token <string list> UNESCAPE_START_AMPERSAND
-%token <string list> SECTION_INVERT_START
-%token <string list> SECTION_START
-%token <string list> SECTION_END
-%token <int * string> PARTIAL_START
-%token <string list> UNESCAPE_START
+%token <string list> ESCAPE
+%token <string list> UNESCAPE
+%token <string list> OPEN_INVERTED_SECTION
+%token <string list> OPEN_SECTION
+%token <string list> CLOSE_SECTION
+%token <int * string> PARTIAL
 %token <string> COMMENT
-%token UNESCAPE_END
 
 %token <string> RAW
 
@@ -56,24 +53,23 @@
 %%
 
 section:
-  | ss = SECTION_INVERT_START END
+  | ss = OPEN_INVERTED_SECTION
     e = mustache_expr
-    se = SECTION_END END {
+    se = CLOSE_SECTION {
     with_loc $sloc
       (Inverted_section (parse_section ss se e))
   }
-  | ss = SECTION_START END
+  | ss = OPEN_SECTION
     e = mustache_expr
-    se = SECTION_END END {
+    se = CLOSE_SECTION {
     with_loc $sloc
       (Section (parse_section ss se e))
   }
 
 mustache_element:
-  | elt = UNESCAPE_START UNESCAPE_END { with_loc $sloc (Unescaped elt) }
-  | elt = UNESCAPE_START_AMPERSAND END { with_loc $sloc (Unescaped elt) }
-  | elt = ESCAPE_START END { with_loc $sloc (Escaped elt) }
-  | elt = PARTIAL_START END {
+  | elt = UNESCAPE { with_loc $sloc (Unescaped elt) }
+  | elt = ESCAPE { with_loc $sloc (Escaped elt) }
+  | elt = PARTIAL {
       with_loc $sloc
         (Partial { indent = fst elt;
                    name = snd elt;