ocamllex: better support for union of character sets (#11166)

Changes

@@ -81,6 +81,10 @@ Working version
   This allows to see optimized bytecode with -dlambda.
   (Jacques Garrigue, review by Gabriel Scherer)
 
+- #11166: ocamllex: the union of two character sets "cset1 | cset2" can now be
+  used in any context where a character set is expected.
+  (Nicolás Ojeda Bär, Martin Jambon, review by Sébastien Hinderer)
+
 ### Manual and documentation:
 
 - #9430, #11291: Document the general desugaring rules for binding operators.

lex/parser.mly

@@ -41,8 +41,9 @@ let rec remove_as = function
   | Alternative (e1, e2) -> Alternative (remove_as e1, remove_as e2)
   | Repetition e -> Repetition (remove_as e)
 
-let as_cset = function
+let rec as_cset = function
   | Characters s -> s
+  | Alternative (e1, e2) -> Cset.union (as_cset e1) (as_cset e2)
   | _ -> raise Cset.Bad
 
 %}

manual/src/cmds/lexyacc.etex

@@ -203,7 +203,10 @@ strings that match @regexp@.
 (option) Match the empty string, or a string matching @regexp@.
 
 \item[@regexp_1 '|' regexp_2@]
-(alternative) Match any string that matches @regexp_1@ or @regexp_2@
+(alternative) Match any string that matches @regexp_1@ or @regexp_2@.
+If both @regexp_1@ and @regexp_2@ are character sets, this constructions
+produces another character set, obtained by taking the union of @regexp_1@ and
+@regexp_2@.
 
 \item[@regexp_1 regexp_2@]
 (concatenation) Match the concatenation of two strings, the first

testsuite/tests/tool-lexyacc/csets.mll

@@ -0,0 +1,22 @@
+(* TEST
+  ocamllex_flags = " -q "
+*)
+
+let digit = ['0'-'9']
+let alpha = ['a'-'z']
+let alpha' = (digit | alpha) # digit
+
+rule read = parse
+| alpha'+ as lxm { Some lxm }
+| digit+ as lxm { Some lxm }
+| eof { None }
+
+{
+let () =
+  let rec aux lexbuf =
+    match read lexbuf with
+    | Some x -> x :: aux lexbuf
+    | None -> []
+  in
+  List.iter print_endline (aux (Lexing.from_string "abc0345ghz"))
+}

testsuite/tests/tool-lexyacc/csets.reference

@@ -0,0 +1,3 @@
+abc
+0345
+ghz