Adding TypedMetavar for AST generic and AST python (part 1, pfff part)

This will help semgrep/semgrep#49

Test plan:
make test

h_program-lang/ast_generic.ml

@@ -262,7 +262,9 @@ and expr =
   | Ref   of tok (* &, address of *) * expr 
   | DeRef of tok (* '*' in C, '!' or '<-' in ML *) * expr 
 
+  (* sgrep: *)
   | Ellipsis of tok (* sgrep: ... in args, stmts, and also types in Python *)
+  | TypedMetavar of ident * tok (* : *) * type_
 
   | OtherExpr of other_expr_operator * any list
 
@@ -929,6 +931,10 @@ let expr_to_pattern e =
   (* TODO: diconstruct e and generate the right pattern (PatLiteral, ...) *)
   OtherPat (OP_ExprPattern, [E e])
 
+let expr_to_type e =
+  (* TODO: diconstruct e and generate the right type (TyBuiltin, ...) *)
+  OtherType (OT_Expr, [E e])
+
 (* see also Java_to_generic.entity_to_param *)
 (* see also python_to_generic.expr_to_attribute *)
 

lang_GENERIC/analyze/lrvalue.ml

@@ -202,7 +202,9 @@ let rec visit_expr hook lhs expr =
      -> error_todo (E expr)
 
   | Seq xs -> List.iter recr xs
-
+
+  (* we should not be called on a sgrep pattern *)
+  | TypedMetavar (_id, _, _t) -> raise Impossible
   | Ellipsis _tok -> ()
 
   | OtherExpr (_other_xxx, anys) -> List.iter (anyhook hook Rhs) anys

lang_GENERIC/parsing/map_ast.ml

@@ -170,6 +170,11 @@ and map_expr x =
       and v2 = map_expr v2
       and v3 = map_expr v3
       in Conditional ((v1, v2, v3))
+  | TypedMetavar ((v1, v2, v3)) ->
+      let v1 = map_ident v1
+      and v2 = map_tok v2
+      and v3 = map_type_ v3
+      in TypedMetavar ((v1, v2, v3))
   | MatchPattern ((v1, v2)) ->
       let v1 = map_expr v1
       and v2 = map_of_list map_action v2

lang_GENERIC/parsing/meta_ast.ml

@@ -162,7 +162,13 @@ and vof_expr =
   | DeRef (t, v1) -> 
       let t = vof_tok t in
       let v1 = vof_expr v1 in Ocaml.VSum (("DeRef", [ t; v1 ]))
-  | Ellipsis v1 -> let v1 = vof_tok v1 in Ocaml.VSum (("Ellipsis", [ v1 ]))
+  | Ellipsis v1 -> 
+      let v1 = vof_tok v1 in Ocaml.VSum (("Ellipsis", [ v1 ]))
+  | TypedMetavar ((v1, v2, v3)) ->
+      let v1 = vof_ident v1
+      and v2 = vof_tok v2
+      and v3 = vof_type_ v3
+      in Ocaml.VSum (("TypedMetavar", [ v1; v2; v3 ]))
   | OtherExpr ((v1, v2)) ->
       let v1 = vof_other_expr_operator v1
       and v2 = Ocaml.vof_list vof_any v2

lang_GENERIC/parsing/visitor_ast.ml

@@ -172,6 +172,8 @@ and v_expr x =
         ()
   | Conditional ((v1, v2, v3)) ->
       let v1 = v_expr v1 and v2 = v_expr v2 and v3 = v_expr v3 in ()
+  | TypedMetavar ((v1, v2, v3)) ->
+      let v1 = v_ident v1 and v2 = v_tok v2 and v3 = v_type_ v3 in ()
   | MatchPattern ((v1, v2)) ->
       let v1 = v_expr v1
       and v2 =

lang_python/analyze/python_to_generic.ml

@@ -103,6 +103,10 @@ let rec expr (x: expr) =
      let v1 = expr v1 in
      let v2 = type_ v2 in
      G.Cast (v2, v1)
+  | TypedMetavar (v1, v2, v3) ->
+     let v1 = name v1 in
+     let v3 = type_ v3 in
+     G.TypedMetavar (v1, v2, v3)
   | ExprStar v1 ->
     let v1 = expr v1 in
     G.Call (G.IdSpecial (G.Spread, fake "spread"), [G.expr_to_arg v1])
@@ -346,7 +350,7 @@ and parameters xs =
 
 and type_ v = 
   let v = expr v in
-  G.OtherType (G.OT_Expr, [G.E v])
+  G.expr_to_type v
 
 and type_parent v = 
   let v = argument v in

lang_python/parsing/ast_python.ml

@@ -133,7 +133,9 @@ type expr =
   (* python3: *)
   (* inside an Assign (or ExprStmt) *)
   | TypedExpr of expr * type_
+  (* sgrep-ext: *)
   | Ellipsis of tok (* should be only in .pyi, types Dict[str,...], or sgrep *)
+  | TypedMetavar of name * tok * type_
 
   | BoolOp of boolop wrap (* op *) * expr list (* values *)
   | BinOp of expr (* left *) * operator wrap (* op *) * expr (* right *)

lang_python/parsing/meta_ast_python.ml

@@ -65,6 +65,11 @@ let rec vof_expr =
       and v2 = vof_type_ v2
       in Ocaml.VSum (("TypedExpr", [ v1; v2 ]))
   | Ellipsis v1 -> let v1 = vof_tok v1 in Ocaml.VSum (("Ellipsis", [ v1 ]))
+  | TypedMetavar ((v1, v2, v3)) ->
+      let v1 = vof_name v1
+      and v2 = vof_tok v2
+      and v3 = vof_type_ v3
+      in Ocaml.VSum (("TypedMetavar", [ v1; v2; v3 ]))
   | BoolOp ((v1, v2)) ->
       let v1 = vof_wrap vof_boolop v1
       and v2 = Ocaml.vof_list vof_expr v2

lang_python/parsing/parser_python.mly

@@ -847,6 +847,11 @@ argument:
   | MULT test      { ArgStar $2 }
   | POW test       { ArgPow $2 }
 
+  /*(* sgrep-ext: difficult to move in atom without s/r conflict so restricted
+     * to argument for now *)*/
+  | NAME COLON test 
+    { Flag_parsing.sgrep_guard (Arg (TypedMetavar ($1, $2, $3))) }
+
   | test EQ test
       { match $1 with
         | Name (id, _, _) -> ArgKwd (id, $3)

lang_python/parsing/visitor_python.ml

@@ -106,6 +106,11 @@ and v_expr (x: expr) =
      let v1 = v_expr v1 in
      let v2 = v_type_ v2 in
      ()
+  | TypedMetavar ((v1, v2, v3)) ->
+     let v1 = v_name v1 in
+     let v2 = v_tok v2 in
+     let v3 = v_type_ v3 in
+     ()
   | ExprStar ((v1)) ->
       let v1 = v_expr v1 in ()
   | Tuple ((v1, v2)) ->