Skip to content

Commit

Permalink
Ignore missing nodes in FFP JSON tests
Browse files Browse the repository at this point in the history 
Summary:
allow-large-files So this is a diff that deletes over 60000 lines and reduces the size of the codebase needed; not really sure if there's any way to get around it and the files are already there.

This diff adds an option --ignore-missing-json that gets rid of missing nodes in the JSON output of hh_parse. It is only used for FFP Json tests, where missing nodes really don't help for what we'd want to test for and add a lot of extra noise when a new feature is being built.

{F359823399}

The only file worth reviewing is generate_full_fidelity_data.ml (and hh_parse.ml), everything else is generated.

Now, when we make a change that adds a new feature to the FFP, the tests act as a regression test to make sure old parsing/tests did not change with the exception of missing nodes.

Reviewed By: oulgen, Wilfred

Differential Revision: D25916261

fbshipit-source-id: 31679f25a53018ae98d467060c9225ca824825c1
  • Loading branch information
@jamesjwu @facebook-github-bot
jamesjwu authored and facebook-github-bot committed Jan 15, 2021
1 parent c3fcbba commit f848b33
Show file tree
Hide file tree
Showing 322 changed files with 3,351 additions and 18,794 deletions.
18 changes: 13 additions & 5 deletions hphp/hack/src/generate_full_fidelity_data.ml
View file
Expand Up @@ -1110,7 +1110,7 @@ SYNTAX
ParserOptions.ffi_t ->
Full_fidelity_syntax_error.t list
val has_leading_trivia : TriviaKind.t -> Token.t -> bool
val to_json : ?with_value:bool -> t -> Hh_json.json
val to_json : ?with_value:bool -> ?ignore_missing:bool -> t -> Hh_json.json
val extract_text : t -> string option
val is_in_body : t -> int -> bool
val syntax_node_to_list : t -> t list
Expand Down Expand Up @@ -2407,25 +2407,33 @@ CHILDREN
| SyntaxList _ -> []
CHILDREN_NAMES
let rec to_json ?(with_value = false) node =
let rec to_json_ ?(with_value = false) ?(ignore_missing = false) node =
let open Hh_json in
let ch = match node.syntax with
| Token t -> [ \"token\", Token.to_json t ]
| SyntaxList x -> [ (\"elements\",
JSON_Array (List.map ~f:(to_json ~with_value) x)) ]
JSON_Array (List.filter_map ~f:(to_json_ ~with_value ~ignore_missing) x)) ]
| _ ->
let rec aux acc c n =
match c, n with
| ([], []) -> acc
| ((hc :: tc), (hn :: tn)) ->
aux ((hn, (to_json ~with_value) hc) :: acc) tc tn
let result = (to_json_ ~with_value ~ignore_missing) hc in
(match result with
| Some r -> aux ((hn, r):: acc) tc tn
| None -> aux acc tc tn)
| _ -> failwith \"mismatch between children and names\" in
List.rev (aux [] (children node) (children_names node)) in
let k = (\"kind\", JSON_String (SyntaxKind.to_string (kind node))) in
let v = if with_value then
(\"value\", SyntaxValue.to_json node.value) :: ch
else ch in
JSON_Object (k :: v)
if ignore_missing && (List.is_empty ch) then None else Some(JSON_Object (k :: v))
let to_json ?(with_value = false) ?(ignore_missing = false) node =
match to_json_ ~with_value ~ignore_missing node with
| Some x -> x
| None -> Hh_json.JSON_Object([])
let binary_operator_kind b =
match syntax b with
Expand Down
14 changes: 12 additions & 2 deletions hphp/hack/src/hh_parse.ml
View file
Expand Up @@ -75,6 +75,7 @@ module FullFidelityParseArgs = struct
disallow_hash_comments: bool;
disallow_fun_and_cls_meth_pseudo_funcs: bool;
enable_coeffects: bool;
ignore_missing_json: bool;
}

let make
Expand Down Expand Up @@ -115,7 +116,8 @@ module FullFidelityParseArgs = struct
enable_xhp_class_modifier
disallow_hash_comments
disallow_fun_and_cls_meth_pseudo_funcs
enable_coeffects =
enable_coeffects
ignore_missing_json =
{
full_fidelity_json;
full_fidelity_dot;
Expand Down Expand Up @@ -155,6 +157,7 @@ module FullFidelityParseArgs = struct
disallow_hash_comments;
disallow_fun_and_cls_meth_pseudo_funcs;
enable_coeffects;
ignore_missing_json;
}

let parse_args () =
Expand Down Expand Up @@ -211,6 +214,7 @@ module FullFidelityParseArgs = struct
let disallow_hash_comments = ref false in
let disallow_fun_and_cls_meth_pseudo_funcs = ref false in
let enable_coeffects = ref false in
let ignore_missing_json = ref false in
let options =
[
(* modes *)
Expand Down Expand Up @@ -359,6 +363,9 @@ No errors are filtered out."
( "--enable-coeffects",
Arg.Set enable_coeffects,
"Allows parsing coeffect syntax" );
( "--ignore-missing-json",
Arg.Set ignore_missing_json,
"Ignore missing nodes in JSON ouput" );
]
in
Arg.parse options push_file usage;
Expand Down Expand Up @@ -417,6 +424,7 @@ No errors are filtered out."
!disallow_hash_comments
!disallow_fun_and_cls_meth_pseudo_funcs
!enable_coeffects
!ignore_missing_json
end

open FullFidelityParseArgs
Expand Down Expand Up @@ -604,7 +612,9 @@ let handle_existing_file args filename =
| None -> ()
end;
( if args.full_fidelity_json then
let json = SyntaxTree.to_json syntax_tree in
let json =
SyntaxTree.to_json ~ignore_missing:args.ignore_missing_json syntax_tree
in
let str = Hh_json.json_to_string json ~pretty:args.pretty_print_json in
Printf.printf "%s\n" str );
( if args.full_fidelity_text_json then
Expand Down
2 changes: 1 addition & 1 deletion hphp/hack/src/parser/full_fidelity_editable_syntax.ml
View file
Expand Up @@ -159,7 +159,7 @@ let offset _ = None

let position _ _ = None

let to_json ?with_value:_ node =
let to_json ?with_value:_ ?ignore_missing:_ node =
let version = Full_fidelity_schema.full_fidelity_schema_version_number in
let tree = EditableSyntax.to_json node in
Hh_json.JSON_Object
Expand Down
16 changes: 12 additions & 4 deletions hphp/hack/src/parser/full_fidelity_syntax.ml
View file
Expand Up @@ -5911,25 +5911,33 @@ module WithToken(Token: TokenType) = struct
]


let rec to_json ?(with_value = false) node =
let rec to_json_ ?(with_value = false) ?(ignore_missing = false) node =
let open Hh_json in
let ch = match node.syntax with
| Token t -> [ "token", Token.to_json t ]
| SyntaxList x -> [ ("elements",
JSON_Array (List.map ~f:(to_json ~with_value) x)) ]
JSON_Array (List.filter_map ~f:(to_json_ ~with_value ~ignore_missing) x)) ]
| _ ->
let rec aux acc c n =
match c, n with
| ([], []) -> acc
| ((hc :: tc), (hn :: tn)) ->
aux ((hn, (to_json ~with_value) hc) :: acc) tc tn
let result = (to_json_ ~with_value ~ignore_missing) hc in
(match result with
| Some r -> aux ((hn, r):: acc) tc tn
| None -> aux acc tc tn)
| _ -> failwith "mismatch between children and names" in
List.rev (aux [] (children node) (children_names node)) in
let k = ("kind", JSON_String (SyntaxKind.to_string (kind node))) in
let v = if with_value then
("value", SyntaxValue.to_json node.value) :: ch
else ch in
JSON_Object (k :: v)
if ignore_missing && (List.is_empty ch) then None else Some(JSON_Object (k :: v))

let to_json ?(with_value = false) ?(ignore_missing = false) node =
match to_json_ ~with_value ~ignore_missing node with
| Some x -> x
| None -> Hh_json.JSON_Object([])

let binary_operator_kind b =
match syntax b with
Expand Down
4 changes: 2 additions & 2 deletions hphp/hack/src/parser/full_fidelity_syntax_tree.ml
View file
Expand Up @@ -134,9 +134,9 @@ all errors that happen in a body. *)
in
remove_cascading e

let to_json ?with_value tree =
let to_json ?with_value ?ignore_missing tree =
let version = Full_fidelity_schema.full_fidelity_schema_version_number in
let root = Syntax.to_json ?with_value tree.root in
let root = Syntax.to_json ?with_value ?ignore_missing tree.root in
let text = Hh_json.JSON_String (SourceText.text tree.text) in
Hh_json.JSON_Object
[
Expand Down
2 changes: 1 addition & 1 deletion hphp/hack/src/parser/full_fidelity_syntax_tree.mli
View file
Expand Up @@ -53,7 +53,7 @@ module WithSyntax (Syntax : Syntax_sig.Syntax_S) : sig

val is_decl : t -> bool

val to_json : ?with_value:bool -> t -> Hh_json.json
val to_json : ?with_value:bool -> ?ignore_missing:bool -> t -> Hh_json.json
end

include module type of
Expand Down
2 changes: 1 addition & 1 deletion hphp/hack/src/parser/syntax_sig.ml
View file
Expand Up @@ -1031,7 +1031,7 @@ module type Syntax_S = sig
ParserOptions.ffi_t ->
Full_fidelity_syntax_error.t list
val has_leading_trivia : TriviaKind.t -> Token.t -> bool
val to_json : ?with_value:bool -> t -> Hh_json.json
val to_json : ?with_value:bool -> ?ignore_missing:bool -> t -> Hh_json.json
val extract_text : t -> string option
val is_in_body : t -> int -> bool
val syntax_node_to_list : t -> t list
Expand Down
18 changes: 3 additions & 15 deletions hphp/hack/test/full_fidelity/cases/abstract_property.php.json.exp
View file
Expand Up @@ -58,7 +58,6 @@
},
{
"kind":"classish_declaration",
"classish_attribute":{"kind":"missing"},
"classish_modifiers":{
"kind":"list",
"elements":[
Expand All @@ -78,7 +77,6 @@
}
]
},
"classish_xhp":{"kind":"missing"},
"classish_keyword":{
"kind":"token",
"token":{
Expand Down Expand Up @@ -107,12 +105,6 @@
"line_number":3
}
},
"classish_type_parameters":{"kind":"missing"},
"classish_extends_keyword":{"kind":"missing"},
"classish_extends_list":{"kind":"missing"},
"classish_implements_keyword":{"kind":"missing"},
"classish_implements_list":{"kind":"missing"},
"classish_where_clause":{"kind":"missing"},
"classish_body":{
"kind":"classish_body",
"classish_body_left_brace":{
Expand All @@ -134,7 +126,6 @@
"elements":[
{
"kind":"property_declaration",
"property_attribute_spec":{"kind":"missing"},
"property_modifiers":{
"kind":"list",
"elements":[
Expand Down Expand Up @@ -205,10 +196,8 @@
"trailing":[],
"line_number":4
}
},
"property_initializer":{"kind":"missing"}
},
"list_separator":{"kind":"missing"}
}
}
}
]
},
Expand Down Expand Up @@ -256,7 +245,6 @@
},
{
"kind":"expression_statement",
"expression_statement_expression":{"kind":"missing"},
"expression_statement_semicolon":{
"kind":"token",
"token":{
Expand Down Expand Up @@ -293,5 +281,5 @@
}
},
"program_text":"<?hh //strict\n\nabstract class C {\n public abstract int $x; // should be illegal\n};\n",
"version":"2020-06-23-0001"
"version":"2021-01-05-0001"
}
54 changes: 11 additions & 43 deletions hphp/hack/test/full_fidelity/cases/abstract_static_props/abstract_const_init.php.json.exp
View file
Expand Up @@ -58,7 +58,6 @@
},
{
"kind":"classish_declaration",
"classish_attribute":{"kind":"missing"},
"classish_modifiers":{
"kind":"list",
"elements":[
Expand Down Expand Up @@ -87,7 +86,6 @@
}
]
},
"classish_xhp":{"kind":"missing"},
"classish_keyword":{
"kind":"token",
"token":{
Expand Down Expand Up @@ -116,12 +114,6 @@
"line_number":4
}
},
"classish_type_parameters":{"kind":"missing"},
"classish_extends_keyword":{"kind":"missing"},
"classish_extends_list":{"kind":"missing"},
"classish_implements_keyword":{"kind":"missing"},
"classish_implements_list":{"kind":"missing"},
"classish_where_clause":{"kind":"missing"},
"classish_body":{
"kind":"classish_body",
"classish_body_left_brace":{
Expand Down Expand Up @@ -179,12 +171,8 @@
"trailing":[],
"line_number":5
}
},
"constructor_call_left_paren":{"kind":"missing"},
"constructor_call_argument_list":{"kind":"missing"},
"constructor_call_right_paren":{"kind":"missing"}
},
"list_separator":{"kind":"missing"}
}
}
}
]
},
Expand Down Expand Up @@ -322,8 +310,7 @@
}
}
}
},
"list_separator":{"kind":"missing"}
}
}
]
},
Expand Down Expand Up @@ -449,8 +436,7 @@
}
}
}
},
"list_separator":{"kind":"missing"}
}
}
]
},
Expand Down Expand Up @@ -489,9 +475,6 @@
},
{
"kind":"classish_declaration",
"classish_attribute":{"kind":"missing"},
"classish_modifiers":{"kind":"missing"},
"classish_xhp":{"kind":"missing"},
"classish_keyword":{
"kind":"token",
"token":{
Expand Down Expand Up @@ -520,12 +503,6 @@
"line_number":8
}
},
"classish_type_parameters":{"kind":"missing"},
"classish_extends_keyword":{"kind":"missing"},
"classish_extends_list":{"kind":"missing"},
"classish_implements_keyword":{"kind":"missing"},
"classish_implements_list":{"kind":"missing"},
"classish_where_clause":{"kind":"missing"},
"classish_body":{
"kind":"classish_body",
"classish_body_left_brace":{
Expand Down Expand Up @@ -583,12 +560,8 @@
"trailing":[],
"line_number":9
}
},
"constructor_call_left_paren":{"kind":"missing"},
"constructor_call_argument_list":{"kind":"missing"},
"constructor_call_right_paren":{"kind":"missing"}
},
"list_separator":{"kind":"missing"}
}
}
}
]
},
Expand Down Expand Up @@ -677,10 +650,8 @@
"trailing":[],
"line_number":9
}
},
"property_initializer":{"kind":"missing"}
},
"list_separator":{"kind":"missing"}
}
}
}
]
},
Expand All @@ -701,7 +672,6 @@
},
{
"kind":"const_declaration",
"const_modifiers":{"kind":"missing"},
"const_keyword":{
"kind":"token",
"token":{
Expand Down Expand Up @@ -753,10 +723,8 @@
"trailing":[],
"line_number":10
}
},
"constant_declarator_initializer":{"kind":"missing"}
},
"list_separator":{"kind":"missing"}
}
}
}
]
},
Expand Down Expand Up @@ -814,5 +782,5 @@
}
},
"program_text":"<?hh // strict\n// Copyright 2004-present Facebook. All Rights Reserved.\n\nabstract class A {\n <<__Const>> abstract static public int $a = 5;\n abstract const int C = 5;\n}\nclass B {\n <<__Const>> static public int $b;\n const int C;\n}\n",
"version":"2020-06-23-0001"
"version":"2021-01-05-0001"
}

0 comments on commit f848b33

Please sign in to comment.