Skip to content
Permalink
Browse files

ignore hack keywords in php files

Summary:
Hack keywords in PHP files should be understood as simple identifiers. For example:
```
class inout {}
```
The simplest way to do that is to have the lexer aware of what it is lexing: PHP or Hack. This is done using a - for now very small - `Lexer_env` module.
The generated `Full_fidelity_token_kind` does the lexing of keywords in function `from_string`. We change the generator so that this function now accept an `is_hack` argument. We also change the `Full_fidelity_schema` so that developers now have to specify whether a given keyword is "hack_only" or "php_and_hack".

Reviewed By: jamesjwu

Differential Revision: D8735588

fbshipit-source-id: 06474fd994885dce3f1ea9fed695bf5ba7bfac33
  • Loading branch information...
CatherineGasnier authored and fredemmott committed Jul 18, 2018
1 parent 9db89dc commit 16a87033933457bdc77b8249dbaa9b30b4f9e098
Showing with 1,764 additions and 423 deletions.
  1. +28 −2 hphp/hack/src/generate_full_fidelity.ml
  2. +4 −0 hphp/hack/src/hh_single_compile.ml
  3. +30 −9 hphp/hack/src/parser/full_fidelity_lexer.ml
  4. +4 −0 hphp/hack/src/parser/full_fidelity_lexer.mli
  5. +184 −184 hphp/hack/src/parser/full_fidelity_token_kind.ml
  6. +2 −1 hphp/hack/src/parser/hack_grammar_descriptor_helper.ml
  7. +246 −215 hphp/hack/src/parser/schema/full_fidelity_schema.ml
  8. +20 −0 hphp/hack/test/full_fidelity/cases/hack_kw_in_php1.php
  9. +86 −0 hphp/hack/test/full_fidelity/cases/hack_kw_in_php1.php.sexp.exp
  10. +9 −0 hphp/hack/test/full_fidelity/cases/hack_kw_in_php2.php
  11. +46 −0 hphp/hack/test/full_fidelity/cases/hack_kw_in_php2.php.sexp.exp
  12. +110 −0 hphp/hack/test/full_fidelity/cases/hack_kw_in_php3.php
  13. +682 −0 hphp/hack/test/full_fidelity/cases/hack_kw_in_php3.php.sexp.exp
  14. +10 −0 hphp/hack/test/full_fidelity/cases/hack_kw_in_php4.php
  15. +61 −0 hphp/hack/test/full_fidelity/cases/hack_kw_in_php4.php.sexp.exp
  16. +9 −0 hphp/hack/test/full_fidelity/cases/hack_kw_in_php_cap1.php
  17. +46 −0 hphp/hack/test/full_fidelity/cases/hack_kw_in_php_cap1.php.sexp.exp
  18. +9 −0 hphp/hack/test/full_fidelity/cases/hack_kw_in_php_cap2.php
  19. +46 −0 hphp/hack/test/full_fidelity/cases/hack_kw_in_php_cap2.php.sexp.exp
  20. +8 −0 hphp/hack/test/full_fidelity/cases/hack_kw_in_php_cap3.php
  21. +34 −0 hphp/hack/test/full_fidelity/cases/hack_kw_in_php_cap3.php.sexp.exp
  22. +10 −0 hphp/hack/test/full_fidelity/cases/hack_kw_in_php_cap4.php
  23. +61 −0 hphp/hack/test/full_fidelity/cases/hack_kw_in_php_cap4.php.sexp.exp
  24. +1 −1 hphp/hack/test/typecheck/class_property_initialization/call_in_static_shape.php
  25. +7 −0 hphp/test/one_bit_refcount_failing_quick
  26. +1 −1 hphp/test/quick/dict/apc.php
  27. +1 −1 hphp/test/quick/dict/unset.php
  28. +1 −1 hphp/test/quick/keyset/serialize.php
  29. +1 −1 hphp/test/quick/vec/dtor.php
  30. +1 −1 hphp/test/quick/vec/member-ops.php
  31. +1 −1 hphp/test/quick/vec/serialize.php
  32. +1 −1 hphp/test/quick/vec/unset.php
  33. +1 −1 hphp/test/quick/xenon/xenon_crash.php
  34. +1 −1 hphp/test/slow/dv_array/hh-only1.php.expectregex
  35. +1 −1 hphp/test/slow/dv_array/hh-only2.php.expectregex
  36. +1 −1 hphp/test/slow/xhp/attr_types.php
@@ -2151,11 +2151,37 @@ module GenerateFFTokenKind = struct
let to_kind_declaration x =
sprintf " | %s\n" x.token_kind

let add_guard_or_pad :
cond:(bool * string) -> ?else_cond:(bool * string) -> string -> string =
fun ~cond:(cond, guard) ?else_cond guards ->
let pad str = String.make (String.length str) ' ' in
let is_only_spaces str = (str = (pad str)) in
let make_same_length str1 str2 =
let blanks n = try String.make n ' ' with Invalid_argument _ -> "" in
let (len1, len2) = (String.length str1, String.length str2) in
let str1 = str1 ^ (blanks (len2 - len1)) in
let str2 = str2 ^ (blanks (len1 - len2)) in
(str1, str2) in
let (else_cond, else_guard) = match else_cond with
| Some (cond, guard) -> cond, guard
| None -> false, "" in
let prefix = if cond || else_cond
then if is_only_spaces guards then "when " else "&& "
else " " in
let (guard, else_guard) = make_same_length guard else_guard in
let guard = if cond then guard
else if else_cond then else_guard
else pad guard in
guards ^ prefix ^ guard ^ " "

let to_from_string x =
let token_text = escape_token_text x.token_text in
let spacer_width = given_text_width - String.length token_text in
let spacer = String.make spacer_width ' ' in
sprintf " | \"%s\"%s -> Some %s\n" token_text spacer x.token_kind
let guards = add_guard_or_pad ""
~cond:(x.is_xhp, "(is_hack || allow_xhp)")
~else_cond:(x.hack_only, "is_hack") in
sprintf " | \"%s\"%s %s-> Some %s\n" token_text spacer guards x.token_kind

let to_to_string x =
let token_text = escape_token_text x.token_text in
@@ -2174,7 +2200,7 @@ KIND_DECLARATIONS_GIVEN_TEXT (* Variable text tokens *)
KIND_DECLARATIONS_VARIABLE_TEXT
[@@deriving show]
let from_string keyword =
let from_string keyword ~is_hack ~allow_xhp =
match keyword with
| \"true\" -> Some BooleanLiteral
| \"false\" -> Some BooleanLiteral
@@ -13,6 +13,7 @@ open Sys_utils
module P = Printf
module SyntaxError = Full_fidelity_syntax_error
module SourceText = Full_fidelity_source_text
module Lex = Full_fidelity_lexer
module Logger = HackcEventLogger

(*****************************************************************************)
@@ -277,11 +278,14 @@ let parse_text compiler_options popt fn text =
not (Hhbc_options.source_mapping !Hhbc_options.compiler_options) in
let enable_hh_syntax =
Hhbc_options.enable_hiphop_syntax !Hhbc_options.compiler_options in
let enable_xhp =
Hhbc_options.enable_xhp !Hhbc_options.compiler_options in
let php5_compat_mode =
not (Hhbc_options.enable_uniform_variable_syntax !Hhbc_options.compiler_options) in
let hacksperimental =
Hhbc_options.hacksperimental !Hhbc_options.compiler_options in
let systemlib_compat_mode = Emit_env.is_systemlib () in
Lex.Env.set ~force_hh:enable_hh_syntax ~enable_xhp;
let env = Full_fidelity_ast.make_env
~parser_options:popt
~ignore_pos
@@ -12,13 +12,29 @@ module TokenKind = Full_fidelity_token_kind
module SourceText = Full_fidelity_source_text
module SyntaxError = Full_fidelity_syntax_error

module Env = struct

let force_hh_opt = ref false
let enable_xhp_opt = ref false
let is_hh_file = ref true

let set_is_hh_file b = is_hh_file := b
let set ~force_hh ~enable_xhp =
force_hh_opt := force_hh;
enable_xhp_opt := enable_xhp

let is_hh () = !is_hh_file || !force_hh_opt
let enable_xhp () = is_hh () || !enable_xhp_opt

end

module Lexer : sig
type t = {
text : SourceText.t;
start : int; (* Both start and offset are absolute offsets in the text. *)
offset : int;
errors : SyntaxError.t list;
hacksperimental : bool
hacksperimental : bool;
} [@@deriving show]
val make : ?hacksperimental:bool -> SourceText.t -> t
val make_at : ?hacksperimental:bool -> SourceText.t -> int -> t
@@ -44,7 +60,7 @@ end = struct
start : int; (* Both start and offset are absolute offsets in the text. *)
offset : int;
errors : SyntaxError.t list;
hacksperimental : bool (* write-once: record updates should not update this field *)
hacksperimental : bool; (* write-once: record updates should not update this field *)
} [@@deriving show]

let make ?(hacksperimental = false) text =
@@ -1397,25 +1413,26 @@ let as_case_insensitive_keyword text =
non-lower versions in our codebase. *)
let lower = String.lowercase_ascii text in
match lower with
| "__halt_compiler" | "abstract" | "and" | "array" | "as" | "bool" | "boolean" | "break"
| "__halt_compiler" | "abstract" | "and" | "array" | "as" | "bool" | "boolean" | "break"
| "callable"
| "case" | "catch" | "class" | "clone" | "const" | "continue" | "declare" | "default"
| "die" | "do" | "echo" | "else" | "elseif" | "empty" | "enddeclare" | "endfor"
| "endforeach" | "endif" | "endswitch" | "endwhile" | "eval" | "exit" | "extends" | "false"
| "final" | "finally" | "for" | "foreach" | "function" | "global" | "goto" | "if"
| "implements" | "include" | "include_once" | "inout" | "instanceof" | "insteadof" | "int"
| "integer"
| "implements" | "include" | "include_once" | "instanceof" | "insteadof" | "int" | "integer"
| "interface" | "isset" | "list" | "namespace" | "new" | "null" | "or" | "parent"
| "print" | "private" | "protected" | "public" | "require" | "require_once"
| "return" | "self" | "static" | "string" | "switch" | "throw" | "trait"
| "try" | "true" | "unset" | "use" | "using" | "var" | "void" | "while"
| "try" | "true" | "unset" | "use" | "var" | "void" | "while"
| "xor" | "yield" -> lower
| "inout" | "using" when Env.is_hh () -> lower
| _ -> text

let as_keyword kind lexer =
if kind = TokenKind.Name then
let text = as_case_insensitive_keyword (current_text lexer) in
match TokenKind.from_string text with
let is_hack = Env.is_hh () and allow_xhp = Env.enable_xhp () in
match TokenKind.from_string text ~is_hack ~allow_xhp with
| Some TokenKind.Let when (not (hacksperimental lexer)) -> TokenKind.Name
| Some keyword -> keyword
| _ -> TokenKind.Name
@@ -1608,8 +1625,12 @@ let skip_to_end_of_markup lexer ~is_leading_section =
let ch1 = peek_char lexer 1 in
let ch2 = peek_char lexer 2 in
match ch0, ch1, ch2 with
| ('H' | 'h'), ('H' | 'h'), _ -> make_long_tag lexer 2
| ('P' | 'p'), ('H' | 'h'), ('P' | 'p') -> make_long_tag lexer 3
| ('H' | 'h'), ('H' | 'h'), _ ->
Env.set_is_hh_file true;
make_long_tag lexer 2
| ('P' | 'p'), ('H' | 'h'), ('P' | 'p') ->
Env.set_is_hh_file false;
make_long_tag lexer 3
| '=', _, _ ->
begin
(* skip = *)
@@ -7,6 +7,10 @@
*
*)

module Env : sig
val set : force_hh:bool -> enable_xhp:bool -> unit
end

module WithToken : functor (Token : Lexable_token_sig.LexableToken_S) -> sig
type t [@@deriving show]
type string_literal_kind =
Oops, something went wrong.

0 comments on commit 16a8703

Please sign in to comment.
You can’t perform that action at this time.