Skip to content
Permalink
Browse files

Hide forward naming table behind API.

Summary: The forward naming table in Hack is a mapping from filenames to the symbols that occur in those files.  It's necessary for almost everything Hack does, but it's also really *really* big, so we'd like to be able to change the backing storage for it in the future.  Since the forward naming table is represented as a raw `Map`, the first step to changing its backing storage is to abstract it behind an API, which this change does.

Reviewed By: 2BitSalute, ljw1004

Differential Revision: D13705295

fbshipit-source-id: 8d3c7b65d1f79b73e2e576bc23127b6bec480376
  • Loading branch information...
Julia Pitts (they/she) authored and hhvm-bot committed Jan 29, 2019
1 parent f4084fa commit 8df7785f3222a8372588baedab2b4437ac1d7379
Showing with 302 additions and 253 deletions.
  1. +1 −1 hphp/hack/src/decl/decl_redecl_service.ml
  2. +1 −1 hphp/hack/src/decl/decl_redecl_service.mli
  3. +1 −1 hphp/hack/src/decl/decl_service.mli
  4. +2 −1 hphp/hack/src/deps/dune
  5. +34 −55 hphp/hack/src/deps/fileInfo.ml
  6. +3 −10 hphp/hack/src/deps/fileInfo.mli
  7. +40 −0 hphp/hack/src/deps/naming_table.ml
  8. +31 −0 hphp/hack/src/deps/naming_table.mli
  9. +2 −2 hphp/hack/src/deps/typing_deps.ml
  10. +1 −1 hphp/hack/src/deps/typing_deps.mli
  11. +8 −6 hphp/hack/src/hh_single_type_check.ml
  12. +2 −2 hphp/hack/src/parser/fixmes.ml
  13. +1 −1 hphp/hack/src/parser/parsing_service.ml
  14. +1 −1 hphp/hack/src/parser/parsing_service.mli
  15. +2 −2 hphp/hack/src/search/signatures/signatureSearchService.ml
  16. +1 −1 hphp/hack/src/search/signatures/signatureSearchService.mli
  17. +1 −1 hphp/hack/src/server/cstSearchService.ml
  18. +1 −1 hphp/hack/src/server/diagnostic_subscription.mli
  19. +4 −4 hphp/hack/src/server/findRefsService.ml
  20. +6 −6 hphp/hack/src/server/methodJumps.ml
  21. +17 −15 hphp/hack/src/server/saveStateService.ml
  22. +2 −2 hphp/hack/src/server/searchServiceRunner.ml
  23. +1 −1 hphp/hack/src/server/serverAiInit.ml
  24. +1 −1 hphp/hack/src/server/serverAiInit.mli
  25. +2 −2 hphp/hack/src/server/serverCheckUtils.ml
  26. +2 −2 hphp/hack/src/server/serverColorFile.ml
  27. +1 −1 hphp/hack/src/server/serverCommand.ml
  28. +7 −7 hphp/hack/src/server/serverCoverageMetric.ml
  29. +4 −4 hphp/hack/src/server/serverEagerInit.ml
  30. +1 −1 hphp/hack/src/server/serverEnv.ml
  31. +1 −1 hphp/hack/src/server/serverEnvBuild.ml
  32. +2 −2 hphp/hack/src/server/serverFindRefs.ml
  33. +2 −2 hphp/hack/src/server/serverHotClasses.ml
  34. +4 −4 hphp/hack/src/server/serverHover.ml
  35. +2 −2 hphp/hack/src/server/serverIdeUtils.ml
  36. +2 −3 hphp/hack/src/server/serverIdeUtils.mli
  37. +2 −2 hphp/hack/src/server/serverInferType.ml
  38. +2 −2 hphp/hack/src/server/serverInferTypeBatch.ml
  39. +2 −2 hphp/hack/src/server/serverInit.ml
  40. +6 −6 hphp/hack/src/server/serverInitCommon.ml
  41. +1 −1 hphp/hack/src/server/serverInitCommon.mli
  42. +1 −1 hphp/hack/src/server/serverInitTypes.ml
  43. +17 −18 hphp/hack/src/server/serverLazyInit.ml
  44. +1 −1 hphp/hack/src/server/serverLint.ml
  45. +1 −1 hphp/hack/src/server/serverMethodJumps.ml
  46. +1 −1 hphp/hack/src/server/serverRefactor.ml
  47. +3 −3 hphp/hack/src/server/serverRpc.ml
  48. +4 −4 hphp/hack/src/server/serverRxApiShared.ml
  49. +53 −50 hphp/hack/src/server/serverTypeCheck.ml
  50. +2 −2 hphp/hack/src/server/serverTypedAst.ml
  51. +1 −1 hphp/hack/src/server/symbolInfoService.ml
  52. +3 −3 hphp/hack/test/integration_ml/saved_state/test_fun_deps_load_from_state.ml
  53. +1 −1 hphp/hack/test/integration_ml/test_get_dependent_classes.ml
  54. +2 −2 hphp/hack/test/integration_ml/test_getfundeps.ml
  55. +2 −2 hphp/hack/test/integration_ml/test_infer_type.ml
  56. +1 −1 hphp/hack/test/integration_ml/test_interrupt.ml
  57. +2 −2 hphp/hack/test/integration_ml/test_isfunlocallable.ml
@@ -36,7 +36,7 @@ let compute_deps_neutral = DepSet.empty, DepSet.empty, DepSet.empty
(*****************************************************************************)

module OnTheFlyStore = GlobalStorage.Make(struct
type t = TypecheckerOptions.t * FileInfo.fast
type t = TypecheckerOptions.t * Naming_table.fast
end)

(*****************************************************************************)
@@ -16,7 +16,7 @@ val redo_type_decl :
conservative_redecl:bool ->
TypecheckerOptions.t ->
FileInfo.names ->
FileInfo.fast ->
Naming_table.fast ->
Errors.t * DepSet.t * DepSet.t * DepSet.t

(**
@@ -22,5 +22,5 @@ type lazy_decl_result = Errors.t
(* Starts the process *)
(*****************************************************************************)
val go: MultiWorker.worker list option -> bucket_size:int -> TypecheckerOptions.t ->
FileInfo.fast -> result
Naming_table.fast -> result
val merge_lazy_decl: lazy_decl_result -> lazy_decl_result -> lazy_decl_result
@@ -2,7 +2,8 @@
(name file_info)
(wrapped false)
(modules
fileInfo)
fileInfo
naming_table)
(libraries
opaque_digest
pos
@@ -122,11 +122,6 @@ type saved = {
}


type fast = names Relative_path.Map.t

(* Object we get from saved state *)
type saved_state_info = saved Relative_path.Map.t

let empty_names = {
n_funs = SSet.empty;
n_classes = SSet.empty;
@@ -141,53 +136,6 @@ let empty_names = {
let name_set_of_idl idl =
List.fold_left idl ~f:(fun acc (_, x) -> SSet.add x acc) ~init:SSet.empty

let saved_to_fast fast =
Relative_path.Map.map fast
begin fun saved -> saved.s_names end

(* Filter out all PHP files from saved fileInfo object. *)
let saved_to_hack_files fast =
saved_to_fast fast


let saved_to_info fast =
Relative_path.Map.fold fast ~init:Relative_path.Map.empty
~f:(fun fn saved acc ->
let {s_names; s_mode; s_hash} = saved in
let {n_funs; n_classes; n_types; n_consts;} = s_names in
let funs = List.map (SSet.elements n_funs)
(fun x -> File (Fun, fn), x) in
let classes = List.map (SSet.elements n_classes)
(fun x -> File (Class, fn), x) in
let typedefs = List.map (SSet.elements n_types)
(fun x -> File (Typedef, fn), x) in
let consts = List.map (SSet.elements n_consts)
(fun x -> File (Const, fn), x) in
let fileinfo = {
file_mode= s_mode;
hash = s_hash;
funs;
classes;
typedefs;
consts;
comments = None;
} in
Relative_path.Map.add acc fn fileinfo
)

let info_to_saved fileinfo =
Relative_path.Map.map fileinfo
(fun info ->
let {funs; classes; typedefs; consts; file_mode= s_mode;
hash= s_hash; comments = _; } = info in
let n_funs = name_set_of_idl funs in
let n_classes = name_set_of_idl classes in
let n_types = name_set_of_idl typedefs in
let n_consts = name_set_of_idl consts in

let s_names = { n_funs; n_classes; n_types; n_consts; } in
{ s_names; s_mode; s_hash; })

let simplify info =
let {funs; classes; typedefs; consts; file_mode = _; comments = _;
hash = _;} = info in
@@ -197,6 +145,40 @@ let simplify info =
let n_consts = name_set_of_idl consts in
{n_funs; n_classes; n_types; n_consts}

let to_saved info =
let {funs; classes; typedefs; consts; file_mode= s_mode;
hash= s_hash; comments = _; } = info in
let n_funs = name_set_of_idl funs in
let n_classes = name_set_of_idl classes in
let n_types = name_set_of_idl typedefs in
let n_consts = name_set_of_idl consts in
let s_names = { n_funs; n_classes; n_types; n_consts; } in
{ s_names; s_mode; s_hash; }

let from_saved fn saved =
let {s_names; s_mode; s_hash} = saved in
let {n_funs; n_classes; n_types; n_consts;} = s_names in
let funs = List.map (SSet.elements n_funs)
(fun x -> File (Fun, fn), x) in
let classes = List.map (SSet.elements n_classes)
(fun x -> File (Class, fn), x) in
let typedefs = List.map (SSet.elements n_types)
(fun x -> File (Typedef, fn), x) in
let consts = List.map (SSet.elements n_consts)
(fun x -> File (Const, fn), x) in
{
file_mode= s_mode;
hash = s_hash;
funs;
classes;
typedefs;
consts;
comments = None;
}

let saved_to_names saved =
saved.s_names

let merge_names t_names1 t_names2 =
let {n_funs; n_classes; n_types; n_consts} = t_names1 in
{
@@ -206,9 +188,6 @@ let merge_names t_names1 t_names2 =
n_consts = SSet.union n_consts t_names2.n_consts;
}

let simplify_fast fast =
Relative_path.Map.map fast simplify

let print_names name =
Printf.printf "Funs:\n";
SSet.iter (Printf.printf "\t%s\n") name.n_funs;
@@ -66,27 +66,20 @@ type names = {
n_consts : SSet.t;
}

type fast = names Relative_path.Map.t


(*****************************************************************************)
(* The record used in our saved state. *)
(*****************************************************************************)
type saved
type saved_state_info = saved Relative_path.Map.t


val empty_names: names

(*****************************************************************************)
(* Functions simplifying the file information. *)
(*****************************************************************************)
val saved_to_info : saved_state_info -> t Relative_path.Map.t
val saved_to_fast: saved_state_info -> fast
val saved_to_hack_files: saved_state_info -> fast
val info_to_saved : t Relative_path.Map.t -> saved_state_info
val simplify: t -> names
val merge_names: names -> names -> names
val simplify_fast: t Relative_path.Map.t -> fast
val print_names : names -> unit
val to_saved : t -> saved
val from_saved : Relative_path.t -> saved -> t
val saved_to_names : saved -> names
val to_string: t -> string
@@ -0,0 +1,40 @@
(**
* Copyright (c) 2015, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the "hack" directory of this source tree.
*
*)

type t = FileInfo.t Relative_path.Map.t
type fast = FileInfo.names Relative_path.Map.t
type saved_state_info = FileInfo.saved Relative_path.Map.t

let combine a b = Relative_path.Map.union a b
let create a = a
let empty = Relative_path.Map.empty
let filter = Relative_path.Map.filter
let fold = Relative_path.Map.fold
let get_files = Relative_path.Map.keys
let get_file_info = Relative_path.Map.get
let get_file_info_unsafe = Relative_path.Map.find_unsafe
let has_file = Relative_path.Map.mem
let iter = Relative_path.Map.iter
let update a key data = Relative_path.Map.add a ~key ~data


let from_saved saved =
Relative_path.Map.fold saved ~init:Relative_path.Map.empty ~f:(fun fn saved acc ->
let file_info = FileInfo.from_saved fn saved in
Relative_path.Map.add acc fn file_info
)

let to_saved a =
Relative_path.Map.map a FileInfo.to_saved

let to_fast a =
Relative_path.Map.map a FileInfo.simplify

let saved_to_fast saved =
Relative_path.Map.map saved FileInfo.saved_to_names
@@ -0,0 +1,31 @@
(**
* Copyright (c) 2015, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the "hack" directory of this source tree.
*
*)

type t
type fast = FileInfo.names Relative_path.Map.t
type saved_state_info = FileInfo.saved Relative_path.Map.t

(* Querying and updating naming tables. *)
val combine : t -> t -> t
val create : FileInfo.t Relative_path.Map.t -> t
val empty : t
val filter : t -> f:(Relative_path.t -> FileInfo.t -> bool) -> t
val fold : t -> init:'b -> f:(Relative_path.t -> FileInfo.t -> 'b -> 'b) -> 'b
val get_files : t -> Relative_path.t list
val get_file_info : t -> Relative_path.t -> FileInfo.t option
val get_file_info_unsafe : t -> Relative_path.t -> FileInfo.t
val has_file : t -> Relative_path.t -> bool
val iter : t -> f:(Relative_path.t -> FileInfo.t -> unit) -> unit
val update : t -> Relative_path.t -> FileInfo.t -> t

(* Converting between different types of naming tables. *)
val from_saved : saved_state_info -> t
val to_saved : t -> saved_state_info
val to_fast: t -> fast
val saved_to_fast: saved_state_info -> fast
@@ -201,9 +201,9 @@ let get_files deps =
with Not_found -> acc
end deps ~init:Relative_path.Set.empty

let update_files fileInfo =
let update_files naming_table =
(* TODO: Figure out if we need GConstName and FunName as well here *)
Relative_path.Map.iter fileInfo begin fun filename info ->
Naming_table.iter naming_table begin fun filename info ->
let {FileInfo.funs; classes; typedefs;
consts;
comments = _;
@@ -46,7 +46,7 @@ val add_idep : Dep.variant -> Dep.variant -> unit
val get_ideps_from_hash : Dep.t -> DepSet.t
val get_ideps : Dep.variant -> DepSet.t
val get_files : DepSet.t -> Relative_path.Set.t
val update_files : FileInfo.t Relative_path.Map.t -> unit
val update_files : Naming_table.t -> unit

(* Add to accumulator all extend dependencies of source_class. Visited is used
* to avoid processing nodes reachable in multiple ways more than once. In other
@@ -923,14 +923,15 @@ let handle_mode

| Dump_inheritance ->
let open ServerCommandTypes.Method_jumps in
Typing_deps.update_files files_info;
Relative_path.Map.iter files_info begin fun fn fileinfo ->
let naming_table = Naming_table.create files_info in
Typing_deps.update_files naming_table;
Naming_table.iter naming_table begin fun fn fileinfo ->
if Relative_path.Map.mem builtins fn then () else begin
List.iter fileinfo.FileInfo.classes begin fun (_p, class_) ->
Printf.printf "Ancestors of %s and their overridden methods:\n"
class_;
let ancestors = MethodJumps.get_inheritance tcopt class_
~filter:No_filter ~find_children:false files_info
~filter:No_filter ~find_children:false naming_table
None in
ClientMethodJumps.print_readable ancestors ~find_children:false;
Printf.printf "\n";
@@ -940,7 +941,7 @@ let handle_mode
Printf.printf "Children of %s and the methods they override:\n"
class_;
let children = MethodJumps.get_inheritance tcopt class_
~filter:No_filter ~find_children:true files_info None in
~filter:No_filter ~find_children:true naming_table None in
ClientMethodJumps.print_readable children ~find_children:true;
Printf.printf "\n";
end;
@@ -1032,13 +1033,14 @@ let handle_mode
)
| Find_refs (line, column) ->
let filename = expect_single_file () in
Typing_deps.update_files files_info;
let naming_table = Naming_table.create files_info in
Typing_deps.update_files naming_table;
Relative_path.set_path_prefix Relative_path.Root (Path.make "/");
Relative_path.set_path_prefix Relative_path.Hhi (Path.make "hhi");
Relative_path.set_path_prefix Relative_path.Tmp (Path.make "tmp");
let genv = ServerEnvBuild.default_genv in
let env = {(ServerEnvBuild.make_env genv.ServerEnv.config) with
ServerEnv.files_info;
ServerEnv.naming_table;
ServerEnv.tcopt = tcopt;
} in
let filename = Relative_path.to_absolute filename in
@@ -92,12 +92,12 @@ let get_unused_fixmes_for codes applied_fixme_map fn acc =
then fixme_pos :: acc
else acc) code_map acc) fixme_map acc

let get_unused_fixmes codes applied_fixmes files_info =
let get_unused_fixmes codes applied_fixmes naming_table =
let applied_fixme_map =
List.fold_left applied_fixmes ~init:Relative_path.Map.empty
~f:begin fun acc (pos,code) ->
let fn = Pos.filename pos in
let line, _, _ = Pos.info_pos pos in
add_applied_fixme acc code fn line end in
Relative_path.Map.fold files_info ~f:(fun fn _ acc ->
Naming_table.fold naming_table ~f:(fun fn _ acc ->
get_unused_fixmes_for codes applied_fixme_map fn acc) ~init:[]
@@ -158,4 +158,4 @@ let go ?(quick = false) workers files_set ~get_next popt ~trace =
parse_sequential ~quick fn content acc popt
) in
if trace then log_parsing_results fast;
fast, errorl, failed_parsing
Naming_table.create fast, errorl, failed_parsing
@@ -15,7 +15,7 @@ val go:
get_next: Relative_path.t list Bucket.next ->
ParserOptions.t ->
trace: bool ->
FileInfo.t Relative_path.Map.t * Errors.t * Relative_path.Set.t
Naming_table.t * Errors.t * Relative_path.Set.t

(* used by hack build *)
val legacy_php_file_info: (Relative_path.t -> FileInfo.t) ref
@@ -179,9 +179,9 @@ let add_function tcopt fun_name =
end
)

let build tcopt fileinfos =
let build tcopt naming_table =
Hh_logger.log "Building Search Index";
Relative_path.Map.iter fileinfos (fun _ value ->
Naming_table.iter naming_table (fun _ value ->
let {FileInfo.funs; _ } = value in
List.iter funs ~f:(fun (pos, fun_name) ->
let path =
@@ -11,7 +11,7 @@ val add_function : TypecheckerOptions.t -> string -> unit
(** Given a function name, look up that function's signature on the typing heap
and mutate the signature search index to include this function. *)

val build : TypecheckerOptions.t -> FileInfo.t Relative_path.Map.t -> unit
val build : TypecheckerOptions.t -> Naming_table.t -> unit
(** Given a Map of file info, retrieve each file's function name and update the
index *)

@@ -702,7 +702,7 @@ let go
let next_files: (Relative_path.t * FileInfo.t * pattern) list Hh_bucket.next =
let with_file_data path =
let path = Relative_path.create_detect_prefix path in
match Relative_path.Map.get env.ServerEnv.files_info path with
match Naming_table.get_file_info env.ServerEnv.naming_table path with
| Some fileinfo -> Some (path, fileinfo, pattern)
| None ->
(* We may not have the file information for a file such as one that we
@@ -19,7 +19,7 @@ val update :
t ->
priority_files:Relative_path.Set.t ->
reparsed:Relative_path.Set.t ->
rechecked:FileInfo.fast ->
rechecked:Naming_table.fast ->
global_errors:Errors.t ->
full_check_done:bool ->
t
Oops, something went wrong.

0 comments on commit 8df7785

Please sign in to comment.
You can’t perform that action at this time.
You signed in with another tab or window. Reload to refresh your session. You signed out in another tab or window. Reload to refresh your session.