main_scheck.ml

(*
 * The author disclaims copyright to this source code.  In place of
 * a legal notice, here is a blessing:
 *
 *    May you do good and not evil.
 *    May you find forgiveness for yourself and forgive others.
 *    May you share freely, never taking more than you give.
 *)

open Common

open Ast_php

module Ast = Ast_php
module V = Visitor_php

module S = Scope_code

(*****************************************************************************)
(* Purpose *)
(*****************************************************************************)

(* A lint-like checker for PHP.
 * 
 * By default 'scheck' performs only a local analysis of the files passed
 * on the command line. It is thus quite fast while still detecting a few
 * important bugs like the use of undefined variables. 
 * 
 * 'scheck' can also leverage more expensive global analysis to find more bugs.
 * Doing so requires a PHP code database which is usually very expensive 
 * to build (see pfff_db_heavy) and very large disk-wise. Fortunately one can 
 * now build this database in memory, on the fly. Indeed, thanks
 * to the include_require_php.ml analysis, we can now
 * build only the db for the files that matters, cutting significantly
 * the time to build the db (going down from 40 000 files to about 1000
 * files on average on facebook code). In a way it is similar
 * to what gcc does when it calls 'cpp' to get the full information for
 * a file. 
 * 
 * 'scheck' can also leverage a light database (see pfff_db) and 
 * use this as a cache.
 * 
 * 'scheck' could also use the heavy database but this requires to have
 * the program linked with Berkeley DB, adding some dependencies to 
 * the user of the program (and is not very multi-user friendly for now).
 * See main_scheck_heavy.ml for such a program.
 * 
 * Note that scheck is mostly for generic bugs (that sometimes
 * requires global analysis). For API-specific bugs, you can use 'sgrep'.
 * 
 * modes:
 *  - local analysis
 *  - perform global analysis "lazily" by building db on-the-fly
 *    of the relevant included files (configurable via a -depth_limit flag)
 *  - TODO leverage global analysis computed previously by pfff_db_light
 *  - leverage global analysis computed by pfff_db_heavy, 
 *    see main_scheck_heavy.ml
 * 
 * current checks:
 *   - variable related (use of undeclared variable, unused variable, etc)
 *   - TODO use/def of entities (e.g. use of undefined class/function/constant
 *     a la checkModule)
 *   - TODO function call related (wrong number of arguments, bad keyword
 *     arguments, etc)
 *   - TODO class related (use of undefined member)
 *   - TODO dead code (dead function in callgraph, dead block in CFG, 
 *     dead assignement in dataflow)
 *   - TODO include/require and file related (including file that do not
 *     exist anymore)
 *   - TODO type related
 *   - TODO resource related (open/close match)
 *   - TODO security related ??
 *   - TODO require_strict() related (see facebook/.../main_linter.ml)
 * 
 * related: 
 *   - TODO lint_php.ml (small syntactic conventions, e.g. bad defines)
 *   - TODO check_code_php.ml (include/require stuff)
 *   - TODO check_module.ml (require_module() stuff), 
 *   - TODO main_linter.ml (require_strict() stuff), 
 *   - TODO main_checker.ml (flib-aware  checker),
 * 
 * todo: make it possible to take a db in parameter so
 * for other functions, we can also get their prototype.
 * 
 * todo: build info about builtins, so when call to preg_match,
 * know that this function takes things via reference.
 * 
 * later: it could later also check javascript, CSS, sql, etc
 * 
 *)

(*****************************************************************************)
(* Flags *)
(*****************************************************************************)

let verbose = ref false

(* action mode *)
let action = ref ""

(* In strict mode, we are more aggressive regarding scope like in
 * JsLint. This is a copy of the same variable in Error_php.ml
 *)
let strict_scope = ref false 

(* running the heavy analysis processing for instance the included files *)
let heavy = ref false

(* depth_limit is used to stop the expensive recursive includes process.
 *
 * I put 5 because it's fast enough at depth 5, and 
 * I think it's good enough as it is probably bad for a file to use
 * something that is distant by more than 5 includes. 
 * 
 * todo: one issue is that some code like facebook uses special 
 *  require/include directives that include_require_php.ml is not aware of.
 *  Maybe we should have a unfacebookizer preprocessor that removes
 *  this sugar. The alternative right now is to copy most of the code
 *  in this file in facebook/qa_code/checker.ml :( and plug in the
 *  special include_require_php.ml hooks. Another alternative is to use
 *  the light_db.json cache.
 *)
let depth_limit = ref (Some 5: int option)

let php_stdlib = 
  ref (Filename.concat Config.path "/data/php_stdlib")

let cache_parse = ref true


(* for ranking errors *)
let rank = ref true

(* for codemap or layer_stat *)
let layer_file = ref (None: filename option)

(*****************************************************************************)
(* Helpers *)
(*****************************************************************************)

let build_mem_db file =

  (* Build the database of information. Some checks needs to have
   * a global view of the code, for instance to know what
   * are the sets of valid protected variable that can be used
   * in a child class.
   * 
   * todo: can probably optimize this later. For instance lazy
   * loading of files, stop when are in flib as modules are
   * not transitive.
   * 
   * see also facebook/.../dependencies.ml
   *)

  (* todo: could infer PHPROOT at least ? just look at
   * the include in the file and see where the files are.
   *)
  let env = 
    Env_php.mk_env (Common.dirname file)
  in
  let root = "/" in (* todo ? *)

  let all_files = 
    Include_require_php.recursive_included_files_of_file 
      ~verbose:!verbose 
      ~depth_limit:!depth_limit
      env file
  in
  (* adding builtins *)
  let builtin_files =
    Lib_parsing_php.find_php_files_of_dir_or_files [!php_stdlib]
  in
        
  let all_files = builtin_files ++ all_files in
  let prj = Database_php.Project (root, None) in
  let prj = Database_php.normalize_project prj in 
  
  let db = 
          Common.save_excursion Flag_analyze_php.verbose_database !verbose 
            (fun()->
              Database_php_build.create_db
                ~db_support:(Database_php.Mem)
                ~phase:2 (* TODO ? *)
                ~files:(Some all_files)
                ~verbose_stats:false
                prj 
            )
  in
  db

(*****************************************************************************)
(* Wrappers *)
(*****************************************************************************)
let pr2_dbg s =
  if !verbose then Common.pr2 s

(*****************************************************************************)
(* Main action *)
(*****************************************************************************)

let main_action xs =

  let files = Lib_parsing_php.find_php_files_of_dir_or_files xs in
  let errors = ref [] in

  Flag_parsing_php.show_parsing_error := false;
  Flag_parsing_php.verbose_lexing := false;
  Error_php.strict := !strict_scope;

  Common.save_excursion Flag_parsing_php.caching_parsing !cache_parse (fun ()->
  files +> List.iter (fun file ->
    try 
      pr2_dbg (spf "processing: %s" file);

      if not !heavy then begin
        let find_entity = None in
        Check_all_php.check_file ~find_entity file
      end else begin
        let db = build_mem_db file in
        let find_entity = Some (Database_php_build.build_entity_finder db) in
        Check_all_php.check_file ~find_entity file
      end
    with 
    | (Timeout | UnixExit _) as exn -> raise exn
    | exn ->
      Common.push2 (spf "PB with %s, exn = %s" file 
                       (Common.string_of_exn exn)) errors;
  );
  );

  let errs = !Error_php._errors +> List.rev in
  let errs = 
    if !rank 
    then Error_php.rank_errors errs +> Common.take_safe 20 
    else errs 
  in

  errs +> List.iter (fun err -> pr (Error_php.string_of_error err));
  Error_php.show_10_most_recurring_unused_variable_names ();
  pr2 (spf "total errors = %d" (List.length !Error_php._errors));

  pr2 "";
  !errors +> List.iter pr2;
  pr2 "";

  !layer_file +> Common.do_option (fun file ->
    (*  a layer needs readable paths, hence the root *)
    let root = Common.common_prefix_of_files_or_dirs xs in

    Layer_checker_php.gen_layer ~root ~output:file !Error_php._errors

  );
  ()

(*****************************************************************************)
(* Extra actions *)
(*****************************************************************************)

(*---------------------------------------------------------------------------*)
(* type inference playground *)
(*---------------------------------------------------------------------------*)

let type_inference file =
  let ast = Parse_php.parse_program file in

  (* PHP Intermediate Language *)
  try
    let pil = Pil_build.pil_of_program ast in

    (* todo: how bootstrap this ? need a bottom-up analysis but
     * we could first start with the types of the PHP builtins that
     * we already have (see builtins_php.mli in lang_php/analyze/).
     *)
    let env = () in

    (* works by side effect on the pil *)
    Type_inference_pil.infer_types env pil;

    (* simple pretty printer *)
    let s = Pretty_print_pil.string_of_program pil in
    pr s;

    (* internal representation pretty printer *)
    let s = Meta_pil.string_of_program
      ~config:{Meta_pil.show_types = true; show_tokens = false}
      pil
    in
    pr s;


  with exn ->
    pr2 "File contain constructions not supported by the PIL; bailing out";
    raise exn

(*---------------------------------------------------------------------------*)
(* Testing *)
(*---------------------------------------------------------------------------*)
let test () =
  let test_files = [
    "tests/php/scheck/variables.php";
    "tests/php/scheck/cfg.php";
    "tests/php/scheck/functions.php";
    "tests/php/scheck/classes.php";
    "tests/php/scheck/static_methods.php";
  ] 
  in
  let test_files = 
    test_files +> List.map (fun s -> Filename.concat Config.path s) in
  
  let (expected_errors :(Common.filename * int (* line *)) list) =
    test_files +> List.map (fun file ->
      Common.cat file +> Common.index_list_1 +> Common.map_filter 
        (fun (s, idx) -> 
          (* Right now we don't care about the actual error messages. We
           * don't check if they match. We are just happy to check for 
           * correct lines error reporting.
           *)
          if s =~ ".*//ERROR:.*" 
          (* + 1 because the comment is one line before *)
          then Some (file, idx + 1) 
          else None
        )
    ) +> List.flatten
  in
  let builtin_files =
    Lib_parsing_php.find_php_files_of_dir_or_files [!php_stdlib]
  in
  let all_files = builtin_files ++ test_files in

  Error_php._errors := [];

  let prj = Database_php.Project ("/", None) in
  let prj = Database_php.normalize_project prj in 
  let db = 
    Common.save_excursion Flag_analyze_php.verbose_database !verbose 
      (fun()->
        Database_php_build.create_db
          ~db_support:(Database_php.Mem)
          ~phase:2 (* TODO ? *)
          ~files:(Some all_files)
          ~verbose_stats:false
          prj 
      )
  in
  let find_entity = Some (Database_php_build.build_entity_finder db) in

  (* run the bugs finders *)
  test_files +> List.iter (Check_all_php.check_file ~find_entity);

  !Error_php._errors +> List.iter (fun e -> pr (Error_php.string_of_error e));
  
  let (actual_errors: (Common.filename * int (* line *)) list) = 
    !Error_php._errors +> Common.map_filter (fun err ->
      let info_opt = Error_php.info_of_error err in
      info_opt +> Common.fmap (fun info ->
        (Ast.file_of_info info, Ast.line_of_info info)
      )
    )
  in
  
  (* diff report *)
  let (common, only_in_expected, only_in_actual) = 
    Common.diff_set_eff expected_errors actual_errors in

  only_in_expected |> List.iter (fun (src, l) ->
    pr2 (spf "this one error is missing: %s:%d" src l);
  );
  only_in_actual |> List.iter (fun (src, l) ->
    pr2 (spf "this one error was not expected: %s:%d" src l);
  );

  if not (null only_in_expected && null only_in_actual)
  then failwith "scheck tests failed"
  else pr2 "scheck tests passed OK"


(*---------------------------------------------------------------------------*)
(* the command line flags *)
(*---------------------------------------------------------------------------*)
let scheck_extra_actions () = [

  "-type_inference", " <file>",
  Common.mk_action_1_arg type_inference;

  "-test", " ",
  Common.mk_action_0_arg test;
]

(*****************************************************************************)
(* The options *)
(*****************************************************************************)

let all_actions () =
 scheck_extra_actions()++
 []

let options () =
  [
    "-verbose", Arg.Set verbose,
    " ";

    "-heavy", Arg.Set heavy,
    " process included files";
    "-depth_limit", Arg.Int (fun i -> depth_limit := Some i), 
    " limit the number of processed includes";
    "-no_caching", Arg.Clear cache_parse, 
    " don't cache parsed ASTs";
     "-php_stdlib", Arg.Set_string php_stdlib, 
     (spf " path to builtins (default = %s)" !php_stdlib);

    "-strict", Arg.Set strict_scope,
    " emulate block scope instead of function scope";
    "-no_scrict_scope", Arg.Clear strict_scope, 
    " use function scope (default)";

    "-no_rank", Arg.Clear rank,
    " ";

    "-gen_layer", Arg.String (fun s -> layer_file := Some s),
    " <file> save result in pfff layer file";

  ] ++
  Flag_analyze_php.cmdline_flags_verbose () ++
  Common.options_of_actions action (all_actions()) ++
  Common.cmdline_flags_devel () ++
  Common.cmdline_flags_verbose () ++
  Common.cmdline_flags_other () ++
  [
  "-version",   Arg.Unit (fun () ->
    pr2 (spf "scheck version: %s" Config.version);
    exit 0;
  ),
    "  guess what";

  (* this can not be factorized in Common *)
  "-date",   Arg.Unit (fun () ->
    pr2 "version: $Date: 2010/04/25 00:44:57 $";
    raise (Common.UnixExit 0)
    ),
  "   guess what";
  ] ++
  []

(*****************************************************************************)
(* Main entry point *)
(*****************************************************************************)

let main () =

  let usage_msg =
    "Usage: " ^ Common.basename Sys.argv.(0) ^
      " [options] <file or dir> " ^ "\n" ^ "Options are:"
  in
  (* does side effect on many global flags *)
  let args = Common.parse_options (options()) usage_msg Sys.argv in

  (* must be done after Arg.parse, because Common.profile is set by it *)
  Common.profile_code "Main total" (fun () ->

    (match args with

    (* --------------------------------------------------------- *)
    (* actions, useful to debug subpart *)
    (* --------------------------------------------------------- *)
    | xs when List.mem !action (Common.action_list (all_actions())) ->
        Common.do_action !action xs (all_actions())

    | _ when not (Common.null_string !action) ->
        failwith ("unrecognized action or wrong params: " ^ !action)

    (* --------------------------------------------------------- *)
    (* main entry *)
    (* --------------------------------------------------------- *)
    | x::xs ->
        main_action (x::xs)

    (* --------------------------------------------------------- *)
    (* empty entry *)
    (* --------------------------------------------------------- *)
    | [] ->
        Common.usage usage_msg (options());
        failwith "too few arguments"
    )
  )

(*****************************************************************************)
let _ =
  Common.main_boilerplate (fun () ->
      main ();
  )