Age and number of authors layer (using git blame), backend part

Summary:
It could be useful in pfff-web to visualize the parts of our codebase where
a file has been modified by many authors (such code could be
more fragile), and where a file has not been modified for a long time.

Another patch will integrate the layers in the pfff-web UI.

Task ID: #

Blame Rev:

Reviewers: sgrimm, eletuchy

CC:

Test Plan:
build layer:
$ ./pfff_db_heavy -gen_nbauthors_layer facebook/tests/mini_www /tmp/layer_nbauthors.json
visualize layer:
$ ./codemap.opt -with_layer /tmp/layer_nbauthors.json facebook/tests/mini_www/

build age layer:
$ ./pfff_db_heavy -gen_age_layer facebook/tests/mini_www /tmp/layer_age.json
visualize:
$ ./codemap.opt -with_layer /tmp/layer_age.json facebook/tests/mini_www/

both visualization seems ok. I've started to compute the layer
for www but git blame on 70000 files takes quite some time. Maybe
it will be finished tomorrow ...

I've also used the layer on the pfff code itself, to see what is hot right now
in pfff, and it seems quite consistent with what I have in mind :)

Revert Plan:

Tags:

- begin *PUBLIC* platform impact section -
Bugzilla: #
- end platform impact -

DiffCamp Revision: 193090

commons/common.mli

@@ -1085,6 +1085,9 @@ val last_day_in_week_of_day : float_time -> float_time
 val day_secs: float_time
 
 val rough_days_since_jesus : date_dmy -> days
+(* to get a positive numbers the second date must be more recent than
+ * the first.
+ *)
 val rough_days_between_dates : date_dmy -> date_dmy -> days
 
 val string_of_unix_time_lfs : Unix.tm -> string

h_program-visual/Makefile

@@ -9,6 +9,7 @@ TARGET=lib
 SRC= flag_program_visual.ml \
      treemap_pl.ml \
      layer_archi.ml \
+     layer_vcs.ml \
      test_program_visual.ml
 
 

h_program-visual/layer_vcs.ml

@@ -0,0 +1,218 @@
+(* Yoann Padioleau
+ *
+ * Copyright (C) 2010 Facebook
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * version 2.1 as published by the Free Software Foundation, with the
+ * special exception on linking described in file license.txt.
+ * 
+ * This library is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the file
+ * license.txt for more details.
+ *)
+
+open Common 
+
+(*****************************************************************************)
+(* Prelude *)
+(*****************************************************************************)
+
+(* 
+ * todo? not sure how to transform those strings below in types so that more
+ * checking is done at compile-time. We could define some AuthorMoreThan20
+ * constructors but we will need to write some boilerplate code
+ * to transform those constructors into strings (which are used
+ * for the "legend" menu in codemap and the pfff-web UI).
+ * Moreover there will be no guarentees that functions like
+ * property_of_nb_authors covers the whole spectrum of constructors.
+ * One good thing we would get is that in the layer generation
+ * we could not generate property that do not exist (right now
+ * it's easy to make a typo in the string and the compiler will not
+ * complain).
+ *)
+
+(*****************************************************************************)
+(* Types and constants *)
+(*****************************************************************************)
+
+let properties_nb_authors = [
+  "authors > 20", "red3";
+  "authors > 10", "red1";
+  "authors > 5", "orange";
+  "authors = 5", "yellow";
+  "authors = 4", "YellowGreen";
+  "authors = 3", "green";
+  "authors = 2", "aquamarine3";
+  "authors = 1", "cyan";
+
+  (* empty files *)
+  "authors = 0", "white";
+]
+
+let properties_age = [
+  "age > 5 years", "blue";
+  "age > 3 years", "DeepSkyBlue1";
+  "age > 1 year",  "cyan";
+  "age > 6 months", "aquamarine3";
+  "age > 3 months", "green";
+  "age > 2 months", "YellowGreen";
+  "age > 1 month", "yellow";
+  "age > 2 weeks", "orange";
+  "age > 1 week", "red1";
+  "age last week", "red3";
+
+  (* empty files *)
+  "no info", "white";
+]
+
+let property_of_nb_authors n =
+  match n with
+  | _ when n > 20 -> "authors > 20"
+  | _ when n > 10 -> "authors > 10"
+  | _ when n > 5 -> "authors > 5"
+  | _ when n <= 5 -> spf "authors = %d" n
+  | _ -> raise Impossible
+
+let property_of_age (Common.Days n) =
+  match n with
+  | _ when n > 5 * 365 -> "age > 5 years"
+  | _ when n > 3 * 365 -> "age > 3 years"
+  | _ when n > 1 * 365 -> "age > 1 year"
+  | _ when n > 6 * 30 -> "age > 6 months"
+  | _ when n > 3 * 30 -> "age > 3 months"
+  | _ when n > 2 * 30 -> "age > 2 months"
+  | _ when n > 1 * 30 -> "age > 1 month"
+  | _ when n > 2 * 7 -> "age > 2 weeks"
+  | _ when n > 1 * 7 -> "age > 1 week"
+  | _ -> "age last week"
+
+(*****************************************************************************)
+(* Main entry point *)
+(*****************************************************************************)
+let gen_nbauthors_layer dir ~output =
+  let dir = Common.realpath dir in
+
+  let files = Common.files_of_dir_or_files_no_vcs_nofilter [dir] in
+  (* filter the cached annotation files generated by Git.annotate below *)
+  let files = files +> Common.exclude (fun f -> f =~ ".*.git_annot$") in
+
+  let layer = { Layer_code.
+     files = files +> Common.index_list_and_total +> 
+      List.map (fun (file, i, total) ->
+        pr2 (spf "processing: %s (%d/%d)" file i total);
+
+        let readable_file = Common.filename_without_leading_path dir file in
+
+        let annots = Git.annotate 
+          ~basedir:dir ~use_cache:true 
+          ~use_dash_C:false (* too slow *)
+          readable_file
+        in
+        let nbauthors = 
+          annots +> Array.to_list 
+          (* don't count the first entry which is the annotation for line 0
+           * which is a dummy value. See git.ml
+           *)
+          +> List.tl 
+          +> List.map (fun (_version, Lib_vcs.Author s, _data) -> s)
+          +> Common.uniq
+          +> List.length
+        in
+        let property = property_of_nb_authors nbauthors in
+
+        readable_file,
+        { Layer_code.
+          (* don't display anything at the line microlevel *)
+          micro_level = [];
+
+          macro_level = [property, 1.];
+        }
+      );
+      kinds = properties_nb_authors;
+  }
+  in
+  pr2 ("generating layer in " ^ output);
+  Layer_code.save_layer layer output
+
+
+let gen_age_layer dir ~output =
+  let dir = Common.realpath dir in
+
+  let files = Common.files_of_dir_or_files_no_vcs_nofilter [dir] in
+  (* filter the cached annotation files generated by Git.annotate below *)
+  let files = files +> Common.exclude (fun f -> f =~ ".*.git_annot$") in
+
+  let layer = { Layer_code.
+     files = files +> Common.index_list_and_total +> 
+      List.map (fun (file, i, total) ->
+        pr2 (spf "processing: %s (%d/%d)" file i total);
+
+        let readable_file = Common.filename_without_leading_path dir file in
+
+        let annots = Git.annotate 
+          ~basedir:dir ~use_cache:true 
+          ~use_dash_C:false (* too slow *)
+          readable_file
+        in
+
+        let annots = 
+          annots +> Array.to_list 
+           (* don't count the first entry which is the annotation for
+            * line 0 which is a dummy value. See git.ml
+            *)
+           +> List.tl 
+        in
+
+        let property = 
+          match annots with
+          | [] -> "no info"
+          | xs ->
+           (* could also decide to use the average date of the file instead *)
+           let max_date_dmy = 
+            xs
+            +> List.map (fun (_version, Lib_vcs.Author _, date_dmy) -> date_dmy)
+            +> Common.maximum_dmy
+           in
+           pr2_gen max_date_dmy;
+           let now_dmy = 
+             Common.today () 
+             +> Common.floattime_to_unixtime +> Common.unixtime_to_dmy 
+           in
+
+           let age_in_days = 
+             Common.rough_days_between_dates max_date_dmy now_dmy
+           in
+           pr2_gen age_in_days;
+
+           property_of_age age_in_days
+        in
+
+        readable_file,
+        { Layer_code.
+          (* don't display anything at the line microlevel for now.
+           * could display the age of each line.
+           *)
+          micro_level = [];
+
+          macro_level = [property, 1.];
+        }
+      );
+      kinds = properties_age;
+  }
+  in
+  pr2 ("generating layer in " ^ output);
+  Layer_code.save_layer layer output
+
+
+(*****************************************************************************)
+(* Actions *)
+(*****************************************************************************)
+
+let actions () = [
+  "-gen_nbauthors_layer", " <git dir> <layerfile>",
+  Common.mk_action_2_arg (fun dir output -> gen_nbauthors_layer dir ~output);
+  "-gen_age_layer", " <git dir> <layerfile>",
+  Common.mk_action_2_arg (fun dir output -> gen_age_layer dir ~output);
+]

h_program-visual/layer_vcs.mli

@@ -0,0 +1,8 @@
+
+val gen_age_layer: 
+  Common.path -> output:Common.filename -> unit
+
+val gen_nbauthors_layer: 
+  Common.path -> output:Common.filename -> unit
+
+val actions : unit -> Common.cmdline_actions

h_version-control/git.ml

@@ -86,7 +86,7 @@ let annotate_regexp =
 
 
 (* related?  git blame and git pickaxe ? *)
-let annotate2 ?(basedir="") ?(use_cache=false) filename = 
+let annotate2 ?(basedir="") ?(use_cache=false) ?(use_dash_C=true) filename = 
 
   let full_filename = Filename.concat basedir filename in
 
@@ -97,7 +97,10 @@ let annotate2 ?(basedir="") ?(use_cache=false) filename =
      * adding HEAD so that can get the full information of a file that
      * has been modified in the working tree.
      *)
-    let cmd = (goto_dir basedir ^ "git annotate -C HEAD -- "^filename^" 2>&1")
+    let cmd = (goto_dir basedir ^ 
+                  spf "git annotate %s HEAD -- %s 2>&1"
+                  (if use_dash_C then "-C" else "")
+                  filename)
     in
     (* pr2 cmd; *)
     let xs = Common.cmd_to_list cmd in
@@ -121,8 +124,9 @@ let annotate2 ?(basedir="") ?(use_cache=false) filename =
     Array.of_list (dummy_annotation::annots)
   )
 
-let annotate ?basedir ?use_cache a = 
-  Common.profile_code "Git.annotate" (fun () -> annotate2 ?basedir ?use_cache a)
+let annotate ?basedir ?use_cache ?use_dash_C a = 
+  Common.profile_code "Git.annotate" (fun () -> 
+    annotate2 ?basedir ?use_cache ?use_dash_C a)
 
 (* ------------------------------------------------------------------------ *)
 let annotate_raw ?(basedir="") filename = 

h_version-control/git.mli

@@ -4,8 +4,11 @@ open Common
 
 (* operations on a singular file *)
 
+(* note that returned array is 0-indexed but the first entry is
+ * a dummy value.
+ *)
 val annotate : 
-  ?basedir:string -> ?use_cache:bool ->
+  ?basedir:string -> ?use_cache:bool -> ?use_dash_C:bool ->
   Common.filename -> Lib_vcs.line_annotation array
 val date_file_creation: 
   ?basedir:string -> Common.filename -> Common.date_dmy

main_db_heavy.ml

@@ -121,6 +121,7 @@ let all_actions () =
   Layer_coverage_php.actions () ++
   Layer_xhprof.actions () ++
   Layer_cyclomatic_php.actions () ++
+  Layer_vcs.actions () ++
   []
 
 let options () =