Version 1.0.0-rc20 is now available at http://coccinelle.lip6.fr/down…

…load.php

This version is not compatible with previous ones, in that initialize and
finalize rules now must have a region for declaring metavariables, just
like other rules.

Unfortunately, I neglected to update the demos accordingly, so there are a
number of demos that now don't work.  I will make a new release (rc21)
that corrects that shortly.

rc20 should be much faster if you need to include a ot of header files.
In particular, we found substantial performance improvements (of around
30%) with the arguments --recursive-includes --relax-include path.

Another significant improvement is that it is now possible to remove
variable declarations that include more than one variable.  For example:

@@ @@
- int x;

should transform

int a,x,y;

into

int a,y;

And

@@ identifier x; @@
- int x;

should completely remove int a,x,y;

changes.txt

@@ -59,6 +59,8 @@
 - Support variable declarations in the first element of a for loop header.
 - Support decimal type with the --ibm option
 - Support some matching on pragmas
+- Initialize and finalize rules contain a metavariable declaration region.
+- Added documentation about iteration.
 
 ** Features:
 - Preserve spacing before // comments when not at the beginning of a line
@@ -153,6 +155,14 @@
   to Nic Volanschi for the bug report.
 - Eliminate the reliance on grep in most cases for worth trying, to improve
   performance.  Thanks to Johannes Berg for the suggestion.
+- Allow multidecls when removing variable names or when modifying type.
+- Take fresh declarations into account when computing get_constants to
+  create dependencies
+- Leave ifdefs-like directives in unparsed code, if they don't have
+  beginning or ending counterparts.
+- Improved interpret_cocci_grep to try to avoid no accumulating lots of
+  anded data.
+- Improve type inference performance when many files included by using maps.
 
 ** Bugfix:
 - Corrected parsing of script strings delimited by a single quote.  Thanks
@@ -300,6 +310,9 @@
   Tegawende Bissyande for finding the problem.
 - Collect all types in get_constants when types mentioned on
   metavariables. Thanks to Michael Stefaniuc for the report.
+- Try to fix pretty printing of directives.
+- Allow renaming types in multidecls, including renames in initializers.
+- Save virtual identifiers from the command line for use in finalize code.
 
 * 0.2.5
 ** Language:

cocci.ml

@@ -986,8 +986,7 @@ let python_code =
     local_python_code ^
     "cocci = Cocci()\n"
 
-let make_init lang code rule_info =
-  let mv = [] in
+let make_init lang code rule_info mv =
   {
   scr_ast_rule = (lang, mv, [], code);
   language = lang;
@@ -1034,11 +1033,10 @@ let prepare_cocci ctls free_var_lists negated_pos_lists
               scr_rule_info = build_rule_info name deps;
 	    }
           in ScriptRuleCocciInfo r
-      | Ast_cocci.InitialScriptRule (name,lang,deps,code) ->
-	  let r = make_init lang code (build_rule_info name deps) in
-	    InitialScriptRuleCocciInfo r
-      | Ast_cocci.FinalScriptRule (name,lang,deps,code) ->
-	  let mv = [] in
+      | Ast_cocci.InitialScriptRule (name,lang,deps,mv,code) ->
+	  let r = make_init lang code (build_rule_info name deps) mv in
+	  InitialScriptRuleCocciInfo r
+      | Ast_cocci.FinalScriptRule (name,lang,deps,mv,code) ->
           let r =
             {
               scr_ast_rule = (lang, mv, [], code);
@@ -1826,7 +1824,8 @@ let initial_final_bigloop2 ty rebuild r =
 
       adjust_pp_with_indent (fun () ->
 	Format.force_newline();
-	Pretty_print_cocci.unparse(rebuild r.scr_ast_rule r.scr_rule_info.dependencies));
+	Pretty_print_cocci.unparse
+	  (rebuild r.scr_ast_rule r.scr_rule_info.dependencies));
     end;
 
   match r.language with
@@ -1906,11 +1905,12 @@ let pre_engine2 (coccifile, isofile) =
     with Not_found ->
       begin
 	Iteration.initialization_stack :=
-	  ((rlang,rname),!Flag.defined_virtual_rules) ::
+	  ((rlang,rname),
+	   (!Flag.defined_virtual_rules,!Flag.defined_virtual_env)) ::
 	  !Iteration.initialization_stack;
 	initial_final_bigloop Initial
-	  (fun (x,_,_,y) -> fun deps ->
-	    Ast_cocci.InitialScriptRule(rname,x,deps,y))
+	  (fun (x,mvs,_,y) -> fun deps ->
+	    Ast_cocci.InitialScriptRule(rname,x,deps,mvs,y))
 	  r
       end in
 
@@ -1945,7 +1945,7 @@ let pre_engine2 (coccifile, isofile) =
 	  used_after = [];
 	  ruleid = (-1);
 	  was_matched = ref false;} in
-      runrule (make_init lgg "" rule_info))
+      runrule (make_init lgg "" rule_info []))
     uninitialized_languages;
 
   ((cocci_infos,parse_strings),constants)
@@ -2028,8 +2028,9 @@ let full_engine a b =
 
 let post_engine2 (cocci_infos,_) =
   List.iter
-    (function ((language,_),virt_rules) ->
+    (function ((language,_),(virt_rules,virt_env)) ->
       Flag.defined_virtual_rules := virt_rules;
+      Flag.defined_virtual_env := virt_env;
       let _ =
 	List.fold_left
 	  (function languages ->
@@ -2038,9 +2039,9 @@ let post_engine2 (cocci_infos,_) =
 		  (if r.language = language && List.mem r.language languages
 		  then failwith ("double finalizer found for "^r.language));
 		  initial_final_bigloop Final
-		    (fun (x,_,_,y) -> fun deps ->
+		    (fun (x,mvs,_,y) -> fun deps ->
 		      Ast_cocci.FinalScriptRule(r.scr_rule_info.rulename,
-						x,deps,y))
+						x,deps,mvs,y))
 		    r;
 		  r.language::languages
 	      | _ -> languages)

commons/common.ml

@@ -423,7 +423,7 @@ let xxx_once f s =
     if not (Hashtbl.mem _already_printed s)
     then begin
       Hashtbl.add _already_printed s true;
-      f ("(ONCE) " ^ s);
+      f ("(ONCE) " ^ s)
     end
 
 let pr2_once s = xxx_once pr2 s
@@ -3529,7 +3529,11 @@ let (with_open_outfile_append: filename -> (((string -> unit) * out_channel) ->
  * question: can we have a signal and so exn when in a exn handler ?
  *)
 
-let interval_timer = ref true
+let interval_timer = ref (
+  try
+    ignore(Unix.getitimer Unix.ITIMER_VIRTUAL);
+    true
+  with Unix.Unix_error(_, _, _) -> false)
 
 let timeout_function timeoutval = fun f ->
   try
@@ -4716,15 +4720,20 @@ let assoc_with_err_msg k l =
 module IntMap = Map.Make
     (struct
       type t = int
-      let compare = compare
+      let compare (x : int) (y : int) = Pervasives.compare x y
     end)
 let intmap_to_list m = IntMap.fold (fun id v acc -> (id, v) :: acc) m []
 let intmap_string_of_t f a = "<Not Yet>"
 
 module IntIntMap = Map.Make
     (struct
       type t = int * int
-      let compare = compare
+      let compare ((x1, y1) : int * int) ((x2, y2) : int * int) =
+	let cmp_x = Pervasives.compare x1 x2 in
+	if cmp_x <> 0 then
+	  cmp_x
+	else
+	  Pervasives.compare y1 y2
 end)
 
 let intintmap_to_list m = IntIntMap.fold (fun id v acc -> (id, v) :: acc) m []

configure

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for coccinelle 1.0.0-rc19.
+# Generated by GNU Autoconf 2.69 for coccinelle 1.0.0-rc20.
 #
 # Report bugs to <cocci@systeme.lip6.fr>.
 #
@@ -639,8 +639,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='coccinelle'
 PACKAGE_TARNAME='coccinelle'
-PACKAGE_VERSION='1.0.0-rc19'
-PACKAGE_STRING='coccinelle 1.0.0-rc19'
+PACKAGE_VERSION='1.0.0-rc20'
+PACKAGE_STRING='coccinelle 1.0.0-rc20'
 PACKAGE_BUGREPORT='cocci@systeme.lip6.fr'
 PACKAGE_URL='http://coccinelle.lip6.fr/'
 
@@ -1498,7 +1498,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures coccinelle 1.0.0-rc19 to adapt to many kinds of systems.
+\`configure' configures coccinelle 1.0.0-rc20 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1564,7 +1564,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of coccinelle 1.0.0-rc19:";;
+     short | recursive ) echo "Configuration of coccinelle 1.0.0-rc20:";;
    esac
   cat <<\_ACEOF
 
@@ -1724,7 +1724,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-coccinelle configure 1.0.0-rc19
+coccinelle configure 1.0.0-rc20
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -1816,7 +1816,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by coccinelle $as_me 1.0.0-rc19, which was
+It was created by coccinelle $as_me 1.0.0-rc20, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
@@ -2688,7 +2688,7 @@ fi
 
 # Define the identity of the package.
  PACKAGE='coccinelle'
- VERSION='1.0.0-rc19'
+ VERSION='1.0.0-rc20'
 
 
 cat >>confdefs.h <<_ACEOF
@@ -17610,7 +17610,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by coccinelle $as_me 1.0.0-rc19, which was
+This file was extended by coccinelle $as_me 1.0.0-rc20, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -17668,7 +17668,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-coccinelle config.status 1.0.0-rc19
+coccinelle config.status 1.0.0-rc20
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 

demos/initvirt.c

@@ -0,0 +1 @@
+int main () { }

demos/initvirt.cocci

@@ -0,0 +1,37 @@
+@initialize:ocaml@
+x << virtual.x;
+@@
+
+let _ = Printf.printf "ocaml start: x is %s\n" x
+
+@initialize:python@
+x << virtual.x;
+@@
+
+print "python start: x is %s" % (x)
+
+@@
+@@
+
+foo();
+
+@script:ocaml@
+@@
+()
+
+@finalize:ocaml@
+x << virtual.x;
+y << virtual.y;
+@@
+
+Printf.printf "ocaml end: x is %s\n" x;
+Printf.printf "ocaml end: y is %s\n" y
+
+@finalize:python@
+x << virtual.x;
+y << virtual.y;
+z << virtual.z;
+@@
+
+print "python end: x is %s" % (x)
+print "python end: y is %s" % (y)

docs/manual/cocci_syntax.tex

@@ -388,14 +388,19 @@ \section{Metavariables for scripts}
   \RULE{\rt{script\_metavariables}}
   \CASE{@ script:\NT{language} \OPT{\NT{rulename}} \OPT{depends on \NT{dep}} @
         \any{\NT{script\_metadecl}} @@}
-  \CASE{@ initialize:\NT{language} \OPT{depends on \NT{dep}} @}
-  \CASE{@ finalize:\NT{language} \OPT{depends on \NT{dep}} @}
+  \CASE{@ initialize:\NT{language} \OPT{depends on \NT{dep}} @
+        \any{\NT{script\_virt\_metadecl}} @@}
+  \CASE{@ finalize:\NT{language} \OPT{depends on \NT{dep}} @
+        \any{\NT{script\_virt\_metadecl}} @@}
 
   \RULE{\rt{language}} \CASE{python} \CASE{ocaml}
 
   \RULE{\rt{script\_metadecl}}
   \CASE{\T{id} <{}< \T{rulename\_id}.\T{id} ;}
   \CASE{\T{id} ;}
+
+  \RULE{\rt{script\_virt\_metadecl}}
+  \CASE{\T{id} <{}< virtual.\T{id} ;}
 \end{grammar}
 
 Currently, the only scripting languages that are supported are Python and
@@ -411,6 +416,12 @@ \section{Metavariables for scripts}
 variables initialized by the other script rules, allowing information to be
 transmitted from the matching process to the finalize rule.
 
+Initialize and finalize rules do have access to virtual metavariables,
+using the usual syntax.  As for other scripting language rules, the rule
+is not run (and essentially does not exist) if some of the required virtual
+metavariables are not bound.  In ocaml, a warning is printed in this case.
+An example is found in demos/initvirt.cocci.
+
 A script metavariable that does not specify an origin, using \texttt{<<},
 is newly declared by the script.  This metavariable should be assigned to a
 string and can be inherited by subsequent rules as an identifier.  In
@@ -1163,6 +1174,64 @@ \section{Command-line semantic match}
 The generated semantic match behaves as though there were a \texttt{*} in front
 of every token.
 
+\section{Iteration}
+
+It is possible to iterate Coccinelle, giving the subsequent iterations a
+different set of virtual rules or virtual identifier bindings.  And example
+is found in {\tt demos/iteration.cocci}.  The example shown there is as
+follows:
+
+\begin{quote}
+\begin{verbatim}
+virtual after_start
+
+@initialize:ocaml@
+
+let tbl = Hashtbl.create(100)
+
+let add_if_not_present from f file =
+try let _ = Hashtbl.find tbl (f,file) in ()
+with Not_found ->
+   Hashtbl.add tbl (f,file) file;
+   let it = new iteration() in
+   (match file with
+     Some fl -> it#set_files [fl]
+   | None -> ());
+   it#add_virtual_rule After_start;
+   it#add_virtual_identifier Err_ptr_function f;
+   it#register()
+\end{verbatim}
+\end{quote}
+
+The virtual rule {\tt after\_start} is used to distinguish between the
+first iteration (in which it is not considered to have matched) and all
+others.  This is done by not mentioning {\tt after\_start} in the command
+line, but adding it on each iteration.
+
+The main code for performing the iteration is found in the function {\tt
+  add\_if\_not\_present}, between the lines calling {\tt new iteration} and
+{\tt register}.  {\tt New iteration} creates a structure representing the
+new iteration.  {\tt set\_files} sets the list of files to be considered on
+the new iteration.  If this function is not called, the new iteration
+treats the same files as the current iteration.  {\tt add\_virtual\_rule A}
+has the same effect as putting {\tt -D a} on the command line.  Note that
+the first letter of the rule name is capitalized, although this is not done
+elsewhere.  {\tt add\_virtual\_identifier X v} has the same effect as
+putting {\tt -D x=v} on the command line.  Note again the case change.
+{\tt extend\_virtual\_identifiers()} (not shown) preserves all virtual
+identifiers of the current iteration that are not overridden by calls to 
+{\tt add\_virtual\_identifier}.  Finally, the call to {\tt register} queues
+the collected information to trigger a new iteration at some time in the
+future.
+
+Modification is not allowed when using iteration.  Thus, it is required to
+use the {\tt --no-show-diff}, unless the semantic patch contains {\tt *}s
+(a semantic match rather than a semantic patch).
+
+The remainder of the code above uses a hash table to ensure that the same
+information is not enqueued more than once.  Coccinelle itself provides no
+support for this.
+
 %%% Local Variables:
 %%% mode: LaTeX
 %%% TeX-master: "main_grammar"