Merge branch 'main' into melange/error-out-melc-missing

* main: (58 commits)
  test: formatting of alternative dune files (ocaml#6567)
  refactor: remove Modules.is_empty (ocaml#6564)
  refactor: module kinds (ocaml#6562)
  refactor(coq): resolve lack of coqc properly
  Cache file contents in action builder by name. (ocaml#6555)
  fix: re-enable dune on older macos sdk's (ocaml#6515)
  fix: do not hide lib interface module (ocaml#6549)
  test: remove pkg-config output for reproducibility (ocaml#6543)
  melange: add test for ocaml flags (ocaml#6548)
  fix: improve virtual library error messages
  test: virtual library and impl locations
  test: alias module regression (ocaml#6544)
  refactor(merlin): dump config sub command (ocaml#6547)
  refactor: simplify merlin (ocaml#6508)
  chore(nix): use nix-overlays for the slim devShell (ocaml#6546)
  fix: module compilation rule env (ocaml#6527)
  chore: update nix (ocaml#6536)
  fix: merlin rules with pp's (ocaml#6474)
  Call [Dune_util.Log.init] as soon as possible (ocaml#6542)
  refactor: speed up stdlib build (ocaml#6524)
  ...

CHANGES.md

@@ -1,10 +1,37 @@
 Unreleased
 ----------
 
+- Do not shadow library interface modules (#6549, fixes #6545, @rgrinberg)
+
+- Move `$ dune ocaml-merlin -dump-config=$dir` to `$ dune ocaml merlin
+  dump-config $dir`. (#6547, @rgrinberg)
+
+- Allow compilation rules to be impacted by `(env ..)` stanzas that modify the
+  environment or set binaries. (#6527, @rgrinberg)
+
+- Fix status line enabled when ANSI colors are forced. (#6503, @MisterDA)
+
 - Coq native mode is now automatically detected by Dune starting with Coq lang
   0.7. `(mode native)` has been deprecated in favour of detection from the
   configuration of Coq. (#6409, @Alizter)
 
+- Print "Leaving Directory" whenever "Entering Directory" is printed. (#6149,
+  fixes #138, @cpitclaudel, @rgrinberg)
+
+- Allow `$ dune ocaml dump-dot-merlin` to run in watch mode. Also this command
+  shouldn't print "Entering Directory" mesages. (#6497, @rgrinberg)
+
+- Fix build with MSVC compiler (#6517, @nojb)
+
+- `dune clean` should no longer fail under Windows due to the inability to
+  remove the `.lock` file. Also, bring the implementation of the global lock
+  under Windows closer to that of Unix. (#6523, @nojb)
+
+- Remove "Entering Directory" messages for `$ dune install`. (#6513,
+  @rgrinberg)
+
+- Fix configurator when using the MSVC compiler (#6538, fixes #6537, @nojb)
+
 3.6.0 (2022-11-14)
 ------------------
 
@@ -178,6 +205,8 @@ Unreleased
 3.4.0 (20-07-2022)
 ------------------
 
+- Do not ignore `C-c` when running `$ dune subst` (#5892, @rgrinberg)
+
 - Make `dune describe` correctly handle overlapping implementations
   for virtual libraries (#5971, fixes #5747, @esope)
 
@@ -623,11 +652,13 @@ Unreleased
 - Fields allowed in the config file are now also allowed in the
   workspace file (#4426, @jeremiedimino)
 
-- Add options to control how Dune should handle stdout and stderr of
-  actions when then succeed. It is now possible to ask Dune to ignore
-  the stdout of actions when they succeed or to request that the
-  stderr of actions must be empty. This allows to reduce the noise of
-  large builds (#4422, #4515, @jeremiedimino)
+- Add CLI flags `--action-<outputs>-on-success ...` (where `<outputs>` is
+  `stdout` or `stderr`) to control how Dune should handle `stdout` and `stderr` of
+  actions when they succeed. It is now possible to ask Dune to ignore the `stdout`
+  of actions when they succeed or to request that the `stderr` of actions must be
+  empty. It is also possible to set these options in the `config` and/or
+  `dune-workspace` files with `(action_<outputs>_on_success ...)`. This feature
+  allows you to reduce the noise of large builds (#4422, #4515, @jeremiedimino)
 
 - The `@all` alias no longer depends directly on copies of files from the source
   directory (#4461, @nojb)
@@ -3061,7 +3092,7 @@ Unreleased
 
 - Print `Entering directory '...'` when the workspace root is not the
   current directory. This allows Emacs and Vim to know where relative
-  filenames should be interpreted from. Fixes #138
+  filenames should be interpreted from. (fixes #138, @jeremiedimino)
 
 - Fix a bug related to `menhir` stanzas: `menhir` stanzas with a
   `merge_into` field that were in `jbuild` files in sub-directories

Makefile

@@ -26,7 +26,8 @@ ppxlib \
 result \
 ctypes \
 "utop>=2.6.0" \
-"melange>=0.3.0"
+"melange>=0.3.0" \
+"mel>=0.3.0" \
 
 # Dependencies recommended for developing dune locally,
 # but not wanted in CI
@@ -125,7 +126,7 @@ all-supported-ocaml-versions: $(BIN)
 
 .PHONY: clean
 clean:
-	rm -rf _boot _build $(BIN)
+	rm -rf _boot _build
 
 distclean: clean
 	rm -f src/dune_rules/setup.ml

bench.Dockerfile

@@ -1,4 +1,4 @@
 FROM ocaml/opam:debian-10-ocaml-4.14
-RUN opam depext patdiff.v0.15.0
+RUN opam depext -u patdiff.v0.15.0
 COPY --chown=opam:opam . bench-dir
 WORKDIR bench-dir

bin/common.ml

@@ -80,14 +80,19 @@ let rpc t =
   | `Forbid_builds -> `Forbid_builds
   | `Allow rpc -> `Allow (Lazy.force rpc)
 
-let forbid_builds t = { t with rpc = `Forbid_builds }
+let forbid_builds t = { t with rpc = `Forbid_builds; no_print_directory = true }
+
+let signal_watcher t =
+  match t.rpc with
+  | `Allow _ -> `Yes
+  | `Forbid_builds ->
+    (* if we aren't building anything, then we don't mind interrupting dune immediately *)
+    `No
 
 let stats t = t.stats
 
 let insignificant_changes t = t.insignificant_changes
 
-let set_print_directory t b = { t with no_print_directory = not b }
-
 let set_promote t v = { t with promote = Some v }
 
 (* To avoid needless recompilations under Windows, where the case of
@@ -111,7 +116,7 @@ let normalize_path path =
 
 let print_entering_message c =
   let cwd = Path.to_absolute_filename Path.root in
-  if cwd <> Fpath.initial_cwd && not c.no_print_directory then
+  if cwd <> Fpath.initial_cwd && not c.no_print_directory then (
     (* Editors such as Emacs parse the output of the build system and interpret
        filenames in error messages relative to where the build system was
        started.
@@ -144,12 +149,18 @@ let print_entering_message c =
             in
             loop ".." (Filename.dirname s)))
     in
-    Console.print [ Pp.verbatim (sprintf "Entering directory '%s'" dir) ]
+    Console.print [ Pp.verbatim (sprintf "Entering directory '%s'" dir) ];
+    at_exit (fun () ->
+        flush stdout;
+        Console.print [ Pp.verbatim (sprintf "Leaving directory '%s'" dir) ]))
 
 let init ?log_file c =
   if c.root.dir <> Filename.current_dir_name then Sys.chdir c.root.dir;
   Path.set_root (normalize_path (Path.External.cwd ()));
   Path.Build.set_build_dir (Path.Outside_build_dir.of_string c.build_dir);
+  (* Once we have the build directory set, initialise the logging. We can't do
+     this earlier, because the build log typically goes into [_build/log]. *)
+  Dune_util.Log.init () ?file:log_file;
   (* We need to print this before reading the workspace file, so that the editor
      can interpret errors in the workspace file. *)
   print_entering_message c;
@@ -162,10 +173,9 @@ let init ?log_file c =
   in
   let config =
     Dune_config.adapt_display config
-      ~output_is_a_tty:(Lazy.force Ansi_color.stderr_supports_color)
+      ~output_is_a_tty:(Lazy.force Ansi_color.output_is_a_tty)
   in
   Dune_config.init config;
-  Dune_util.Log.init () ?file:log_file;
   Dune_engine.Execution_parameters.init
     (let open Memo.O in
     let+ w = Dune_rules.Workspace.workspace () in

bin/common.mli

@@ -14,6 +14,8 @@ val rpc :
 
 val forbid_builds : t -> t
 
+val signal_watcher : t -> [ `Yes | `No ]
+
 val stats : t -> Dune_stats.t option
 
 val print_metrics : t -> bool
@@ -62,9 +64,6 @@ val term_with_default_root_is_cwd : t Cmdliner.Term.t
 
 val envs : Cmdliner.Cmd.Env.info list
 
-(** Set whether Dune should print the "Entering directory '<dir>'" message *)
-val set_print_directory : t -> bool -> t
-
 val set_promote : t -> Dune_engine.Clflags.Promote.t -> t
 
 val debug_backtraces : bool Cmdliner.Term.t

bin/describe.ml

@@ -225,7 +225,6 @@ module Crawl = struct
   module Deps : sig
     val read :
          options:options
-      -> use_pp:bool
       -> obj_dir:Path.Build.t Obj_dir.t
       -> modules:Modules.t
       -> Module.t
@@ -255,23 +254,11 @@ module Crawl = struct
         subject to a preprocessing phase. Two lists are returned. The first list
         contains the direct dependencies of the interface file, whereas the
         second list contains the direct dependencies of the implementation file. *)
-    let read ~options ~use_pp ~obj_dir ~modules unit =
+    let read ~options ~obj_dir ~modules unit =
       let no_deps = ([], []) in
       match options.with_deps with
       | false -> Memo.return (Action_builder.return no_deps)
       | true -> (
-        let unit =
-          (* This translation is necessary to handle files that are not in the
-             "ocaml" dialect, such as Reason files *)
-          Module.ml_source unit
-        in
-        let unit =
-          (* For files that are subject to a preprocessing phase, only the
-             preprocessed files have an attached ocamldep rule: the source files
-             have no such rule attached. In such a case, we refer to the
-             preprocessed module instead of the source module. *)
-          if use_pp then Module.pped unit else unit
-        in
         Memo.return
         @@
         match Module.kind unit with
@@ -323,23 +310,6 @@ module Crawl = struct
         in
         module_ ~obj_dir ~deps_for_intf ~deps_for_impl m :: acc)
 
-  (** [module_uses_pp per_module_pp module_] tells whether the module [module_]
-      needs some pre-processing stage, according to the per-module information
-      [per_module_pp]. The per-module information can be retrieved from the
-      library or executable [module_] belongs to. *)
-  let module_uses_pp
-      (per_module_pp :
-        Preprocess.With_instrumentation.t Preprocess.Per_module.t)
-      (module_ : Module.t) : bool =
-    let open Preprocess in
-    match
-      Per_module.find (Module.name module_)
-        (Per_module.without_instrumentation per_module_pp)
-    with
-    | No_preprocessing -> false
-    | Future_syntax _ | Action _ -> true
-    | Pps pps -> not @@ List.is_empty @@ pps.Pps.pps
-
   (** Builds a workspace item for the provided executables object *)
   let executables sctx ~options ~project ~dir (exes : Dune_file.Executables.t) :
       (Descr.Item.t * Lib.Set.t) option Memo.t =
@@ -348,9 +318,18 @@ module Crawl = struct
       Dir_contents.get sctx ~dir >>= Dir_contents.ocaml
       >>| Ml_sources.modules_and_obj_dir ~for_:(Exe { first_exe })
     in
+
+    let pp_map =
+      Staged.unstage
+      @@
+      let version = (Super_context.context sctx).version in
+      Preprocessing.pped_modules_map
+        (Preprocess.Per_module.without_instrumentation exes.buildable.preprocess)
+        version
+    in
     let deps_of module_ =
-      let use_pp = module_uses_pp exes.buildable.preprocess module_ in
-      Deps.read ~options ~use_pp ~obj_dir ~modules:modules_ module_
+      let module_ = pp_map module_ in
+      Deps.read ~options ~obj_dir ~modules:modules_ module_
     in
     let obj_dir = Obj_dir.of_local obj_dir in
     let* scope =
@@ -392,12 +371,18 @@ module Crawl = struct
           >>= Dir_contents.ocaml
           >>| Ml_sources.modules_and_obj_dir ~for_:(Library name)
           >>= fun (modules_, obj_dir_) ->
+          let pp_map =
+            Staged.unstage
+            @@
+            let version = (Super_context.context sctx).version in
+            Preprocessing.pped_modules_map
+              (Preprocess.Per_module.without_instrumentation
+                 (Lib_info.preprocess info))
+              version
+          in
           let deps_of module_ =
-            let use_pp =
-              module_uses_pp (Lib_info.preprocess @@ Lib.info lib) module_
-            in
-            Deps.read ~options ~use_pp ~obj_dir:obj_dir_ ~modules:modules_
-              module_
+            Deps.read ~options ~obj_dir:obj_dir_ ~modules:modules_
+              (pp_map module_)
           in
           modules ~obj_dir ~deps_of modules_
       in

bin/import.ml

@@ -149,8 +149,9 @@ module Scheduler = struct
     let stats = Common.stats common in
     let config =
       let insignificant_changes = Common.insignificant_changes common in
+      let signal_watcher = Common.signal_watcher common in
       Dune_config.for_scheduler dune_config stats ~insignificant_changes
-        ~signal_watcher:`Yes
+        ~signal_watcher
     in
     let f =
       match Common.rpc common with
@@ -170,9 +171,10 @@ module Scheduler = struct
     in
     let stats = Common.stats common in
     let config =
+      let signal_watcher = Common.signal_watcher common in
       let insignificant_changes = Common.insignificant_changes common in
       Dune_config.for_scheduler dune_config stats ~insignificant_changes
-        ~signal_watcher:`Yes
+        ~signal_watcher
     in
     let file_watcher = Common.file_watcher common in
     let run () =

bin/ocaml_cmd.ml

@@ -9,4 +9,5 @@ let group =
     ; Ocaml_merlin.Dump_dot_merlin.command
     ; Top.command
     ; Top.module_command
+    ; Ocaml_merlin.group
     ]