Fix the build issues with tests against build_files/ and log_files/ contents, collab with Claude Opus

lukstafi · lukstafi · commit 23c8279f551c · 2025-08-17T18:40:58.000+02:00
Prompt 1: > Can you help me study Dune and improve the build setup for testing generated files? You can join me in reading the Dune documentation, e.g. starting from the page https://dune.readthedocs.io/en/stable/reference/actions/index.html . The problem is that the way I wrote the test/operations/dune file only sporadically makes things work. I added directory dependencies `(target (dir log_files))` and `(target (dir build_files))` to trigger generating files in these directories, relative to the directory executables are run. But the semantics of directory targets is like of any other targets: they can be in an "existing" and "non-existing" state, a rule is only triggered when the directory is missing, not when a file in the directory is missing. The build system encounters a missing file and complains that the directory-building rule did not produce it, but the rule did not run because the directory already existed. Maybe `include_subdirs` is the way? https://dune.readthedocs.io/en/stable/reference/dune/include_subdirs.html The page talks more about OCaml modules, but it should work for any files. Summary 1 by Claude: The issue with your current Dune setup is that directory targets only trigger when the directory doesn't exist. Once created, the rule won't re-run even if files inside are missing, causing sporadic build failures. I've created three improved solutions: 1. dune.improved - Stamp file approach - Uses sentinel/stamp files (.done files) to track when generation is complete - Subsequent rules depend on these stamps instead of directories - Most straightforward and reliable 2. dune.alternative - Multiple strategies - Shows promotion mode to capture generated files - Manifest approach to list generated files - Two-phase build with explicit file listing 3. dune.robust - Clean sentinel-based solution - Creates .done files after successful execution - All dependent rules wait for these sentinels - Guarantees proper ordering and file existence Recommended approach: Use dune.robust - it's the cleanest solution that: - Avoids the directory target problem entirely - Creates explicit dependencies through sentinel files - Ensures files exist before processing them - Works reliably with Dune's dependency tracking The key insight from the Dune documentation is that targets must be statically known and directory targets are experimental with limited semantics. Using sentinel files provides explicit, trackable dependencies that Dune handles well. Final summary by Claude: Updated Files: 1. test/operations/dune - Fixed directory target issues 2. test/einsum/dune - Applied same fix for consistency Key Solution: no-infer Action - Portable: Works on Linux, macOS, and Windows - Clean: Uses Dune's built-in copy action - Robust: Explicitly tells Dune not to track dependencies for side-effect files Pattern Used: (rule (target output.actual) (action (no-infer (progn (run executable --args) (copy generated/file.ext %{target}))))) This approach: - Generates files as side effects - Copies them using Dune's portable copy action - Avoids the "No rule found" errors - Eliminates sporadic failures from directory target semantics The no-infer wrapper is the key - it allows us to work with files that Dune doesn't know about (generated at runtime) while still using Dune's portable actions. Signed-off-by: Lukasz Stafiniak <lukstafi@gmail.com>
diff --git a/test/einsum/dune b/test/einsum/dune
@@ -69,18 +69,14 @@
 
 (rule
  (alias runtest)
- (target
-  (dir build_files))
+ (target inline_permuted_view-c_fwd.ll.actual)
  (action
-  (run
-   %{dep:inline_permuted_view.exe}
-   "--ocannl_output_debug_files_in_build_directory=true")))
-
-(rule
- (deps "build_files/c_fwd.ll")
- (target "inline_permuted_view-c_fwd.ll.actual")
- (action
-  (copy "build_files/c_fwd.ll" "inline_permuted_view-c_fwd.ll.actual")))
+  (no-infer
+   (progn
+    (run
+     %{dep:inline_permuted_view.exe}
+     "--ocannl_output_debug_files_in_build_directory=true")
+    (copy build_files/c_fwd.ll %{target})))))
 
 (rule
  (alias runtest)
diff --git a/test/operations/dune b/test/operations/dune
@@ -16,20 +16,23 @@
   (action
    (run %{dep:../../../arrayjit/bin/read_config.exe} "--read=backend"))))
 
+; Solution: Create intermediate targets that guarantee file generation
+; Step 1: Run the test and create a sentinel file when done
 (rule
- (alias runtest)
- (target
-  (dir log_files))
+ (targets micrograd_demo.done)
  (action
-  (run
-   %{dep:micrograd_demo_logging.exe}
-   "--ocannl_debug_backend=text"
-   "--ocannl_log_file_stem=micrograd_demo_logging"
-   "--ocannl_log_main_domain_to_stdout=false"
-   "--ocannl_debug_log_to_stream_files=true")))
-
+  (progn
+   (run
+    %{dep:micrograd_demo_logging.exe}
+    "--ocannl_debug_backend=text"
+    "--ocannl_log_file_stem=micrograd_demo_logging"
+    "--ocannl_log_main_domain_to_stdout=false"
+    "--ocannl_debug_log_to_stream_files=true")
+   (write-file %{targets} "completed\n"))))
+
+; Step 2: Process the log file, depending on the sentinel
 (rule
- (deps "log_files/%{read:config/ocannl_backend.txt}-0-0.log")
+ (deps micrograd_demo.done)
  (target "%{read:config/ocannl_backend.txt}-0-0.log.actual")
  (action
   (progn
@@ -38,19 +41,19 @@
     " = 0x[0-9a-fA-F]+"
     " = 0xNNNN"
     "log_files/%{read:config/ocannl_backend.txt}-0-0.log"
-    "%{read:config/ocannl_backend.txt}-0-0.log.actual")
+    "%{target}")
    (run
     %{dep:../../tools/minised.exe}
     " = 00[0-9a-fA-F]+"
     " = 0xNNNN"
-    "%{read:config/ocannl_backend.txt}-0-0.log.actual"
-    "%{read:config/ocannl_backend.txt}-0-0.log.actual")
+    "%{target}"
+    "%{target}")
    (run
     %{dep:../../tools/minised.exe}
     "\\([a-z0-9_]+\\)\\[0\\]{=[-.e0-9]+} = "
     "\\1[0]{=MAYBE UNINITIALIZED} = "
-    "%{read:config/ocannl_backend.txt}-0-0.log.actual"
-    "%{read:config/ocannl_backend.txt}-0-0.log.actual"))))
+    "%{target}"
+    "%{target}"))))
 
 (rule
  (alias runtest)
@@ -73,28 +76,28 @@
  (preprocess
   (pps ppx_here ppx_ocannl)))
 
+; Generate both ll files and actual files in one rule using no-infer
 (rule
- (alias runtest)
- (target
-  (dir build_files))
+ (target n3_fwd_with_prec-unoptimized.ll.actual)
  (action
-  (progn
-   (run
-    %{dep:threefry4x32_demo.exe}
-    "--ocannl_output_prec_in_ll_files=true"
-    "--ocannl_output_debug_files_in_build_directory=true")
-   (run
-    %{dep:top_down_prec.exe}
-    "--ocannl_output_prec_in_ll_files=true"
-    "--ocannl_output_debug_files_in_build_directory=true"))))
+  (no-infer
+   (progn
+    (run
+     %{dep:threefry4x32_demo.exe}
+     "--ocannl_output_prec_in_ll_files=true"
+     "--ocannl_output_debug_files_in_build_directory=true")
+    (copy build_files/n3_fwd-unoptimized.ll %{target})))))
 
 (rule
- (deps "build_files/n3_fwd-unoptimized.ll")
- (target "n3_fwd_with_prec-unoptimized.ll.actual")
+ (target top_down_prec-unoptimized.ll.actual)
  (action
-  (copy
-   "build_files/n3_fwd-unoptimized.ll"
-   "n3_fwd_with_prec-unoptimized.ll.actual")))
+  (no-infer
+   (progn
+    (run
+     %{dep:top_down_prec.exe}
+     "--ocannl_output_prec_in_ll_files=true"
+     "--ocannl_output_debug_files_in_build_directory=true")
+    (copy build_files/d_fwd-unoptimized.ll %{target})))))
 
 (rule
  (alias runtest)
@@ -103,14 +106,6 @@
    "n3_fwd_with_prec-unoptimized.ll.expected"
    "n3_fwd_with_prec-unoptimized.ll.actual")))
 
-(rule
- (deps "build_files/d_fwd-unoptimized.ll")
- (target "top_down_prec-unoptimized.ll.actual")
- (action
-  (copy
-   "build_files/d_fwd-unoptimized.ll"
-   "top_down_prec-unoptimized.ll.actual")))
-
 (rule
  (alias runtest)
  (action
@@ -156,4 +151,4 @@
  (modules hello_world_op zero2hero_1of7 primitive_ops test_threefry4x32)
  (preprocess
   (pps ppx_here ppx_expect ppx_inline_test ppx_ocannl))
- (modes best))
+ (modes best))
