From 20b47403dcf1bd1de4c3eb3c02e5b51940b561a2 Mon Sep 17 00:00:00 2001 From: Brian Nguyen Date: Mon, 6 Jul 2026 19:03:33 -0500 Subject: [PATCH 1/2] [None][build] Use cp -rL and skip up-to-date copies in copy_resolving_symlink copy_resolving_symlink was unconditionally deleting and re-copying large directory trees (e.g. CUTLASS headers) on every build. On network-mounted filesystems this can take several minutes even when nothing changed. Add an mtime check: if the destination already exists and is at least as new as the resolved source, skip the copy. The source mtime advances whenever CMake rebuilds the target, so this correctly invalidates the cache on real rebuilds while skipping no-op incremental runs. When a copy is needed, shell out to cp -rL instead of Python copytree. cp uses kernel-level copy primitives that are significantly faster on NFS. Falls back to copytree if cp is unavailable. Signed-off-by: Brian Nguyen --- scripts/build_wheel.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/scripts/build_wheel.py b/scripts/build_wheel.py index b984152c94cb..b8502ddef0e2 100755 --- a/scripts/build_wheel.py +++ b/scripts/build_wheel.py @@ -822,7 +822,12 @@ def symlink_remove_dst_tree(src, dst, dirs_exist_ok=True): # Helper function to resolve symlinks and copy actual content def copy_resolving_symlink(src_path, dst_path): - """Copy file or directory, resolving symlinks to copy actual content.""" + """Copy file or directory, resolving symlinks to copy actual content. + + Skips the copy when dst already exists and is at least as new as src + (mtime comparison). This avoids redundant large tree copies (e.g. + the CUTLASS headers) on incremental rebuilds over slow filesystems. + """ if src_path.is_symlink(): resolved_src = src_path.resolve() else: @@ -830,10 +835,20 @@ def copy_resolving_symlink(src_path, dst_path): if resolved_src.is_dir(): if dst_path.exists(): + if dst_path.stat().st_mtime >= resolved_src.stat().st_mtime: + return # destination is up to date rmtree(dst_path) - # Use symlinks=False (default) to follow symlinks and copy actual content - # This ensures nested symlinks are also resolved - copytree(resolved_src, dst_path, symlinks=False) + # Shell out to cp -rL: dereferences symlinks like copytree(symlinks=False) + # but uses kernel-level copy primitives, which is significantly faster + # than Python's file-by-file copytree on network-mounted filesystems. + # Fall back to copytree if cp is unavailable (non-Linux or minimal containers). + try: + run(["cp", "-rL", + str(resolved_src), + str(dst_path)], + check=True) + except (FileNotFoundError, Exception): + copytree(resolved_src, dst_path, symlinks=False) else: if dst_path.is_dir(): dst_path = dst_path / src_path.name From ed774dca738cd6981278bd4624a5aeb995c73ea6 Mon Sep 17 00:00:00 2001 From: Brian Nguyen Date: Mon, 6 Jul 2026 19:31:14 -0500 Subject: [PATCH 2/2] [None][build] use stamp file and shutil.which in copy_resolving_symlink Replace mtime-on-dst check with a stamp file so partial/interrupted copies are not incorrectly skipped on the next build. Narrow the except clause to FileNotFoundError only so cp errors propagate instead of silently falling back to copytree. Use shutil.which("cp") to resolve the binary path and skip the subprocess path entirely when cp is not available. Signed-off-by: Brian Nguyen --- scripts/build_wheel.py | 36 +++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/scripts/build_wheel.py b/scripts/build_wheel.py index b8502ddef0e2..5d48337d5889 100755 --- a/scripts/build_wheel.py +++ b/scripts/build_wheel.py @@ -824,9 +824,13 @@ def symlink_remove_dst_tree(src, dst, dirs_exist_ok=True): def copy_resolving_symlink(src_path, dst_path): """Copy file or directory, resolving symlinks to copy actual content. - Skips the copy when dst already exists and is at least as new as src - (mtime comparison). This avoids redundant large tree copies (e.g. - the CUTLASS headers) on incremental rebuilds over slow filesystems. + Skips the copy when dst already exists and a stamp file confirms the + previous copy completed successfully, and the stamp is at least as new + as src. Using a stamp (rather than dst mtime) avoids two pitfalls: + - Directory mtime on Linux only reflects direct-child changes, not + deeply nested ones, so a modified source file may not update it. + - An interrupted copy leaves dst with a fresh mtime that would + incorrectly satisfy an mtime check, causing the next build to skip. """ if src_path.is_symlink(): resolved_src = src_path.resolve() @@ -834,21 +838,31 @@ def copy_resolving_symlink(src_path, dst_path): resolved_src = src_path if resolved_src.is_dir(): - if dst_path.exists(): - if dst_path.stat().st_mtime >= resolved_src.stat().st_mtime: + stamp = dst_path.parent / f".{dst_path.name}.stamp" + if dst_path.exists() and stamp.exists(): + if stamp.stat().st_mtime >= resolved_src.stat().st_mtime: return # destination is up to date rmtree(dst_path) + elif dst_path.exists(): + rmtree( + dst_path) # dst exists but no stamp — interrupted copy + stamp.unlink(missing_ok=True) # Shell out to cp -rL: dereferences symlinks like copytree(symlinks=False) # but uses kernel-level copy primitives, which is significantly faster # than Python's file-by-file copytree on network-mounted filesystems. # Fall back to copytree if cp is unavailable (non-Linux or minimal containers). - try: - run(["cp", "-rL", - str(resolved_src), - str(dst_path)], - check=True) - except (FileNotFoundError, Exception): + cp_bin = shutil.which("cp") + if cp_bin is not None: + try: + run([cp_bin, "-rL", + str(resolved_src), + str(dst_path)], + check=True) + except FileNotFoundError: + copytree(resolved_src, dst_path, symlinks=False) + else: copytree(resolved_src, dst_path, symlinks=False) + stamp.touch() else: if dst_path.is_dir(): dst_path = dst_path / src_path.name