multi-thread mv

chapmanjacobd · Jun 14, 2024 · ae22dc8 · ae22dc8
1 parent 7a38b11
commit ae22dc8
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 16 deletions.
diff --git a/xklb/folders/merge_mv.py b/xklb/folders/merge_mv.py
@@ -9,7 +9,6 @@ def parse_args():
     parser.add_argument("--copy", "--cp", "-c", action="store_true", help="Copy instead of move")
     arggroups.clobber(parser)
     arggroups.debug(parser)
-    parser.set_defaults(threads=1)
 
     arggroups.paths_or_stdin(parser, destination=True)
     parser.add_argument("destination", help="Destination directory")
@@ -43,7 +42,9 @@ def gen_src_dest(args, sources, destination):
                     file_dest = os.path.join(file_dest, os.path.basename(source))
                 file_dest = os.path.join(file_dest, os.path.relpath(p, source))
 
-                yield p, file_dest
+                src, dest = devices.clobber(args, p, file_dest)
+                if src:
+                    yield src, dest
 
         else:
             file_dest = destination
@@ -52,7 +53,9 @@ def gen_src_dest(args, sources, destination):
             if path_utils.is_folder_dest(source, file_dest):
                 file_dest = os.path.join(file_dest, os.path.basename(source))
 
-            yield source, file_dest
+            src, dest = devices.clobber(args, source, file_dest)
+            if src:
+                yield src, dest
 
 
 def mmv_folders(args, mv_fn, sources, destination):
@@ -64,14 +67,9 @@ def mmv_folders(args, mv_fn, sources, destination):
     else:
         sources = (os.path.realpath(s) for s in sources)
 
-    def move_file(src, dest):
-        src, dest = devices.clobber(args, src, dest)
-        if src:
-            mv_fn(args, src, dest)
-
-    with concurrent.futures.ThreadPoolExecutor(max_workers=1) as ex:
+    with concurrent.futures.ThreadPoolExecutor(max_workers=args.threads) as ex:
         for f in concurrent.futures.as_completed(
-            [ex.submit(move_file, src, dest) for src, dest in gen_src_dest(args, sources, destination)]
+            [ex.submit(mv_fn, args, src, dest) for src, dest in gen_src_dest(args, sources, destination)]
         ):
             f.result()
 

diff --git a/xklb/folders/mergerfs_cp.py b/xklb/folders/mergerfs_cp.py
@@ -19,15 +19,14 @@
 
 from xklb import usage
 from xklb.folders import merge_mv
-from xklb.utils import arggroups, argparse_utils, consts, devices, processes
+from xklb.utils import arggroups, argparse_utils, consts, processes
 
 
 def parse_args():
     parser = argparse_utils.ArgumentParser(usage=usage.mergerfs_cp)
     arggroups.clobber(parser)
     parser.set_defaults(file_over_file="delete-dest-hash rename-dest")
     arggroups.debug(parser)
-    parser.set_defaults(threads=1)
 
     arggroups.paths_or_stdin(parser, destination=True)
     parser.add_argument("destination", help="Destination directory")
@@ -75,10 +74,7 @@ def mergerfs_cp_file(args, merger_fs_src, destination):
         if os.path.exists(source):
             found_file = True
             destination = os.path.join(srcmount, os.path.relpath(destination, args.mergerfs_mount))
-
-            source, destination = devices.clobber(args, source, destination)
-            if source:
-                processes.cmd(*args.cp_args, source, destination, strict=False, quiet=False, error_verbosity=2)
+            processes.cmd(*args.cp_args, source, destination, strict=False, quiet=False, error_verbosity=2)
 
     if not found_file:
         print(f"Could not find srcmount of {merger_fs_src}")