ENH start re-writting delete_folder

joblib · Dec 23, 2019 · 17110f0 · 17110f0
1 parent c9d0840
commit 17110f0
Show file tree

Hide file tree

Showing 4 changed files with 17 additions and 8 deletions.
diff --git a/joblib/_memmapping_reducer.py b/joblib/_memmapping_reducer.py
@@ -417,7 +417,8 @@ def _cleanup():
         delete_folder = __import__(
             pool_module_name, fromlist=['delete_folder']).delete_folder
         try:
-            delete_folder(pool_folder)
+            # delete_folder(pool_folder)
+            pass
         except WindowsError:
             warnings.warn("Failed to clean temporary folder: {}"
                           .format(pool_folder))

diff --git a/joblib/disk.py b/joblib/disk.py
@@ -8,13 +8,15 @@
 # License: BSD Style, 3 clauses.
 
 
+import atexit
 import os
 import sys
 import time
 import errno
 import shutil
 import warnings
 
+from .externals.loky.backend import resource_tracker
 
 try:
     WindowsError
@@ -110,16 +112,22 @@ def delete_folder(folder_path, onerror=None):
             # allow the rmtree to fail once, wait and re-try.
             # if the error is raised again, fail
             err_count = 0
+            files = os.listdir(folder_path)
+            did_error = False
             while True:
                 try:
                     shutil.rmtree(folder_path, False, None)
                     break
                 except (OSError, WindowsError):
                     err_count += 1
                     if err_count > RM_SUBDIRS_N_RETRY:
-                        warnings.warn(
-                            "Unable to delete folder {} after {} tentatives."
-                            .format(folder_path, RM_SUBDIRS_N_RETRY))
-                        # raise
-                        break
+                        atexit.register(resource_tracker.maybe_unlink,
+                                        (folder_path, "folder"))
+                        did_error = True
+
                     time.sleep(RM_SUBDIRS_RETRY_TIME)
+            if not did_error:
+                print("OKOKOK\n\n\n\n\n")
+                for file in files:
+                    resource_tracker.unregister(
+                        os.path.join(folder_path, file), rtype="file")
diff --git a/joblib/executor.py b/joblib/executor.py
@@ -61,7 +61,7 @@ def apply_async(self, func, args):
 
     def terminate(self):
         self._executor.shutdown()
-        delete_folder(self._temp_folder)
+        # delete_folder(self._temp_folder)
 
     def map(self, f, *args):
         res = self._executor.map(f, *args)

diff --git a/joblib/pool.py b/joblib/pool.py
@@ -326,4 +326,4 @@ def terminate(self):
                     if i + 1 == n_retries:
                         warnings.warn("Failed to terminate worker processes in"
                                       " multiprocessing pool: %r" % e)
-        delete_folder(self._temp_folder)
+        # delete_folder(self._temp_folder)