From 41d1c3903614a7c7b9945bee22a774a9a028a901 Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Thu, 14 Mar 2019 13:26:10 +0100 Subject: [PATCH] Fix unittest bug when running on nfs filesystems * Retry `rmtree` when an `OSError` with `Errno 16` is thrown, in the custom ReFrame `rmtree` implementation. --- reframe/utility/os_ext.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/reframe/utility/os_ext.py b/reframe/utility/os_ext.py index 5c9e761169..74a5eeec3a 100644 --- a/reframe/utility/os_ext.py +++ b/reframe/utility/os_ext.py @@ -176,13 +176,17 @@ def ignore(dir, contents): def rmtree(*args, max_retries=3, **kwargs): """Persistent version of ``shutil.rmtree()``. - If ``shutil.rmtree()`` with ``ENOTEMPTY``, retry up to ``max_retries`` - times to delete the directory. + If ``shutil.rmtree()`` fails with ``ENOTEMPTY`` or ``EBUSY``, retry up to + ``max_retries`times to delete the directory. This version of ``rmtree()`` is mostly provided to work around a race condition between when ``sacct`` reports a job as completed and when the Slurm epilog runs. See https://github.com/eth-cscs/reframe/issues/291 for more information. + Furthermore, it offers a work around for nfs file systems where a ``.nfs*`` + file may be present during the ``rmtree()`` call which throws a busy + device/resource error. See https://github.com/eth-cscs/reframe/issues/712 + for more information. ``args`` and ``kwargs`` are passed through to ``shutil.rmtree()``. @@ -200,7 +204,7 @@ def rmtree(*args, max_retries=3, **kwargs): except OSError as e: if i == max_retries: raise - elif e.errno == errno.ENOTEMPTY: + elif e.errno in {errno.ENOTEMPTY, errno.EBUSY}: pass else: raise