rename utils.misc to utils.file_utils; add safe_rename

dataworkspaces/resources/snapshot_utils.py

@@ -12,7 +12,7 @@
 import click
 
 from dataworkspaces.errors import ConfigurationError
-from dataworkspaces.utils.misc import remove_dir_if_empty
+from dataworkspaces.utils.file_utils import remove_dir_if_empty
 
 # Timestamps have the form '2018-09-30T14:09:05'
 TEMPLATE_VAR_PATS = {

dataworkspaces/utils/file_utils.py

@@ -0,0 +1,39 @@
+# Copyright 2018,2019 by MPI-SWS and Data-ken Research. Licensed under Apache 2.0. See LICENSE.txt.
+"""
+File-related utilities
+"""
+
+import os
+from os.path import dirname
+import shutil
+
+from dataworkspaces.errors import ConfigurationError
+
+def remove_dir_if_empty(path, base_dir, verbose=False):
+    """Remove an empty directory and any parents that are empty,
+    up to base_dir.
+    """
+    if path==base_dir:
+        return
+    elif len(os.listdir(path))==0:
+        os.rmdir(path)
+        if verbose:
+            print("Removing (now) empty directory %s" % path)
+        remove_dir_if_empty(dirname(path), base_dir, verbose)
+
+
+def safe_rename(src, dest):
+    """Safe replacement for os.rename(). The problem is that os.rename()
+    does not work across file systems. In that case, you need to actually
+    copy the file
+    """
+    try:
+        os.rename(src, dest)
+    except OSError:
+        try:
+            shutil.copyfile(src, dest)
+            os.remove(src)
+        except Exception as e:
+            raise ConfigurationError("Unable to copy %s to %s: %s"%
+                                     (src, dest, e)) from e
+

dataworkspaces/utils/git_utils.py

@@ -14,7 +14,7 @@
 from .subprocess_utils import \
     find_exe, call_subprocess,\
     call_subprocess_for_rc
-from .misc import remove_dir_if_empty
+from .file_utils import remove_dir_if_empty
 from dataworkspaces.errors import ConfigurationError, InternalError
 
 def is_git_repo(dirpath):

tests/Makefile

@@ -4,7 +4,7 @@ DATAWORKSPACES:=$(shell cd ../dataworkspaces; pwd)
 help:
 	@echo targets are: test clean mypy help install-rclone-deb
 
-UNIT_TESTS=test_git_utils test_move_results test_api test_git_fat_integration test_lineage_utils test_lineage test_snapshots test_jupyter_kit test_push_pull
+UNIT_TESTS=test_git_utils test_move_results test_api test_git_fat_integration test_lineage_utils test_lineage test_snapshots test_jupyter_kit test_push_pull test_file_utils
 
 MYPY_UTILS=workspace_utils.py lineage_utils.py
 MYPY_APIS=lineage.py

tests/test_file_utils.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+"""
+Test file utilities
+"""
+import os.path
+import unittest
+import tempfile
+import sys
+import shutil
+
+TEMPDIR=os.path.abspath(os.path.expanduser(__file__)).replace('.py', '_data')
+
+OTHER_FS='/tmp'
+
+try:
+    import dataworkspaces
+except ImportError:
+    sys.path.append(os.path.abspath(".."))
+
+from dataworkspaces.utils.file_utils import safe_rename
+
+class TestFileUtils(unittest.TestCase):
+    def setUp(self):
+        if os.path.exists(TEMPDIR):
+            shutil.rmtree(TEMPDIR)
+        os.mkdir(TEMPDIR)
+
+    def tearDown(self):
+        if os.path.exists(TEMPDIR):
+            shutil.rmtree(TEMPDIR)
+
+    def test_safe_rename(self):
+        """This test needs to be run in a case where OTHER_FS is on a separate
+        filesystem - the underlying issue is that os.rename() does not work
+        across filesystems and this version does copying as a fallback.
+        """
+        data = "this is a test\n"
+        with tempfile.NamedTemporaryFile(dir=OTHER_FS, delete=False) as testfile:
+            testfile.write(data.encode('utf-8'))
+        try:
+            dest = os.path.join(TEMPDIR, 'data.txt')
+            safe_rename(testfile.name, dest)
+            self.assertTrue(os.path.exists(dest))
+            with open(dest, 'r') as f:
+                data2 = f.read()
+            self.assertEqual(data, data2)
+        finally:
+            if os.path.exists(testfile.name):
+                os.remove(testfile.name)
+
+
+
+if __name__ == '__main__':
+    unittest.main()