Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Optimize move.move_file for moving files between different OSFS instances. #523

Merged
merged 42 commits into from
May 2, 2022
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
db837c5
add optimization to move
tfeldmann Feb 8, 2022
d6db460
update contributors
tfeldmann Feb 8, 2022
43dc890
add test
tfeldmann Feb 8, 2022
cb49b6e
optimization not dependent of OSFS anymore
tfeldmann Mar 24, 2022
1f7b0ff
Merge branch 'master' into move_file-optimization
tfeldmann Mar 24, 2022
448f15f
remove unneeded import
tfeldmann Mar 24, 2022
68846ce
lock src and dst for move operation
tfeldmann Mar 24, 2022
003e1b7
handle ValueError
tfeldmann Mar 24, 2022
fa1803b
fix unbound var
tfeldmann Mar 24, 2022
86adaa8
set "read_only": True in `WrapReadOnly.getmeta()`
tfeldmann Mar 25, 2022
014123b
handle ro FSs in move_file optimization
tfeldmann Mar 25, 2022
14d4fa2
add test for two different MemFS instances
tfeldmann Mar 25, 2022
a97c0c3
support FS URLs
tfeldmann Mar 25, 2022
bbe4389
clear use of fs url instead of str
tfeldmann Mar 25, 2022
69022aa
add test for read-only sources
tfeldmann Mar 25, 2022
545e016
more tests for read only sources
tfeldmann Mar 25, 2022
fd577df
clarify fallthrough and add writeable=True
tfeldmann Mar 25, 2022
4edb1e0
cleanup destination if move fails
tfeldmann Mar 25, 2022
20203d9
update changelog
tfeldmann Mar 25, 2022
ad7a970
raise exc in mange_fs if fs is not writeable
tfeldmann Mar 25, 2022
1267194
don't run tests twice in parameterized_class
tfeldmann Mar 25, 2022
b0963bf
remove unneeded import
tfeldmann Mar 25, 2022
4a102ce
update changelog
tfeldmann Mar 25, 2022
763efc2
rename test case
tfeldmann Mar 25, 2022
e490452
formatting
tfeldmann Mar 25, 2022
0c02f53
add python2.7 commonpath backport
tfeldmann Mar 25, 2022
9be0381
fix path in ResourceReadOnly exception
tfeldmann Mar 25, 2022
baf7019
test_move cleanup
tfeldmann Mar 25, 2022
69d1906
check time on if self.preserve_time is set
tfeldmann Mar 25, 2022
7993af0
update changelog wording
tfeldmann Mar 25, 2022
9cfc5dd
revert import order
tfeldmann Mar 25, 2022
54d3374
ignore flake8 C401 in _pathcompat
tfeldmann Mar 25, 2022
c051f23
fix codestyle and typecheck errors
tfeldmann Mar 25, 2022
a549250
removed duplicated test code
tfeldmann Mar 25, 2022
44cbe31
add `cleanup_dest_on_error` in `move_file`
tfeldmann Mar 25, 2022
696c4ca
use non-overlapping osfs urls
tfeldmann Mar 25, 2022
891d1fc
use OSFS instead of fs_open, rename cleanup_dst_on_error param
tfeldmann Mar 28, 2022
4d80b22
fix `supports_rename` in `WrapReadOnly`
tfeldmann Mar 28, 2022
78f6696
cleanup test_move filenames
tfeldmann Mar 28, 2022
46fbc71
Fix circular import between `fs.base`, `fs.osfs` and `fs.move`
althonos Mar 30, 2022
5675f84
cleanup and docs
tfeldmann Apr 1, 2022
da33922
test `src` first, then `dst`
tfeldmann Apr 1, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion fs/move.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ def move_file(
preserve_time (bool): If `True`, try to preserve mtime of the
resources (defaults to `False`).
cleanup_dst_on_error (bool): If `True`, tries to delete the file copied to
`dst_fs` if deleting the file from `src_fs` fails (defaults to `True`).
``dst_fs`` if deleting the file from ``src_fs`` fails (defaults to `True`).

"""
with manage_fs(src_fs, writeable=True) as _src_fs:
Expand Down
39 changes: 17 additions & 22 deletions tests/test_move.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
except ImportError:
import mock

from parameterized import parameterized_class
from parameterized import parameterized, parameterized_class


import fs.move
Expand Down Expand Up @@ -80,10 +80,10 @@ def test_move_dir(self):

fs.move.move_dir(src_fs, "/foo", dst_fs, "/", preserve_time=self.preserve_time)

self.assertTrue(dst_fs.isdir("bar"))
self.assertTrue(dst_fs.isfile("bar/baz.txt"))
self.assertFalse(src_fs.exists("foo"))
self.assertTrue(src_fs.isfile("test.txt"))
self.assertTrue(dst_fs.isdir("bar"))
self.assertTrue(dst_fs.isfile("bar/baz.txt"))

if self.preserve_time:
dst_file2_info = dst_fs.getinfo("bar/baz.txt", namespaces)
Expand All @@ -104,16 +104,16 @@ def test_move_file_fs_urls(self):
# create a temp dir to work on
with open_fs("temp://") as tmp:
path = tmp.getsyspath("/")
subdir_src = tmp.makedir("subdir_src")
subdir_src.writetext("file.txt", "Content")
tmp.makedir("subdir_src")
tmp.writetext("subdir_src/file.txt", "Content")
tmp.makedir("subdir_dst")
fs.move.move_file(
"osfs://" + join(path, "subdir_src"),
"file.txt",
"osfs://" + join(path, "subdir_dst"),
"target.txt",
)
self.assertFalse(subdir_src.exists("file.txt"))
self.assertFalse(tmp.exists("subdir_src/file.txt"))
self.assertEqual(tmp.readtext("subdir_dst/target.txt"), "Content")

def test_move_file_same_fs_read_only_source(self):
Expand All @@ -124,10 +124,10 @@ def test_move_file_same_fs_read_only_source(self):
dst = tmp.makedir("sub")
with self.assertRaises(ResourceReadOnly):
fs.move.move_file(src, "file.txt", dst, "target_file.txt")
self.assertTrue(src.exists("file.txt"))
self.assertFalse(
dst.exists("target_file.txt"), "file should not have been copied over"
)
self.assertTrue(src.exists("file.txt"))

def test_move_file_read_only_mem_source(self):
tfeldmann marked this conversation as resolved.
Show resolved Hide resolved
with open_fs("mem://") as src, open_fs("mem://") as dst:
Expand All @@ -136,40 +136,35 @@ def test_move_file_read_only_mem_source(self):
src_ro = read_only(src)
with self.assertRaises(ResourceReadOnly):
fs.move.move_file(src_ro, "file.txt", dst_sub, "target.txt")
self.assertTrue(src.exists("file.txt"))
self.assertFalse(
dst_sub.exists("target.txt"), "file should not have been copied over"
)
self.assertTrue(src.exists("file.txt"))

def test_move_file_read_only_mem_dest(self):
with open_fs("mem://") as src, open_fs("mem://") as dst:
src.writetext("file.txt", "Content")
dst_ro = read_only(dst)
with self.assertRaises(ResourceReadOnly):
fs.move.move_file(src, "file.txt", dst_ro, "target.txt")
self.assertTrue(src.exists("file.txt"))
self.assertFalse(
dst_ro.exists("target.txt"), "file should not have been copied over"
)
self.assertTrue(src.exists("file.txt"))

def test_move_file_cleanup_on_error(self):
with open_fs("mem://") as src, open_fs("mem://") as dst:
src.writetext("file.txt", "Content")
with mock.patch.object(src, "remove") as mck:
mck.side_effect = FSError
with self.assertRaises(FSError):
fs.move.move_file(src, "file.txt", dst, "target.txt")
self.assertTrue(src.exists("file.txt"))
self.assertFalse(dst.exists("target.txt"))

def test_move_file_no_cleanup_on_error(self):
@parameterized.expand([(True,), (False,)])
def test_move_file_cleanup_on_error(self, cleanup):
with open_fs("mem://") as src, open_fs("mem://") as dst:
src.writetext("file.txt", "Content")
with mock.patch.object(src, "remove") as mck:
mck.side_effect = FSError
with self.assertRaises(FSError):
fs.move.move_file(
src, "file.txt", dst, "target.txt", cleanup_dst_on_error=False
src,
"file.txt",
dst,
"target.txt",
cleanup_dst_on_error=cleanup,
)
self.assertTrue(src.exists("file.txt"))
self.assertTrue(dst.exists("target.txt"))
self.assertEqual(not dst.exists("target.txt"), cleanup)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Might be marginally more readable as

self.assertEqual(dst.exists("target.txt"), not cleanup)