From 09223442f572695c22bdf2959206411e764aa85c Mon Sep 17 00:00:00 2001 From: rxxg Date: Wed, 8 Jan 2020 16:16:21 +0100 Subject: [PATCH 1/5] Use speedcopy module for faster copies across network boundaries --- dvc/system.py | 4 ++-- setup.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dvc/system.py b/dvc/system.py index d32663b4af..a6fc77bf4a 100644 --- a/dvc/system.py +++ b/dvc/system.py @@ -1,7 +1,7 @@ import errno import logging import os -import shutil +import speedcopy from dvc.compat import fspath from dvc.exceptions import DvcException @@ -18,7 +18,7 @@ def is_unix(): @staticmethod def copy(src, dest): src, dest = fspath(src), fspath(dest) - return shutil.copyfile(src, dest) + return speedcopy.copyfile(src, dest) @staticmethod def hardlink(source, link_name): diff --git a/setup.py b/setup.py index f8c4fda597..4aaf0b7b1b 100644 --- a/setup.py +++ b/setup.py @@ -75,6 +75,7 @@ def run(self): "win-unicode-console>=0.5; sys_platform == 'win32'", "pywin32>=225; sys_platform == 'win32'", "networkx>=2.1,<2.4", + "speedcopy>=2", ] From 811fc5317c4e18f402aa0631a44faabb38eea576 Mon Sep 17 00:00:00 2001 From: rxxg Date: Thu, 9 Jan 2020 14:25:01 +0100 Subject: [PATCH 2/5] Use speedcopy module with bugfix --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4aaf0b7b1b..5008f9ce38 100644 --- a/setup.py +++ b/setup.py @@ -75,7 +75,7 @@ def run(self): "win-unicode-console>=0.5; sys_platform == 'win32'", "pywin32>=225; sys_platform == 'win32'", "networkx>=2.1,<2.4", - "speedcopy>=2", + "speedcopy @ git+https://github.com/rxxg/speedcopy@5", ] From 674541f9225ee96acd237bfb8791ace3738ffa49 Mon Sep 17 00:00:00 2001 From: rxxg Date: Thu, 9 Jan 2020 17:57:15 +0100 Subject: [PATCH 3/5] Use pyfastcopy for non-Windows systems using Python < 3.8 --- dvc/system.py | 19 +++++++++++++++---- setup.py | 1 + 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/dvc/system.py b/dvc/system.py index a6fc77bf4a..7a6a702502 100644 --- a/dvc/system.py +++ b/dvc/system.py @@ -1,7 +1,7 @@ import errno import logging import os -import speedcopy +import platform from dvc.compat import fspath from dvc.exceptions import DvcException @@ -18,7 +18,20 @@ def is_unix(): @staticmethod def copy(src, dest): src, dest = fspath(src), fspath(dest) - return speedcopy.copyfile(src, dest) + system = platform.system() + if system == "Windows": + import speedcopy + + return speedcopy.copyfile(src, dest) + else: + import shutil + import sys + + if sys.version_info < (3, 8): + # Importing the module monkey-patches shutil.copyfile + import pyfastcopy # noqa: F401 + + return shutil.copyfile(src, dest) @staticmethod def hardlink(source, link_name): @@ -90,8 +103,6 @@ def _reflink_linux(src, dst): @staticmethod def reflink(source, link_name): - import platform - source, link_name = fspath(source), fspath(link_name) system = platform.system() diff --git a/setup.py b/setup.py index 5008f9ce38..13792fbc44 100644 --- a/setup.py +++ b/setup.py @@ -76,6 +76,7 @@ def run(self): "pywin32>=225; sys_platform == 'win32'", "networkx>=2.1,<2.4", "speedcopy @ git+https://github.com/rxxg/speedcopy@5", + "pyfastcopy>=1.0.3", ] From 8fca139246cc15e7ca0ce772d9e76ca8862b8e56 Mon Sep 17 00:00:00 2001 From: rxxg Date: Wed, 15 Jan 2020 12:50:43 +0100 Subject: [PATCH 4/5] Use released version of speedcopy library --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 13792fbc44..a3fe8421f6 100644 --- a/setup.py +++ b/setup.py @@ -75,7 +75,7 @@ def run(self): "win-unicode-console>=0.5; sys_platform == 'win32'", "pywin32>=225; sys_platform == 'win32'", "networkx>=2.1,<2.4", - "speedcopy @ git+https://github.com/rxxg/speedcopy@5", + "speedcopy>=2.0.1", "pyfastcopy>=1.0.3", ] From 083a630b44287acc323602d7060ea676db086dad Mon Sep 17 00:00:00 2001 From: rxxg Date: Thu, 16 Jan 2020 17:08:07 +0100 Subject: [PATCH 5/5] Fallback to standard Python code in case of missing packages --- dvc/system.py | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/dvc/system.py b/dvc/system.py index 7a6a702502..c300daac57 100644 --- a/dvc/system.py +++ b/dvc/system.py @@ -2,13 +2,30 @@ import logging import os import platform +import shutil from dvc.compat import fspath from dvc.exceptions import DvcException - logger = logging.getLogger(__name__) +if platform.system() == "Windows": + try: + import speedcopy + + speedcopy.patch_copyfile() + except ImportError: + pass +else: + import sys + + if sys.version_info < (3, 8): + try: + # Importing the module monkey-patches shutil.copyfile + import pyfastcopy # noqa: F401 + except ImportError: + pass + class System(object): @staticmethod @@ -18,20 +35,7 @@ def is_unix(): @staticmethod def copy(src, dest): src, dest = fspath(src), fspath(dest) - system = platform.system() - if system == "Windows": - import speedcopy - - return speedcopy.copyfile(src, dest) - else: - import shutil - import sys - - if sys.version_info < (3, 8): - # Importing the module monkey-patches shutil.copyfile - import pyfastcopy # noqa: F401 - - return shutil.copyfile(src, dest) + return shutil.copyfile(src, dest) @staticmethod def hardlink(source, link_name):