From 15eda82ee0dddf96653675c8b2cf7463b4f21f8e Mon Sep 17 00:00:00 2001 From: Robert Gibson Date: Wed, 8 Jan 2020 16:16:21 +0100 Subject: [PATCH 01/13] Use speedcopy module for faster copies across network boundaries --- dvc/system.py | 4 ++-- setup.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dvc/system.py b/dvc/system.py index 26e8497cd1..e41e9f8b81 100644 --- a/dvc/system.py +++ b/dvc/system.py @@ -1,7 +1,7 @@ import errno import logging import os -import shutil +import speedcopy from dvc.compat import fspath @@ -17,7 +17,7 @@ def is_unix(): @staticmethod def copy(src, dest): src, dest = fspath(src), fspath(dest) - return shutil.copyfile(src, dest) + return speedcopy.copyfile(src, dest) @staticmethod def hardlink(source, link_name): diff --git a/setup.py b/setup.py index 9c86e09035..38e871ea2e 100644 --- a/setup.py +++ b/setup.py @@ -75,6 +75,7 @@ def run(self): "win-unicode-console>=0.5; sys_platform == 'win32'", "pywin32>=225; sys_platform == 'win32'", "networkx>=2.1,<2.4", + "speedcopy>=2", ] From 262ca923500b56dd66d29e48621f7a9a7a43adf3 Mon Sep 17 00:00:00 2001 From: Robert Gibson Date: Thu, 9 Jan 2020 14:25:01 +0100 Subject: [PATCH 02/13] Use speedcopy module with bugfix --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 38e871ea2e..7cf5031e10 100644 --- a/setup.py +++ b/setup.py @@ -75,7 +75,7 @@ def run(self): "win-unicode-console>=0.5; sys_platform == 'win32'", "pywin32>=225; sys_platform == 'win32'", "networkx>=2.1,<2.4", - "speedcopy>=2", + "speedcopy @ git+https://github.com/rxxg/speedcopy@5", ] From 4c53c05acf1113d06fe0ffd95e7f21105fb1de95 Mon Sep 17 00:00:00 2001 From: Robert Gibson Date: Thu, 9 Jan 2020 17:57:15 +0100 Subject: [PATCH 03/13] Use pyfastcopy for non-Windows systems using Python < 3.8 --- dvc/system.py | 18 +++++++++++++++--- setup.py | 1 + 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/dvc/system.py b/dvc/system.py index e41e9f8b81..2c969ff44f 100644 --- a/dvc/system.py +++ b/dvc/system.py @@ -1,7 +1,7 @@ import errno import logging import os -import speedcopy +import platform from dvc.compat import fspath @@ -17,7 +17,20 @@ def is_unix(): @staticmethod def copy(src, dest): src, dest = fspath(src), fspath(dest) - return speedcopy.copyfile(src, dest) + system = platform.system() + if system == "Windows": + import speedcopy + + return speedcopy.copyfile(src, dest) + else: + import shutil + import sys + + if sys.version < (3, 8): + # Importing the module monkey-patches shutil.copyfile + import pyfastcopy # noqa: F401 + + return shutil.copyfile(src, dest) @staticmethod def hardlink(source, link_name): @@ -125,7 +138,6 @@ def _reflink_linux(src, dst): @staticmethod def reflink(source, link_name): - import platform from dvc.exceptions import DvcException source, link_name = fspath(source), fspath(link_name) diff --git a/setup.py b/setup.py index 7cf5031e10..90d61f1627 100644 --- a/setup.py +++ b/setup.py @@ -76,6 +76,7 @@ def run(self): "pywin32>=225; sys_platform == 'win32'", "networkx>=2.1,<2.4", "speedcopy @ git+https://github.com/rxxg/speedcopy@5", + "pyfastcopy>=1.0.3", ] From e4ad0e48de3c298a222228c4a569e072a71614b6 Mon Sep 17 00:00:00 2001 From: Robert Gibson Date: Fri, 10 Jan 2020 08:59:50 +0100 Subject: [PATCH 04/13] Use pyfastcopy for non-Windows system using Python < 3.8 --- dvc/system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/system.py b/dvc/system.py index 2c969ff44f..79a82582aa 100644 --- a/dvc/system.py +++ b/dvc/system.py @@ -26,7 +26,7 @@ def copy(src, dest): import shutil import sys - if sys.version < (3, 8): + if sys.version_info < (3, 8): # Importing the module monkey-patches shutil.copyfile import pyfastcopy # noqa: F401 From 55d768843cfa5f9abaa46dc968d2532553dad2dd Mon Sep 17 00:00:00 2001 From: Robert Gibson Date: Wed, 8 Jan 2020 16:16:21 +0100 Subject: [PATCH 05/13] Use speedcopy module for faster copies across network boundaries --- dvc/system.py | 4 ++-- setup.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dvc/system.py b/dvc/system.py index d32663b4af..a6fc77bf4a 100644 --- a/dvc/system.py +++ b/dvc/system.py @@ -1,7 +1,7 @@ import errno import logging import os -import shutil +import speedcopy from dvc.compat import fspath from dvc.exceptions import DvcException @@ -18,7 +18,7 @@ def is_unix(): @staticmethod def copy(src, dest): src, dest = fspath(src), fspath(dest) - return shutil.copyfile(src, dest) + return speedcopy.copyfile(src, dest) @staticmethod def hardlink(source, link_name): diff --git a/setup.py b/setup.py index 035253aaaf..e8801b44dc 100644 --- a/setup.py +++ b/setup.py @@ -75,6 +75,7 @@ def run(self): "win-unicode-console>=0.5; sys_platform == 'win32'", "pywin32>=225; sys_platform == 'win32'", "networkx>=2.1,<2.4", + "speedcopy>=2", ] From ff5d5b195f675540a5d68cde0b6cabe0ce4a10c0 Mon Sep 17 00:00:00 2001 From: Robert Gibson Date: Thu, 9 Jan 2020 14:25:01 +0100 Subject: [PATCH 06/13] Use speedcopy module with bugfix --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e8801b44dc..bc36c0d244 100644 --- a/setup.py +++ b/setup.py @@ -75,7 +75,7 @@ def run(self): "win-unicode-console>=0.5; sys_platform == 'win32'", "pywin32>=225; sys_platform == 'win32'", "networkx>=2.1,<2.4", - "speedcopy>=2", + "speedcopy @ git+https://github.com/rxxg/speedcopy@5", ] From 8075750757d44bbde8d1eb792f04e9700ed607ad Mon Sep 17 00:00:00 2001 From: Robert Gibson Date: Thu, 9 Jan 2020 17:57:15 +0100 Subject: [PATCH 07/13] Use pyfastcopy for non-Windows systems using Python < 3.8 --- dvc/system.py | 19 +++++++++++++++---- setup.py | 1 + 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/dvc/system.py b/dvc/system.py index a6fc77bf4a..82dfd2f561 100644 --- a/dvc/system.py +++ b/dvc/system.py @@ -1,7 +1,7 @@ import errno import logging import os -import speedcopy +import platform from dvc.compat import fspath from dvc.exceptions import DvcException @@ -18,7 +18,20 @@ def is_unix(): @staticmethod def copy(src, dest): src, dest = fspath(src), fspath(dest) - return speedcopy.copyfile(src, dest) + system = platform.system() + if system == "Windows": + import speedcopy + + return speedcopy.copyfile(src, dest) + else: + import shutil + import sys + + if sys.version < (3, 8): + # Importing the module monkey-patches shutil.copyfile + import pyfastcopy # noqa: F401 + + return shutil.copyfile(src, dest) @staticmethod def hardlink(source, link_name): @@ -90,8 +103,6 @@ def _reflink_linux(src, dst): @staticmethod def reflink(source, link_name): - import platform - source, link_name = fspath(source), fspath(link_name) system = platform.system() diff --git a/setup.py b/setup.py index bc36c0d244..f6fbea549c 100644 --- a/setup.py +++ b/setup.py @@ -76,6 +76,7 @@ def run(self): "pywin32>=225; sys_platform == 'win32'", "networkx>=2.1,<2.4", "speedcopy @ git+https://github.com/rxxg/speedcopy@5", + "pyfastcopy>=1.0.3", ] From f75d29f84a7f3002e288c92f18898638bd24a368 Mon Sep 17 00:00:00 2001 From: Robert Gibson Date: Fri, 10 Jan 2020 08:59:50 +0100 Subject: [PATCH 08/13] Use pyfastcopy for non-Windows system using Python < 3.8 --- dvc/system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/system.py b/dvc/system.py index 82dfd2f561..7a6a702502 100644 --- a/dvc/system.py +++ b/dvc/system.py @@ -27,7 +27,7 @@ def copy(src, dest): import shutil import sys - if sys.version < (3, 8): + if sys.version_info < (3, 8): # Importing the module monkey-patches shutil.copyfile import pyfastcopy # noqa: F401 From eabc10d8084a7d94f7234935b04a087a2aafc561 Mon Sep 17 00:00:00 2001 From: rxxg Date: Wed, 8 Jan 2020 16:16:21 +0100 Subject: [PATCH 09/13] Use speedcopy module for faster copies across network boundaries --- dvc/system.py | 4 ++-- setup.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dvc/system.py b/dvc/system.py index d32663b4af..a6fc77bf4a 100644 --- a/dvc/system.py +++ b/dvc/system.py @@ -1,7 +1,7 @@ import errno import logging import os -import shutil +import speedcopy from dvc.compat import fspath from dvc.exceptions import DvcException @@ -18,7 +18,7 @@ def is_unix(): @staticmethod def copy(src, dest): src, dest = fspath(src), fspath(dest) - return shutil.copyfile(src, dest) + return speedcopy.copyfile(src, dest) @staticmethod def hardlink(source, link_name): diff --git a/setup.py b/setup.py index 035253aaaf..e8801b44dc 100644 --- a/setup.py +++ b/setup.py @@ -75,6 +75,7 @@ def run(self): "win-unicode-console>=0.5; sys_platform == 'win32'", "pywin32>=225; sys_platform == 'win32'", "networkx>=2.1,<2.4", + "speedcopy>=2", ] From 0c715760686ee5002a0b036a8737386b9bd85a9b Mon Sep 17 00:00:00 2001 From: rxxg Date: Thu, 9 Jan 2020 14:25:01 +0100 Subject: [PATCH 10/13] Use speedcopy module with bugfix --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e8801b44dc..bc36c0d244 100644 --- a/setup.py +++ b/setup.py @@ -75,7 +75,7 @@ def run(self): "win-unicode-console>=0.5; sys_platform == 'win32'", "pywin32>=225; sys_platform == 'win32'", "networkx>=2.1,<2.4", - "speedcopy>=2", + "speedcopy @ git+https://github.com/rxxg/speedcopy@5", ] From 364a6321829678d6d670cdf166e7f7f236d921ab Mon Sep 17 00:00:00 2001 From: rxxg Date: Thu, 9 Jan 2020 17:57:15 +0100 Subject: [PATCH 11/13] Use pyfastcopy for non-Windows systems using Python < 3.8 --- dvc/system.py | 19 +++++++++++++++---- setup.py | 1 + 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/dvc/system.py b/dvc/system.py index a6fc77bf4a..82dfd2f561 100644 --- a/dvc/system.py +++ b/dvc/system.py @@ -1,7 +1,7 @@ import errno import logging import os -import speedcopy +import platform from dvc.compat import fspath from dvc.exceptions import DvcException @@ -18,7 +18,20 @@ def is_unix(): @staticmethod def copy(src, dest): src, dest = fspath(src), fspath(dest) - return speedcopy.copyfile(src, dest) + system = platform.system() + if system == "Windows": + import speedcopy + + return speedcopy.copyfile(src, dest) + else: + import shutil + import sys + + if sys.version < (3, 8): + # Importing the module monkey-patches shutil.copyfile + import pyfastcopy # noqa: F401 + + return shutil.copyfile(src, dest) @staticmethod def hardlink(source, link_name): @@ -90,8 +103,6 @@ def _reflink_linux(src, dst): @staticmethod def reflink(source, link_name): - import platform - source, link_name = fspath(source), fspath(link_name) system = platform.system() diff --git a/setup.py b/setup.py index bc36c0d244..f6fbea549c 100644 --- a/setup.py +++ b/setup.py @@ -76,6 +76,7 @@ def run(self): "pywin32>=225; sys_platform == 'win32'", "networkx>=2.1,<2.4", "speedcopy @ git+https://github.com/rxxg/speedcopy@5", + "pyfastcopy>=1.0.3", ] From 4ac124f5f62687d3204f8bbffb86a960e2f118b3 Mon Sep 17 00:00:00 2001 From: rxxg Date: Fri, 10 Jan 2020 08:59:50 +0100 Subject: [PATCH 12/13] Use pyfastcopy for non-Windows system using Python < 3.8 --- dvc/system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/system.py b/dvc/system.py index 82dfd2f561..7a6a702502 100644 --- a/dvc/system.py +++ b/dvc/system.py @@ -27,7 +27,7 @@ def copy(src, dest): import shutil import sys - if sys.version < (3, 8): + if sys.version_info < (3, 8): # Importing the module monkey-patches shutil.copyfile import pyfastcopy # noqa: F401 From 0838cbd98e0cd7a3957fb05e70ce5e7ad0a9e6dd Mon Sep 17 00:00:00 2001 From: rxxg Date: Thu, 9 Jan 2020 17:57:15 +0100 Subject: [PATCH 13/13] Use pyfastcopy for non-Windows systems using Python < 3.8 --- dvc/system.py | 19 +++++++++++++++---- setup.py | 1 + 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/dvc/system.py b/dvc/system.py index a6fc77bf4a..7a6a702502 100644 --- a/dvc/system.py +++ b/dvc/system.py @@ -1,7 +1,7 @@ import errno import logging import os -import speedcopy +import platform from dvc.compat import fspath from dvc.exceptions import DvcException @@ -18,7 +18,20 @@ def is_unix(): @staticmethod def copy(src, dest): src, dest = fspath(src), fspath(dest) - return speedcopy.copyfile(src, dest) + system = platform.system() + if system == "Windows": + import speedcopy + + return speedcopy.copyfile(src, dest) + else: + import shutil + import sys + + if sys.version_info < (3, 8): + # Importing the module monkey-patches shutil.copyfile + import pyfastcopy # noqa: F401 + + return shutil.copyfile(src, dest) @staticmethod def hardlink(source, link_name): @@ -90,8 +103,6 @@ def _reflink_linux(src, dst): @staticmethod def reflink(source, link_name): - import platform - source, link_name = fspath(source), fspath(link_name) system = platform.system() diff --git a/setup.py b/setup.py index bc36c0d244..f6fbea549c 100644 --- a/setup.py +++ b/setup.py @@ -76,6 +76,7 @@ def run(self): "pywin32>=225; sys_platform == 'win32'", "networkx>=2.1,<2.4", "speedcopy @ git+https://github.com/rxxg/speedcopy@5", + "pyfastcopy>=1.0.3", ]