Merge pull request #1323 from adisbladis/scp-to-rsync

Use rsync instead of scp for file copying

default.nix

@@ -18,6 +18,7 @@ in pkgs.poetry2nix.mkPoetryApplication {
 
   propagatedBuildInputs = [
     pkgs.openssh
+    pkgs.rsync
   ];
 
   nativeBuildInputs = [

nixops/backends/__init__.py

@@ -419,27 +419,35 @@ def copy_closure_to(self, path):
             env=env,
         )
 
-    def get_scp_name(self):
+    def _get_scp_name(self) -> str:
         ssh_name = self.get_ssh_name()
         # ipv6 addresses have to be wrapped in brackets for scp
         if ":" in ssh_name:
             return "[%s]" % (ssh_name)
         return ssh_name
 
-    def upload_file(self, source, target, recursive=False):
+    def _fmt_rsync_command(self, *args: str, recursive: bool = False) -> List[str]:
         master = self.ssh.get_master()
-        cmdline = ["scp"] + self.get_ssh_flags(True) + master.opts
+
+        ssh_cmdline: List[str] = ["ssh"] + self.get_ssh_flags() + master.opts
+        cmdline = ["rsync", "-e", nixops.util.shlex_join(ssh_cmdline)]
         if recursive:
             cmdline += ["-r"]
-        cmdline += [source, "root@" + self.get_scp_name() + ":" + target]
+
+        cmdline.extend(args)
+
+        return cmdline
+
+    def upload_file(self, source: str, target: str, recursive: bool = False):
+        cmdline = self._fmt_rsync_command(
+            source, "root@" + self._get_scp_name() + ":" + target, recursive=recursive,
+        )
         return self._logged_exec(cmdline)
 
-    def download_file(self, source, target, recursive=False):
-        master = self.ssh.get_master()
-        cmdline = ["scp"] + self.get_ssh_flags(True) + master.opts
-        if recursive:
-            cmdline += ["-r"]
-        cmdline += ["root@" + self.get_scp_name() + ":" + source, target]
+    def download_file(self, source: str, target: str, recursive: bool = False):
+        cmdline = self._fmt_rsync_command(
+            "root@" + self._get_scp_name() + ":" + source, target, recursive=recursive,
+        )
         return self._logged_exec(cmdline)
 
     def get_console_output(self):

nixops/util.py

@@ -18,6 +18,7 @@
 import re
 import typeguard
 import inspect
+import shlex
 from typing import (
     Callable,
     List,
@@ -33,6 +34,7 @@
     Hashable,
     TypeVar,
     Generic,
+    Iterable,
 )
 
 # the following ansi_ imports are for backwards compatability. They
@@ -43,6 +45,11 @@
 from io import StringIO
 
 
+def shlex_join(split_command: Iterable[str]) -> str:
+    """Backport of shlex.join from python 3.8"""
+    return " ".join(shlex.quote(arg) for arg in split_command)
+
+
 devnull = open(os.devnull, "r+")
 
 

shell.nix

@@ -13,6 +13,7 @@ in pkgs.mkShell {
     })
     pkgs.openssh
     pkgs.poetry
+    pkgs.rsync  # Included by default on NixOS
   ];
 
   shellHook = ''