Ignore gitignore when rsyncing flambe (#40)

* Ignore gitignore when rsyncing flambe and fix gitignore

* Update docs and fix flake8

* Hot fix

* Update bandit

* Fix tests

.gitignore

@@ -274,18 +274,6 @@ tags
 # Persistent undo
 [._]*.un~
 
-### VirtualEnv ###
-# Virtualenv
-# http://iamzed.com/2009/05/07/a-primer-on-virtualenv/
-[Bb]in
-[Ii]nclude
-[Ll]ib
-[Ll]ib64
-[Ll]ocal
-[Ss]cripts
-pyvenv.cfg
-pip-selfcheck.json
-
 ### VisualStudioCode ###
 .vscode/
 

bandit.yaml

@@ -1,5 +1,5 @@
 ### profile may optionally select or skip tests
 
-skips: ['B301','B311','B403','B404','B603','B605','B601','B607','B307', 'B110']
+skips: ['B301','B311','B403','B404','B603','B605','B601','B607','B307', 'B110', 'B602']
 
 # See bandit --help for details on each skip

flambe/cluster/instance/instance.py

@@ -413,7 +413,7 @@ def run_cmds(self, setup_cmds: List[str]) -> None:
                 raise errors.RemoteCommandError(f"Error executing {s} in {self.host}. " +
                                                 f"{ret.msg}")
 
-    def send_rsync(self, host_path: str, remote_path: str) -> None:
+    def send_rsync(self, host_path: str, remote_path: str, filter_param: str = "") -> None:
         """Send a local file or folder to a remote instance with rsync.
 
         Parameters
@@ -422,6 +422,9 @@ def send_rsync(self, host_path: str, remote_path: str) -> None:
             The local filename or folder
         remote_path : str
             The remote filename or folder to use
+        filter_param : str
+            The filter parameter to be passed to rsync.
+            For example, "--filter=':- .gitignore'"
 
         Raises
         ------
@@ -438,12 +441,14 @@ def send_rsync(self, host_path: str, remote_path: str) -> None:
 
         _to = f"{self.username}@{self.host if self.use_public else self.private_host}:{remote_path}"
 
-        cmd = ["rsync", "-ae",
-               f"ssh -i {self.key} -o StrictHostKeyChecking=no",
-               _from, _to]
+        cmd = (
+            f'rsync {filter_param} -ae "ssh -i {self.key} -o StrictHostKeyChecking=no" '
+            f'{_from} {_to}'
+        )
         try:
             subprocess.check_call(cmd, stdout=subprocess.DEVNULL,
-                                  stderr=subprocess.DEVNULL)
+                                  stderr=subprocess.DEVNULL,
+                                  shell=True)
             logger.debug(f"rsync {host_path} -> {remote_path} successful")
         except subprocess.CalledProcessError as e:
             raise errors.RemoteFileTransferError(e)
@@ -623,8 +628,13 @@ def install_flambe(self) -> None:
 
         else:
             origin = get_flambe_repo_location()
+            # Avoid rsyncing resources from gitignore
+            filter_param = ""
+            if os.path.exists(os.path.join(origin, ".gitignore")):
+                filter_param = f"--filter=':- {os.path.join(origin, '.gitignore')}'"
+
             destiny = os.path.join(self.get_home_path(), "extensions", "flambe")
-            self.send_rsync(origin, destiny)
+            self.send_rsync(origin, destiny, filter_param)
             logger.debug(f"Sent flambe {origin} -> {destiny}")
             pip_destiny = destiny if not self.contains_gpu() else f"{destiny}[cuda]"
             ret = self._run_cmd(

tests/unit/cluster/test_instance.py

@@ -149,7 +149,9 @@ def test_install_flambe_gpu(mock_contains_gpu, mock_run_cmd, get_instance):
 @mock.patch('flambe.cluster.instance.instance.Instance.get_home_path')
 @mock.patch('flambe.cluster.instance.instance.Instance.send_rsync')
 @mock.patch('flambe.cluster.instance.instance.Instance._run_cmd')
-def test_install_flambe_debug(mock_run_cmd, mock_rsync, mock_get_home_path, mock_flambe_loc, get_instance):
+@mock.patch('os.path.exists')
+def test_install_flambe_debug(mock_os_exists, mock_run_cmd, mock_rsync, mock_get_home_path, mock_flambe_loc, get_instance):
+    mock_os_exists.return_value = False
     mock_flambe_loc.return_value = "/home/user/flambe"
 
     mock_get_home_path.return_value = "/home/ubuntu"
@@ -165,7 +167,32 @@ def test_install_flambe_debug(mock_run_cmd, mock_rsync, mock_get_home_path, mock
     mock_flambe_loc.assert_called_once()
 
     mock_get_home_path.assert_called_once()
-    mock_rsync.assert_called_once_with("/home/user/flambe", "/home/ubuntu/extensions/flambe")
+    mock_rsync.assert_called_once_with("/home/user/flambe", "/home/ubuntu/extensions/flambe", "")
+
+
+@mock.patch('flambe.cluster.instance.instance.get_flambe_repo_location')
+@mock.patch('flambe.cluster.instance.instance.Instance.get_home_path')
+@mock.patch('flambe.cluster.instance.instance.Instance.send_rsync')
+@mock.patch('flambe.cluster.instance.instance.Instance._run_cmd')
+@mock.patch('os.path.exists')
+def test_install_flambe_debug_2(mock_os_exists, mock_run_cmd, mock_rsync, mock_get_home_path, mock_flambe_loc, get_instance):
+    mock_os_exists.return_value = True
+    mock_flambe_loc.return_value = "/home/user/flambe"
+
+    mock_get_home_path.return_value = "/home/ubuntu"
+    mock_run_cmd.return_value = RemoteCommand(True, b"")
+
+    ins = get_instance(debug=True)
+
+    ins.install_flambe()
+
+    cmd = f'python3 -m pip install --user --upgrade  /home/ubuntu/extensions/flambe'
+    mock_run_cmd.assert_called_with(cmd, retries=3)
+
+    mock_flambe_loc.assert_called_once()
+
+    mock_get_home_path.assert_called_once()
+    mock_rsync.assert_called_once_with("/home/user/flambe", "/home/ubuntu/extensions/flambe", "--filter=':- /home/user/flambe/.gitignore'")
 
 
 @mock.patch('flambe.cluster.instance.instance.Instance._run_cmd')