Rsync threaded

api/api/common.py

@@ -8,7 +8,7 @@
 from api.exceptions import FileSynchronizationException
 from pyactor.context import set_context, create_host
 from swiftclient import client as swift_client
-
+import threading
 import errno
 import hashlib
 import calendar
@@ -154,22 +154,28 @@ def get_project_list():
 def rsync_dir_with_nodes(directory):
     # retrieve nodes
     nodes = get_all_registered_nodes()
+    already_sync = list()
     for node in nodes:
-        logger.info("Rsync - pushing to "+node['type']+":"+node['name'])
-        if not node.viewkeys() & {'ssh_username', 'ssh_password'}:
-            raise FileSynchronizationException("SSH credentials missing. Please, set the credentials for this "+node['type']+" node: "+node['name'])
-
-        # Directory is only synchronized if node status is UP
-        if calendar.timegm(time.gmtime()) - int(float(node['last_ping'])) <= NODE_STATUS_THRESHOLD:
-            # The basename of the path is not needed because it will be the same as source dir
-            dest_directory = os.path.dirname(directory)
-            data = {'directory': directory, 'dest_directory': dest_directory, 'node_ip': node['ip'],
-                    'ssh_username': node['ssh_username'], 'ssh_password': node['ssh_password']}
-            rsync_command = 'sshpass -p {ssh_password} rsync --progress --delete -avrz -e ssh {directory} {ssh_username}@{node_ip}:{dest_directory}'.format(**data)
-
-            ret = os.system(rsync_command)
-            if ret != 0:
-                raise FileSynchronizationException("An error occurred copying files to Swift nodes. Please check the SSH credentials of this "+node['type']+" node: "+node['name'])
+        if node['ip'] not in already_sync:
+            already_sync.append(node['ip'])
+            to_json_bools(node, 'ssh_access')
+            if not node['ssh_access']:
+                raise FileSynchronizationException("SSH credentials missing. Please, set the credentials for this "+node['type']+" node: "+node['name'])
+
+            # Directory is only synchronized if node status is UP
+            if calendar.timegm(time.gmtime()) - int(float(node['last_ping'])) <= NODE_STATUS_THRESHOLD:
+                # The basename of the path is not needed because it will be the same as source dir
+                logger.info("Rsync - pushing to "+node['type']+":"+node['name'])
+                dest_directory = os.path.dirname(directory)
+                data = {'directory': directory, 'dest_directory': dest_directory, 'node_ip': node['ip'],
+                        'ssh_username': node['ssh_username'], 'ssh_password': node['ssh_password']}
+
+                threading.Thread(target=rsync, args=(node, data)).start()
+
+
+def rsync(node, data):
+    rsync_command = 'sshpass -p {ssh_password} rsync --progress --delete -avrz -e ssh {directory} {ssh_username}@{node_ip}:{dest_directory}'.format(**data)
+    os.system(rsync_command)
 
 
 def get_all_registered_nodes():