Merge pull request #63 from gabrielelanaro/develop

rsync Tree functionality

We now have a built-in mechanism for syncing Treants using rsync as the transfer engine.

src/datreant/core/rsync.py

@@ -0,0 +1,46 @@
+"""Rsync wrapper"""
+
+import subprocess
+import os
+
+
+def rsync(source, dest, compress=True, backup=False, dry=False,
+          include=None, exclude=None, rsync_path='/usr/bin/rsync'):
+    opts = ['-r']  # Recursive
+
+    if compress:
+        opts.append('-c')
+
+    if backup:
+        opts.append('-b')
+
+    if dry:
+        opts.append('--dry-run')
+
+    if include:
+        opts.extend(['--include=*/'])
+
+        if isinstance(include, list):
+            opts.extend(["--include={}".format(inc) for inc in include])
+        elif isinstance(include, str):
+            opts.append("--include={}".format(include))
+
+        opts.extend(['--exclude=*'])
+
+    if exclude:
+        if isinstance(exclude, list):
+            opts.extend(["--exclude={}".format(exc) for exc in exclude])
+        elif isinstance(exclude, str):
+            opts.append("--exclude={}".format(exclude))
+
+    source = os.path.join(source, '')  # Add trailing slash
+    cmd = [rsync_path] + opts + [source, dest]
+    print(cmd)
+    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    out, err = p.communicate()
+    if p.returncode != 0:
+        raise Exception('\n'.join([
+            'Syncing error: rsync returned status code {}',
+            'Standard error: {}'
+            'Standard output: {}']).format(
+            p.returncode, err, out))

src/datreant/core/tests/test_rsync.py

@@ -0,0 +1,133 @@
+"""Tests for syncing directories using rsync
+
+"""
+
+import datreant.core as dtr
+import pytest
+import os
+import py.path
+
+
+def test_sync_local(tmpdir):
+    '''Test that syncronization works locally'''
+    with tmpdir.as_cwd():
+        sequoia = dtr.Tree("sequoia").makedirs()
+        sequoia2 = dtr.Tree("sequoia2").makedirs()
+        py.path.local('sequoia/hello.txt').write('hello')
+
+        assert os.path.exists('sequoia/hello.txt')
+
+        sequoia.sync(sequoia2)
+        assert os.path.exists('sequoia2/hello.txt')
+
+
+def _create_tree(name, files=[]):
+    tree = dtr.Tree(name).makedirs()
+    for file_ in files:
+        py.path.local(os.path.join(tree.abspath, file_)
+                      ).write('hello', ensure=True)
+
+    return tree
+
+
+def test_excludes(tmpdir):
+    with tmpdir.as_cwd():
+        sequoia = _create_tree('sequoia', ['hello.txt', 'data/hello.txt',
+                                           'data/world.dat', 'world.dat'])
+        sequoia2 = dtr.Tree("sequoia2").makedirs()
+        sequoia3 = dtr.Tree("sequoia3").makedirs()
+
+        sequoia.sync(sequoia2, exclude='*.txt')
+
+        assert os.path.exists('sequoia2/world.dat')
+        assert os.path.exists('sequoia2/data/world.dat')
+
+        assert not os.path.exists('sequoia2/hello.txt')
+        assert not os.path.exists('sequoia2/data/hello.txt')
+
+        sequoia.sync(sequoia3, exclude=['*.txt', '*.dat'])
+        assert not os.path.exists('sequoia3/hello.txt')
+        assert not os.path.exists('sequoia3/world.dat')
+
+        assert os.path.exists('sequoia3/data/')
+
+
+def test_includes(tmpdir):
+    with tmpdir.as_cwd():
+        sequoia = _create_tree('sequoia', ['hello.txt', 'data/hello.txt',
+                                           'data/world.dat', 'world.dat'])
+        sequoia2 = dtr.Tree("sequoia2").makedirs()
+        sequoia3 = dtr.Tree("sequoia3").makedirs()
+        sequoia4 = dtr.Tree("sequoia4").makedirs()
+
+        # Only txt
+        sequoia.sync(sequoia2, include='*.txt')
+
+        assert os.path.exists('sequoia2/data/hello.txt')
+        assert os.path.exists('sequoia2/hello.txt')
+
+        assert not os.path.exists('sequoia2/world.dat')
+        assert not os.path.exists('sequoia2/data/world.dat')
+
+        # Only txt and dat
+        sequoia.sync(sequoia3, include=['*.txt', '*.dat'])
+
+        assert os.path.exists('sequoia3/data/hello.txt')
+        assert os.path.exists('sequoia3/hello.txt')
+
+        assert os.path.exists('sequoia3/world.dat')
+        assert os.path.exists('sequoia3/data/world.dat')
+
+        # We can also test include/excludes at the same time
+        sequoia.sync(sequoia4, exclude='*.txt', include=['data/*'])
+
+        assert os.path.exists('sequoia4/data/world.dat')
+        assert not os.path.exists('sequoia4/hello.txt')
+        assert not os.path.exists('sequoia4/world.dat')
+
+
+# def test_sync_remote(tmpdir):
+#     remote_path = 'user@host:/tmp/sequoia'
+#     with tmpdir.as_cwd():
+#         sequoia = dtr.Tree("sequoia").makedirs()
+#         py.path.local('sequoia/hello.txt').write('hello')
+#
+#         sequoia2 = dtr.Tree("sequoia2").makedirs()
+#
+#         # Upload
+#         sequoia.sync(remote_path)
+#
+#         # Download
+#         sequoia2.sync(remote_path, mode='download')
+#
+#         assert os.path.exists('sequoia2/hello.txt')
+
+
+# def test_rsync_tree(tmpdir):
+#
+#     # We want to sync this tree to another directory
+#     with tmpdir.as_cwd():
+#         # Let's assume we have an existing tree
+#         dtr.Tree("sequoia").makedirs()
+#
+#         # We create a new tree
+#         t = dtr.Tree('sequoia2')
+#
+#         # Options
+#         t.sync('sequoia',
+#                mode='download',
+#                compress=True,
+#                backup=True,
+#                dry=True,
+#                include=["*.txt"],
+#                exclude=[])
+#
+#         # Upload content to sequoia
+#         t.sync('sequoia', mode='upload')
+#
+#         # Download content from sequoia
+#         t.sync('sequoia', mode='download')
+#
+#         # Remote sync
+#         t.sync('sequoia2', host='127.0.0.1', user='xxx', password='xxx')
+#         t.sync('sequoia2', host='127.0.0.1', user='xxx', key='./cloud.pem')

src/datreant/core/trees.py

@@ -13,6 +13,7 @@
 
 from .util import makedirs
 from .manipulators import discover
+from .rsync import rsync
 from . import _TREELIMBS
 
 
@@ -467,3 +468,26 @@ def make(self):
         self.makedirs()
 
         return self
+
+    def sync(self, dest, mode='upload', compress=True, backup=False, dry=False,
+             include=None, exclude=None, rsync_path='/usr/bin/rsync'):
+        """Syncronize directories using rsync.
+
+        Parameters
+        ----------
+        dest: destination
+        """
+        if isinstance(dest, Veg):
+            dest = dest.abspath
+
+        if mode == 'download':
+            source = dest
+            dest = self.abspath
+        elif mode == 'upload':
+            source = self.abspath
+        else:
+            raise ValueError("Sync mode can be only 'upload' or 'download'.")
+        # Here we do some massaging before passing to the rsync function
+        return rsync(source, dest, compress=compress, backup=backup,
+                     dry=dry, include=include,
+                     exclude=exclude, rsync_path=rsync_path)