Merge pull request #74 from datreant/rsync-cleanup

Rsync API cleanup

Closes #73.

docs/api.rst

@@ -9,3 +9,4 @@ This is an overview of the ``datreant.core`` API.
     api_treants.rst
     api_filesystem.rst
     api_bundle.rst
+    api_rsync.rst

docs/api_rsync.rst

@@ -0,0 +1,13 @@
+Treant synchronization
+======================
+These are the API components of ``datreant.core`` for synchronizing
+Treants locally and remotely.
+
+.. _Sync_api:
+
+Sync
+----
+The synchronization functionality is provided by the rsync wrapper function.
+The function is used by the :py:meth:`datreant.core.Tree.sync` method.
+
+.. autofunction:: datreant.core.rsync.rsync

docs/trees.rst

@@ -18,7 +18,7 @@ We can use Trees and Leaves directly to manipulate them ::
     >>> t = dtr.Tree('moe')
     >>> t
     <Tree: 'moe'>
-    
+
     >>> l = dtr.Leaf('curly.txt')
     >>> l
     <Leaf: 'curly.txt'>
@@ -28,7 +28,7 @@ necessarily have to exist. Just as with Treants, more than one instance
 of a Tree or Leaf can point to the same place.
 
 
-Working with Trees 
+Working with Trees
 ==================
 **Tree** objects can be used to introspect downward into their directory
 structure. Since a Tree is essentially a container for its own child Trees and
@@ -66,6 +66,22 @@ in which case whether a Tree or Leaf is returned is dependent on an ending
           sensitive to ending ``/`` separators to determine whether to give a
           Tree or a Leaf.
 
+Synchronizing Trees
+==================
+
+Synchronization of tree contents can be performed through the
+:py:meth:`datreant.core.Tree.sync` method. Synchronization can be performed
+both locally and remotely, and is done through the rsync command::
+
+    >>> sequoia = dtr.Tree('sequoia')
+    >>> oak = dtr.Tree('oak')
+    >>> sequoia.sync(oak, mode="download")  # Sync contents from oak to sequoia
+    >>> sequoia.sync("/tmp/sequoia", mode="upload")  # Sync to a local directory
+    >>> sequoia.sync("user@host:/directory")  # Sync remotely
+
+.. note:: To be able to sync remotely, it is necessary to have passwordless
+          ssh access (through key file) to the server.
+
 API Reference: Tree
 -------------------
 See the :ref:`Tree_api` API reference for more details.
@@ -116,7 +132,7 @@ Or getting things back later::
     array([[ 1.28609187, -0.08739047,  1.23335427],
            [ 1.85979027,  0.37250825,  0.89576077],
            [-0.77038908, -0.02746453, -0.13723022],
-           ..., 
+           ...,
            [-0.76445797,  0.94284523,  0.29052753],
            [-0.44437005, -0.91921603, -0.4978258 ],
            [-0.70563139, -0.62811205,  0.60291534]])
@@ -127,6 +143,6 @@ a file::
     >>> t['about_moe.txt'].read()
     'Moe is not a nice person.\n'
 
-API Reference: Leaf 
+API Reference: Leaf
 -------------------
 See the :ref:`Leaf_api` API reference for more details.

src/datreant/core/rsync.py

@@ -4,11 +4,44 @@
 import os
 
 
-def rsync(source, dest, compress=True, backup=False, dry=False,
+def rsync(source, dest, compress=True, backup=False, dry=False, checksum=True,
           include=None, exclude=None, rsync_path='/usr/bin/rsync'):
+    """Wrapper function for rsync. There are some minor differences with the
+    standard rsync behaviour:
+
+    - The include option will exclude every other path.
+    - The exclude statements will take place after the include statements
+
+    Parameters
+    ----------
+    source : str
+        Source directory for the sync
+    dest : str
+        Dest directory for the sync
+    compress : bool
+        If True, use gzip compression to reduce the data transfered over
+        the network
+    backup : bool
+        If True, pre-existing files are renamed with a "~" extension before
+        they are replaced
+    dry: bool
+        If True, do a dry-run. Useful for debugging purposes
+    checksum: bool
+        Perform a checksum to determine if file content has changed.
+    include: str or list
+        Paths (wildcards are allowed) to be included. If this option is used,
+        every other path is excluded
+    exclude: str or list
+        Paths to be excluded from the copy
+    rsync_path: str
+        Path where to find the rsync executable
+    """
     opts = ['-r']  # Recursive
 
     if compress:
+        opts.append('-z')
+
+    if checksum:
         opts.append('-c')
 
     if backup:
@@ -37,6 +70,7 @@ def rsync(source, dest, compress=True, backup=False, dry=False,
     cmd = [rsync_path] + opts + [source, dest]
     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     out, err = p.communicate()
+
     if p.returncode != 0:
         raise Exception('\n'.join([
             'Syncing error: rsync returned status code {}',

src/datreant/core/trees.py

@@ -469,25 +469,36 @@ def make(self):
 
         return self
 
-    def sync(self, dest, mode='upload', compress=True, backup=False, dry=False,
-             include=None, exclude=None, rsync_path='/usr/bin/rsync'):
-        """Syncronize directories using rsync.
+    def sync(self, other, mode='upload', compress=True, checksum=True,
+             backup=False, dry=False, include=None, exclude=None,
+             rsync_path='/usr/bin/rsync'):
+        """Synchronize directories using rsync.
 
         Parameters
         ----------
-        dest: destination
+        other: str or Tree
+            Other end of the sync, can be either a path or another Tree.
+        mode: str
+            Either ``"upload"`` if uploading to  `other`, or ``"download"`` if
+            downloading from `other`
+
+
+        The other options are described in the
+        :py:func:`datreant.core.rsync.rsync` documentation.
+
         """
-        if isinstance(dest, Veg):
-            dest = dest.abspath
+        if isinstance(other, Tree):
+            other = other.abspath
 
         if mode == 'download':
-            source = dest
+            source = other
             dest = self.abspath
         elif mode == 'upload':
             source = self.abspath
+            dest = other
         else:
             raise ValueError("Sync mode can be only 'upload' or 'download'.")
         # Here we do some massaging before passing to the rsync function
         return rsync(source, dest, compress=compress, backup=backup,
-                     dry=dry, include=include,
+                     dry=dry, include=include, checksum=checksum,
                      exclude=exclude, rsync_path=rsync_path)