rclone instructions draft

source/user/howto/rclone.rst

@@ -65,3 +65,94 @@ A more reliable way can be to mount either rclone directory or directly rclone.c
 One has, however, later call rclone with "--config" option to point to the rclone.conf file, e.g::
 
         $ rclone --config /rclone/rclone.conf ls deep-nextcloud:/Datasets/dogs_breed/models
+
+Example code on usage of rclone from python
+-------------------------
+
+**Simple example**
+
+A simple call of rclone from python is via subprocess.Popen()::
+
+        import subprocess
+ 
+        # from deep-nextcloud into the container
+        command = (['rclone', 'copy', 'deep-nextcloud:/Datasets/dogs_breed/data', '/srv/dogs_breed_det/data'])
+ 
+         result = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+         output, error = result.communicate()
+
+**Advanced examples**
+
+More advanced usage includes calling rclone with various options (ls, copy, check) in order to check file existence at Source, check if after copying two versions match exactly.
+
+* rclone_call::
+
+		 def rclone_call(src_path, dest_dir, cmd = 'copy', get_output=False):
+   		 """ Function
+     		   rclone calls
+ 		 """
+   			 if cmd == 'copy':
+   		        	command = (['rclone', 'copy', '--progress', src_path, dest_dir])
+   		 	 elif cmd == 'ls':
+    		      	  command = (['rclone', 'ls', src_path])
+    			 elif cmd == 'check':
+     		      	  command = (['rclone', 'check', src_path, dest_dir])
+     
+  			 if get_output:
+      		  		 result = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+  		 	 else:
+      		 		 result = subprocess.Popen(command, stderr=subprocess.PIPE)
+  		 	 output, error = result.communicate()
+   			 return output, error
+
+* rclone_copy::
+
+		def rclone_copy(src_path, dest_dir, src_type='file'):
+   		 """ Function for rclone call to copy data (sync?)
+    		:param src_path: full path to source (file or directory)
+   		 :param dest_dir: full path to destination directory (not file!)
+   		 :param src_type: if source is file (default) or directory
+   		 :return: if destination was downloaded, and possible error
+  		  """
+ 
+   		 error_out = None
+     
+   		 if src_type == 'file':
+    		    src_dir = os.path.dirname(src_path)
+       		 dest_file = src_path.split('/')[-1]
+      		  dest_path = os.path.join(dest_dir, dest_file)
+   		 else:
+        		src_dir = src_path
+      		  dest_path =  dest_dir
+ 
+  		  # check first if we find src_path
+  		  output, error = rclone_call(src_path, dest_dir, cmd='ls')
+    		if error:
+       		 print('[ERROR] %s (src):\n%s' % (src_path, error))
+       		 error_out = error
+        		dest_exist = False
+   		 else:
+      		  # if src_path exists, copy it
+      		  output, error = rclone_call(src_path, dest_dir, cmd='copy')
+       		 if not error:
+       		     # compare two directories, if copied file appears in output
+        		    # as not found or not matching -> Error
+         		   print('[INFO] File %s copied. Check if (src) and (dest) really match..' % (dest_file))
+         		   output, error = rclone_call(src_dir, dest_dir, cmd='check')
+          		  if 'ERROR : ' + dest_file in error:
+           		     print('[ERROR] %s (src) and %s (dest) do not match!' % (src_path, dest_path))
+            		    error_out = 'Copy failed: ' + src_path + ' (src) and ' + \
+              		               dest_path + ' (dest) do not match'
+             		   dest_exist = False
+           		 else:
+               		 output, error = rclone_call(dest_path, dest_dir,
+                   		                         cmd='ls', get_output = True)
+               		 file_size = [ elem for elem in output.split(' ') if elem.isdigit() ][0]
+               		 print('[INFO] Checked: Successfully copied to %s %s bytes' % (dest_path, file_size))
+              		  dest_exist = True
+        		else:
+          		  print('[ERROR] %s (src):\n%s' % (dest_path, error))
+          		  error_out = error
+            dest_exist = False
+ 
+    return dest_exist, error_out