Create simple copyfile overlord module that is able to send big files.

func/minion/modules/copyfile.py

@@ -46,6 +46,47 @@ def checksum(self, thing):
 
         return thissum.hexdigest()
 
+    def open(self, filepath, mode=None, uid=-1, gid=-1):
+        dirpath = os.path.dirname(filepath)
+        if not os.path.exists(dirpath):
+            os.makedirs(dirpath)
+
+        # Create empty file
+        try:
+            fo = open(filepath, 'w')
+            fo.close()
+            del fo
+        except (IOError, OSError), e:
+            # XXX logger output here
+            return -1
+
+        try:
+            # we could intify the mode here if it's a string
+            if mode:
+                os.chmod(filepath, mode)
+            if uid != -1 or gid != -1:
+                os.chown(filepath, uid, gid)
+        except (IOError, OSError), e:
+            return -1
+
+        return filepath
+
+    def append(self, filepath, filebuf):
+        if not os.path.exists(filepath):
+            # file disaperead
+            return -1
+
+        try:
+            fo = open(filepath, 'a')
+            fo.write(filebuf.data)
+            fo.close()
+            del fo
+        except (IOError, OSError), e:
+            # XXX logger output here
+            return -1
+
+        return 1
+
 
     def copyfile(self, filepath, filebuf, mode=0644, uid=0, gid=0, force=None):
         # -1 = problem file was not copied

func/overlord/cmd_modules/copyfile.py

@@ -49,19 +49,5 @@ def do(self, args):
 
         self.server_spec = self.parentCommand.server_spec
         self.getOverlord()
-
-        try:
-            fb = open(self.options.filename, "r").read()
-        except IOError, e:
-            sys.stderr.write("Unable to open file: %s: %s\n" % (self.options.filename, e))
-            return
-
-        st = os.stat(self.options.filename)
-        mode = stat.S_IMODE(st.st_mode)
-        uid = st.st_uid
-        gid = st.st_gid
 
-
-        data = xmlrpclib.Binary(fb)
-        results = self.overlord_obj.run("copyfile", "copyfile", [self.options.remotepath, data,
-                                                                 mode, uid, gid])
+        return self.overlord_obj.local.copyfile.send(self.options.filename, self.options.remotepath)

func/overlord/modules/copyfile.py

@@ -0,0 +1,27 @@
+from func.overlord import overlord_module
+import os
+import stat
+import xmlrpclib
+
+class copyfile(overlord_module.BaseModule):
+    def send(self, localpath, remotepath, bufsize=60000):
+        try:
+            f = open(localpath, "r")
+        except IOError, e:
+            sys.stderr.write("Unable to open file: %s: %s\n" % (self.options.filename, e))
+            return
+
+        st = os.stat(localpath)
+        mode = stat.S_IMODE(st.st_mode)
+        uid = st.st_uid
+        gid = st.st_gid
+
+        self.parent.run("copyfile", "open", [remotepath, mode, uid, gid])
+
+        while True: 
+            data=f.read(bufsize)
+            if data:
+                self.parent.run("copyfile", "append", [remotepath, xmlrpclib.Binary(data)])
+            else:
+                break
+