improve efficiency of image transfer during migration

This reduces time to transfer a qcow2 image
with a virtual size of 10G, over GigE,
from about 7 minutes to about 30 seconds.

There are multiple inefficiencies in the existing process.
Taking an example of a qcow2 image with 10G virtual size,
the process was:

  qcow2 -> raw -> read -> send -> write -> qcow2

qcow2 to raw takes 20s,
transfer of the resultant 10G is another 4m9s, and
conversion back to qcow takes 2m33s.
I.E. a total of about 7 minutes.

So instead we try to avoid the initial qcow2 to raw
conversion completely, which results in the whole
process completing in about 30s, in the common
case where no conversion to raw is done on the destination.

We also optimize the case where the source qcow2
image doesn't have a backing file, and then directly
copy the source image without merging a backing file.

Note this will also improve the situation when
resizing/migrating within the same host as
needles conversions are avoided in that case too.

We also optimize the case where raw images are being used
by trying to use `rsync -Sz` rather than `scp`.
That compresses runs of zeros and create sparse destination files.
Testing a 10G raw image showed a saving of 30s in transfer time.
Also the network was greatly reduced (corresponding to holes
in the source), as was space usage at the destination.
This gain is limited though by rsync inefficiently reading
all the holes at the source:
https://bugzilla.samba.org/show_bug.cgi?id=8918

Thanks to David Naori <dnaori@redhat.com> for testing and ideas.

Change-Id: I9e87f912ef2717221c244241cda2f1027a4ca66a

nova/tests/test_libvirt.py

@@ -3485,6 +3485,9 @@ def test_finish_migration(self):
                       'local_gb': 10, 'backing_file': '/base/disk.local'}]
         disk_info_text = jsonutils.dumps(disk_info)
 
+        def fake_can_resize_fs(path, size, use_cow=False):
+            return False
+
         def fake_extend(path, size):
             pass
 
@@ -3513,6 +3516,8 @@ def fake_get_info(instance):
 
         self.flags(use_cow_images=True)
         self.stubs.Set(libvirt_driver.disk, 'extend', fake_extend)
+        self.stubs.Set(libvirt_driver.disk, 'can_resize_fs',
+                       fake_can_resize_fs)
         self.stubs.Set(self.libvirtconnection, 'to_xml', fake_to_xml)
         self.stubs.Set(self.libvirtconnection, 'plug_vifs', fake_plug_vifs)
         self.stubs.Set(self.libvirtconnection, '_create_image',

nova/virt/disk/api.py

@@ -124,6 +124,31 @@ def extend(image, size):
     resize2fs(image)
 
 
+def can_resize_fs(image, size, use_cow=False):
+    """Check whether we can resize contained file system."""
+
+    # Check that we're increasing the size
+    virt_size = get_image_virtual_size(image)
+    if virt_size >= size:
+        return False
+
+    # Check the image is unpartitioned
+    if use_cow:
+        # Try to mount an unpartitioned qcow2 image
+        try:
+            inject_data(image, use_cow=True)
+        except exception.NovaException:
+            return False
+    else:
+        # For raw, we can directly inspect the file system
+        try:
+            utils.execute('e2label', image)
+        except exception.ProcessExecutionError:
+            return False
+
+    return True
+
+
 def bind(src, target, instance_name):
     """Bind device to a filesytem"""
     if src:

nova/virt/libvirt/driver.py

@@ -2763,7 +2763,6 @@ def migrate_disk_and_power_off(self, context, instance, dest,
         self.power_off(instance)
 
         # copy disks to destination
-        # if disk type is qcow2, convert to raw then send to dest.
         # rename instance dir to +_resize at first for using
         # shared storage for instance dir (eg. NFS).
         same_host = (dest == self.get_host_ip_addr())
@@ -2772,28 +2771,29 @@ def migrate_disk_and_power_off(self, context, instance, dest,
         try:
             utils.execute('mv', inst_base, inst_base_resize)
             if same_host:
+                dest = None
                 utils.execute('mkdir', '-p', inst_base)
             else:
                 utils.execute('ssh', dest, 'mkdir', '-p', inst_base)
             for info in disk_info:
                 # assume inst_base == dirname(info['path'])
-                to_path = "%s:%s" % (dest, info['path'])
-                fname = os.path.basename(info['path'])
+                img_path = info['path']
+                fname = os.path.basename(img_path)
                 from_path = os.path.join(inst_base_resize, fname)
-                if info['type'] == 'qcow2':
+                if info['type'] == 'qcow2' and info['backing_file']:
                     tmp_path = from_path + "_rbase"
+                    # merge backing file
                     utils.execute('qemu-img', 'convert', '-f', 'qcow2',
-                                  '-O', 'raw', from_path, tmp_path)
+                                  '-O', 'qcow2', from_path, tmp_path)
+
                     if same_host:
-                        utils.execute('mv', tmp_path, info['path'])
+                        utils.execute('mv', tmp_path, img_path)
                     else:
-                        utils.execute('scp', tmp_path, to_path)
+                        libvirt_utils.copy_image(tmp_path, img_path, host=dest)
                         utils.execute('rm', '-f', tmp_path)
-                else:  # raw
-                    if same_host:
-                        utils.execute('cp', from_path, info['path'])
-                    else:
-                        utils.execute('scp', from_path, to_path)
+
+                else:  # raw or qcow2 with no backing file
+                    libvirt_utils.copy_image(from_path, img_path, host=dest)
         except Exception, e:
             try:
                 if os.path.exists(inst_base_resize):
@@ -2828,12 +2828,28 @@ def finish_migration(self, context, migration, instance, disk_info,
         for info in disk_info:
             fname = os.path.basename(info['path'])
             if fname == 'disk':
-                disk.extend(info['path'],
-                            instance['root_gb'] * 1024 * 1024 * 1024)
+                size = instance['root_gb']
             elif fname == 'disk.local':
-                disk.extend(info['path'],
-                            instance['ephemeral_gb'] * 1024 * 1024 * 1024)
-            if FLAGS.use_cow_images:
+                size = instance['ephemeral_gb']
+            else:
+                size = 0
+            size *= 1024 * 1024 * 1024
+
+            # If we have a non partitioned image that we can extend
+            # then ensure we're in 'raw' format so we can extend file system.
+            fmt = info['type']
+            if (size and fmt == 'qcow2' and
+                disk.can_resize_fs(info['path'], size, use_cow=True)):
+                path_raw = info['path'] + '_raw'
+                utils.execute('qemu-img', 'convert', '-f', 'qcow2',
+                              '-O', 'raw', info['path'], path_raw)
+                utils.execute('mv', path_raw, info['path'])
+                fmt = 'raw'
+
+            if size:
+                disk.extend(info['path'], size)
+
+            if fmt == 'raw' and FLAGS.use_cow_images:
                 # back to qcow2 (no backing_file though) so that snapshot
                 # will be available
                 path_qcow = info['path'] + '_qcow'

nova/virt/libvirt/utils.py

@@ -180,17 +180,35 @@ def get_disk_backing_file(path):
     return backing_file
 
 
-def copy_image(src, dest):
-    """Copy a disk image
+def copy_image(src, dest, host=None):
+    """Copy a disk image to an existing directory
 
     :param src: Source image
     :param dest: Destination path
+    :param host: Remote host
     """
-    # We shell out to cp because that will intelligently copy
-    # sparse files.  I.E. holes will not be written to DEST,
-    # rather recreated efficiently.  In addition, since
-    # coreutils 8.11, holes can be read efficiently too.
-    execute('cp', src, dest)
+
+    if not host:
+        # We shell out to cp because that will intelligently copy
+        # sparse files.  I.E. holes will not be written to DEST,
+        # rather recreated efficiently.  In addition, since
+        # coreutils 8.11, holes can be read efficiently too.
+        execute('cp', src, dest)
+    else:
+        dest = "%s:%s" % (host, dest)
+        # Try rsync first as that can compress and create sparse dest files.
+        # Note however that rsync currently doesn't read sparse files
+        # efficiently: https://bugzilla.samba.org/show_bug.cgi?id=8918
+        # At least network traffic is mitigated with compression.
+        try:
+            # Do a relatively light weight test first, so that we
+            # can fall back to scp, without having run out of space
+            # on the destination for example.
+            execute('rsync', '--sparse', '--compress', '--dry-run', src, dest)
+        except exception.ProcessExecutionError:
+            execute('scp', src, dest)
+        else:
+            execute('rsync', '--sparse', '--compress', src, dest)
 
 
 def mkfs(fs, path, label=None):