Merge "Fixes _cleanup_rbd code to capture ImageBusy exception"

openstack · Apr 1, 2015 · ef42f74 · ef42f74
2 parents ab25f5f + a37bc78
commit ef42f74
Show file tree

Hide file tree

Showing 2 changed files with 31 additions and 9 deletions.
diff --git a/nova/tests/unit/virt/libvirt/test_rbd.py b/nova/tests/unit/virt/libvirt/test_rbd.py
@@ -296,11 +296,14 @@ def _test_cleanup_exception(self, exception_name,
         rbd.list.return_value = ['12345_test', '111_test']
 
         client = mock_client.return_value
-        self.driver.cleanup_volumes(instance)
-        rbd.remove.assert_called_once_with(client.ioctx, '12345_test')
+        with mock.patch('eventlet.greenthread.sleep'):
+            self.driver.cleanup_volumes(instance)
+        rbd.remove.assert_any_call(client.ioctx, '12345_test')
+        # NOTE(danms): 10 retries + 1 final attempt to propagate = 11
+        self.assertEqual(11, len(rbd.remove.call_args_list))
 
     def test_cleanup_volumes_fail_not_found(self):
-        self._test_cleanup_exception('ImageNotFound')
+        self._test_cleanup_exception('ImageBusy')
 
     def test_cleanup_volumes_fail_snapshots(self):
         self._test_cleanup_exception('ImageHasSnapshots')

diff --git a/nova/virt/libvirt/rbd_utils.py b/nova/virt/libvirt/rbd_utils.py
@@ -32,6 +32,7 @@
 from nova.i18n import _
 from nova.i18n import _LE
 from nova.i18n import _LW
+from nova.openstack.common import loopingcall
 from nova import utils
 
 LOG = logging.getLogger(__name__)
@@ -254,19 +255,37 @@ def import_image(self, base, name):
         utils.execute('rbd', 'import', *args)
 
     def cleanup_volumes(self, instance):
+        def _cleanup_vol(ioctx, volume, retryctx):
+            try:
+                rbd.RBD().remove(client.ioctx, volume)
+                raise loopingcall.LoopingCallDone(retvalue=False)
+            except (rbd.ImageBusy, rbd.ImageHasSnapshots):
+                LOG.warn(_LW('rbd remove %(volume)s in pool %(pool)s '
+                             'failed'),
+                         {'volume': volume, 'pool': self.pool})
+            retryctx['retries'] -= 1
+            if retryctx['retries'] <= 0:
+                raise loopingcall.LoopingCallDone()
+
         with RADOSClient(self, self.pool) as client:
 
             def belongs_to_instance(disk):
                 return disk.startswith(instance.uuid)
 
             volumes = rbd.RBD().list(client.ioctx)
             for volume in filter(belongs_to_instance, volumes):
-                try:
-                    rbd.RBD().remove(client.ioctx, volume)
-                except (rbd.ImageNotFound, rbd.ImageHasSnapshots):
-                    LOG.warn(_LW('rbd remove %(volume)s in pool %(pool)s '
-                                 'failed'),
-                             {'volume': volume, 'pool': self.pool})
+                # NOTE(danms): We let it go for ten seconds
+                retryctx = {'retries': 10}
+                timer = loopingcall.FixedIntervalLoopingCall(
+                    _cleanup_vol, client.ioctx, volume, retryctx)
+                timed_out = timer.start(interval=1).wait()
+                if timed_out:
+                    # NOTE(danms): Run this again to propagate the error, but
+                    # if it succeeds, don't raise the loopingcall exception
+                    try:
+                        _cleanup_vol(client.ioctx, volume, retryctx)
+                    except loopingcall.LoopingCallDone:
+                        pass
 
     def get_pool_info(self):
         with RADOSClient(self) as client: