Fix: API returns 503 if one of the store is mis-configured

If user mis-configures one of the multiple stores defined in glance-api
configuratio file, then service returns 503 for all API requests.

Made provision to exclude faulty store and resume service normally
to function wiht other configured stores.

Closes-Bug: #1875281
Change-Id: I685a0ecbfba7cf7dbce3fd2eb84f97bd8ffbfcf3
(cherry picked from commit 98b9091)
(cherry picked from commit 17b66ab)

glance_store/_drivers/rbd.py

@@ -271,10 +271,16 @@ def get_connection(self, conffile, rados_id):
 
         try:
             client.connect(timeout=self.connect_timeout)
-        except rados.Error:
-            msg = _LE("Error connecting to ceph cluster.")
-            LOG.exception(msg)
-            raise exceptions.BackendException()
+        except (rados.Error, rados.ObjectNotFound) as e:
+            if self.backend_group and len(self.conf.enabled_backends) > 1:
+                reason = _("Error in store configuration: %s") % e
+                LOG.debug(reason)
+                raise exceptions.BadStoreConfiguration(
+                    store_name=self.backend_group, reason=reason)
+            else:
+                msg = _LE("Error connecting to ceph cluster.")
+                LOG.exception(msg)
+                raise exceptions.BackendException()
         try:
             yield client
         finally:

glance_store/tests/unit/test_multistore_rbd.py

@@ -35,6 +35,9 @@ class MockRados(object):
     class Error(Exception):
         pass
 
+    class ObjectNotFound(Exception):
+        pass
+
     class ioctx(object):
         def __init__(self, *args, **kwargs):
             pass
@@ -442,11 +445,12 @@ def test_rados_connect_timeout(self, mock_rados_connect):
     @mock.patch.object(MockRados.Rados, 'connect', side_effect=MockRados.Error)
     def test_rados_connect_error(self, _):
         rbd_store.rados.Error = MockRados.Error
+        rbd_store.rados.ObjectNotFound = MockRados.ObjectNotFound
 
         def test():
             with self.store.get_connection('conffile', 'rados_id'):
                 pass
-        self.assertRaises(exceptions.BackendException, test)
+        self.assertRaises(exceptions.BadStoreConfiguration, test)
 
     def test_create_image_conf_features(self):
         # Tests that we use non-0 features from ceph.conf and cast to int.

glance_store/tests/unit/test_rbd_store.py

@@ -34,6 +34,9 @@ class MockRados(object):
     class Error(Exception):
         pass
 
+    class ObjectNotFound(Exception):
+        pass
+
     class ioctx(object):
         def __init__(self, *args, **kwargs):
             pass
@@ -450,6 +453,7 @@ def test_rados_connect_timeout(self, mock_rados_connect):
     @mock.patch.object(MockRados.Rados, 'connect', side_effect=MockRados.Error)
     def test_rados_connect_error(self, _):
         rbd_store.rados.Error = MockRados.Error
+        rbd_store.rados.ObjectNotFound = MockRados.ObjectNotFound
 
         def test():
             with self.store.get_connection('conffile', 'rados_id'):