Add multi-store support

Made provision for multi-store support. Added new config option
'enabled_backends' which will be a comma separated Key:Value pair
of store identifier and store type.

DocImpact
Depends-On: https://review.openstack.org/573648
Implements: blueprint multi-store

Change-Id: I9cfa066bdce51619a78ce86a8b1f1f8d05e5bfb6

glance/api/v2/discovery.py

@@ -14,8 +14,10 @@
 # limitations under the License.
 
 from oslo_config import cfg
+import webob.exc
 
 from glance.common import wsgi
+from glance.i18n import _
 
 
 CONF = cfg.CONF
@@ -34,6 +36,27 @@ def get_image_import(self, req):
             'import-methods': import_methods
         }
 
+    def get_stores(self, req):
+        # TODO(abhishekk): This will be removed after config options
+        # 'stores' and 'default_store' are removed.
+        enabled_backends = CONF.enabled_backends
+        if not enabled_backends:
+            msg = _("Multi backend is not supported at this site.")
+            raise webob.exc.HTTPNotFound(explanation=msg)
+
+        backends = []
+        for backend in enabled_backends:
+            stores = {}
+            stores['id'] = backend
+            description = getattr(CONF, backend).store_description
+            if description:
+                stores['description'] = description
+            if backend == CONF.glance_store.default_backend:
+                stores['default'] = "true"
+            backends.append(stores)
+
+        return {'stores': backends}
+
 
 def create_resource():
     return wsgi.Resource(InfoController())

glance/api/v2/image_data.py

@@ -100,6 +100,18 @@ def _delete(self, image_repo, image):
 
     @utils.mutating
     def upload(self, req, image_id, data, size):
+        backend = None
+        if CONF.enabled_backends:
+            backend = req.headers.get('x-image-meta-store',
+                                      CONF.glance_store.default_backend)
+
+            try:
+                glance_store.get_store_from_store_identifier(backend)
+            except glance_store.UnknownScheme as exc:
+                raise webob.exc.HTTPBadRequest(explanation=exc.msg,
+                                               request=req,
+                                               content_type='text/plain')
+
         image_repo = self.gateway.get_repo(req.context)
         image = None
         refresher = None
@@ -129,7 +141,7 @@ def upload(self, req, image_id, data, size):
                                  encodeutils.exception_to_unicode(e))
 
                 image_repo.save(image, from_state='queued')
-                image.set_data(data, size)
+                image.set_data(data, size, backend=backend)
 
                 try:
                     image_repo.save(image, from_state='saving')
@@ -274,9 +286,16 @@ def stage(self, req, image_id, data, size):
         # NOTE(jokke): this is horrible way to do it but as long as
         # glance_store is in a shape it is, the only way. Don't hold me
         # accountable for it.
+        # TODO(abhishekk): After removal of backend module from glance_store
+        # need to change this to use multi_backend module.
         def _build_staging_store():
             conf = cfg.ConfigOpts()
-            backend.register_opts(conf)
+
+            try:
+                backend.register_opts(conf)
+            except cfg.DuplicateOptError:
+                pass
+
             conf.set_override('filesystem_store_datadir',
                               CONF.node_staging_uri[7:],
                               group='glance_store')

glance/api/v2/images.py

@@ -94,10 +94,6 @@ def import_image(self, req, image_id, body):
         task_factory = self.gateway.get_task_factory(req.context)
         executor_factory = self.gateway.get_task_executor_factory(req.context)
         task_repo = self.gateway.get_task_repo(req.context)
-
-        task_input = {'image_id': image_id,
-                      'import_req': body}
-
         import_method = body.get('method').get('name')
         uri = body.get('method').get('uri')
 
@@ -121,11 +117,26 @@ def import_image(self, req, image_id, body):
             if not getattr(image, 'disk_format', None):
                 msg = _("'disk_format' needs to be set before import")
                 raise exception.Conflict(msg)
+
+            backend = None
+            if CONF.enabled_backends:
+                backend = req.headers.get('x-image-meta-store',
+                                          CONF.glance_store.default_backend)
+                try:
+                    glance_store.get_store_from_store_identifier(backend)
+                except glance_store.UnknownScheme:
+                    msg = _("Store for scheme %s not found") % backend
+                    LOG.warn(msg)
+                    raise exception.Conflict(msg)
         except exception.Conflict as e:
             raise webob.exc.HTTPConflict(explanation=e.msg)
         except exception.NotFound as e:
             raise webob.exc.HTTPNotFound(explanation=e.msg)
 
+        task_input = {'image_id': image_id,
+                      'import_req': body,
+                      'backend': backend}
+
         if (import_method == 'web-download' and
            not utils.validate_import_uri(uri)):
                 LOG.debug("URI for web-download does not pass filtering: %s",
@@ -324,7 +335,10 @@ def delete(self, req, image_id):
 
             if image.status == 'uploading':
                 file_path = str(CONF.node_staging_uri + '/' + image.image_id)
-                self.store_api.delete_from_backend(file_path)
+                if CONF.enabled_backends:
+                    self.store_api.delete(file_path, None)
+                else:
+                    self.store_api.delete_from_backend(file_path)
 
             image.delete()
             image_repo.remove(image)
@@ -926,6 +940,20 @@ def _get_image_locations(image):
             image_view['file'] = self._get_image_href(image, 'file')
             image_view['schema'] = '/v2/schemas/image'
             image_view = self.schema.filter(image_view)  # domain
+
+            # add store information to image
+            if CONF.enabled_backends:
+                locations = _get_image_locations(image)
+                if locations:
+                    stores = []
+                    for loc in locations:
+                        backend = loc['metadata'].get('backend')
+                        if backend:
+                            stores.append(backend)
+
+                    if stores:
+                        image_view['stores'] = ",".join(stores)
+
             return image_view
         except exception.Forbidden as e:
             raise webob.exc.HTTPForbidden(explanation=e.msg)
@@ -941,6 +969,11 @@ def create(self, response, image):
                               ','.join(CONF.enabled_import_methods))
             response.headerlist.append(import_methods)
 
+        if CONF.enabled_backends:
+            enabled_backends = ("OpenStack-image-store-ids",
+                                ','.join(CONF.enabled_backends.keys()))
+            response.headerlist.append(enabled_backends)
+
     def show(self, response, image):
         image_view = self._format_image(image)
         body = json.dumps(image_view, ensure_ascii=False)
@@ -1107,6 +1140,11 @@ def get_base_properties():
             'readOnly': True,
             'description': _('An image file url'),
         },
+        'backend': {
+            'type': 'string',
+            'readOnly': True,
+            'description': _('Backend store to upload image to'),
+        },
         'schema': {
             'type': 'string',
             'readOnly': True,

glance/api/v2/router.py

@@ -565,5 +565,14 @@ def __init__(self, mapper):
                        controller=reject_method_resource,
                        action='reject',
                        allowed_methods='GET')
+        mapper.connect('/info/stores',
+                       controller=info_resource,
+                       action='get_stores',
+                       conditions={'method': ['GET']},
+                       body_reject=True)
+        mapper.connect('/info/stores',
+                       controller=reject_method_resource,
+                       action='reject',
+                       allowed_methods='GET')
 
         super(API, self).__init__(mapper)

glance/async/flows/_internal_plugins/web_download.py

@@ -61,8 +61,14 @@ def _build_store(self):
         # glance_store refactor is done. A good thing is that glance_store is
         # under our team's management and it gates on Glance so changes to
         # this API will (should?) break task's tests.
+        # TODO(abhishekk): After removal of backend module from glance_store
+        # need to change this to use multi_backend module.
         conf = cfg.ConfigOpts()
-        backend.register_opts(conf)
+        try:
+            backend.register_opts(conf)
+        except cfg.DuplicateOptError:
+            pass
+
         conf.set_override('filesystem_store_datadir',
                           CONF.node_staging_uri[7:],
                           group='glance_store')

glance/async/flows/api_image_import.py

@@ -86,7 +86,10 @@ def execute(self, file_path):
 
         :param file_path: path to the file being deleted
         """
-        store_api.delete_from_backend(file_path)
+        if CONF.enabled_backends:
+            store_api.delete(file_path, None)
+        else:
+            store_api.delete_from_backend(file_path)
 
 
 class _VerifyStaging(task.Task):
@@ -122,6 +125,8 @@ def __init__(self, task_id, task_type, task_repo, uri):
         self._build_store()
 
     def _build_store(self):
+        # TODO(abhishekk): After removal of backend module from glance_store
+        # need to change this to use multi_backend module.
         # NOTE(jokke): If we want to use some other store for staging, we can
         # implement the logic more general here. For now this should do.
         # NOTE(flaper87): Due to the nice glance_store api (#sarcasm), we're
@@ -133,7 +138,10 @@ def _build_store(self):
         # under our team's management and it gates on Glance so changes to
         # this API will (should?) break task's tests.
         conf = cfg.ConfigOpts()
-        backend.register_opts(conf)
+        try:
+            backend.register_opts(conf)
+        except cfg.DuplicateOptError:
+            pass
         conf.set_override('filesystem_store_datadir',
                           CONF.node_staging_uri[7:],
                           group='glance_store')
@@ -159,12 +167,13 @@ def execute(self):
 
 class _ImportToStore(task.Task):
 
-    def __init__(self, task_id, task_type, image_repo, uri, image_id):
+    def __init__(self, task_id, task_type, image_repo, uri, image_id, backend):
         self.task_id = task_id
         self.task_type = task_type
         self.image_repo = image_repo
         self.uri = uri
         self.image_id = image_id
+        self.backend = backend
         super(_ImportToStore, self).__init__(
             name='%s-ImportToStore-%s' % (task_type, task_id))
 
@@ -215,7 +224,8 @@ def execute(self, file_path=None):
         # will need the file path anyways for our delete workflow for now.
         # For future proofing keeping this as is.
         image = self.image_repo.get(self.image_id)
-        image_import.set_image_data(image, file_path or self.uri, self.task_id)
+        image_import.set_image_data(image, file_path or self.uri, self.task_id,
+                                    backend=self.backend)
 
         # NOTE(flaper87): We need to save the image again after the locations
         # have been set in the image.
@@ -306,6 +316,7 @@ def get_flow(**kwargs):
     image_id = kwargs.get('image_id')
     import_method = kwargs.get('import_req')['method']['name']
     uri = kwargs.get('import_req')['method'].get('uri')
+    backend = kwargs.get('backend')
 
     separator = ''
     if not CONF.node_staging_uri.endswith('/'):
@@ -332,7 +343,8 @@ def get_flow(**kwargs):
                                      task_type,
                                      image_repo,
                                      file_uri,
-                                     image_id)
+                                     image_id,
+                                     backend)
     flow.add(import_to_store)
 
     delete_task = lf.Flow(task_type).add(_DeleteFromFS(task_id, task_type))

glance/async/taskflow_executor.py

@@ -129,6 +129,7 @@ def _get_flow(self, task):
             if task.type == 'api_image_import':
                 kwds['image_id'] = task_input['image_id']
                 kwds['import_req'] = task_input['import_req']
+                kwds['backend'] = task_input['backend']
             return driver.DriverManager('glance.flows', task.type,
                                         invoke_on_load=True,
                                         invoke_kwds=kwds).driver

glance/common/scripts/image_import/main.py

@@ -137,13 +137,13 @@ def create_image(image_repo, image_factory, image_properties, task_id):
     return image
 
 
-def set_image_data(image, uri, task_id):
+def set_image_data(image, uri, task_id, backend=None):
     data_iter = None
     try:
         LOG.info(_LI("Task %(task_id)s: Got image data uri %(data_uri)s to be "
                  "imported"), {"data_uri": uri, "task_id": task_id})
         data_iter = script_utils.get_image_data_iter(uri)
-        image.set_data(data_iter)
+        image.set_data(data_iter, backend=backend)
     except Exception as e:
         with excutils.save_and_reraise_exception():
             LOG.warn(_LW("Task %(task_id)s failed with exception %(error)s") %

glance/common/store_utils.py

@@ -46,7 +46,15 @@ def safe_delete_from_backend(context, image_id, location):
     """
 
     try:
-        ret = store_api.delete_from_backend(location['url'], context=context)
+        if CONF.enabled_backends:
+            backend = location['metadata'].get('backend')
+            ret = store_api.delete(location['url'],
+                                   backend,
+                                   context=context)
+        else:
+            ret = store_api.delete_from_backend(location['url'],
+                                                context=context)
+
         location['status'] = 'deleted'
         if 'id' in location:
             db_api.get_api().image_location_delete(context, image_id,
@@ -133,5 +141,9 @@ def validate_external_location(uri):
     # TODO(zhiyan): This function could be moved to glance_store.
     # TODO(gm): Use a whitelist of allowed schemes
     scheme = urlparse.urlparse(uri).scheme
-    return (scheme in store_api.get_known_schemes() and
+    known_schemes = store_api.get_known_schemes()
+    if CONF.enabled_backends:
+        known_schemes = store_api.get_known_schemes_for_multi_store()
+
+    return (scheme in known_schemes and
             scheme not in RESTRICTED_URI_SCHEMAS)

glance/common/wsgi.py

@@ -317,6 +317,13 @@
                       '"HTTP_X_FORWARDED_PROTO".')),
 ]
 
+store_opts = [
+    cfg.DictOpt('enabled_backends',
+                help=_('Key:Value pair of store identifier and store type. '
+                       'In case of multiple backends should be separated'
+                       'using comma.')),
+]
+
 
 LOG = logging.getLogger(__name__)
 
@@ -325,6 +332,7 @@
 CONF.register_opts(socket_opts)
 CONF.register_opts(eventlet_opts)
 CONF.register_opts(wsgi_opts)
+CONF.register_opts(store_opts)
 profiler_opts.set_defaults(CONF)
 
 ASYNC_EVENTLET_THREAD_POOL_LIST = []
@@ -448,6 +456,13 @@ def initialize_glance_store():
     glance_store.verify_default_store()
 
 
+def initialize_multi_store():
+    """Initialize glance multi store backends."""
+    glance_store.register_store_opts(CONF)
+    glance_store.create_multi_stores(CONF)
+    glance_store.verify_store()
+
+
 def get_asynchronous_eventlet_pool(size=1000):
     """Return eventlet pool to caller.
 
@@ -599,7 +614,10 @@ def configure(self, old_conf=None, has_changed=None):
         self.client_socket_timeout = CONF.client_socket_timeout or None
         self.configure_socket(old_conf, has_changed)
         if self.initialize_glance_store:
-            initialize_glance_store()
+            if CONF.enabled_backends:
+                initialize_multi_store()
+            else:
+                initialize_glance_store()
 
     def reload(self):
         """