Merge "Add GridFS store"

glance/store/gridfs.py

@@ -0,0 +1,201 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2013 Red Hat, Inc
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Storage backend for GridFS"""
+from __future__ import absolute_import
+
+from oslo.config import cfg
+import urlparse
+
+from glance.common import exception
+import glance.openstack.common.log as logging
+import glance.store
+import glance.store.base
+import glance.store.location
+
+try:
+    import gridfs
+    import gridfs.errors
+    import pymongo
+    import pymongo.uri_parser as uri_parser
+except ImportError:
+    pymongo = None
+
+LOG = logging.getLogger(__name__)
+
+gridfs_opts = [
+    cfg.StrOpt('mongodb_store_uri'),
+    cfg.StrOpt('mongodb_store_db', default=None),
+]
+
+CONF = cfg.CONF
+CONF.register_opts(gridfs_opts)
+
+
+class StoreLocation(glance.store.location.StoreLocation):
+    """
+    Class describing an gridfs URI:
+
+        gridfs://<IMAGE_ID>
+
+    Connection information has been consciously omitted for
+    security reasons, since this location will be stored in glance's
+    database and can be queried from outside.
+
+    Note(flaper87): Make connection info available if user wants so
+    by adding a new configuration parameter `mongdb_store_insecure`.
+    """
+
+    def get_uri(self):
+        return "gridfs://%s" % self.specs.get("image_id")
+
+    def parse_uri(self, uri):
+        """
+        This method should fix any issue with the passed URI. Right now,
+        it just sets image_id value in the specs dict.
+
+        :param uri: Current set URI
+        """
+        parsed = urlparse.urlparse(uri)
+        assert parsed.scheme in ('gridfs',)
+        self.specs["image_id"] = parsed.netloc
+
+
+class Store(glance.store.base.Store):
+    """GridFS adapter"""
+
+    EXAMPLE_URL = "gridfs://<IMAGE_ID>"
+
+    def get_schemes(self):
+        return ('gridfs',)
+
+    def configure_add(self):
+        """
+        Configure the Store to use the stored configuration options
+        Any store that needs special configuration should implement
+        this method. If the store was not able to successfully configure
+        itself, it should raise `exception.BadStoreConfiguration`
+        """
+        if pymongo is None:
+            msg = _("Missing dependecies: pymongo")
+            raise exception.BadStoreConfiguration(msg)
+
+        self.mongodb_uri = self._option_get('mongodb_store_uri')
+
+        parsed = uri_parser.parse_uri(self.mongodb_uri)
+        self.mongodb_db = self._option_get('mongodb_store_db') or \
+            parsed.get("database")
+
+        self.mongodb = pymongo.MongoClient(self.mongodb_uri)
+        self.fs = gridfs.GridFS(self.mongodb[self.mongodb_db])
+
+    def _option_get(self, param):
+        result = getattr(CONF, param)
+        if not result:
+            reason = _("Could not find %(param)s in configuration "
+                       "options.") % locals()
+            LOG.debug(reason)
+            raise exception.BadStoreConfiguration(store_name="gridfs",
+                                                  reason=reason)
+        return result
+
+    def get(self, location):
+        """
+        Takes a `glance.store.location.Location` object that indicates
+        where to find the image file, and returns a tuple of generator
+        (for reading the image file) and image_size
+
+        :param location `glance.store.location.Location` object, supplied
+                        from glance.store.location.get_location_from_uri()
+        :raises `glance.exception.NotFound` if image does not exist
+        """
+        image = self._get_file(location)
+        return (image, image.length)
+
+    def get_size(self, location):
+        """
+        Takes a `glance.store.location.Location` object that indicates
+        where to find the image file, and returns the image_size (or 0
+        if unavailable)
+
+        :param location `glance.store.location.Location` object, supplied
+                        from glance.store.location.get_location_from_uri()
+        """
+        try:
+            key = self._get_file(location)
+            return key.length
+        except Exception:
+            return 0
+
+    def _get_file(self, location):
+        store_location = location
+        if isinstance(location, glance.store.location.Location):
+            store_location = location.store_location
+        try:
+
+            parsed = urlparse.urlparse(store_location.get_uri())
+            return self.fs.get(parsed.netloc)
+        except gridfs.errors.NoFile:
+            msg = _("Could not find %s image in GridFS") % \
+                store_location.get_uri()
+            LOG.debug(msg)
+            raise exception.NotFound(msg)
+
+    def add(self, image_id, image_file, image_size):
+        """
+        Stores an image file with supplied identifier to the backend
+        storage system and returns a tuple containing information
+        about the stored image.
+
+        :param image_id: The opaque image identifier
+        :param image_file: The image data to write, as a file-like object
+        :param image_size: The size of the image data to write, in bytes
+
+        :retval tuple of URL in backing store, bytes written, and checksum
+        :raises `glance.common.exception.Duplicate` if the image already
+                existed
+        """
+        loc = StoreLocation({'image_id': image_id})
+
+        if self.fs.exists(image_id):
+            raise exception.Duplicate(_("GridFS already has an image at "
+                                        "location %s") % loc.get_uri())
+
+        LOG.debug(_("Adding a new image to GridFS with id %s and size %s") %
+                 (image_id, image_size))
+
+        self.fs.put(image_file, _id=image_id)
+        image = self._get_file(loc)
+
+        LOG.debug(_("Uploaded image %s, md5 %s, length %s to GridFS") %
+                 (image._id, image.md5, image.length))
+
+        return (loc.get_uri(), image.length, image.md5)
+
+    def delete(self, location):
+        """
+        Takes a `glance.store.location.Location` object that indicates
+        where to find the image file to delete
+
+        :location `glance.store.location.Location` object, supplied
+                  from glance.store.location.get_location_from_uri()
+
+        :raises NotFound if image does not exist
+        """
+        image = self._get_file(location)
+        self.fs.delete(image._id)
+        LOG.debug("Deleted image %s from GridFS")

glance/tests/functional/store/test_gridfs.py

@@ -0,0 +1,89 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2013 Red Hat, Inc
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+"""
+Functional tests for the gridfs store interface
+
+Set the GLANCE_TEST_GRIDFS_CONF environment variable to the location
+of a Glance config that defines how to connect to a functional
+GridFS backend
+"""
+
+import ConfigParser
+import os
+import os.path
+
+import oslo.config.cfg
+import testtools
+
+import glance.store.gridfs
+import glance.tests.functional.store as store_tests
+
+
+try:
+    import gridfs
+    import pymongo
+except ImportError:
+    gridfs = None
+
+
+def read_config(path):
+    cp = ConfigParser.RawConfigParser()
+    cp.read(path)
+    return cp
+
+
+def parse_config(config):
+    out = {}
+    options = [
+        'mongodb_store_db',
+        'mongodb_store_uri']
+
+    for option in options:
+        out[option] = config.defaults()[option]
+
+    return out
+
+
+class TestGridfsStore(store_tests.BaseTestCase, testtools.TestCase):
+
+    store_cls_path = 'glance.store.gridfs.Store'
+    store_cls = glance.store.gridfs.Store
+    store_name = 'gridfs'
+
+    def setUp(self):
+        config_path = os.environ.get('GLANCE_TEST_GRIDFS_CONF')
+        if not config_path or not gridfs:
+            msg = "GLANCE_TEST_GRIDFS_CONF environ not set."
+            self.skipTest(msg)
+
+        oslo.config.cfg.CONF(args=[], default_config_files=[config_path])
+
+        raw_config = read_config(config_path)
+        self.gfs_config = parse_config(raw_config)
+        super(TestGridfsStore, self).setUp()
+
+    def get_store(self, **kwargs):
+        store = self.store_cls(context=kwargs.get('context'))
+        store.configure()
+        store.configure_add()
+        return store
+
+    def stash_image(self, image_id, image_data):
+        conn = pymongo.MongoClient(self.gfs_config.get("mongodb_store_uri"))
+        fs = gridfs.GridFS(conn[self.gfs_config.get("mongodb_store_db")])
+        fs.put(image_data, _id=image_id)
+        return 'gridfs://%s' % image_id