Adds Driver Layer to Image Cache

Fixes LP Bug#879136 - keyerror: 'image' when doing nova image-list
Fixes LP Bug#819936 - New image cache breaks Glance on Windows

This patch refactors the image cache further by adding an
adaptable driver layer to the cache. The existing filesystem-based
driver that depended on python-xattr and conditional fstab support
has been moved to /glance/image_cache/drivers/xattr.py, and a new
default driver is now based on SQLite and has no special requirements.

The image cache now contains a simple interface for pruning the
cache. Instead of the logic being contained in
/glance/image_cache/pruner.py, now the prune logic is self-contained
within the ImageCache.prune() method, with pruning calling the
simple well-defined driver methods of get_least_recently_accessed()
and get_cache_size().

Adds a functional test case for the caching middleware and adds
documentation on how to configure the image cache drivers.

TODO: cache-manage middleware...
TODO: cache management docs

Change-Id: Id7ae73549d6bb39222eb7ac0427b0083fd1af3ec

bin/glance-cache-reaper → bin/glance-cache-cleaner

@@ -19,16 +19,16 @@
 #    under the License.
 
 """
-Glance Image Cache Invalid Cache Entry and Stalled Image  Reaper
+Glance Image Cache Invalid Cache Entry and Stalled Image cleaner
 
 This is meant to be run as a periodic task from cron.
 
 If something goes wrong while we're caching an image (for example the fetch
 times out, or an exception is raised), we create an 'invalid' entry. These
 entires are left around for debugging purposes. However, after some period of
-time, we want to cleans these up, aka reap them.
+time, we want to clean these up.
 
-Also, if an incomplete image hangs around past the image_cache_stall_timeout
+Also, if an incomplete image hangs around past the image_cache_stall_time
 period, we automatically sweep it up.
 """
 
@@ -70,7 +70,7 @@ if __name__ == '__main__':
     (options, args) = config.parse_options(oparser)
 
     try:
-        conf, app = config.load_paste_app('glance-reaper', options, args)
+        conf, app = config.load_paste_app('glance-cleaner', options, args)
         app.run()
     except RuntimeError, e:
         sys.exit("ERROR: %s" % e)

doc/source/configuring.rst

@@ -507,15 +507,43 @@ Configuration Options Affecting the Image Cache
 
 One main configuration file option affects the image cache.
 
- * ``image_cache_datadir=PATH``
+ * ``image_cache_dir=PATH``
 
 Required when image cache middleware is enabled.
 
 Default: ``/var/lib/glance/image-cache``
 
-This is the root directory where the image cache will write its
-cached image files. Make sure the directory is writeable by the
-user running the ``glance-api`` server
+This is the base directory the image cache can write files to.
+Make sure the directory is writeable by the user running the
+``glance-api`` server
+
+ * ``image_cache_driver=DRIVER``
+
+Optional. Choice of ``sqlite`` or ``xattr``
+
+Default: ``sqlite``
+
+The default ``sqlite`` cache driver has no special dependencies, other
+than the ``python-sqlite3`` library, which is installed on virtually
+all operating systems with modern versions of Python. It stores
+information about the cached files in a SQLite database.
+
+The ``xattr`` cache driver required the ``python-xattr>=0.6.0`` library
+and requires that the filesystem containing ``image_cache_dir`` have
+access times tracked for all files (in other words, the noatime option
+CANNOT be set for that filesystem). In addition, ``user_xattr`` must be
+set on the filesystem's description line in fstab. Because of these
+requirements, the ``xattr`` cache driver is not available on Windows.
+
+ * ``image_cache_sqlite_db=DB_FILE``
+
+Optional.
+
+Default: ``cache.db``
+
+When using the ``sqlite`` cache driver, you can set the name of the database
+that will be used to store the cached images information. The database
+is always contained in the ``image_cache_dir``.
 
 Configuring the Glance Registry
 -------------------------------

etc/glance-api.conf

@@ -178,8 +178,8 @@ scrubber_datadir = /var/lib/glance/scrubber
 
 # =============== Image Cache Options =============================
 
-# Directory that the Image Cache writes data to
-image_cache_datadir = /var/lib/glance/image-cache/
+# Base directory that the Image Cache uses
+image_cache_dir = /var/lib/glance/image-cache/
 
 [pipeline:glance-api]
 pipeline = versionnegotiation context apiv1app

etc/glance-cache.conf

@@ -11,11 +11,11 @@ log_file = /var/log/glance/image-cache.log
 use_syslog = False
 
 # Directory that the Image Cache writes data to
-image_cache_datadir = /var/lib/glance/image-cache/
+image_cache_dir = /var/lib/glance/image-cache/
 
 # Number of seconds after which we should consider an incomplete image to be
 # stalled and eligible for reaping
-image_cache_stall_timeout = 86400
+image_cache_stall_time = 86400
 
 # image_cache_invalid_entry_grace_period - seconds
 #
@@ -27,18 +27,8 @@ image_cache_stall_timeout = 86400
 # are elibible to be reaped.
 image_cache_invalid_entry_grace_period = 3600
 
-image_cache_max_size_bytes = 1073741824
-
-# Percentage of the cache that should be freed (in addition to the overage)
-# when the cache is pruned
-#
-# A percentage of 0% means we prune only as many files as needed to remain
-# under the cache's max_size. This is space efficient but will lead to
-# constant pruning as the size bounces just-above and just-below the max_size.
-#
-# To mitigate this 'thrashing', you can specify an additional amount of the
-# cache that should be tossed out on each prune.
-image_cache_percent_extra_to_free = 0.20
+# Max cache size in bytes
+image_cache_max_size = 1073741824
 
 # Address to find the registry server
 registry_host = 0.0.0.0
@@ -52,5 +42,5 @@ paste.app_factory = glance.image_cache.pruner:app_factory
 [app:glance-prefetcher]
 paste.app_factory = glance.image_cache.prefetcher:app_factory
 
-[app:glance-reaper]
-paste.app_factory = glance.image_cache.reaper:app_factory
+[app:glance-cleaner]
+paste.app_factory = glance.image_cache.cleaner:app_factory

glance/api/middleware/cache.py

@@ -27,7 +27,8 @@
 import httplib
 import logging
 import re
-import shutil
+
+import webob
 
 from glance import image_cache
 from glance import registry
@@ -36,8 +37,6 @@
 from glance.common import utils
 from glance.common import wsgi
 
-import webob
-
 logger = logging.getLogger(__name__)
 get_images_re = re.compile(r'^(/v\d+)*/images/(.+)$')
 
@@ -48,8 +47,7 @@ def __init__(self, app, options):
         self.options = options
         self.cache = image_cache.ImageCache(options)
         self.serializer = images.ImageSerializer()
-        logger.info(_("Initialized image cache middleware using datadir: %s"),
-                    options.get('image_cache_datadir'))
+        logger.info(_("Initialized image cache middleware"))
         super(CacheFilter, self).__init__(app)
 
     def process_request(self, request):
@@ -67,7 +65,15 @@ def process_request(self, request):
             return None
 
         image_id = match.group(2)
-        if self.cache.hit(image_id):
+
+        # /images/detail is unfortunately supported, so here we
+        # cut out those requests and anything with a query
+        # parameter...
+        # See LP Bug #879136
+        if '?' in image_id or image_id == 'detail':
+            return None
+
+        if self.cache.is_cached(image_id):
             logger.debug(_("Cache hit for image '%s'"), image_id)
             image_iterator = self.get_from_cache(image_id)
             context = request.context
@@ -83,24 +89,6 @@ def process_request(self, request):
                         "however the registry did not contain metadata for "
                         "that image!" % image_id)
                 logger.error(msg)
-                return None
-
-        # Make sure we're not already prefetching or caching the image
-        # that just generated the miss
-        if self.cache.is_image_currently_prefetching(image_id):
-            logger.debug(_("Image '%s' is already being prefetched,"
-                         " not tee'ing into the cache"), image_id)
-            return None
-        elif self.cache.is_image_currently_being_written(image_id):
-            logger.debug(_("Image '%s' is already being cached,"
-                         " not tee'ing into the cache"), image_id)
-            return None
-
-        # NOTE(sirp): If we're about to download and cache an
-        # image which is currently in the prefetch queue, just
-        # delete the queue items since we're caching it anyway
-        if self.cache.is_image_queued_for_prefetch(image_id):
-            self.cache.delete_queued_prefetch_image(image_id)
         return None
 
     def process_response(self, resp):
@@ -120,27 +108,13 @@ def process_response(self, resp):
             return resp
 
         image_id = match.group(2)
-        if not self.cache.hit(image_id):
-            # Make sure we're not already prefetching or caching the image
-            # that just generated the miss
-            if self.cache.is_image_currently_prefetching(image_id):
-                logger.debug(_("Image '%s' is already being prefetched,"
-                             " not tee'ing into the cache"), image_id)
-                return resp
-            if self.cache.is_image_currently_being_written(image_id):
-                logger.debug(_("Image '%s' is already being cached,"
-                             " not tee'ing into the cache"), image_id)
-                return resp
-
-        logger.debug(_("Tee'ing image '%s' into cache"), image_id)
-        # TODO(jaypipes): This is so incredibly wasteful, but because
-        # the image cache needs the image's name, we have to do this.
-        # In the next iteration, remove the image cache's need for
-        # any attribute other than the id...
-        image_meta = registry.get_image_metadata(request.context,
-                                                 image_id)
-        resp.app_iter = self.get_from_store_tee_into_cache(
-            image_meta, resp.app_iter)
+        if '?' in image_id or image_id == 'detail':
+            return resp
+
+        if self.cache.is_cached(image_id):
+            return resp
+
+        resp.app_iter = self.cache.get_caching_iter(image_id, resp.app_iter)
         return resp
 
     def get_status_code(self, response):
@@ -152,13 +126,6 @@ def get_status_code(self, response):
             return response.status_int
         return response.status
 
-    def get_from_store_tee_into_cache(self, image_meta, image_iterator):
-        """Called if cache miss"""
-        with self.cache.open(image_meta, "wb") as cache_file:
-            for chunk in image_iterator:
-                cache_file.write(chunk)
-                yield chunk
-
     def get_from_cache(self, image_id):
         """Called if cache hit"""
         with self.cache.open_for_read(image_id) as cache_file:

glance/common/exception.py

@@ -117,6 +117,11 @@ class BadStoreConfiguration(GlanceException):
                "Reason: %(reason)s")
 
 
+class BadDriverConfiguration(GlanceException):
+    message = _("Driver %(driver_name)s could not be configured correctly. "
+               "Reason: %(reason)s")
+
+
 class StoreDeleteNotSupported(GlanceException):
     message = _("Deleting images from this store is not supported.")