Skip to content

Commit

Permalink
Correcting image cleanup in cache drivers
Browse files Browse the repository at this point in the history 
Fixes bug 888382. Includes some removal of unused code.

Change-Id: I53923bcf1fe13d63f7242cf328eb53e37d28285e
  • Loading branch information
Brian Waldon committed Nov 10, 2011
1 parent 98cefb7 commit 0135c42
Show file tree
Hide file tree
Showing 4 changed files with 47 additions and 147 deletions.
72 changes: 3 additions & 69 deletions glance/image_cache/drivers/sqlite.py
View file
Expand Up @@ -132,7 +132,8 @@ def initialize_db(self):
msg = _("Failed to initialize the image cache database. "
"Got error: %s") % e
logger.error(msg)
raise BadDriverConfiguration(driver_name='sqlite', reason=msg)
raise exception.BadDriverConfiguration(driver_name='sqlite',
reason=msg)

def get_cache_size(self):
"""
Expand Down Expand Up @@ -253,7 +254,7 @@ def clean(self):
DEFAULT_STALL_TIME))
now = time.time()
older_than = now - incomplete_stall_time
self.delete_incomplete_files(older_than)
self.delete_stalled_files(older_than)

def get_least_recently_accessed(self):
"""
Expand Down Expand Up @@ -407,73 +408,6 @@ def queue_image(self, image_id):

return True

def _base_entries(self, basepath):
def iso8601_from_timestamp(timestamp):
return datetime.datetime.utcfromtimestamp(timestamp)\
.isoformat()

for path in self.self.get_cache_files(basepath):
filename = os.path.basename(path)
try:
image_id = int(filename)
except ValueError, TypeError:
continue

entry = {}
entry['id'] = image_id
entry['path'] = path
entry['name'] = self.driver.get_attr(image_id, 'active',
'image_name',
default='UNKNOWN')

mtime = os.path.getmtime(path)
entry['last_modified'] = iso8601_from_timestamp(mtime)

atime = os.path.getatime(path)
entry['last_accessed'] = iso8601_from_timestamp(atime)

entry['size'] = os.path.getsize(path)

entry['expected_size'] = self.driver.get_attr(image_id,
'active', 'expected_size', default='UNKNOWN')

yield entry

def invalid_entries(self):
"""Cache info for invalid cached images"""
for entry in self._base_entries(self.invalid_path):
path = entry['path']
entry['error'] = self.driver.get_attr(image_id, 'invalid',
'error',
default='UNKNOWN')
yield entry

def incomplete_entries(self):
"""Cache info for incomplete cached images"""
for entry in self._base_entries(self.incomplete_path):
yield entry

def prefetch_entries(self):
"""Cache info for both queued and in-progress prefetch jobs"""
both_entries = itertools.chain(
self._base_entries(self.prefetch_path),
self._base_entries(self.prefetching_path))

for entry in both_entries:
path = entry['path']
entry['status'] = 'in-progress' if 'prefetching' in path\
else 'queued'
yield entry

def entries(self):
"""Cache info for currently cached images"""
for entry in self._base_entries(self.path):
path = entry['path']
entry['hits'] = self.driver.get_attr(image_id, 'active',
'hits',
default='UNKNOWN')
yield entry

def delete_invalid_files(self):
"""
Removes any invalid cache entries
Expand Down
103 changes: 26 additions & 77 deletions glance/image_cache/drivers/xattr.py
View file
Expand Up @@ -71,6 +71,8 @@

logger = logging.getLogger(__name__)

DEFAULT_STALL_TIME = 86400 # 24 hours


class Driver(base.Driver):

Expand Down Expand Up @@ -348,90 +350,23 @@ def get_cache_queue(self):
items.sort()
return [image_id for (mtime, image_id) in items]

def _base_entries(self, basepath):
def iso8601_from_timestamp(timestamp):
return datetime.datetime.utcfromtimestamp(timestamp)\
.isoformat()

for path in self.get_all_regular_files(basepath):
filename = os.path.basename(path)
try:
image_id = int(filename)
except ValueError, TypeError:
continue

entry = {}
entry['id'] = image_id
entry['path'] = path
entry['name'] = self.driver.get_attr(image_id, 'active',
'image_name',
default='UNKNOWN')

mtime = os.path.getmtime(path)
entry['last_modified'] = iso8601_from_timestamp(mtime)

atime = os.path.getatime(path)
entry['last_accessed'] = iso8601_from_timestamp(atime)

entry['size'] = os.path.getsize(path)

entry['expected_size'] = self.driver.get_attr(image_id,
'active', 'expected_size', default='UNKNOWN')

yield entry

def invalid_entries(self):
"""Cache info for invalid cached images"""
for entry in self._base_entries(self.invalid_path):
path = entry['path']
entry['error'] = self.driver.get_attr(image_id, 'invalid',
'error',
default='UNKNOWN')
yield entry

def incomplete_entries(self):
"""Cache info for incomplete cached images"""
for entry in self._base_entries(self.incomplete_path):
yield entry

def prefetch_entries(self):
"""Cache info for both queued and in-progress prefetch jobs"""
both_entries = itertools.chain(
self._base_entries(self.prefetch_path),
self._base_entries(self.prefetching_path))

for entry in both_entries:
path = entry['path']
entry['status'] = 'in-progress' if 'prefetching' in path\
else 'queued'
yield entry

def entries(self):
"""Cache info for currently cached images"""
for entry in self._base_entries(self.path):
path = entry['path']
entry['hits'] = self.driver.get_attr(image_id, 'active',
'hits',
default='UNKNOWN')
yield entry

def _reap_old_files(self, dirpath, entry_type, grace=None):
"""
"""
now = time.time()
reaped = 0
for path in self.get_all_regular_files(dirpath):
for path in get_all_regular_files(dirpath):
mtime = os.path.getmtime(path)
age = now - mtime
if not grace:
logger.debug(_("No grace period, reaping '%(path)s'"
" immediately"), locals())
self._delete_file(path)
delete_cached_file(path)
reaped += 1
elif age > grace:
logger.debug(_("Cache entry '%(path)s' exceeds grace period, "
"(%(age)i s > %(grace)i s)"), locals())
self._delete_file(path)
delete_cached_file(path)
reaped += 1

logger.info(_("Reaped %(reaped)s %(entry_type)s cache entries"),
Expand All @@ -444,14 +379,28 @@ def reap_invalid(self, grace=None):
:param grace: Number of seconds to keep an invalid entry around for
debugging purposes. If None, then delete immediately.
"""
return self._reap_old_files(self.invalid_path, 'invalid', grace=grace)
return self._reap_old_files(self.invalid_dir, 'invalid', grace=grace)

def reap_stalled(self, grace=None):
"""Remove any stalled cache entries
:param grace: Number of seconds to keep an invalid entry around for
debugging purposes. If None, then delete immediately.
"""
return self._reap_old_files(self.incomplete_dir, 'stalled',
grace=grace)

def clean(self):
"""
Delete any image files in the invalid directory and any
files in the incomplete directory that are older than a
configurable amount of time.
"""
self.reap_invalid()

def reap_stalled(self):
"""Remove any stalled cache entries"""
stall_timeout = int(self.options.get('image_cache_stall_timeout',
86400))
return self._reap_old_files(self.incomplete_path, 'stalled',
grace=stall_timeout)
incomplete_stall_time = int(self.options.get('image_cache_stall_time',
DEFAULT_STALL_TIME))
self.reap_stalled(incomplete_stall_time)


def get_all_regular_files(basepath):
Expand Down
2 changes: 1 addition & 1 deletion glance/image_cache/queue_image.py
View file
Expand Up @@ -81,4 +81,4 @@ def run(self, images):
def app_factory(global_config, **local_conf):
conf = global_config.copy()
conf.update(local_conf)
return Prefetcher(conf)
return Queuer(conf)
17 changes: 17 additions & 0 deletions glance/tests/unit/test_image_cache.py
View file
Expand Up @@ -123,6 +123,23 @@ def test_delete_all(self):
for image_id in (1, 2):
self.assertFalse(self.cache.is_cached(image_id))

@skip_if_disabled
def test_clean_stalled(self):
"""
Test the clean method removes expected images
"""
incomplete_file_path = os.path.join(self.cache_dir, 'incomplete', '1')
incomplete_file = open(incomplete_file_path, 'w')
incomplete_file.write(FIXTURE_DATA)
incomplete_file.close()

self.assertTrue(os.path.exists(incomplete_file_path))

self.cache.options['image_cache_stall_time'] = 0
self.cache.clean()

self.assertFalse(os.path.exists(incomplete_file_path))

@skip_if_disabled
def test_prune(self):
"""
Expand Down

0 comments on commit 0135c42

Please sign in to comment.