Permalink
Find file
Fetching contributors…
Cannot retrieve contributors at this time
231 lines (190 sloc) 8.05 KB
"Modified memcached cache backend"
import time
from threading import local
from django.core.cache.backends.base import BaseCache, InvalidCacheBackendError
from django.utils.hashcompat import sha_constructor
from django.utils.encoding import smart_str
from django.conf import settings
try:
from django.utils import importlib
except ImportError:
import importlib
try:
import pylibmc as memcache
NotFoundError = memcache.NotFound
using_pylibmc = True
except ImportError:
using_pylibmc = False
try:
import memcache
NotFoundError = ValueError
except ImportError:
raise InvalidCacheBackendError('Memcached cache backend requires ' +
'either the "pylibmc" or "memcache" library')
# Flavor is used amongst multiple apps to differentiate the "flavor" of the
# environment. Examples of flavors are 'prod', 'staging', 'dev', and 'test'.
FLAVOR = getattr(settings, 'FLAVOR', '')
CACHE_VERSION = str(getattr(settings, 'CACHE_VERSION', 1))
CACHE_BEHAVIORS = getattr(settings, 'CACHE_BEHAVIORS', {'hash': 'crc'})
CACHE_KEY_MODULE = getattr(settings, 'CACHE_KEY_MODULE', 'newcache')
CACHE_HERD_TIMEOUT = getattr(settings, 'CACHE_HERD_TIMEOUT', 60)
class Marker(object):
pass
MARKER = Marker()
def get_key(key):
"""
Returns a hashed, versioned, flavored version of the string that was input.
"""
hashed = sha_constructor(smart_str(key)).hexdigest()
return ''.join((FLAVOR, '-', CACHE_VERSION, '-', hashed))
key_func = importlib.import_module(CACHE_KEY_MODULE).get_key
class CacheClass(BaseCache):
def __init__(self, server, params):
super(CacheClass, self).__init__(params)
self._servers = server.split(';')
self._use_binary = bool(params.get('binary'))
self._local = local()
@property
def _cache(self):
"""
Implements transparent thread-safe access to a memcached client.
"""
client = getattr(self._local, 'client', None)
if client:
return client
# Use binary mode if it's both supported and requested
if using_pylibmc and self._use_binary:
client = memcache.Client(self._servers, binary=True)
else:
client = memcache.Client(self._servers)
# If we're using pylibmc, set the behaviors according to settings
if using_pylibmc:
client.behaviors = CACHE_BEHAVIORS
self._local.client = client
return client
def _pack_value(self, value, timeout):
"""
Packs a value to include a marker (to indicate that it's a packed
value), the value itself, and the value's timeout information.
"""
herd_timeout = (timeout or self.default_timeout) + int(time.time())
return (MARKER, value, herd_timeout)
def _unpack_value(self, value, default=None):
"""
Unpacks a value and returns a tuple whose first element is the value,
and whose second element is whether it needs to be herd refreshed.
"""
try:
marker, unpacked, herd_timeout = value
except (ValueError, TypeError):
return value, False
if not isinstance(marker, Marker):
return value, False
if herd_timeout < int(time.time()):
return unpacked, True
return unpacked, False
def _get_memcache_timeout(self, timeout):
"""
Memcached deals with long (> 30 days) timeouts in a special
way. Call this function to obtain a safe value for your timeout.
"""
if timeout is None:
timeout = self.default_timeout
if timeout > 2592000: # 60*60*24*30, 30 days
# See http://code.google.com/p/memcached/wiki/FAQ
# "You can set expire times up to 30 days in the future. After that
# memcached interprets it as a date, and will expire the item after
# said date. This is a simple (but obscure) mechanic."
#
# This means that we have to switch to absolute timestamps.
timeout += int(time.time())
return timeout
def add(self, key, value, timeout=None, herd=True):
# If the user chooses to use the herd mechanism, then encode some
# timestamp information into the object to be persisted into memcached
if herd and timeout != 0:
packed = self._pack_value(value, timeout)
real_timeout = (self._get_memcache_timeout(timeout) +
CACHE_HERD_TIMEOUT)
else:
packed = value
real_timeout = self._get_memcache_timeout(timeout)
return self._cache.add(key_func(key), packed, real_timeout)
def get(self, key, default=None):
encoded_key = key_func(key)
packed = self._cache.get(encoded_key)
if packed is None:
return default
val, refresh = self._unpack_value(packed)
# If the cache has expired according to the embedded timeout, then
# shove it back into the cache for a while, but act as if it was a
# cache miss.
if refresh:
self._cache.set(encoded_key, val,
self._get_memcache_timeout(CACHE_HERD_TIMEOUT))
return default
return val
def set(self, key, value, timeout=None, herd=True):
# If the user chooses to use the herd mechanism, then encode some
# timestamp information into the object to be persisted into memcached
if herd and timeout != 0:
packed = self._pack_value(value, timeout)
real_timeout = (self._get_memcache_timeout(timeout) +
CACHE_HERD_TIMEOUT)
else:
packed = value
real_timeout = self._get_memcache_timeout(timeout)
return self._cache.set(key_func(key), packed, real_timeout)
def delete(self, key):
self._cache.delete(key_func(key))
def get_many(self, keys):
# First, map all of the keys through our key function
rvals = map(key_func, keys)
packed_resp = self._cache.get_multi(rvals)
resp = {}
reinsert = {}
for key, packed in packed_resp.iteritems():
# If it was a miss, treat it as a miss to our response & continue
if packed is None:
resp[key] = packed
continue
val, refresh = self._unpack_value(packed)
if refresh:
reinsert[key] = val
resp[key] = None
else:
resp[key] = val
# If there are values to re-insert for a short period of time, then do
# so now.
if reinsert:
self._cache.set_multi(reinsert,
self._get_memcache_timeout(CACHE_HERD_TIMEOUT))
# Build a reverse map of encoded keys to the original keys, so that
# the returned dict's keys are what users expect (in that they match
# what the user originally entered)
reverse = dict(zip(rvals, keys))
return dict(((reverse[k], v) for k, v in resp.iteritems()))
def close(self, **kwargs):
self._cache.disconnect_all()
def incr(self, key, delta=1):
try:
return self._cache.incr(key_func(key), delta)
except NotFoundError:
raise ValueError("Key '%s' not found" % (key,))
def decr(self, key, delta=1):
try:
return self._cache.decr(key_func(key), delta)
except NotFoundError:
raise ValueError("Key '%s' not found" % (key,))
def set_many(self, data, timeout=None, herd=True):
if herd and timeout != 0:
safe_data = dict(((key_func(k), self._pack_value(v, timeout))
for k, v in data.iteritems()))
else:
safe_data = dict((
(key_func(k), v) for k, v in data.iteritems()))
self._cache.set_multi(safe_data, self._get_memcache_timeout(timeout))
def delete_many(self, keys):
self._cache.delete_multi(map(key_func, keys))
def clear(self):
self._cache.flush_all()