cache: use redis as transverse cache

fix #2619
http://rietveld.coopengo.com/15420001/

trytond/cache.py

@@ -6,9 +6,12 @@
 from sql import Table
 from sql.functions import CurrentTimestamp
 
+from trytond.config import config
 from trytond.transaction import Transaction
 
-__all__ = ['Cache', 'LRUDict']
+from trytond.cache_redis import Redis
+
+__all__ = ['_Cache', 'Cache', 'LRUDict']
 
 
 def freeze(o):
@@ -20,7 +23,7 @@ def freeze(o):
         return o
 
 
-class Cache(object):
+class _Cache(object):
     """
     A key value LRU cache with size limit.
     """
@@ -32,6 +35,8 @@ def __init__(self, name, size_limit=1024, context=True):
         self.size_limit = size_limit
         self.context = context
         self._cache = {}
+        assert name not in set([i._name for i in self._cache_instance]), \
+            '%s is already used' % name
         self._cache_instance.append(self)
         self._name = name
         self._timestamp = None
@@ -66,43 +71,42 @@ def set(self, key, value):
                 pass
         return value
 
+    def _empty(self, dbname):
+        self._cache[dbname] = LRUDict(self.size_limit)
+
     def clear(self):
         cursor = Transaction().cursor
-        Cache.reset(cursor.dbname, self._name)
+        with self._resets_lock:
+            self._resets.setdefault(cursor.dbname, set())
+            self._resets[cursor.dbname].add(self._name)
         with self._lock:
-            self._cache[cursor.dbname] = LRUDict(self.size_limit)
+            self._empty(cursor.dbname)
 
-    @staticmethod
-    def clean(dbname):
+    @classmethod
+    def clean(cls, dbname):
         with Transaction().new_cursor():
             cursor = Transaction().cursor
             table = Table('ir_cache')
             cursor.execute(*table.select(table.timestamp, table.name))
             timestamps = {}
             for timestamp, name in cursor.fetchall():
                 timestamps[name] = timestamp
-        for inst in Cache._cache_instance:
+        for inst in cls._cache_instance:
             if inst._name in timestamps:
                 with inst._lock:
                     if (not inst._timestamp
                             or timestamps[inst._name] > inst._timestamp):
                         inst._timestamp = timestamps[inst._name]
-                        inst._cache[dbname] = LRUDict(inst.size_limit)
-
-    @staticmethod
-    def reset(dbname, name):
-        with Cache._resets_lock:
-            Cache._resets.setdefault(dbname, set())
-            Cache._resets[dbname].add(name)
+                        inst._empty(dbname)
 
-    @staticmethod
-    def resets(dbname):
+    @classmethod
+    def resets(cls, dbname):
         with Transaction().new_cursor():
             cursor = Transaction().cursor
             table = Table('ir_cache')
-            with Cache._resets_lock:
-                Cache._resets.setdefault(dbname, set())
-                for name in Cache._resets[dbname]:
+            with cls._resets_lock:
+                cls._resets.setdefault(dbname, set())
+                for name in cls._resets[dbname]:
                     cursor.execute(*table.select(table.name,
                             where=table.name == name))
                     if cursor.fetchone():
@@ -114,7 +118,7 @@ def resets(dbname):
                         cursor.execute(*table.insert(
                                 [table.timestamp, table.name],
                                 [[CurrentTimestamp(), name]]))
-                Cache._resets[dbname].clear()
+                cls._resets[dbname].clear()
             cursor.commit()
 
     @classmethod
@@ -123,6 +127,30 @@ def drop(cls, dbname):
             inst._cache.pop(dbname, None)
 
 
+class Cache(object):
+    def __new__(cls, *args, **kwargs):
+        use_redis = config.get('cache', 'redis', default=None)
+        if use_redis is None:
+            return _Cache(*args, **kwargs)
+        else:
+            return Redis(*args, **kwargs)
+
+    @staticmethod
+    def clean(dbname):
+        _Cache.clean(dbname)
+        Redis.clean(dbname)
+
+    @staticmethod
+    def resets(dbname):
+        _Cache.resets(dbname)
+        Redis.resets(dbname)
+
+    @staticmethod
+    def drop(dbname):
+        _Cache.drop(dbname)
+        Redis.drop(dbname)
+
+
 class LRUDict(OrderedDict):
     """
     Dictionary with a size limit.

trytond/cache_redis.py

@@ -0,0 +1,144 @@
+from threading import Lock
+import msgpack
+from decimal import Decimal
+import datetime
+from urlparse import urlparse
+import redis
+
+from trytond.config import config
+from trytond.transaction import Transaction
+
+__all__ = ['Redis']
+
+
+def freeze(o):
+    if isinstance(o, (set, tuple, list)):
+        return tuple(freeze(x) for x in o)
+    elif isinstance(o, dict):
+        return frozenset((x, freeze(y)) for x, y in o.iteritems())
+    else:
+        return o
+
+
+def encode_hook(o):
+    if isinstance(o, Decimal):
+        return {
+            '__decimal__': True,
+            'data': str(o)
+        }
+    if isinstance(o, datetime.datetime):
+        return {
+            '__datetime__': True,
+            'data': (o.year, o.month, o.day, o.hour, o.minute, o.second,
+                o.microsecond)
+        }
+    if isinstance(o, datetime.date):
+        return {
+            '__date__': True,
+            'data': (o.year, o.month, o.day)
+        }
+    if isinstance(o, datetime.time):
+        return {
+            '__time__': True,
+            'data': (o.hour, o.minute, o.second, o.microsecond)
+        }
+    if isinstance(o, datetime.timedelta):
+        return {
+            '__timedelta__': True,
+            'data': o.total_seconds()
+        }
+    if isinstance(o, set):
+        return {
+            '__set__': True,
+            'data': tuple(o)
+        }
+    return o
+
+
+def decode_hook(o):
+    if '__decimal__' in o:
+        return Decimal(o['data'])
+    elif '__datetime__' in o:
+        return datetime.datetime(*o['data'])
+    elif '__date__' in o:
+        return datetime.date(*o['data'])
+    elif '__time__' in o:
+        return datetime.time(*o['data'])
+    elif '__timedelta__' in o:
+        return datetime.timedelta(o['data'])
+    elif '__set__' in o:
+        return set(o['data'])
+    return o
+
+
+class Redis(object):
+    _cache_instance = []
+    _client = None
+    _client_check_lock = Lock()
+
+    @classmethod
+    def ensure_client(cls):
+        with cls._client_check_lock:
+            if cls._client is None:
+                redis_url = config.get('cache', 'redis')
+                url = urlparse(redis_url)
+                assert url.scheme == 'redis', 'invalid redis url'
+                host = url.hostname
+                port = url.port
+                db = url.path.strip('/')
+                cls._client = redis.StrictRedis(host=host, port=port, db=db)
+
+    def __init__(self, name, size_limit=1024, context=True):
+        self.context = context
+        assert name not in set([i._name for i in self._cache_instance]), \
+            '%s is already used' % name
+        self._cache_instance.append(self)
+        self._name = name
+        self.ensure_client()
+
+    def _namespace(self, dbname=None):
+        if dbname is None:
+            dbname = Transaction().cursor.dbname
+        return '%s:%s' % (self._name, dbname)
+
+    def _key(self, key):
+        if self.context:
+            t = Transaction()
+            res = freeze((key, t.user, t.context))
+        else:
+            res = freeze(key)
+        return '%x' % hash(res)
+
+    def get(self, key, default=None):
+        namespace = self._namespace()
+        key = self._key(key)
+        result = self._client.hget(namespace, key)
+        if result is None:
+            return default
+        else:
+            return msgpack.unpackb(result,  encoding='utf-8',
+                object_hook=decode_hook)
+
+    def set(self, key, value):
+        namespace = self._namespace()
+        key = self._key(key)
+        value = msgpack.packb(value, use_bin_type=True, default=encode_hook)
+        self._client.hset(namespace, key, value)
+
+    def clear(self):
+        namespace = self._namespace()
+        self._client.delete(namespace)
+
+    @classmethod
+    def clean(cls, dbname):
+        pass
+
+    @classmethod
+    def resets(cls, dbname):
+        pass
+
+    @classmethod
+    def drop(cls, dbname):
+        if cls._client is not None:
+            for inst in cls._cache_instance:
+                cls._client.delete(inst._namespace(dbname))

trytond/ir/translation.py

@@ -75,7 +75,7 @@ class Translation(ModelSQL, ModelView):
     overriding_module = fields.Char('Overriding Module', readonly=True)
     _translation_cache = Cache('ir.translation', size_limit=10240,
         context=False)
-    _get_language_cache = Cache('ir.translation')
+    _get_language_cache = Cache('ir.translation.lang')
 
     @classmethod
     def __setup__(cls):

trytond/ir/ui/view.py

@@ -16,7 +16,7 @@
 from trytond.transaction import Transaction
 from trytond.wizard import Wizard, StateView, Button
 from trytond.pool import Pool
-from trytond.cache import Cache
+from trytond.cache import _Cache as Cache
 from trytond.rpc import RPC
 
 __all__ = [