Better ThreaedDict implementation using threadlocal (tx Ben Hoyt)

Using threading.local instead of managing thread-local state manually
improved the performance by 50x. Also added a threadlocal implementation
for python 2.3 as it doesn't have threading.local.

web/application.py

@@ -107,15 +107,9 @@ def _unload(self):
                 web.ctx.fullpath = oldctx.fullpath
 
     def _cleanup(self):
-        #@@@
-        # Since the CherryPy Webserver uses thread pool, the thread-local state is never cleared.
-        # This interferes with the other requests. 
-        # clearing the thread-local storage to avoid that.
-        # see utils.ThreadedDict for details
-        import threading
-        t = threading.currentThread()
-        if hasattr(t, '_d'):
-            del t._d
+        # Threads can be recycled by WSGI servers.
+        # Clearing up all thread-local state to avoid interefereing with subsequent requests.
+        utils.ThreadedDict.clear_all()
 
     def add_mapping(self, pattern, classname):
         self.mapping += (pattern, classname)

web/python23.py

@@ -0,0 +1,46 @@
+"""Python 2.3 compatabilty"""
+import threading
+
+class threadlocal(object):
+    """Implementation of threading.local for python2.3.
+    """
+    def __getattribute__(self, name):
+        if name == "__dict__":
+            return threadlocal._getd(self)
+        elif name.startswith("__"):
+            # for handling special atrributes like __class__ etc.
+            return object.__getattribute__(self, name)
+        else:
+            try:
+                return self.__dict__[name]
+            except KeyError:
+                raise AttributeError, name
+
+    def __setattr__(self, name, value):
+        self.__dict__[name] = value
+
+    def __delattr__(self, name):
+        try:
+            del self.__dict__[name]
+        except KeyError:
+            raise AttributeError, name
+
+    def _getd(self):
+        t = threading.currentThread()
+        if not hasattr(t, '_d'):
+            # using __dict__ of thread as thread local storage
+            t._d = {}
+
+        _id = id(self)
+        # there could be multiple instances of threadlocal.
+        # use id(self) as key
+        if _id not in t._d:
+            t._d[_id] = {}
+        return t._d[_id]
+
+if __name__ == '__main__':
+     d = threadlocal()
+     d.x = 1
+     print d.__dict__
+     print d.x
+

web/session.py

@@ -41,12 +41,13 @@ def __init__(self, message):
 class Session(utils.ThreadedDict):
     """Session management for web.py
     """
+    __slots__ = ["store", "_initializer", "_last_cleanup_time", "_config"]
 
     def __init__(self, app, store, initializer=None):
-        self.__dict__['store'] = store
-        self.__dict__['_initializer'] = initializer
-        self.__dict__['_last_cleanup_time'] = 0
-        self.__dict__['_config'] = utils.storage(web.config.session_parameters)
+        self.store = store
+        self._initializer = initializer
+        self._last_cleanup_time = 0
+        self._config = utils.storage(web.config.session_parameters)
 
         if app:
             app.add_processor(self._processor)
@@ -137,7 +138,7 @@ def _cleanup(self):
         timeout = self._config.timeout
         if current_time - self._last_cleanup_time > timeout:
             self.store.cleanup(timeout)
-            self.__dict__['_last_cleanup_time'] = current_time
+            self._last_cleanup_time = current_time
 
     def expired(self):
         """Called when an expired session is atime"""

web/utils.py

@@ -43,6 +43,11 @@
 try: set
 except NameError:
     from sets import Set as set
+
+try:
+    from threading import local as threadlocal
+except ImportError:
+    from python23 import threadlocal
 
 class Storage(dict):
     """
@@ -1153,7 +1158,7 @@ def tryall(context, prefix=None):
     for (key, value) in results.iteritems():
         print ' '*2, str(key)+':', value
 
-class ThreadedDict:
+class ThreadedDict(threadlocal):
     """
     Thread local storage.
     
@@ -1170,30 +1175,87 @@ class ThreadedDict:
         >>> d.x
         1
     """
-    def __getattr__(self, key):
-        return getattr(self._getd(), key)
+    _instances = set()
+
+    def __init__(self):
+        ThreadedDict._instances.add(self)
+
+    def __del__(self):
+        ThreadedDict._instances.remove(self)
+
+    def __hash__(self):
+        return id(self)
+
+    def clear_all():
+        """Clears all ThreadedDict instances.
+        """
+        for t in ThreadedDict._instances:
+            t.clear()
+    clear_all = staticmethod(clear_all)
+
+    # Define all these methods to more or less fully emulate dict -- attribute access
+    # is built into threading.local.
 
-    def __setattr__(self, key, value):
-        return setattr(self._getd(), key, value)
+    def __getitem__(self, key):
+        return self.__dict__[key]
 
-    def __delattr__(self, key):
-        return delattr(self._getd(), key)
+    def __setitem__(self, key, value):
+        self.__dict__[key] = value
 
-    def __hash__(self): 
-        return id(self)
+    def __delitem__(self, key):
+        del self.__dict__[key]
+
+    def __contains__(self, key):
+        return key in self.__dict__
+
+    has_key = __contains__
+
+    def clear(self):
+        self.__dict__.clear()
 
-    def _getd(self):
-        t = threading.currentThread()
-        if not hasattr(t, '_d'):
-            # using __dict__ of thread as thread local storage
-            t._d = {}
+    def copy(self):
+        return self.__dict__.copy()
 
-        # there could be multiple instances of ThreadedDict.
-        # use self as key
-        if self not in t._d:
-            t._d[self] = storage()
-        return t._d[self]
+    def get(self, key, default=None):
+        return self.__dict__.get(key, default)
 
+    def items(self):
+        return self.__dict__.items()
+
+    def iteritems(self):
+        return self.__dict__.iteritems()
+
+    def keys(self):
+        return self.__dict__.keys()
+
+    def iterkeys(self):
+        return self.__dict__.iterkeys()
+
+    iter = iterkeys
+
+    def values(self):
+        return self.__dict__.values()
+
+    def itervalues(self):
+        return self.__dict__.itervalues()
+
+    def pop(self, key, *args):
+        return self.__dict__.pop(key, *args)
+
+    def popitem(self):
+        return self.__dict__.popitem()
+
+    def setdefault(self, key, default=None):
+        return self.__dict__.setdefault(key, default)
+
+    def update(self, *args, **kwargs):
+        self.__dict__.update(*args, **kwargs)
+
+    def __repr__(self):
+        return '<ThreadedDict %r>' % self.__dict__
+
+    __str__ = __repr__
+
 threadeddict = ThreadedDict
 
 def autoassign(self, locals):