working on pubsub app

stdnet/apps/pubsub.py

@@ -1,54 +1,76 @@
+import logging
+
 from inspect import isclass
+from collections import deque
 
 from stdnet import getdb, AsyncObject
 from stdnet.utils.encoders import Json
 from stdnet.utils import is_string
+
+
+logger = logging.getLogger('stdnet.pubsub')
+
+
+class PubSubBase(object):
+    pickler = Json
 
-
-class Publisher(object):
-    '''Class which publish messages to message queues.'''
-    def __init__(self, server = None, pickler = Json):
+    def __init__(self, server=None, pickler=None):
+        pickler = pickler or self.pickler
         if isclass(pickler):
             pickler = pickler()
         self.pickler = pickler
-        self.client = getdb(server).client
+        self.server = getdb(server)
 
+
+class Publisher(PubSubBase):
+    '''Class which publish messages to message queues.'''        
     def publish(self, channel, data):
         data = self.pickler.dumps(data)
-        #return self.backend.publish(channel, data)
-        return self.client.execute_command('PUBLISH', channel, data)
+        return self.server.publish(channel, data)
 
 
-class Subscriber(AsyncObject):
-    '''Subscribe to '''
-    def __init__(self, server = None, pickler = Json):
-        if isclass(pickler):
-            pickler = pickler()
-        self.pickler = pickler
-        self.client = getdb(server).subscriber()
+class Subscriber(PubSubBase):
+    '''A subscriber to channels'''
+    def __init__(self, server=None, pickler=None):
+        super(Subscriber, self).__init__(server, pickler)
+        self.channels = {}
+        self.patterns = {}
+        self._subscriber = self.server.subscriber(
+                                message_callback=self.message_callback)
 
     def disconnect(self):
-        self.client.disconnect()
+        self._subscriber.disconnect()
 
     def subscription_count(self):
-        return self.client.subscription_count()
+        return self._subscriber.subscription_count()
 
-    def subscribe(self, channels):
-        return self.client.subscribe(channels)
+    def subscribe(self, *channels):
+        return self._subscriber.subscribe(self.channel_list(channels))
 
-    def unsubscribe(self, channels = None):
-        return self.client.unsubscribe(channels)
-
-    def psubscribe(self, channels):
-        return self.client.psubscribe(channels)
+    def unsubscribe(self, *channels):
+        return self._subscriber.unsubscribe(self.channel_list(channels))
 
-    def punsubscribe(self, channels = None):
-        return self.client.punsubscribe(channels)
+    def psubscribe(self, *channels):
+        return self._subscriber.psubscribe(self.channel_list(channels))
 
-    def pull(self, timeout = None, count = None):
-        '''Retrieve new messages from the subscribed channels.
+    def punsubscribe(self, *channels):
+        return self._subscriber.punsubscribe(self.channel_list(channels))
 
-:parameter timeout: Optional timeout in seconds.
-:parameter count: Optional number of messages to retrieve.'''
-        return self.client.pull(timeout, count, self.pickler.loads)
-
+    def message_callback(self, command, channel, message=None):
+        if command == 'subscribe':
+            self.channels[channel] = deque()
+        elif command == 'unsubscribe':
+            self.channels.pop(channel, None)
+        elif channel in self.channels:
+            self.channels.append(message)
+        else:
+            logger.warn('Got message for unsubscribed channel "%s"' % channel)
+
+    def channel_list(self, channels):
+        ch = []
+        for channel in channels:
+            if not isinstance(channel, (list, tuple)):
+                ch.append(channel)
+            else:
+                ch.extend(channel)
+        return ch

stdnet/backends/base.py

@@ -203,7 +203,7 @@ class BackendDataServer(object):
     structure_module = None
     struct_map = {}
 
-    def __init__(self, name, address, pickler = None,
+    def __init__(self, name, address, pickler=None,
                  charset='utf-8', connection_string='',
                  prefix=None, **params):
         self.__name = name
@@ -233,9 +233,6 @@ def __eq__(self, other):
     def issame(self, other):
         return self.client == other.client
 
-    def cursor(self, pipelined=False):
-        return self
-
     def disconnect(self):
         '''Disconnect the connection.'''
         pass
@@ -244,13 +241,6 @@ def __repr__(self):
         return self.connection_string
     __str__ = __repr__
 
-    def isempty(self):
-        '''Returns ``True`` if the database has no keys.'''
-        keys = self.keys()
-        if not hasattr(keys,'__len__'):
-            keys = list(keys)
-        return len(keys)
-
     def make_objects(self, meta, data, related_fields=None):
         '''Generator of :class:`stdnet.odm.StdModel` instances with data
 from database.
@@ -289,8 +279,11 @@ def make_objects(self, meta, data, related_fields=None):
     def objects_from_db(self, meta, data, related_fields=None):
         return list(self.make_objects(meta, data, related_fields))
 
-    def structure(self, instance, client = None):
-        '''Create a backend :class:`stdnet.odm.Structure` handler.'''
+    def structure(self, instance, client=None):
+        '''Create a backend :class:`stdnet.odm.Structure` handler.
+        
+:parameter instance: a :class:`stdnet.odm.Structure`
+:parameter client: Optional client handler'''
         struct = self.struct_map.get(instance._meta.name)
         if struct is None:
             raise ModelNotAvailable('structure "{0}" is not available for\
@@ -315,36 +308,40 @@ def basekey(self, meta, *args):
 
     # PURE VIRTUAL METHODS
 
-    def setup_connection(self, address):  # pragma: no cover
+    def setup_connection(self, address):
         '''Callback during initialization. Implementation should override
 this function for customizing their handling of connection parameters. It
 must return a instance of the backend handler.'''
         raise NotImplementedError()
 
-    def execute_session(self, session, callback):   # pragma: no cover
+    def execute_session(self, session, callback):
         '''Execute a :class:`stdnet.odm.Session` in the backend server.'''
         raise NotImplementedError()
 
-    def model_keys(self, meta):     # pragma: no cover
+    def model_keys(self, meta):
         '''Return a list of database keys used by model *model*'''
         raise NotImplementedError()
 
-    def instance_keys(self, obj):   # pragma: no cover
+    def instance_keys(self, obj):
         '''Return a list of database keys used by instance *obj*'''
         raise NotImplementedError()
 
-    def as_cache(self):     # pragma: no cover
+    def as_cache(self):
         raise NotImplementedError('This backend cannot be used as cache')
 
-    def clear(self):    # pragma: no cover
+    def clear(self):
         """Remove *all* values from the database at once."""
         raise NotImplementedError()
 
-    def flush(self, meta=None, pattern=None):   # pragma: no cover
+    def flush(self, meta=None, pattern=None):
         '''Flush all model keys from the database'''
         raise NotImplementedError()
 
-    def subscriber(self):   # pragma: no cover
+    def publish(self, channel, message):
+        '''Publish a message to a *channel*'''
+        raise NotImplementedError('This backend cannot publish messages')
+
+    def subscriber(self, **kwargs):
         raise NotImplementedError()
 
 

stdnet/backends/main.py

@@ -64,7 +64,7 @@ def get_connection_string(scheme, address, params):
 
 def getdb(backend_uri=None, **kwargs):
     '''get a backend database'''
-    if isinstance(backend_uri,BackendDataServer):
+    if isinstance(backend_uri, BackendDataServer):
         return backend_uri
     backend_uri = backend_uri or settings.DEFAULT_BACKEND
     if not backend_uri:

stdnet/backends/redisb.py

@@ -782,9 +782,6 @@ def get(self, id, default = None):
             return self.pickler.loads(v)
         else:
             return default
-
-    def cursor(self, pipelined = False):
-        return self.client.pipeline() if pipelined else self.client
 
     def disconnect(self):
         self.client.connection_pool.disconnect()
@@ -969,5 +966,8 @@ def flush_structure(self, sm, pipe):
             pipe.add_callback(
                         partial(structure_session_callback,sm))
 
-    def subscriber(self):
-        return redis.Subscriber(self.client)    
+    def publish(self, channel, message):
+        return self.client.execute_command('PUBLISH', channel, message)
+
+    def subscriber(self, **kwargs):
+        return redis.Subscriber(self.client, **kwargs)    

stdnet/lib/redis/connection.py

@@ -98,7 +98,7 @@ def __init__(self, client, connection, command_name, args,
         self.connection = connection
         self.command_name = command_name
         self.args = args
-        self.release_connection = release_connection
+        self._release_connection = release_connection
         self.options = options
         self.tried = 0
         self._raw_response = []
@@ -116,6 +116,13 @@ def __init__(self, client, connection, command_name, args,
         else:
             self.command = None
 
+    @property
+    def release_connection(self):
+        if self.connection.streaming:
+            return False
+        else:
+            return self._release_connection
+
     @property
     def num_responses(self):
         if self.command_name:
@@ -265,12 +272,14 @@ class Connection(object):
     def __init__(self, pool, password=None,
                  socket_timeout=None, encoding='utf-8',
                  encoding_errors='strict', reader_class=None,
-                 decode = False, **kwargs):
+                 decode = False, streaming=False,
+                 **kwargs):
         self.pool = pool
         self.password = password
         self.socket_timeout = socket_timeout
         self.encoding = encoding
         self.encoding_errors = encoding_errors
+        self.streaming = streaming
         self.__sock = None
         if reader_class is None:
             if settings.REDIS_PY_PARSER: