Skip to content
Browse files

adding some basic data type handling

The basic code comes from python-memcache.
The compression code was ripped out (for now)
and the focus is on getting strings, ints, longs,
and dictionaries in via the .set() API.

Dicitonaries can now be stored without any additional
json.dumps() code, but end up as strings on output.

This is a known shortcoming, but will not be addressed
until we have some consistent data type handling plan
among the SDKs.

Change-Id: I3b39eee1579ef860d87a5cb94b4bf630a9cfe7d2
Reviewed-on: http://review.couchbase.org/19877
Reviewed-by: Pavel Paulau <pavel.paulau@gmail.com>
Tested-by: Benjamin Young <benjamin@couchbase.com>
  • Loading branch information...
1 parent 5bb6178 commit d7049edd8f42f5954b8ea58a75bf2ef89378cc43 @BigBlueHat BigBlueHat committed with BigBlueHat Aug 14, 2012
Showing with 88 additions and 2 deletions.
  1. +10 −1 couchbase/client.py
  2. +5 −0 couchbase/constants.py
  3. +55 −1 couchbase/memcachedclient.py
  4. +18 −0 couchbase/tests/test_client.py
View
11 couchbase/client.py
@@ -282,7 +282,16 @@ def save(self, document):
def __setitem__(self, key, value):
if isinstance(value, dict):
- self.set(key, value['expiration'], value['flags'], value['value'])
+ if 'expiration' in value or 'flags' in value:
+ assert 'value' in value
+ if isinstance(value['value'], dict):
+ v = json.dumps(value['value'])
+ else:
+ v = value['value']
+ self.set(key, value.get('expiration', 0),
+ value.get('flags', 0), v)
+ else:
+ self.set(key, 0, 0, json.dumps(value))
else:
self.set(key, 0, 0, value)
View
5 couchbase/constants.py
@@ -192,6 +192,11 @@ class MemcachedConstants(object):
ERR_AUTH = 0x20
ERR_AUTH_CONTINUE = 0x21
+ FLAG_PICKLE = 0x1 # 1<<0 in python-memcache
+ FLAG_INTEGER = 0x2 # 1<<1 in python-memcache
+ FLAG_LONG = 0x4 # 1<<2 in python-memcache
+ FLAG_COMPRESSED = 0x8 # 1<<3 in python-memcache
+
class VBucketAwareConstants(MemcachedConstants):
"""Constants for the vBucket aware extensions to the memcached protocol."""
View
56 couchbase/memcachedclient.py
@@ -21,11 +21,16 @@
import zlib
import struct
import warnings
+import json
+import cPickle as pickle
+import cStringIO as StringIO
from couchbase.logger import logger
from couchbase.constants import MemcachedConstants
from couchbase.exception import MemcachedError
+log = logger("client")
+
class MemcachedClient(object):
"""Simple memcached client."""
@@ -150,8 +155,32 @@ def _set_vbucket_id(self, key, vbucket):
else:
self.vbucketId = vbucket
+ def _val_to_store_info(self, val):
+ """Using some prior art from python-memcache
+
+ Transform val to a storable representation, returning a tuple of the
+ flags, the length of the new value, and the new value itself."""
+ flags = 0
+ if isinstance(val, str):
+ pass
+ elif isinstance(val, int):
+ flags |= MemcachedConstants.FLAG_INTEGER
+ val = "%d" % val
+ elif isinstance(val, long):
+ flags |= MemcachedConstants.FLAG_LONG
+ val = "%d" % val
+ else:
+ flags |= MemcachedConstants.FLAG_PICKLE
+ file = StringIO()
+ pickler = pickle.Pickler(file)
+ pickler.dump(val)
+ val = file.getvalue()
+
+ return (flags, len(val), val)
+
def set(self, key, exp, flags, val, vbucket=-1):
"""Set a value in the memcached server."""
+ flags, len, val = self._val_to_store_info(val)
self._set_vbucket_id(key, vbucket)
return self._mutate(MemcachedConstants.CMD_SET, key, exp, flags, 0,
val)
@@ -168,10 +197,35 @@ def replace(self, key, exp, flags, val, vbucket=-1):
return self._mutate(MemcachedConstants.CMD_REPLACE, key, exp, flags, 0,
val)
+ def _recv_value(self, val, flags=None):
+ """python-memcache flag parsing. Including here for compatibility with
+ prior art in the Python world.
+
+ From that baseline, a JSON flag (0x0) and handling/parsing has been
+ added."""
+ if flags & MemcachedConstants.FLAG_INTEGER:
+ val = int(val)
+ elif flags & MemcachedConstants.FLAG_LONG:
+ val = long(val)
+ elif flags & MemcachedConstants.FLAG_PICKLE:
+ try:
+ file = StringIO(val)
+ unpickler = pickle.Unpickler(file)
+ val = unpickler.load()
+ except Exception, e:
+ log.error('Pickle error: %s\n' % e)
+ return None
+ else:
+ log.warn("unknown flags on get: %x\n" % flags)
+
+ return val
+
def _parse_get(self, data, klen=0):
flags = struct.unpack(MemcachedConstants.GET_RES_FMT, data[-1][:4])[0]
rv = data[-1][4 + klen:]
- if isinstance(rv, str) and rv.isdigit():
+ if flags:
+ rv = self._recv_value(rv, flags)
+ elif isinstance(rv, str) and rv.isdigit():
rv = int(rv)
return flags, data[1], rv
View
18 couchbase/tests/test_client.py
@@ -19,6 +19,7 @@
import warnings
import uuid
import time
+import json
from nose.plugins.attrib import attr
from nose.plugins.skip import SkipTest
@@ -224,5 +225,22 @@ def test_gat(self):
time.sleep(3)
self.assertTrue(self.client.get(key)[2] == value)
+ @attr(cbv="1.0.0")
+ def test_setitem(self):
+ # test int
+ self.client['int'] = 10
+ self.assertEqual(self.client['int'][2], 10)
+ # test long
+ self.client['long'] = long(10)
+ self.assertEqual(self.client['long'][2], long(10))
+ # test string
+ self.client['str'] = 'string'
+ self.assertEqual(self.client['str'][2], 'string')
+ # test json
+ # dictionaries are serialized to JSON objects
+ self.client['json'] = {'json':'obj'}
+ # but come out as strings for now
+ self.assertEqual(self.client['json'][2], json.dumps({'json':'obj'}))
+
if __name__ == "__main__":
unittest.main()

0 comments on commit d7049ed

Please sign in to comment.
Something went wrong with that request. Please try again.