Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Loading…

Please pull to pick up Python client v0.1.2 #39

Merged
2 commits merged into from

1 participant

@jtraupman-li

Made some changes to the Python client to fix problems we were having:

  1. Added some exception handling for malformed JSON, since we were choking on JSON serializers with schemas defined with single quotes instead of double quotes. The Python JSON lib doesn't like this, and we still don't handle such schemas correctly, but we don't completely crash now.

  2. Changed the metadata bootstrap to only parse the XML corresponding the store we're trying to connect to. This should both speed up initializations a little as well as prevent errors in other stores from prevent the client from connecting.

  3. Explicitly added a check with a proper exception for trying to connect to a non-existent store.

Inc'ed the python client version number and added a few tests, too.

Jonathan Tra... added some commits
Jonathan Traupman add some handling for malformed JSON, change metadata bootstrap to on…
…ly parse store that we're trying to connect to
0691960
Jonathan Traupman add check for missing store 9c67920
This issue was closed.
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Feb 2, 2011
  1. add some handling for malformed JSON, change metadata bootstrap to on…

    Jonathan Traupman authored
    …ly parse store that we're trying to connect to
  2. add check for missing store

    Jonathan Traupman authored
This page is out of date. Refresh to see the latest.
View
2  clients/python/README
@@ -60,4 +60,4 @@ and delete() methods. For example:
> client.get("foo")
[]
-The test suite contains many other usage examples.
+The test suite contains many other usage examples.
View
2  clients/python/setup.py
@@ -1,7 +1,7 @@
from setuptools import setup
setup(name='voldemort',
- version='0.1.1',
+ version='0.1.2',
description='Python Voldemort Client',
long_description=('Pure python client for accessing Voldemort key/value stores. ' +
'Supports both raw and JSON stores. Only supports the tcp protocol ' +
View
13 clients/python/tests/test_client.py
@@ -300,3 +300,16 @@ def test_raw_versions(self):
resp = s.get(1)
self.assertEquals(resp, [])
+
+ def test_missing_store(self):
+ """
+ Tests that we get an exception when we try to get a non-existent store.
+ """
+
+ try:
+ s = StoreClient('does-not-exist', [('localhost', 6666)])
+ except VoldemortException:
+ self.assertTrue(True)
+ return
+
+ self.assertTrue(False)
View
45 clients/python/voldemort/client.py
@@ -112,12 +112,22 @@ def __init__(self, store_node):
self.preferred_writes = _int_or_none(_child_text(store_node, "preferred-writes", required=False))
key_serializer_node = _child(store_node, "key-serializer")
- self.key_serializer_type = _child_text(key_serializer_node, "type")
- self.key_serializer = self._create_serializer(self.key_serializer_type, key_serializer_node)
+ try:
+ self.key_serializer_type = _child_text(key_serializer_node, "type")
+ self.key_serializer = self._create_serializer(self.key_serializer_type, key_serializer_node)
+ except serialization.SerializationException, e:
+ logging.warn("Error while creating key serializer for store [%s]: %s" % (self.name, e))
+ self.key_serializer_type = "invalid"
+ self.key_serializer = serialization.UnimplementedSerializer("invalid")
value_serializer_node = _child(store_node, "value-serializer")
- self.value_serializer_type = _child_text(value_serializer_node, "type")
- self.value_serializer = self._create_serializer(self.value_serializer_type, value_serializer_node)
+ try:
+ self.value_serializer_type = _child_text(value_serializer_node, "type")
+ self.value_serializer = self._create_serializer(self.value_serializer_type, value_serializer_node)
+ except serialization.SerializationException, e:
+ logging.warn("Error while creating value serializer for store [%s]: %s" % (self.name, e))
+ self.value_serializer_type = "invalid"
+ self.value_serializer = serialization.UnimplementedSerializer("invalid")
def _create_serializer(self, serializer_type, serializer_node):
if serializer_type not in serialization.SERIALIZER_CLASSES:
@@ -126,10 +136,16 @@ def _create_serializer(self, serializer_type, serializer_node):
return serialization.SERIALIZER_CLASSES[serializer_type].create_from_xml(serializer_node)
@staticmethod
- def parse_stores_xml(xml):
+ def parse_stores_xml(xml, store_name):
doc = minidom.parseString(xml)
- stores = [Store(store_node) for store_node in doc.getElementsByTagName("store")]
- return dict((store.name, store) for store in stores)
+ store_nodes = doc.getElementsByTagName("store")
+ for store_node in store_nodes:
+ name = _child_text(store_node, "name")
+ if name == store_name:
+ return Store(store_node)
+
+ return None
+
class VoldemortException(Exception):
def __init__(self, msg, code = 1):
@@ -147,13 +163,16 @@ def __init__(self, store_name, bootstrap_urls, reconnect_interval = 500, conflic
self.store_name = store_name
self.request_count = 0
self.conflict_resolver = conflict_resolver
- self.nodes, self.stores = self._bootstrap_metadata(bootstrap_urls)
+ self.nodes, self.store = self._bootstrap_metadata(bootstrap_urls, store_name)
+ if not self.store:
+ raise VoldemortException("Cannot find store [%s] at %s" % (store_name, bootstrap_urls))
+
self.node_id = random.randint(0, len(self.nodes) - 1)
self.node_id, self.connection = self._reconnect()
self.reconnect_interval = reconnect_interval
self.open = True
- self.key_serializer = self.stores[store_name].key_serializer
- self.value_serializer = self.stores[store_name].value_serializer
+ self.key_serializer = self.store.key_serializer
+ self.value_serializer = self.store.value_serializer
def _make_connection(self, host, port):
protocol = 'pb0'
@@ -236,7 +255,7 @@ def _receive_response(self, connection):
## Bootstrap cluster metadata from a list of urls of nodes in the cluster.
## The urls are tuples in the form (host, port).
## A dictionary of node_id => node is returned.
- def _bootstrap_metadata(self, bootstrap_urls):
+ def _bootstrap_metadata(self, bootstrap_urls, store_name):
random.shuffle(bootstrap_urls)
for host, port in bootstrap_urls:
logging.debug('Attempting to bootstrap metadata from ' + host + ':' + str(port))
@@ -249,9 +268,9 @@ def _bootstrap_metadata(self, bootstrap_urls):
nodes = Node.parse_cluster(cluster_xmls[0][0])
logging.debug('Bootstrap from ' + host + ':' + str(port) + ' succeeded, found ' + str(len(nodes)) + " nodes.")
stores_xml = self._get_with_connection(connection, 'metadata', 'stores.xml', should_route=False)[0][0]
- stores = Store.parse_stores_xml(stores_xml)
+ store = Store.parse_stores_xml(stores_xml, store_name)
- return nodes, stores
+ return nodes, store
except socket.error, (err_num, message):
logging.warn('Metadata bootstrap from ' + host + ':' + str(port) + " failed: " + message)
finally:
View
20 clients/python/voldemort/serialization/json_serializer.py
@@ -201,6 +201,14 @@ def create_from_xml(node):
Traceback (most recent call last):
...
SerializationException: Schema info has duplicates of version: 0
+
+ JSON with single quotes is NOT valid JSON, even though voldemort doesn't necessarily check for this:
+ >>> xml = minidom.parseString("<serializer><type>json</type>" +
+ ... "<schema-info version=\"0\">{ 'foo':'int32' }</schema-info></serializer>")
+ >>> s = JsonTypeSerializer.create_from_xml(xml)
+ Traceback (most recent call last):
+ ...
+ SerializationException: Error decoding schema JSON
"""
typedef = dict()
@@ -228,11 +236,13 @@ def create_from_xml(node):
if not has_version and len(typedef) > 1:
raise SerializationException('Schema info has version="none" and multiple versions')
- if not has_version:
- return JsonTypeSerializer(typedef[0])
- else:
- return JsonTypeSerializer(typedef)
-
+ try:
+ if not has_version:
+ return JsonTypeSerializer(typedef[0])
+ else:
+ return JsonTypeSerializer(typedef)
+ except simplejson.JSONDecodeError:
+ raise SerializationException('Error decoding schema JSON')
def read(self, input):
r"""
Something went wrong with that request. Please try again.