Permalink
Browse files

cleanup python client and add JSON serialization support to it

  • Loading branch information...
1 parent edc7f3a commit 2100b2050184ef354ff156fcb3319d4392af192d Jonathan Traupman committed Nov 25, 2010
View
@@ -2,6 +2,7 @@
src.dir=src
java.dir=src/java
python.dir=clients/python
+python.proto.dir=clients/python/voldemort/protocol
protobuff.dir=src/proto
lib.dir=lib
classes.dir=dist/classes
View
@@ -22,7 +22,7 @@
<isset property="env.BUILD_NUMBER" />
<not>
<equals arg1="" arg2="${env.BUILD_NUMBER}" trim="yes"/>
- </not>
+ </not>
</and>
</condition>
@@ -112,17 +112,17 @@
<fileset dir="${testclasses.dir}" />
</jar>
</target>
-
+
<target name="protobuff" description="Generate source files from .proto files">
<pathconvert property="proto.sources" pathsep=" ">
<path id="proto-files">
<fileset dir="${protobuff.dir}" />
</path>
</pathconvert>
- <property name="proto.path" location="${protobuff.dir}"/>
+ <property name="proto.path" location="${protobuff.dir}"/>
<property name="javaout.path" location="${java.dir}"/>
- <property name="pythonout.path" location="${python.dir}"/>
+ <property name="pythonout.path" location="${python.proto.dir}"/>
<exec executable="protoc" failonerror="true">
<arg value="--proto_path=${proto.path}"/>
<arg value="--java_out=${javaout.path}"/>
@@ -151,7 +151,7 @@
</fileset>
</jar>
</target>
-
+
<target name="alljar" depends="build, contrib-build" description="Build a jar file that includes all contrib code.">
<jar destfile="${dist.dir}/${name}-${curr.release}-all.jar">
<fileset dir="${classes.dir}">
@@ -166,7 +166,7 @@
</fileset>
</jar>
</target>
-
+
<target name="war" depends="build" description="Build server war file">
<war destfile="${dist.dir}/${name}.war" webxml="web.xml" basedir="${classes.dir}">
<classes dir="${classes.dir}"/>
@@ -312,7 +312,7 @@
<report todir="${contribtesthtml.dir}" format="frames" />
</junitreport>
</target>
-
+
<macrodef name="create-release-artifacts">
<attribute name="version" />
<sequential>
@@ -340,12 +340,12 @@
<target name="snapshot" description="Create a release-snapshot zip file with everything pre-built.">
<create-release-artifacts version="${curr.release.snapshot}" />
</target>
-
+
<target name="release" description="Create a release zip file with everything pre-built.">
<create-release-artifacts version="${curr.release}" />
</target>
-
- <target name="hadoop-benchmark-jar" depends="build, contrib-build"
+
+ <target name="hadoop-benchmark-jar" depends="build, contrib-build"
description="Build a jar file that includes all contrib code plus the necessary jars for running the hadoop benchmark.">
<jar destfile="${dist.dir}/hadoop-benchmark.jar">
<fileset dir="${classes.dir}">
View
@@ -0,0 +1,63 @@
+ABOUT
+
+This directory contains a pure Python implementation of a voldemort client.
+It supports both raw (string) and JSON serialized stores. Only the protocol
+buffer interface over TCP is supported for talking to the server. Only
+server-side routing is supported.
+
+INSTALLING
+
+To install the module, you will need the following dependencies:
+- nose >= 0.11
+- simplejson >= 2.1.1
+- Google protobuf > 2.3.0
+
+The setup process will automatically install nose and simplejson, since they
+are well behaved Python packages. The protobuf module will need to be downloaded
+from https://code.google.com/p/protobuf/downloads/list and installed manually.
+
+Once the dependencies, run the test suite to sanity check things. You need to
+first start up a Voldemort server locally, pointing to the config files in
+tests/voldemort_config. From the root voldemort of the voldemort source tree, run:
+
+> bin/voldemort-server clients/python/tests/voldemort_config
+
+In a separate shell, change into the clients/python directory and run:
+
+> python setup.py nosetests
+
+If all tests pass, you can install the package with the command:
+
+> python setup.py install
+
+This may need to be run as root if you don't have permissions to install to your
+local python library.
+
+USING THE MODULE
+
+To use the client, simple import it into your program with the statement:
+
+import voldemort
+
+To create a client connection, instantiate a StoreClient object:
+
+client = voldemort.StoreClient('store_name', [('node1', 6666), ('node2', 6666)])
+
+The values of the store name and cluster nodes/ports will depend on your particular
+Voldemort setup. The key and value serialization type will be determined
+automatically during client initialization using the values in your cluster's
+stores.xml file.
+
+The StoreClient object implements the get(), get_all(), put(), maybe_put(),
+and delete() methods. For example:
+
+> v1 = client.put("foo", "hello")
+> resp = client.get("foo")
+> resp[0][0]
+"hello"
+
+> client.delete("foo")
+> client.get("foo")
+[]
+
+The test suite contains many other usage examples.
View
@@ -0,0 +1,5 @@
+[nosetests]
+detailed-errors=1
+with-doctest=1
+
+
View
@@ -0,0 +1,15 @@
+from setuptools import setup
+
+setup(name='voldemort',
+ version='0.1',
+ description='Python Voldemort Client',
+ long_description=('Pure python client for accessing Voldemort key/value stores. ' +
+ 'Supports both raw and JSON stores. Only supports the tcp protocol ' +
+ 'buffer interface with server-side routing.'),
+ packages=['voldemort', 'voldemort.protocol', 'voldemort.serialization'],
+ author='LinkedIn Corporation',
+ license='Apache 2.0',
+ url='http://project-voldemort.com',
+ install_requires=['protobuf>=2.3.0', 'simplejson>=2.1.1'],
+ setup_requires=['nose>=0.11'],
+)
View
@@ -1,59 +0,0 @@
-import logging
-import time
-from voldemort import StoreClient
-
-if __name__ == '__main__':
-
- logging.basicConfig(level=logging.INFO,)
-
- ## some random tests
- s = StoreClient('test', [('localhost', 6666)])
- version = s.put("hello", "1")
- assert s.get("hello")[0][0] == "1"
- s.put("hello", "2", version)
- assert s.get("hello")[0][0] == "2"
- s.put("hello", "3")
- assert s.get("hello")[0][0] == "3"
- s.delete("hello")
- assert len(s.get("hello")) == 0
-
- ## test get_all
- pairs = [("a1", "1"), ("a2", "2"), ("a3", "3"), ("a4", "4")]
- for k, v in pairs:
- s.put(k, v)
-
- vals = s.get_all([k for k, v in pairs])
- for k, v in pairs:
- assert vals[k][0][0] == v
-
- requests = 10000
-
- ## Time get requests
- s.put("hello", "world")
- start = time.time()
- for i in xrange(requests):
- s.get('hello')
- print requests/(time.time() - start), ' get requests per second'
-
- ## Time put requests
- version = s.put('abc', 'def')
- start = time.time()
- for i in xrange(requests):
- version = s.put('abc', 'def', version)
- print requests/(time.time() - start), ' put requests per second'
-
- ## Time get_all requests
- keys = [k for k,v in pairs]
- start = time.time()
- for i in xrange(requests):
- vals = s.get_all(keys)
- print requests/(time.time() - start), ' get_all requests per second'
-
- ## Time delete requests
- version = None
- for i in xrange(requests):
- version = s.put(str(i), str(i))
- start = time.time()
- for i in xrange(requests):
- vals = s.delete(str(i), version)
- print requests/(time.time() - start), ' delete requests per second'
Oops, something went wrong.

0 comments on commit 2100b20

Please sign in to comment.