Skip to content
Browse files

0.1.0 commit

  • Loading branch information...
0 parents commit b4783f09abc9e7041f57ad00cca4256dc4125775 @srinikom committed
115 README.textile
@@ -0,0 +1,115 @@
+h2. leveldb-server
+Async leveldb-server based on zeromq.
+More info about leveldb
+More info about zeromq
+
+h3. License
+New BSD license. Please see license.txt for more details
+
+h2. Features
+
+h3. leveldb features
+* Very simple key-value storage
+* Data is sorted by key - allows @ range @ queries
+* Data is automatically compressed
+* Can act as presistant cache
+* For safebox it replaced memcached+mysql
+* @cp level.db backup.db@ for backup
+
+h3. leveldb-server features
+* Networking/wiring from @ zeromq @ messaging library
+* Async server for scalability and capacity
+* Sync client for easy coding
+
+h3. leveldb-client features
+* python client
+* easy to get multiple language bindings. See here for zmq client library http://www.zeromq.org/bindings:_start
+
+<pre>
+>>> db.put("k3", "v3")
+'True'
+>>> db.get("k3")
+'v3'
+>>> db.range()
+'[{"k1": "v1"}, {"k2": "v2"}, {"k3": "v3"}]'
+>>> db.delete('k1')
+>>>
+</pre>
+
+Will be adding high availability, replication and autosharding using the same zeromq framework.
+
+h3. Dependencies
+<pre>
+python
+zmq
+pyzmq
+leveldb
+pyleveldb
+</pre>
+
+h2. Getting Started
+
+h3. Installing zeromq
+
+<pre>
+wget http://download.zeromq.org/zeromq-2.1.10.tar.gz
+tar xvfz zeromq-2.1.10.tar.gz
+cd zeromq-2.1.10
+sudo ./configure
+sudo make
+sudo make install
+</pre>
+
+h3. Installing pyzmq
+<pre>
+wget https://github.com/zeromq/pyzmq/downloads/pyzmq-2.1.10.tar.gz
+tar xvfz pyzmq-2.1.10.tar.gz
+cd pyzmq-2.1.10/
+sudo python setup.py configure --zmq=/usr/local/lib/
+sudo python setup.py install
+</pre>
+
+h3. Installing leveldb and pyleveldb
+<pre>
+svn checkout http://py-leveldb.googlecode.com/svn/trunk/ py-leveldb-read-only
+cd py-leveldb-read-only
+sudo compile_leveldb.sh
+sudo python setup.py install
+</pre>
+
+h3. Starting the leveldb-server
+<pre>
+> python leveldb-server.py -h
+Usage: leveldb-server.py
+ -p [port and host settings] Default: tcp://127.0.0.1:5147
+ -f [database file name] Default: level.db
+
+leveldb-server
+
+Options:
+ --version show program's version number and exit
+ -h, --help show this help message and exit
+ -p HOST, --host=HOST
+ -d DBFILE, --dbfile=DBFILE
+> python leveldb-server.py
+</pre>
+
+h3. Using the leveldb-client-py
+<pre>
+> sudo python setup.py install
+> python
+>>> from leveldbClient import database
+>>> db = database.leveldb()
+>>> db.get("Key")
+>>> db.put("K", "V")
+>>> db.range()
+>>> db.range(start, end)
+>>> db.delete("K")
+</pre>
+
+h2. Backing up the database
+<pre>
+cp -rpf level.db backup.db
+</pre>
+
+
9 clients/py/build/lib.linux-i686-2.6/leveldbClient/__init__.py
@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+#Copyright (c) 2011 Fabula Solutions. All rights reserved.
+#Use of this source code is governed by a BSD-style license that can be
+#found in the license.txt file.
+
+"""The client for leveldb-server."""
+
+version = "0.1.0"
+version_info = (0, 1, 0)
43 clients/py/build/lib.linux-i686-2.6/leveldbClient/database.py
@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+#Copyright (c) 2011 Fabula Solutions. All rights reserved.
+#Use of this source code is governed by a BSD-style license that can be
+#found in the license.txt file.
+
+# leveldb client
+import zmq
+import threading
+import time
+import json
+
+class leveldb(object):
+ """leveldb client"""
+ def __init__(self, host="tcp://127.0.0.1:5147", timeout=10*1000):
+ self.host = host
+ self.timeout = timeout
+ self.connect()
+
+ def connect(self):
+ self.context = zmq.Context()
+ self.socket = self.context.socket(zmq.XREQ)
+ self.socket.connect(self.host)
+
+ def get(self, key):
+ self.socket.send_multipart(['get', json.dumps(key)])
+ return self.socket.recv_multipart()[0]
+
+ def put(self, key, value):
+ self.socket.send_multipart(['put', json.dumps([key, value])])
+ return self.socket.recv_multipart()[0]
+
+ def delete(self, key):
+ self.socket.send_multipart(['delete', json.dumps(key)])
+ return self.socket.recv_multipart()[0]
+
+ def range(self, start=None, end=None):
+ self.socket.send_multipart(['range', json.dumps([start, end])])
+ return self.socket.recv_multipart()[0]
+
+ def close(self):
+ self.socket.close()
+ self.context.term()
+
BIN clients/py/dist/leveldb_server_client-0.1.0-py2.6.egg
Binary file not shown.
9 clients/py/leveldbClient/__init__.py
@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+#Copyright (c) 2011 Fabula Solutions. All rights reserved.
+#Use of this source code is governed by a BSD-style license that can be
+#found in the license.txt file.
+
+"""The client for leveldb-server."""
+
+version = "0.1.0"
+version_info = (0, 1, 0)
43 clients/py/leveldbClient/database.py
@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+#Copyright (c) 2011 Fabula Solutions. All rights reserved.
+#Use of this source code is governed by a BSD-style license that can be
+#found in the license.txt file.
+
+# leveldb client
+import zmq
+import threading
+import time
+import json
+
+class leveldb(object):
+ """leveldb client"""
+ def __init__(self, host="tcp://127.0.0.1:5147", timeout=10*1000):
+ self.host = host
+ self.timeout = timeout
+ self.connect()
+
+ def connect(self):
+ self.context = zmq.Context()
+ self.socket = self.context.socket(zmq.XREQ)
+ self.socket.connect(self.host)
+
+ def get(self, key):
+ self.socket.send_multipart(['get', json.dumps(key)])
+ return self.socket.recv_multipart()[0]
+
+ def put(self, key, value):
+ self.socket.send_multipart(['put', json.dumps([key, value])])
+ return self.socket.recv_multipart()[0]
+
+ def delete(self, key):
+ self.socket.send_multipart(['delete', json.dumps(key)])
+ return self.socket.recv_multipart()[0]
+
+ def range(self, start=None, end=None):
+ self.socket.send_multipart(['range', json.dumps([start, end])])
+ return self.socket.recv_multipart()[0]
+
+ def close(self):
+ self.socket.close()
+ self.context.term()
+
10 clients/py/leveldb_server_client.egg-info/PKG-INFO
@@ -0,0 +1,10 @@
+Metadata-Version: 1.0
+Name: leveldb-server-client
+Version: 0.1.0
+Summary: client for leveldb-server
+Home-page: https://github.com/srinikom/leveldb-server
+Author: Srini
+Author-email: srini@fabulasolutions.com
+License: BSD
+Description: UNKNOWN
+Platform: UNKNOWN
7 clients/py/leveldb_server_client.egg-info/SOURCES.txt
@@ -0,0 +1,7 @@
+setup.py
+leveldbClient/__init__.py
+leveldbClient/database.py
+leveldb_server_client.egg-info/PKG-INFO
+leveldb_server_client.egg-info/SOURCES.txt
+leveldb_server_client.egg-info/dependency_links.txt
+leveldb_server_client.egg-info/top_level.txt
1 clients/py/leveldb_server_client.egg-info/dependency_links.txt
@@ -0,0 +1 @@
+
1 clients/py/leveldb_server_client.egg-info/top_level.txt
@@ -0,0 +1 @@
+leveldbClient
29 clients/py/setup.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+#Copyright (c) 2011 Fabula Solutions. All rights reserved.
+#Use of this source code is governed by a BSD-style license that can be
+#found in the license.txt file.
+
+import distutils.core
+import sys
+# Importing setuptools adds some features like "setup.py develop", but
+# it's optional so swallow the error if it's not there.
+try:
+ import setuptools
+except ImportError:
+ pass
+
+kwargs = {}
+
+version = "0.1.0"
+
+distutils.core.setup(
+ name="leveldb-server client",
+ version=version,
+ packages = ["leveldbClient"],
+ author="Srini",
+ author_email="srini@fabulasolutions.com",
+ url="https://github.com/srinikom/leveldb-server",
+ license="BSD",
+ description="client for leveldb-server",
+ **kwargs
+)
133 leveldb-server.py
@@ -0,0 +1,133 @@
+#Copyright (c) 2011 Fabula Solutions. All rights reserved.
+#Use of this source code is governed by a BSD-style license that can be
+#found in the license.txt file.
+
+# leveldb server
+import threading
+import zmq
+import leveldb
+import json
+import optparse
+
+class workerThread(threading.Thread):
+ """workerThread"""
+ def __init__(self, context, db):
+ threading.Thread.__init__ (self)
+ self.context = context
+ self.db = db
+
+ def run(self):
+ socket = self.context.socket(zmq.XREQ)
+ socket.connect('inproc://backend')
+ while True:
+ msg = socket.recv_multipart()
+ if len(msg) != 3:
+ value = 'None'
+ reply = [msg[0], value]
+ socket.send_multipart(reply)
+ continue
+ id = msg[0]
+ op = msg[1]
+ data = json.loads(msg[2])
+ reply = [id]
+ if op == 'get':
+ try:
+ value = self.db.Get(data)
+ except:
+ value = ""
+ reply.append(value)
+
+ elif op == 'put':
+ try:
+ self.db.Put(data[0], data[1])
+ value = "True"
+ except:
+ value = ""
+ reply.append(value)
+
+ elif op == 'delete':
+ self.db.Delete(data)
+ value = ""
+ reply.append(value)
+
+ elif op == 'range':
+ start = data[0]
+ end = data[1]
+ if start and end:
+ try:
+ arr = []
+ for value in self.db.RangeIter(start, end):
+ arr.append({value[0]: value[1]})
+ reply.append(json.dumps(arr))
+ except:
+ value = ""
+ reply.append(value)
+ else:
+ try:
+ arr = []
+ for value in self.db.RangeIter():
+ arr.append({value[0]: value[1]})
+ reply.append(json.dumps(arr))
+ except:
+ value = ""
+ reply.append(value)
+ else:
+ value = ""
+ reply.append(value)
+ #print reply
+ socket.send_multipart(reply)
+
+ socket.close()
+
+if __name__ == "__main__":
+ optparser = optparse.OptionParser(
+ prog='leveldb-server.py',
+ version='0.1.1',
+ description='leveldb-server',
+ usage='%prog \n\t-p [port and host settings] Default: tcp://127.0.0.1:5147\n' + \
+ '\t-f [database file name] Default: level.db')
+ optparser.add_option('--host', '-p', dest='host',
+ default='tcp://127.0.0.1:5147')
+ optparser.add_option('--dbfile', '-d', dest='dbfile',
+ default='level.db')
+ options, arguments = optparser.parse_args()
+
+ if not (options.host and options.dbfile):
+ optparser.print_help()
+
+ print "Starting leveldb-server %s" % options.host
+ context = zmq.Context()
+ frontend = context.socket(zmq.XREP)
+ frontend.bind(options.host)
+ backend = context.socket(zmq.XREQ)
+ backend.bind('inproc://backend')
+
+ poll = zmq.Poller()
+ poll.register(frontend, zmq.POLLIN)
+ poll.register(backend, zmq.POLLIN)
+
+ db = leveldb.LevelDB(options.dbfile)
+
+ workers = []
+ for i in xrange(3):
+ worker = workerThread(context, db)
+ worker.start()
+ workers.append(worker)
+
+ while True:
+ sockets = dict(poll.poll())
+ if frontend in sockets:
+ if sockets[frontend] == zmq.POLLIN:
+ msg = frontend.recv_multipart()
+ backend.send_multipart(msg)
+
+ if backend in sockets:
+ if sockets[backend] == zmq.POLLIN:
+ msg = backend.recv_multipart()
+ frontend.send_multipart(msg)
+
+ #never here
+ frontend.close()
+ backend.close()
+ context.term()
+
28 license.txt
@@ -0,0 +1,28 @@
+Copyright (c) 2011 Fabula Solutions. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+ * Neither the name of Fabula Solutions nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+

0 comments on commit b4783f0

Please sign in to comment.
Something went wrong with that request. Please try again.