Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Newer
Older
100644 881 lines (744 sloc) 29.796 kB
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
1 #!/usr/bin/python
2 # -*- coding: utf-8 -*-
3 #
4 # Simple key-value datastore
8a87e13 @ownport add few TODO and billets for sqlite implementation
authored
5
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
6 # - support only mysql database
7 # - console support added
8 #
9 # some ideas taked from PyMongo interface http://api.mongodb.org/python/current/index.html
10 # kvlite2 tutorial http://code.google.com/p/kvlite/wiki/kvlite2
11 #
12 # TODO autocommit for put()
8a87e13 @ownport add few TODO and billets for sqlite implementation
authored
13 # TODO synchronise documents between few datastores
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
14 #
15 #
16 __author__ = 'Andrey Usov <http://devel.ownport.net>'
17 __version__ = '0.3'
18 __license__ = """
19 Redistribution and use in source and binary forms, with or without modification,
20 are permitted provided that the following conditions are met:
21
22 * Redistributions of source code must retain the above copyright notice,
23 this list of conditions and the following disclaimer.
24 * Redistributions in binary form must reproduce the above copyright notice,
25 this list of conditions and the following disclaimer in the documentation
26 and/or other materials provided with the distribution.
27
28 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS'
29 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
31 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
32 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
33 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
34 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
35 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
36 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
37 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
38 POSSIBILITY OF SUCH DAMAGE."""
39
212f56d @ownport added sqlite3 support
authored
40 import os
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
41 import cmd
42 import sys
43 import zlib
fc6d7df @ownport update scheme and code
authored
44 import uuid
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
45 import pprint
212f56d @ownport added sqlite3 support
authored
46 import sqlite3
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
47 import binascii
48
1b145ca @ownport added MysqlConnectionManager + tests
authored
49 __all__ = ['open', 'remove',]
50
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
51 try:
52 import MySQLdb
53 except ImportError:
54 print >> sys.stderr, 'Error! MySQLdb package is not installed, please install python-mysqldb'
55 sys.exit()
56
212f56d @ownport added sqlite3 support
authored
57
fc6d7df @ownport update scheme and code
authored
58 try:
59 import cPickle as pickle
60 except ImportError:
61 import pickle
212f56d @ownport added sqlite3 support
authored
62 # TODO add test cases for serializers
8a87e13 @ownport add few TODO and billets for sqlite implementation
authored
63 # TODO add support user specific serializators
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
64
22f983f @ownport migration from old scheme to new
authored
65 SUPPORTED_BACKENDS = ['mysql', 'sqlite', ]
66
212f56d @ownport added sqlite3 support
authored
67 '''
68 A collection is a group of documents stored in kvlite2,
69 and can be thought of as roughly the equivalent of a
70 table in a relational database.
71
72
73
74 For using JSON as serialization
75
76 >>> import json
77 >>> collection = open('sqlite://test.sqlite:test', serializer=json)
78 >>>
79
80 '''
81 # -----------------------------------------------------------------
82 # cPickleSerializer class
83 # -----------------------------------------------------------------
84
85 class cPickleSerializer(object):
86 ''' cPickleSerializer '''
87
88 @staticmethod
89 def dumps(v):
90 ''' dumps value '''
91 if isinstance(v, unicode):
92 v = str(v)
93 return pickle.dumps(v)
94
95 @staticmethod
96 def loads(v):
97 ''' loads value '''
98 if isinstance(v, unicode):
99 v = str(v)
100 return pickle.loads(v)
101
102 # -----------------------------------------------------------------
103 # CompressedJsonSerializer class
104 # -----------------------------------------------------------------
105
106 class CompressedJsonSerializer(object):
107 ''' CompressedJsonSerializer '''
108
109 @staticmethod
110 def dumps(v):
111 ''' dumps value '''
112 return zlib.compress(json_encode(v))
113
114 @staticmethod
115 def loads(v):
116 ''' loads value '''
117 return json_decode(zlib.decompress(v))
118
119 # -----------------------------------------------------------------
120 # SERIALIZERS
121 # -----------------------------------------------------------------
122
123 SERIALIZERS = {
124 'pickle': cPickleSerializer,
125 'completed_json': CompressedJsonSerializer,
126 }
127
e9990ae @ownport move old code to MysqlCollection
authored
128
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
129 # -----------------------------------------------------------------
8ffdeaf @ownport added open() function for creation and opening collection
authored
130 # KVLite utils
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
131 # -----------------------------------------------------------------
212f56d @ownport added sqlite3 support
authored
132 def open(uri, serializer=cPickleSerializer):
8ffdeaf @ownport added open() function for creation and opening collection
authored
133 '''
134 open collection by URI,
135 if collection does not exist kvlite will try to create it
136
137 in case of successful opening or creation new collection
138 return Collection object
fc6d7df @ownport update scheme and code
authored
139
140 serializer: the class or module to serialize msgs with, must have
141 methods or functions named ``dumps`` and ``loads``,
142 `pickle <http://docs.python.org/library/pickle.html>`_ is the default,
143 use ``None`` to store messages in plain text (suitable for strings,
144 integers, etc)
8ffdeaf @ownport added open() function for creation and opening collection
authored
145 '''
fc6d7df @ownport update scheme and code
authored
146 # TODO save kvlite configuration in database
1b145ca @ownport added MysqlConnectionManager + tests
authored
147
fc6d7df @ownport update scheme and code
authored
148 manager = CollectionManager(uri)
149 params = manager.parse_uri(uri)
150 if params['collection'] not in manager.collections():
151 manager.create(params['collection'])
212f56d @ownport added sqlite3 support
authored
152 return manager.collection_class(manager.connection, params['collection'], serializer)
1b145ca @ownport added MysqlConnectionManager + tests
authored
153
154 def remove(uri):
155 '''
156 remove collection by URI
157 '''
22f983f @ownport migration from old scheme to new
authored
158 backend, rest_uri = uri.split('://')
159 if backend in SUPPORTED_BACKENDS:
160 if backend == 'mysql':
1b145ca @ownport added MysqlConnectionManager + tests
authored
161 MysqlCollection(uri).remove()
22f983f @ownport migration from old scheme to new
authored
162 elif backend == 'sqlite':
1b145ca @ownport added MysqlConnectionManager + tests
authored
163 SqliteCollection(uri).remove()
22f983f @ownport migration from old scheme to new
authored
164 else:
1b145ca @ownport added MysqlConnectionManager + tests
authored
165 raise NotImplementedError('unknown backend: {}'.format(uri))
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
166 else:
22f983f @ownport migration from old scheme to new
authored
167 raise RuntimeError('Unknown backend: {}'.format(backend))
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
168
fc6d7df @ownport update scheme and code
authored
169 def get_uuid(amount=100):
170 ''' return UUIDs '''
171
172 # TODO check another approach for generation UUIDs, more fast
173 # TODO get uuids from file as approach to speedup UUIDs generation
174
175 uuids = list()
176 for _ in xrange(amount):
177 u = str(uuid.uuid4()).replace('-', '')
178 uuids.append(("%040s" % u).replace(' ','0'))
179 return uuids
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
180
181 # -----------------------------------------------------------------
1b145ca @ownport added MysqlConnectionManager + tests
authored
182 # CollectionManager class
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
183 # -----------------------------------------------------------------
1b145ca @ownport added MysqlConnectionManager + tests
authored
184 class CollectionManager(object):
185 ''' Collection Manager'''
186
22f983f @ownport migration from old scheme to new
authored
187 def __init__(self, uri):
1b145ca @ownport added MysqlConnectionManager + tests
authored
188 backend, rest_uri = uri.split('://')
189 if backend in SUPPORTED_BACKENDS:
190 if backend == 'mysql':
212f56d @ownport added sqlite3 support
authored
191 self.backend_manager = MysqlCollectionManager(uri)
1b145ca @ownport added MysqlConnectionManager + tests
authored
192 elif backend == 'sqlite':
212f56d @ownport added sqlite3 support
authored
193 self.backend_manager = SqliteCollectionManager(uri)
1b145ca @ownport added MysqlConnectionManager + tests
authored
194 else:
195 raise NotImplementedError()
196 else:
197 raise RuntimeError('Unknown backend: {}'.format(backend))
fc6d7df @ownport update scheme and code
authored
198
212f56d @ownport added sqlite3 support
authored
199 def parse_uri(self, uri):
200 ''' parse_uri '''
201 return self.backend_manager.parse_uri(uri)
202
fc6d7df @ownport update scheme and code
authored
203 def create(self, name):
204 ''' create collection '''
212f56d @ownport added sqlite3 support
authored
205 self.backend_manager.create(name)
fc6d7df @ownport update scheme and code
authored
206
212f56d @ownport added sqlite3 support
authored
207 @property
208 def collection_class(self):
fc6d7df @ownport update scheme and code
authored
209 ''' return object MysqlCollection or SqliteCollection '''
212f56d @ownport added sqlite3 support
authored
210 return self.backend_manager.collection_class
211
212 @property
213 def connection(self):
214 ''' return reference to backend connection '''
215 return self.backend_manager.connection
fc6d7df @ownport update scheme and code
authored
216
217 def collections(self):
218 ''' return list of collections '''
212f56d @ownport added sqlite3 support
authored
219 return self.backend_manager.collections()
fc6d7df @ownport update scheme and code
authored
220
221 def remove(self, name):
222 ''' remove collection '''
212f56d @ownport added sqlite3 support
authored
223 self.backend_manager.remove(name)
1b145ca @ownport added MysqlConnectionManager + tests
authored
224
225 # -----------------------------------------------------------------
226 # MysqlCollectionManager class
227 # -----------------------------------------------------------------
228 class MysqlCollectionManager(object):
229 ''' MysqlCollectionManager '''
230
231 def __init__(self, uri):
22f983f @ownport migration from old scheme to new
authored
232
fc6d7df @ownport update scheme and code
authored
233 params = self.parse_uri(uri)
212f56d @ownport added sqlite3 support
authored
234
1b145ca @ownport added MysqlConnectionManager + tests
authored
235 try:
236 self.__conn = MySQLdb.connect(
237 host=params['host'], port = params['port'],
238 user=params['username'], passwd=params['password'],
239 db=params['db'])
240 except MySQLdb.OperationalError,err:
241 raise RuntimeError(err)
242 self.__cursor = self.__conn.cursor()
22f983f @ownport migration from old scheme to new
authored
243
244 @staticmethod
fc6d7df @ownport update scheme and code
authored
245 def parse_uri(uri):
246 '''parse URI
22f983f @ownport migration from old scheme to new
authored
247
248 return driver, user, password, host, port, database, table
249 '''
fc6d7df @ownport update scheme and code
authored
250 parsed_uri = dict()
251 parsed_uri['backend'], rest_uri = uri.split('://', 1)
252 parsed_uri['username'], rest_uri = rest_uri.split(':', 1)
253 parsed_uri['password'], rest_uri = rest_uri.split('@', 1)
254
255 if ':' in rest_uri:
256 parsed_uri['host'], rest_uri = rest_uri.split(':', 1)
257 parsed_uri['port'], rest_uri = rest_uri.split('/', 1)
258 parsed_uri['port'] = int(parsed_uri['port'])
22f983f @ownport migration from old scheme to new
authored
259 else:
fc6d7df @ownport update scheme and code
authored
260 parsed_uri['host'], rest_uri = rest_uri.split('/')
261 parsed_uri['port'] = 3306
22f983f @ownport migration from old scheme to new
authored
262
fc6d7df @ownport update scheme and code
authored
263 if '.' in rest_uri:
264 parsed_uri['db'], parsed_uri['collection'] = rest_uri.split('.', 1)
265 else:
266 parsed_uri['db'] = rest_uri
267 parsed_uri['collection'] = None
268 return parsed_uri
212f56d @ownport added sqlite3 support
authored
269
1b145ca @ownport added MysqlConnectionManager + tests
authored
270 def create(self, name):
22f983f @ownport migration from old scheme to new
authored
271 ''' create collection '''
1b145ca @ownport added MysqlConnectionManager + tests
authored
272
22f983f @ownport migration from old scheme to new
authored
273 SQL_CREATE_TABLE = '''CREATE TABLE IF NOT EXISTS %s (
274 __rowid__ INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
275 k BINARY(20) NOT NULL,
276 v MEDIUMBLOB,
277 UNIQUE KEY (k) ) ENGINE=InnoDB;'''
278
1b145ca @ownport added MysqlConnectionManager + tests
authored
279 self.__cursor.execute(SQL_CREATE_TABLE % name)
280 self.__conn.commit()
281
212f56d @ownport added sqlite3 support
authored
282 @property
283 def connection(self):
284 ''' return connection '''
285 return self.__conn
286
287 @property
288 def collection_class(self):
fc6d7df @ownport update scheme and code
authored
289 ''' return MysqlCollection object'''
212f56d @ownport added sqlite3 support
authored
290 return MysqlCollection
1b145ca @ownport added MysqlConnectionManager + tests
authored
291
292 def collections(self):
293 ''' return collection list'''
294 self.__cursor.execute('SHOW TABLES;')
fc6d7df @ownport update scheme and code
authored
295 return [t[0] for t in self.__cursor.fetchall()]
1b145ca @ownport added MysqlConnectionManager + tests
authored
296
297
298 def remove(self, name):
299 ''' remove collection '''
300 if name in self.collections():
301 self.__cursor.execute('DROP TABLE %s;' % name)
302 self.__conn.commit()
303 else:
304 raise RuntimeError('No collection with name: {}'.format(name))
305
306 def close(self):
307 ''' close connection to database '''
308 self.__conn.close()
309
310 # -----------------------------------------------------------------
311 # MysqlCollection class
312 # -----------------------------------------------------------------
313 class MysqlCollection(object):
314 ''' Mysql Connection '''
315
212f56d @ownport added sqlite3 support
authored
316 def __init__(self, connection, collection_name, serializer=cPickleSerializer):
fc6d7df @ownport update scheme and code
authored
317
318 self.__conn = connection
1b145ca @ownport added MysqlConnectionManager + tests
authored
319 self.__cursor = self.__conn.cursor()
fc6d7df @ownport update scheme and code
authored
320 self.__collection = collection_name
212f56d @ownport added sqlite3 support
authored
321 self.__serializer = serializer
322
fc6d7df @ownport update scheme and code
authored
323 self.__uuid_cache = list()
22f983f @ownport migration from old scheme to new
authored
324
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
325 def get_uuid(self):
fc6d7df @ownport update scheme and code
authored
326 """
327 if mysql connection is available more fast way to use this method
328 than global function - get_uuid()
329
330 return id based on uuid """
331
332 if not self.__uuid_cache:
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
333 self.__cursor.execute('SELECT %s;' % ','.join(['uuid()' for _ in range(100)]))
334 for uuid in self.__cursor.fetchone():
335 u = uuid.split('-')
336 u.reverse()
337 u = ("%040s" % ''.join(u)).replace(' ','0')
fc6d7df @ownport update scheme and code
authored
338 self.__uuid_cache.append(u)
339 return self.__uuid_cache.pop()
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
340
341 def __get_many(self):
342 ''' return all docs '''
343 rowid = 0
344 while True:
345 SQL_SELECT_MANY = 'SELECT __rowid__, k,v FROM %s WHERE __rowid__ > %d LIMIT 1000 ;' % (self.__collection, rowid)
346 self.__cursor.execute(SQL_SELECT_MANY)
347 result = self.__cursor.fetchall()
348 if not result:
349 break
350 for r in result:
351 rowid = r[0]
352 k = binascii.b2a_hex(r[1])
353 try:
212f56d @ownport added sqlite3 support
authored
354 v = self.__serializer.loads(r[2])
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
355 except Exception, err:
212f56d @ownport added sqlite3 support
authored
356 raise RuntimeError('key %s, %s' % (k, err))
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
357 yield (k, v)
358
359 def get(self, k=None):
360 '''
361 return document by key from collection
362 return documents if key is not defined
363 '''
364 if k:
365 if len(k) > 40:
366 raise WronKeyValue()
367 SQL = 'SELECT k,v FROM %s WHERE k = ' % self.__collection
368 try:
369 self.__cursor.execute(SQL + "%s", binascii.a2b_hex(k))
370 except TypeError, err:
371 raise WronKeyValue(err)
372 result = self.__cursor.fetchone()
373 if result:
374 try:
212f56d @ownport added sqlite3 support
authored
375 v = self.__serializer.loads(result[1])
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
376 except Exception, err:
212f56d @ownport added sqlite3 support
authored
377 raise RuntimeError('key %s, %s' % (k, err))
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
378 return (binascii.b2a_hex(result[0]), v)
379 else:
380 return (None, None)
381 else:
382 return self.__get_many()
383
384 def put(self, k, v):
385 ''' put document in collection '''
212f56d @ownport added sqlite3 support
authored
386
387 # TODO many k/v in put()
388
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
389 if len(k) > 40:
fc6d7df @ownport update scheme and code
authored
390 raise RuntimeError('The length of key is more than 40 bytes')
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
391 SQL_INSERT = 'INSERT INTO %s (k,v) ' % self.__collection
392 SQL_INSERT += 'VALUES (%s,%s) ON DUPLICATE KEY UPDATE v=%s;;'
212f56d @ownport added sqlite3 support
authored
393 v = self.__serializer.dumps(v)
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
394 try:
395 self.__cursor.execute(SQL_INSERT, (binascii.a2b_hex(k), v, v))
396 except TypeError, err:
fc6d7df @ownport update scheme and code
authored
397 raise RuntimeError(err)
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
398
399 def delete(self, k):
400 ''' delete document by k '''
401 if len(k) > 40:
402 raise WronKeyValue()
403 SQL_DELETE = '''DELETE FROM %s WHERE k = ''' % self.__collection
404 try:
405 self.__cursor.execute(SQL_DELETE + "%s;", binascii.a2b_hex(k))
406 except TypeError, err:
407 raise WronKeyValue(err)
408
409 def keys(self):
410 ''' return document keys in collection'''
411 rowid = 0
412 while True:
413 SQL_SELECT_MANY = 'SELECT __rowid__, k FROM %s WHERE __rowid__ > %d LIMIT 1000 ;' % (self.__collection, rowid)
414 self.__cursor.execute(SQL_SELECT_MANY)
415 result = self.__cursor.fetchall()
416 if not result:
417 break
418 for r in result:
419 rowid = r[0]
420 k = binascii.b2a_hex(r[1])
421 yield k
422
212f56d @ownport added sqlite3 support
authored
423 @property
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
424 def count(self):
425 ''' return amount of documents in collection'''
426 self.__cursor.execute('SELECT count(*) FROM %s;' % self.__collection)
427 return int(self.__cursor.fetchone()[0])
428
429 def commit(self):
430 self.__conn.commit()
431
432 def close(self):
433 ''' close connection to database '''
434 self.__conn.close()
435
fc6d7df @ownport update scheme and code
authored
436 # -----------------------------------------------------------------
437 # SqliteCollectionManager class
438 # -----------------------------------------------------------------
439 class SqliteCollectionManager(object):
440 ''' Sqlite Collection Manager '''
212f56d @ownport added sqlite3 support
authored
441 def __init__(self, uri):
442
443 params = self.parse_uri(uri)
444
445 self.__conn = sqlite3.connect(params['db'])
446 self.__cursor = self.__conn.cursor()
447
448 @staticmethod
449 def parse_uri(uri):
450 '''parse URI
451
452 return driver, database, collection
453 '''
454 parsed_uri = dict()
455 parsed_uri['backend'], rest_uri = uri.split('://', 1)
456 if ':' in rest_uri:
457 parsed_uri['db'], parsed_uri['collection'] = rest_uri.split(':',1)
458 else:
459 parsed_uri['db'] = rest_uri
460 parsed_uri['collection'] = None
461 if parsed_uri['db'] == 'memory':
462 parsed_uri['db'] = ':memory:'
463 return parsed_uri
464
465 @property
466 def connection(self):
467 ''' return connection '''
468 return self.__conn
469
470 @property
471 def collection_class(self):
472 ''' return SqliteCollection object'''
473 return SqliteCollection
474
475 def collections(self):
476 ''' return collection list'''
477
478 self.__cursor.execute('SELECT name FROM sqlite_master WHERE type="table";')
479 return [t[0] for t in self.__cursor.fetchall()]
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
480
212f56d @ownport added sqlite3 support
authored
481 def create(self, name):
482 ''' create collection '''
483
484 SQL_CREATE_TABLE = '''CREATE TABLE IF NOT EXISTS %s (
485 k NOT NULL, v, UNIQUE (k) );'''
486
487 self.__cursor.execute(SQL_CREATE_TABLE % name)
488 self.__conn.commit()
489
490 def remove(self, name):
491 ''' remove collection '''
492 if name in self.collections():
493 self.__cursor.execute('DROP TABLE %s;' % name)
494 self.__conn.commit()
495 else:
496 raise RuntimeError('No collection with name: {}'.format(name))
497
498 def close(self):
499 ''' close connection to database '''
500 self.__conn.close()
501
22f983f @ownport migration from old scheme to new
authored
502 # -----------------------------------------------------------------
503 # SqliteCollection class
504 # -----------------------------------------------------------------
e9990ae @ownport move old code to MysqlCollection
authored
505 class SqliteCollection(object):
22f983f @ownport migration from old scheme to new
authored
506 ''' Sqlite Collection'''
507
212f56d @ownport added sqlite3 support
authored
508 def __init__(self, connection, collection_name, serializer=cPickleSerializer):
e9990ae @ownport move old code to MysqlCollection
authored
509
212f56d @ownport added sqlite3 support
authored
510 self.__conn = connection
511 self.__cursor = self.__conn.cursor()
512 self.__collection = collection_name
513 self.__serializer = serializer
e9990ae @ownport move old code to MysqlCollection
authored
514
212f56d @ownport added sqlite3 support
authored
515 self.__uuid_cache = list()
e9990ae @ownport move old code to MysqlCollection
authored
516
212f56d @ownport added sqlite3 support
authored
517 def get_uuid(self):
518 """ return id based on uuid """
e9990ae @ownport move old code to MysqlCollection
authored
519
212f56d @ownport added sqlite3 support
authored
520 if not self.__uuid_cache:
521 for uuid in get_uuid():
522 self.__uuid_cache.append(uuid)
523 return self.__uuid_cache.pop()
524
525 def put(self, k, v):
526 ''' put document in collection '''
e9990ae @ownport move old code to MysqlCollection
authored
527
212f56d @ownport added sqlite3 support
authored
528 # TODO many k/v in put()
529
530 if len(k) > 40:
531 raise RuntimeError('The length of key is more than 40 bytes')
532 SQL_INSERT = 'INSERT OR REPLACE INTO %s (k,v) ' % self.__collection
533 SQL_INSERT += 'VALUES (?,?)'
534 v = self.__serializer.dumps(v)
535 try:
536 self.__cursor.execute(SQL_INSERT, (k, v))
537 except TypeError, err:
538 raise RuntimeError(err)
e9990ae @ownport move old code to MysqlCollection
authored
539
212f56d @ownport added sqlite3 support
authored
540 def __get_many(self):
541 ''' return all docs '''
542 rowid = 0
543 while True:
544 SQL_SELECT_MANY = 'SELECT rowid, k,v FROM %s WHERE rowid > %d LIMIT 1000 ;' % (self.__collection, rowid)
545 self.__cursor.execute(SQL_SELECT_MANY)
546 result = self.__cursor.fetchall()
547 if not result:
548 break
549 for r in result:
550 rowid = r[0]
551 k = r[1]
552 try:
553 v = self.__serializer.loads(r[2])
554 except Exception, err:
555 raise RuntimeError('key %s, %s' % (k, err))
556 yield (k, v)
557
558 def get(self, k=None):
559 '''
560 return document by key from collection
561 return documents if key is not defined
e9990ae @ownport move old code to MysqlCollection
authored
562 '''
212f56d @ownport added sqlite3 support
authored
563 if k:
564 if len(k) > 40:
565 raise RuntimeError('The key length is more than 40 bytes')
566 SQL = 'SELECT k,v FROM %s WHERE k = ?;' % self.__collection
567 try:
568 self.__cursor.execute(SQL, (k,))
569 except TypeError, err:
570 raise WronKeyValue(err)
571 result = self.__cursor.fetchone()
572 if result:
573 try:
574 v = self.__serializer.loads(result[1])
575 except Exception, err:
576 raise RuntimeError('key %s, %s' % (k, err))
577 return (result[0], v)
578 else:
579 return (None, None)
580 else:
581 return self.__get_many()
582
583 def keys(self):
584 ''' return document keys in collection'''
585 rowid = 0
586 while True:
587 SQL_SELECT_MANY = 'SELECT rowid, k FROM %s WHERE rowid > %d LIMIT 1000 ;' % (self.__collection, rowid)
588 self.__cursor.execute(SQL_SELECT_MANY)
589 result = self.__cursor.fetchall()
590 if not result:
591 break
592 for r in result:
593 rowid = r[0]
594 yield r[1]
e9990ae @ownport move old code to MysqlCollection
authored
595
212f56d @ownport added sqlite3 support
authored
596 def delete(self, k):
597 ''' delete document by k '''
598 if len(k) > 40:
599 raise RuntimeError('The key length is more than 40 bytes')
600 SQL_DELETE = '''DELETE FROM %s WHERE k = ?;''' % self.__collection
601 try:
602 self.__cursor.execute(SQL_DELETE, (k,))
603 except TypeError, err:
604 raise WronKeyValue(err)
605
606 @property
607 def count(self):
608 ''' return amount of documents in collection'''
609 self.__cursor.execute('SELECT count(*) FROM %s;' % self.__collection)
610 return int(self.__cursor.fetchone()[0])
611
612 def commit(self):
613 self.__conn.commit()
614
615 def close(self):
616 ''' close connection to database '''
617 self.__conn.close()
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
618
619 # -----------------------------------------------------------------
620 # Console class
621 # -----------------------------------------------------------------
622 class Console(cmd.Cmd):
623 def __init__(self):
624 cmd.Cmd.__init__(self)
625 self.prompt = "kvlite> "
626 self.ruler = '-'
627
628 self.__history_size = 20
629 self.__history = list()
630 self.__kvlite_colls = dict()
631 self.__current_coll_name = 'kvlite'
632 self.__current_coll = None
633
634 def emptyline(self):
635 return False
636
637 def do_help(self, arg):
638 ''' help <command>\tshow <command> help'''
639 if arg:
640 try:
641 func = getattr(self, 'help_' + arg)
642 except AttributeError:
643 try:
644 doc=getattr(self, 'do_' + arg).__doc__
645 if doc:
646 self.stdout.write("%s\n"%str(doc))
647 return
648 except AttributeError:
649 pass
650 self.stdout.write("%s\n"%str(self.nohelp % (arg,)))
651 return
652 else:
653 names = [
654 '', 'do_help', 'do_version', 'do_licence', 'do_history', 'do_exit', '',
655 'do_create', 'do_use', 'do_show', 'do_remove', 'do_import', 'do_export', '',
656 'do_hash', 'do_keys', 'do_items', 'do_get', 'do_put', 'do_delete', 'do_count', ''
657 ]
658 for name in names:
659 if not name:
660 print
661 else:
662 print getattr(self, name).__doc__
663
664 def do_history(self,line):
665 ''' history\t\tshow commands history '''
666 for i, line in enumerate(self.__history):
667 print "0%d. %s" % (i+1, line)
668
669 def precmd(self, line):
670 if len(self.__history) == self.__history_size:
671 prev_line = self.__history.pop(0)
672 if line and line not in self.__history:
673 self.__history.append(line)
674 return line
675
676 def do_version(self, line):
677 ''' version\t\tshow kvlite version'''
678 print 'version: %s' % __version__
679
680 def do_licence(self, line):
681 ''' licence\t\tshow licence'''
682 print __license__
683 print
684
685 def do_exit(self, line):
686 ''' exit\t\t\texit from console '''
687 return True
688
689 def do_import(self, filename):
690 ''' import <filename>\timport collection configuration from JSON file'''
691 import os
692
693 if not filename:
694 print getattr(self, 'do_import').__doc__
695 return
696 filename = filename.rstrip().lstrip()
697
698 if os.path.isfile(filename):
699 for k, v in json_decode(open(filename).read()).items():
700 self.__kvlite_colls[k] = v
701 print 'Import completed'
702 else:
703 print 'Error! File %s does not exists' % filename
704
705 def do_export(self, filename):
706 ''' export <filename>\texport collection configurations to JSON file'''
707 # TODO check if file exists. If yes, import about it
708 if not filename:
709 print getattr(self, 'do_import').__doc__
710 return
711 filename = filename.rstrip().lstrip()
712 json_file = open(filename, 'w')
713 json_file.write(json_encode(self.__kvlite_colls))
714 json_file.close()
715 print 'Export completed to file: %s' % filename
716
717 def do_show(self, line):
718 ''' show collections\tlist of available collections (defined in settings.py)'''
719 if line == 'collections':
720 for coll in self.__kvlite_colls:
721 print ' %s' % coll
722 else:
723 print 'Unknown argument: %s' % line
724
725 def do_use(self, collection_name):
726 ''' use <collection>\tuse the collection as the default (current) collection'''
727 if collection_name in self.__kvlite_colls:
728 self.prompt = '%s>' % collection_name
729 self.__current_coll_name = collection_name
730 self.__current_coll = Collection(self.__kvlite_colls[self.__current_coll_name])
731 return
732 else:
733 print 'Error! Unknown collection: %s' % collection_name
734
735 def do_create(self, line):
736 ''' create <name> <uri>\tcreate new collection (if not exists)'''
737 try:
738 name, uri = [i for i in line.split(' ') if i <> '']
739 except ValueError:
740 print getattr(self, 'do_create').__doc__
741 return
742
743 if name in self.__kvlite_colls:
744 print 'Warning! Collection name already defined: %s, %s' % (name, self.__kvlite_colls[name])
745 print 'If needed please change collection name'
746 return
747 try:
748 if is_collection_exists(uri):
749 self.__kvlite_colls[name] = uri
750 print 'Connection exists, the reference added to collection list'
751 return
752 else:
753 create_collection(uri)
754 self.__kvlite_colls[name] = uri
755 print 'Collection created and added to collection list'
756 return
757 except WrongURIException:
758 print 'Error! Incorrect URI'
759 return
760 except ConnectionError, err:
761 print 'Connection Error! Please check URI, %s' % str(err)
762 return
763
764 def do_remove(self, name):
765 ''' remove <collection>\tremove collection'''
766 if name not in self.__kvlite_colls:
767 print 'Error! Collection name does not exist: %s' % name
768 return
769 try:
770 if is_collection_exists(self.__kvlite_colls[name]):
771 delete_collection(self.__kvlite_colls[name])
772 del self.__kvlite_colls[name]
773 print 'Collection %s deleted' % name
774 return
775 else:
776 print 'Error! Collection does not exist, %s' % self.__kvlite_colls[name]
777 except WrongURIException:
778 print 'Error! Incorrect URI'
779 return
780 except ConnectionError, err:
781 print 'Connection Error! Please check URI, %s' % str(err)
782 return
783
784 def do_hash(self, line):
785 ''' hash [string]\tgenerate sha1 hash, random if string is not defined'''
786 import hashlib
787 import datetime
788 if not line:
789 str_now = str(datetime.datetime.now())
790 print 'Random sha1 hash:', hashlib.sha1(str_now).hexdigest()
791 else:
792 line = line.rstrip().lstrip()
793 print 'sha1 hash:', hashlib.sha1(line).hexdigest()
794
795 def do_keys(self, line):
796 ''' keys\t\t\tlist of keys '''
797 if not self.__current_coll_name in self.__kvlite_colls:
798 print 'Error! Unknown collection: %s' % self.__current_coll_name
799 return
800 for k,v in self.__current_coll.get():
801 print k
802
803 def do_items(self, line):
804 ''' items\t\tlist of collection's items '''
805 if not self.__current_coll_name in self.__kvlite_colls:
806 print 'Error! Unknown collection: %s' % self.__current_coll_name
807 return
808 for k,v in self.__current_coll.get():
809 print k
810 pprint.pprint(v)
811 print
812
813 def do_count(self, args):
814 ''' count\t\tshow the amount of entries in collection '''
815 if self.__current_coll:
816 print self.__current_coll.count()
817
818 def do_get(self, key):
819 ''' get <key>\t\tshow collection entry by key'''
820 if not key:
821 print getattr(self, 'do_get').__doc__
822 return
823 for key in [k for k in key.split(' ') if k <> '']:
824 if self.__current_coll:
825 k, v = self.__current_coll.get(key)
826 print k
827 pprint.pprint(v)
828 print
829 else:
830 print 'Error! The collection is not selected, please use collection'
831 return
832
833 def do_put(self, line):
834 ''' put <key> <value>\tstore entry to collection'''
835 try:
836 k,v = [i for i in line.split(' ',1) if i <> '']
837 except ValueError:
838 print getattr(self, 'do_put').__doc__
839 return
840
841 try:
842 v = json_decode(v)
843 except ValueError, err:
844 print 'Value decoding error!', err
845 return
846
847 if self.__current_coll:
848 try:
849 self.__current_coll.put(k, v)
850 self.__current_coll.commit()
851 print 'Done'
852 return
853 except WronKeyValue, err:
854 print 'Error! Incorrect key/value,', err
855 return
856 else:
857 print 'Error! The collection is not selected, please use collection'
858 return
859
860
861 def do_delete(self, key):
862 ''' delete <key>\t\tdelete entry by key '''
863 key = key.rstrip().lstrip()
864 if self.__current_coll.get(key) <> (None, None):
865 self.__current_coll.delete(key)
866 self.__current_coll.commit()
867 print 'Done'
868 return
869 else:
870 print 'Error! The key %s does not exist' % key
871 return
872
873 # ----------------------------------
874 # main
875 # ----------------------------------
876 if __name__ == '__main__':
877 console = Console()
878 console.cmdloop()
879
880
Something went wrong with that request. Please try again.