Skip to content
Newer
Older
100644 610 lines (525 sloc) 20.8 KB
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
1 #!/usr/bin/python
2 # -*- coding: utf-8 -*-
3 #
4 # Simple key-value datastore
8a87e13 @ownport add few TODO and billets for sqlite implementation
authored
5
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
6 # - support only mysql database
7 # - console support added
8 #
9 # some ideas taked from PyMongo interface http://api.mongodb.org/python/current/index.html
10 # kvlite2 tutorial http://code.google.com/p/kvlite/wiki/kvlite2
11 #
12 # TODO autocommit for put()
8a87e13 @ownport add few TODO and billets for sqlite implementation
authored
13 # TODO synchronise documents between few datastores
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
14 #
15 #
16 __author__ = 'Andrey Usov <http://devel.ownport.net>'
17 __version__ = '0.3'
18 __license__ = """
19 Redistribution and use in source and binary forms, with or without modification,
20 are permitted provided that the following conditions are met:
21
22 * Redistributions of source code must retain the above copyright notice,
23 this list of conditions and the following disclaimer.
24 * Redistributions in binary form must reproduce the above copyright notice,
25 this list of conditions and the following disclaimer in the documentation
26 and/or other materials provided with the distribution.
27
28 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS'
29 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
31 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
32 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
33 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
34 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
35 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
36 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
37 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
38 POSSIBILITY OF SUCH DAMAGE."""
39
40 import re
41 import cmd
42 import sys
43 import zlib
44 import pprint
45 import binascii
46
1b145ca @ownport added MysqlConnectionManager + tests
authored
47 __all__ = ['open', 'remove',]
48
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
49 try:
50 import MySQLdb
51 except ImportError:
52 print >> sys.stderr, 'Error! MySQLdb package is not installed, please install python-mysqldb'
53 sys.exit()
54
8a87e13 @ownport add few TODO and billets for sqlite implementation
authored
55 # TODO add deferent serialization on user choice (pickle & json)
56 # TODO add support user specific serializators
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
57 from json import loads as json_decode
58 from json import dumps as json_encode
59
22f983f @ownport migration from old scheme to new
authored
60 SUPPORTED_BACKENDS = ['mysql', 'sqlite', ]
61
e9990ae @ownport move old code to MysqlCollection
authored
62
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
63 # -----------------------------------------------------------------
8ffdeaf @ownport added open() function for creation and opening collection
authored
64 # KVLite utils
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
65 # -----------------------------------------------------------------
8ffdeaf @ownport added open() function for creation and opening collection
authored
66 def open(uri):
67 '''
68 open collection by URI,
69 if collection does not exist kvlite will try to create it
70
71 in case of successful opening or creation new collection
72 return Collection object
73 '''
1b145ca @ownport added MysqlConnectionManager + tests
authored
74 manager = CollectionManager(uri)
75
76
77 def remove(uri):
78 '''
79 remove collection by URI
80 '''
22f983f @ownport migration from old scheme to new
authored
81 backend, rest_uri = uri.split('://')
82 if backend in SUPPORTED_BACKENDS:
83 if backend == 'mysql':
1b145ca @ownport added MysqlConnectionManager + tests
authored
84 MysqlCollection(uri).remove()
22f983f @ownport migration from old scheme to new
authored
85 elif backend == 'sqlite':
1b145ca @ownport added MysqlConnectionManager + tests
authored
86 SqliteCollection(uri).remove()
22f983f @ownport migration from old scheme to new
authored
87 else:
1b145ca @ownport added MysqlConnectionManager + tests
authored
88 raise NotImplementedError('unknown backend: {}'.format(uri))
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
89 else:
22f983f @ownport migration from old scheme to new
authored
90 raise RuntimeError('Unknown backend: {}'.format(backend))
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
91
92
93 # -----------------------------------------------------------------
1b145ca @ownport added MysqlConnectionManager + tests
authored
94 # CollectionManager class
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
95 # -----------------------------------------------------------------
1b145ca @ownport added MysqlConnectionManager + tests
authored
96 class CollectionManager(object):
97 ''' Collection Manager'''
98
22f983f @ownport migration from old scheme to new
authored
99 def __init__(self, uri):
1b145ca @ownport added MysqlConnectionManager + tests
authored
100 backend, rest_uri = uri.split('://')
101 if backend in SUPPORTED_BACKENDS:
102 if backend == 'mysql':
103 return MysqlCollection(uri)
104 elif backend == 'sqlite':
105 return SqliteCollection(uri)
106 else:
107 raise NotImplementedError()
108 else:
109 raise RuntimeError('Unknown backend: {}'.format(backend))
22f983f @ownport migration from old scheme to new
authored
110
1b145ca @ownport added MysqlConnectionManager + tests
authored
111
112 # -----------------------------------------------------------------
113 # MysqlCollectionManager class
114 # -----------------------------------------------------------------
115 class MysqlCollectionManager(object):
116 ''' MysqlCollectionManager '''
117
118 def __init__(self, uri):
22f983f @ownport migration from old scheme to new
authored
119
1b145ca @ownport added MysqlConnectionManager + tests
authored
120 params = self._parse_uri(uri)
121 self.__collection = params['collection']
122 try:
123 self.__conn = MySQLdb.connect(
124 host=params['host'], port = params['port'],
125 user=params['username'], passwd=params['password'],
126 db=params['db'])
127 except MySQLdb.OperationalError,err:
128 raise RuntimeError(err)
129 self.__cursor = self.__conn.cursor()
22f983f @ownport migration from old scheme to new
authored
130
131 @staticmethod
132 def _parse_uri(uri):
133 ''' parse URI
134
135 return driver, user, password, host, port, database, table
136 '''
137 result = {}
1b145ca @ownport added MysqlConnectionManager + tests
authored
138 m = re.search(r'(?P<backend>\w+)://(?P<username>.+):(?P<password>.+)@(?P<host>.+?):?(?P<port>\d*)\/(?P<db>.+)\.(?P<collection>.+)', uri, re.I)
22f983f @ownport migration from old scheme to new
authored
139 try:
140 result = dict(m.groupdict())
1b145ca @ownport added MysqlConnectionManager + tests
authored
141 except AttributeError, e:
142 raise RuntimeError(e)
22f983f @ownport migration from old scheme to new
authored
143
144 if result['port'] <> '':
145 result['port'] = int(result['port'])
146 else:
147 result['port'] = 3306
148 return result
149
1b145ca @ownport added MysqlConnectionManager + tests
authored
150 def create(self, name):
22f983f @ownport migration from old scheme to new
authored
151 ''' create collection '''
1b145ca @ownport added MysqlConnectionManager + tests
authored
152
22f983f @ownport migration from old scheme to new
authored
153 SQL_CREATE_TABLE = '''CREATE TABLE IF NOT EXISTS %s (
154 __rowid__ INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
155 k BINARY(20) NOT NULL,
156 v MEDIUMBLOB,
157 UNIQUE KEY (k) ) ENGINE=InnoDB;'''
158
1b145ca @ownport added MysqlConnectionManager + tests
authored
159 self.__cursor.execute(SQL_CREATE_TABLE % name)
160 self.__conn.commit()
161
162 @property
163 def collection(self):
164 ''' return collection name from uri'''
165 return self.__collection
166
167 def collections(self):
168 ''' return collection list'''
169 self.__cursor.execute('SHOW TABLES;')
170 return [i[0] for i in self.__cursor.fetchall()]
171
172
173 def remove(self, name):
174 ''' remove collection '''
175 if name in self.collections():
176 self.__cursor.execute('DROP TABLE %s;' % name)
177 self.__conn.commit()
178 else:
179 raise RuntimeError('No collection with name: {}'.format(name))
180
181 def close(self):
182 ''' close connection to database '''
183 self.__conn.close()
184
185 # -----------------------------------------------------------------
186 # MysqlCollection class
187 # -----------------------------------------------------------------
188 class MysqlCollection(object):
189 ''' Mysql Connection '''
190
191 def __init__(self, uri):
192
193 params = self._parse_uri(uri)
194 try:
195 self.__conn = MySQLdb.connect(host=params['host'], port = params['port'],
196 user=params['usr'], passwd=params['pwd'], db=params['db'])
197 except MySQLdb.OperationalError,err:
198 raise RuntimeError(err)
199 self.__cursor = self.__conn.cursor()
22f983f @ownport migration from old scheme to new
authored
200
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
201 def get_uuid(self):
202 """ return id based on uuid """
8a87e13 @ownport add few TODO and billets for sqlite implementation
authored
203 # TODO add generation UUID in case of use sqlite database
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
204 if not self.__uuids:
205 self.__cursor.execute('SELECT %s;' % ','.join(['uuid()' for _ in range(100)]))
206 for uuid in self.__cursor.fetchone():
207 u = uuid.split('-')
208 u.reverse()
209 u = ("%040s" % ''.join(u)).replace(' ','0')
210 self.__uuids.append(u)
211 return self.__uuids.pop()
212
213 def __get_many(self):
214 ''' return all docs '''
215 rowid = 0
216 while True:
217 SQL_SELECT_MANY = 'SELECT __rowid__, k,v FROM %s WHERE __rowid__ > %d LIMIT 1000 ;' % (self.__collection, rowid)
218 self.__cursor.execute(SQL_SELECT_MANY)
219 result = self.__cursor.fetchall()
220 if not result:
221 break
222 for r in result:
223 rowid = r[0]
224 k = binascii.b2a_hex(r[1])
225 try:
226 v = self.unpack(r[2])
227 except Exception, err:
228 raise ValueUnpackError('key %s, %s' % (k, err))
229 yield (k, v)
230
231 def get(self, k=None):
232 '''
233 return document by key from collection
234 return documents if key is not defined
235 '''
236 if k:
237 if len(k) > 40:
238 raise WronKeyValue()
239 SQL = 'SELECT k,v FROM %s WHERE k = ' % self.__collection
240 try:
241 self.__cursor.execute(SQL + "%s", binascii.a2b_hex(k))
242 except TypeError, err:
243 raise WronKeyValue(err)
244 result = self.__cursor.fetchone()
245 if result:
246 try:
247 v = self.unpack(result[1])
248 except Exception, err:
249 raise ValueUnpackError('key %s, %s' % (k, err))
250 return (binascii.b2a_hex(result[0]), v)
251 else:
252 return (None, None)
253 else:
254 return self.__get_many()
255
256 def put(self, k, v):
257 ''' put document in collection '''
258 if len(k) > 40:
259 raise WronKeyValue()
260 SQL_INSERT = 'INSERT INTO %s (k,v) ' % self.__collection
261 SQL_INSERT += 'VALUES (%s,%s) ON DUPLICATE KEY UPDATE v=%s;;'
262 v = self.pack(v)
263 try:
264 self.__cursor.execute(SQL_INSERT, (binascii.a2b_hex(k), v, v))
265 except TypeError, err:
266 raise WronKeyValue(err)
267
268 def delete(self, k):
269 ''' delete document by k '''
270 if len(k) > 40:
271 raise WronKeyValue()
272 SQL_DELETE = '''DELETE FROM %s WHERE k = ''' % self.__collection
273 try:
274 self.__cursor.execute(SQL_DELETE + "%s;", binascii.a2b_hex(k))
275 except TypeError, err:
276 raise WronKeyValue(err)
277
278 def keys(self):
279 ''' return document keys in collection'''
280 rowid = 0
281 while True:
282 SQL_SELECT_MANY = 'SELECT __rowid__, k FROM %s WHERE __rowid__ > %d LIMIT 1000 ;' % (self.__collection, rowid)
283 self.__cursor.execute(SQL_SELECT_MANY)
284 result = self.__cursor.fetchall()
285 if not result:
286 break
287 for r in result:
288 rowid = r[0]
289 k = binascii.b2a_hex(r[1])
290 yield k
291
292 def count(self):
293 ''' return amount of documents in collection'''
294 self.__cursor.execute('SELECT count(*) FROM %s;' % self.__collection)
295 return int(self.__cursor.fetchone()[0])
296
297 def commit(self):
298 self.__conn.commit()
299
300 def close(self):
301 ''' close connection to database '''
302 self.__conn.close()
303
304
22f983f @ownport migration from old scheme to new
authored
305 # -----------------------------------------------------------------
306 # SqliteCollection class
307 # -----------------------------------------------------------------
e9990ae @ownport move old code to MysqlCollection
authored
308 class SqliteCollection(object):
22f983f @ownport migration from old scheme to new
authored
309 ''' Sqlite Collection'''
310
311 def __init__(self, uri):
312 raise NotImplementedError('SqliteCollection is not implemented yet')
e9990ae @ownport move old code to MysqlCollection
authored
313
314 class Collection(object):
315 '''
316 kvlite2 collection
317
318 A collection is a group of documents stored in kvlite2,
319 and can be thought of as roughly the equivalent of a
320 table in a relational database.
321
322 '''
323 def __init__(self, db_uri):
324 '''
325 db_uri - URI to databases,
326 URI format: driver://username:passwd@host[:port]/database.collection
327 '''
328 params = parse_uri(db_uri)
329 self.__conn = MySQLdb.connect(host=params['host'], port = params['port'],
330 user=params['usr'], passwd=params['pwd'], db=params['db'])
331 self.__collection = params['coll']
332 self.__cursor = self.__conn.cursor()
333 self.__uuids = []
334
335 def pack(self, v):
336 ''' pack value
337
338 Note: before pack the value it's better to encode it by base64
339 '''
340 return zlib.compress(json_encode(v))
341
342 def unpack(self, v):
343 ''' unpack value
344 '''
345 return json_decode(zlib.decompress(v))
346
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
347
348 # -----------------------------------------------------------------
349 # Console class
350 # -----------------------------------------------------------------
351 class Console(cmd.Cmd):
352 def __init__(self):
353 cmd.Cmd.__init__(self)
354 self.prompt = "kvlite> "
355 self.ruler = '-'
356
357 self.__history_size = 20
358 self.__history = list()
359 self.__kvlite_colls = dict()
360 self.__current_coll_name = 'kvlite'
361 self.__current_coll = None
362
363 def emptyline(self):
364 return False
365
366 def do_help(self, arg):
367 ''' help <command>\tshow <command> help'''
368 if arg:
369 try:
370 func = getattr(self, 'help_' + arg)
371 except AttributeError:
372 try:
373 doc=getattr(self, 'do_' + arg).__doc__
374 if doc:
375 self.stdout.write("%s\n"%str(doc))
376 return
377 except AttributeError:
378 pass
379 self.stdout.write("%s\n"%str(self.nohelp % (arg,)))
380 return
381 else:
382 names = [
383 '', 'do_help', 'do_version', 'do_licence', 'do_history', 'do_exit', '',
384 'do_create', 'do_use', 'do_show', 'do_remove', 'do_import', 'do_export', '',
385 'do_hash', 'do_keys', 'do_items', 'do_get', 'do_put', 'do_delete', 'do_count', ''
386 ]
387 for name in names:
388 if not name:
389 print
390 else:
391 print getattr(self, name).__doc__
392
393 def do_history(self,line):
394 ''' history\t\tshow commands history '''
395 for i, line in enumerate(self.__history):
396 print "0%d. %s" % (i+1, line)
397
398 def precmd(self, line):
399 if len(self.__history) == self.__history_size:
400 prev_line = self.__history.pop(0)
401 if line and line not in self.__history:
402 self.__history.append(line)
403 return line
404
405 def do_version(self, line):
406 ''' version\t\tshow kvlite version'''
407 print 'version: %s' % __version__
408
409 def do_licence(self, line):
410 ''' licence\t\tshow licence'''
411 print __license__
412 print
413
414 def do_exit(self, line):
415 ''' exit\t\t\texit from console '''
416 return True
417
418 def do_import(self, filename):
419 ''' import <filename>\timport collection configuration from JSON file'''
420 import os
421
422 if not filename:
423 print getattr(self, 'do_import').__doc__
424 return
425 filename = filename.rstrip().lstrip()
426
427 if os.path.isfile(filename):
428 for k, v in json_decode(open(filename).read()).items():
429 self.__kvlite_colls[k] = v
430 print 'Import completed'
431 else:
432 print 'Error! File %s does not exists' % filename
433
434 def do_export(self, filename):
435 ''' export <filename>\texport collection configurations to JSON file'''
436 # TODO check if file exists. If yes, import about it
437 if not filename:
438 print getattr(self, 'do_import').__doc__
439 return
440 filename = filename.rstrip().lstrip()
441 json_file = open(filename, 'w')
442 json_file.write(json_encode(self.__kvlite_colls))
443 json_file.close()
444 print 'Export completed to file: %s' % filename
445
446 def do_show(self, line):
447 ''' show collections\tlist of available collections (defined in settings.py)'''
448 if line == 'collections':
449 for coll in self.__kvlite_colls:
450 print ' %s' % coll
451 else:
452 print 'Unknown argument: %s' % line
453
454 def do_use(self, collection_name):
455 ''' use <collection>\tuse the collection as the default (current) collection'''
456 if collection_name in self.__kvlite_colls:
457 self.prompt = '%s>' % collection_name
458 self.__current_coll_name = collection_name
459 self.__current_coll = Collection(self.__kvlite_colls[self.__current_coll_name])
460 return
461 else:
462 print 'Error! Unknown collection: %s' % collection_name
463
464 def do_create(self, line):
465 ''' create <name> <uri>\tcreate new collection (if not exists)'''
466 try:
467 name, uri = [i for i in line.split(' ') if i <> '']
468 except ValueError:
469 print getattr(self, 'do_create').__doc__
470 return
471
472 if name in self.__kvlite_colls:
473 print 'Warning! Collection name already defined: %s, %s' % (name, self.__kvlite_colls[name])
474 print 'If needed please change collection name'
475 return
476 try:
477 if is_collection_exists(uri):
478 self.__kvlite_colls[name] = uri
479 print 'Connection exists, the reference added to collection list'
480 return
481 else:
482 create_collection(uri)
483 self.__kvlite_colls[name] = uri
484 print 'Collection created and added to collection list'
485 return
486 except WrongURIException:
487 print 'Error! Incorrect URI'
488 return
489 except ConnectionError, err:
490 print 'Connection Error! Please check URI, %s' % str(err)
491 return
492
493 def do_remove(self, name):
494 ''' remove <collection>\tremove collection'''
495 if name not in self.__kvlite_colls:
496 print 'Error! Collection name does not exist: %s' % name
497 return
498 try:
499 if is_collection_exists(self.__kvlite_colls[name]):
500 delete_collection(self.__kvlite_colls[name])
501 del self.__kvlite_colls[name]
502 print 'Collection %s deleted' % name
503 return
504 else:
505 print 'Error! Collection does not exist, %s' % self.__kvlite_colls[name]
506 except WrongURIException:
507 print 'Error! Incorrect URI'
508 return
509 except ConnectionError, err:
510 print 'Connection Error! Please check URI, %s' % str(err)
511 return
512
513 def do_hash(self, line):
514 ''' hash [string]\tgenerate sha1 hash, random if string is not defined'''
515 import hashlib
516 import datetime
517 if not line:
518 str_now = str(datetime.datetime.now())
519 print 'Random sha1 hash:', hashlib.sha1(str_now).hexdigest()
520 else:
521 line = line.rstrip().lstrip()
522 print 'sha1 hash:', hashlib.sha1(line).hexdigest()
523
524 def do_keys(self, line):
525 ''' keys\t\t\tlist of keys '''
526 if not self.__current_coll_name in self.__kvlite_colls:
527 print 'Error! Unknown collection: %s' % self.__current_coll_name
528 return
529 for k,v in self.__current_coll.get():
530 print k
531
532 def do_items(self, line):
533 ''' items\t\tlist of collection's items '''
534 if not self.__current_coll_name in self.__kvlite_colls:
535 print 'Error! Unknown collection: %s' % self.__current_coll_name
536 return
537 for k,v in self.__current_coll.get():
538 print k
539 pprint.pprint(v)
540 print
541
542 def do_count(self, args):
543 ''' count\t\tshow the amount of entries in collection '''
544 if self.__current_coll:
545 print self.__current_coll.count()
546
547 def do_get(self, key):
548 ''' get <key>\t\tshow collection entry by key'''
549 if not key:
550 print getattr(self, 'do_get').__doc__
551 return
552 for key in [k for k in key.split(' ') if k <> '']:
553 if self.__current_coll:
554 k, v = self.__current_coll.get(key)
555 print k
556 pprint.pprint(v)
557 print
558 else:
559 print 'Error! The collection is not selected, please use collection'
560 return
561
562 def do_put(self, line):
563 ''' put <key> <value>\tstore entry to collection'''
564 try:
565 k,v = [i for i in line.split(' ',1) if i <> '']
566 except ValueError:
567 print getattr(self, 'do_put').__doc__
568 return
569
570 try:
571 v = json_decode(v)
572 except ValueError, err:
573 print 'Value decoding error!', err
574 return
575
576 if self.__current_coll:
577 try:
578 self.__current_coll.put(k, v)
579 self.__current_coll.commit()
580 print 'Done'
581 return
582 except WronKeyValue, err:
583 print 'Error! Incorrect key/value,', err
584 return
585 else:
586 print 'Error! The collection is not selected, please use collection'
587 return
588
589
590 def do_delete(self, key):
591 ''' delete <key>\t\tdelete entry by key '''
592 key = key.rstrip().lstrip()
593 if self.__current_coll.get(key) <> (None, None):
594 self.__current_coll.delete(key)
595 self.__current_coll.commit()
596 print 'Done'
597 return
598 else:
599 print 'Error! The key %s does not exist' % key
600 return
601
602 # ----------------------------------
603 # main
604 # ----------------------------------
605 if __name__ == '__main__':
606 console = Console()
607 console.cmdloop()
608
609
Something went wrong with that request. Please try again.