Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100644 589 lines (509 sloc) 20.698 kb
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
1 #!/usr/bin/python
2 # -*- coding: utf-8 -*-
3 #
4 # Simple key-value datastore
8a87e13 @ownport add few TODO and billets for sqlite implementation
authored
5
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
6 # - support only mysql database
7 # - console support added
8 #
9 # some ideas taked from PyMongo interface http://api.mongodb.org/python/current/index.html
10 # kvlite2 tutorial http://code.google.com/p/kvlite/wiki/kvlite2
11 #
12 # TODO autocommit for put()
8a87e13 @ownport add few TODO and billets for sqlite implementation
authored
13 # TODO synchronise documents between few datastores
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
14 #
15 #
16 __author__ = 'Andrey Usov <http://devel.ownport.net>'
17 __version__ = '0.3'
18 __license__ = """
19 Redistribution and use in source and binary forms, with or without modification,
20 are permitted provided that the following conditions are met:
21
22 * Redistributions of source code must retain the above copyright notice,
23 this list of conditions and the following disclaimer.
24 * Redistributions in binary form must reproduce the above copyright notice,
25 this list of conditions and the following disclaimer in the documentation
26 and/or other materials provided with the distribution.
27
28 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS'
29 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
31 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
32 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
33 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
34 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
35 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
36 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
37 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
38 POSSIBILITY OF SUCH DAMAGE."""
39
40 import re
41 import cmd
42 import sys
43 import zlib
44 import pprint
45 import binascii
46
47 try:
48 import MySQLdb
49 except ImportError:
50 print >> sys.stderr, 'Error! MySQLdb package is not installed, please install python-mysqldb'
51 sys.exit()
52
8a87e13 @ownport add few TODO and billets for sqlite implementation
authored
53 # TODO add deferent serialization on user choice (pickle & json)
54 # TODO add support user specific serializators
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
55 from json import loads as json_decode
56 from json import dumps as json_encode
57
58 # TODO describe this Exception
59 class WronKeyValue(Exception): pass
60
61 # in case when URI for connection defined incorrectly this exception is raised
62 class WrongURIException(Exception): pass
63
64 # in case value cannot be unpacked
65 class ValueUnpackError(Exception): pass
66
67 # exception raised in case of connection error
68 class ConnectionError(Exception): pass
69
22f983f @ownport migration from old scheme to new
authored
70 SUPPORTED_BACKENDS = ['mysql', 'sqlite', ]
71
e9990ae @ownport move old code to MysqlCollection
authored
72
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
73 # -----------------------------------------------------------------
8ffdeaf @ownport added open() function for creation and opening collection
authored
74 # KVLite utils
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
75 # -----------------------------------------------------------------
8ffdeaf @ownport added open() function for creation and opening collection
authored
76 def open(uri):
77 '''
78 open collection by URI,
79 if collection does not exist kvlite will try to create it
80
81 in case of successful opening or creation new collection
82 return Collection object
83 '''
22f983f @ownport migration from old scheme to new
authored
84 backend, rest_uri = uri.split('://')
85 if backend in SUPPORTED_BACKENDS:
86 if backend == 'mysql':
87 return MysqlCollection(uri)
88 elif backend == 'sqlite':
89 return SqliteCollection(uri)
90 else:
91 raise NotImplementedError()
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
92 else:
22f983f @ownport migration from old scheme to new
authored
93 raise RuntimeError('Unknown backend: {}'.format(backend))
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
94
95 def is_collection_exists(URI):
96 ''' check if collection exists '''
97 params = parse_uri(URI)
98 try:
99 conn = MySQLdb.connect(host=params['host'], port = params['port'],
100 user=params['usr'], passwd=params['pwd'], db=params['db'])
101 except MySQLdb.OperationalError,err:
102 raise ConnectionError(err)
103 cursor = conn.cursor()
104 cursor.execute('SHOW TABLES;')
105 for r in cursor.fetchall():
106 if r[0] == params['coll']:
107 return True
108 return False
109
110 def collection_names(URI):
111 ''' return list of collection names'''
112 params = parse_uri(URI)
113 conn = MySQLdb.connect(host=params['host'], port = params['port'],
114 user=params['usr'], passwd=params['pwd'], db=params['db'])
115 cursor = conn.cursor()
116 cursor.execute('SHOW TABLES;')
117 return cursor.fetchall()
118
119
120 def delete_collection(URI):
121 ''' delete collection '''
122 if is_collection_exists(URI):
123 params = parse_uri(URI)
124 conn = MySQLdb.connect(host=params['host'], port = params['port'],
125 user=params['usr'], passwd=params['pwd'], db=params['db'])
126 cursor = conn.cursor()
127 cursor.execute('DROP TABLE %s;' % params['coll'])
128 conn.commit()
129
130 # -----------------------------------------------------------------
22f983f @ownport migration from old scheme to new
authored
131 # MysqlCollection class
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
132 # -----------------------------------------------------------------
8a87e13 @ownport add few TODO and billets for sqlite implementation
authored
133
134 class MysqlConnection(object):
e9990ae @ownport move old code to MysqlCollection
authored
135 ''' Mysql Connection '''
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
136
22f983f @ownport migration from old scheme to new
authored
137 def __init__(self, uri):
138
139 params = parse_uri(URI)
140
141
142 raise NotImplementedError('MysqlCollection is not implemented yet')
143
144 @staticmethod
145 def _parse_uri(uri):
146 ''' parse URI
147
148 return driver, user, password, host, port, database, table
149 '''
150 result = {}
151 m = re.search(r'(?P<drv>\w+)://(?P<usr>.+):(?P<pwd>.+)@(?P<host>.+?):?(?P<port>\d*)\/(?P<db>.+)\.(?P<coll>.+)', uri, re.I)
152 try:
153 result = dict(m.groupdict())
154 except AttributeError,e:
155 raise WrongURIException(e)
156
157 if result['port'] <> '':
158 result['port'] = int(result['port'])
159 else:
160 result['port'] = 3306
161 return result
162
163 def _create(self, params):
164 ''' create collection '''
165 try:
166 conn = MySQLdb.connect(host=params['host'], port = params['port'],
167 user=params['usr'], passwd=params['pwd'], db=params['db'])
168 except MySQLdb.OperationalError,err:
169 raise ConnectionError(err)
170 cursor = conn.cursor()
171 SQL_CREATE_TABLE = '''CREATE TABLE IF NOT EXISTS %s (
172 __rowid__ INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
173 k BINARY(20) NOT NULL,
174 v MEDIUMBLOB,
175 UNIQUE KEY (k) ) ENGINE=InnoDB;'''
176
177 cursor.execute(SQL_CREATE_TABLE % params['coll'])
178 conn.commit()
179
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
180 def get_uuid(self):
181 """ return id based on uuid """
8a87e13 @ownport add few TODO and billets for sqlite implementation
authored
182 # TODO add generation UUID in case of use sqlite database
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
183 if not self.__uuids:
184 self.__cursor.execute('SELECT %s;' % ','.join(['uuid()' for _ in range(100)]))
185 for uuid in self.__cursor.fetchone():
186 u = uuid.split('-')
187 u.reverse()
188 u = ("%040s" % ''.join(u)).replace(' ','0')
189 self.__uuids.append(u)
190 return self.__uuids.pop()
191
192 def __get_many(self):
193 ''' return all docs '''
194 rowid = 0
195 while True:
196 SQL_SELECT_MANY = 'SELECT __rowid__, k,v FROM %s WHERE __rowid__ > %d LIMIT 1000 ;' % (self.__collection, rowid)
197 self.__cursor.execute(SQL_SELECT_MANY)
198 result = self.__cursor.fetchall()
199 if not result:
200 break
201 for r in result:
202 rowid = r[0]
203 k = binascii.b2a_hex(r[1])
204 try:
205 v = self.unpack(r[2])
206 except Exception, err:
207 raise ValueUnpackError('key %s, %s' % (k, err))
208 yield (k, v)
209
210 def get(self, k=None):
211 '''
212 return document by key from collection
213 return documents if key is not defined
214 '''
215 if k:
216 if len(k) > 40:
217 raise WronKeyValue()
218 SQL = 'SELECT k,v FROM %s WHERE k = ' % self.__collection
219 try:
220 self.__cursor.execute(SQL + "%s", binascii.a2b_hex(k))
221 except TypeError, err:
222 raise WronKeyValue(err)
223 result = self.__cursor.fetchone()
224 if result:
225 try:
226 v = self.unpack(result[1])
227 except Exception, err:
228 raise ValueUnpackError('key %s, %s' % (k, err))
229 return (binascii.b2a_hex(result[0]), v)
230 else:
231 return (None, None)
232 else:
233 return self.__get_many()
234
235 def put(self, k, v):
236 ''' put document in collection '''
237 if len(k) > 40:
238 raise WronKeyValue()
239 SQL_INSERT = 'INSERT INTO %s (k,v) ' % self.__collection
240 SQL_INSERT += 'VALUES (%s,%s) ON DUPLICATE KEY UPDATE v=%s;;'
241 v = self.pack(v)
242 try:
243 self.__cursor.execute(SQL_INSERT, (binascii.a2b_hex(k), v, v))
244 except TypeError, err:
245 raise WronKeyValue(err)
246
247 def delete(self, k):
248 ''' delete document by k '''
249 if len(k) > 40:
250 raise WronKeyValue()
251 SQL_DELETE = '''DELETE FROM %s WHERE k = ''' % self.__collection
252 try:
253 self.__cursor.execute(SQL_DELETE + "%s;", binascii.a2b_hex(k))
254 except TypeError, err:
255 raise WronKeyValue(err)
256
257 def keys(self):
258 ''' return document keys in collection'''
259 rowid = 0
260 while True:
261 SQL_SELECT_MANY = 'SELECT __rowid__, k FROM %s WHERE __rowid__ > %d LIMIT 1000 ;' % (self.__collection, rowid)
262 self.__cursor.execute(SQL_SELECT_MANY)
263 result = self.__cursor.fetchall()
264 if not result:
265 break
266 for r in result:
267 rowid = r[0]
268 k = binascii.b2a_hex(r[1])
269 yield k
270
271 def count(self):
272 ''' return amount of documents in collection'''
273 self.__cursor.execute('SELECT count(*) FROM %s;' % self.__collection)
274 return int(self.__cursor.fetchone()[0])
275
276 def commit(self):
277 self.__conn.commit()
278
279 def close(self):
280 ''' close connection to database '''
281 self.__conn.close()
282
283
22f983f @ownport migration from old scheme to new
authored
284 # -----------------------------------------------------------------
285 # SqliteCollection class
286 # -----------------------------------------------------------------
e9990ae @ownport move old code to MysqlCollection
authored
287 class SqliteCollection(object):
22f983f @ownport migration from old scheme to new
authored
288 ''' Sqlite Collection'''
289
290 def __init__(self, uri):
291 raise NotImplementedError('SqliteCollection is not implemented yet')
e9990ae @ownport move old code to MysqlCollection
authored
292
293 class Collection(object):
294 '''
295 kvlite2 collection
296
297 A collection is a group of documents stored in kvlite2,
298 and can be thought of as roughly the equivalent of a
299 table in a relational database.
300
301 '''
302 def __init__(self, db_uri):
303 '''
304 db_uri - URI to databases,
305 URI format: driver://username:passwd@host[:port]/database.collection
306 '''
307 params = parse_uri(db_uri)
308 self.__conn = MySQLdb.connect(host=params['host'], port = params['port'],
309 user=params['usr'], passwd=params['pwd'], db=params['db'])
310 self.__collection = params['coll']
311 self.__cursor = self.__conn.cursor()
312 self.__uuids = []
313
314 def pack(self, v):
315 ''' pack value
316
317 Note: before pack the value it's better to encode it by base64
318 '''
319 return zlib.compress(json_encode(v))
320
321 def unpack(self, v):
322 ''' unpack value
323 '''
324 return json_decode(zlib.decompress(v))
325
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
326
327 # -----------------------------------------------------------------
328 # Console class
329 # -----------------------------------------------------------------
330 class Console(cmd.Cmd):
331 def __init__(self):
332 cmd.Cmd.__init__(self)
333 self.prompt = "kvlite> "
334 self.ruler = '-'
335
336 self.__history_size = 20
337 self.__history = list()
338 self.__kvlite_colls = dict()
339 self.__current_coll_name = 'kvlite'
340 self.__current_coll = None
341
342 def emptyline(self):
343 return False
344
345 def do_help(self, arg):
346 ''' help <command>\tshow <command> help'''
347 if arg:
348 try:
349 func = getattr(self, 'help_' + arg)
350 except AttributeError:
351 try:
352 doc=getattr(self, 'do_' + arg).__doc__
353 if doc:
354 self.stdout.write("%s\n"%str(doc))
355 return
356 except AttributeError:
357 pass
358 self.stdout.write("%s\n"%str(self.nohelp % (arg,)))
359 return
360 else:
361 names = [
362 '', 'do_help', 'do_version', 'do_licence', 'do_history', 'do_exit', '',
363 'do_create', 'do_use', 'do_show', 'do_remove', 'do_import', 'do_export', '',
364 'do_hash', 'do_keys', 'do_items', 'do_get', 'do_put', 'do_delete', 'do_count', ''
365 ]
366 for name in names:
367 if not name:
368 print
369 else:
370 print getattr(self, name).__doc__
371
372 def do_history(self,line):
373 ''' history\t\tshow commands history '''
374 for i, line in enumerate(self.__history):
375 print "0%d. %s" % (i+1, line)
376
377 def precmd(self, line):
378 if len(self.__history) == self.__history_size:
379 prev_line = self.__history.pop(0)
380 if line and line not in self.__history:
381 self.__history.append(line)
382 return line
383
384 def do_version(self, line):
385 ''' version\t\tshow kvlite version'''
386 print 'version: %s' % __version__
387
388 def do_licence(self, line):
389 ''' licence\t\tshow licence'''
390 print __license__
391 print
392
393 def do_exit(self, line):
394 ''' exit\t\t\texit from console '''
395 return True
396
397 def do_import(self, filename):
398 ''' import <filename>\timport collection configuration from JSON file'''
399 import os
400
401 if not filename:
402 print getattr(self, 'do_import').__doc__
403 return
404 filename = filename.rstrip().lstrip()
405
406 if os.path.isfile(filename):
407 for k, v in json_decode(open(filename).read()).items():
408 self.__kvlite_colls[k] = v
409 print 'Import completed'
410 else:
411 print 'Error! File %s does not exists' % filename
412
413 def do_export(self, filename):
414 ''' export <filename>\texport collection configurations to JSON file'''
415 # TODO check if file exists. If yes, import about it
416 if not filename:
417 print getattr(self, 'do_import').__doc__
418 return
419 filename = filename.rstrip().lstrip()
420 json_file = open(filename, 'w')
421 json_file.write(json_encode(self.__kvlite_colls))
422 json_file.close()
423 print 'Export completed to file: %s' % filename
424
425 def do_show(self, line):
426 ''' show collections\tlist of available collections (defined in settings.py)'''
427 if line == 'collections':
428 for coll in self.__kvlite_colls:
429 print ' %s' % coll
430 else:
431 print 'Unknown argument: %s' % line
432
433 def do_use(self, collection_name):
434 ''' use <collection>\tuse the collection as the default (current) collection'''
435 if collection_name in self.__kvlite_colls:
436 self.prompt = '%s>' % collection_name
437 self.__current_coll_name = collection_name
438 self.__current_coll = Collection(self.__kvlite_colls[self.__current_coll_name])
439 return
440 else:
441 print 'Error! Unknown collection: %s' % collection_name
442
443 def do_create(self, line):
444 ''' create <name> <uri>\tcreate new collection (if not exists)'''
445 try:
446 name, uri = [i for i in line.split(' ') if i <> '']
447 except ValueError:
448 print getattr(self, 'do_create').__doc__
449 return
450
451 if name in self.__kvlite_colls:
452 print 'Warning! Collection name already defined: %s, %s' % (name, self.__kvlite_colls[name])
453 print 'If needed please change collection name'
454 return
455 try:
456 if is_collection_exists(uri):
457 self.__kvlite_colls[name] = uri
458 print 'Connection exists, the reference added to collection list'
459 return
460 else:
461 create_collection(uri)
462 self.__kvlite_colls[name] = uri
463 print 'Collection created and added to collection list'
464 return
465 except WrongURIException:
466 print 'Error! Incorrect URI'
467 return
468 except ConnectionError, err:
469 print 'Connection Error! Please check URI, %s' % str(err)
470 return
471
472 def do_remove(self, name):
473 ''' remove <collection>\tremove collection'''
474 if name not in self.__kvlite_colls:
475 print 'Error! Collection name does not exist: %s' % name
476 return
477 try:
478 if is_collection_exists(self.__kvlite_colls[name]):
479 delete_collection(self.__kvlite_colls[name])
480 del self.__kvlite_colls[name]
481 print 'Collection %s deleted' % name
482 return
483 else:
484 print 'Error! Collection does not exist, %s' % self.__kvlite_colls[name]
485 except WrongURIException:
486 print 'Error! Incorrect URI'
487 return
488 except ConnectionError, err:
489 print 'Connection Error! Please check URI, %s' % str(err)
490 return
491
492 def do_hash(self, line):
493 ''' hash [string]\tgenerate sha1 hash, random if string is not defined'''
494 import hashlib
495 import datetime
496 if not line:
497 str_now = str(datetime.datetime.now())
498 print 'Random sha1 hash:', hashlib.sha1(str_now).hexdigest()
499 else:
500 line = line.rstrip().lstrip()
501 print 'sha1 hash:', hashlib.sha1(line).hexdigest()
502
503 def do_keys(self, line):
504 ''' keys\t\t\tlist of keys '''
505 if not self.__current_coll_name in self.__kvlite_colls:
506 print 'Error! Unknown collection: %s' % self.__current_coll_name
507 return
508 for k,v in self.__current_coll.get():
509 print k
510
511 def do_items(self, line):
512 ''' items\t\tlist of collection's items '''
513 if not self.__current_coll_name in self.__kvlite_colls:
514 print 'Error! Unknown collection: %s' % self.__current_coll_name
515 return
516 for k,v in self.__current_coll.get():
517 print k
518 pprint.pprint(v)
519 print
520
521 def do_count(self, args):
522 ''' count\t\tshow the amount of entries in collection '''
523 if self.__current_coll:
524 print self.__current_coll.count()
525
526 def do_get(self, key):
527 ''' get <key>\t\tshow collection entry by key'''
528 if not key:
529 print getattr(self, 'do_get').__doc__
530 return
531 for key in [k for k in key.split(' ') if k <> '']:
532 if self.__current_coll:
533 k, v = self.__current_coll.get(key)
534 print k
535 pprint.pprint(v)
536 print
537 else:
538 print 'Error! The collection is not selected, please use collection'
539 return
540
541 def do_put(self, line):
542 ''' put <key> <value>\tstore entry to collection'''
543 try:
544 k,v = [i for i in line.split(' ',1) if i <> '']
545 except ValueError:
546 print getattr(self, 'do_put').__doc__
547 return
548
549 try:
550 v = json_decode(v)
551 except ValueError, err:
552 print 'Value decoding error!', err
553 return
554
555 if self.__current_coll:
556 try:
557 self.__current_coll.put(k, v)
558 self.__current_coll.commit()
559 print 'Done'
560 return
561 except WronKeyValue, err:
562 print 'Error! Incorrect key/value,', err
563 return
564 else:
565 print 'Error! The collection is not selected, please use collection'
566 return
567
568
569 def do_delete(self, key):
570 ''' delete <key>\t\tdelete entry by key '''
571 key = key.rstrip().lstrip()
572 if self.__current_coll.get(key) <> (None, None):
573 self.__current_coll.delete(key)
574 self.__current_coll.commit()
575 print 'Done'
576 return
577 else:
578 print 'Error! The key %s does not exist' % key
579 return
580
581 # ----------------------------------
582 # main
583 # ----------------------------------
584 if __name__ == '__main__':
585 console = Console()
586 console.cmdloop()
587
588
Something went wrong with that request. Please try again.