Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100644 567 lines (492 sloc) 19.825 kb
67bd9d4 Andrey Usov Migration from http://code.google.com/p/kvlite/
authored
1 #!/usr/bin/python
2 # -*- coding: utf-8 -*-
3 #
4 # Simple key-value datastore
8a87e13 Andrey Usov add few TODO and billets for sqlite implementation
authored
5
67bd9d4 Andrey Usov Migration from http://code.google.com/p/kvlite/
authored
6 # - support only mysql database
7 # - console support added
8 #
9 # some ideas taked from PyMongo interface http://api.mongodb.org/python/current/index.html
10 # kvlite2 tutorial http://code.google.com/p/kvlite/wiki/kvlite2
11 #
12 # TODO autocommit for put()
8a87e13 Andrey Usov add few TODO and billets for sqlite implementation
authored
13 # TODO synchronise documents between few datastores
67bd9d4 Andrey Usov Migration from http://code.google.com/p/kvlite/
authored
14 #
15 #
16 __author__ = 'Andrey Usov <http://devel.ownport.net>'
17 __version__ = '0.3'
18 __license__ = """
19 Redistribution and use in source and binary forms, with or without modification,
20 are permitted provided that the following conditions are met:
21
22 * Redistributions of source code must retain the above copyright notice,
23 this list of conditions and the following disclaimer.
24 * Redistributions in binary form must reproduce the above copyright notice,
25 this list of conditions and the following disclaimer in the documentation
26 and/or other materials provided with the distribution.
27
28 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS'
29 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
31 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
32 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
33 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
34 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
35 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
36 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
37 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
38 POSSIBILITY OF SUCH DAMAGE."""
39
40 import re
41 import cmd
42 import sys
43 import zlib
44 import pprint
45 import binascii
46
47 try:
48 import MySQLdb
49 except ImportError:
50 print >> sys.stderr, 'Error! MySQLdb package is not installed, please install python-mysqldb'
51 sys.exit()
52
8a87e13 Andrey Usov add few TODO and billets for sqlite implementation
authored
53 # TODO add deferent serialization on user choice (pickle & json)
54 # TODO add support user specific serializators
67bd9d4 Andrey Usov Migration from http://code.google.com/p/kvlite/
authored
55 from json import loads as json_decode
56 from json import dumps as json_encode
57
58 # TODO describe this Exception
59 class WronKeyValue(Exception): pass
60
61 # in case when URI for connection defined incorrectly this exception is raised
62 class WrongURIException(Exception): pass
63
64 # in case value cannot be unpacked
65 class ValueUnpackError(Exception): pass
66
67 # exception raised in case of connection error
68 class ConnectionError(Exception): pass
69
e9990ae Andrey Usov move old code to MysqlCollection
authored
70
67bd9d4 Andrey Usov Migration from http://code.google.com/p/kvlite/
authored
71 # -----------------------------------------------------------------
8ffdeaf Andrey Usov added open() function for creation and opening collection
authored
72 # KVLite utils
67bd9d4 Andrey Usov Migration from http://code.google.com/p/kvlite/
authored
73 # -----------------------------------------------------------------
8ffdeaf Andrey Usov added open() function for creation and opening collection
authored
74 def open(uri):
75 '''
76 open collection by URI,
77 if collection does not exist kvlite will try to create it
78
79 in case of successful opening or creation new collection
80 return Collection object
81 '''
82 raise NotImplementedError('kvlite.open()')
83
84
67bd9d4 Andrey Usov Migration from http://code.google.com/p/kvlite/
authored
85 def parse_uri(uri):
86 ''' parse URI
87
88 return driver, user, password, host, port, database, table
89 '''
60974ff Andrey Usov fix typos in TODO
authored
90 # TODO add support different scheme of databases
67bd9d4 Andrey Usov Migration from http://code.google.com/p/kvlite/
authored
91 result = {}
92 m = re.search(r'(?P<drv>\w+)://(?P<usr>.+):(?P<pwd>.+)@(?P<host>.+?):?(?P<port>\d*)\/(?P<db>.+)\.(?P<coll>.+)', uri, re.I)
93 try:
94 result = dict(m.groupdict())
95 except AttributeError,e:
96 raise WrongURIException(e)
97
98 if result['port'] <> '':
99 result['port'] = int(result['port'])
100 else:
101 result['port'] = 3306
102 return result
103
104 def create_collection(URI):
105 ''' create collection '''
106 params = parse_uri(URI)
107 try:
108 conn = MySQLdb.connect(host=params['host'], port = params['port'],
109 user=params['usr'], passwd=params['pwd'], db=params['db'])
110 except MySQLdb.OperationalError,err:
111 raise ConnectionError(err)
112 cursor = conn.cursor()
113 SQL_CREATE_TABLE = '''CREATE TABLE IF NOT EXISTS %s (
114 __rowid__ INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
115 k BINARY(20) NOT NULL,
116 v MEDIUMBLOB,
117 UNIQUE KEY (k) ) ENGINE=InnoDB;'''
118
119 cursor.execute(SQL_CREATE_TABLE % params['coll'])
120 conn.commit()
121
122 def is_collection_exists(URI):
123 ''' check if collection exists '''
124 params = parse_uri(URI)
125 try:
126 conn = MySQLdb.connect(host=params['host'], port = params['port'],
127 user=params['usr'], passwd=params['pwd'], db=params['db'])
128 except MySQLdb.OperationalError,err:
129 raise ConnectionError(err)
130 cursor = conn.cursor()
131 cursor.execute('SHOW TABLES;')
132 for r in cursor.fetchall():
133 if r[0] == params['coll']:
134 return True
135 return False
136
137 def collection_names(URI):
138 ''' return list of collection names'''
139 params = parse_uri(URI)
140 conn = MySQLdb.connect(host=params['host'], port = params['port'],
141 user=params['usr'], passwd=params['pwd'], db=params['db'])
142 cursor = conn.cursor()
143 cursor.execute('SHOW TABLES;')
144 return cursor.fetchall()
145
146
147 def delete_collection(URI):
148 ''' delete collection '''
149 if is_collection_exists(URI):
150 params = parse_uri(URI)
151 conn = MySQLdb.connect(host=params['host'], port = params['port'],
152 user=params['usr'], passwd=params['pwd'], db=params['db'])
153 cursor = conn.cursor()
154 cursor.execute('DROP TABLE %s;' % params['coll'])
155 conn.commit()
156
157 # -----------------------------------------------------------------
8a87e13 Andrey Usov add few TODO and billets for sqlite implementation
authored
158 # Collections class
67bd9d4 Andrey Usov Migration from http://code.google.com/p/kvlite/
authored
159 # -----------------------------------------------------------------
8a87e13 Andrey Usov add few TODO and billets for sqlite implementation
authored
160
161 class MysqlConnection(object):
e9990ae Andrey Usov move old code to MysqlCollection
authored
162 ''' Mysql Connection '''
67bd9d4 Andrey Usov Migration from http://code.google.com/p/kvlite/
authored
163
164 def get_uuid(self):
165 """ return id based on uuid """
8a87e13 Andrey Usov add few TODO and billets for sqlite implementation
authored
166 # TODO add generation UUID in case of use sqlite database
67bd9d4 Andrey Usov Migration from http://code.google.com/p/kvlite/
authored
167 if not self.__uuids:
168 self.__cursor.execute('SELECT %s;' % ','.join(['uuid()' for _ in range(100)]))
169 for uuid in self.__cursor.fetchone():
170 u = uuid.split('-')
171 u.reverse()
172 u = ("%040s" % ''.join(u)).replace(' ','0')
173 self.__uuids.append(u)
174 return self.__uuids.pop()
175
176 def __get_many(self):
177 ''' return all docs '''
178 rowid = 0
179 while True:
180 SQL_SELECT_MANY = 'SELECT __rowid__, k,v FROM %s WHERE __rowid__ > %d LIMIT 1000 ;' % (self.__collection, rowid)
181 self.__cursor.execute(SQL_SELECT_MANY)
182 result = self.__cursor.fetchall()
183 if not result:
184 break
185 for r in result:
186 rowid = r[0]
187 k = binascii.b2a_hex(r[1])
188 try:
189 v = self.unpack(r[2])
190 except Exception, err:
191 raise ValueUnpackError('key %s, %s' % (k, err))
192 yield (k, v)
193
194 def get(self, k=None):
195 '''
196 return document by key from collection
197 return documents if key is not defined
198 '''
199 if k:
200 if len(k) > 40:
201 raise WronKeyValue()
202 SQL = 'SELECT k,v FROM %s WHERE k = ' % self.__collection
203 try:
204 self.__cursor.execute(SQL + "%s", binascii.a2b_hex(k))
205 except TypeError, err:
206 raise WronKeyValue(err)
207 result = self.__cursor.fetchone()
208 if result:
209 try:
210 v = self.unpack(result[1])
211 except Exception, err:
212 raise ValueUnpackError('key %s, %s' % (k, err))
213 return (binascii.b2a_hex(result[0]), v)
214 else:
215 return (None, None)
216 else:
217 return self.__get_many()
218
219 def put(self, k, v):
220 ''' put document in collection '''
221 if len(k) > 40:
222 raise WronKeyValue()
223 SQL_INSERT = 'INSERT INTO %s (k,v) ' % self.__collection
224 SQL_INSERT += 'VALUES (%s,%s) ON DUPLICATE KEY UPDATE v=%s;;'
225 v = self.pack(v)
226 try:
227 self.__cursor.execute(SQL_INSERT, (binascii.a2b_hex(k), v, v))
228 except TypeError, err:
229 raise WronKeyValue(err)
230
231 def delete(self, k):
232 ''' delete document by k '''
233 if len(k) > 40:
234 raise WronKeyValue()
235 SQL_DELETE = '''DELETE FROM %s WHERE k = ''' % self.__collection
236 try:
237 self.__cursor.execute(SQL_DELETE + "%s;", binascii.a2b_hex(k))
238 except TypeError, err:
239 raise WronKeyValue(err)
240
241 def keys(self):
242 ''' return document keys in collection'''
243 rowid = 0
244 while True:
245 SQL_SELECT_MANY = 'SELECT __rowid__, k FROM %s WHERE __rowid__ > %d LIMIT 1000 ;' % (self.__collection, rowid)
246 self.__cursor.execute(SQL_SELECT_MANY)
247 result = self.__cursor.fetchall()
248 if not result:
249 break
250 for r in result:
251 rowid = r[0]
252 k = binascii.b2a_hex(r[1])
253 yield k
254
255 def count(self):
256 ''' return amount of documents in collection'''
257 self.__cursor.execute('SELECT count(*) FROM %s;' % self.__collection)
258 return int(self.__cursor.fetchone()[0])
259
260 def commit(self):
261 self.__conn.commit()
262
263 def close(self):
264 ''' close connection to database '''
265 self.__conn.close()
266
267
e9990ae Andrey Usov move old code to MysqlCollection
authored
268 class SqliteCollection(object):
269 pass
270
271 class Collection(object):
272 '''
273 kvlite2 collection
274
275 A collection is a group of documents stored in kvlite2,
276 and can be thought of as roughly the equivalent of a
277 table in a relational database.
278
279 '''
280 def __init__(self, db_uri):
281 '''
282 db_uri - URI to databases,
283 URI format: driver://username:passwd@host[:port]/database.collection
284 '''
285 params = parse_uri(db_uri)
286 self.__conn = MySQLdb.connect(host=params['host'], port = params['port'],
287 user=params['usr'], passwd=params['pwd'], db=params['db'])
288 self.__collection = params['coll']
289 self.__cursor = self.__conn.cursor()
290 self.__uuids = []
291
292 def pack(self, v):
293 ''' pack value
294
295 Note: before pack the value it's better to encode it by base64
296 '''
297 return zlib.compress(json_encode(v))
298
299 def unpack(self, v):
300 ''' unpack value
301 '''
302 return json_decode(zlib.decompress(v))
303
67bd9d4 Andrey Usov Migration from http://code.google.com/p/kvlite/
authored
304
305 # -----------------------------------------------------------------
306 # Console class
307 # -----------------------------------------------------------------
308 class Console(cmd.Cmd):
309 def __init__(self):
310 cmd.Cmd.__init__(self)
311 self.prompt = "kvlite> "
312 self.ruler = '-'
313
314 self.__history_size = 20
315 self.__history = list()
316 self.__kvlite_colls = dict()
317 self.__current_coll_name = 'kvlite'
318 self.__current_coll = None
319
320 def emptyline(self):
321 return False
322
323 def do_help(self, arg):
324 ''' help <command>\tshow <command> help'''
325 if arg:
326 try:
327 func = getattr(self, 'help_' + arg)
328 except AttributeError:
329 try:
330 doc=getattr(self, 'do_' + arg).__doc__
331 if doc:
332 self.stdout.write("%s\n"%str(doc))
333 return
334 except AttributeError:
335 pass
336 self.stdout.write("%s\n"%str(self.nohelp % (arg,)))
337 return
338 else:
339 names = [
340 '', 'do_help', 'do_version', 'do_licence', 'do_history', 'do_exit', '',
341 'do_create', 'do_use', 'do_show', 'do_remove', 'do_import', 'do_export', '',
342 'do_hash', 'do_keys', 'do_items', 'do_get', 'do_put', 'do_delete', 'do_count', ''
343 ]
344 for name in names:
345 if not name:
346 print
347 else:
348 print getattr(self, name).__doc__
349
350 def do_history(self,line):
351 ''' history\t\tshow commands history '''
352 for i, line in enumerate(self.__history):
353 print "0%d. %s" % (i+1, line)
354
355 def precmd(self, line):
356 if len(self.__history) == self.__history_size:
357 prev_line = self.__history.pop(0)
358 if line and line not in self.__history:
359 self.__history.append(line)
360 return line
361
362 def do_version(self, line):
363 ''' version\t\tshow kvlite version'''
364 print 'version: %s' % __version__
365
366 def do_licence(self, line):
367 ''' licence\t\tshow licence'''
368 print __license__
369 print
370
371 def do_exit(self, line):
372 ''' exit\t\t\texit from console '''
373 return True
374
375 def do_import(self, filename):
376 ''' import <filename>\timport collection configuration from JSON file'''
377 import os
378
379 if not filename:
380 print getattr(self, 'do_import').__doc__
381 return
382 filename = filename.rstrip().lstrip()
383
384 if os.path.isfile(filename):
385 for k, v in json_decode(open(filename).read()).items():
386 self.__kvlite_colls[k] = v
387 print 'Import completed'
388 else:
389 print 'Error! File %s does not exists' % filename
390
391 def do_export(self, filename):
392 ''' export <filename>\texport collection configurations to JSON file'''
393 # TODO check if file exists. If yes, import about it
394 if not filename:
395 print getattr(self, 'do_import').__doc__
396 return
397 filename = filename.rstrip().lstrip()
398 json_file = open(filename, 'w')
399 json_file.write(json_encode(self.__kvlite_colls))
400 json_file.close()
401 print 'Export completed to file: %s' % filename
402
403 def do_show(self, line):
404 ''' show collections\tlist of available collections (defined in settings.py)'''
405 if line == 'collections':
406 for coll in self.__kvlite_colls:
407 print ' %s' % coll
408 else:
409 print 'Unknown argument: %s' % line
410
411 def do_use(self, collection_name):
412 ''' use <collection>\tuse the collection as the default (current) collection'''
413 if collection_name in self.__kvlite_colls:
414 self.prompt = '%s>' % collection_name
415 self.__current_coll_name = collection_name
416 self.__current_coll = Collection(self.__kvlite_colls[self.__current_coll_name])
417 return
418 else:
419 print 'Error! Unknown collection: %s' % collection_name
420
421 def do_create(self, line):
422 ''' create <name> <uri>\tcreate new collection (if not exists)'''
423 try:
424 name, uri = [i for i in line.split(' ') if i <> '']
425 except ValueError:
426 print getattr(self, 'do_create').__doc__
427 return
428
429 if name in self.__kvlite_colls:
430 print 'Warning! Collection name already defined: %s, %s' % (name, self.__kvlite_colls[name])
431 print 'If needed please change collection name'
432 return
433 try:
434 if is_collection_exists(uri):
435 self.__kvlite_colls[name] = uri
436 print 'Connection exists, the reference added to collection list'
437 return
438 else:
439 create_collection(uri)
440 self.__kvlite_colls[name] = uri
441 print 'Collection created and added to collection list'
442 return
443 except WrongURIException:
444 print 'Error! Incorrect URI'
445 return
446 except ConnectionError, err:
447 print 'Connection Error! Please check URI, %s' % str(err)
448 return
449
450 def do_remove(self, name):
451 ''' remove <collection>\tremove collection'''
452 if name not in self.__kvlite_colls:
453 print 'Error! Collection name does not exist: %s' % name
454 return
455 try:
456 if is_collection_exists(self.__kvlite_colls[name]):
457 delete_collection(self.__kvlite_colls[name])
458 del self.__kvlite_colls[name]
459 print 'Collection %s deleted' % name
460 return
461 else:
462 print 'Error! Collection does not exist, %s' % self.__kvlite_colls[name]
463 except WrongURIException:
464 print 'Error! Incorrect URI'
465 return
466 except ConnectionError, err:
467 print 'Connection Error! Please check URI, %s' % str(err)
468 return
469
470 def do_hash(self, line):
471 ''' hash [string]\tgenerate sha1 hash, random if string is not defined'''
472 import hashlib
473 import datetime
474 if not line:
475 str_now = str(datetime.datetime.now())
476 print 'Random sha1 hash:', hashlib.sha1(str_now).hexdigest()
477 else:
478 line = line.rstrip().lstrip()
479 print 'sha1 hash:', hashlib.sha1(line).hexdigest()
480
481 def do_keys(self, line):
482 ''' keys\t\t\tlist of keys '''
483 if not self.__current_coll_name in self.__kvlite_colls:
484 print 'Error! Unknown collection: %s' % self.__current_coll_name
485 return
486 for k,v in self.__current_coll.get():
487 print k
488
489 def do_items(self, line):
490 ''' items\t\tlist of collection's items '''
491 if not self.__current_coll_name in self.__kvlite_colls:
492 print 'Error! Unknown collection: %s' % self.__current_coll_name
493 return
494 for k,v in self.__current_coll.get():
495 print k
496 pprint.pprint(v)
497 print
498
499 def do_count(self, args):
500 ''' count\t\tshow the amount of entries in collection '''
501 if self.__current_coll:
502 print self.__current_coll.count()
503
504 def do_get(self, key):
505 ''' get <key>\t\tshow collection entry by key'''
506 if not key:
507 print getattr(self, 'do_get').__doc__
508 return
509 for key in [k for k in key.split(' ') if k <> '']:
510 if self.__current_coll:
511 k, v = self.__current_coll.get(key)
512 print k
513 pprint.pprint(v)
514 print
515 else:
516 print 'Error! The collection is not selected, please use collection'
517 return
518
519 def do_put(self, line):
520 ''' put <key> <value>\tstore entry to collection'''
521 try:
522 k,v = [i for i in line.split(' ',1) if i <> '']
523 except ValueError:
524 print getattr(self, 'do_put').__doc__
525 return
526
527 try:
528 v = json_decode(v)
529 except ValueError, err:
530 print 'Value decoding error!', err
531 return
532
533 if self.__current_coll:
534 try:
535 self.__current_coll.put(k, v)
536 self.__current_coll.commit()
537 print 'Done'
538 return
539 except WronKeyValue, err:
540 print 'Error! Incorrect key/value,', err
541 return
542 else:
543 print 'Error! The collection is not selected, please use collection'
544 return
545
546
547 def do_delete(self, key):
548 ''' delete <key>\t\tdelete entry by key '''
549 key = key.rstrip().lstrip()
550 if self.__current_coll.get(key) <> (None, None):
551 self.__current_coll.delete(key)
552 self.__current_coll.commit()
553 print 'Done'
554 return
555 else:
556 print 'Error! The key %s does not exist' % key
557 return
558
559 # ----------------------------------
560 # main
561 # ----------------------------------
562 if __name__ == '__main__':
563 console = Console()
564 console.cmdloop()
565
566
Something went wrong with that request. Please try again.