Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100644 882 lines (744 sloc) 29.793 kb
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
1 #!/usr/bin/python
2 # -*- coding: utf-8 -*-
3 #
4 # Simple key-value datastore
8a87e13 @ownport add few TODO and billets for sqlite implementation
authored
5
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
6 # - support only mysql database
7 # - console support added
8 #
9 # some ideas taked from PyMongo interface http://api.mongodb.org/python/current/index.html
10 # kvlite2 tutorial http://code.google.com/p/kvlite/wiki/kvlite2
11 #
12 # TODO autocommit for put()
8a87e13 @ownport add few TODO and billets for sqlite implementation
authored
13 # TODO synchronise documents between few datastores
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
14 #
15 #
16 __author__ = 'Andrey Usov <http://devel.ownport.net>'
17 __version__ = '0.3'
18 __license__ = """
19 Redistribution and use in source and binary forms, with or without modification,
20 are permitted provided that the following conditions are met:
21
22 * Redistributions of source code must retain the above copyright notice,
23 this list of conditions and the following disclaimer.
24 * Redistributions in binary form must reproduce the above copyright notice,
25 this list of conditions and the following disclaimer in the documentation
26 and/or other materials provided with the distribution.
27
28 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS'
29 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
31 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
32 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
33 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
34 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
35 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
36 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
37 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
38 POSSIBILITY OF SUCH DAMAGE."""
39
212f56d @ownport added sqlite3 support
authored
40 import os
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
41 import cmd
42 import sys
274b369 @ownport separate kvlite and console
authored
43 import json
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
44 import zlib
fc6d7df @ownport update scheme and code
authored
45 import uuid
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
46 import pprint
212f56d @ownport added sqlite3 support
authored
47 import sqlite3
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
48 import binascii
49
1b145ca @ownport added MysqlConnectionManager + tests
authored
50 __all__ = ['open', 'remove',]
51
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
52 try:
53 import MySQLdb
54 except ImportError:
55 print >> sys.stderr, 'Error! MySQLdb package is not installed, please install python-mysqldb'
56 sys.exit()
57
fc6d7df @ownport update scheme and code
authored
58 try:
59 import cPickle as pickle
60 except ImportError:
61 import pickle
212f56d @ownport added sqlite3 support
authored
62 # TODO add test cases for serializers
8a87e13 @ownport add few TODO and billets for sqlite implementation
authored
63 # TODO add support user specific serializators
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
64
22f983f @ownport migration from old scheme to new
authored
65 SUPPORTED_BACKENDS = ['mysql', 'sqlite', ]
66
212f56d @ownport added sqlite3 support
authored
67 '''
68 A collection is a group of documents stored in kvlite2,
69 and can be thought of as roughly the equivalent of a
70 table in a relational database.
71
72
73
74 For using JSON as serialization
75
76 >>> import json
77 >>> collection = open('sqlite://test.sqlite:test', serializer=json)
78 >>>
79
80 '''
81 # -----------------------------------------------------------------
82 # cPickleSerializer class
83 # -----------------------------------------------------------------
84
85 class cPickleSerializer(object):
86 ''' cPickleSerializer '''
87
88 @staticmethod
89 def dumps(v):
90 ''' dumps value '''
91 if isinstance(v, unicode):
92 v = str(v)
93 return pickle.dumps(v)
94
95 @staticmethod
96 def loads(v):
97 ''' loads value '''
98 if isinstance(v, unicode):
99 v = str(v)
100 return pickle.loads(v)
101
102 # -----------------------------------------------------------------
103 # CompressedJsonSerializer class
104 # -----------------------------------------------------------------
105
106 class CompressedJsonSerializer(object):
107 ''' CompressedJsonSerializer '''
108
109 @staticmethod
110 def dumps(v):
111 ''' dumps value '''
274b369 @ownport separate kvlite and console
authored
112 return zlib.compress(json.dumps(v))
212f56d @ownport added sqlite3 support
authored
113
114 @staticmethod
115 def loads(v):
116 ''' loads value '''
274b369 @ownport separate kvlite and console
authored
117 return json.loads(zlib.decompress(v))
212f56d @ownport added sqlite3 support
authored
118
119 # -----------------------------------------------------------------
120 # SERIALIZERS
121 # -----------------------------------------------------------------
122
123 SERIALIZERS = {
124 'pickle': cPickleSerializer,
125 'completed_json': CompressedJsonSerializer,
126 }
127
e9990ae @ownport move old code to MysqlCollection
authored
128
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
129 # -----------------------------------------------------------------
8ffdeaf @ownport added open() function for creation and opening collection
authored
130 # KVLite utils
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
131 # -----------------------------------------------------------------
212f56d @ownport added sqlite3 support
authored
132 def open(uri, serializer=cPickleSerializer):
8ffdeaf @ownport added open() function for creation and opening collection
authored
133 '''
134 open collection by URI,
135 if collection does not exist kvlite will try to create it
136
137 in case of successful opening or creation new collection
138 return Collection object
fc6d7df @ownport update scheme and code
authored
139
140 serializer: the class or module to serialize msgs with, must have
141 methods or functions named ``dumps`` and ``loads``,
142 `pickle <http://docs.python.org/library/pickle.html>`_ is the default,
143 use ``None`` to store messages in plain text (suitable for strings,
144 integers, etc)
8ffdeaf @ownport added open() function for creation and opening collection
authored
145 '''
fc6d7df @ownport update scheme and code
authored
146 # TODO save kvlite configuration in database
1b145ca @ownport added MysqlConnectionManager + tests
authored
147
fc6d7df @ownport update scheme and code
authored
148 manager = CollectionManager(uri)
149 params = manager.parse_uri(uri)
150 if params['collection'] not in manager.collections():
151 manager.create(params['collection'])
212f56d @ownport added sqlite3 support
authored
152 return manager.collection_class(manager.connection, params['collection'], serializer)
1b145ca @ownport added MysqlConnectionManager + tests
authored
153
154 def remove(uri):
155 '''
156 remove collection by URI
157 '''
22f983f @ownport migration from old scheme to new
authored
158 backend, rest_uri = uri.split('://')
159 if backend in SUPPORTED_BACKENDS:
160 if backend == 'mysql':
1b145ca @ownport added MysqlConnectionManager + tests
authored
161 MysqlCollection(uri).remove()
22f983f @ownport migration from old scheme to new
authored
162 elif backend == 'sqlite':
1b145ca @ownport added MysqlConnectionManager + tests
authored
163 SqliteCollection(uri).remove()
22f983f @ownport migration from old scheme to new
authored
164 else:
1b145ca @ownport added MysqlConnectionManager + tests
authored
165 raise NotImplementedError('unknown backend: {}'.format(uri))
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
166 else:
22f983f @ownport migration from old scheme to new
authored
167 raise RuntimeError('Unknown backend: {}'.format(backend))
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
168
fc6d7df @ownport update scheme and code
authored
169 def get_uuid(amount=100):
170 ''' return UUIDs '''
171
172 # TODO check another approach for generation UUIDs, more fast
173 # TODO get uuids from file as approach to speedup UUIDs generation
174
175 uuids = list()
176 for _ in xrange(amount):
177 u = str(uuid.uuid4()).replace('-', '')
178 uuids.append(("%040s" % u).replace(' ','0'))
179 return uuids
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
180
181 # -----------------------------------------------------------------
1b145ca @ownport added MysqlConnectionManager + tests
authored
182 # CollectionManager class
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
183 # -----------------------------------------------------------------
1b145ca @ownport added MysqlConnectionManager + tests
authored
184 class CollectionManager(object):
185 ''' Collection Manager'''
186
22f983f @ownport migration from old scheme to new
authored
187 def __init__(self, uri):
274b369 @ownport separate kvlite and console
authored
188
189 self.backend_manager = None
190
1b145ca @ownport added MysqlConnectionManager + tests
authored
191 backend, rest_uri = uri.split('://')
192 if backend in SUPPORTED_BACKENDS:
193 if backend == 'mysql':
212f56d @ownport added sqlite3 support
authored
194 self.backend_manager = MysqlCollectionManager(uri)
1b145ca @ownport added MysqlConnectionManager + tests
authored
195 elif backend == 'sqlite':
212f56d @ownport added sqlite3 support
authored
196 self.backend_manager = SqliteCollectionManager(uri)
1b145ca @ownport added MysqlConnectionManager + tests
authored
197 else:
198 raise RuntimeError('Unknown backend: {}'.format(backend))
fc6d7df @ownport update scheme and code
authored
199
212f56d @ownport added sqlite3 support
authored
200 def parse_uri(self, uri):
201 ''' parse_uri '''
202 return self.backend_manager.parse_uri(uri)
203
fc6d7df @ownport update scheme and code
authored
204 def create(self, name):
205 ''' create collection '''
212f56d @ownport added sqlite3 support
authored
206 self.backend_manager.create(name)
fc6d7df @ownport update scheme and code
authored
207
212f56d @ownport added sqlite3 support
authored
208 @property
209 def collection_class(self):
fc6d7df @ownport update scheme and code
authored
210 ''' return object MysqlCollection or SqliteCollection '''
212f56d @ownport added sqlite3 support
authored
211 return self.backend_manager.collection_class
212
213 @property
214 def connection(self):
215 ''' return reference to backend connection '''
216 return self.backend_manager.connection
fc6d7df @ownport update scheme and code
authored
217
218 def collections(self):
219 ''' return list of collections '''
212f56d @ownport added sqlite3 support
authored
220 return self.backend_manager.collections()
fc6d7df @ownport update scheme and code
authored
221
222 def remove(self, name):
223 ''' remove collection '''
212f56d @ownport added sqlite3 support
authored
224 self.backend_manager.remove(name)
1b145ca @ownport added MysqlConnectionManager + tests
authored
225
226 # -----------------------------------------------------------------
227 # MysqlCollectionManager class
228 # -----------------------------------------------------------------
229 class MysqlCollectionManager(object):
230 ''' MysqlCollectionManager '''
231
232 def __init__(self, uri):
22f983f @ownport migration from old scheme to new
authored
233
fc6d7df @ownport update scheme and code
authored
234 params = self.parse_uri(uri)
212f56d @ownport added sqlite3 support
authored
235
1b145ca @ownport added MysqlConnectionManager + tests
authored
236 try:
237 self.__conn = MySQLdb.connect(
238 host=params['host'], port = params['port'],
239 user=params['username'], passwd=params['password'],
240 db=params['db'])
241 except MySQLdb.OperationalError,err:
242 raise RuntimeError(err)
243 self.__cursor = self.__conn.cursor()
22f983f @ownport migration from old scheme to new
authored
244
245 @staticmethod
fc6d7df @ownport update scheme and code
authored
246 def parse_uri(uri):
247 '''parse URI
22f983f @ownport migration from old scheme to new
authored
248
249 return driver, user, password, host, port, database, table
250 '''
fc6d7df @ownport update scheme and code
authored
251 parsed_uri = dict()
252 parsed_uri['backend'], rest_uri = uri.split('://', 1)
253 parsed_uri['username'], rest_uri = rest_uri.split(':', 1)
254 parsed_uri['password'], rest_uri = rest_uri.split('@', 1)
255
256 if ':' in rest_uri:
257 parsed_uri['host'], rest_uri = rest_uri.split(':', 1)
258 parsed_uri['port'], rest_uri = rest_uri.split('/', 1)
259 parsed_uri['port'] = int(parsed_uri['port'])
22f983f @ownport migration from old scheme to new
authored
260 else:
fc6d7df @ownport update scheme and code
authored
261 parsed_uri['host'], rest_uri = rest_uri.split('/')
262 parsed_uri['port'] = 3306
22f983f @ownport migration from old scheme to new
authored
263
fc6d7df @ownport update scheme and code
authored
264 if '.' in rest_uri:
265 parsed_uri['db'], parsed_uri['collection'] = rest_uri.split('.', 1)
266 else:
267 parsed_uri['db'] = rest_uri
268 parsed_uri['collection'] = None
269 return parsed_uri
212f56d @ownport added sqlite3 support
authored
270
1b145ca @ownport added MysqlConnectionManager + tests
authored
271 def create(self, name):
22f983f @ownport migration from old scheme to new
authored
272 ''' create collection '''
1b145ca @ownport added MysqlConnectionManager + tests
authored
273
22f983f @ownport migration from old scheme to new
authored
274 SQL_CREATE_TABLE = '''CREATE TABLE IF NOT EXISTS %s (
275 __rowid__ INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
276 k BINARY(20) NOT NULL,
277 v MEDIUMBLOB,
278 UNIQUE KEY (k) ) ENGINE=InnoDB;'''
279
1b145ca @ownport added MysqlConnectionManager + tests
authored
280 self.__cursor.execute(SQL_CREATE_TABLE % name)
281 self.__conn.commit()
282
212f56d @ownport added sqlite3 support
authored
283 @property
284 def connection(self):
285 ''' return connection '''
286 return self.__conn
287
288 @property
289 def collection_class(self):
fc6d7df @ownport update scheme and code
authored
290 ''' return MysqlCollection object'''
212f56d @ownport added sqlite3 support
authored
291 return MysqlCollection
1b145ca @ownport added MysqlConnectionManager + tests
authored
292
293 def collections(self):
294 ''' return collection list'''
295 self.__cursor.execute('SHOW TABLES;')
fc6d7df @ownport update scheme and code
authored
296 return [t[0] for t in self.__cursor.fetchall()]
1b145ca @ownport added MysqlConnectionManager + tests
authored
297
298
299 def remove(self, name):
300 ''' remove collection '''
301 if name in self.collections():
302 self.__cursor.execute('DROP TABLE %s;' % name)
303 self.__conn.commit()
304 else:
305 raise RuntimeError('No collection with name: {}'.format(name))
306
307 def close(self):
308 ''' close connection to database '''
309 self.__conn.close()
310
311 # -----------------------------------------------------------------
312 # MysqlCollection class
313 # -----------------------------------------------------------------
314 class MysqlCollection(object):
315 ''' Mysql Connection '''
316
212f56d @ownport added sqlite3 support
authored
317 def __init__(self, connection, collection_name, serializer=cPickleSerializer):
fc6d7df @ownport update scheme and code
authored
318
319 self.__conn = connection
1b145ca @ownport added MysqlConnectionManager + tests
authored
320 self.__cursor = self.__conn.cursor()
fc6d7df @ownport update scheme and code
authored
321 self.__collection = collection_name
212f56d @ownport added sqlite3 support
authored
322 self.__serializer = serializer
323
fc6d7df @ownport update scheme and code
authored
324 self.__uuid_cache = list()
22f983f @ownport migration from old scheme to new
authored
325
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
326 def get_uuid(self):
fc6d7df @ownport update scheme and code
authored
327 """
328 if mysql connection is available more fast way to use this method
329 than global function - get_uuid()
330
331 return id based on uuid """
332
333 if not self.__uuid_cache:
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
334 self.__cursor.execute('SELECT %s;' % ','.join(['uuid()' for _ in range(100)]))
335 for uuid in self.__cursor.fetchone():
336 u = uuid.split('-')
337 u.reverse()
338 u = ("%040s" % ''.join(u)).replace(' ','0')
fc6d7df @ownport update scheme and code
authored
339 self.__uuid_cache.append(u)
340 return self.__uuid_cache.pop()
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
341
342 def __get_many(self):
343 ''' return all docs '''
344 rowid = 0
345 while True:
346 SQL_SELECT_MANY = 'SELECT __rowid__, k,v FROM %s WHERE __rowid__ > %d LIMIT 1000 ;' % (self.__collection, rowid)
347 self.__cursor.execute(SQL_SELECT_MANY)
348 result = self.__cursor.fetchall()
349 if not result:
350 break
351 for r in result:
352 rowid = r[0]
353 k = binascii.b2a_hex(r[1])
354 try:
212f56d @ownport added sqlite3 support
authored
355 v = self.__serializer.loads(r[2])
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
356 except Exception, err:
212f56d @ownport added sqlite3 support
authored
357 raise RuntimeError('key %s, %s' % (k, err))
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
358 yield (k, v)
359
360 def get(self, k=None):
361 '''
362 return document by key from collection
363 return documents if key is not defined
364 '''
365 if k:
366 if len(k) > 40:
367 raise WronKeyValue()
368 SQL = 'SELECT k,v FROM %s WHERE k = ' % self.__collection
369 try:
370 self.__cursor.execute(SQL + "%s", binascii.a2b_hex(k))
371 except TypeError, err:
372 raise WronKeyValue(err)
373 result = self.__cursor.fetchone()
374 if result:
375 try:
212f56d @ownport added sqlite3 support
authored
376 v = self.__serializer.loads(result[1])
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
377 except Exception, err:
212f56d @ownport added sqlite3 support
authored
378 raise RuntimeError('key %s, %s' % (k, err))
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
379 return (binascii.b2a_hex(result[0]), v)
380 else:
381 return (None, None)
382 else:
383 return self.__get_many()
384
385 def put(self, k, v):
386 ''' put document in collection '''
212f56d @ownport added sqlite3 support
authored
387
388 # TODO many k/v in put()
389
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
390 if len(k) > 40:
fc6d7df @ownport update scheme and code
authored
391 raise RuntimeError('The length of key is more than 40 bytes')
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
392 SQL_INSERT = 'INSERT INTO %s (k,v) ' % self.__collection
393 SQL_INSERT += 'VALUES (%s,%s) ON DUPLICATE KEY UPDATE v=%s;;'
212f56d @ownport added sqlite3 support
authored
394 v = self.__serializer.dumps(v)
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
395 try:
396 self.__cursor.execute(SQL_INSERT, (binascii.a2b_hex(k), v, v))
397 except TypeError, err:
fc6d7df @ownport update scheme and code
authored
398 raise RuntimeError(err)
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
399
400 def delete(self, k):
401 ''' delete document by k '''
402 if len(k) > 40:
403 raise WronKeyValue()
404 SQL_DELETE = '''DELETE FROM %s WHERE k = ''' % self.__collection
405 try:
406 self.__cursor.execute(SQL_DELETE + "%s;", binascii.a2b_hex(k))
407 except TypeError, err:
408 raise WronKeyValue(err)
409
410 def keys(self):
411 ''' return document keys in collection'''
412 rowid = 0
413 while True:
414 SQL_SELECT_MANY = 'SELECT __rowid__, k FROM %s WHERE __rowid__ > %d LIMIT 1000 ;' % (self.__collection, rowid)
415 self.__cursor.execute(SQL_SELECT_MANY)
416 result = self.__cursor.fetchall()
417 if not result:
418 break
419 for r in result:
420 rowid = r[0]
421 k = binascii.b2a_hex(r[1])
422 yield k
423
212f56d @ownport added sqlite3 support
authored
424 @property
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
425 def count(self):
426 ''' return amount of documents in collection'''
427 self.__cursor.execute('SELECT count(*) FROM %s;' % self.__collection)
428 return int(self.__cursor.fetchone()[0])
429
430 def commit(self):
431 self.__conn.commit()
432
433 def close(self):
434 ''' close connection to database '''
435 self.__conn.close()
436
fc6d7df @ownport update scheme and code
authored
437 # -----------------------------------------------------------------
438 # SqliteCollectionManager class
439 # -----------------------------------------------------------------
440 class SqliteCollectionManager(object):
441 ''' Sqlite Collection Manager '''
212f56d @ownport added sqlite3 support
authored
442 def __init__(self, uri):
443
444 params = self.parse_uri(uri)
445
446 self.__conn = sqlite3.connect(params['db'])
447 self.__cursor = self.__conn.cursor()
448
449 @staticmethod
450 def parse_uri(uri):
451 '''parse URI
452
453 return driver, database, collection
454 '''
455 parsed_uri = dict()
456 parsed_uri['backend'], rest_uri = uri.split('://', 1)
457 if ':' in rest_uri:
458 parsed_uri['db'], parsed_uri['collection'] = rest_uri.split(':',1)
459 else:
460 parsed_uri['db'] = rest_uri
461 parsed_uri['collection'] = None
462 if parsed_uri['db'] == 'memory':
463 parsed_uri['db'] = ':memory:'
464 return parsed_uri
465
466 @property
467 def connection(self):
468 ''' return connection '''
469 return self.__conn
470
471 @property
472 def collection_class(self):
473 ''' return SqliteCollection object'''
474 return SqliteCollection
475
476 def collections(self):
477 ''' return collection list'''
478
479 self.__cursor.execute('SELECT name FROM sqlite_master WHERE type="table";')
480 return [t[0] for t in self.__cursor.fetchall()]
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
481
212f56d @ownport added sqlite3 support
authored
482 def create(self, name):
483 ''' create collection '''
484
485 SQL_CREATE_TABLE = '''CREATE TABLE IF NOT EXISTS %s (
486 k NOT NULL, v, UNIQUE (k) );'''
487
488 self.__cursor.execute(SQL_CREATE_TABLE % name)
489 self.__conn.commit()
490
491 def remove(self, name):
492 ''' remove collection '''
493 if name in self.collections():
494 self.__cursor.execute('DROP TABLE %s;' % name)
495 self.__conn.commit()
496 else:
497 raise RuntimeError('No collection with name: {}'.format(name))
498
499 def close(self):
500 ''' close connection to database '''
501 self.__conn.close()
502
22f983f @ownport migration from old scheme to new
authored
503 # -----------------------------------------------------------------
504 # SqliteCollection class
505 # -----------------------------------------------------------------
e9990ae @ownport move old code to MysqlCollection
authored
506 class SqliteCollection(object):
22f983f @ownport migration from old scheme to new
authored
507 ''' Sqlite Collection'''
508
212f56d @ownport added sqlite3 support
authored
509 def __init__(self, connection, collection_name, serializer=cPickleSerializer):
e9990ae @ownport move old code to MysqlCollection
authored
510
212f56d @ownport added sqlite3 support
authored
511 self.__conn = connection
512 self.__cursor = self.__conn.cursor()
513 self.__collection = collection_name
514 self.__serializer = serializer
e9990ae @ownport move old code to MysqlCollection
authored
515
212f56d @ownport added sqlite3 support
authored
516 self.__uuid_cache = list()
e9990ae @ownport move old code to MysqlCollection
authored
517
212f56d @ownport added sqlite3 support
authored
518 def get_uuid(self):
519 """ return id based on uuid """
e9990ae @ownport move old code to MysqlCollection
authored
520
212f56d @ownport added sqlite3 support
authored
521 if not self.__uuid_cache:
522 for uuid in get_uuid():
523 self.__uuid_cache.append(uuid)
524 return self.__uuid_cache.pop()
525
526 def put(self, k, v):
527 ''' put document in collection '''
e9990ae @ownport move old code to MysqlCollection
authored
528
212f56d @ownport added sqlite3 support
authored
529 # TODO many k/v in put()
530
531 if len(k) > 40:
532 raise RuntimeError('The length of key is more than 40 bytes')
533 SQL_INSERT = 'INSERT OR REPLACE INTO %s (k,v) ' % self.__collection
534 SQL_INSERT += 'VALUES (?,?)'
535 v = self.__serializer.dumps(v)
536 try:
537 self.__cursor.execute(SQL_INSERT, (k, v))
538 except TypeError, err:
539 raise RuntimeError(err)
e9990ae @ownport move old code to MysqlCollection
authored
540
212f56d @ownport added sqlite3 support
authored
541 def __get_many(self):
542 ''' return all docs '''
543 rowid = 0
544 while True:
545 SQL_SELECT_MANY = 'SELECT rowid, k,v FROM %s WHERE rowid > %d LIMIT 1000 ;' % (self.__collection, rowid)
546 self.__cursor.execute(SQL_SELECT_MANY)
547 result = self.__cursor.fetchall()
548 if not result:
549 break
550 for r in result:
551 rowid = r[0]
552 k = r[1]
553 try:
554 v = self.__serializer.loads(r[2])
555 except Exception, err:
556 raise RuntimeError('key %s, %s' % (k, err))
557 yield (k, v)
558
559 def get(self, k=None):
560 '''
561 return document by key from collection
562 return documents if key is not defined
e9990ae @ownport move old code to MysqlCollection
authored
563 '''
212f56d @ownport added sqlite3 support
authored
564 if k:
565 if len(k) > 40:
566 raise RuntimeError('The key length is more than 40 bytes')
567 SQL = 'SELECT k,v FROM %s WHERE k = ?;' % self.__collection
568 try:
569 self.__cursor.execute(SQL, (k,))
570 except TypeError, err:
571 raise WronKeyValue(err)
572 result = self.__cursor.fetchone()
573 if result:
574 try:
575 v = self.__serializer.loads(result[1])
576 except Exception, err:
577 raise RuntimeError('key %s, %s' % (k, err))
578 return (result[0], v)
579 else:
580 return (None, None)
581 else:
582 return self.__get_many()
583
584 def keys(self):
585 ''' return document keys in collection'''
586 rowid = 0
587 while True:
588 SQL_SELECT_MANY = 'SELECT rowid, k FROM %s WHERE rowid > %d LIMIT 1000 ;' % (self.__collection, rowid)
589 self.__cursor.execute(SQL_SELECT_MANY)
590 result = self.__cursor.fetchall()
591 if not result:
592 break
593 for r in result:
594 rowid = r[0]
595 yield r[1]
e9990ae @ownport move old code to MysqlCollection
authored
596
212f56d @ownport added sqlite3 support
authored
597 def delete(self, k):
598 ''' delete document by k '''
599 if len(k) > 40:
600 raise RuntimeError('The key length is more than 40 bytes')
601 SQL_DELETE = '''DELETE FROM %s WHERE k = ?;''' % self.__collection
602 try:
603 self.__cursor.execute(SQL_DELETE, (k,))
604 except TypeError, err:
605 raise WronKeyValue(err)
606
607 @property
608 def count(self):
609 ''' return amount of documents in collection'''
610 self.__cursor.execute('SELECT count(*) FROM %s;' % self.__collection)
611 return int(self.__cursor.fetchone()[0])
612
613 def commit(self):
614 self.__conn.commit()
615
616 def close(self):
617 ''' close connection to database '''
618 self.__conn.close()
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored
619
620 # -----------------------------------------------------------------
621 # Console class
622 # -----------------------------------------------------------------
623 class Console(cmd.Cmd):
624 def __init__(self):
625 cmd.Cmd.__init__(self)
626 self.prompt = "kvlite> "
627 self.ruler = '-'
628
629 self.__history_size = 20
630 self.__history = list()
631 self.__kvlite_colls = dict()
632 self.__current_coll_name = 'kvlite'
633 self.__current_coll = None
634
635 def emptyline(self):
636 return False
637
638 def do_help(self, arg):
639 ''' help <command>\tshow <command> help'''
640 if arg:
641 try:
642 func = getattr(self, 'help_' + arg)
643 except AttributeError:
644 try:
645 doc=getattr(self, 'do_' + arg).__doc__
646 if doc:
647 self.stdout.write("%s\n"%str(doc))
648 return
649 except AttributeError:
650 pass
651 self.stdout.write("%s\n"%str(self.nohelp % (arg,)))
652 return
653 else:
654 names = [
655 '', 'do_help', 'do_version', 'do_licence', 'do_history', 'do_exit', '',
656 'do_create', 'do_use', 'do_show', 'do_remove', 'do_import', 'do_export', '',
657 'do_hash', 'do_keys', 'do_items', 'do_get', 'do_put', 'do_delete', 'do_count', ''
658 ]
659 for name in names:
660 if not name:
661 print
662 else:
663 print getattr(self, name).__doc__
664
665 def do_history(self,line):
666 ''' history\t\tshow commands history '''
667 for i, line in enumerate(self.__history):
668 print "0%d. %s" % (i+1, line)
669
670 def precmd(self, line):
671 if len(self.__history) == self.__history_size:
672 prev_line = self.__history.pop(0)
673 if line and line not in self.__history:
674 self.__history.append(line)
675 return line
676
677 def do_version(self, line):
678 ''' version\t\tshow kvlite version'''
679 print 'version: %s' % __version__
680
681 def do_licence(self, line):
682 ''' licence\t\tshow licence'''
683 print __license__
684 print
685
686 def do_exit(self, line):
687 ''' exit\t\t\texit from console '''
688 return True
689
690 def do_import(self, filename):
691 ''' import <filename>\timport collection configuration from JSON file'''
692 import os
693
694 if not filename:
695 print getattr(self, 'do_import').__doc__
696 return
697 filename = filename.rstrip().lstrip()
698
699 if os.path.isfile(filename):
700 for k, v in json_decode(open(filename).read()).items():
701 self.__kvlite_colls[k] = v
702 print 'Import completed'
703 else:
704 print 'Error! File %s does not exists' % filename
705
706 def do_export(self, filename):
707 ''' export <filename>\texport collection configurations to JSON file'''
708 # TODO check if file exists. If yes, import about it
709 if not filename:
710 print getattr(self, 'do_import').__doc__
711 return
712 filename = filename.rstrip().lstrip()
713 json_file = open(filename, 'w')
714 json_file.write(json_encode(self.__kvlite_colls))
715 json_file.close()
716 print 'Export completed to file: %s' % filename
717
718 def do_show(self, line):
719 ''' show collections\tlist of available collections (defined in settings.py)'''
720 if line == 'collections':
721 for coll in self.__kvlite_colls:
722 print ' %s' % coll
723 else:
724 print 'Unknown argument: %s' % line
725
726 def do_use(self, collection_name):
727 ''' use <collection>\tuse the collection as the default (current) collection'''
728 if collection_name in self.__kvlite_colls:
729 self.prompt = '%s>' % collection_name
730 self.__current_coll_name = collection_name
731 self.__current_coll = Collection(self.__kvlite_colls[self.__current_coll_name])
732 return
733 else:
734 print 'Error! Unknown collection: %s' % collection_name
735
736 def do_create(self, line):
737 ''' create <name> <uri>\tcreate new collection (if not exists)'''
738 try:
739 name, uri = [i for i in line.split(' ') if i <> '']
740 except ValueError:
741 print getattr(self, 'do_create').__doc__
742 return
743
744 if name in self.__kvlite_colls:
745 print 'Warning! Collection name already defined: %s, %s' % (name, self.__kvlite_colls[name])
746 print 'If needed please change collection name'
747 return
748 try:
749 if is_collection_exists(uri):
750 self.__kvlite_colls[name] = uri
751 print 'Connection exists, the reference added to collection list'
752 return
753 else:
754 create_collection(uri)
755 self.__kvlite_colls[name] = uri
756 print 'Collection created and added to collection list'
757 return
758 except WrongURIException:
759 print 'Error! Incorrect URI'
760 return
761 except ConnectionError, err:
762 print 'Connection Error! Please check URI, %s' % str(err)
763 return
764
765 def do_remove(self, name):
766 ''' remove <collection>\tremove collection'''
767 if name not in self.__kvlite_colls:
768 print 'Error! Collection name does not exist: %s' % name
769 return
770 try:
771 if is_collection_exists(self.__kvlite_colls[name]):
772 delete_collection(self.__kvlite_colls[name])
773 del self.__kvlite_colls[name]
774 print 'Collection %s deleted' % name
775 return
776 else:
777 print 'Error! Collection does not exist, %s' % self.__kvlite_colls[name]
778 except WrongURIException:
779 print 'Error! Incorrect URI'
780 return
781 except ConnectionError, err:
782 print 'Connection Error! Please check URI, %s' % str(err)
783 return
784
785 def do_hash(self, line):
786 ''' hash [string]\tgenerate sha1 hash, random if string is not defined'''
787 import hashlib
788 import datetime
789 if not line:
790 str_now = str(datetime.datetime.now())
791 print 'Random sha1 hash:', hashlib.sha1(str_now).hexdigest()
792 else:
793 line = line.rstrip().lstrip()
794 print 'sha1 hash:', hashlib.sha1(line).hexdigest()
795
796 def do_keys(self, line):
797 ''' keys\t\t\tlist of keys '''
798 if not self.__current_coll_name in self.__kvlite_colls:
799 print 'Error! Unknown collection: %s' % self.__current_coll_name
800 return
801 for k,v in self.__current_coll.get():
802 print k
803
804 def do_items(self, line):
805 ''' items\t\tlist of collection's items '''
806 if not self.__current_coll_name in self.__kvlite_colls:
807 print 'Error! Unknown collection: %s' % self.__current_coll_name
808 return
809 for k,v in self.__current_coll.get():
810 print k
811 pprint.pprint(v)
812 print
813
814 def do_count(self, args):
815 ''' count\t\tshow the amount of entries in collection '''
816 if self.__current_coll:
817 print self.__current_coll.count()
818
819 def do_get(self, key):
820 ''' get <key>\t\tshow collection entry by key'''
821 if not key:
822 print getattr(self, 'do_get').__doc__
823 return
824 for key in [k for k in key.split(' ') if k <> '']:
825 if self.__current_coll:
826 k, v = self.__current_coll.get(key)
827 print k
828 pprint.pprint(v)
829 print
830 else:
831 print 'Error! The collection is not selected, please use collection'
832 return
833
834 def do_put(self, line):
835 ''' put <key> <value>\tstore entry to collection'''
836 try:
837 k,v = [i for i in line.split(' ',1) if i <> '']
838 except ValueError:
839 print getattr(self, 'do_put').__doc__
840 return
841
842 try:
843 v = json_decode(v)
844 except ValueError, err:
845 print 'Value decoding error!', err
846 return
847
848 if self.__current_coll:
849 try:
850 self.__current_coll.put(k, v)
851 self.__current_coll.commit()
852 print 'Done'
853 return
854 except WronKeyValue, err:
855 print 'Error! Incorrect key/value,', err
856 return
857 else:
858 print 'Error! The collection is not selected, please use collection'
859 return
860
861
862 def do_delete(self, key):
863 ''' delete <key>\t\tdelete entry by key '''
864 key = key.rstrip().lstrip()
865 if self.__current_coll.get(key) <> (None, None):
866 self.__current_coll.delete(key)
867 self.__current_coll.commit()
868 print 'Done'
869 return
870 else:
871 print 'Error! The key %s does not exist' % key
872 return
873
874 # ----------------------------------
875 # main
876 # ----------------------------------
877 if __name__ == '__main__':
878 console = Console()
879 console.cmdloop()
880
881
Something went wrong with that request. Please try again.