Skip to content
Newer
Older
100644 598 lines (486 sloc) 19.3 KB
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored Jun 27, 2012
1 #!/usr/bin/python
2 # -*- coding: utf-8 -*-
3 #
4 # Simple key-value datastore
8a87e13 @ownport add few TODO and billets for sqlite implementation
authored Sep 7, 2012
5
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored Jun 27, 2012
6 # - support only mysql database
7 # - console support added
8 #
9 # some ideas taked from PyMongo interface http://api.mongodb.org/python/current/index.html
10 # kvlite2 tutorial http://code.google.com/p/kvlite/wiki/kvlite2
11 #
12 # TODO autocommit for put()
8a87e13 @ownport add few TODO and billets for sqlite implementation
authored Sep 7, 2012
13 # TODO synchronise documents between few datastores
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored Jun 27, 2012
14 #
15 #
16 __author__ = 'Andrey Usov <http://devel.ownport.net>'
17 __version__ = '0.3'
18 __license__ = """
19 Redistribution and use in source and binary forms, with or without modification,
20 are permitted provided that the following conditions are met:
21
22 * Redistributions of source code must retain the above copyright notice,
23 this list of conditions and the following disclaimer.
24 * Redistributions in binary form must reproduce the above copyright notice,
25 this list of conditions and the following disclaimer in the documentation
26 and/or other materials provided with the distribution.
27
28 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS'
29 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
31 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
32 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
33 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
34 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
35 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
36 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
37 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
38 POSSIBILITY OF SUCH DAMAGE."""
39
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
40 import os
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored Jun 27, 2012
41 import sys
274b369 @ownport separate kvlite and console
authored Sep 11, 2012
42 import json
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored Jun 27, 2012
43 import zlib
fc6d7df @ownport update scheme and code
authored Sep 10, 2012
44 import uuid
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
45 import sqlite3
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored Jun 27, 2012
46 import binascii
e8c408a @ownport preparation for merge kvlite/sqlite branch with master
authored Sep 12, 2012
47 import cPickle as pickle
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored Jun 27, 2012
48
1b145ca @ownport added MysqlConnectionManager + tests
authored Sep 9, 2012
49 __all__ = ['open', 'remove',]
50
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored Jun 27, 2012
51 try:
52 import MySQLdb
53 except ImportError:
54 print >> sys.stderr, 'Error! MySQLdb package is not installed, please install python-mysqldb'
55 sys.exit()
56
e8c408a @ownport preparation for merge kvlite/sqlite branch with master
authored Sep 12, 2012
57
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
58 # TODO add test cases for serializers
8a87e13 @ownport add few TODO and billets for sqlite implementation
authored Sep 7, 2012
59 # TODO add support user specific serializators
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored Jun 27, 2012
60
22f983f @ownport migration from old scheme to new
authored Sep 7, 2012
61 SUPPORTED_BACKENDS = ['mysql', 'sqlite', ]
62
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
63 '''
64 A collection is a group of documents stored in kvlite2,
65 and can be thought of as roughly the equivalent of a
66 table in a relational database.
67
68
69
70 For using JSON as serialization
71
72 >>> import json
73 >>> collection = open('sqlite://test.sqlite:test', serializer=json)
74 >>>
75
76 '''
77 # -----------------------------------------------------------------
78 # cPickleSerializer class
79 # -----------------------------------------------------------------
80
81 class cPickleSerializer(object):
82 ''' cPickleSerializer '''
83
84 @staticmethod
85 def dumps(v):
86 ''' dumps value '''
87 if isinstance(v, unicode):
88 v = str(v)
89 return pickle.dumps(v)
90
91 @staticmethod
92 def loads(v):
93 ''' loads value '''
94 return pickle.loads(v)
95
96 # -----------------------------------------------------------------
97 # CompressedJsonSerializer class
98 # -----------------------------------------------------------------
99
100 class CompressedJsonSerializer(object):
101 ''' CompressedJsonSerializer '''
102
e8c408a @ownport preparation for merge kvlite/sqlite branch with master
authored Sep 12, 2012
103 # TODO issue: sqlite doesn't support binary values
104
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
105 @staticmethod
106 def dumps(v):
107 ''' dumps value '''
274b369 @ownport separate kvlite and console
authored Sep 11, 2012
108 return zlib.compress(json.dumps(v))
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
109
110 @staticmethod
111 def loads(v):
112 ''' loads value '''
274b369 @ownport separate kvlite and console
authored Sep 11, 2012
113 return json.loads(zlib.decompress(v))
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
114
115 # -----------------------------------------------------------------
116 # SERIALIZERS
117 # -----------------------------------------------------------------
118
119 SERIALIZERS = {
120 'pickle': cPickleSerializer,
121 'completed_json': CompressedJsonSerializer,
122 }
123
e9990ae @ownport move old code to MysqlCollection
authored Sep 7, 2012
124
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored Jun 27, 2012
125 # -----------------------------------------------------------------
8ffdeaf @ownport added open() function for creation and opening collection
authored Sep 7, 2012
126 # KVLite utils
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored Jun 27, 2012
127 # -----------------------------------------------------------------
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
128 def open(uri, serializer=cPickleSerializer):
8ffdeaf @ownport added open() function for creation and opening collection
authored Sep 7, 2012
129 '''
130 open collection by URI,
131 if collection does not exist kvlite will try to create it
132
133 in case of successful opening or creation new collection
134 return Collection object
fc6d7df @ownport update scheme and code
authored Sep 10, 2012
135
136 serializer: the class or module to serialize msgs with, must have
137 methods or functions named ``dumps`` and ``loads``,
138 `pickle <http://docs.python.org/library/pickle.html>`_ is the default,
139 use ``None`` to store messages in plain text (suitable for strings,
140 integers, etc)
8ffdeaf @ownport added open() function for creation and opening collection
authored Sep 7, 2012
141 '''
fc6d7df @ownport update scheme and code
authored Sep 10, 2012
142 # TODO save kvlite configuration in database
1b145ca @ownport added MysqlConnectionManager + tests
authored Sep 9, 2012
143
fc6d7df @ownport update scheme and code
authored Sep 10, 2012
144 manager = CollectionManager(uri)
145 params = manager.parse_uri(uri)
146 if params['collection'] not in manager.collections():
147 manager.create(params['collection'])
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
148 return manager.collection_class(manager.connection, params['collection'], serializer)
1b145ca @ownport added MysqlConnectionManager + tests
authored Sep 9, 2012
149
150 def remove(uri):
151 '''
152 remove collection by URI
153 '''
e8c408a @ownport preparation for merge kvlite/sqlite branch with master
authored Sep 12, 2012
154 manager = CollectionManager(uri)
155 params = manager.parse_uri(uri)
156 if params['collection'] in manager.collections():
157 manager.remove(params['collection'])
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored Jun 27, 2012
158
fc6d7df @ownport update scheme and code
authored Sep 10, 2012
159 def get_uuid(amount=100):
160 ''' return UUIDs '''
161
162 # TODO check another approach for generation UUIDs, more fast
163 # TODO get uuids from file as approach to speedup UUIDs generation
164
165 uuids = list()
166 for _ in xrange(amount):
167 u = str(uuid.uuid4()).replace('-', '')
168 uuids.append(("%040s" % u).replace(' ','0'))
169 return uuids
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored Jun 27, 2012
170
171 # -----------------------------------------------------------------
1b145ca @ownport added MysqlConnectionManager + tests
authored Sep 9, 2012
172 # CollectionManager class
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored Jun 27, 2012
173 # -----------------------------------------------------------------
1b145ca @ownport added MysqlConnectionManager + tests
authored Sep 9, 2012
174 class CollectionManager(object):
175 ''' Collection Manager'''
176
22f983f @ownport migration from old scheme to new
authored Sep 7, 2012
177 def __init__(self, uri):
274b369 @ownport separate kvlite and console
authored Sep 11, 2012
178
179 self.backend_manager = None
180
1b145ca @ownport added MysqlConnectionManager + tests
authored Sep 9, 2012
181 backend, rest_uri = uri.split('://')
182 if backend in SUPPORTED_BACKENDS:
183 if backend == 'mysql':
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
184 self.backend_manager = MysqlCollectionManager(uri)
1b145ca @ownport added MysqlConnectionManager + tests
authored Sep 9, 2012
185 elif backend == 'sqlite':
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
186 self.backend_manager = SqliteCollectionManager(uri)
1b145ca @ownport added MysqlConnectionManager + tests
authored Sep 9, 2012
187 else:
188 raise RuntimeError('Unknown backend: {}'.format(backend))
fc6d7df @ownport update scheme and code
authored Sep 10, 2012
189
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
190 def parse_uri(self, uri):
191 ''' parse_uri '''
192 return self.backend_manager.parse_uri(uri)
193
fc6d7df @ownport update scheme and code
authored Sep 10, 2012
194 def create(self, name):
195 ''' create collection '''
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
196 self.backend_manager.create(name)
fc6d7df @ownport update scheme and code
authored Sep 10, 2012
197
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
198 @property
199 def collection_class(self):
fc6d7df @ownport update scheme and code
authored Sep 10, 2012
200 ''' return object MysqlCollection or SqliteCollection '''
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
201 return self.backend_manager.collection_class
202
203 @property
204 def connection(self):
205 ''' return reference to backend connection '''
206 return self.backend_manager.connection
fc6d7df @ownport update scheme and code
authored Sep 10, 2012
207
208 def collections(self):
209 ''' return list of collections '''
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
210 return self.backend_manager.collections()
fc6d7df @ownport update scheme and code
authored Sep 10, 2012
211
212 def remove(self, name):
213 ''' remove collection '''
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
214 self.backend_manager.remove(name)
1b145ca @ownport added MysqlConnectionManager + tests
authored Sep 9, 2012
215
216 # -----------------------------------------------------------------
217 # MysqlCollectionManager class
218 # -----------------------------------------------------------------
219 class MysqlCollectionManager(object):
220 ''' MysqlCollectionManager '''
221
222 def __init__(self, uri):
22f983f @ownport migration from old scheme to new
authored Sep 7, 2012
223
fc6d7df @ownport update scheme and code
authored Sep 10, 2012
224 params = self.parse_uri(uri)
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
225
1b145ca @ownport added MysqlConnectionManager + tests
authored Sep 9, 2012
226 try:
227 self.__conn = MySQLdb.connect(
228 host=params['host'], port = params['port'],
229 user=params['username'], passwd=params['password'],
230 db=params['db'])
231 except MySQLdb.OperationalError,err:
232 raise RuntimeError(err)
233 self.__cursor = self.__conn.cursor()
22f983f @ownport migration from old scheme to new
authored Sep 7, 2012
234
235 @staticmethod
fc6d7df @ownport update scheme and code
authored Sep 10, 2012
236 def parse_uri(uri):
237 '''parse URI
22f983f @ownport migration from old scheme to new
authored Sep 7, 2012
238
239 return driver, user, password, host, port, database, table
240 '''
fc6d7df @ownport update scheme and code
authored Sep 10, 2012
241 parsed_uri = dict()
242 parsed_uri['backend'], rest_uri = uri.split('://', 1)
243 parsed_uri['username'], rest_uri = rest_uri.split(':', 1)
244 parsed_uri['password'], rest_uri = rest_uri.split('@', 1)
245
246 if ':' in rest_uri:
247 parsed_uri['host'], rest_uri = rest_uri.split(':', 1)
248 parsed_uri['port'], rest_uri = rest_uri.split('/', 1)
249 parsed_uri['port'] = int(parsed_uri['port'])
22f983f @ownport migration from old scheme to new
authored Sep 7, 2012
250 else:
fc6d7df @ownport update scheme and code
authored Sep 10, 2012
251 parsed_uri['host'], rest_uri = rest_uri.split('/')
252 parsed_uri['port'] = 3306
22f983f @ownport migration from old scheme to new
authored Sep 7, 2012
253
fc6d7df @ownport update scheme and code
authored Sep 10, 2012
254 if '.' in rest_uri:
255 parsed_uri['db'], parsed_uri['collection'] = rest_uri.split('.', 1)
256 else:
257 parsed_uri['db'] = rest_uri
258 parsed_uri['collection'] = None
259 return parsed_uri
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
260
1b145ca @ownport added MysqlConnectionManager + tests
authored Sep 9, 2012
261 def create(self, name):
22f983f @ownport migration from old scheme to new
authored Sep 7, 2012
262 ''' create collection '''
1b145ca @ownport added MysqlConnectionManager + tests
authored Sep 9, 2012
263
22f983f @ownport migration from old scheme to new
authored Sep 7, 2012
264 SQL_CREATE_TABLE = '''CREATE TABLE IF NOT EXISTS %s (
265 __rowid__ INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
266 k BINARY(20) NOT NULL,
267 v MEDIUMBLOB,
268 UNIQUE KEY (k) ) ENGINE=InnoDB;'''
269
1b145ca @ownport added MysqlConnectionManager + tests
authored Sep 9, 2012
270 self.__cursor.execute(SQL_CREATE_TABLE % name)
271 self.__conn.commit()
272
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
273 @property
274 def connection(self):
275 ''' return connection '''
276 return self.__conn
277
278 @property
279 def collection_class(self):
fc6d7df @ownport update scheme and code
authored Sep 10, 2012
280 ''' return MysqlCollection object'''
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
281 return MysqlCollection
1b145ca @ownport added MysqlConnectionManager + tests
authored Sep 9, 2012
282
283 def collections(self):
284 ''' return collection list'''
285 self.__cursor.execute('SHOW TABLES;')
fc6d7df @ownport update scheme and code
authored Sep 10, 2012
286 return [t[0] for t in self.__cursor.fetchall()]
1b145ca @ownport added MysqlConnectionManager + tests
authored Sep 9, 2012
287
288
289 def remove(self, name):
290 ''' remove collection '''
291 if name in self.collections():
292 self.__cursor.execute('DROP TABLE %s;' % name)
293 self.__conn.commit()
294 else:
295 raise RuntimeError('No collection with name: {}'.format(name))
296
297 def close(self):
298 ''' close connection to database '''
299 self.__conn.close()
300
301 # -----------------------------------------------------------------
302 # MysqlCollection class
303 # -----------------------------------------------------------------
304 class MysqlCollection(object):
305 ''' Mysql Connection '''
306
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
307 def __init__(self, connection, collection_name, serializer=cPickleSerializer):
fc6d7df @ownport update scheme and code
authored Sep 10, 2012
308
309 self.__conn = connection
1b145ca @ownport added MysqlConnectionManager + tests
authored Sep 9, 2012
310 self.__cursor = self.__conn.cursor()
fc6d7df @ownport update scheme and code
authored Sep 10, 2012
311 self.__collection = collection_name
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
312 self.__serializer = serializer
313
fc6d7df @ownport update scheme and code
authored Sep 10, 2012
314 self.__uuid_cache = list()
22f983f @ownport migration from old scheme to new
authored Sep 7, 2012
315
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored Jun 27, 2012
316 def get_uuid(self):
fc6d7df @ownport update scheme and code
authored Sep 10, 2012
317 """
318 if mysql connection is available more fast way to use this method
319 than global function - get_uuid()
320
321 return id based on uuid """
322
323 if not self.__uuid_cache:
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored Jun 27, 2012
324 self.__cursor.execute('SELECT %s;' % ','.join(['uuid()' for _ in range(100)]))
325 for uuid in self.__cursor.fetchone():
326 u = uuid.split('-')
327 u.reverse()
328 u = ("%040s" % ''.join(u)).replace(' ','0')
fc6d7df @ownport update scheme and code
authored Sep 10, 2012
329 self.__uuid_cache.append(u)
330 return self.__uuid_cache.pop()
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored Jun 27, 2012
331
332 def __get_many(self):
333 ''' return all docs '''
334 rowid = 0
335 while True:
336 SQL_SELECT_MANY = 'SELECT __rowid__, k,v FROM %s WHERE __rowid__ > %d LIMIT 1000 ;' % (self.__collection, rowid)
337 self.__cursor.execute(SQL_SELECT_MANY)
338 result = self.__cursor.fetchall()
339 if not result:
340 break
341 for r in result:
342 rowid = r[0]
343 k = binascii.b2a_hex(r[1])
344 try:
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
345 v = self.__serializer.loads(r[2])
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored Jun 27, 2012
346 except Exception, err:
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
347 raise RuntimeError('key %s, %s' % (k, err))
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored Jun 27, 2012
348 yield (k, v)
349
350 def get(self, k=None):
351 '''
352 return document by key from collection
353 return documents if key is not defined
354 '''
355 if k:
356 if len(k) > 40:
e8c408a @ownport preparation for merge kvlite/sqlite branch with master
authored Sep 12, 2012
357 raise RuntimeError('The length of key is more than 40 bytes')
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored Jun 27, 2012
358 SQL = 'SELECT k,v FROM %s WHERE k = ' % self.__collection
359 try:
360 self.__cursor.execute(SQL + "%s", binascii.a2b_hex(k))
e8c408a @ownport preparation for merge kvlite/sqlite branch with master
authored Sep 12, 2012
361 except Exception, err:
362 raise RuntimeError(err)
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored Jun 27, 2012
363 result = self.__cursor.fetchone()
364 if result:
e8c408a @ownport preparation for merge kvlite/sqlite branch with master
authored Sep 12, 2012
365 v = self.__serializer.loads(result[1])
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored Jun 27, 2012
366 return (binascii.b2a_hex(result[0]), v)
367 else:
368 return (None, None)
369 else:
370 return self.__get_many()
371
372 def put(self, k, v):
373 ''' put document in collection '''
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
374
375 # TODO many k/v in put()
376
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored Jun 27, 2012
377 if len(k) > 40:
fc6d7df @ownport update scheme and code
authored Sep 10, 2012
378 raise RuntimeError('The length of key is more than 40 bytes')
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored Jun 27, 2012
379 SQL_INSERT = 'INSERT INTO %s (k,v) ' % self.__collection
380 SQL_INSERT += 'VALUES (%s,%s) ON DUPLICATE KEY UPDATE v=%s;;'
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
381 v = self.__serializer.dumps(v)
e8c408a @ownport preparation for merge kvlite/sqlite branch with master
authored Sep 12, 2012
382 self.__cursor.execute(SQL_INSERT, (binascii.a2b_hex(k), v, v))
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored Jun 27, 2012
383
384 def delete(self, k):
385 ''' delete document by k '''
386 if len(k) > 40:
e8c408a @ownport preparation for merge kvlite/sqlite branch with master
authored Sep 12, 2012
387 raise RuntimeError('The length of key is more than 40 bytes')
388
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored Jun 27, 2012
389 SQL_DELETE = '''DELETE FROM %s WHERE k = ''' % self.__collection
e8c408a @ownport preparation for merge kvlite/sqlite branch with master
authored Sep 12, 2012
390 self.__cursor.execute(SQL_DELETE + "%s;", binascii.a2b_hex(k))
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored Jun 27, 2012
391
392 def keys(self):
393 ''' return document keys in collection'''
394 rowid = 0
395 while True:
396 SQL_SELECT_MANY = 'SELECT __rowid__, k FROM %s WHERE __rowid__ > %d LIMIT 1000 ;' % (self.__collection, rowid)
397 self.__cursor.execute(SQL_SELECT_MANY)
398 result = self.__cursor.fetchall()
399 if not result:
400 break
401 for r in result:
402 rowid = r[0]
403 k = binascii.b2a_hex(r[1])
404 yield k
405
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
406 @property
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored Jun 27, 2012
407 def count(self):
408 ''' return amount of documents in collection'''
409 self.__cursor.execute('SELECT count(*) FROM %s;' % self.__collection)
410 return int(self.__cursor.fetchone()[0])
411
412 def commit(self):
413 self.__conn.commit()
414
415 def close(self):
416 ''' close connection to database '''
417 self.__conn.close()
418
fc6d7df @ownport update scheme and code
authored Sep 10, 2012
419 # -----------------------------------------------------------------
420 # SqliteCollectionManager class
421 # -----------------------------------------------------------------
422 class SqliteCollectionManager(object):
423 ''' Sqlite Collection Manager '''
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
424 def __init__(self, uri):
425
426 params = self.parse_uri(uri)
427
428 self.__conn = sqlite3.connect(params['db'])
429 self.__cursor = self.__conn.cursor()
e8c408a @ownport preparation for merge kvlite/sqlite branch with master
authored Sep 12, 2012
430 self.__conn.text_factory = str
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
431
432 @staticmethod
433 def parse_uri(uri):
434 '''parse URI
435
436 return driver, database, collection
437 '''
438 parsed_uri = dict()
439 parsed_uri['backend'], rest_uri = uri.split('://', 1)
440 if ':' in rest_uri:
441 parsed_uri['db'], parsed_uri['collection'] = rest_uri.split(':',1)
442 else:
443 parsed_uri['db'] = rest_uri
444 parsed_uri['collection'] = None
445 if parsed_uri['db'] == 'memory':
446 parsed_uri['db'] = ':memory:'
447 return parsed_uri
448
449 @property
450 def connection(self):
451 ''' return connection '''
452 return self.__conn
453
454 @property
455 def collection_class(self):
456 ''' return SqliteCollection object'''
457 return SqliteCollection
458
459 def collections(self):
460 ''' return collection list'''
461
462 self.__cursor.execute('SELECT name FROM sqlite_master WHERE type="table";')
463 return [t[0] for t in self.__cursor.fetchall()]
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored Jun 27, 2012
464
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
465 def create(self, name):
466 ''' create collection '''
467
468 SQL_CREATE_TABLE = '''CREATE TABLE IF NOT EXISTS %s (
469 k NOT NULL, v, UNIQUE (k) );'''
470
471 self.__cursor.execute(SQL_CREATE_TABLE % name)
472 self.__conn.commit()
473
474 def remove(self, name):
475 ''' remove collection '''
476 if name in self.collections():
477 self.__cursor.execute('DROP TABLE %s;' % name)
478 self.__conn.commit()
479 else:
480 raise RuntimeError('No collection with name: {}'.format(name))
481
482 def close(self):
483 ''' close connection to database '''
484 self.__conn.close()
485
22f983f @ownport migration from old scheme to new
authored Sep 7, 2012
486 # -----------------------------------------------------------------
487 # SqliteCollection class
488 # -----------------------------------------------------------------
e9990ae @ownport move old code to MysqlCollection
authored Sep 7, 2012
489 class SqliteCollection(object):
22f983f @ownport migration from old scheme to new
authored Sep 7, 2012
490 ''' Sqlite Collection'''
491
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
492 def __init__(self, connection, collection_name, serializer=cPickleSerializer):
e9990ae @ownport move old code to MysqlCollection
authored Sep 7, 2012
493
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
494 self.__conn = connection
495 self.__cursor = self.__conn.cursor()
496 self.__collection = collection_name
497 self.__serializer = serializer
e9990ae @ownport move old code to MysqlCollection
authored Sep 7, 2012
498
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
499 self.__uuid_cache = list()
e9990ae @ownport move old code to MysqlCollection
authored Sep 7, 2012
500
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
501 def get_uuid(self):
502 """ return id based on uuid """
e9990ae @ownport move old code to MysqlCollection
authored Sep 7, 2012
503
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
504 if not self.__uuid_cache:
505 for uuid in get_uuid():
506 self.__uuid_cache.append(uuid)
507 return self.__uuid_cache.pop()
508
509 def put(self, k, v):
510 ''' put document in collection '''
e9990ae @ownport move old code to MysqlCollection
authored Sep 7, 2012
511
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
512 # TODO many k/v in put()
513
514 if len(k) > 40:
515 raise RuntimeError('The length of key is more than 40 bytes')
516 SQL_INSERT = 'INSERT OR REPLACE INTO %s (k,v) ' % self.__collection
517 SQL_INSERT += 'VALUES (?,?)'
518 v = self.__serializer.dumps(v)
e8c408a @ownport preparation for merge kvlite/sqlite branch with master
authored Sep 12, 2012
519 self.__cursor.execute(SQL_INSERT, (k, v))
e9990ae @ownport move old code to MysqlCollection
authored Sep 7, 2012
520
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
521 def __get_many(self):
522 ''' return all docs '''
523 rowid = 0
524 while True:
525 SQL_SELECT_MANY = 'SELECT rowid, k,v FROM %s WHERE rowid > %d LIMIT 1000 ;' % (self.__collection, rowid)
526 self.__cursor.execute(SQL_SELECT_MANY)
527 result = self.__cursor.fetchall()
528 if not result:
529 break
530 for r in result:
531 rowid = r[0]
532 k = r[1]
533 try:
534 v = self.__serializer.loads(r[2])
535 except Exception, err:
536 raise RuntimeError('key %s, %s' % (k, err))
537 yield (k, v)
538
539 def get(self, k=None):
540 '''
541 return document by key from collection
542 return documents if key is not defined
e9990ae @ownport move old code to MysqlCollection
authored Sep 7, 2012
543 '''
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
544 if k:
545 if len(k) > 40:
546 raise RuntimeError('The key length is more than 40 bytes')
547 SQL = 'SELECT k,v FROM %s WHERE k = ?;' % self.__collection
548 try:
549 self.__cursor.execute(SQL, (k,))
e8c408a @ownport preparation for merge kvlite/sqlite branch with master
authored Sep 12, 2012
550 except Exception, err:
551 raise RuntimeError(err)
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
552 result = self.__cursor.fetchone()
553 if result:
554 try:
555 v = self.__serializer.loads(result[1])
556 except Exception, err:
557 raise RuntimeError('key %s, %s' % (k, err))
558 return (result[0], v)
559 else:
560 return (None, None)
561 else:
562 return self.__get_many()
563
564 def keys(self):
565 ''' return document keys in collection'''
566 rowid = 0
567 while True:
568 SQL_SELECT_MANY = 'SELECT rowid, k FROM %s WHERE rowid > %d LIMIT 1000 ;' % (self.__collection, rowid)
569 self.__cursor.execute(SQL_SELECT_MANY)
570 result = self.__cursor.fetchall()
571 if not result:
572 break
573 for r in result:
574 rowid = r[0]
575 yield r[1]
e9990ae @ownport move old code to MysqlCollection
authored Sep 7, 2012
576
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
577 def delete(self, k):
578 ''' delete document by k '''
579 if len(k) > 40:
580 raise RuntimeError('The key length is more than 40 bytes')
581 SQL_DELETE = '''DELETE FROM %s WHERE k = ?;''' % self.__collection
e8c408a @ownport preparation for merge kvlite/sqlite branch with master
authored Sep 12, 2012
582 self.__cursor.execute(SQL_DELETE, (k,))
212f56d @ownport added sqlite3 support
authored Sep 10, 2012
583
584 @property
585 def count(self):
586 ''' return amount of documents in collection'''
587 self.__cursor.execute('SELECT count(*) FROM %s;' % self.__collection)
588 return int(self.__cursor.fetchone()[0])
589
590 def commit(self):
591 self.__conn.commit()
592
593 def close(self):
594 ''' close connection to database '''
595 self.__conn.close()
67bd9d4 @ownport Migration from http://code.google.com/p/kvlite/
authored Jun 27, 2012
596
597
Something went wrong with that request. Please try again.