Skip to content
This repository
Browse code

Remove kvstore and move everything to park 0.9.1

  • Loading branch information...
commit ac5e9ccb05079f71bf9f9afa1aceccba39f689e8 1 parent b095d8d
Peter Teichman authored September 05, 2012
4  cobe/brain.py
@@ -3,10 +3,10 @@
3 3
 import itertools
4 4
 import logging
5 5
 import math
  6
+import park
6 7
 
7 8
 from cobe.analysis import (
8 9
     AccentNormalizer, StemNormalizer, WhitespaceAnalyzer)
9  
-from cobe.kvstore import SqliteStore
10 10
 from cobe.model import Model
11 11
 from cobe.search import RandomWalkSearcher
12 12
 from cobe.utils import itime
@@ -42,7 +42,7 @@ class Brain(object):
42 42
     def __init__(self, filename):
43 43
         self.analyzer = StandardAnalyzer()
44 44
 
45  
-        store = SqliteStore(filename)
  45
+        store = park.SQLiteStore(filename)
46 46
 
47 47
         self.model = Model(self.analyzer, store)
48 48
         self.searcher = RandomWalkSearcher(self.model)
6  cobe/commands.py
@@ -4,6 +4,7 @@
4 4
 import fileinput
5 5
 import logging
6 6
 import os
  7
+import park
7 8
 import re
8 9
 import readline
9 10
 import sys
@@ -11,7 +12,6 @@
11 12
 from . import analysis
12 13
 
13 14
 from .brain import Brain
14  
-from .kvstore import SqliteStore
15 15
 from .model import Model
16 16
 from .varint import decode, decode_one, encode_one
17 17
 
@@ -26,7 +26,7 @@ def add_subparser(cls, parser):
26 26
 
27 27
     @staticmethod
28 28
     def run(args):
29  
-        store = SqliteStore("cobe.store")
  29
+        store = park.SQLiteStore("cobe.store")
30 30
         analyzer = analysis.WhitespaceAnalyzer()
31 31
         model = Model(analyzer, store)
32 32
 
@@ -40,7 +40,7 @@ def run(args):
40 40
 
41 41
         print "3-gram counts:"
42 42
         get_token = model.tokens.get_token
43  
-        for ngram, count in model._prefix_items("3", skip_prefix=True):
  43
+        for ngram, count in model._prefix_items("3", strip_prefix=True):
44 44
             # This needs a more efficient way to get the token ids,
45 45
             # maybe a simple varint-aware string split.
46 46
             grams = [get_token(encode_one(i)) for i in decode(ngram)]
312  cobe/kvstore.py
... ...
@@ -1,312 +0,0 @@
1  
-# Copyright (C) 2012 Peter Teichman
2  
-
3  
-import abc
4  
-import logging
5  
-import os
6  
-import sqlite3
7  
-
8  
-from .utils import ibatch
9  
-
10  
-logger = logging.getLogger(__name__)
11  
-
12  
-
13  
-class KVStore(object):
14  
-    """An abstract key-value interface with support for range iteration."""
15  
-    __metaclass__ = abc.ABCMeta
16  
-
17  
-    @abc.abstractmethod
18  
-    def get(self, key, default=None):  # pragma: no cover
19  
-        """Get the value associated with a key.
20  
-
21  
-        Returns:
22  
-            The specified default if the key is not present in the
23  
-            store. This method does not raise KeyError for missing
24  
-            keys.
25  
-
26  
-        """
27  
-        pass
28  
-
29  
-    @abc.abstractmethod
30  
-    def put(self, key, value):  # pragma: no cover
31  
-        """Set the value associated with a key.
32  
-
33  
-        Both the key and value can be binary values.
34  
-
35  
-        """
36  
-        pass
37  
-
38  
-    @abc.abstractmethod
39  
-    def put_many(self, items):  # pragma: no cover
40  
-        """Set many key-value pairs.
41  
-
42  
-        This method should take advantage of performance or atomicity
43  
-        features of the underlying store.
44  
-
45  
-        Args:
46  
-            items: An iterator producing (key, value) tuples.
47  
-
48  
-        """
49  
-        pass
50  
-
51  
-    @abc.abstractmethod
52  
-    def keys(self, key_from=None, key_to=None):  # pragma: no cover
53  
-        """Get a lexically sorted range of keys.
54  
-
55  
-        Args:
56  
-            key_from: Lower bound (inclusive), or None for unbounded.
57  
-            key_to: Upper bound (inclusive), or None for unbounded.
58  
-
59  
-        Returns:
60  
-            An iterator yielding all keys from the store where
61  
-            key_from <= key <= key_to.
62  
-
63  
-        """
64  
-        pass
65  
-
66  
-    @abc.abstractmethod
67  
-    def items(self, key_from=None, key_to=None):  # pragma: no cover
68  
-        """Get a lexically sorted range of (key, value) tuples.
69  
-
70  
-        Args:
71  
-            key_from: Lower bound (inclusive), or None for unbounded.
72  
-            key_to: Upper bound (inclusive), or None for unbounded.
73  
-
74  
-        Returns:
75  
-            An iterator yielding all (key, value) pairs from the store
76  
-            where key_from <= key <= key_to.
77  
-
78  
-        """
79  
-        pass
80  
-
81  
-    def prefix_items(self, prefix, skip_prefix=False):
82  
-        """yield all (key, value) pairs with keys that begin with $prefix.
83  
-
84  
-        Args:
85  
-            prefix: lexical prefix for keys to search.
86  
-
87  
-            skip_prefix: a boolean to specify whether the prefix
88  
-                should be stripped from the generated keys.
89  
-
90  
-        Yields:
91  
-            All the (key, value) pairs in the store where the keys
92  
-                begin with the requested prefix.
93  
-
94  
-        """
95  
-        items = self.items(key_from=prefix)
96  
-
97  
-        start = 0
98  
-        if skip_prefix:
99  
-            start = len(prefix)
100  
-
101  
-        for key, value in items:
102  
-            if not key.startswith(prefix):
103  
-                break
104  
-            yield key[start:], value
105  
-
106  
-    def prefix_keys(self, prefix, skip_prefix=False):
107  
-        """yield all keys that begin with $prefix.
108  
-
109  
-        Args:
110  
-            prefix: lexical prefix for keys to search.
111  
-
112  
-            skip_prefix: a boolean to specify whether the prefix
113  
-                should be stripped from the generated keys.
114  
-
115  
-        Yields:
116  
-            All the keys in the store that begin with the requested prefix.
117  
-
118  
-        """
119  
-        keys = self.keys(key_from=prefix)
120  
-
121  
-        start = 0
122  
-        if skip_prefix:
123  
-            start = len(prefix)
124  
-
125  
-        for key in keys:
126  
-            if not key.startswith(prefix):
127  
-                break
128  
-            yield key[start:]
129  
-
130  
-
131  
-class BsddbStore(KVStore):
132  
-    def __init__(self, path):
133  
-        import bsddb
134  
-        self.bsddb = bsddb
135  
-        self.db = bsddb.btopen(path)
136  
-
137  
-    def get(self, key, default=None):
138  
-        if not key in self.db:
139  
-            return default
140  
-
141  
-        return self.db[key]
142  
-
143  
-    def put(self, key, value):
144  
-        self.db[key] = value
145  
-
146  
-    def put_many(self, items):
147  
-        for key, value in items:
148  
-            self.put(key, value)
149  
-
150  
-    def _iter(self, key_from=None, key_to=None):
151  
-        db = self.db
152  
-
153  
-        def is_done(key):
154  
-            return key_to is not None and key > key_to
155  
-
156  
-        # This is pretty messy, as bsddb throws different exceptions
157  
-        # for first() and set_location() on empty databases.
158  
-        try:
159  
-            if key_from is None:
160  
-                key, value = db.first()
161  
-            else:
162  
-                key, value = db.set_location(key_from)
163  
-        except (KeyError, self.bsddb.error):
164  
-            return
165  
-
166  
-        while not is_done(key):
167  
-            yield key, value
168  
-            try:
169  
-                key, value = db.next()
170  
-            except self.bsddb.error:
171  
-                return
172  
-
173  
-    def keys(self, key_from=None, key_to=None):
174  
-        for key, value in self._iter(key_from=key_from, key_to=key_to):
175  
-            yield key
176  
-
177  
-    def items(self, key_from=None, key_to=None):
178  
-        for key, value in self._iter(key_from=key_from, key_to=key_to):
179  
-            yield key, value
180  
-
181  
-
182  
-class SqliteStore(KVStore):
183  
-    def __init__(self, path):
184  
-        need_schema = not os.path.exists(path)
185  
-
186  
-        self.conn = sqlite3.connect(path)
187  
-
188  
-        # Don't create unicode objects for retrieved values
189  
-        self.conn.text_factory = buffer
190  
-
191  
-        # Disable the SQLite cache. Its pages tend to get swapped
192  
-        # out, even if the database file is in buffer cache.
193  
-        c = self.conn.cursor()
194  
-        c.execute("PRAGMA cache_size=0")
195  
-        c.execute("PRAGMA page_size=4096")
196  
-
197  
-        # Use write-ahead logging if it's available, otherwise truncate
198  
-        journal_mode, = c.execute("PRAGMA journal_mode=WAL").fetchone()
199  
-        if journal_mode != "wal":
200  
-            c.execute("PRAGMA journal_mode=truncate")
201  
-
202  
-        # Speed-for-reliability tradeoffs
203  
-        c.execute("PRAGMA temp_store=memory")
204  
-        c.execute("PRAGMA synchronous=OFF")
205  
-
206  
-        if need_schema:
207  
-            self._create_db(self.conn)
208  
-
209  
-    def _create_db(self, conn):
210  
-        logger.debug("Creating SqliteStore schema")
211  
-        c = conn.cursor()
212  
-
213  
-        c.execute("""
214  
-CREATE TABLE kv (
215  
-    key BLOB NOT NULL PRIMARY KEY,
216  
-    value BLOB NOT NULL)""")
217  
-
218  
-        conn.commit()
219  
-
220  
-    def get(self, key, default=None):
221  
-        q = "SELECT value FROM kv WHERE key = ?"
222  
-        c = self.conn.cursor()
223  
-
224  
-        row = c.execute(q, (sqlite3.Binary(key),)).fetchone()
225  
-        if not row:
226  
-            return default
227  
-
228  
-        return str(row[0])
229  
-
230  
-    def put(self, key, value):
231  
-        q = "INSERT OR REPLACE INTO kv (key, value) VALUES (?, ?)"
232  
-        self.conn.execute(q, (sqlite3.Binary(key), sqlite3.Binary(value)))
233  
-
234  
-    def put_many(self, items):
235  
-        q = "INSERT OR REPLACE INTO kv (key, value) VALUES (?, ?)"
236  
-        c = self.conn.cursor()
237  
-
238  
-        blob = sqlite3.Binary
239  
-        for batch in ibatch(items, 30000):
240  
-            items = ((blob(key), blob(value)) for key, value in batch)
241  
-
242  
-            c.executemany(q, items)
243  
-            self.conn.commit()
244  
-
245  
-    def _range_where(self, key_from=None, key_to=None):
246  
-        if key_from is not None and key_to is None:
247  
-            return "WHERE key >= :key_from"
248  
-
249  
-        if key_from is None and key_to is not None:
250  
-            return "WHERE key <= :key_to"
251  
-
252  
-        if key_from is not None and key_to is not None:
253  
-            return "WHERE key BETWEEN :key_from AND :key_to"
254  
-
255  
-        return ""
256  
-
257  
-    def items(self, key_from=None, key_to=None):
258  
-        q = "SELECT key, value FROM kv %s ORDER BY key " \
259  
-            % self._range_where(key_from, key_to)
260  
-
261  
-        if key_from is not None:
262  
-            key_from = sqlite3.Binary(key_from)
263  
-
264  
-        if key_to is not None:
265  
-            key_to = sqlite3.Binary(key_to)
266  
-
267  
-        c = self.conn.cursor()
268  
-        for key, value in c.execute(q, dict(key_from=key_from, key_to=key_to)):
269  
-            yield str(key), str(value)
270  
-
271  
-    def keys(self, key_from=None, key_to=None):
272  
-        q = "SELECT key FROM kv %s ORDER BY key " \
273  
-            % self._range_where(key_from, key_to)
274  
-
275  
-        if key_from is not None:
276  
-            key_from = sqlite3.Binary(key_from)
277  
-
278  
-        if key_to is not None:
279  
-            key_to = sqlite3.Binary(key_to)
280  
-
281  
-        c = self.conn.cursor()
282  
-        for key, in c.execute(q, dict(key_from=key_from, key_to=key_to)):
283  
-            yield str(key)
284  
-
285  
-
286  
-class LevelDBStore(KVStore):
287  
-    def __init__(self, path):
288  
-        import leveldb
289  
-        self.leveldb = leveldb
290  
-        self.kv = leveldb.LevelDB(path)
291  
-
292  
-    def get(self, key, default=None):
293  
-        return self.kv.Get(key, default=default)
294  
-
295  
-    def put(self, key, value):
296  
-        self.kv.Put(key, value)
297  
-
298  
-    def put_many(self, items):
299  
-        for batch in ibatch(items, 30000):
300  
-            db_batch = self.leveldb.WriteBatch()
301  
-
302  
-            for key, value in batch:
303  
-                db_batch.Put(key, value)
304  
-
305  
-            self.kv.Write(db_batch)
306  
-
307  
-    def items(self, key_from=None, key_to=None):
308  
-        return self.kv.RangeIter(key_from=key_from, key_to=key_to)
309  
-
310  
-    def keys(self, key_from=None, key_to=None):
311  
-        return self.kv.RangeIter(key_from=key_from, key_to=key_to,
312  
-                                 include_value=False)
12  cobe/model.py
@@ -126,7 +126,7 @@ def __init__(self, analyzer, store, n=3):
126 126
         # Leverage LevelDB's sorting to extract all tokens (the things
127 127
         # prefixed with the token key for an empty string)
128 128
         token_prefix = self._token_key("")
129  
-        all_tokens = self.store.prefix_items(token_prefix, skip_prefix=True)
  129
+        all_tokens = self.store.prefix_items(token_prefix, strip_prefix=True)
130 130
 
131 131
         self.tokens.load(all_tokens)
132 132
 
@@ -263,7 +263,7 @@ def choose_random_context(self, token, rng=random):
263 263
         token_id = self.tokens.get_id(token)
264 264
 
265 265
         prefix = self._tokens_count_key((token_id,), self.orders[0])
266  
-        items = list(self.store.prefix_keys(prefix, skip_prefix=True))
  266
+        items = list(self.store.prefix_keys(prefix, strip_prefix=True))
267 267
 
268 268
         if len(items):
269 269
             context = rng.choice(items)
@@ -280,7 +280,7 @@ def choose_random_word(self, context, rng=random):
280 280
         # with the key for token_ids
281 281
         key = self._tokens_count_key(token_ids, len(token_ids) + 1)
282 282
 
283  
-        items = list(self.store.prefix_keys(key, skip_prefix=True))
  283
+        items = list(self.store.prefix_keys(key, strip_prefix=True))
284 284
 
285 285
         if len(items):
286 286
             token_id = rng.choice(items)
@@ -358,7 +358,7 @@ def get_norm_tokens(self, prefix, norm):
358 358
         key = self._norm_key(prefix, norm)
359 359
         get_token = self.tokens.get_token
360 360
 
361  
-        for token_id in self.store.prefix_keys(key, skip_prefix=True):
  361
+        for token_id in self.store.prefix_keys(key, strip_prefix=True):
362 362
             yield get_token(token_id)
363 363
 
364 364
     def search_bfs(self, context, end, filter=None):
@@ -383,7 +383,7 @@ def search_bfs(self, context, end, filter=None):
383 383
             token_ids = path[-n:]
384 384
             key = self._tokens_count_key(token_ids, len(token_ids) + 1)
385 385
 
386  
-            next_tokens = list(self.store.prefix_keys(key, skip_prefix=True))
  386
+            next_tokens = list(self.store.prefix_keys(key, strip_prefix=True))
387 387
             if next_tokens and filter is not None:
388 388
                 next_tokens = filter(next_tokens)
389 389
 
@@ -410,7 +410,7 @@ def search_bfs_reverse(self, context, end, filter=None):
410 410
             token_ids = path[:n]
411 411
             key = self._tokens_reverse_key(token_ids)
412 412
 
413  
-            prev_tokens = list(self.store.prefix_keys(key, skip_prefix=True))
  413
+            prev_tokens = list(self.store.prefix_keys(key, strip_prefix=True))
414 414
             if prev_tokens and filter is not None:
415 415
                 prev_tokens = filter(prev_tokens)
416 416
 
8  cobe/utils.py
@@ -4,14 +4,6 @@
4 4
 import time
5 5
 
6 6
 
7  
-def ibatch(iterable, size):
8  
-    """Yield a series of batches from iterable, each size elements long."""
9  
-    source = iter(iterable)
10  
-    while True:
11  
-        batch = itertools.islice(source, size)
12  
-        yield itertools.chain([batch.next()], batch)
13  
-
14  
-
15 7
 def itime(iterable, seconds):
16 8
     """Yield items from iterable until a time duration has passed.
17 9
 
14  docs/index.rst
Source Rendered
@@ -33,20 +33,6 @@ Contents:
33 33
 .. autoclass:: NgramCounter
34 34
     :members:
35 35
 
36  
-.. automodule:: cobe.kvstore
37  
-
38  
-.. autoclass:: KVStore
39  
-    :members:
40  
-
41  
-.. autoclass:: BsddbStore
42  
-    :members:
43  
-
44  
-.. autoclass:: SqliteStore
45  
-    :members:
46  
-
47  
-.. autoclass:: LevelDBStore
48  
-    :members:
49  
-
50 36
 .. automodule:: cobe.model
51 37
 
52 38
 .. autoclass:: Model
1  setup.py
@@ -50,6 +50,7 @@ def run(self):
50 50
     install_requires = [
51 51
         "PyStemmer==1.2.0",
52 52
         "argparse==1.2.1",
  53
+        "park==0.9.1",
53 54
         "python-irclib==0.4.8"
54 55
         ],
55 56
 
4  tests/test_analysis.py
... ...
@@ -1,10 +1,10 @@
1 1
 # Copyright (C) 2012 Peter Teichman
2 2
 # coding=utf8
3 3
 
  4
+import park
4 5
 import unittest2 as unittest
5 6
 
6 7
 from cobe import analysis
7  
-from cobe import kvstore
8 8
 from cobe import model
9 9
 from cobe import search
10 10
 
@@ -130,7 +130,7 @@ def test_conflated_query(self):
130 130
         analyzer = analysis.WhitespaceAnalyzer()
131 131
         analyzer.add_token_normalizer(analysis.LowercaseNormalizer())
132 132
 
133  
-        m = model.Model(analyzer, kvstore.SqliteStore(":memory:"))
  133
+        m = model.Model(analyzer, park.SQLiteStore(":memory:"))
134 134
         m.train(u"This is a test")
135 135
         m.train(u"this is a test")
136 136
 
282  tests/test_kvstore.py
... ...
@@ -1,282 +0,0 @@
1  
-# Copyright (C) 2012 Peter Teichman
2  
-
3  
-import operator
4  
-import os
5  
-import shutil
6  
-import unittest2 as unittest
7  
-
8  
-import cobe.kvstore
9  
-
10  
-
11  
-class KVStoreBase(object):
12  
-    """Base tests for KV Stores"""
13  
-
14  
-    def test_get_default(self):
15  
-        self.assertIsNone(self.store.get("missing"))
16  
-
17  
-    def test_put_get(self):
18  
-        key, value = "test_key", "test_value"
19  
-
20  
-        self.assertIsNone(self.store.get(key, None))
21  
-        self.store.put(key, value)
22  
-        self.assertEqual(value, self.store.get(key))
23  
-
24  
-    def test_null_key(self):
25  
-        key, value = "\x00", "test_value"
26  
-
27  
-        self.assertIsNone(self.store.get(key, None))
28  
-        self.store.put(key, value)
29  
-
30  
-        self.assertEqual(value, self.store.get(key))
31  
-
32  
-        self.assertEqual([key], list(self.store.keys()))
33  
-        self.assertEqual([(key, value)], list(self.store.items()))
34  
-
35  
-    def test_null_value(self):
36  
-        key, value = "test_key", "\x00"
37  
-
38  
-        self.assertIsNone(self.store.get(key, None))
39  
-        self.store.put(key, value)
40  
-
41  
-        self.assertEqual(value, self.store.get(key))
42  
-
43  
-        self.assertEqual([key], list(self.store.keys()))
44  
-        self.assertEqual([(key, value)], list(self.store.items()))
45  
-
46  
-    def test_replace(self):
47  
-        key = "foo"
48  
-        self.assertIsNone(self.store.get(key, None))
49  
-
50  
-        self.store.put(key, "bar")
51  
-        self.assertEqual("bar", self.store.get(key))
52  
-
53  
-        self.store.put(key, "baz")
54  
-        self.assertEqual("baz", self.store.get(key))
55  
-
56  
-    def test_put_many(self):
57  
-        items = [
58  
-            ("one", "value1"),
59  
-            ("two", "value2"),
60  
-            ("three", "value3"),
61  
-            ("four", "value4"),
62  
-            ("five", "value5"),
63  
-            ("six", "value6"),
64  
-            ("seven", "value7"),
65  
-            ("eight", "value8"),
66  
-            ("nine", "value9")
67  
-            ]
68  
-
69  
-        self.store.put_many(items)
70  
-
71  
-        for key, value in items:
72  
-            self.assertEqual(value, self.store.get(key))
73  
-
74  
-    def test_no_keys(self):
75  
-        self.assertEqual([], list(self.store.keys()))
76  
-
77  
-        self.assertEqual([], list(self.store.keys(key_from="foo")))
78  
-        self.assertEqual([], list(self.store.keys(key_to="bar")))
79  
-
80  
-        self.assertEqual([], list(self.store.keys(key_from="foo",
81  
-                                                  key_to="bar")))
82  
-
83  
-    def test_no_items(self):
84  
-        self.assertEqual([], list(self.store.items()))
85  
-
86  
-        self.assertEqual([], list(self.store.items(key_from="foo")))
87  
-        self.assertEqual([], list(self.store.items(key_to="bar")))
88  
-
89  
-        self.assertEqual([], list(self.store.items(key_from="foo",
90  
-                                                   key_to="bar")))
91  
-
92  
-    def test_keys(self):
93  
-        items = [
94  
-            ("one", "value1"),
95  
-            ("two", "value2"),
96  
-            ("three", "value3"),
97  
-            ("four", "value4"),
98  
-            ("five", "value5"),
99  
-            ("six", "value6"),
100  
-            ("seven", "value7"),
101  
-            ("eight", "value8"),
102  
-            ("nine", "value9")
103  
-            ]
104  
-
105  
-        for key, value in items:
106  
-            self.store.put(key, value)
107  
-
108  
-        # Sorted order is: eight five four nine one seven six three two
109  
-        keys = list(self.store.keys())
110  
-        expected = "eight five four nine one seven six three two".split()
111  
-        self.assertEqual(expected, keys)
112  
-
113  
-        # Test key_from on keys that are present and missing in the db
114  
-        keys = list(self.store.keys(key_from="four"))
115  
-        expected = "four nine one seven six three two".split()
116  
-        self.assertEqual(expected, keys)
117  
-
118  
-        keys = list(self.store.keys(key_from="fo"))
119  
-        expected = "four nine one seven six three two".split()
120  
-        self.assertEqual(expected, keys)
121  
-
122  
-        # Test key_to
123  
-        keys = list(self.store.keys(key_to="six"))
124  
-        expected = "eight five four nine one seven six".split()
125  
-        self.assertEqual(expected, keys)
126  
-
127  
-        keys = list(self.store.keys(key_to="si"))
128  
-        expected = "eight five four nine one seven".split()
129  
-        self.assertEqual(expected, keys)
130  
-
131  
-        # And test them both together
132  
-        keys = list(self.store.keys(key_from="five", key_to="three"))
133  
-        expected = "five four nine one seven six three".split()
134  
-        self.assertEqual(expected, keys)
135  
-
136  
-    def test_prefix_keys(self):
137  
-        # Fake some interesting keys and values to make sure the
138  
-        # prefix iterators are working
139  
-        store = self.store
140  
-
141  
-        store.put("a/", "a")
142  
-        store.put("a/b", "b")
143  
-        store.put("a/c", "c")
144  
-        store.put("a/d", "d")
145  
-        store.put("a/e", "e")
146  
-        store.put("a/f", "f")
147  
-        store.put("b/", "b")
148  
-        store.put("c/", "c")
149  
-        store.put("d/", "d")
150  
-
151  
-        a_list = list(store.prefix_keys("a/"))
152  
-        self.assertEqual("a/ a/b a/c a/d a/e a/f".split(), a_list)
153  
-
154  
-        a_list = list(store.prefix_keys("a/", skip_prefix=True))
155  
-        self.assertEqual(["", "b", "c", "d", "e", "f"], a_list)
156  
-
157  
-        self.assertEqual(["b/"], list(store.prefix_keys("b/")))
158  
-        self.assertEqual(["c/"], list(store.prefix_keys("c/")))
159  
-        self.assertEqual(["d/"], list(store.prefix_keys("d/")))
160  
-
161  
-    def test_items(self):
162  
-        put_items = dict([
163  
-            ("one", "value1"),
164  
-            ("two", "value2"),
165  
-            ("three", "value3"),
166  
-            ("four", "value4"),
167  
-            ("five", "value5"),
168  
-            ("six", "value6"),
169  
-            ("seven", "value7"),
170  
-            ("eight", "value8"),
171  
-            ("nine", "value9")
172  
-            ])
173  
-
174  
-        for key, value in put_items.items():
175  
-            self.store.put(key, value)
176  
-
177  
-        # Sorted order is: eight five four nine one seven six three two
178  
-        keys = list(self.store.items())
179  
-        expected = sorted(put_items.items(), key=operator.itemgetter(0))
180  
-        self.assertEqual(expected, keys)
181  
-
182  
-        # Test key_from on keys that are present and missing in the db
183  
-        keys = list(self.store.items(key_from="four"))
184  
-        self.assertEqual(expected[2:], keys)
185  
-
186  
-        keys = list(self.store.items(key_from="fo"))
187  
-        self.assertEqual(expected[2:], keys)
188  
-
189  
-        # Test key_to
190  
-        keys = list(self.store.items(key_to="six"))
191  
-        self.assertEqual(expected[:7], keys)
192  
-
193  
-        keys = list(self.store.items(key_to="si"))
194  
-        self.assertEqual(expected[:6], keys)
195  
-
196  
-        # And test them both together
197  
-        keys = list(self.store.items(key_from="five", key_to="three"))
198  
-        self.assertEqual(expected[1:8], keys)
199  
-
200  
-    def test_prefix_items(self):
201  
-        # Fake some interesting keys and values to make sure the
202  
-        # prefix iterators are working
203  
-        store = self.store
204  
-
205  
-        store.put("a/", "a")
206  
-        store.put("a/b", "b")
207  
-        store.put("a/c", "c")
208  
-        store.put("a/d", "d")
209  
-        store.put("a/e", "e")
210  
-        store.put("a/f", "f")
211  
-        store.put("b/", "b")
212  
-        store.put("c/", "c")
213  
-        store.put("d/", "d")
214  
-
215  
-        expected = [("a/", "a"),
216  
-                    ("a/b", "b"),
217  
-                    ("a/c", "c"),
218  
-                    ("a/d", "d"),
219  
-                    ("a/e", "e"),
220  
-                    ("a/f", "f")]
221  
-
222  
-        a_list = list(store.prefix_items("a/"))
223  
-        self.assertEqual(expected, a_list)
224  
-
225  
-        expected = [("", "a"),
226  
-                    ("b", "b"),
227  
-                    ("c", "c"),
228  
-                    ("d", "d"),
229  
-                    ("e", "e"),
230  
-                    ("f", "f")]
231  
-
232  
-        a_list = list(store.prefix_items("a/", skip_prefix=True))
233  
-        self.assertEqual(expected, a_list)
234  
-
235  
-
236  
-class TestBsddbStore(unittest.TestCase, KVStoreBase):
237  
-    DB = "tests.test_bsddb_store"
238  
-
239  
-    def setUp(self):
240  
-        try:
241  
-            import bsddb
242  
-            bsddb  # reference bsddb to satisfy pyflakes
243  
-        except ImportError:  # pragma: no cover
244  
-            raise unittest.SkipTest("bsddb not installed")
245  
-
246  
-        self.store = cobe.kvstore.BsddbStore(self.DB)
247  
-
248  
-        def cleanup():
249  
-            self.store.db.close()
250  
-            if os.path.exists(self.DB):
251  
-                os.unlink(self.DB)
252  
-
253  
-        self.addCleanup(cleanup)
254  
-
255  
-
256  
-class TestSqliteStore(unittest.TestCase, KVStoreBase):
257  
-    DB = "tests.test_sqlite_store"
258  
-
259  
-    def setUp(self):
260  
-        self.store = cobe.kvstore.SqliteStore(self.DB)
261  
-
262  
-        def cleanup():
263  
-            if os.path.exists(self.DB):
264  
-                os.unlink(self.DB)
265  
-
266  
-        self.addCleanup(cleanup)
267  
-
268  
-
269  
-class TestLevelDBStore(unittest.TestCase, KVStoreBase):
270  
-    DBDIR = "tests.test_leveldb_store"
271  
-
272  
-    def setUp(self):
273  
-        try:
274  
-            self.store = cobe.kvstore.LevelDBStore(self.DBDIR)
275  
-        except ImportError:  # pragma: no cover
276  
-            raise unittest.SkipTest("py-leveldb not installed")
277  
-
278  
-        def cleanup():
279  
-            if os.path.exists(self.DBDIR):
280  
-                shutil.rmtree(self.DBDIR)
281  
-
282  
-        self.addCleanup(cleanup)
4  tests/test_model.py
... ...
@@ -1,11 +1,11 @@
1 1
 # Copyright (C) 2012 Peter Teichman
2 2
 
  3
+import park
3 4
 import random
4 5
 import unittest2 as unittest
5 6
 
6 7
 from cobe.analysis import LowercaseNormalizer, WhitespaceAnalyzer
7 8
 from cobe.model import Model, TokenRegistry
8  
-from cobe.kvstore import SqliteStore
9 9
 
10 10
 
11 11
 class TestTokenRegistry(unittest.TestCase):
@@ -33,7 +33,7 @@ def test_non_unicode(self):
33 33
 class TestModel(unittest.TestCase):
34 34
     def setUp(self):
35 35
         self.analyzer = WhitespaceAnalyzer()
36  
-        self.store = SqliteStore(":memory:")
  36
+        self.store = park.SQLiteStore(":memory:")
37 37
         self.model = Model(self.analyzer, self.store)
38 38
 
39 39
     def test_init(self):
4  tests/test_search.py
... ...
@@ -1,9 +1,9 @@
1 1
 # Copyright (C) 2012 Peter Teichman
2 2
 
  3
+import park
3 4
 import unittest2 as unittest
4 5
 
5 6
 from cobe import analysis
6  
-from cobe import kvstore
7 7
 from cobe import model
8 8
 from cobe import search
9 9
 
@@ -23,7 +23,7 @@ def test_terms(self):
23 23
 class RandomWalkSearcherTest(unittest.TestCase):
24 24
     def setUp(self):
25 25
         self.analyzer = analysis.WhitespaceAnalyzer()
26  
-        self.store = kvstore.SqliteStore(":memory:")
  26
+        self.store = park.SQLiteStore(":memory:")
27 27
         self.model = model.Model(self.analyzer, self.store)
28 28
 
29 29
     def test_list_strip(self):

0 notes on commit ac5e9cc

Please sign in to comment.
Something went wrong with that request. Please try again.