-
Notifications
You must be signed in to change notification settings - Fork 3
/
README.txt
363 lines (303 loc) · 10.2 KB
/
README.txt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
Currently requires:
- Python 2.7
- CouchDB 1.0.1+ (untested on lower, 0.11 will probably work)
- http://pypi.python.org/pypi/CouchDB 0.8+ (untested on lower, 0.7 will probably work)
Source: http://github.com/wickedgrey/couchable
Package: http://pypi.python.org/pypi/couchable
API docs: http://packages.python.org/couchable
Blog: http://blog.nopinch.net/tag/couchable/
Example of Use
===============
import numpy
import couchdb
import couchable
couchable.registerAttachmentType(numpy.ndarray,
lambda obj: obj.dumps(),
lambda data: numpy.loads(data),
'application/octet-stream', gzip=True)
class BaseClass(object):
def __init__(self, shape, type_=numpy.uint16):
self.vol = numpy.zeros(shape, type_)
# ...
class ClassA(object):
def __init__(self, name):
self.name = name
# ...
couchable.registerDocType(ClassA,
lambda obj, cdb: couchable.newid(obj, lambda x: x.name),
lambda obj, cdb: None)
class SubClassB(BaseClass):
def __init__(self, container):
BaseClass.__init__(self, container.vol.shape, numpy.float32, container.pixelLength_mm)
self.container = container
# ...
couchable.registerDocType(SubClassB)
class Container(BaseClass):
def __init__(self):
BaseClass.__init__(self, (300, 400, 200), numpy.bool_)
# ...
couchable.registerDocType(Container)
def main(options, arguments):
cdb = couchable.CouchableDb(options.cdb_name)
view = couchdb.design.ViewDefinition('generic', 'byclass',
'''
function(doc) {
if ('couchable:' in doc) {
info = doc['couchable:'];
emit([info.module, info.class, doc._id], doc);
}
}''')
view.sync(cdb.db)
# ...
viewResult = view(cdb.db, include_docs=True, startkey=['example', 'Container'], endkey=['example', 'Container', {}])
container_list = cdb.load(viewResult.rows)
# ...
Examples of the JSON Structure of Stored Objects
================================================
All of the object dumps are taken from Futon.
>>> import couchable
>>> cdb=couchable.CouchableDb('example')
>>> class SimpleDoc(couchable.CouchableDoc):
... def __init__(self, **kwargs):
... for name, value in kwargs.items():
... setattr(self, name, value)
...
>>> a = SimpleDoc(name='AAA')
>>> cdb.store(a)
'main__.SimpleDoc:2a208810-467f-4feb-a5bb-98d0beb1e5e7'
{
"_id": "main__.SimpleDoc:2a208810-467f-4feb-a5bb-98d0beb1e5e7",
"_rev": "1-315ed02172dddb449a4ab38e54b8bb85",
"couchable:": {
"class": "SimpleDoc",
"module": "__main__"
},
"name": "AAA"
}
The key things to note are:
- The automatically generated id includes some hints about the type of the
object. It is only made if the object doesn't have an ID already.
- The object metadata is stored in "couchable:".
- Normal field names are stored at the top level of the object.
>>> a.int_ = 1234
>>> a.long_ = 1234567890L
>>> a.str_ = 'some str'
>>> cdb.store(a)
'main__.SimpleDoc:2a208810-467f-4feb-a5bb-98d0beb1e5e7'
{
"_id": "main__.SimpleDoc:2a208810-467f-4feb-a5bb-98d0beb1e5e7",
"_rev": "2-618b01362f076539f1abb481585b2f64",
"couchable:": {
"class": "SimpleDoc",
"module": "__main__"
},
"long_": 1234567890,
"name": "AAA",
"str_": "some str",
"int_": 1234
}
Numbers, strings and lists are stored in native JSON. Dicts with string keys
are as well, though non-string keys are disallowed by JSON, and so require
special handling (see below).
>>> del a.int_
>>> del a.long_
>>> del a.str_
>>> a.tuple_ = (1, 'two', 3.3)
>>> cdb.store(a)
'main__.SimpleDoc:2a208810-467f-4feb-a5bb-98d0beb1e5e7'
{
"_id": "main__.SimpleDoc:2a208810-467f-4feb-a5bb-98d0beb1e5e7",
"_rev": "3-a24d5422be27e8dca0f4b4375718bceb",
"name": "AAA",
"couchable:": {
"class": "SimpleDoc",
"module": "__main__"
},
"tuple_": {
"couchable:": {
"args": [
[
1,
"two",
3.3
]
],
"class": "tuple",
"module": "__builtin__",
"kwargs": {
}
}
}
}
Tuples do not have a native JSON representation, so they get treated much like
arbitrary objects do (note: since tuples don't have a __dict__, they need
special case support). Here, the "args" and "kwargs" will be passed into the
constructor of the type when it's time to load this object.
>>> del a.tuple_
>>> a._implementationDetail = 'this is private'
>>> cdb.store(a)
'main__.SimpleDoc:2a208810-467f-4feb-a5bb-98d0beb1e5e7'
{
"_id": "main__.SimpleDoc:2a208810-467f-4feb-a5bb-98d0beb1e5e7",
"_rev": "5-5ac8698c7b9567642484e728e60b95b1",
"couchable:": {
"private": {
"_implementationDetail": "this is private"
},
"class": "SimpleDoc",
"module": "__main__"
},
"name": "AAA"
}
CouchDB reserves all top-level fields that start with an underscore. Since
python makes liberal use of underscores to denote "private" or implementation
detail fields, those end up inside of a second-level dictionary inside of the
"couchable:" dict. During loading, these will end up back inside the object
__dict__.
>>> del a._implementationDetail
>>> a.reserved = 'couchable: reserving the string "couchable:" since 2010'
>>> cdb.store(a)
'main__.SimpleDoc:2a208810-467f-4feb-a5bb-98d0beb1e5e7'
{
"_id": "main__.SimpleDoc:2a208810-467f-4feb-a5bb-98d0beb1e5e7",
"_rev": "6-a7ef4d2ba59a68128b076cf5d5d0aaee",
"couchable:": {
"class": "SimpleDoc",
"module": "__main__"
},
"reserved": "couchable:append:str:couchable: reserving the string \"couchable:\" since 2010",
"name": "AAA"
}
Any string that starts with "couchable:" is escaped, since couchable makes
heavy use of that prefix. See below.
>>> del a.reserved
>>> a.dict_ = {'foo':'FOO', 123:'bar', (45, 67):'baz'}
>>> cdb.store(a)
'main__.SimpleDoc:2a208810-467f-4feb-a5bb-98d0beb1e5e7'
{
"_id": "main__.SimpleDoc:2a208810-467f-4feb-a5bb-98d0beb1e5e7",
"_rev": "7-4706c617a5f8900956c76ecb6f2e2daa",
"couchable:": {
"keys": {
"couchable:key:tuple:(45, 67)": {
"couchable:": {
"args": [[45, 67]],
"class": "tuple",
"module": "__builtin__",
"kwargs": {
}
}
}
},
"class": "SimpleDoc",
"module": "__main__"
},
"dict_": {
"couchable:key:tuple:(45, 67)": "baz",
"foo": "FOO",
"couchable:repr:int:123": "bar"
},
"name": "AAA"
}
As we saw above, tuples are supported using similar structures to those used
by arbitrary objects. However, JSON only supports strings as dictionary keys.
We solve this by replacing the object-as-key with a string that acts as a
pointer to the actual object, which is stored inside doc['couchable:']['keys']
as the full object.
Similarly, ints cannot be JSON dictonary keys, but we can use the int repr to
fully represent the object, and so we do so in-place (note that this wouldn't
work for tuples because tuples can contain arbitrarily complex objects, not
just the ints that we have in the example).
>>> del a.dict_
>>> a.nested = {(1,1):{(2,2):{(3,3):'four'}}}
>>> cdb.store(a)
'main__.SimpleDoc:2a208810-467f-4feb-a5bb-98d0beb1e5e7'
{
"_id": "main__.SimpleDoc:2a208810-467f-4feb-a5bb-98d0beb1e5e7",
"_rev": "10-1fe944276e27bef87259857be9d2abd3",
"nested": {
"couchable:key:tuple:(1, 1)": {
"couchable:key:tuple:(2, 2)": {
"couchable:key:tuple:(3, 3)": "four"
}
}
},
"couchable:": {
"keys": {
"couchable:key:tuple:(1, 1)": {
"couchable:": {
"args": [[1, 1]],
"class": "tuple",
"module": "__builtin__",
"kwargs": {
}
}
},
"couchable:key:tuple:(2, 2)": {
"couchable:": {
"args": [[2, 2]],
"class": "tuple",
"module": "__builtin__",
"kwargs": {
}
}
},
"couchable:key:tuple:(3, 3)": {
"couchable:": {
"args": [[3, 3]],
"class": "tuple",
"module": "__builtin__",
"kwargs": {
}
}
}
},
"class": "SimpleDoc",
"module": "__main__"
},
"name": "AAA"
}
Here we have nested dictionaries with tuple keys, and dict values. All of the
tuples are replaced with string pointers, and fully saved in 'keys'. Note
that 'keys' is a *document* level construct, not an object level one. 'keys'
will never appear outside of doc['couchable:'] (at least not outside of naming
coincidences).
>>> del a.nested
>>> b=SimpleDoc(name='BBB')
>>> a.bbb = b
>>> cdb.store(a)
'main__.SimpleDoc:2a208810-467f-4feb-a5bb-98d0beb1e5e7'
{
"_id": "main__.SimpleDoc:2a208810-467f-4feb-a5bb-98d0beb1e5e7",
"_rev": "11-5eccb48b99431431c9f91b5e84a4ed7b",
"bbb": "couchable:id:main__.SimpleDoc:544a1408-d3f3-41c4-a428-9f0d2d8e2372",
"couchable:": {
"class": "SimpleDoc",
"module": "__main__"
},
"name": "AAA"
}
{
"_id": "main__.SimpleDoc:544a1408-d3f3-41c4-a428-9f0d2d8e2372",
"_rev": "1-577fefe455995539e11c992b7b46e10a",
"couchable:": {
"class": "SimpleDoc",
"module": "__main__"
},
"name": "BBB"
}
Here, we show the behavior when storing multiple objects, each of which is
flagged as needing to be a full document. Similar to the other string
pointers encountered already, couchable:id points to an object stored in an
entirely different document.
Known Limitations
=================
In general, couchable doesn't play well with classes that override __new__ in
odd ways.
Couchable cannot (and will probably never) store instances of the following
types:
- Tuple subclasses that override __new__ that *don't* do so in a way that is
compatible with collections.namedtuple.
- Programmatically defined classes that are not importable (they're basically
impossible to reconstruct during loading).
This list may not be exhaustive; unknown limitations may exist.