Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Getting close to the first beta release. Fixed a problem with private…

… keys. Beefed up the README.
  • Loading branch information...
commit 589726e6e49858576f43872d8e7372b0aaa35f93 1 parent 4764b3d
@elistevens authored
View
274 README.txt
@@ -3,22 +3,276 @@ Currently requires:
- CouchDB 0.11+ (untested on lower)
- http://pypi.python.org/pypi/CouchDB 0.7+ (untested on lower)
+Examples of the JSON Structure of Stored Objects
+================================================
+
+All of the object dumps are taken from Futon.
+
>>> import couchable
->>> cdb=couchable.CouchableDb('testing')
+>>> cdb=couchable.CouchableDb('example')
>>> class SimpleDoc(couchable.CouchableDoc):
... def __init__(self, **kwargs):
... for name, value in kwargs.items():
... setattr(self, name, value)
...
>>> a = SimpleDoc(name='AAA')
->>> b = SimpleDoc(name='BBB', a=a)
->>> c = SimpleDoc(name='CCC', a=a)
->>> id_list = cdb.store([b, c])
->>> id_list
-['main__.SimpleDoc:...', 'main__.SimpleDoc:...']
->>> b, c = cdb.load(id_list)
->>> assert b.a is c.a
->>> cdb.db[b._id]
-<Document 'main__.SimpleDoc:...'@'...' {'a': 'couchable:id:main__.SimpleDoc:...', 'couchable:': {'class': 'SimpleDoc', 'module': '__main__'}, 'name': 'BBB'}>
+>>> cdb.store(a)
+'main__.SimpleDoc:2a208810-467f-4feb-a5bb-98d0beb1e5e7'
+
+{
+ "_id": "main__.SimpleDoc:2a208810-467f-4feb-a5bb-98d0beb1e5e7",
+ "_rev": "1-315ed02172dddb449a4ab38e54b8bb85",
+ "couchable:": {
+ "class": "SimpleDoc",
+ "module": "__main__"
+ },
+ "name": "AAA"
+}
+
+The key things to note are:
+- The automatically generated id includes some hints about the type of the
+ object. It is only made if the object doesn't have an ID already.
+- The object metadata is stored in "couchable:".
+- Normal field names are stored at the top level of the object.
+
+
+>>> a.int_ = 1234
+>>> a.long_ = 1234567890L
+>>> a.str_ = 'some str'
+>>> cdb.store(a)
+'main__.SimpleDoc:2a208810-467f-4feb-a5bb-98d0beb1e5e7'
+
+{
+ "_id": "main__.SimpleDoc:2a208810-467f-4feb-a5bb-98d0beb1e5e7",
+ "_rev": "2-618b01362f076539f1abb481585b2f64",
+ "couchable:": {
+ "class": "SimpleDoc",
+ "module": "__main__"
+ },
+ "long_": 1234567890,
+ "name": "AAA",
+ "str_": "some str",
+ "int_": 1234
+}
+
+Numbers, strings and lists are stored in native JSON. Dicts with string keys
+are as well, though non-string keys are disallowed by JSON, and so require
+special handling (see below).
+
+
+>>> del a.int_
+>>> del a.long_
+>>> del a.str_
+>>> a.tuple_ = (1, 'two', 3.3)
+>>> cdb.store(a)
+'main__.SimpleDoc:2a208810-467f-4feb-a5bb-98d0beb1e5e7'
+
+{
+ "_id": "main__.SimpleDoc:2a208810-467f-4feb-a5bb-98d0beb1e5e7",
+ "_rev": "3-a24d5422be27e8dca0f4b4375718bceb",
+ "name": "AAA",
+ "couchable:": {
+ "class": "SimpleDoc",
+ "module": "__main__"
+ },
+ "tuple_": {
+ "couchable:": {
+ "args": [
+ [
+ 1,
+ "two",
+ 3.3
+ ]
+ ],
+ "class": "tuple",
+ "module": "__builtin__",
+ "kwargs": {
+ }
+ }
+ }
+}
+
+Tuples do not have a native JSON representation, so they get treated much like
+arbitrary objects do (note: since tuples don't have a __dict__, they need
+special case support). Here, the "args" and "kwargs" will be passed into the
+constructor of the type when it's time to load this object.
+
+
+>>> del a.tuple_
+>>> a._implementationDetail = 'this is private'
+>>> cdb.store(a)
+'main__.SimpleDoc:2a208810-467f-4feb-a5bb-98d0beb1e5e7'
+
+{
+ "_id": "main__.SimpleDoc:2a208810-467f-4feb-a5bb-98d0beb1e5e7",
+ "_rev": "5-5ac8698c7b9567642484e728e60b95b1",
+ "couchable:": {
+ "private": {
+ "_implementationDetail": "this is private"
+ },
+ "class": "SimpleDoc",
+ "module": "__main__"
+ },
+ "name": "AAA"
+}
+
+CouchDB reserves all top-level fields that start with an underscore. Since
+python makes liberal use of underscores to denote "private" or implementation
+detail fields, those end up inside of a second-level dictionary inside of the
+"couchable:" dict. During loading, these will end up back inside the object
+__dict__.
+
+
+>>> del a._implementationDetail
+>>> a.reserved = 'couchable: reserving the string "couchable:" since 2010'
+>>> cdb.store(a)
+'main__.SimpleDoc:2a208810-467f-4feb-a5bb-98d0beb1e5e7'
+
+{
+ "_id": "main__.SimpleDoc:2a208810-467f-4feb-a5bb-98d0beb1e5e7",
+ "_rev": "6-a7ef4d2ba59a68128b076cf5d5d0aaee",
+ "couchable:": {
+ "class": "SimpleDoc",
+ "module": "__main__"
+ },
+ "reserved": "couchable:append:str:couchable: reserving the string \"couchable:\" since 2010",
+ "name": "AAA"
+}
+
+Any string that starts with "couchable:" is escaped, since couchable makes
+heavy use of that prefix. See below.
+
+
+>>> del a.reserved
+>>> a.dict_ = {'foo':'FOO', 123:'bar', (45, 67):'baz'}
+>>> cdb.store(a)
+'main__.SimpleDoc:2a208810-467f-4feb-a5bb-98d0beb1e5e7'
+
+{
+ "_id": "main__.SimpleDoc:2a208810-467f-4feb-a5bb-98d0beb1e5e7",
+ "_rev": "7-4706c617a5f8900956c76ecb6f2e2daa",
+ "couchable:": {
+ "keys": {
+ "couchable:key:tuple:(45, 67)": {
+ "couchable:": {
+ "args": [[45, 67]],
+ "class": "tuple",
+ "module": "__builtin__",
+ "kwargs": {
+ }
+ }
+ }
+ },
+ "class": "SimpleDoc",
+ "module": "__main__"
+ },
+ "dict_": {
+ "couchable:key:tuple:(45, 67)": "baz",
+ "foo": "FOO",
+ "couchable:repr:int:123": "bar"
+ },
+ "name": "AAA"
+}
+
+As we saw above, tuples are supported using similar structures to those used
+by arbitrary objects. However, JSON only supports strings as dictionary keys.
+We solve this by replacing the object-as-key with a string that acts as a
+pointer to the actual object, which is stored inside doc['couchable:']['keys']
+as the full object.
+
+Similarly, ints cannot be JSON dictonary keys, but we can use the int repr to
+fully represent the object, and so we do so in-place (note that this wouldn't
+work for tuples because tuples can contain arbitrarily complex objects, not
+just the ints that we have in the example).
+
+
+>>> del a.dict_
+>>> a.nested = {(1,1):{(2,2):{(3,3):'four'}}}
+>>> cdb.store(a)
+'main__.SimpleDoc:2a208810-467f-4feb-a5bb-98d0beb1e5e7'
+
+{
+ "_id": "main__.SimpleDoc:2a208810-467f-4feb-a5bb-98d0beb1e5e7",
+ "_rev": "10-1fe944276e27bef87259857be9d2abd3",
+ "nested": {
+ "couchable:key:tuple:(1, 1)": {
+ "couchable:key:tuple:(2, 2)": {
+ "couchable:key:tuple:(3, 3)": "four"
+ }
+ }
+ },
+ "couchable:": {
+ "keys": {
+ "couchable:key:tuple:(1, 1)": {
+ "couchable:": {
+ "args": [[1, 1]],
+ "class": "tuple",
+ "module": "__builtin__",
+ "kwargs": {
+ }
+ }
+ },
+ "couchable:key:tuple:(2, 2)": {
+ "couchable:": {
+ "args": [[2, 2]],
+ "class": "tuple",
+ "module": "__builtin__",
+ "kwargs": {
+ }
+ }
+ },
+ "couchable:key:tuple:(3, 3)": {
+ "couchable:": {
+ "args": [[3, 3]],
+ "class": "tuple",
+ "module": "__builtin__",
+ "kwargs": {
+ }
+ }
+ }
+ },
+ "class": "SimpleDoc",
+ "module": "__main__"
+ },
+ "name": "AAA"
+}
+
+Here we have nested dictionaries with tuple keys, and dict values. All of the
+tuples are replaced with string pointers, and fully saved in 'keys'. Note
+that 'keys' is a *document* level construct, not an object level one. 'keys'
+will never appear outside of doc['couchable:'] (at least not outside of naming
+coincidences).
+
+
+>>> del a.nested
+>>> b=SimpleDoc(name='BBB')
+>>> a.bbb = b
+>>> cdb.store(a)
+'main__.SimpleDoc:2a208810-467f-4feb-a5bb-98d0beb1e5e7'
+
+{
+ "_id": "main__.SimpleDoc:2a208810-467f-4feb-a5bb-98d0beb1e5e7",
+ "_rev": "11-5eccb48b99431431c9f91b5e84a4ed7b",
+ "bbb": "couchable:id:main__.SimpleDoc:544a1408-d3f3-41c4-a428-9f0d2d8e2372",
+ "couchable:": {
+ "class": "SimpleDoc",
+ "module": "__main__"
+ },
+ "name": "AAA"
+}
+
+{
+ "_id": "main__.SimpleDoc:544a1408-d3f3-41c4-a428-9f0d2d8e2372",
+ "_rev": "1-577fefe455995539e11c992b7b46e10a",
+ "couchable:": {
+ "class": "SimpleDoc",
+ "module": "__main__"
+ },
+ "name": "BBB"
+}
+Here, we show the behavior when storing multiple objects, each of which is
+flagged as needing to be a full document. Similar to the other string
+pointers encountered already, couchable:id points to an object stored in an
+entirely different document.
View
17 couchable/core.py
@@ -158,9 +158,8 @@ def findHandler(cls_or_name, handler_dict):
class CouchableDb(object):
"""
- >>> import couchable
- >>> cdb=couchable.CouchableDb('testing')
- >>> class SimpleDoc(couchable.CouchableDoc):
+ >>> cdb=CouchableDb('testing')
+ >>> class SimpleDoc(CouchableDoc):
... def __init__(self, **kwargs):
... for name, value in kwargs.items():
... setattr(self, name, value)
@@ -187,9 +186,9 @@ def __init__(self, name=None, url=None, db=None):
server = couchdb.Server(url)
try:
- db = server['pykour']
+ db = server[name]
except:
- db = server.create('pykour')
+ db = server.create(name)
assert db not in self._wrapper_cache
@@ -212,6 +211,7 @@ def store(self, what):
# Actually (finally) send the data to couchdb.
try:
#pprint.pprint([(x[0]._id, getattr(x[0], '_rev', None)) for x in self._done_dict.values()])
+ #print datetime.datetime.now(), "214: self.db.update"
ret_list = self.db.update([x[1] for x in self._done_dict.values()])
except:
#print self._done_dict.values()
@@ -222,6 +222,7 @@ def store(self, what):
obj, doc, attachment_list = store_tuple
if success:
for content, content_name, content_type in attachment_list:
+ #print datetime.datetime.now(), "225: self.db.put_attachment"
self.db.put_attachment(doc, content, content_name, content_type)
# This is important, even if there are no attachments
@@ -546,8 +547,8 @@ def _pack_dict_keyMeansObject(self, parent_doc, data, attachment_list, name, isO
for k,v in data.items() if k not in private_keys and k != '_attachments'}
if private_keys:
- doc.setdefault(FIELD_NAME, {})
- doc[FIELD_NAME]['private'] = {self._pack(parent_doc, k, attachment_list, '{}>{}'.format(name, str(k)), True):
+ parent_doc.setdefault(FIELD_NAME, {})
+ parent_doc[FIELD_NAME]['private'] = {self._pack(parent_doc, k, attachment_list, '{}>{}'.format(name, str(k)), True):
self._pack(parent_doc, v, attachment_list, '{}.{}'.format(name, str(k)), False)
for k,v in data.items() if k in private_keys}
@@ -595,6 +596,7 @@ def _unpack(self, parent_doc, doc, loaded_dict, inst=None):
# FIXME: error?
print type_str, data, _attachment_handlers
+ #print datetime.datetime.now(), "599: self.db.get_attachment"
attachment_response = self.db.get_attachment(parent_doc, data)
return handler_tuple[1](attachment_response.read())
else:
@@ -685,6 +687,7 @@ def load(self, what, loaded=None):
def _load(self, _id, loaded_dict):
if _id not in loaded_dict:
#try:
+ #print datetime.datetime.now(), "690: self.db[_id]"
loaded_dict[_id] = self.db[_id]
#except:
# print "problem:", _id
View
32 couchable/test_couchable.py
@@ -25,6 +25,7 @@
import gc
import random
import sys
+import time
import unittest
# 3rd party packages
@@ -84,6 +85,7 @@ def setUp(self):
self.server = couchdb.Server()
try:
self.server.delete('testing')
+ pass
except:
pass
@@ -102,6 +104,7 @@ def setUp(self):
def tearDown(self):
del self.simple_dict
+ #@unittest.skip('''Playing with ouput''')
@dumpcdb
def test_docs(self):
# doctest returns a tuple of (failed, attempted)
@@ -125,7 +128,7 @@ def test_simple(self):
self.assertEqual(getattr(obj, key), value)
@dumpcdb
- def test_zLast_nonStrKeys(self):
+ def test_nonStrKeys(self):
d = {1234:'ints', (1,2,3,4):'tuples', frozenset([1,1,2,2,3,3]): 'frozenset'}
obj = Simple(d=d)
@@ -137,10 +140,25 @@ def test_zLast_nonStrKeys(self):
obj = self.cdb.load(_id)
+ self.assertEqual(type(obj), Simple)
+
for key, value in d.items():
self.assertIn(key, obj.d)
self.assertEqual(obj.d[key], value)
+ @dumpcdb
+ def test_private(self):
+ a = SimpleDoc(name='AAA', _implementationDetail='foo')
+
+ a_id = self.cdb.store(a)
+
+ del a
+ self.assertFalse(self.cdb._obj_by_id)
+
+ a = self.cdb.load(a_id)
+
+ self.assertEqual(type(a), SimpleDoc)
+
@dumpcdb
def test_multidoc(self):
@@ -223,6 +241,18 @@ def test_docCycles(self):
finally:
sys.setrecursionlimit(limit)
+
+ @unittest.skip("""This fails intermittently. I suspect a problem with the base couchdb library, but I can't pin it down yet.""")
+ @dumpcdb
+ def test_wait(self):
+ a = SimpleDoc(name='AAA')
+ a_id = self.cdb.store(a)
+
+ time.sleep(300)
+
+ b = SimpleDoc(name='BBB', a=a)
+ b_id = self.cdb.store(b)
+
@unittest.skip("""still implementing tests for this...""")
@dumpcdb
View
110 couchable/test_couchdb.py
@@ -0,0 +1,110 @@
+# Copyright (c) 2010 Eli Stevens
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+
+# stdlib
+import copy
+import doctest
+import gc
+import random
+import sys
+import time
+import unittest
+
+# 3rd party packages
+import couchdb
+
+# in-house
+#import couchable
+#import couchable.core
+
+#def dumpcdb(func):
+# def test_dumpcdb_(self):
+# try:
+# func(self)
+# except:
+# for _id in sorted(self.cdb.db):
+# print _id
+# doc = self.cdb.db[_id]
+# for key in sorted(doc):
+# print '{:>15}: {}'.format(key, doc[key])
+#
+# raise
+#
+# return test_dumpcdb_
+#
+#class Simple(object):
+# def __init__(self, **kwargs):
+# for name, value in kwargs.items():
+# setattr(self, name, value)
+#
+#class SimpleDoc(couchable.CouchableDoc):
+# def __init__(self, **kwargs):
+# for name, value in kwargs.items():
+# setattr(self, name, value)
+#
+#class SimpleAttachment(couchable.CouchableAttachment):
+# def __init__(self, **kwargs):
+# for name, value in kwargs.items():
+# setattr(self, name, value)
+#
+## in progress...
+#class AftermarketDoc(object):
+# def __init__(self, **kwargs):
+# for name, value in kwargs.items():
+# setattr(self, name, value)
+#
+#class AftermarketAttachment(object):
+# def __init__(self, **kwargs):
+# for name, value in kwargs.items():
+# setattr(self, name, value)
+## end in progress
+
+class TestCouchdb(unittest.TestCase):
+ def setUp(self):
+ self.server = couchdb.Server()
+ try:
+ self.server.delete('testing')
+ pass
+ except:
+ pass
+
+ self.db = self.server.create('testing')
+
+
+ def tearDown(self):
+ del self.db
+ del self.server
+
+
+ @unittest.skip('''Haven't been able to get a clean testcase repro yet.''')
+ def test_wait(self):
+ a = {'name':'AAA'}
+ #a['_id'], a['_rev'] = self.db.save(a)
+ self.db.update([a])
+
+
+ time.sleep(300)
+
+ b = {'name':'BBB'}
+ self.db.update([b])
+
+
+# eof
Please sign in to comment.
Something went wrong with that request. Please try again.