Skip to content
Browse files

atomic ops discussion

  • Loading branch information...
1 parent b516f62 commit 7023d6da0431d863249856b9475347d82fda6a62 @el33th4x0r el33th4x0r committed
Showing with 135 additions and 106 deletions.
  1. +135 −106 doc/05.asynchronous.rst
View
241 doc/05.asynchronous.rst
@@ -326,9 +326,8 @@ process; or fail the operation, leaving the object unchanged.
>>> c.delete('userlocks', 'jsmith1')
True
-This operation approximates the test-and-set mechanism used for implementing
-concurrency controls in memory. It's easy to build ``lock()`` and ``unlock()``
-using these controls:
+Here's an illustrative example that shows how a test-and-set mechanism can be used to implement
+``lock()`` and ``unlock()``.
.. sourcecode:: pycon
@@ -340,11 +339,15 @@ using these controls:
>>> lock(c, 'jsmith1')
>>> unlock(c, 'jsmith1')
-Note that a real implementation might want to make provisions for when clients
-fail while holding the lock.
+Note that a real implementation will not want to busy-loop, and will want to make provisions
+for when clients fail while holding the lock.
-TODO
-----
+A Comprehensive Walk
+--------------------
+
+Having built an intuition for how to structure and use the atomic operations, let's go through them
+and illustrate the various atomic operations for each of the different types. So, let's first
+create a space that can allow us to do this:
.. sourcecode:: pycon
@@ -352,186 +355,212 @@ TODO
>>> c = hyperclient.Client('127.0.0.1', 1982)
>>> c.add_space('''space alldatatypes key k attributes string s, int i, float f, list(string) ls, set(string) ss, map(string, string) mss, map(string, int) msi''')
- >>> # put_if_not_exist
- >>> c.put_if_not_exist('alldatatypes', 'gun', {})
+The key-based operations ``put_if_not_exist`` and ``cond_put`` can be used to create the object if it does not exist, and to modify it only if certain fields match expected
+values, respectively.
+
+.. sourcecode:: pycon
+
+ >>> c.put_if_not_exist('alldatatypes', 'somekey', {'s': 'initial value'})
True
- >>> c.put_if_not_exist('alldatatypes', 'gun', {})
+ >>> c.put_if_not_exist('alldatatypes', 'somekey', {'s': 'initial value'})
False
>>> # cond_put. First is predicate. May be any valid search predicate
- >>> c.cond_put('alldatatypes', 'gun', {'s': ''}, {'s': 'some string'})
+ >>> c.cond_put('alldatatypes', 'somekey', {'s': 'initial value'}, {'s': 'some string'})
True
- >>> c.cond_put('alldatatypes', 'gun', {'s': ''}, {'s': 'some string'})
+ >>> c.cond_put('alldatatypes', 'somekey', {'s': 'initial value'}, {'s': 'some string'})
False
- >>> c.get('alldatatypes', 'gun')
+
+Note how the first operations succeeds, and the second one fails. Let's check to make sure that our object
+reflects the changes we have applied:
+
+.. sourcecode:: pycon
+
+ >>> c.get('alldatatypes', 'somekey')
{'f': 0.0, 'i': 0, 'mss': {}, 'ss': set([]), 's': 'some string', 'ls': [], 'msi': {}}
- >>> # atomic_add
- >>> c.atomic_add('alldatatypes', 'gun', {'i': 1, 'f': 0.25})
+Let's now perform some atomic operations on integers and floats. These are self-explanatory, so we'll let the code do the talking. You will note that the float "f" and integer "i" fields are the ones of interest here, the rest are non-changing:
+
+.. sourcecode:: pycon
+
+ >>> c.atomic_add('alldatatypes', 'somekey', {'i': 1, 'f': 0.25})
True
- >>> c.get('alldatatypes', 'gun')
+ >>> c.get('alldatatypes', 'somekey')
{'f': 0.25, 'i': 1, 'mss': {}, 'ss': set([]), 's': 'some string', 'ls': [], 'msi': {}}
- >>> # atomic_sub
- >>> c.atomic_sub('alldatatypes', 'gun', {'i': 2, 'f': 0.5})
+ >>> c.atomic_sub('alldatatypes', 'somekey', {'i': 2, 'f': 0.5})
True
- >>> c.get('alldatatypes', 'gun')
+ >>> c.get('alldatatypes', 'somekey')
{'f': -0.25, 'i': -1, 'mss': {}, 'ss': set([]), 's': 'some string', 'ls': [], 'msi': {}}
- >>> # atomic_mul
- >>> c.atomic_mul('alldatatypes', 'gun', {'i': 2, 'f': 4.})
+ >>> c.atomic_mul('alldatatypes', 'somekey', {'i': 2, 'f': 4.})
True
- >>> c.get('alldatatypes', 'gun')
+ >>> c.get('alldatatypes', 'somekey')
{'f': -1.0, 'i': -2, 'mss': {}, 'ss': set([]), 's': 'some string', 'ls': [], 'msi': {}}
- >>> # atomic_div
- >>> c.atomic_div('alldatatypes', 'gun', {'i': 2, 'f': 4.})
+ >>> c.atomic_div('alldatatypes', 'somekey', {'i': 2, 'f': 4.})
True
- >>> c.get('alldatatypes', 'gun')
+ >>> c.get('alldatatypes', 'somekey')
{'f': -0.25, 'i': -1, 'mss': {}, 'ss': set([]), 's': 'some string', 'ls': [], 'msi': {}}
- >>> # atomic_and
- >>> c.put('alldatatypes', 'gun', {'i': 0xdeadbeefcafe})
+ >>> c.put('alldatatypes', 'somekey', {'i': 0xdeadbeefcafe})
True
- >>> c.atomic_and('alldatatypes', 'gun', {'i': 0xffffffff0000})
+ >>> c.atomic_and('alldatatypes', 'somekey', {'i': 0xffffffff0000})
True
- >>> c.get('alldatatypes', 'gun')
+ >>> c.get('alldatatypes', 'somekey')
{'f': -0.25, 'i': 244837814042624, 'mss': {}, 'ss': set([]), 's': 'some string', 'ls': [], 'msi': {}}
+ >>> print "0x%x" % (c.get('alldatatypes', 'somekey')['i'],)
+ 0xdeadbeef0000
- >>> # atomic_or
- >>> c.atomic_or('alldatatypes', 'gun', {'i': 0x00000000cafe})
+ >>> c.atomic_or('alldatatypes', 'somekey', {'i': 0x00000000cafe})
True
- >>> c.get('alldatatypes', 'gun')
- {'f': -0.25, 'i': 244837814094590, 'mss': {}, 'ss': set([]), 's': 'some string', 'ls': [], 'msi': {}}
+ >>> print "0x%x" % (c.get('alldatatypes', 'somekey')['i'],)
+ 0xdeadbeefcafe
- >>> # atomic_xor
- >>> c.atomic_xor('alldatatypes', 'gun', {'i': 0xdea5a0403af3})
+ >>> c.atomic_xor('alldatatypes', 'somekey', {'i': 0xdea5a0403af3})
True
- >>> c.get('alldatatypes', 'gun')
- {'f': -0.25, 'i': 34874585101, 'mss': {}, 'ss': set([]), 's': 'some string', 'ls': [], 'msi': {}}
+ >>> print "0x%x" % (c.get('alldatatypes', 'somekey')['i'],)
+ 0x81eaff00d
+
+Ok, now let's perform some atomic operations on strings:
- >>> # string_prepend
- >>> c.string_prepend('alldatatypes', 'gun', {'s': '->'})
+.. sourcecode:: pycon
+
+ >>> c.string_prepend('alldatatypes', 'somekey', {'s': '->'})
True
- >>> c.get('alldatatypes', 'gun')
- {'f': -0.25, 'i': 34874585101, 'mss': {}, 'ss': set([]), 's': '->some string', 'ls': [], 'msi': {}}
+ >>> c.get('alldatatypes', 'somekey')['s']
+ ->some string
- >>> # string_append
- >>> c.string_append('alldatatypes', 'gun', {'s': '<-'})
+ >>> c.string_append('alldatatypes', 'somekey', {'s': '<-'})
True
- >>> c.get('alldatatypes', 'gun')
- {'f': -0.25, 'i': 34874585101, 'mss': {}, 'ss': set([]), 's': '->some string<-', 'ls': [], 'msi': {}}
+ >>> c.get('alldatatypes', 'somekey')['s']
+ ->some string<-
+
+Lists provide atomic operations as well, to push new items on the left or the right of the list:
- >>> # list_lpush
- >>> c.put('alldatatypes', 'gun', {'ls': ['B']})
+.. sourcecode:: pycon
+
+ >>> c.put('alldatatypes', 'somekey', {'ls': ['B']})
True
- >>> c.list_lpush('alldatatypes', 'gun', {'ls': 'A'})
+ >>> c.list_lpush('alldatatypes', 'somekey', {'ls': 'A'})
True
- >>> c.get('alldatatypes', 'gun')
- {'f': -0.25, 'i': 34874585101, 'mss': {}, 'ss': set([]), 's': '->some string<-', 'ls': ['A', 'B'], 'msi': {}}
+ >>> c.get('alldatatypes', 'somekey')['ls']
+ ['A', 'B']
- >>> # list_rpush
- >>> c.list_rpush('alldatatypes', 'gun', {'ls': 'C'})
+ >>> c.list_rpush('alldatatypes', 'somekey', {'ls': 'C'})
True
- >>> c.get('alldatatypes', 'gun')
- {'f': -0.25, 'i': 34874585101, 'mss': {}, 'ss': set([]), 's': '->some string<-', 'ls': ['A', 'B', 'C'], 'msi': {}}
+ >>> c.get('alldatatypes', 'somekey')['ls']
+ ['A', 'B', 'C']
+
+Sets provide the whole range of atomic set operations:
+
+.. sourcecode:: pycon
- >>> # set_add
- >>> c.set_add('alldatatypes', 'gun', {'ss': 'C'})
+ >>> c.set_add('alldatatypes', 'somekey', {'ss': 'C'})
True
- >>> c.get('alldatatypes', 'gun')
- {'f': -0.25, 'i': 34874585101, 'mss': {}, 'ss': set(['C']), 's': '->some string<-', 'ls': ['A', 'B', 'C'], 'msi': {}}
+ >>> c.get('alldatatypes', 'somekey')['ss']
+ set(['C'])
- >>> # set_remove
- >>> c.set_remove('alldatatypes', 'gun', {'ss': 'C'})
+ >>> c.set_remove('alldatatypes', 'somekey', {'ss': 'C'})
True
- >>> c.get('alldatatypes', 'gun')
- {'f': -0.25, 'i': 34874585101, 'mss': {}, 'ss': set([]), 's': '->some string<-', 'ls': ['A', 'B', 'C'], 'msi': {}}
+ >>> c.get('alldatatypes', 'somekey')['ss']
+ set([])
- >>> # set_union
- >>> c.set_union('alldatatypes', 'gun', {'ss': set(['A', 'B', 'C'])})
+ >>> c.set_union('alldatatypes', 'somekey', {'ss': set(['A', 'B', 'C'])})
True
- >>> c.get('alldatatypes', 'gun')
- {'f': -0.25, 'i': 34874585101, 'mss': {}, 'ss': set(['A', 'C', 'B']), 's': '->some string<-', 'ls': ['A', 'B', 'C'], 'msi': {}}
+ >>> c.get('alldatatypes', 'somekey')['ss']
+ set(['A', 'C', 'B'])
- >>> # set_intersect
- >>> c.set_intersect('alldatatypes', 'gun', {'ss': set(['A', 'B', 'Z'])})
+ >>> c.set_intersect('alldatatypes', 'somekey', {'ss': set(['A', 'B', 'Z'])})
True
- >>> c.get('alldatatypes', 'gun')
- {'f': -0.25, 'i': 34874585101, 'mss': {}, 'ss': set(['A', 'B']), 's': '->some string<-', 'ls': ['A', 'B', 'C'], 'msi': {}}
+ >>> c.get('alldatatypes', 'somekey')['ss']
+ set(['A', 'B'])
+
+Finally, we have maps. Maps provide two kinds of atomic operations: the kind that atomically manipulates the map itself, and the kind that atomically manipulates an element of the map.
+
+Let's atomically add an element to the ``mss`` field while atomically adding another element, with the same key, to the ``msi`` field.
+
+.. sourcecode:: pycon
- >>> # map_add
- >>> c.map_add('alldatatypes', 'gun', {'mss': {'mapkey': 'mapvalue'}, 'msi': {'mapkey': 16}})
+ >>> c.map_add('alldatatypes', 'somekey', {'mss': {'mapkey': 'mapvalue'}, 'msi': {'mapkey': 16}})
True
- >>> c.get('alldatatypes', 'gun')
+ >>> c.get('alldatatypes', 'somekey')
{'f': -0.25, 'i': 34874585101, 'mss': {'mapkey': 'mapvalue'}, 'ss': set(['A', 'B']), 's': '->some string<-', 'ls': ['A', 'B', 'C'], 'msi': {'mapkey': 16}}
- >>> c.map_add('alldatatypes', 'gun', {'mss': {'mapkey': 'mapvalue', 'tmp': 'delete me'}, 'msi': {'mapkey': 16}})
+
+Let's add another field to one of the maps:
+
+.. sourcecode:: pycon
+
+ >>> c.map_add('alldatatypes', 'somekey', {'mss': {'tmp': 'delete me'}})
True
- >>> c.get('alldatatypes', 'gun')
+ >>> c.get('alldatatypes', 'somekey')
{'f': -0.25, 'i': 34874585101, 'mss': {'tmp': 'delete me', 'mapkey': 'mapvalue'}, 'ss': set(['A', 'B']), 's': '->some string<-', 'ls': ['A', 'B', 'C'], 'msi': {'mapkey': 16}}
- >>> # Map remove This is a clumsy API. I should make it work with a set
- >>> # rather than dict
- >>> c.map_remove('alldatatypes', 'gun', {'mss': {'tmp': 'X'}})
+Let's now atomically delete that field. We need only specify its key, the value does not matter for the ``map_remove`` operation.
+
+.. sourcecode:: pycon
+
+ >>> c.map_remove('alldatatypes', 'somekey', {'mss': {'tmp': 'X'}})
True
- >>> c.get('alldatatypes', 'gun')
+ >>> c.get('alldatatypes', 'somekey')
{'f': -0.25, 'i': 34874585101, 'mss': {'mapkey': 'mapvalue'}, 'ss': set(['A', 'B']), 's': '->some string<-', 'ls': ['A', 'B', 'C'], 'msi': {'mapkey': 16}}
- >>> # map_atomic_add
- >>> c.map_atomic_add('alldatatypes', 'gun', {'msi': {'mapkey': 16}})
+
+Now we can perform all of the preceding atomic operations on each of the elements of the maps, atomically:
+
+.. sourcecode:: pycon
+
+ >>> c.map_atomic_add('alldatatypes', 'somekey', {'msi': {'mapkey': 16}})
True
- >>> c.get('alldatatypes', 'gun')
+ >>> c.get('alldatatypes', 'somekey')
{'f': -0.25, 'i': 34874585101, 'mss': {'mapkey': 'mapvalue'}, 'ss': set(['A', 'B']), 's': '->some string<-', 'ls': ['A', 'B', 'C'], 'msi': {'mapkey': 32}}
- >>> # map_atomic_sub
- >>> c.map_atomic_sub('alldatatypes', 'gun', {'msi': {'mapkey': -32}})
+ >>> c.map_atomic_sub('alldatatypes', 'somekey', {'msi': {'mapkey': -32}})
True
- >>> c.get('alldatatypes', 'gun')
+ >>> c.get('alldatatypes', 'somekey')
{'f': -0.25, 'i': 34874585101, 'mss': {'mapkey': 'mapvalue'}, 'ss': set(['A', 'B']), 's': '->some string<-', 'ls': ['A', 'B', 'C'], 'msi': {'mapkey': 64}}
- >>> # map_atomic_mul
- >>> c.map_atomic_mul('alldatatypes', 'gun', {'msi': {'mapkey': 4}})
+ >>> c.map_atomic_mul('alldatatypes', 'somekey', {'msi': {'mapkey': 4}})
True
- >>> c.get('alldatatypes', 'gun')
+ >>> c.get('alldatatypes', 'somekey')
{'f': -0.25, 'i': 34874585101, 'mss': {'mapkey': 'mapvalue'}, 'ss': set(['A', 'B']), 's': '->some string<-', 'ls': ['A', 'B', 'C'], 'msi': {'mapkey': 256}}
- >>> # map_atomic_div
- >>> c.map_atomic_div('alldatatypes', 'gun', {'msi': {'mapkey': 64}})
+ >>> c.map_atomic_div('alldatatypes', 'somekey', {'msi': {'mapkey': 64}})
True
- >>> c.get('alldatatypes', 'gun')
+ >>> c.get('alldatatypes', 'somekey')
{'f': -0.25, 'i': 34874585101, 'mss': {'mapkey': 'mapvalue'}, 'ss': set(['A', 'B']), 's': '->some string<-', 'ls': ['A', 'B', 'C'], 'msi': {'mapkey': 4}}
- >>> # map_atomic_and
- >>> c.map_atomic_and('alldatatypes', 'gun', {'msi': {'mapkey': 2}})
+ >>> c.map_atomic_and('alldatatypes', 'somekey', {'msi': {'mapkey': 2}})
True
- >>> c.get('alldatatypes', 'gun')
+ >>> c.get('alldatatypes', 'somekey')
{'f': -0.25, 'i': 34874585101, 'mss': {'mapkey': 'mapvalue'}, 'ss': set(['A', 'B']), 's': '->some string<-', 'ls': ['A', 'B', 'C'], 'msi': {'mapkey': 0}}
- >>> # map_atomic_or
- >>> c.map_atomic_or('alldatatypes', 'gun', {'msi': {'mapkey': 5}})
+ >>> c.map_atomic_or('alldatatypes', 'somekey', {'msi': {'mapkey': 5}})
True
- >>> c.get('alldatatypes', 'gun')
+ >>> c.get('alldatatypes', 'somekey')
{'f': -0.25, 'i': 34874585101, 'mss': {'mapkey': 'mapvalue'}, 'ss': set(['A', 'B']), 's': '->some string<-', 'ls': ['A', 'B', 'C'], 'msi': {'mapkey': 5}}
- >>> # map_atomic_xor
- >>> c.map_atomic_xor('alldatatypes', 'gun', {'msi': {'mapkey': 7}})
+ >>> c.map_atomic_xor('alldatatypes', 'somekey', {'msi': {'mapkey': 7}})
True
- >>> c.get('alldatatypes', 'gun')
+ >>> c.get('alldatatypes', 'somekey')
{'f': -0.25, 'i': 34874585101, 'mss': {'mapkey': 'mapvalue'}, 'ss': set(['A', 'B']), 's': '->some string<-', 'ls': ['A', 'B', 'C'], 'msi': {'mapkey': 2}}
- >>> # map_string_prepend
- >>> c.map_string_prepend('alldatatypes', 'gun', {'mss': {'mapkey': '->'}})
+ >>> c.map_string_prepend('alldatatypes', 'somekey', {'mss': {'mapkey': '->'}})
True
- >>> c.get('alldatatypes', 'gun')
+ >>> c.get('alldatatypes', 'somekey')
{'f': -0.25, 'i': 34874585101, 'mss': {'mapkey': '->mapvalue'}, 'ss': set(['A', 'B']), 's': '->some string<-', 'ls': ['A', 'B', 'C'], 'msi': {'mapkey': 2}}
- >>> # map_string_append
- >>> c.map_string_append('alldatatypes', 'gun', {'mss': {'mapkey': '<-'}})
+ >>> c.map_string_append('alldatatypes', 'somekey', {'mss': {'mapkey': '<-'}})
True
- >>> c.get('alldatatypes', 'gun')
+ >>> c.get('alldatatypes', 'somekey')
{'f': -0.25, 'i': 34874585101, 'mss': {'mapkey': '->mapvalue<-'}, 'ss': set(['A', 'B']), 's': '->some string<-', 'ls': ['A', 'B', 'C'], 'msi': {'mapkey': 2}}
+HyperDex's atomic operations are extensive and very expressive. And they are guaranteed to be applied in the same order on all replicas, so the state of the object you are operating
+on is guaranteed to be the same, regardless of failovers.
+
+
+
.. todo::
.. sourcecode:: pycon

0 comments on commit 7023d6d

Please sign in to comment.
Something went wrong with that request. Please try again.