Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Fetching contributors…

Cannot retrieve contributors at this time

executable file 649 lines (540 sloc) 25.274 kb
#!/usr/bin/env python
# Copyright (c) 2012, Cornell University
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
# * Neither the name of HyperDex nor the names of its contributors may be
# used to endorse or promote products derived from this software without
# specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
'''Construct the space shown below and fuzz test it.'''
import collections
import copy
import operator
import pprint
import random
import string
import sys
import time
import hyperclient
from hyperclient import *
HYPERCLIENT_SUCCESS = 8448
HYPERCLIENT_NOTFOUND = 8449
HYPERCLIENT_SEARCHDONE = 8450
HYPERCLIENT_CMPFAIL = 8451
HYPERCLIENT_UNKNOWNSPACE = 8512
HYPERCLIENT_COORDFAIL = 8513
HYPERCLIENT_SERVERERROR = 8514
HYPERCLIENT_POLLFAILED = 8515
HYPERCLIENT_OVERFLOW = 8516
HYPERCLIENT_RECONFIGURE = 8517
HYPERCLIENT_TIMEOUT = 8519
HYPERCLIENT_UNKNOWNATTR = 8520
HYPERCLIENT_DUPEATTR = 8521
HYPERCLIENT_NONEPENDING = 8523
HYPERCLIENT_DONTUSEKEY = 8524
HYPERCLIENT_WRONGTYPE = 8525
HYPERCLIENT_NOMEM = 8526
HYPERCLIENT_EXCEPTION = 8574
HYPERCLIENT_ZERO = 8575
HYPERCLIENT_A = 8576
HYPERCLIENT_B = 8577
CHARSET = string.ascii_letters + string.digits + string.punctuation
ATTRS = ('s', 'i', 'ls', 'li', 'ss', 'si', 'mss', 'msi', 'mis', 'mii')
INT64_MIN = -9223372036854775808
INT64_MAX = 9223372036854775807
def generate_bytes():
return ''.join([random.choice(CHARSET) for i in xrange(random.randint(0, 16))])
def generate_int():
return random.randint(INT64_MIN, INT64_MAX)
def generate_list_bytes():
return [generate_bytes() for i in xrange(random.randint(0, 16))]
def generate_list_int():
return [generate_int() for i in xrange(random.randint(0, 16))]
def generate_set_bytes():
return set(generate_list_bytes())
def generate_set_int():
return set(generate_list_int())
def generate_map_bytes_bytes():
return dict([(generate_bytes(), generate_bytes()) for i in xrange(random.randint(0, 16))])
def generate_map_bytes_int():
return dict([(generate_bytes(), generate_int()) for i in xrange(random.randint(0, 16))])
def generate_map_int_bytes():
return dict([(generate_int(), generate_bytes()) for i in xrange(random.randint(0, 16))])
def generate_map_int_int():
return dict([(generate_int(), generate_int()) for i in xrange(random.randint(0, 16))])
def generate_put():
key = str(random.randint(0, 1024))
attr_gen = {'s': generate_bytes,
'i': generate_int,
'ls': generate_list_bytes,
'li': generate_list_int,
'ss': generate_set_bytes,
'si': generate_set_int,
'mss': generate_map_bytes_bytes,
'msi': generate_map_bytes_int,
'mis': generate_map_int_bytes,
'mii': generate_map_int_int}
value = {}
for attr in random.sample(ATTRS, random.randint(0, len(ATTRS))):
value[attr] = attr_gen[attr]()
return key, value
def generate_del():
key = str(random.randint(0, 1024))
return key,
def generate_atomic_math():
key = str(random.randint(0, 1024))
val = {}
if random.random() < 0.9:
val['i'] = generate_int()
return key, val
def generate_string_pend():
key = str(random.randint(0, 1024))
val = {}
if random.random() < 0.9:
val['s'] = generate_bytes()
return key, val
def generate_list_push():
key = str(random.randint(0, 1024))
val = {}
if random.random() < 0.9:
val['li'] = generate_int()
if random.random() < 0.9:
val['ls'] = generate_bytes()
return key, val
def generate_set_op():
key = str(random.randint(0, 1024))
val = {}
if random.random() < 0.9:
val['si'] = generate_int()
if random.random() < 0.9:
val['ss'] = generate_bytes()
return key, val
def generate_set_set():
key = str(random.randint(0, 1024))
val = {}
if random.random() < 0.9:
val['si'] = generate_set_int()
if random.random() < 0.9:
val['ss'] = generate_set_bytes()
return key, val
def generate_map_op():
key = str(random.randint(0, 1024))
val = {'mss': {}, 'msi': {}, 'mis': {}, 'mii': {}}
if random.random() < 0.9:
val['mss'][str(random.randint(0, 1024))] = generate_bytes()
if random.random() < 0.9:
val['msi'][str(random.randint(0, 1024))] = generate_int()
if random.random() < 0.9:
val['mis'][random.randint(0, 1024)] = generate_bytes()
if random.random() < 0.9:
val['mii'][random.randint(0, 1024)] = generate_int()
return key, val
def generate_map_remove():
key = str(random.randint(0, 1024))
val = {}
if random.random() < 0.9:
val['mss'] = str(random.randint(0, 1024))
if random.random() < 0.9:
val['msi'] = str(random.randint(0, 1024))
if random.random() < 0.9:
val['mis'] = random.randint(0, 1024)
if random.random() < 0.9:
val['mii'] = random.randint(0, 1024)
return key, val
def generate_map_atomic_math():
key = str(random.randint(0, 1024))
val = {'msi': {}, 'mii': {}}
num = random.randint(0, 1024)
keys = [str(k) for k in random.sample(range(0, 1025), num)]
for k in keys:
val['msi'][k] = generate_int()
num = random.randint(0, 1024)
for k in random.sample(range(0, 1025), num):
val['mii'][k] = generate_int()
return key, val
def generate_map_string_pend():
key = str(random.randint(0, 1024))
val = {'mss': {}, 'mis': {}}
num = random.randint(0, 1024)
num = random.randint(0, 1024)
keys = [str(k) for k in random.sample(range(0, 1025), num)]
for k in keys:
val['mss'][k] = generate_bytes()
num = random.randint(0, 1024)
for k in random.sample(range(0, 1025), num):
val['mis'][k] = generate_bytes()
return key, val
def random_generator():
op_gen = {'put': generate_put
,'del': generate_del
,'atomic_add': generate_atomic_math
,'atomic_sub': generate_atomic_math
,'atomic_mul': generate_atomic_math
,'atomic_div': generate_atomic_math
,'atomic_mod': generate_atomic_math
,'atomic_and': generate_atomic_math
,'atomic_or': generate_atomic_math
,'atomic_xor': generate_atomic_math
,'string_prepend': generate_string_pend
,'string_append': generate_string_pend
,'list_lpush': generate_list_push
,'list_rpush': generate_list_push
,'set_add': generate_set_op
,'set_remove': generate_set_op
,'set_intersect': generate_set_set
,'set_union': generate_set_set
,'map_add': generate_map_op
,'map_remove': generate_map_remove
,'map_atomic_add': generate_map_atomic_math
,'map_atomic_sub': generate_map_atomic_math
,'map_atomic_mul': generate_map_atomic_math
,'map_atomic_div': generate_map_atomic_math
,'map_atomic_mod': generate_map_atomic_math
,'map_atomic_and': generate_map_atomic_math
,'map_atomic_or': generate_map_atomic_math
,'map_atomic_xor': generate_map_atomic_math
,'map_string_prepend': generate_map_string_pend
,'map_string_append': generate_map_string_pend
}
while True:
op = random.choice(op_gen.keys())
args = op_gen[op]()
yield op, args
def inlineexception(f):
def newf(*args, **kwargs):
try:
return f(*args, **kwargs)
except HyperClientException as e:
return e
return newf
class HyperDexClient(object):
def __init__(self, addr, port, space):
self._client = hyperclient.Client(addr, port)
try:
self._client.rm_space('fuzz')
except:
pass
try:
self._client.add_space('''
space fuzz
key string k
attributes
string s,
int i,
list(string) ls,
list(int) li,
set(string) ss,
set(int) si,
map(string, string) mss,
map(string, int) msi,
map(int, string) mis,
map(int, int) mii
create 1 partition
''')
time.sleep(1)
except:
pass
self._space = space
@inlineexception
def op_put(self, key, value):
assert self._client.put(self._space, key, value)
return self._client.get(self._space, key)
@inlineexception
def op_del(self, key):
return self._client.delete(self._space, key)
@inlineexception
def _atomic(self, key, ops, func):
func(self._space, key, ops)
return self._client.get(self._space, key)
def op_atomic_add(self, key, nums):
return self._atomic(key, nums, self._client.atomic_add)
def op_atomic_sub(self, key, nums):
return self._atomic(key, nums, self._client.atomic_sub)
def op_atomic_mul(self, key, nums):
return self._atomic(key, nums, self._client.atomic_mul)
def op_atomic_div(self, key, nums):
return self._atomic(key, nums, self._client.atomic_div)
def op_atomic_mod(self, key, nums):
return self._atomic(key, nums, self._client.atomic_mod)
def op_atomic_and(self, key, nums):
return self._atomic(key, nums, self._client.atomic_and)
def op_atomic_or(self, key, nums):
return self._atomic(key, nums, self._client.atomic_or)
def op_atomic_xor(self, key, nums):
return self._atomic(key, nums, self._client.atomic_xor)
def op_string_prepend(self, key, s):
return self._atomic(key, s, self._client.string_prepend)
def op_string_append(self, key, s):
return self._atomic(key, s, self._client.string_append)
def op_list_lpush(self, key, s):
return self._atomic(key, s, self._client.list_lpush)
def op_list_rpush(self, key, s):
return self._atomic(key, s, self._client.list_rpush)
def op_set_add(self, key, s):
return self._atomic(key, s, self._client.set_add)
def op_set_remove(self, key, s):
return self._atomic(key, s, self._client.set_remove)
def op_set_intersect(self, key, s):
return self._atomic(key, s, self._client.set_intersect)
def op_set_union(self, key, s):
return self._atomic(key, s, self._client.set_union)
def op_map_add(self, key, ops):
return self._atomic(key, ops, self._client.map_add)
def op_map_remove(self, key, ops):
return self._atomic(key, ops, self._client.map_remove)
def op_map_atomic_add(self, key, nums):
return self._atomic(key, nums, self._client.map_atomic_add)
def op_map_atomic_sub(self, key, nums):
return self._atomic(key, nums, self._client.map_atomic_sub)
def op_map_atomic_mul(self, key, nums):
return self._atomic(key, nums, self._client.map_atomic_mul)
def op_map_atomic_div(self, key, nums):
return self._atomic(key, nums, self._client.map_atomic_div)
def op_map_atomic_mod(self, key, nums):
return self._atomic(key, nums, self._client.map_atomic_mod)
def op_map_atomic_and(self, key, nums):
return self._atomic(key, nums, self._client.map_atomic_and)
def op_map_atomic_or(self, key, nums):
return self._atomic(key, nums, self._client.map_atomic_or)
def op_map_atomic_xor(self, key, nums):
return self._atomic(key, nums, self._client.map_atomic_xor)
def op_map_string_prepend(self, key, s):
return self._atomic(key, s, self._client.map_string_prepend)
def op_map_string_append(self, key, s):
return self._atomic(key, s, self._client.map_string_append)
def _add_overflow_check(func):
def _func(a, b):
x = func(a, b)
if x < INT64_MIN or x > INT64_MAX:
raise HyperClientException(HYPERCLIENT_OVERFLOW)
return x
return _func
def _set_add(x, y):
x.add(y)
return x
def _set_remove(x, y):
if y in x:
x.remove(y)
return x
def _map_add(x, y):
for k, v in y.iteritems():
x[k] = v
return x
def _map_remove(x, y):
if y in x:
del x[y]
return x
class HyperDexModel(object):
def __init__(self):
def stored():
return {'s': '',
'i': 0,
'ls': list(),
'li': list(),
'ss': set(),
'si': set(),
'mss': dict(),
'msi': dict(),
'mis': dict(),
'mii': dict()}
self._kvs = collections.defaultdict(stored)
self._put_types = {
's': lambda x: isinstance(x, bytes),
'i': lambda x: isinstance(x, int),
'ls': lambda x: isinstance(x, list) and all([isinstance(e, bytes) for e in x]),
'li': lambda x: isinstance(x, list) and all([isinstance(e, int) for e in x]),
'ss': lambda x: isinstance(x, set) and all([isinstance(e, bytes) for e in x]),
'si': lambda x: isinstance(x, set) and all([isinstance(e, int) for e in x]),
'mss': lambda x: isinstance(x, dict) and all([isinstance(k, bytes) and isinstance(v, bytes) for (k, v) in x.iteritems()]),
'msi': lambda x: isinstance(x, dict) and all([isinstance(k, bytes) and isinstance(v, int) for (k, v) in x.iteritems()]),
'mis': lambda x: isinstance(x, dict) and all([isinstance(k, int) and isinstance(v, bytes) for (k, v) in x.iteritems()]),
'mii': lambda x: isinstance(x, dict) and all([isinstance(k, int) and isinstance(v, int) for (k, v) in x.iteritems()])}
@inlineexception
def op_put(self, key, value):
for k, v in value.iteritems():
if k not in self._put_types:
return HyperClientException(HYPERCLIENT_UNKNOWNATTR, k)
if not self._put_types[k](v):
return HyperClientException(HYPERCLIENT_WRONGTYPE, k)
for k, v in value.iteritems():
self._kvs[key][k] = v
return self._kvs[key]
@inlineexception
def op_del(self, key):
if key in self._kvs:
del self._kvs[key]
return True
else:
return False
@inlineexception
def _atomic(self, key, ops, func, validate):
if key not in self._kvs:
return HyperClientException(HYPERCLIENT_NOTFOUND)
for k, v in ops.iteritems():
if k not in self._put_types:
return HyperClientException(HYPERCLIENT_UNKNOWNATTR, k)
if not validate.get(k, lambda x: False)(v):
return HyperClientException(HYPERCLIENT_WRONGTYPE, k)
newobj = self._kvs[key].copy()
newobj = copy.deepcopy(self._kvs[key])
for k, v in ops.iteritems():
newobj[k] = func(newobj[k], v)
self._kvs[key] = newobj
return self._kvs[key]
@inlineexception
def _atomic_map(self, key, ops, func, default, validate):
for k, m in ops.iteritems():
for mk, mv in m.iteritems():
if k not in self._put_types:
return HyperClientException(HYPERCLIENT_UNKNOWNATTR, k)
if not validate.get(k, lambda x, y: False)(mk, mv):
return HyperClientException(HYPERCLIENT_WRONGTYPE, k)
if key not in self._kvs:
return HyperClientException(HYPERCLIENT_NOTFOUND)
newobj = copy.deepcopy(self._kvs[key])
for k, m in ops.iteritems():
for mk, mv in m.iteritems():
if mk not in newobj[k]:
newobj[k][mk] = default
newobj[k][mk] = func(newobj[k][mk], mv)
self._kvs[key] = newobj
return self._kvs[key]
def op_atomic_add(self, key, nums):
return self._atomic(key, nums, _add_overflow_check(operator.add),
{'i': lambda x: isinstance(x, int)})
def op_atomic_sub(self, key, nums):
return self._atomic(key, nums, _add_overflow_check(operator.sub),
{'i': lambda x: isinstance(x, int)})
def op_atomic_mul(self, key, nums):
return self._atomic(key, nums, _add_overflow_check(operator.mul),
{'i': lambda x: isinstance(x, int)})
def op_atomic_div(self, key, nums):
return self._atomic(key, nums, _add_overflow_check(operator.div),
{'i': lambda x: isinstance(x, int)})
def op_atomic_mod(self, key, nums):
return self._atomic(key, nums, _add_overflow_check(operator.mod),
{'i': lambda x: isinstance(x, int)})
def op_atomic_and(self, key, nums):
return self._atomic(key, nums, lambda x, y: x & y,
{'i': lambda x: isinstance(x, int)})
def op_atomic_or(self, key, nums):
return self._atomic(key, nums, lambda x, y: x | y,
{'i': lambda x: isinstance(x, int)})
def op_atomic_xor(self, key, nums):
return self._atomic(key, nums, lambda x, y: x ^ y,
{'i': lambda x: isinstance(x, int)})
def op_string_prepend(self, key, s):
return self._atomic(key, s, lambda x, y: y + x,
{'s': lambda x: isinstance(x, bytes)})
def op_string_append(self, key, s):
return self._atomic(key, s, lambda x, y: x + y,
{'s': lambda x: isinstance(x, bytes)})
def op_list_lpush(self, key, s):
return self._atomic(key, s, lambda x, y: [y] + x,
{'li': lambda x: isinstance(x, int),
'ls': lambda x: isinstance(x, bytes)})
def op_list_rpush(self, key, s):
return self._atomic(key, s, lambda x, y: x + [y],
{'li': lambda x: isinstance(x, int),
'ls': lambda x: isinstance(x, bytes)})
def op_set_add(self, key, s):
return self._atomic(key, s, _set_add,
{'si': lambda x: isinstance(x, int),
'ss': lambda x: isinstance(x, bytes)})
def op_set_remove(self, key, s):
return self._atomic(key, s, _set_remove,
{'si': lambda x: isinstance(x, int),
'ss': lambda x: isinstance(x, bytes)})
def op_set_intersect(self, key, s):
return self._atomic(key, s, lambda x, y: x & y,
{'si': lambda x: isinstance(x, set) and all([isinstance(e, int) for e in x]),
'ss': lambda x: isinstance(x, set) and all([isinstance(e, bytes) for e in x])})
def op_set_union(self, key, s):
return self._atomic(key, s, lambda x, y: x | y,
{'si': lambda x: isinstance(x, set) and all([isinstance(e, int) for e in x]),
'ss': lambda x: isinstance(x, set) and all([isinstance(e, bytes) for e in x])})
def op_map_add(self, key, ops):
return self._atomic(key, ops, _map_add,
{'msi': lambda d: isinstance(d, dict) and all([isinstance(x, bytes) and isinstance(y, int) for x, y in d.iteritems()]),
'mii': lambda d: isinstance(d, dict) and all([isinstance(x, int) and isinstance(y, int) for x, y in d.iteritems()]),
'mss': lambda d: isinstance(d, dict) and all([isinstance(x, bytes) and isinstance(y, bytes) for x, y in d.iteritems()]),
'mis': lambda d: isinstance(d, dict) and all([isinstance(x, int) and isinstance(y, bytes) for x, y in d.iteritems()])})
def op_map_remove(self, key, s):
return self._atomic(key, s, _map_remove,
{'msi': lambda x: isinstance(x, bytes),
'mii': lambda x: isinstance(x, int),
'mss': lambda x: isinstance(x, bytes),
'mis': lambda x: isinstance(x, int)})
def op_map_atomic_add(self, key, nums):
return self._atomic_map(key, nums, _add_overflow_check(operator.add), 0,
{'msi': lambda x, y: isinstance(x, bytes) and isinstance(y, int),
'mii': lambda x, y: isinstance(x, int) and isinstance(y, int)})
def op_map_atomic_sub(self, key, nums):
return self._atomic_map(key, nums, _add_overflow_check(operator.sub), 0,
{'msi': lambda x, y: isinstance(x, bytes) and isinstance(y, int),
'mii': lambda x, y: isinstance(x, int) and isinstance(y, int)})
def op_map_atomic_mul(self, key, nums):
return self._atomic_map(key, nums, _add_overflow_check(operator.mul), 0,
{'msi': lambda x, y: isinstance(x, bytes) and isinstance(y, int),
'mii': lambda x, y: isinstance(x, int) and isinstance(y, int)})
def op_map_atomic_div(self, key, nums):
return self._atomic_map(key, nums, _add_overflow_check(operator.div), 0,
{'msi': lambda x, y: isinstance(x, bytes) and isinstance(y, int),
'mii': lambda x, y: isinstance(x, int) and isinstance(y, int)})
def op_map_atomic_mod(self, key, nums):
return self._atomic_map(key, nums, _add_overflow_check(operator.mod), 0,
{'msi': lambda x, y: isinstance(x, bytes) and isinstance(y, int),
'mii': lambda x, y: isinstance(x, int) and isinstance(y, int)})
def op_map_atomic_and(self, key, nums):
return self._atomic_map(key, nums, _add_overflow_check(lambda x, y: x & y), 0,
{'msi': lambda x, y: isinstance(x, bytes) and isinstance(y, int),
'mii': lambda x, y: isinstance(x, int) and isinstance(y, int)})
def op_map_atomic_or(self, key, nums):
return self._atomic_map(key, nums, _add_overflow_check(lambda x, y: x | y), 0,
{'msi': lambda x, y: isinstance(x, bytes) and isinstance(y, int),
'mii': lambda x, y: isinstance(x, int) and isinstance(y, int)})
def op_map_atomic_xor(self, key, nums):
return self._atomic_map(key, nums, _add_overflow_check(lambda x, y: x ^ y), 0,
{'msi': lambda x, y: isinstance(x, bytes) and isinstance(y, int),
'mii': lambda x, y: isinstance(x, int) and isinstance(y, int)})
def op_map_string_prepend(self, key, nums):
return self._atomic_map(key, nums, lambda x, y: y + x, '',
{'mss': lambda x, y: isinstance(x, bytes) and isinstance(y, bytes),
'mis': lambda x, y: isinstance(x, int) and isinstance(y, bytes)})
def op_map_string_append(self, key, nums):
return self._atomic_map(key, nums, lambda x, y: x + y, '',
{'mss': lambda x, y: isinstance(x, bytes) and isinstance(y, bytes),
'mis': lambda x, y: isinstance(x, int) and isinstance(y, bytes)})
if __name__ == '__main__':
random.seed(0)
hyperdex = HyperDexClient('127.0.0.1', 1982, 'fuzz')
model = HyperDexModel()
for op, args in random_generator():
#print op, args
m = getattr(model, 'op_' + op)(*args)
h = getattr(hyperdex, 'op_' + op)(*args)
if m != h:
print 'There was a mismatch:', op, args
print 'Model:'
pprint.pprint(m)
print 'HyperDex:'
pprint.pprint(h)
sys.exit(1)
Jump to Line
Something went wrong with that request. Please try again.