110 boto/sdb/db/manager/sdbmanager.py
View file
@@ -15,7 +15,7 @@
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
@@ -32,34 +32,39 @@

ISO8601 = '%Y-%m-%dT%H:%M:%SZ'


class TimeDecodeError(Exception):
pass


class SDBConverter(object):
"""
Responsible for converting base Python types to format compatible with underlying
database. For SimpleDB, that means everything needs to be converted to a string
when stored in SimpleDB and from a string when retrieved.
To convert a value, pass it to the encode or decode method. The encode method
will take a Python native value and convert to DB format. The decode method will
take a DB format value and convert it to Python native format. To find the appropriate
method to call, the generic encode/decode methods will look for the type-specific
method by searching for a method called "encode_<type name>" or "decode_<type name>".
Responsible for converting base Python types to format compatible
with underlying database. For SimpleDB, that means everything
needs to be converted to a string when stored in SimpleDB and from
a string when retrieved.
To convert a value, pass it to the encode or decode method. The
encode method will take a Python native value and convert to DB
format. The decode method will take a DB format value and convert
it to Python native format. To find the appropriate method to
call, the generic encode/decode methods will look for the
type-specific method by searching for a method
called"encode_<type name>" or "decode_<type name>".
"""
def __init__(self, manager):
self.manager = manager
self.type_map = { bool : (self.encode_bool, self.decode_bool),
int : (self.encode_int, self.decode_int),
long : (self.encode_long, self.decode_long),
float : (self.encode_float, self.decode_float),
Model : (self.encode_reference, self.decode_reference),
Key : (self.encode_reference, self.decode_reference),
datetime : (self.encode_datetime, self.decode_datetime),
date : (self.encode_date, self.decode_date),
time : (self.encode_time, self.decode_time),
Blob: (self.encode_blob, self.decode_blob),
str: (self.encode_string, self.decode_string),
self.type_map = {bool: (self.encode_bool, self.decode_bool),
int: (self.encode_int, self.decode_int),
long: (self.encode_long, self.decode_long),
float: (self.encode_float, self.decode_float),
Model: (self.encode_reference, self.decode_reference),
Key: (self.encode_reference, self.decode_reference),
datetime: (self.encode_datetime, self.decode_datetime),
date: (self.encode_date, self.decode_date),
time: (self.encode_time, self.decode_time),
Blob: (self.encode_blob, self.decode_blob),
str: (self.encode_string, self.decode_string),
}

def encode(self, item_type, value):
@@ -92,7 +97,7 @@ def encode_list(self, prop, value):
# We support lists up to 1,000 attributes, since
# SDB technically only supports 1024 attributes anyway
values = {}
for k,v in enumerate(value):
for k, v in enumerate(value):
values["%03d" % k] = v
return self.encode_map(prop, values)

@@ -128,7 +133,7 @@ def decode_list(self, prop, value):
dec_val = {}
for val in value:
if val != None:
k,v = self.decode_map_element(item_type, val)
k, v = self.decode_map_element(item_type, val)
try:
k = int(k)
except:
@@ -143,7 +148,7 @@ def decode_map(self, prop, value):
ret_value = {}
item_type = getattr(prop, "item_type")
for val in value:
k,v = self.decode_map_element(item_type, val)
k, v = self.decode_map_element(item_type, val)
ret_value[k] = v
return ret_value

@@ -152,7 +157,7 @@ def decode_map_element(self, item_type, value):
import urllib
key = value
if ":" in value:
key, value = value.split(':',1)
key, value = value.split(':', 1)
key = urllib.unquote(key)
if Model in item_type.mro():
value = item_type(id=value)
@@ -346,7 +351,6 @@ def encode_blob(self, value):
key.set_contents_from_string(value.value)
return value.id


def decode_blob(self, value):
if not value:
return None
@@ -369,12 +373,14 @@ def decode_blob(self, value):

def encode_string(self, value):
"""Convert ASCII, Latin-1 or UTF-8 to pure Unicode"""
if not isinstance(value, str): return value
if not isinstance(value, str):
return value
try:
return unicode(value, 'utf-8')
except: # really, this should throw an exception.
# in the interest of not breaking current
# systems, however:
except:
# really, this should throw an exception.
# in the interest of not breaking current
# systems, however:
arr = []
for ch in value:
arr.append(unichr(ord(ch)))
@@ -385,10 +391,12 @@ def decode_string(self, value):
return the value as-is"""
return value


class SDBManager(object):

def __init__(self, cls, db_name, db_user, db_passwd,
db_host, db_port, db_table, ddl_dir, enable_ssl, consistent=None):
db_host, db_port, db_table, ddl_dir, enable_ssl,
consistent=None):
self.cls = cls
self.db_name = db_name
self.db_user = db_user
@@ -442,7 +450,7 @@ def _object_lister(self, cls, query_lister):
obj = self.get_object(cls, item.name, item)
if obj:
yield obj

def encode_value(self, prop, value):
if value == None:
return None
@@ -467,10 +475,10 @@ def get_blob_bucket(self, bucket_name=None):
except:
self.bucket = s3.create_bucket(bucket_name)
return self.bucket

def load_object(self, obj):
if not obj._loaded:
a = self.domain.get_attributes(obj.id,consistent_read=self.consistent)
a = self.domain.get_attributes(obj.id, consistent_read=self.consistent)
if '__type__' in a:
for prop in obj.properties(hidden=False):
if prop.name in a:
@@ -481,11 +489,11 @@ def load_object(self, obj):
except Exception, e:
boto.log.exception(e)
obj._loaded = True

def get_object(self, cls, id, a=None):
obj = None
if not a:
a = self.domain.get_attributes(id,consistent_read=self.consistent)
a = self.domain.get_attributes(id, consistent_read=self.consistent)
if '__type__' in a:
if not cls or a['__type__'] != cls.__name__:
cls = find_class(a['__module__'], a['__type__'])
@@ -502,7 +510,7 @@ def get_object(self, cls, id, a=None):
s = '(%s) class %s.%s not found' % (id, a['__module__'], a['__type__'])
boto.log.info('sdbmanager: %s' % s)
return obj

def get_object_from_id(self, id):
return self.get_object(None, id)

@@ -527,14 +535,13 @@ def count(self, cls, filters, quick=True, sort_by=None, select=None):
return count
return count


def _build_filter(self, property, name, op, val):
if name == "__id__":
name = 'itemName()'
if name != "itemName()":
name = '`%s`' % name
if val == None:
if op in ('is','='):
if op in ('is', '='):
return "%(name)s is null" % {"name": name}
elif op in ('is not', '!='):
return "%s is not null" % name
@@ -560,10 +567,10 @@ def _build_filter_part(self, cls, filters, order_by=None, select=None):

if order_by:
if order_by[0] == "-":
order_by_method = "DESC";
order_by_method = "DESC"
order_by = order_by[1:]
else:
order_by_method = "ASC";
order_by_method = "ASC"

if select:
if order_by and order_by in select:
@@ -612,7 +619,7 @@ def _build_filter_part(self, cls, filters, order_by=None, select=None):
type_query = "(`__type__` = '%s'" % cls.__name__
for subclass in self._get_all_decendents(cls).keys():
type_query += " or `__type__` = '%s'" % subclass
type_query +=")"
type_query += ")"
query_parts.append(type_query)

order_by_query = ""
@@ -646,9 +653,9 @@ def save_object(self, obj, expected_value=None):
if not obj.id:
obj.id = str(uuid.uuid4())

attrs = {'__type__' : obj.__class__.__name__,
'__module__' : obj.__class__.__module__,
'__lineage__' : obj.get_lineage()}
attrs = {'__type__': obj.__class__.__name__,
'__module__': obj.__class__.__module__,
'__lineage__': obj.get_lineage()}
del_attrs = []
for property in obj.properties(hidden=False):
value = property.get_value_for_datastore(obj)
@@ -695,10 +702,10 @@ def set_property(self, prop, obj, name, value):
raise SDBPersistenceError("Error: %s must be unique!" % prop.name)
except(StopIteration):
pass
self.domain.put_attributes(obj.id, {name : value}, replace=True)
self.domain.put_attributes(obj.id, {name: value}, replace=True)

def get_property(self, prop, obj, name):
a = self.domain.get_attributes(obj.id,consistent_read=self.consistent)
a = self.domain.get_attributes(obj.id, consistent_read=self.consistent)

# try to get the attribute value from SDB
if name in a:
@@ -709,18 +716,17 @@ def get_property(self, prop, obj, name):
raise AttributeError, '%s not found' % name

def set_key_value(self, obj, name, value):
self.domain.put_attributes(obj.id, {name : value}, replace=True)
self.domain.put_attributes(obj.id, {name: value}, replace=True)

def delete_key_value(self, obj, name):
self.domain.delete_attributes(obj.id, name)

def get_key_value(self, obj, name):
a = self.domain.get_attributes(obj.id, name,consistent_read=self.consistent)
a = self.domain.get_attributes(obj.id, name, consistent_read=self.consistent)
if name in a:
return a[name]
else:
return None

def get_raw_item(self, obj):
return self.domain.get_item(obj.id)

4 boto/sdb/db/property.py
View file
@@ -488,7 +488,7 @@ def __set__(self, obj, value):
This causes bad things to happen"""
if value != None and (obj.id == value or (hasattr(value, "id") and obj.id == value.id)):
raise ValueError, "Can not associate an object with itself!"
return super(ReferenceProperty, self).__set__(obj,value)
return super(ReferenceProperty, self).__set__(obj, value)

def __property_config__(self, model_class, property_name):
Property.__property_config__(self, model_class, property_name)
@@ -643,7 +643,7 @@ def __set__(self, obj, value):
value = [value]
elif value == None: # Override to allow them to set this to "None" to remove everything
value = []
return super(ListProperty, self).__set__(obj,value)
return super(ListProperty, self).__set__(obj, value)


class MapProperty(Property):
2 boto/sdb/db/test_db.py
View file
@@ -153,7 +153,7 @@ def test_list():
t = TestList()
_objects['test_list_t'] = t
t.name = 'a list of ints'
t.nums = [1,2,3,4,5]
t.nums = [1, 2, 3, 4, 5]
t.put()
tt = TestList.get_by_id(t.id)
_objects['test_list_tt'] = tt
2 boto/swf/layer1.py
View file
@@ -62,7 +62,7 @@ class Layer1(AWSAuthConnection):
'com.amazonaws.swf.base.model#OperationNotPermittedFault':
swf_exceptions.SWFOperationNotPermittedError,
'com.amazonaws.swf.base.model#TypeAlreadyExistsFault':
swf_exceptions.SWFTypeAlreadyExistsError ,
swf_exceptions.SWFTypeAlreadyExistsError,
}

ResponseError = SWFResponseError
8 boto/utils.py
View file
@@ -132,7 +132,7 @@ def canonical_string(method, path, headers, expires=None,
qsa = [ a.split('=', 1) for a in qsa]
qsa = [ unquote_v(a) for a in qsa if a[0] in qsa_of_interest ]
if len(qsa) > 0:
qsa.sort(cmp=lambda x,y:cmp(x[0], y[0]))
qsa.sort(cmp=lambda x, y:cmp(x[0], y[0]))
qsa = [ '='.join(a) for a in qsa ]
buf += '?'
buf += '&'.join(qsa)
@@ -224,7 +224,7 @@ def get_instance_metadata(version='latest', url='http://169.254.169.254'):

def get_instance_userdata(version='latest', sep=None,
url='http://169.254.169.254'):
ud_url = '%s/%s/user-data' % (url,version)
ud_url = '%s/%s/user-data' % (url, version)
user_data = retry_url(ud_url, retry_on_404=False)
if user_data:
if sep:
@@ -650,7 +650,7 @@ def write_mime_multipart(content, compress=False, deftype='text/plain', delimite
:rtype: str:
"""
wrapper = MIMEMultipart()
for name,con in content:
for name, con in content:
definite_type = guess_mime_type(con, deftype)
maintype, subtype = definite_type.split('/', 1)
if maintype == 'text':
@@ -696,7 +696,7 @@ def guess_mime_type(content, deftype):
'#cloud-boothook' : 'text/cloud-boothook'
}
rtype = deftype
for possible_type,mimetype in starts_with_mappings.items():
for possible_type, mimetype in starts_with_mappings.items():
if content.startswith(possible_type):
rtype = mimetype
break
6 tests/db/test_lists.py
View file
@@ -46,13 +46,13 @@ def teardown_class(cls):
def test_list_order(self):
"""Testing the order of lists"""
t = SimpleListModel()
t.nums = [5,4,1,3,2]
t.nums = [5, 4, 1, 3, 2]
t.strs = ["B", "C", "A", "D", "Foo"]
t.put()
self.objs.append(t)
time.sleep(3)
t = SimpleListModel.get_by_id(t.id)
assert(t.nums == [5,4,1,3,2])
assert(t.nums == [5, 4, 1, 3, 2])
assert(t.strs == ["B", "C", "A", "D", "Foo"])

def test_old_compat(self):
@@ -81,7 +81,7 @@ def test_query_equals(self):
time.sleep(3)
assert(SimpleListModel.find(strs="Bizzle").count() == 1)
assert(SimpleListModel.find(strs="Bar").count() == 1)
assert(SimpleListModel.find(strs=["Bar","Bizzle"]).count() == 1)
assert(SimpleListModel.find(strs=["Bar", "Bizzle"]).count() == 1)

def test_query_not_equals(self):
"""Test a not equal filter"""
4 tests/db/test_password.py
View file
@@ -81,7 +81,7 @@ class MyModel(Model):
id= obj.id
time.sleep(5)
obj = MyModel.get_by_id(id)
self.assertEquals(obj.password,'bar')
self.assertEquals(obj.password, 'bar')
self.assertEquals(str(obj.password), expected)
#hmac.new('mysecret','bar').hexdigest())

@@ -98,7 +98,7 @@ def test_aaa_default_password_property(self):

def test_password_constructor_hashfunc(self):
import hmac
myhashfunc=lambda msg: hmac.new('mysecret',msg)
myhashfunc=lambda msg: hmac.new('mysecret', msg)
cls = self.test_model(hashfunc=myhashfunc)
obj = cls()
obj.password='hello'
14 tests/db/test_sequence.py
View file
@@ -69,7 +69,7 @@ def test_sequence_simple_int(self):
assert(s2.val == 3)

def test_sequence_simple_string(self):
from boto.sdb.db.sequence import Sequence,increment_string
from boto.sdb.db.sequence import Sequence, increment_string
s = Sequence(fnc=increment_string)
self.sequences.append(s)
assert(s.val == "A")
@@ -80,26 +80,26 @@ def test_fib(self):
from boto.sdb.db.sequence import fib
# Just check the first few numbers in the sequence
lv = 0
for v in [1,2,3,5,8,13,21,34,55,89,144]:
assert(fib(v,lv) == lv+v)
lv = fib(v,lv)
for v in [1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144]:
assert(fib(v, lv) == lv+v)
lv = fib(v, lv)

def test_sequence_fib(self):
"""Test the fibonacci sequence"""
from boto.sdb.db.sequence import Sequence,fib
from boto.sdb.db.sequence import Sequence, fib
s = Sequence(fnc=fib)
s2 = Sequence(s.id)
self.sequences.append(s)
assert(s.val == 1)
# Just check the first few numbers in the sequence
for v in [1,2,3,5,8,13,21,34,55,89,144]:
for v in [1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144]:
assert(s.next() == v)
assert(s.val == v)
assert(s2.val == v) # it shouldn't matter which reference we use since it's garunteed to be consistent

def test_sequence_string(self):
"""Test the String incrementation sequence"""
from boto.sdb.db.sequence import Sequence,increment_string
from boto.sdb.db.sequence import Sequence, increment_string
s = Sequence(fnc=increment_string)
self.sequences.append(s)
assert(s.val == "A")
2 tests/dynamodb/test_layer2.py
View file
@@ -274,7 +274,7 @@ def test_layer2_basic(self):
item3['FalseBoolean'] = False

# Test some set values
integer_set = set([1,2,3,4,5])
integer_set = set([1, 2, 3, 4, 5])
float_set = set([1.1, 2.2, 3.3, 4.4, 5.5])
mixed_set = set([1, 2, 3.3, 4, 5.555])
str_set = set(['foo', 'bar', 'fie', 'baz'])
2 tests/mturk/create_hit_external.py
View file
@@ -10,7 +10,7 @@ def test_create_hit_external(self):
q = ExternalQuestion(external_url=external_url, frame_height=800)
conn = SetHostMTurkConnection()
keywords=['boto', 'test', 'doctest']
create_hit_rs = conn.create_hit(question=q, lifetime=60*65,max_assignments=2,title="Boto External Question Test", keywords=keywords,reward = 0.05, duration=60*6,approval_delay=60*60, annotation='An annotation from boto external question test', response_groups=['Minimal','HITDetail','HITQuestion','HITAssignmentSummary',])
create_hit_rs = conn.create_hit(question=q, lifetime=60*65, max_assignments=2, title="Boto External Question Test", keywords=keywords, reward = 0.05, duration=60*6, approval_delay=60*60, annotation='An annotation from boto external question test', response_groups=['Minimal', 'HITDetail', 'HITQuestion', 'HITAssignmentSummary',])
assert(create_hit_rs.status == True)

if __name__ == "__main__":
2 tests/mturk/create_hit_with_qualifications.py
View file
@@ -8,7 +8,7 @@ def test():
keywords=['boto', 'test', 'doctest']
qualifications = Qualifications()
qualifications.add(PercentAssignmentsApprovedRequirement(comparator="GreaterThan", integer_value="95"))
create_hit_rs = conn.create_hit(question=q, lifetime=60*65,max_assignments=2,title="Boto External Question Test", keywords=keywords,reward = 0.05, duration=60*6,approval_delay=60*60, annotation='An annotation from boto external question test', qualifications=qualifications)
create_hit_rs = conn.create_hit(question=q, lifetime=60*65, max_assignments=2, title="Boto External Question Test", keywords=keywords, reward = 0.05, duration=60*6, approval_delay=60*60, annotation='An annotation from boto external question test', qualifications=qualifications)
assert(create_hit_rs.status == True)
print create_hit_rs.HITTypeId

15 tests/mturk/support.py
View file
@@ -1,8 +1,7 @@

import sys

# use unittest2 under Python 2.6 and earlier.
if sys.version_info >= (2,7):
import unittest
else:
import unittest2 as unittest
import sys

# use unittest2 under Python 2.6 and earlier.
if sys.version_info >= (2, 7):
import unittest
else:
import unittest2 as unittest
2 tests/s3/test_gsconnection.py
View file
@@ -92,7 +92,7 @@ def test_1_basic(self):
fp = open('foobar1', 'wb')
k.get_contents_to_file(fp)
fp.close()
fp2.seek(0,0)
fp2.seek(0, 0)
fp = open('foobar1', 'rb')
assert (fp2.read() == fp.read()), 'Chunked Transfer corrupted the Data'
fp.close()
2 tests/s3/test_key.py
View file
@@ -136,7 +136,7 @@ def test_set_contents_with_md5(self):
# let's try a wrong md5 by just altering it.
k = self.bucket.new_key("k")
sfp.seek(0)
hexdig,base64 = k.compute_md5(sfp)
hexdig, base64 = k.compute_md5(sfp)
bad_md5 = (hexdig, base64[3:])
try:
k.set_contents_from_file(sfp, md5=bad_md5)
4 tests/s3/test_mfa.py
View file
@@ -51,7 +51,7 @@ def test_mfadel(self):

# Check enabling mfa worked.
i = 0
for i in range(1,8):
for i in range(1, 8):
time.sleep(2**i)
d = self.bucket.get_versioning_status()
if d['Versioning'] == 'Enabled' and d['MfaDelete'] == 'Enabled':
@@ -82,7 +82,7 @@ def test_mfadel(self):

# Lastly, check disabling mfa worked.
i = 0
for i in range(1,8):
for i in range(1, 8):
time.sleep(2**i)
d = self.bucket.get_versioning_status()
if d['Versioning'] == 'Suspended' and d['MfaDelete'] != 'Enabled':
2 tests/s3/test_multidelete.py
View file
@@ -150,7 +150,7 @@ def test_1(self):
nkeys = 100

# create a bunch of keynames
key_names = ['key-%03d' % i for i in range(0,nkeys)]
key_names = ['key-%03d' % i for i in range(0, nkeys)]

# create the corresponding keys
for key_name in key_names:
18 tests/utils/test_password.py
View file
@@ -28,23 +28,23 @@
class TestPassword(unittest.TestCase):
"""Test basic password functionality"""

def clstest(self,cls):
def clstest(self, cls):

"""Insure that password.__eq__ hashes test value before compare"""

password=cls('foo')
log.debug( "Password %s" % password )
self.assertNotEquals(password , 'foo')
self.assertNotEquals(password, 'foo')

password.set('foo')
hashed = str(password)
self.assertEquals(password , 'foo')
self.assertEquals(password, 'foo')
self.assertEquals(password.str, hashed)

password = cls(hashed)
self.assertNotEquals(password.str , 'foo')
self.assertEquals(password , 'foo')
self.assertEquals(password.str , hashed)
self.assertNotEquals(password.str, 'foo')
self.assertEquals(password, 'foo')
self.assertEquals(password.str, hashed)


def test_aaa_version_1_9_default_behavior(self):
@@ -67,7 +67,7 @@ def test_hmac(self):
from boto.utils import Password
import hmac

def hmac_hashfunc(cls,msg):
def hmac_hashfunc(cls, msg):
log.debug("\n%s %s" % (cls.__class__, cls) )
return hmac.new('mysecretkey', msg)

@@ -78,7 +78,7 @@ class HMACPassword(Password):
password=HMACPassword()
password.set('foo')

self.assertEquals(str(password), hmac.new('mysecretkey','foo').hexdigest())
self.assertEquals(str(password), hmac.new('mysecretkey', 'foo').hexdigest())

def test_constructor(self):
from boto.utils import Password
@@ -88,7 +88,7 @@ def test_constructor(self):

password = Password(hashfunc=hmac_hashfunc)
password.set('foo')
self.assertEquals(password.str, hmac.new('mysecretkey','foo').hexdigest())
self.assertEquals(password.str, hmac.new('mysecretkey', 'foo').hexdigest())