Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

PEP8 and pyflakes cleanup.

  • Loading branch information...
commit bc6b6105dbed87b33aaa20688bb44ca541ff0688 1 parent 9fb28d9
@garnaat garnaat authored
Showing with 151 additions and 93 deletions.
  1. +6 −5 boto/resultset.py
  2. +17 −16 boto/storage_uri.py
  3. +128 −72 boto/utils.py
View
11 boto/resultset.py
@@ -19,14 +19,15 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
+
class ResultSet(list):
"""
The ResultSet is used to pass results back from the Amazon services
to the client. It is light wrapper around Python's :py:class:`list` class,
- with some additional methods for parsing XML results from AWS.
- Because I don't really want any dependencies on external libraries,
- I'm using the standard SAX parser that comes with Python. The good news is
- that it's quite fast and efficient but it makes some things rather
+ with some additional methods for parsing XML results from AWS.
+ Because I don't really want any dependencies on external libraries,
+ I'm using the standard SAX parser that comes with Python. The good news is
+ that it's quite fast and efficient but it makes some things rather
difficult.
You can pass in, as the marker_elem parameter, a list of tuples.
@@ -114,6 +115,7 @@ def endElement(self, name, value, connection):
else:
setattr(self, name, value)
+
class BooleanResult(object):
def __init__(self, marker_elem=None):
@@ -154,4 +156,3 @@ def endElement(self, name, value, connection):
self.request_id = value
else:
setattr(self, name, value)
-
View
33 boto/storage_uri.py
@@ -20,7 +20,6 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
-import boto
import os
from boto.exception import BotoClientError
from boto.exception import InvalidUriError
@@ -64,12 +63,12 @@ def equals(self, uri):
def check_response(self, resp, level, uri):
if resp is None:
- raise InvalidUriError('Attempt to get %s for "%s" failed.\nThis '
- 'can happen if the URI refers to a non-'
- 'existent object or if you meant to\noperate '
- 'on a directory (e.g., leaving off -R option '
- 'on gsutil cp, mv, or ls of a\nbucket)' %
- (level, uri))
+ raise InvalidUriError('Attempt to get %s for "%s" failed.\n'
+ 'This can happen if the URI refers to'
+ 'a non-existent object or if you meant to\n'
+ 'operate on a directory (e.g., leaving off'
+ '-R option on gsutil cp, mv, or ls of a\n'
+ 'bucket)' % (level, uri))
def connect(self, access_key_id=None, secret_access_key=None, **kwargs):
"""
@@ -239,8 +238,9 @@ def clone_replace_name(self, new_name):
@param new_name: new object name
"""
if not self.bucket_name:
- raise InvalidUriError('clone_replace_name() on bucket-less URI %s' %
- self.uri)
+ msg = 'clone_replace_name() on bucket-less URI %s' % self.uri
+ raise InvalidUriError(msg)
+
return BucketStorageUri(
self.scheme, bucket_name=self.bucket_name, object_name=new_name,
debug=self.debug,
@@ -303,12 +303,12 @@ def get_subresource(self, subresource, validate=True, headers=None,
def add_group_email_grant(self, permission, email_address, recursive=False,
validate=True, headers=None):
if self.scheme != 'gs':
- raise ValueError('add_group_email_grant() not supported for %s '
- 'URIs.' % self.scheme)
+ raise ValueError('add_group_email_grant() not supported for %s '
+ 'URIs.' % self.scheme)
if self.object_name:
if recursive:
- raise ValueError('add_group_email_grant() on key-ful URI cannot '
- 'specify recursive=True')
+ raise ValueError('add_group_email_grant() on key-ful URI '
+ 'cannot specify recursive=True')
key = self.get_key(validate, headers)
self.check_response(key, 'key', self.uri)
key.add_group_email_grant(permission, email_address, headers)
@@ -452,8 +452,10 @@ def set_canned_acl(self, acl_str, validate=True, headers=None,
def set_def_canned_acl(self, acl_str, validate=True, headers=None,
version_id=None):
- """sets or updates a bucket's default object acl to a predefined
- (canned) value"""
+ """
+ Sets or updates a bucket's default object acl to a predefined
+ (canned) value.
+ """
if not self.object_name:
raise InvalidUriError('set_canned_acl on object-less URI (%s)' %
self.uri)
@@ -493,7 +495,6 @@ def disable_logging(self, validate=True, headers=None, version_id=None):
bucket.disable_logging(headers=headers)
-
class FileStorageUri(StorageUri):
"""
StorageUri subclass that handles files in the local file system.
View
200 boto/utils.py
@@ -16,7 +16,7 @@
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
-# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
@@ -38,7 +38,6 @@
Some handy utility functions used by several classes.
"""
-import imp
import subprocess
import time
import logging.handlers
@@ -60,29 +59,34 @@
_hashfn = md5.md5
# List of Query String Arguments of Interest
-qsa_of_interest = ['acl', 'cors', 'defaultObjectAcl', 'location', 'logging',
- 'partNumber', 'policy', 'requestPayment', 'torrent',
- 'versioning', 'versionId', 'versions', 'website',
- 'uploads', 'uploadId', 'response-content-type',
- 'response-content-language', 'response-expires',
+qsa_of_interest = ['acl', 'cors', 'defaultObjectAcl', 'location', 'logging',
+ 'partNumber', 'policy', 'requestPayment', 'torrent',
+ 'versioning', 'versionId', 'versions', 'website',
+ 'uploads', 'uploadId', 'response-content-type',
+ 'response-content-language', 'response-expires',
'response-cache-control', 'response-content-disposition',
'response-content-encoding', 'delete', 'lifecycle']
+
def unquote_v(nv):
if len(nv) == 1:
return nv
else:
return (nv[0], compat.unquote(nv[1]))
-# generates the aws canonical string for the given parameters
+
def canonical_string(method, path, headers, expires=None,
provider=None):
+ """
+ Generates the aws canonical string for the given parameters.
+ """
if not provider:
provider = boto.provider.get_default()
interesting_headers = {}
for key in headers:
lk = key.lower()
- if headers[key] != None and (lk in ['content-md5', 'content-type', 'date'] or
+ hdrs = ['content-md5', 'content-type', 'date']
+ if headers[key] != None and (lk in hdrs or
lk.startswith(provider.header_prefix)):
interesting_headers[lk] = headers[key].strip()
@@ -113,21 +117,22 @@ def canonical_string(method, path, headers, expires=None,
# don't include anything after the first ? in the resource...
# unless it is one of the QSA of interest, defined above
- t = path.split('?')
+ t = path.split('?')
buf += t[0]
if len(t) > 1:
qsa = t[1].split('&')
- qsa = [ a.split('=', 1) for a in qsa]
- qsa = [ unquote_v(a) for a in qsa if a[0] in qsa_of_interest ]
+ qsa = [a.split('=', 1) for a in qsa]
+ qsa = [unquote_v(a) for a in qsa if a[0] in qsa_of_interest]
if len(qsa) > 0:
- qsa.sort(cmp=lambda x,y:cmp(x[0], y[0]))
- qsa = [ '='.join(a) for a in qsa ]
+ qsa.sort(key=lambda x: x[0])
+ qsa = ['='.join(a) for a in qsa]
buf += '?'
buf += '&'.join(qsa)
return buf
+
def merge_meta(headers, metadata, provider=None):
if not provider:
provider = boto.provider.get_default()
@@ -143,6 +148,7 @@ def merge_meta(headers, metadata, provider=None):
return final_headers
+
def get_aws_metadata(headers, provider=None):
if not provider:
provider = boto.provider.get_default()
@@ -158,6 +164,7 @@ def get_aws_metadata(headers, provider=None):
del headers[hkey]
return metadata
+
def retry_url(url, retry_on_404=True, num_retries=10):
for i in range(0, num_retries):
try:
@@ -175,10 +182,11 @@ def retry_url(url, retry_on_404=True, num_retries=10):
except:
pass
boto.log.exception('Caught exception reading instance data')
- time.sleep(2**i)
+ time.sleep(2 ** i)
boto.log.error('Unable to read instance data, giving up')
return ''
+
def _get_instance_metadata(url):
d = {}
data = retry_url(url)
@@ -190,7 +198,7 @@ def _get_instance_metadata(url):
else:
p = field.find('=')
if p > 0:
- key = field[p+1:]
+ key = field[p + 1:]
resource = field[0:p] + '/openssh-key'
else:
key = resource = field
@@ -201,6 +209,7 @@ def _get_instance_metadata(url):
d[key] = val
return d
+
def get_instance_metadata(version='latest', url='http://169.254.169.254'):
"""
Returns the instance metadata as a nested Python dictionary.
@@ -211,9 +220,10 @@ def get_instance_metadata(version='latest', url='http://169.254.169.254'):
"""
return _get_instance_metadata('%s/%s/meta-data/' % (url, version))
+
def get_instance_userdata(version='latest', sep=None,
url='http://169.254.169.254'):
- ud_url = '%s/%s/user-data' % (url,version)
+ ud_url = '%s/%s/user-data' % (url, version)
user_data = retry_url(ud_url, retry_on_404=False)
if user_data:
if sep:
@@ -226,12 +236,14 @@ def get_instance_userdata(version='latest', sep=None,
ISO8601 = '%Y-%m-%dT%H:%M:%SZ'
ISO8601_MS = '%Y-%m-%dT%H:%M:%S.%fZ'
-
+
+
def get_ts(ts=None):
if not ts:
ts = time.gmtime()
return time.strftime(ISO8601, ts)
+
def parse_ts(ts):
ts = ts.strip()
try:
@@ -241,6 +253,7 @@ def parse_ts(ts):
dt = datetime.datetime.strptime(ts, ISO8601_MS)
return dt
+
def find_class(module_name, class_name=None):
if class_name:
module_name = "%s.%s" % (module_name, class_name)
@@ -256,7 +269,8 @@ def find_class(module_name, class_name=None):
return c
except:
return None
-
+
+
def update_dme(username, password, dme_id, ip_address):
"""
Update your Dynamic DNS record with DNSMadeEasy.com
@@ -266,11 +280,13 @@ def update_dme(username, password, dme_id, ip_address):
s = compat.urlopen(dme_url % (username, password, dme_id, ip_address))
return s.read()
+
def fetch_file(uri, file=None, username=None, password=None):
"""
- Fetch a file based on the URI provided. If you do not pass in a file pointer
- a tempfile.NamedTemporaryFile, or None if the file could not be
- retrieved is returned.
+ Fetch a file based on the URI provided. If you do not pass in a
+ file pointer a tempfile.NamedTemporaryFile, or None if the file could
+ not be retrieved is returned.
+
The URI can be either an HTTP url, or "s3://bucket_name/key_name"
"""
boto.log.info('Fetching %s' % uri)
@@ -279,7 +295,8 @@ def fetch_file(uri, file=None, username=None, password=None):
try:
if uri.startswith('s3://'):
bucket_name, key_name = uri[len('s3://'):].split('/', 1)
- c = boto.connect_s3(aws_access_key_id=username, aws_secret_access_key=password)
+ c = boto.connect_s3(aws_access_key_id=username,
+ aws_secret_access_key=password)
bucket = c.get_bucket(bucket_name)
key = bucket.get_key(key_name)
key.get_contents_to_file(file)
@@ -299,20 +316,23 @@ def fetch_file(uri, file=None, username=None, password=None):
file = None
return file
+
class ShellCommand(object):
- def __init__(self, command, wait=True, fail_fast=False, cwd = None):
+ def __init__(self, command, wait=True, fail_fast=False, cwd=None):
self.exit_code = 0
self.command = command
self.log_fp = compat.StringIO()
self.wait = wait
self.fail_fast = fail_fast
- self.run(cwd = cwd)
+ self.run(cwd=cwd)
def run(self, cwd=None):
boto.log.info('running:%s' % self.command)
- self.process = subprocess.Popen(self.command, shell=True, stdin=subprocess.PIPE,
- stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+ self.process = subprocess.Popen(self.command, shell=True,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
cwd=cwd)
if(self.wait):
while self.process.poll() == None:
@@ -324,7 +344,9 @@ def run(self, cwd=None):
self.exit_code = self.process.returncode
if self.fail_fast and self.exit_code != 0:
- raise Exception("Command " + self.command + " failed with status " + self.exit_code)
+ msg = "Command " + self.command
+ msg += " failed with status " + self.exit_code
+ raise Exception(msg)
return self.exit_code
@@ -334,12 +356,15 @@ def setReadOnly(self, value):
def getStatus(self):
return self.exit_code
- status = property(getStatus, setReadOnly, None, 'The exit code for the command')
+ status = property(getStatus, setReadOnly, None,
+ 'The exit code for the command')
def getOutput(self):
return self.log_fp.getvalue()
- output = property(getOutput, setReadOnly, None, 'The STDIN and STDERR output of the command')
+ output = property(getOutput, setReadOnly, None,
+ 'The STDIN and STDERR output of the command')
+
class AuthSMTPHandler(logging.handlers.SMTPHandler):
"""
@@ -347,25 +372,29 @@ class AuthSMTPHandler(logging.handlers.SMTPHandler):
to accept a username and password on the constructor and to then use those
credentials to authenticate with the SMTP server. To use this, you could
add something like this in your boto config file:
-
+
[handler_hand07]
class=boto.utils.AuthSMTPHandler
level=WARN
formatter=form07
- args=('localhost', 'username', 'password', 'from@abc', ['user1@abc', 'user2@xyz'], 'Logger Subject')
+ args=('localhost', 'username', 'password',
+ 'from@abc', ['user1@abc', 'user2@xyz'], 'Logger Subject')
"""
- def __init__(self, mailhost, username, password, fromaddr, toaddrs, subject):
+ def __init__(self, mailhost, username, password,
+ fromaddr, toaddrs, subject):
"""
Initialize the handler.
We have extended the constructor to accept a username/password
for SMTP authentication.
"""
- logging.handlers.SMTPHandler.__init__(self, mailhost, fromaddr, toaddrs, subject)
+ logging.handlers.SMTPHandler.__init__(self, mailhost,
+ fromaddr, toaddrs,
+ subject)
self.username = username
self.password = password
-
+
def emit(self, record):
"""
Emit a record.
@@ -393,32 +422,34 @@ def emit(self, record):
except:
self.handleError(record)
+
class LRUCache(dict):
- """A dictionary-like object that stores only a certain number of items, and
+ """
+ A dictionary-like object that stores only a certain number of items, and
discards its least recently used item when full.
-
+
>>> cache = LRUCache(3)
>>> cache['A'] = 0
>>> cache['B'] = 1
>>> cache['C'] = 2
>>> len(cache)
3
-
+
>>> cache['A']
0
-
- Adding new items to the cache does not increase its size. Instead, the least
- recently used item is dropped:
-
+
+ Adding new items to the cache does not increase its size. Instead,
+ the least recently used item is dropped:
+
>>> cache['D'] = 3
>>> len(cache)
3
>>> 'B' in cache
False
-
+
Iterating over the cache returns the keys, starting with the most recently
used:
-
+
>>> for key in cache:
... print key
D
@@ -430,14 +461,16 @@ class LRUCache(dict):
by Mike Bayer and released under the MIT license (Genshi uses the
BSD License). See:
- http://svn.myghty.org/myghtyutils/trunk/lib/myghtyutils/util.py
+ http://svn.myghty.org/myghtyutils/trunk/lib/myghtyutils/util.py
"""
class _Item(object):
+
def __init__(self, key, value):
self.previous = self.next = None
self.key = key
self.value = value
+
def __repr__(self):
return repr(self.value)
@@ -512,26 +545,29 @@ def _update_item(self, item):
item.next = self.head
self.head.previous = self.head = item
+
class Password(object):
"""
Password object that stores itself as hashed.
Hash defaults to SHA512 if available, MD5 otherwise.
"""
- hashfunc=_hashfn
+ hashfunc = _hashfn
+
def __init__(self, str=None, hashfunc=None):
"""
- Load the string from an initial value, this should be the raw hashed password.
+ Load the string from an initial value, this should be the raw
+ hashed password.
"""
self.str = str
if hashfunc:
- self.hashfunc = hashfunc
+ self.hashfunc = hashfunc
def set(self, value):
self.str = self.hashfunc(value).hexdigest()
-
+
def __str__(self):
return str(self.str)
-
+
def __eq__(self, other):
if other == None:
return False
@@ -543,22 +579,28 @@ def __len__(self):
else:
return 0
-def notify(subject, body=None, html_body=None, to_string=None, attachments=None, append_instance_id=True):
+
+def notify(subject, body=None, html_body=None, to_string=None,
+ attachments=None, append_instance_id=True):
attachments = attachments or []
if append_instance_id:
- subject = "[%s] %s" % (boto.config.get_value("Instance", "instance-id"), subject)
+ subject = "[%s] %s" % (boto.config.get_value("Instance",
+ "instance-id"),
+ subject)
if not to_string:
to_string = boto.config.get_value('Notification', 'smtp_to', None)
if to_string:
try:
- from_string = boto.config.get_value('Notification', 'smtp_from', 'boto')
+ from_string = boto.config.get_value('Notification',
+ 'smtp_from',
+ 'boto')
msg = compat.MIMEMultipart()
msg['From'] = from_string
msg['Reply-To'] = from_string
msg['To'] = to_string
msg['Date'] = compat.formatdate(localtime=True)
msg['Subject'] = subject
-
+
if body:
msg.attach(compat.MIMEText(body))
@@ -571,11 +613,15 @@ def notify(subject, body=None, html_body=None, to_string=None, attachments=None,
for part in attachments:
msg.attach(part)
- smtp_host = boto.config.get_value('Notification', 'smtp_host', 'localhost')
+ smtp_host = boto.config.get_value('Notification',
+ 'smtp_host',
+ 'localhost')
# Alternate port support
if boto.config.get_value("Notification", "smtp_port"):
- server = smtplib.SMTP(smtp_host, int(boto.config.get_value("Notification", "smtp_port")))
+ server = smtplib.SMTP(smtp_host,
+ int(boto.config.get_value("Notification",
+ "smtp_port")))
else:
server = smtplib.SMTP(smtp_host)
@@ -593,6 +639,7 @@ def notify(subject, body=None, html_body=None, to_string=None, attachments=None,
except:
boto.log.exception('notify failed')
+
def get_utf8_value(value):
if not isinstance(value, compat.string_types) and not isinstance(value, compat.text_type):
value = str(value)
@@ -601,6 +648,7 @@ def get_utf8_value(value):
else:
return value
+
def mklist(value):
if not isinstance(value, list):
if isinstance(value, tuple):
@@ -609,6 +657,7 @@ def mklist(value):
value = [value]
return value
+
def pythonize_name(name, sep='_'):
s = ''
if name[0].isupper:
@@ -620,26 +669,31 @@ def pythonize_name(name, sep='_'):
s += c
return s
-def write_mime_multipart(content, compress=False, deftype='text/plain', delimiter=':'):
+
+def write_mime_multipart(content, compress=False,
+ deftype='text/plain',
+ delimiter=':'):
"""Description:
:param content: A list of tuples of name-content pairs. This is used
instead of a dict to ensure that scripts run in order
:type list of tuples:
- :param compress: Use gzip to compress the scripts, defaults to no compression
- :type bool:
+ :type compress: bool
+ :param compress: Use gzip to compress the scripts, defaults to
+ no compression
- :param deftype: The type that should be assumed if nothing else can be figured out
- :type str:
+ :type deftype: str
+ :param deftype: The type that should be assumed if nothing else
+ can be figured out
+ :type delimiter: str
:param delimiter: mime delimiter
- :type str:
+ :rtype: str
:return: Final mime multipart
- :rtype: str:
"""
wrapper = compat.MIMEMultipart()
- for name,con in content:
+ for name, con in content:
definite_type = guess_mime_type(con, deftype)
maintype, subtype = definite_type.split('/', 1)
if maintype == 'text':
@@ -664,6 +718,7 @@ def write_mime_multipart(content, compress=False, deftype='text/plain', delimite
return rcontent
+
def guess_mime_type(content, deftype):
"""Description: Guess the mime type of a block of text
:param content: content we're finding the type of
@@ -676,21 +731,22 @@ def guess_mime_type(content, deftype):
:return: <description>
"""
#Mappings recognized by cloudinit
- starts_with_mappings={
- '#include' : 'text/x-include-url',
- '#!' : 'text/x-shellscript',
- '#cloud-config' : 'text/cloud-config',
- '#upstart-job' : 'text/upstart-job',
- '#part-handler' : 'text/part-handler',
- '#cloud-boothook' : 'text/cloud-boothook'
+ starts_with_mappings = {
+ '#include': 'text/x-include-url',
+ '#!': 'text/x-shellscript',
+ '#cloud-config': 'text/cloud-config',
+ '#upstart-job': 'text/upstart-job',
+ '#part-handler': 'text/part-handler',
+ '#cloud-boothook': 'text/cloud-boothook'
}
rtype = deftype
- for possible_type,mimetype in starts_with_mappings.items():
+ for possible_type, mimetype in starts_with_mappings.items():
if content.startswith(possible_type):
rtype = mimetype
break
return(rtype)
+
def compute_md5(fp, buf_size=8192, size=None):
"""
Compute MD5 hash on passed file and return results in a tuple of values.
Please sign in to comment.
Something went wrong with that request. Please try again.