From fe2be48deb4f835ec999e236cce173244fbd74ef Mon Sep 17 00:00:00 2001 From: Alon Swartz Date: Tue, 10 Aug 2010 12:18:42 +0300 Subject: [PATCH] updated to boto 1.9b-4 (from Debian Sid) - removed ._.py binary files - removed packaging files --- bin/list_instances | 5 +- bin/sdbadmin | 12 +- boto/__init__.py | 14 - boto/cloudfront/__init__.py | 5 +- boto/cloudfront/distribution.py | 4 +- boto/cloudfront/identity.py | 28 +- boto/connection.py | 13 +- boto/contrib/ymlmessage.py | 30 +- boto/ec2/address.py | 13 +- boto/ec2/autoscale/__init__.py | 3 + boto/ec2/autoscale/group.py | 1 + boto/ec2/autoscale/instance.py | 12 +- boto/ec2/blockdevicemapping.py | 31 +- boto/ec2/buyreservation.py | 2 +- boto/ec2/cloudwatch/__init__.py | 17 +- boto/ec2/connection.py | 78 +++-- boto/ec2/elb/loadbalancer.py | 2 + boto/ec2/image.py | 11 +- boto/ec2/instance.py | 10 +- boto/ec2/securitygroup.py | 7 +- boto/ec2/snapshot.py | 5 +- boto/ec2/spotinstancerequest.py | 7 +- boto/ec2/volume.py | 11 +- boto/exception.py | 9 +- boto/fps/connection.py | 257 +++++++------- boto/manage/cmdshell.py | 8 +- boto/manage/propget.py | 2 + boto/manage/server.py | 24 +- boto/manage/task.py | 16 +- boto/manage/volume.py | 9 +- boto/mapreduce/partitiondb.py | 23 +- boto/mapreduce/queuetools.py | 2 +- boto/mashups/server.py | 15 +- boto/mturk/connection.py | 27 +- boto/mturk/question.py | 51 ++- boto/pyami/bootstrap.py | 2 +- boto/pyami/config.py | 1 + boto/pyami/installers/__init__.py | 12 +- boto/pyami/installers/ubuntu/ebs.py | 5 +- boto/pyami/installers/ubuntu/installer.py | 2 +- boto/pyami/installers/ubuntu/trac.py | 196 ++++++----- boto/pyami/launch_ami.py | 6 +- boto/pyami/scriptbase.py | 4 +- boto/pyami/startup.py | 6 +- boto/rds/__init__.py | 12 +- boto/rds/dbsecuritygroup.py | 4 +- boto/resultset.py | 6 - boto/s3/acl.py | 3 +- boto/s3/bucket.py | 388 +++++----------------- boto/s3/bucketlistresultset.py | 42 --- boto/s3/connection.py | 19 +- boto/s3/key.py | 72 +--- boto/sdb/__init__.py | 2 +- boto/sdb/connection.py | 3 +- boto/sdb/db/blob.py | 2 +- boto/sdb/db/manager/__init__.py | 2 - boto/sdb/db/manager/pgmanager.py | 10 +- boto/sdb/db/manager/sdbmanager.py | 37 +-- boto/sdb/db/manager/xmlmanager.py | 2 + boto/sdb/db/model.py | 3 +- boto/sdb/db/property.py | 16 +- boto/sdb/db/test_db.py | 5 +- boto/sdb/domain.py | 14 +- boto/sdb/item.py | 8 - boto/sdb/persist/checker.py | 1 + boto/sdb/persist/object.py | 2 +- boto/sdb/persist/property.py | 3 +- boto/sdb/persist/test_persist.py | 5 +- boto/sdb/queryresultset.py | 1 + boto/services/bs.py | 1 + boto/services/message.py | 1 + boto/services/result.py | 11 +- boto/services/service.py | 6 +- boto/services/sonofmmm.py | 3 +- boto/services/submit.py | 5 +- boto/sqs/__init__.py | 3 +- boto/sqs/connection.py | 32 +- boto/sqs/queue.py | 5 +- boto/tests/test.py | 4 - boto/tests/test_ec2connection.py | 2 +- boto/tests/test_s3connection.py | 8 +- boto/tests/test_sqsconnection.py | 4 +- boto/utils.py | 37 +-- boto/vpc/__init__.py | 11 +- boto/vpc/vpnconnection.py | 2 +- docs/fabfile.py | 33 ++ docs/source/conf-orig.py | 206 ++++++++++++ docs/source/index.rst | 32 +- docs/source/ref/index.rst | 1 - docs/source/vpc_tut.rst | 2 +- setup.py | 7 +- 91 files changed, 917 insertions(+), 1159 deletions(-) create mode 100644 docs/fabfile.py create mode 100644 docs/source/conf-orig.py diff --git a/bin/list_instances b/bin/list_instances index 19e1c9b..56ad53b 100755 --- a/bin/list_instances +++ b/bin/list_instances @@ -2,9 +2,8 @@ import boto ec2 = boto.connect_ec2() -print "%-15s %-15s %-30s %s" % ("ID", 'Zone', "Groups", "Hostname") -print "-"*105 +print "%-20s %-30s %s" % ("ID", "Groups", "Hostname") for r in ec2.get_all_instances(): groups = [g.id for g in r.groups] for i in r.instances: - print "%-15s %-15s %-30s %s" % (i.id, i.placement, ','.join(groups), i.public_dns_name) + print "%-20s %-30s %s" % (i.id, ','.join(groups), i.public_dns_name) diff --git a/bin/sdbadmin b/bin/sdbadmin index e8ff9b5..24c2f69 100755 --- a/bin/sdbadmin +++ b/bin/sdbadmin @@ -25,7 +25,6 @@ VERSION = "%prog version 1.0" import boto import time -from boto import sdb def choice_input(options, default=None, title=None): """ @@ -74,13 +73,13 @@ def load_db(domain, file): """ domain.from_xml(file) -def create_db(domain_name, region_name): +def create_db(domain_name): """Create a new DB :param domain: Name of the domain to create :type domain: str """ - sdb = boto.sdb.connect_to_region(region_name) + sdb = boto.connect_sdb() return sdb.create_domain(domain_name) if __name__ == "__main__": @@ -97,15 +96,14 @@ if __name__ == "__main__": parser.add_option("-a", "--all-domains", help="Operate on all domains", action="store_true", default=False, dest="all_domains") parser.add_option("-d", "--domain", help="Do functions on domain (may be more then one)", action="append", dest="domains") parser.add_option("-f", "--file", help="Input/Output file we're operating on", dest="file_name") - parser.add_option("-r", "--region", help="Region (e.g. us-east-1[default] or eu-west-1)", default="us-east-1", dest="region_name") + (options, args) = parser.parse_args() if options.create: - for domain_name in options.domains: - create_db(domain_name, options.region_name) + create_db(*args) exit() - sdb = boto.sdb.connect_to_region(options.region_name) + sdb = boto.connect_sdb() if options.list: for db in sdb.get_all_domains(): print db diff --git a/boto/__init__.py b/boto/__init__.py index fc69327..fc2e592 100644 --- a/boto/__init__.py +++ b/boto/__init__.py @@ -224,20 +224,6 @@ def connect_rds(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): from boto.rds import RDSConnection return RDSConnection(aws_access_key_id, aws_secret_access_key, **kwargs) -def connect_emr(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): - """ - :type aws_access_key_id: string - :param aws_access_key_id: Your AWS Access Key ID - - :type aws_secret_access_key: string - :param aws_secret_access_key: Your AWS Secret Access Key - - :rtype: :class:`boto.emr.EmrConnection` - :return: A connection to Elastic mapreduce - """ - from boto.emr import EmrConnection - return EmrConnection(aws_access_key_id, aws_secret_access_key, **kwargs) - def check_extensions(module_name, module_path): """ This function checks for extensions to boto modules. It should be called in the diff --git a/boto/cloudfront/__init__.py b/boto/cloudfront/__init__.py index 28309ff..d03d6eb 100644 --- a/boto/cloudfront/__init__.py +++ b/boto/cloudfront/__init__.py @@ -23,12 +23,11 @@ import xml.sax import base64 import time +import boto.utils from boto.connection import AWSAuthConnection from boto import handler -from boto.cloudfront.distribution import Distribution, DistributionSummary, DistributionConfig -from boto.cloudfront.distribution import StreamingDistribution, StreamingDistributionSummary, StreamingDistributionConfig +from boto.cloudfront.distribution import * from boto.cloudfront.identity import OriginAccessIdentity -from boto.cloudfront.identity import OriginAccessIdentitySummary from boto.cloudfront.identity import OriginAccessIdentityConfig from boto.resultset import ResultSet from boto.cloudfront.exception import CloudFrontServerError diff --git a/boto/cloudfront/distribution.py b/boto/cloudfront/distribution.py index ead6e36..cd36add 100644 --- a/boto/cloudfront/distribution.py +++ b/boto/cloudfront/distribution.py @@ -22,7 +22,7 @@ import uuid from boto.cloudfront.identity import OriginAccessIdentity from boto.cloudfront.object import Object, StreamingObject -from boto.cloudfront.signers import ActiveTrustedSigners, TrustedSigners +from boto.cloudfront.signers import Signer, ActiveTrustedSigners, TrustedSigners from boto.cloudfront.logging import LoggingInfo from boto.s3.acl import ACL @@ -69,7 +69,7 @@ def to_xml(self): s += '\n' if self.origin_access_identity: val = self.get_oai_value() - s += '%s\n' % val + s += '%s\n' % val if self.trusted_signers: s += '\n' for signer in self.trusted_signers: diff --git a/boto/cloudfront/identity.py b/boto/cloudfront/identity.py index 3bc5cfd..711b8b7 100644 --- a/boto/cloudfront/identity.py +++ b/boto/cloudfront/identity.py @@ -50,7 +50,7 @@ def endElement(self, name, value, connection): setattr(self, name, value) def update(self, comment=None): - new_config = OriginAccessIdentityConfig(self.connection, + new_config = OriginAccessIdentifyConfig(self.connection, self.config.caller_reference, self.config.comment) if comment != None: @@ -62,7 +62,7 @@ def delete(self): return self.connection.delete_distribution(self.id, self.etag) def uri(self): - return 'origin-access-identity/cloudfront/%s' % self.id + return 'origin-access-identity/cloudfront/%s' % id class OriginAccessIdentityConfig: @@ -94,29 +94,5 @@ def endElement(self, name, value, connection): else: setattr(self, name, value) -class OriginAccessIdentitySummary: - def __init__(self, connection=None, id='', - s3_user_id='', comment=''): - self.connection = connection - self.id = id - self.s3_user_id = s3_user_id - self.comment = comment - self.etag = None - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'Id': - self.id = value - elif name == 'S3CanonicalUserId': - self.s3_user_id = value - elif name == 'Comment': - self.comment = value - else: - setattr(self, name, value) - - def get_origin_access_identity(self): - return self.connection.get_origin_access_identity_info(self.id) - diff --git a/boto/connection.py b/boto/connection.py index 4f321e1..9a443f7 100644 --- a/boto/connection.py +++ b/boto/connection.py @@ -50,7 +50,7 @@ import xml.sax import Queue import boto -from boto.exception import BotoClientError, BotoServerError +from boto.exception import AWSConnectionError, BotoClientError, BotoServerError from boto.resultset import ResultSet import boto.utils from boto import config, UserAgent, handler @@ -477,12 +477,11 @@ def close(self): boto.log.debug('closing all HTTP connections') self.connection = None # compat field - if hasattr(self, '_cache') and isinstance(self._cache, dict): - hosts = list(self._cache.keys()) - for host in hosts: - conn = self._cache[host] - conn.close() - del self._cache[host] + hosts = list(self._cache.keys()) + for host in hosts: + conn = self._cache[host] + conn.close() + del self._cache[host] class AWSQueryConnection(AWSAuthConnection): diff --git a/boto/contrib/ymlmessage.py b/boto/contrib/ymlmessage.py index b9a2c93..22e5c62 100644 --- a/boto/contrib/ymlmessage.py +++ b/boto/contrib/ymlmessage.py @@ -29,24 +29,24 @@ import yaml class YAMLMessage(Message): - """ - The YAMLMessage class provides a YAML compatible message. Encoding and - decoding are handled automaticaly. + """ + The YAMLMessage class provides a YAML compatible message. Encoding and + decoding are handled automaticaly. - Access this message data like such: + Access this message data like such: - m.data = [ 1, 2, 3] - m.data[0] # Returns 1 + m.data = [ 1, 2, 3] + m.data[0] # Returns 1 - This depends on the PyYAML package - """ + This depends on the PyYAML package + """ - def __init__(self, queue=None, body='', xml_attrs=None): - self.data = None - Message.__init__(self, queue, body) + def __init__(self, queue=None, body='', xml_attrs=None): + self.data = None + Message.__init__(self, queue, body) - def set_body(self, body): - self.data = yaml.load(body) + def set_body(self, body): + self.data = yaml.load(body) - def get_body(self): - return yaml.dump(self.data) + def get_body(self): + return yaml.dump(self.data) diff --git a/boto/ec2/address.py b/boto/ec2/address.py index 60ed406..b2af107 100644 --- a/boto/ec2/address.py +++ b/boto/ec2/address.py @@ -1,4 +1,4 @@ -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the @@ -44,15 +44,8 @@ def endElement(self, name, value, connection): else: setattr(self, name, value) - def release(self): - return self.connection.release_address(self.public_ip) + def delete(self): + return self.connection.delete_address(self.public_ip) - delete = release - def associate(self, instance_id): - return self.connection.associate_address(instance_id, self.public_ip) - def disassociate(self): - return self.connection.disassociate_address(self.public_ip) - - diff --git a/boto/ec2/autoscale/__init__.py b/boto/ec2/autoscale/__init__.py index a06781f..d7c5946 100644 --- a/boto/ec2/autoscale/__init__.py +++ b/boto/ec2/autoscale/__init__.py @@ -25,7 +25,10 @@ """ import boto +from boto import config from boto.connection import AWSQueryConnection +from boto.resultset import ResultSet +from boto.ec2.regioninfo import RegionInfo from boto.ec2.autoscale.request import Request from boto.ec2.autoscale.trigger import Trigger from boto.ec2.autoscale.launchconfig import LaunchConfiguration diff --git a/boto/ec2/autoscale/group.py b/boto/ec2/autoscale/group.py index 3fa6d68..d9df39f 100644 --- a/boto/ec2/autoscale/group.py +++ b/boto/ec2/autoscale/group.py @@ -21,6 +21,7 @@ import weakref +from boto.ec2.zone import Zone from boto.ec2.elb.listelement import ListElement from boto.resultset import ResultSet from boto.ec2.autoscale.trigger import Trigger diff --git a/boto/ec2/autoscale/instance.py b/boto/ec2/autoscale/instance.py index 2e9ae46..33f2ae6 100644 --- a/boto/ec2/autoscale/instance.py +++ b/boto/ec2/autoscale/instance.py @@ -43,11 +43,9 @@ def endElement(self, name, value, connection): else: setattr(self, name, value) - # BUG: self.get_object is not defined - # BUG: Request is not defined - # def terminate(self): - # """ Terminate this instance. """ - # params = {'LaunchConfigurationName' : self.instance_id} - # return self.get_object('DeleteLaunchConfiguration', params, - # Request) + def terminate(self): + """ Terminate this instance. """ + params = {'LaunchConfigurationName' : self.instance_id} + return self.get_object('DeleteLaunchConfiguration', params, + Request) diff --git a/boto/ec2/blockdevicemapping.py b/boto/ec2/blockdevicemapping.py index f315fe9..ef7163a 100644 --- a/boto/ec2/blockdevicemapping.py +++ b/boto/ec2/blockdevicemapping.py @@ -20,11 +20,10 @@ # IN THE SOFTWARE. # -class BlockDeviceType(object): +class EBSBlockDeviceType(object): def __init__(self, connection=None): self.connection = connection - self.ephemeral_name = None self.volume_id = None self.snapshot_id = None self.status = None @@ -38,8 +37,6 @@ def startElement(self, name, attrs, connection): def endElement(self, name, value, connection): if name =='volumeId': self.volume_id = value - elif name == 'virtualName': - self.ephemeral_name = value elif name =='snapshotId': self.snapshot_id = value elif name == 'volumeSize': @@ -56,9 +53,6 @@ def endElement(self, name, value, connection): else: setattr(self, name, value) -# for backwards compatibility -EBSBlockDeviceType = BlockDeviceType - class BlockDeviceMapping(dict): def __init__(self, connection=None): @@ -69,7 +63,7 @@ def __init__(self, connection=None): def startElement(self, name, attrs, connection): if name == 'ebs': - self.current_value = BlockDeviceType(self) + self.current_value = EBSBlockDeviceType(self) return self.current_value def endElement(self, name, value, connection): @@ -81,18 +75,15 @@ def endElement(self, name, value, connection): def build_list_params(self, params, prefix=''): i = 1 for dev_name in self: - pre = '%sBlockDeviceMapping.%d' % (prefix, i) + pre = '%sBlockDeviceMapping.%d' % (pre, i) params['%s.DeviceName' % pre] = dev_name - block_dev = self[dev_name] - if block_dev.ephemeral_name: - params['%s.VirtualName' % pre] = block_dev.ephemeral_name + ebs = self[dev_name] + if ebs.snapshot_id: + params['%s.Ebs.SnapshotId' % pre] = ebs.snapshot_id + if ebs.size: + params['%s.Ebs.VolumeSize' % pre] = ebs.size + if ebs.delete_on_termination: + params['%s.Ebs.DeleteOnTermination' % pre] = 'true' else: - if block_dev.snapshot_id: - params['%s.Ebs.SnapshotId' % pre] = block_dev.snapshot_id - if block_dev.size: - params['%s.Ebs.VolumeSize' % pre] = block_dev.size - if block_dev.delete_on_termination: - params['%s.Ebs.DeleteOnTermination' % pre] = 'true' - else: - params['%s.Ebs.DeleteOnTermination' % pre] = 'false' + params['%s.Ebs.DeleteOnTermination' % pre] = 'false' i += 1 diff --git a/boto/ec2/buyreservation.py b/boto/ec2/buyreservation.py index 45b21a1..ba65590 100644 --- a/boto/ec2/buyreservation.py +++ b/boto/ec2/buyreservation.py @@ -20,7 +20,7 @@ # IN THE SOFTWARE. import boto.ec2 -from boto.sdb.db.property import StringProperty, IntegerProperty +from boto.sdb.db.property import * from boto.manage import propget InstanceTypes = ['m1.small', 'm1.large', 'm1.xlarge', 'c1.medium', 'c1.xlarge'] diff --git a/boto/ec2/cloudwatch/__init__.py b/boto/ec2/cloudwatch/__init__.py index 1cb8719..1c606a1 100644 --- a/boto/ec2/cloudwatch/__init__.py +++ b/boto/ec2/cloudwatch/__init__.py @@ -140,6 +140,7 @@ from boto.ec2.cloudwatch.metric import Metric from boto.ec2.cloudwatch.datapoint import Datapoint import boto +import datetime class CloudWatchConnection(AWSQueryConnection): @@ -193,21 +194,13 @@ def get_metric_statistics(self, period, start_time, end_time, measure_name, self.build_list_params(params, statistics, 'Statistics.member.%d') return self.get_list('GetMetricStatistics', params, [('member', Datapoint)]) - def list_metrics(self, next_token=None): + def list_metrics(self): """ Returns a list of the valid metrics for which there is recorded data available. - - :type next_token: string - :param next_token: A maximum of 500 metrics will be returned at one time. - If more results are available, the ResultSet returned - will contain a non-Null next_token attribute. Passing - that token as a parameter to list_metrics will retrieve - the next page of metrics. """ - params = {} - if next_token: - params['NextToken'] = next_token - return self.get_list('ListMetrics', params, [('member', Metric)]) + response = self.make_request('ListMetrics') + body = response.read() + return self.get_list('ListMetrics', None, [('member', Metric)]) diff --git a/boto/ec2/connection.py b/boto/ec2/connection.py index cc0c005..e14c9bc 100644 --- a/boto/ec2/connection.py +++ b/boto/ec2/connection.py @@ -24,8 +24,10 @@ """ import urllib +import xml.sax import base64 import boto +from boto import config from boto.connection import AWSQueryConnection from boto.resultset import ResultSet from boto.ec2.image import Image, ImageAttribute @@ -43,6 +45,7 @@ from boto.ec2.spotinstancerequest import SpotInstanceRequest from boto.ec2.spotpricehistory import SpotPriceHistory from boto.ec2.spotdatafeedsubscription import SpotDatafeedSubscription +from boto.ec2.launchspecification import LaunchSpecification from boto.exception import EC2ResponseError #boto.set_stream_logger('ec2') @@ -564,52 +567,61 @@ def get_instance_attribute(self, instance_id, attribute): params['Attribute'] = attribute return self.get_object('DescribeInstanceAttribute', params, InstanceAttribute) - def modify_instance_attribute(self, instance_id, attribute, value): + def modify_image_attribute(self, image_id, attribute='launchPermission', + operation='add', user_ids=None, groups=None, + product_codes=None): """ - Changes an attribute of an instance + Changes an attribute of an image. + See http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/ApiReference-query-ModifyImageAttribute.html - :type instance_id: string - :param instance_id: The instance id you wish to change + :type image_id: string + :param image_id: The image id you wish to change :type attribute: string - :param attribute: The attribute you wish to change. - AttributeName - Expected value (default) - instanceType - A valid instance type (m1.small) - kernel - Kernel ID (None) - ramdisk - Ramdisk ID (None) - userData - Base64 encoded String (None) - disableApiTermination - Boolean (true) - instanceInitiatedShutdownBehavior - stop|terminate - rootDeviceName - device name (None) - - :type value: string - :param value: The new value for the attribute + :param attribute: The attribute you wish to change - :rtype: bool - :return: Whether the operation succeeded or not + :type operation: string + :param operation: Either add or remove (this is required for changing launchPermissions) + + :type user_ids: list + :param user_ids: The Amazon IDs of users to add/remove attributes + + :type groups: list + :param groups: The groups to add/remove attributes + + :type product_codes: list + :param product_codes: Amazon DevPay product code. Currently only one + product code can be associated with an AMI. Once + set, the product code cannot be changed or reset. """ - params = {'InstanceId' : instance_id, + params = {'ImageId' : image_id, 'Attribute' : attribute, - 'Value' : value} - return self.get_status('ModifyInstanceAttribute', params) + 'OperationType' : operation} + if user_ids: + self.build_list_params(params, user_ids, 'UserId') + if groups: + self.build_list_params(params, groups, 'UserGroup') + if product_codes: + self.build_list_params(params, product_codes, 'ProductCode') + return self.get_status('ModifyImageAttribute', params) - def reset_instance_attribute(self, instance_id, attribute): + def reset_image_attribute(self, image_id, attribute='launchPermission'): """ - Resets an attribute of an instance to its default value. + Resets an attribute of an AMI to its default value. + See http://docs.amazonwebservices.com/AWSEC2/2008-02-01/DeveloperGuide/ApiReference-Query-ResetImageAttribute.html - :type instance_id: string - :param instance_id: ID of the instance + :type image_id: string + :param image_id: ID of the AMI for which an attribute will be described :type attribute: string - :param attribute: The attribute to reset. Valid values are: - kernel|ramdisk + :param attribute: The attribute to reset :rtype: bool :return: Whether the operation succeeded or not """ - params = {'InstanceId' : instance_id, + params = {'ImageId' : image_id, 'Attribute' : attribute} - return self.get_status('ResetInstanceAttribute', params) + return self.get_status('ResetImageAttribute', params) # Spot Instances @@ -804,7 +816,7 @@ def get_spot_datafeed_subscription(self): Return the current spot instance data feed subscription associated with this account, if any. - :rtype: :class:`boto.ec2.spotdatafeedsubscription.SpotDatafeedSubscription` + :rtype: :class:`boto.ec2.spotdatafeedsubscription.SpotDatafeedSubscription :return: The datafeed subscription object or None """ return self.get_object('DescribeSpotDatafeedSubscription', @@ -816,15 +828,13 @@ def create_spot_datafeed_subscription(self, bucket, prefix): :type bucket: str or unicode :param bucket: The name of the bucket where spot instance data - will be written. The account issuing this request - must have FULL_CONTROL access to the bucket - specified in the request. + will be written. :type prefix: str or unicode :param prefix: An optional prefix that will be pre-pended to all data files written to the bucket. - :rtype: :class:`boto.ec2.spotdatafeedsubscription.SpotDatafeedSubscription` + :rtype: :class:`boto.ec2.spotdatafeedsubscription.SpotDatafeedSubscription :return: The datafeed subscription object or None """ params = {'Bucket' : bucket} diff --git a/boto/ec2/elb/loadbalancer.py b/boto/ec2/elb/loadbalancer.py index 0a90389..2902107 100644 --- a/boto/ec2/elb/loadbalancer.py +++ b/boto/ec2/elb/loadbalancer.py @@ -20,8 +20,10 @@ # IN THE SOFTWARE. from boto.ec2.elb.healthcheck import HealthCheck +from boto.ec2.elb.instancestate import InstanceState from boto.ec2.elb.listener import Listener from boto.ec2.elb.listelement import ListElement +from boto.ec2.zone import Zone from boto.ec2.instanceinfo import InstanceInfo from boto.resultset import ResultSet diff --git a/boto/ec2/image.py b/boto/ec2/image.py index c9b7fec..8ef2513 100644 --- a/boto/ec2/image.py +++ b/boto/ec2/image.py @@ -116,8 +116,7 @@ def run(self, min_count=1, max_count=1, key_name=None, security_groups=None, user_data=None, addressing_type=None, instance_type='m1.small', placement=None, kernel_id=None, ramdisk_id=None, - monitoring_enabled=False, subnet_id=None, - block_device_map=None): + monitoring_enabled=False, subnet_id=None): """ Runs this instance. @@ -157,11 +156,6 @@ def run(self, min_count=1, max_count=1, key_name=None, :type subnet_id: string :param subnet_id: The subnet ID within which to launch the instances for VPC. - :type block_device_map: :class:`boto.ec2.blockdevicemapping.BlockDeviceMapping` - :param block_device_map: A BlockDeviceMapping data structure - describing the EBS volumes associated - with the Image. - :rtype: Reservation :return: The :class:`boto.ec2.instance.Reservation` associated with the request for machines """ @@ -170,8 +164,7 @@ def run(self, min_count=1, max_count=1, key_name=None, user_data, addressing_type, instance_type, placement, kernel_id, ramdisk_id, - monitoring_enabled, subnet_id, - block_device_map) + monitoring_enabled, subnet_id) def deregister(self): return self.connection.deregister_image(self.id) diff --git a/boto/ec2/instance.py b/boto/ec2/instance.py index 421f576..5932c4e 100644 --- a/boto/ec2/instance.py +++ b/boto/ec2/instance.py @@ -207,16 +207,8 @@ def update(self): self._update(rs[0].instances[0]) return self.state - def terminate(self): - rs = self.connection.terminate_instances([self.id]) - self._update(rs[0]) - def stop(self): - rs = self.connection.stop_instances([self.id]) - self._update(rs[0]) - - def start(self): - rs = self.connection.start_instances([self.id]) + rs = self.connection.terminate_instances([self.id]) self._update(rs[0]) def reboot(self): diff --git a/boto/ec2/securitygroup.py b/boto/ec2/securitygroup.py index 61b0a00..6f17ad3 100644 --- a/boto/ec2/securitygroup.py +++ b/boto/ec2/securitygroup.py @@ -23,7 +23,6 @@ Represents an EC2 Security Group """ from boto.ec2.ec2object import EC2Object -from boto.exception import BotoClientError class SecurityGroup(EC2Object): @@ -61,9 +60,9 @@ def endElement(self, name, value, connection): self.status = True else: raise Exception( - 'Unexpected value of status %s for group %s'%( + 'Unexpected value of status %s for image %s'%( value, - self.name + self.id ) ) else: @@ -246,7 +245,7 @@ def endElement(self, name, value, connection): else: setattr(self, name, value) - def add_grant(self, name=None, owner_id=None, cidr_ip=None): + def add_grant(self, owner_id=None, name=None, cidr_ip=None): grant = GroupOrCIDR(self) grant.owner_id = owner_id grant.name = name diff --git a/boto/ec2/snapshot.py b/boto/ec2/snapshot.py index 3d4398e..33b53b0 100644 --- a/boto/ec2/snapshot.py +++ b/boto/ec2/snapshot.py @@ -52,10 +52,7 @@ def endElement(self, name, value, connection): elif name == 'ownerId': self.owner_id = value elif name == 'volumeSize': - try: - self.volume_size = int(value) - except: - self.volume_size = value + self.volume_size = int(value) elif name == 'description': self.description = value else: diff --git a/boto/ec2/spotinstancerequest.py b/boto/ec2/spotinstancerequest.py index 3014c7a..5b1d7ce 100644 --- a/boto/ec2/spotinstancerequest.py +++ b/boto/ec2/spotinstancerequest.py @@ -40,9 +40,9 @@ def startElement(self, name, attrs, connection): def endElement(self, name, value, connection): if name == 'code': - self.code = value + self.code = code elif name == 'message': - self.message = value + self.message = message setattr(self, name, value) class SpotInstanceRequest(EC2Object): @@ -61,7 +61,6 @@ def __init__(self, connection=None): self.availability_zone_group = None self.create_time = None self.launch_specification = None - self.instance_id = None def __repr__(self): return 'SpotInstanceRequest:%s' % self.id @@ -97,8 +96,6 @@ def endElement(self, name, value, connection): self.availability_zone_group = value elif name == 'createTime': self.create_time = value - elif name == 'instanceId': - self.instance_id = value else: setattr(self, name, value) diff --git a/boto/ec2/volume.py b/boto/ec2/volume.py index 7e5cb1c..200ca90 100644 --- a/boto/ec2/volume.py +++ b/boto/ec2/volume.py @@ -20,7 +20,7 @@ # IN THE SOFTWARE. """ -Represents an EC2 Elastic Block Storage Volume +Represents an EC2 Elastic IP Volume """ from boto.ec2.ec2object import EC2Object @@ -156,7 +156,14 @@ def volume_state(self): def attachment_state(self): """ - Get the attachment state. + Get the attachmentSet information for the volume. This info is stored + in a dictionary object and contains at least the following info: + + - volumeId + - instanceId + - device + - status + - attachTime """ state = None if self.attach_data: diff --git a/boto/exception.py b/boto/exception.py index c11a4d1..ba65694 100644 --- a/boto/exception.py +++ b/boto/exception.py @@ -22,11 +22,11 @@ """ Exception classes - Subclassing allows you to check for specific errors """ -import base64 -import xml.sax + from boto import handler from boto.resultset import ResultSet +import xml.sax class BotoClientError(StandardError): """ @@ -244,11 +244,6 @@ def _cleanupParsedProperties(self): for p in ('errors'): setattr(self, p, None) -class EmrResponseError(BotoServerError): - """ - Error in response from EMR - """ - class _EC2Error: def __init__(self, connection=None): diff --git a/boto/fps/connection.py b/boto/fps/connection.py index 0e0d4e8..0f14775 100644 --- a/boto/fps/connection.py +++ b/boto/fps/connection.py @@ -24,6 +24,7 @@ import uuid import boto import boto.utils +import urllib from boto import handler from boto.connection import AWSQueryConnection from boto.resultset import ResultSet @@ -31,142 +32,142 @@ class FPSConnection(AWSQueryConnection): - APIVersion = '2007-01-08' - SignatureVersion = '1' + APIVersion = '2007-01-08' + SignatureVersion = '1' - def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, - is_secure=True, port=None, proxy=None, proxy_port=None, - host='fps.sandbox.amazonaws.com', debug=0, - https_connection_factory=None): - AWSQueryConnection.__init__(self, aws_access_key_id, - aws_secret_access_key, - is_secure, port, proxy, proxy_port, - host, debug, https_connection_factory) - - def install_payment_instruction(self, instruction, token_type="Unrestricted", transaction_id=None): - """ - InstallPaymentInstruction - instruction: The PaymentInstruction to send, for example: - - MyRole=='Caller' orSay 'Roles do not match'; - - token_type: Defaults to "Unrestricted" - transaction_id: Defaults to a new ID - """ + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, port=None, proxy=None, proxy_port=None, + host='fps.sandbox.amazonaws.com', debug=0, + https_connection_factory=None): + AWSQueryConnection.__init__(self, aws_access_key_id, + aws_secret_access_key, + is_secure, port, proxy, proxy_port, + host, debug, https_connection_factory) + + def install_payment_instruction(self, instruction, token_type="Unrestricted", transaction_id=None): + """ + InstallPaymentInstruction + instruction: The PaymentInstruction to send, for example: + + MyRole=='Caller' orSay 'Roles do not match'; + + token_type: Defaults to "Unrestricted" + transaction_id: Defaults to a new ID + """ - if(transaction_id == None): - transaction_id = uuid.uuid4() - params = {} - params['PaymentInstruction'] = instruction - params['TokenType'] = token_type - params['CallerReference'] = transaction_id - response = self.make_request("InstallPaymentInstruction", params) - return response - - def install_caller_instruction(self, token_type="Unrestricted", transaction_id=None): - """ - Set us up as a caller - This will install a new caller_token into the FPS section. - This should really only be called to regenerate the caller token. - """ - response = self.install_payment_instruction("MyRole=='Caller';", token_type=token_type, transaction_id=transaction_id) - body = response.read() - if(response.status == 200): - rs = ResultSet() - h = handler.XmlHandler(rs, self) - xml.sax.parseString(body, h) - caller_token = rs.TokenId - try: - boto.config.save_system_option("FPS", "caller_token", caller_token) - except(IOError): - boto.config.save_user_option("FPS", "caller_token", caller_token) - return caller_token - else: - raise FPSResponseError(response.status, response.reason, body) + if(transaction_id == None): + transaction_id = uuid.uuid4() + params = {} + params['PaymentInstruction'] = instruction + params['TokenType'] = token_type + params['CallerReference'] = transaction_id + response = self.make_request("InstallPaymentInstruction", params) + return response + + def install_caller_instruction(self, token_type="Unrestricted", transaction_id=None): + """ + Set us up as a caller + This will install a new caller_token into the FPS section. + This should really only be called to regenerate the caller token. + """ + response = self.install_payment_instruction("MyRole=='Caller';", token_type=token_type, transaction_id=transaction_id) + body = response.read() + if(response.status == 200): + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + caller_token = rs.TokenId + try: + boto.config.save_system_option("FPS", "caller_token", caller_token) + except(IOError): + boto.config.save_user_option("FPS", "caller_token", caller_token) + return caller_token + else: + raise FPSResponseError(response.status, respons.reason, body) - def install_recipient_instruction(self, token_type="Unrestricted", transaction_id=None): - """ - Set us up as a Recipient - This will install a new caller_token into the FPS section. - This should really only be called to regenerate the recipient token. - """ - response = self.install_payment_instruction("MyRole=='Recipient';", token_type=token_type, transaction_id=transaction_id) - body = response.read() - if(response.status == 200): - rs = ResultSet() - h = handler.XmlHandler(rs, self) - xml.sax.parseString(body, h) - recipient_token = rs.TokenId - try: - boto.config.save_system_option("FPS", "recipient_token", recipient_token) - except(IOError): - boto.config.save_user_option("FPS", "recipient_token", recipient_token) + def install_recipient_instruction(self, token_type="Unrestricted", transaction_id=None): + """ + Set us up as a Recipient + This will install a new caller_token into the FPS section. + This should really only be called to regenerate the recipient token. + """ + response = self.install_payment_instruction("MyRole=='Recipient';", token_type=token_type, transaction_id=transaction_id) + body = response.read() + if(response.status == 200): + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + recipient_token = rs.TokenId + try: + boto.config.save_system_option("FPS", "recipient_token", recipient_token) + except(IOError): + boto.config.save_user_option("FPS", "recipient_token", recipient_token) - return recipient_token - else: - raise FPSResponseError(response.status, response.reason, body) + return recipient_token + else: + raise FPSResponseError(response.status, respons.reason, body) - def make_url(self, returnURL, paymentReason, pipelineName, **params): - """ - Generate the URL with the signature required for a transaction - """ - params['callerKey'] = str(self.aws_access_key_id) - params['returnURL'] = str(returnURL) - params['paymentReason'] = str(paymentReason) - params['pipelineName'] = pipelineName + def make_url(self, returnURL, paymentReason, pipelineName, **params): + """ + Generate the URL with the signature required for a transaction + """ + params['callerKey'] = str(self.aws_access_key_id) + params['returnURL'] = str(returnURL) + params['paymentReason'] = str(paymentReason) + params['pipelineName'] = pipelineName - if(not params.has_key('callerReference')): - params['callerReference'] = str(uuid.uuid4()) + if(not params.has_key('callerReference')): + params['callerReference'] = str(uuid.uuid4()) - url = "" - keys = params.keys() - keys.sort() - for k in keys: - url += "&%s=%s" % (k, urllib.quote_plus(str(params[k]))) + url = "" + keys = params.keys() + keys.sort() + for k in keys: + url += "&%s=%s" % (k, urllib.quote_plus(str(params[k]))) - url = "/cobranded-ui/actions/start?%s" % ( url[1:]) - signature= boto.utils.encode(self.aws_secret_access_key, url, True) - return "https://authorize.payments-sandbox.amazon.com%s&awsSignature=%s" % (url, signature) + url = "/cobranded-ui/actions/start?%s" % ( url[1:]) + signature= boto.utils.encode(self.aws_secret_access_key, url, True) + return "https://authorize.payments-sandbox.amazon.com%s&awsSignature=%s" % (url, signature) - def make_payment(self, amount, sender_token, charge_fee_to="Recipient", reference=None, senderReference=None, recipientReference=None, senderDescription=None, recipientDescription=None, callerDescription=None, metadata=None, transactionDate=None): - """ - Make a payment transaction - You must specify the amount and the sender token. - """ - params = {} - params['RecipientTokenId'] = boto.config.get("FPS", "recipient_token") - params['CallerTokenId'] = boto.config.get("FPS", "caller_token") - params['SenderTokenId'] = sender_token - params['TransactionAmount.Amount'] = str(amount) - params['TransactionAmount.CurrencyCode'] = "USD" - params['ChargeFeeTo'] = charge_fee_to + def make_payment(self, amount, sender_token, charge_fee_to="Recipient", reference=None, senderReference=None, recipientReference=None, senderDescription=None, recipientDescription=None, callerDescription=None, metadata=None, transactionDate=None): + """ + Make a payment transaction + You must specify the amount and the sender token. + """ + params = {} + params['RecipientTokenId'] = boto.config.get("FPS", "recipient_token") + params['CallerTokenId'] = boto.config.get("FPS", "caller_token") + params['SenderTokenId'] = sender_token + params['TransactionAmount.Amount'] = str(amount) + params['TransactionAmount.CurrencyCode'] = "USD" + params['ChargeFeeTo'] = charge_fee_to - if(transactionDate != None): - params['TransactionDate'] = transactionDate - if(senderReference != None): - params['SenderReference'] = senderReference - if(recipientReference != None): - params['RecipientReference'] = recipientReference - if(senderDescription != None): - params['SenderDescription'] = senderDescription - if(recipientDescription != None): - params['RecipientDescription'] = recipientDescription - if(callerDescription != None): - params['CallerDescription'] = callerDescription - if(metadata != None): - params['MetaData'] = metadata - if(transactionDate != None): - params['TransactionDate'] = transactionDate - if(reference == None): - reference = uuid.uuid4() - params['CallerReference'] = reference + if(transactionDate != None): + params['TransactionDate'] = transactionDate + if(senderReference != None): + params['SenderReference'] = senderReference + if(recipientReference != None): + params['RecipientReference'] = recipientReference + if(senderDescription != None): + params['SenderDescription'] = senderDescription + if(recipientDescription != None): + params['RecipientDescription'] = recipientDescription + if(callerDescription != None): + params['CallerDescription'] = callerDescription + if(metadata != None): + params['MetaData'] = metadata + if(transactionDate != None): + params['TransactionDate'] = transactionDate + if(reference == None): + reference = uuid.uuid4() + params['CallerReference'] = reference - response = self.make_request("Pay", params) - body = response.read() - if(response.status == 200): - rs = ResultSet() - h = handler.XmlHandler(rs, self) - xml.sax.parseString(body, h) - return rs - else: - raise FPSResponseError(response.status, response.reason, body) + response = self.make_request("Pay", params) + body = response.read() + if(response.status == 200): + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs + else: + raise FPSResponseError(response.status, response.reason, body) diff --git a/boto/manage/cmdshell.py b/boto/manage/cmdshell.py index 5d287ab..340b1e2 100644 --- a/boto/manage/cmdshell.py +++ b/boto/manage/cmdshell.py @@ -21,14 +21,10 @@ from boto.mashups.interactive import interactive_shell import boto -import os -import time -import shutil +import os, time, shutil import StringIO import paramiko import socket -import subprocess - class SSHClient(object): @@ -59,7 +55,7 @@ def connect(self): except paramiko.BadHostKeyException: print "%s has an entry in ~/.ssh/known_hosts and it doesn't match" % self.server.hostname print 'Edit that file to remove the entry and then hit return to try again' - raw_input('Hit Enter when ready') + rawinput('Hit Enter when ready') retry += 1 except EOFError: print 'Unexpected Error from SSH Connection, retry in 5 seconds' diff --git a/boto/manage/propget.py b/boto/manage/propget.py index 45b2ff2..172e1aa 100644 --- a/boto/manage/propget.py +++ b/boto/manage/propget.py @@ -19,6 +19,8 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. +import os +from boto.sdb.db.property import * def get(prop, choices=None): prompt = prop.verbose_name diff --git a/boto/manage/server.py b/boto/manage/server.py index dfb1ecf..cc623ef 100644 --- a/boto/manage/server.py +++ b/boto/manage/server.py @@ -1,5 +1,4 @@ # Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ -# Copyright (c) 2010 Chris Moyer http://coredumped.org/ # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the @@ -28,7 +27,7 @@ from boto.mashups.iobject import IObject from boto.pyami.config import BotoConfigPath, Config from boto.sdb.db.model import Model -from boto.sdb.db.property import StringProperty, IntegerProperty, BooleanProperty, CalculatedProperty +from boto.sdb.db.property import * from boto.manage import propget from boto.ec2.zone import Zone from boto.ec2.keypair import KeyPair @@ -50,7 +49,7 @@ def __init__(self, server, uname='root'): def copy_x509(self, key_file, cert_file): print '\tcopying cert and pk over to /mnt directory on server' - self.ssh_client.open_sftp() + sftp_client = self.ssh_client.open_sftp() path, name = os.path.split(key_file) self.remote_key_file = '/mnt/%s' % name self.ssh_client.put_file(key_file, self.remote_key_file) @@ -104,16 +103,16 @@ def bundle(self, bucket=None, prefix=None, key_file=None, cert_file=None, self.copy_x509(key_file, cert_file) if not fp: fp = StringIO.StringIO() - fp.write('sudo mv %s /mnt/boto.cfg; ' % BotoConfigPath) - fp.write('mv ~/.ssh/authorized_keys /mnt/authorized_keys; ') + fp.write('mv %s /mnt/boto.cfg; ' % BotoConfigPath) + fp.write('mv /root/.ssh/authorized_keys /mnt/authorized_keys; ') if clear_history: fp.write('history -c; ') fp.write(self.bundle_image(prefix, size, ssh_key)) fp.write('; ') fp.write(self.upload_bundle(bucket, prefix, ssh_key)) fp.write('; ') - fp.write('sudo mv /mnt/boto.cfg %s; ' % BotoConfigPath) - fp.write('mv /mnt/authorized_keys ~/.ssh/authorized_keys') + fp.write('mv /mnt/boto.cfg %s; ' % BotoConfigPath) + fp.write('mv /mnt/authorized_keys /root/.ssh/authorized_keys\n') command = fp.getvalue() print 'running the following command on the remote server:' print command @@ -122,7 +121,7 @@ def bundle(self, bucket=None, prefix=None, key_file=None, cert_file=None, print '\t%s' % t[1] print '...complete!' print 'registering image...' - self.image_id = self.server.ec2.register_image(name=prefix, image_location='%s/%s.manifest.xml' % (bucket, prefix)) + self.image_id = self.server.ec2.register_image('%s/%s.manifest.xml' % (bucket, prefix)) return self.image_id class CommandLineGetter(object): @@ -174,8 +173,10 @@ def get_zone(self, params): def get_ami_id(self, params): ami = params.get('ami', None) if isinstance(ami, str) or isinstance(ami, unicode): - for a in self.ec2.get_all_images(): + ami_list = self.get_ami_list() + for l,a in ami_list: if a.id == ami: + ami = a params['ami'] = a if not params.get('ami', None): prop = StringProperty(name='ami', verbose_name='AMI', @@ -320,7 +321,6 @@ def create(cls, config_file=None, logical_volume = None, cfg = None, **params): instances = reservation.instances if elastic_ip != None and instances.__len__() > 0: instance = instances[0] - print 'Waiting for instance to start so we can set its elastic IP address...' while instance.update() != 'running': time.sleep(1) instance.use_ip(elastic_ip) @@ -527,12 +527,12 @@ def run(self, command): return status def get_bundler(self, uname='root'): - self.get_ssh_key_file() + ssh_key_file = self.get_ssh_key_file() return Bundler(self, uname) def get_ssh_client(self, uname='root'): from boto.manage.cmdshell import SSHClient - self.get_ssh_key_file() + ssh_key_file = self.get_ssh_key_file() return SSHClient(self, uname=uname) def install(self, pkg): diff --git a/boto/manage/task.py b/boto/manage/task.py index 2f9d7d0..5fb234d 100644 --- a/boto/manage/task.py +++ b/boto/manage/task.py @@ -21,7 +21,7 @@ # import boto -from boto.sdb.db.property import StringProperty, DateTimeProperty, IntegerProperty +from boto.sdb.db.property import * from boto.sdb.db.model import Model import datetime, subprocess, StringIO, time @@ -72,6 +72,7 @@ def check(self): If it's an hourly task and it's never been run, run it now. If it's a daily task and it's never been run and the hour is right, run it now. """ + need_to_run = False boto.log.info('checking Task[%s]-now=%s, last=%s' % (self.name, self.now, self.last_executed)) if self.hourly and not self.last_executed: @@ -81,7 +82,7 @@ def check(self): if int(self.hour) == self.now.hour: return 0 else: - return max( (int(self.hour)-self.now.hour), (self.now.hour-int(self.hour)) )*60*60 + return max((int(self.hour) - self.now.hour),0)*60*60 delta = self.now - self.last_executed if self.hourly: @@ -90,13 +91,10 @@ def check(self): else: return 60*60 - delta.seconds else: - if int(self.hour) == self.now.hour: - if delta.days >= 1: - return 0 - else: - return 82800 # 23 hours, just to be safe + if delta.days >= 1: + return 0 else: - return max( (int(self.hour)-self.now.hour), (self.now.hour-int(self.hour)) )*60*60 + return min(60*60*24-delta.seconds, 43200) def _run(self, msg, vtimeout): boto.log.info('Task[%s] - running:%s' % (self.name, self.command)) @@ -129,7 +127,7 @@ def run(self, msg, vtimeout=60): self._run(msg, vtimeout) queue = msg.queue new_msg = queue.new_message(self.id) - new_msg = queue.write(new_msg) + new_msg = queue.write(msg) self.message_id = new_msg.id self.put() boto.log.info('Task[%s] - new message id=%s' % (self.name, new_msg.id)) diff --git a/boto/manage/volume.py b/boto/manage/volume.py index 66a458f..bed5594 100644 --- a/boto/manage/volume.py +++ b/boto/manage/volume.py @@ -21,16 +21,13 @@ from __future__ import with_statement from boto.sdb.db.model import Model -from boto.sdb.db.property import StringProperty, IntegerProperty, ListProperty, ReferenceProperty, CalculatedProperty +from boto.sdb.db.property import * from boto.manage.server import Server from boto.manage import propget import boto.ec2 -import time -import traceback +import time, traceback from contextlib import closing import dateutil.parser -import datetime - class CommandLineGetter(object): @@ -307,7 +304,7 @@ def snapshot(self): else: snapshot = self.server.ec2.create_snapshot(self.volume_id) boto.log.info('Snapshot of Volume %s created: %s' % (self.name, snapshot)) - except Exception: + except Exception, e: boto.log.info('Snapshot error') boto.log.info(traceback.format_exc()) finally: diff --git a/boto/mapreduce/partitiondb.py b/boto/mapreduce/partitiondb.py index 25cf135..c5b0475 100644 --- a/boto/mapreduce/partitiondb.py +++ b/boto/mapreduce/partitiondb.py @@ -20,25 +20,22 @@ # IN THE SOFTWARE. # -import random -import os -import datetime +import random, time, os, datetime +import boto from boto.sdb.persist.object import SDBObject -from boto.sdb.persist.property import StringProperty, ObjectProperty, DateTimeProperty, ObjectListProperty, S3KeyProperty - - -HEX_DIGITS = '0123456789abcdef' +from boto.sdb.persist.property import * class Identifier(object): - @staticmethod - def gen(prefix): + _hex_digits = '0123456789abcdef' + + @classmethod + def gen(cls, prefix): suffix = '' for i in range(0,8): - suffix += random.choice(HEX_DIGITS) - return prefix + '-' + suffix - - + suffix += random.choice(cls._hex_digits) + return ts + '-' + suffix + class Version(SDBObject): name = StringProperty() diff --git a/boto/mapreduce/queuetools.py b/boto/mapreduce/queuetools.py index db1e495..3e08a10 100644 --- a/boto/mapreduce/queuetools.py +++ b/boto/mapreduce/queuetools.py @@ -1,5 +1,5 @@ #!/usr/bin/python -import socket +import socket, sys from lqs import LQSServer, LQSMessage import boto from boto.sqs.jsonmessage import JSONMessage diff --git a/boto/mashups/server.py b/boto/mashups/server.py index 6cea106..48f637b 100644 --- a/boto/mashups/server.py +++ b/boto/mashups/server.py @@ -22,17 +22,15 @@ """ High-level abstraction of an EC2 server """ -import boto -import boto.utils +import boto, boto.utils from boto.mashups.iobject import IObject from boto.pyami.config import Config, BotoConfigPath from boto.mashups.interactive import interactive_shell from boto.sdb.db.model import Model -from boto.sdb.db.property import StringProperty +from boto.sdb.db.property import * import os import StringIO - class ServerSet(list): def __getattr__(self, name): @@ -59,11 +57,7 @@ def map(self, *args): class Server(Model): - @property - def ec2(self): - if self._ec2 is None: - self._ec2 = boto.connect_ec2() - return self._ec2 + ec2 = boto.connect_ec2() @classmethod def Inventory(cls): @@ -93,7 +87,6 @@ def __init__(self, id=None, **kw): self._ssh_client = None self._pkey = None self._config = None - self._ec2 = None name = StringProperty(unique=True, verbose_name="Name") instance_id = StringProperty(verbose_name="Instance ID") @@ -304,7 +297,7 @@ def bundle_image(self, prefix, key_file, cert_file, size): sftp_client.remove(BotoConfigPath) except: pass - command = 'sudo ec2-bundle-vol ' + command = 'ec2-bundle-vol ' command += '-c %s -k %s ' % (remote_cert_file, remote_key_file) command += '-u %s ' % self._reservation.owner_id command += '-p %s ' % prefix diff --git a/boto/mturk/connection.py b/boto/mturk/connection.py index f064554..261e2a7 100644 --- a/boto/mturk/connection.py +++ b/boto/mturk/connection.py @@ -31,7 +31,7 @@ class MTurkConnection(AWSQueryConnection): - APIVersion = '2008-08-02' + APIVersion = '2006-10-31' SignatureVersion = '1' def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, @@ -317,12 +317,12 @@ def expire_hit(self, hit_id): """ Expire a HIT that is no longer needed. - The effect is identical to the HIT expiring on its own. The HIT no longer appears on the - Mechanical Turk web site, and no new Workers are allowed to accept the HIT. Workers who - have accepted the HIT prior to expiration are allowed to complete it or return it, or - allow the assignment duration to elapse (abandon the HIT). Once all remaining assignments - have been submitted, the expired HIT becomes "reviewable", and will be returned by a call - to GetReviewableHITs. + The effect is identical to the HIT expiring on its own. The HIT no longer appears on the + Mechanical Turk web site, and no new Workers are allowed to accept the HIT. Workers who + have accepted the HIT prior to expiration are allowed to complete it or return it, or + allow the assignment duration to elapse (abandon the HIT). Once all remaining assignments + have been submitted, the expired HIT becomes "reviewable", and will be returned by a call + to GetReviewableHITs. """ params = {'HITId' : hit_id,} return self._process_request('ForceExpireHIT', params) @@ -501,15 +501,4 @@ class QuestionFormAnswer(BaseAutoResultElement): *NOTE* - currently really only supports free-text answers """ - def __init__(self, connection): - BaseAutoResultElement.__init__(self, connection) - self.fields = [] - self.qid = None - - def endElement(self, name, value, connection): - if name == 'QuestionIdentifier': - self.qid = value - elif name == 'FreeText' and self.qid: - self.fields.append((self.qid,value)) - elif name == 'Answer': - self.qid = None + pass diff --git a/boto/mturk/question.py b/boto/mturk/question.py index d4d9734..89f1a45 100644 --- a/boto/mturk/question.py +++ b/boto/mturk/question.py @@ -87,10 +87,10 @@ def get_binary_xml(self, field, value): %s %s -""" % (value['type'], - value['subtype'], - value['dataurl'], - value['alttext']) +""" % (value['binary_type'], + value['binary_subtype'], + value['binary'], + value['binary_alttext']) def get_application_xml(self, field, value): raise NotImplementedError("Application question content is not yet supported.") @@ -138,21 +138,37 @@ def get_as_xml(self): class QuestionForm(object): QUESTIONFORM_SCHEMA_LOCATION = "http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2005-10-01/QuestionForm.xsd" - QUESTIONFORM_XML_TEMPLATE = """%s""" - - def __init__(self): - self.items = [] + QUESTIONFORM_XML_TEMPLATE = """%s""" # % (ns, questions_xml) + + def __init__(self, questions=None, overview=None): + if questions is None or type(questions) is not list: + raise ValueError("Must pass a list of Question instances to QuestionForm constructor") + else: + self.questions = questions + self.overview = overview - def append(self, item): - "Expects field type and value" - self.items.append(item) - + def get_as_xml(self): - xml = '' - for item in self.items: - xml = xml + item.get_as_xml() - return QuestionForm.QUESTIONFORM_XML_TEMPLATE % (QuestionForm.QUESTIONFORM_SCHEMA_LOCATION, xml) - + if self.overview: + overview_xml = self.overview.get_as_xml() + questions_xml = "".join([q.get_as_xml() for q in self.questions]) + qf_xml = overview_xml + questions_xml + return QuestionForm.QUESTIONFORM_XML_TEMPLATE % (QuestionForm.QUESTIONFORM_SCHEMA_LOCATION, qf_xml) + + #def startElement(self, name, attrs, connection): + # return None + # + #def endElement(self, name, value, connection): + # + # #if name == 'Amount': + # # self.amount = float(value) + # #elif name == 'CurrencyCode': + # # self.currency_code = value + # #elif name == 'FormattedPrice': + # # self.formatted_price = value + # + # pass # What's this method for? I don't get it. + class QuestionContent(object): QUESTIONCONTENT_XML_TEMPLATE = """%s""" @@ -296,6 +312,7 @@ def __init__(self, min=1, max=1, style=None, selections=None, type='text', other self.other = other def get_as_xml(self): + xml = "" if self.type == 'text': TYPE_TAG = "Text" elif self.type == 'binary': diff --git a/boto/pyami/bootstrap.py b/boto/pyami/bootstrap.py index 44bcc2d..59462ca 100644 --- a/boto/pyami/bootstrap.py +++ b/boto/pyami/bootstrap.py @@ -19,7 +19,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # -import os +import os, pwd import boto from boto.utils import get_instance_metadata, get_instance_userdata from boto.pyami.config import Config, BotoConfigPath diff --git a/boto/pyami/config.py b/boto/pyami/config.py index ea0c3a1..831acc4 100644 --- a/boto/pyami/config.py +++ b/boto/pyami/config.py @@ -50,6 +50,7 @@ def __init__(self, path=None, fp=None, do_load=True): def load_credential_file(self, path): """Load a credential file as is setup like the Java utilities""" + config = ConfigParser.ConfigParser() c_data = StringIO.StringIO() c_data.write("[Credentials]\n") for line in open(path, "r").readlines(): diff --git a/boto/pyami/installers/__init__.py b/boto/pyami/installers/__init__.py index cc68926..4dcf2f4 100644 --- a/boto/pyami/installers/__init__.py +++ b/boto/pyami/installers/__init__.py @@ -20,7 +20,7 @@ # IN THE SOFTWARE. # from boto.pyami.scriptbase import ScriptBase - +import os class Installer(ScriptBase): """ @@ -31,7 +31,7 @@ def add_cron(self, name, minute, hour, mday, month, wday, who, command, env=None """ Add an entry to the system crontab. """ - raise NotImplementedError + raise NotImplimented() def add_init_script(self, file): """ @@ -42,23 +42,23 @@ def add_env(self, key, value): """ Add an environemnt variable """ - raise NotImplementedError + raise NotImplimented() def stop(self, service_name): """ Stop a service. """ - raise NotImplementedError + raise NotImplimented() def start(self, service_name): """ Start a service. """ - raise NotImplementedError + raise NotImplimented() def install(self): """ Do whatever is necessary to "install" the package. """ - raise NotImplementedError + raise NotImplimented() diff --git a/boto/pyami/installers/ubuntu/ebs.py b/boto/pyami/installers/ubuntu/ebs.py index 5486add..2cf0f22 100644 --- a/boto/pyami/installers/ubuntu/ebs.py +++ b/boto/pyami/installers/ubuntu/ebs.py @@ -77,13 +77,10 @@ def main(self): """ BackupCleanupScript= """#!/usr/bin/env python -import boto -from boto.manage.volume import Volume - # Cleans Backups of EBS volumes for v in Volume.all(): - v.trim_snapshots(True) + v.trim_snapshot(True) """ class EBSInstaller(Installer): diff --git a/boto/pyami/installers/ubuntu/installer.py b/boto/pyami/installers/ubuntu/installer.py index 370d63f..0169950 100644 --- a/boto/pyami/installers/ubuntu/installer.py +++ b/boto/pyami/installers/ubuntu/installer.py @@ -92,5 +92,5 @@ def install(self): """ This is the only method you need to override """ - raise NotImplementedError + raise NotImplimented() diff --git a/boto/pyami/installers/ubuntu/trac.py b/boto/pyami/installers/ubuntu/trac.py index ef83af7..c97ddd2 100644 --- a/boto/pyami/installers/ubuntu/trac.py +++ b/boto/pyami/installers/ubuntu/trac.py @@ -24,116 +24,114 @@ import os class Trac(Installer): - """ - Install Trac and DAV-SVN - Sets up a Vhost pointing to [Trac]->home - Using the config parameter [Trac]->hostname - Sets up a trac environment for every directory found under [Trac]->data_dir + """ + Install Trac and DAV-SVN + Sets up a Vhost pointing to [Trac]->home + Using the config parameter [Trac]->hostname + Sets up a trac environment for every directory found under [Trac]->data_dir - [Trac] - name = My Foo Server - hostname = trac.foo.com - home = /mnt/sites/trac - data_dir = /mnt/trac - svn_dir = /mnt/subversion - server_admin = root@foo.com - sdb_auth_domain = users - # Optional - SSLCertificateFile = /mnt/ssl/foo.crt - SSLCertificateKeyFile = /mnt/ssl/foo.key - SSLCertificateChainFile = /mnt/ssl/FooCA.crt + [Trac] + name = My Foo Server + hostname = trac.foo.com + home = /mnt/sites/trac + data_dir = /mnt/trac + svn_dir = /mnt/subversion + server_admin = root@foo.com + sdb_auth_domain = users + # Optional + SSLCertificateFile = /mnt/ssl/foo.crt + SSLCertificateKeyFile = /mnt/ssl/foo.key + SSLCertificateChainFile = /mnt/ssl/FooCA.crt - """ + """ - def install(self): - self.run('apt-get -y install trac', notify=True, exit_on_error=True) - self.run('apt-get -y install libapache2-svn', notify=True, exit_on_error=True) - self.run("a2enmod ssl") - self.run("a2enmod mod_python") - self.run("a2enmod dav_svn") - self.run("a2enmod rewrite") - # Make sure that boto.log is writable by everyone so that subversion post-commit hooks can - # write to it. - self.run("touch /var/log/boto.log") - self.run("chmod a+w /var/log/boto.log") + def install(self): + self.run('apt-get -y install trac', notify=True, exit_on_error=True) + self.run('apt-get -y install libapache2-svn', notify=True, exit_on_error=True) + self.run("a2enmod ssl") + self.run("a2enmod python") + self.run("a2enmod dav_svn") + self.run("a2enmod rewrite") - def setup_vhost(self): - domain = boto.config.get("Trac", "hostname").strip() - if domain: - domain_info = domain.split('.') - cnf = open("/etc/apache2/sites-available/%s" % domain_info[0], "w") - cnf.write("NameVirtualHost *:80\n") - if boto.config.get("Trac", "SSLCertificateFile"): - cnf.write("NameVirtualHost *:443\n\n") - cnf.write("\n") - cnf.write("\tServerAdmin %s\n" % boto.config.get("Trac", "server_admin").strip()) - cnf.write("\tServerName %s\n" % domain) - cnf.write("\tRewriteEngine On\n") - cnf.write("\tRewriteRule ^(.*)$ https://%s$1\n" % domain) - cnf.write("\n\n") + def setup_vhost(self): + domain = boto.config.get("Trac", "hostname").strip() + if domain: + cnf = open("/etc/apache2/sites-available/%s" % domain, "w") + cnf.write("NameVirtualHost *:80\n") + if boto.config.get("Trac", "SSLCertificateFile"): + cnf.write("NameVirtualHost *:443\n\n") + cnf.write("\n") + cnf.write("\tServerAdmin %s\n" % boto.config.get("Trac", "server_admin").strip()) + cnf.write("\tServerName %s\n" % domain) + cnf.write("\tRewriteEngine On\n") + cnf.write("\tRewriteRule ^(.*)$ https://%s$1\n" % domain) + cnf.write("\n\n") - cnf.write("\n") - else: - cnf.write("\n") + cnf.write("\n") + else: + cnf.write("\n") - cnf.write("\tServerAdmin %s\n" % boto.config.get("Trac", "server_admin").strip()) - cnf.write("\tServerName %s\n" % domain) - cnf.write("\tDocumentRoot %s\n" % boto.config.get("Trac", "home").strip()) + cnf.write("\tServerAdmin %s\n" % boto.config.get("Trac", "server_admin").strip()) + cnf.write("\tServerName %s\n" % domain) + cnf.write("\tDocumentRoot %s\n" % boto.config.get("Trac", "home").strip()) - cnf.write("\t\n" % boto.config.get("Trac", "home").strip()) - cnf.write("\t\tOptions FollowSymLinks Indexes MultiViews\n") - cnf.write("\t\tAllowOverride All\n") - cnf.write("\t\tOrder allow,deny\n") - cnf.write("\t\tallow from all\n") - cnf.write("\t\n") + cnf.write("\t\n" % boto.config.get("Trac", "home").strip()) + cnf.write("\t\tOptions FollowSymLinks Indexes MultiViews\n") + cnf.write("\t\tAllowOverride All\n") + cnf.write("\t\tOrder allow,deny\n") + cnf.write("\t\tallow from all\n") + cnf.write("\t\n") - cnf.write("\t\n") - cnf.write("\t\tAuthType Basic\n") - cnf.write("\t\tAuthName \"%s\"\n" % boto.config.get("Trac", "name")) - cnf.write("\t\tRequire valid-user\n") - cnf.write("\t\tAuthUserFile /mnt/apache/passwd/passwords\n") - cnf.write("\t\n") + cnf.write("\t\n") + cnf.write("\t\tAuthType Basic\n") + cnf.write("\t\tAuthName \"%s\"\n" % boto.config.get("Trac", "name")) + cnf.write("\t\tRequire valid-user\n") + cnf.write("\t\tAuthBasicAuthoritative off\n") + cnf.write("\t\tAuthUserFile /dev/null\n") + cnf.write("\t\tPythonAuthenHandler marajo.web.authen_handler\n") + cnf.write("\t\tPythonOption SDBDomain %s\n" % boto.config.get("Trac", "sdb_auth_domain")) + cnf.write("\t\n") - data_dir = boto.config.get("Trac", "data_dir") - for env in os.listdir(data_dir): - if(env[0] != "."): - cnf.write("\t\n" % env) - cnf.write("\t\tSetHandler mod_python\n") - cnf.write("\t\tPythonInterpreter main_interpreter\n") - cnf.write("\t\tPythonHandler trac.web.modpython_frontend\n") - cnf.write("\t\tPythonOption TracEnv %s/%s\n" % (data_dir, env)) - cnf.write("\t\tPythonOption TracUriRoot /trac/%s\n" % env) - cnf.write("\t\n") + data_dir = boto.config.get("Trac", "data_dir") + for env in os.listdir(data_dir): + if(env[0] != "."): + cnf.write("\t\n" % env) + cnf.write("\t\tSetHandler mod_python\n") + cnf.write("\t\tPythonInterpreter main_interpreter\n") + cnf.write("\t\tPythonHandler trac.web.modpython_frontend\n") + cnf.write("\t\tPythonOption TracEnv %s/%s\n" % (data_dir, env)) + cnf.write("\t\tPythonOption TracUriRoot /trac%s\n" % env) + cnf.write("\t\n") - svn_dir = boto.config.get("Trac", "svn_dir") - for env in os.listdir(svn_dir): - if(env[0] != "."): - cnf.write("\t\n" % env) - cnf.write("\t\tDAV svn\n") - cnf.write("\t\tSVNPath %s/%s\n" % (svn_dir, env)) - cnf.write("\t\n") + svn_dir = boto.config.get("Trac", "svn_dir") + for env in os.listdir(svn_dir): + if(env[0] != "."): + cnf.write("\t\n" % env) + cnf.write("\t\tDAV svn\n") + cnf.write("\t\tSVNPath %s/%s\n" % (svn_dir, env)) + cnf.write("\t\n") - cnf.write("\tErrorLog /var/log/apache2/error.log\n") - cnf.write("\tLogLevel warn\n") - cnf.write("\tCustomLog /var/log/apache2/access.log combined\n") - cnf.write("\tServerSignature On\n") - SSLCertificateFile = boto.config.get("Trac", "SSLCertificateFile") - if SSLCertificateFile: - cnf.write("\tSSLEngine On\n") - cnf.write("\tSSLCertificateFile %s\n" % SSLCertificateFile) + cnf.write("\tErrorLog /var/log/apache2/error.log\n") + cnf.write("\tLogLevel warn\n") + cnf.write("\tCustomLog /var/log/apache2/access.log combined\n") + cnf.write("\tServerSignature On\n") + SSLCertificateFile = boto.config.get("Trac", "SSLCertificateFile") + if SSLCertificateFile: + cnf.write("\tSSLEngine On\n") + cnf.write("\tSSLCertificateFile %s\n" % SSLCertificateFile) - SSLCertificateKeyFile = boto.config.get("Trac", "SSLCertificateKeyFile") - if SSLCertificateKeyFile: - cnf.write("\tSSLCertificateKeyFile %s\n" % SSLCertificateKeyFile) + SSLCertificateKeyFile = boto.config.get("Trac", "SSLCertificateKeyFile") + if SSLCertificateKeyFile: + cnf.write("\tSSLCertificateKeyFile %s\n" % SSLCertificateKeyFile) - SSLCertificateChainFile = boto.config.get("Trac", "SSLCertificateChainFile") - if SSLCertificateChainFile: - cnf.write("\tSSLCertificateChainFile %s\n" % SSLCertificateChainFile) - cnf.write("\n") - cnf.close() - self.run("a2ensite %s" % domain_info[0]) - self.run("/etc/init.d/apache2 force-reload") + SSLCertificateChainFile = boto.config.get("Trac", "SSLCertificateChainFile") + if SSLCertificateChainFile: + cnf.write("\tSSLCertificateChainFile %s\n" % SSLCertificateChainFile) + cnf.write("\n") + cnf.close() + self.run("a2ensite %s" % domain) + self.run("/etc/init.d/apache2 force-reload") - def main(self): - self.install() - self.setup_vhost() + def main(self): + self.install() + self.setup_vhost() diff --git a/boto/pyami/launch_ami.py b/boto/pyami/launch_ami.py index 243d56d..c49c2a3 100755 --- a/boto/pyami/launch_ami.py +++ b/boto/pyami/launch_ami.py @@ -20,11 +20,9 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # -import getopt -import sys -import imp -import time +import getopt, sys, imp, time import boto +from boto.utils import get_instance_userdata usage_string = """ SYNOPSIS diff --git a/boto/pyami/scriptbase.py b/boto/pyami/scriptbase.py index ef8bd28..6fe92aa 100644 --- a/boto/pyami/scriptbase.py +++ b/boto/pyami/scriptbase.py @@ -1,5 +1,5 @@ -import os -import sys +import os, sys, time, traceback +import smtplib from boto.utils import ShellCommand, get_ts import boto import boto.utils diff --git a/boto/pyami/startup.py b/boto/pyami/startup.py index 8443bff..d6f1376 100644 --- a/boto/pyami/startup.py +++ b/boto/pyami/startup.py @@ -19,12 +19,12 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # -import sys +import os, sys, traceback, StringIO import boto from boto.utils import find_class from boto import config from boto.pyami.scriptbase import ScriptBase - +from boto.utils import find_class class Startup(ScriptBase): @@ -44,7 +44,7 @@ def run_scripts(self): s.main() else: boto.log.warning('Trouble parsing script: %s' % script) - except Exception: + except Exception, e: boto.log.exception('Problem Running Script: %s' % script) def main(self): diff --git a/boto/rds/__init__.py b/boto/rds/__init__.py index e48a55d..92b7199 100644 --- a/boto/rds/__init__.py +++ b/boto/rds/__init__.py @@ -20,9 +20,15 @@ # IN THE SOFTWARE. # +import xml.sax +import base64 +import time +import boto import boto.utils import urllib from boto.connection import AWSQueryConnection +from boto import handler +from boto.resultset import ResultSet from boto.rds.dbinstance import DBInstance from boto.rds.dbsecuritygroup import DBSecurityGroup from boto.rds.parametergroup import ParameterGroup @@ -106,7 +112,7 @@ def create_dbinstance(self, id, allocated_storage, instance_class, db.m2.2xlarge | db.m2.4xlarge :type engine: str - :param engine: Name of database engine. Must be MySQL5.1 for now. +. :param engine: Name of database engine. Must be MySQL5.1 for now. :type master_username: str :param master_username: Name of master user for the DBInstance. @@ -736,8 +742,8 @@ def restore_dbinstance_from_point_in_time(self, source_instance_id, params['UseLatestRestorableTime'] = 'true' elif restore_time: params['RestoreTime'] = restore_time.isoformat() - if dbinstance_class: - params['DBInstanceClass'] = dbinstance_class + if instance_class: + params['DBInstanceClass'] = instance_class if port: params['Port'] = port if availability_zone: diff --git a/boto/rds/dbsecuritygroup.py b/boto/rds/dbsecuritygroup.py index 24cdad2..9ec6cc0 100644 --- a/boto/rds/dbsecuritygroup.py +++ b/boto/rds/dbsecuritygroup.py @@ -74,7 +74,7 @@ def authorize(self, cidr_ip=None, ec2_group=None): @type cidr_ip: string @param cidr_ip: A valid CIDR IP range to authorize - @type ec2_group: :class:`boto.ec2.securitygroup.SecurityGroup>` + @type ec2_group: :class:`boto.ec2.securitygroup.SecurityGroup>`b @rtype: bool @return: True if successful. @@ -99,7 +99,7 @@ def revoke(self, cidr_ip=None, ec2_group=None): @type cidr_ip: string @param cidr_ip: A valid CIDR IP range to authorize - @type ec2_group: :class:`boto.ec2.securitygroup.SecurityGroup>` + @type ec2_group: :class:`boto.ec2.securitygroup.SecurityGroup>`b @rtype: bool @return: True if successful. diff --git a/boto/resultset.py b/boto/resultset.py index cf6f1fd..aab1b68 100644 --- a/boto/resultset.py +++ b/boto/resultset.py @@ -44,8 +44,6 @@ def __init__(self, marker_elem=None): else: self.markers = [] self.marker = None - self.key_marker = None - self.version_id_marker = None self.is_truncated = False self.next_token = None self.status = True @@ -69,10 +67,6 @@ def endElement(self, name, value, connection): self.is_truncated = self.to_boolean(value) elif name == 'Marker': self.marker = value - elif name == 'KeyMarker': - self.key_marker = value - elif name == 'VersionIdMarker': - self.version_id_marker = value elif name == 'Prefix': self.prefix = value elif name == 'return': diff --git a/boto/s3/acl.py b/boto/s3/acl.py index 59d3687..702551e 100644 --- a/boto/s3/acl.py +++ b/boto/s3/acl.py @@ -20,12 +20,11 @@ # IN THE SOFTWARE. from boto.s3.user import User - +import StringIO CannedACLStrings = ['private', 'public-read', 'public-read-write', 'authenticated-read'] - class Policy: def __init__(self, parent=None): diff --git a/boto/s3/bucket.py b/boto/s3/bucket.py index d2135f2..297f0a2 100644 --- a/boto/s3/bucket.py +++ b/boto/s3/bucket.py @@ -22,17 +22,15 @@ import boto from boto import handler from boto.resultset import ResultSet -from boto.s3.acl import Policy, CannedACLStrings, Grant +from boto.s3.acl import Policy, CannedACLStrings, ACL, Grant +from boto.s3.user import User from boto.s3.key import Key from boto.s3.prefix import Prefix -from boto.s3.deletemarker import DeleteMarker from boto.exception import S3ResponseError, S3PermissionsError, S3CopyError from boto.s3.bucketlistresultset import BucketListResultSet -from boto.s3.bucketlistresultset import VersionedBucketListResultSet import boto.utils import xml.sax import urllib -import re S3Permissions = ['READ', 'WRITE', 'READ_ACP', 'WRITE_ACP', 'FULL_CONTROL'] @@ -57,15 +55,6 @@ class Bucket: %s """ - VersioningBody = """ - - %s - %s - """ - - VersionRE = '([A-Za-z]+)' - MFADeleteRE = '([A-Za-z]+)' - def __init__(self, connection=None, name=None, key_class=Key): self.name = name self.connection = connection @@ -117,7 +106,7 @@ def lookup(self, key_name, headers=None): """ return self.get_key(key_name, headers=headers) - def get_key(self, key_name, headers=None, version_id=None): + def get_key(self, key_name, headers=None): """ Check to see if a particular key exists within the bucket. This method uses a HEAD request to check for the existance of the key. @@ -129,15 +118,9 @@ def get_key(self, key_name, headers=None, version_id=None): :rtype: :class:`boto.s3.key.Key` :returns: A Key object from this bucket. """ - if version_id: - query_args = 'versionId=%s' % version_id - else: - query_args = None - response = self.connection.make_request('HEAD', self.name, key_name, - headers=headers, - query_args=query_args) + response = self.connection.make_request('HEAD', self.name, key_name, headers=headers) if response.status == 200: - response.read() + body = response.read() k = self.key_class(self) k.metadata = boto.utils.get_aws_metadata(response.msg) k.etag = response.getheader('etag') @@ -146,11 +129,10 @@ def get_key(self, key_name, headers=None, version_id=None): k.last_modified = response.getheader('last-modified') k.size = int(response.getheader('content-length')) k.name = key_name - k.handle_version_headers(response) return k else: if response.status == 404: - response.read() + body = response.read() return None else: raise S3ResponseError(response.status, response.reason, '') @@ -166,14 +148,13 @@ def list(self, prefix='', delimiter='', marker='', headers=None): :type prefix: string :param prefix: allows you to limit the listing to a particular - prefix. For example, if you call the method with - prefix='/foo/' then the iterator will only cycle - through the keys that begin with the string '/foo/'. + prefix. For example, if you call the method with prefix='/foo/' + then the iterator will only cycle through the keys that begin with + the string '/foo/'. :type delimiter: string :param delimiter: can be used in conjunction with the prefix - to allow you to organize and browse your keys - hierarchically. See: + to allow you to organize and browse your keys hierarchically. See: http://docs.amazonwebservices.com/AmazonS3/2006-03-01/ for more details. @@ -185,137 +166,53 @@ def list(self, prefix='', delimiter='', marker='', headers=None): """ return BucketListResultSet(self, prefix, delimiter, marker, headers) - def list_versions(self, prefix='', delimiter='', key_marker='', - version_id_marker='', headers=None): + def get_all_keys(self, headers=None, **params): """ - List key objects within a bucket. This returns an instance of an - BucketListResultSet that automatically handles all of the result - paging, etc. from S3. You just need to keep iterating until - there are no more results. - Called with no arguments, this will return an iterator object across - all keys within the bucket. + A lower-level method for listing contents of a bucket. This closely models the actual S3 + API and requires you to manually handle the paging of results. For a higher-level method + that handles the details of paging for you, you can use the list method. + + :type maxkeys: int + :param maxkeys: The maximum number of keys to retrieve :type prefix: string - :param prefix: allows you to limit the listing to a particular - prefix. For example, if you call the method with - prefix='/foo/' then the iterator will only cycle - through the keys that begin with the string '/foo/'. - - :type delimiter: string - :param delimiter: can be used in conjunction with the prefix - to allow you to organize and browse your keys - hierarchically. See: - http://docs.amazonwebservices.com/AmazonS3/2006-03-01/ - for more details. - + :param prefix: The prefix of the keys you want to retrieve + :type marker: string :param marker: The "marker" of where you are in the result set - :rtype: :class:`boto.s3.bucketlistresultset.BucketListResultSet` - :return: an instance of a BucketListResultSet that handles paging, etc - """ - return VersionedBucketListResultSet(self, prefix, delimiter, key_marker, - version_id_marker, headers) + :type delimiter: string + :param delimiter: "If this optional, Unicode string parameter is included with your request, then keys that contain the same string between the prefix and the first occurrence of the delimiter will be rolled up into a single result element in the CommonPrefixes collection. These rolled-up keys are not returned elsewhere in the response." - def _get_all(self, element_map, initial_query_string='', - headers=None, **params): + :rtype: ResultSet + :return: The result from S3 listing the keys requested + + """ l = [] for k,v in params.items(): - k = k.replace('_', '-') if k == 'maxkeys': k = 'max-keys' if isinstance(v, unicode): v = v.encode('utf-8') - if v is not None and v != '': + if v is not None: l.append('%s=%s' % (urllib.quote(k), urllib.quote(str(v)))) if len(l): - s = initial_query_string + '&' + '&'.join(l) + s = '&'.join(l) else: - s = initial_query_string + s = None response = self.connection.make_request('GET', self.name, headers=headers, query_args=s) body = response.read() boto.log.debug(body) if response.status == 200: - rs = ResultSet(element_map) + rs = ResultSet([('Contents', self.key_class), + ('CommonPrefixes', Prefix)]) h = handler.XmlHandler(rs, self) xml.sax.parseString(body, h) return rs else: raise S3ResponseError(response.status, response.reason, body) - def get_all_keys(self, headers=None, **params): - """ - A lower-level method for listing contents of a bucket. - This closely models the actual S3 API and requires you to manually - handle the paging of results. For a higher-level method - that handles the details of paging for you, you can use the list method. - - :type max_keys: int - :param max_keys: The maximum number of keys to retrieve - - :type prefix: string - :param prefix: The prefix of the keys you want to retrieve - - :type marker: string - :param marker: The "marker" of where you are in the result set - - :type delimiter: string - :param delimiter: If this optional, Unicode string parameter - is included with your request, then keys that - contain the same string between the prefix and - the first occurrence of the delimiter will be - rolled up into a single result element in the - CommonPrefixes collection. These rolled-up keys - are not returned elsewhere in the response. - - :rtype: ResultSet - :return: The result from S3 listing the keys requested - - """ - return self._get_all([('Contents', self.key_class), - ('CommonPrefixes', Prefix)], - '', headers, **params) - - def get_all_versions(self, headers=None, **params): - """ - A lower-level, version-aware method for listing contents of a bucket. - This closely models the actual S3 API and requires you to manually - handle the paging of results. For a higher-level method - that handles the details of paging for you, you can use the list method. - - :type max_keys: int - :param max_keys: The maximum number of keys to retrieve - - :type prefix: string - :param prefix: The prefix of the keys you want to retrieve - - :type key_marker: string - :param key_marker: The "marker" of where you are in the result set - with respect to keys. - - :type version_id_marker: string - :param version_id_marker: The "marker" of where you are in the result - set with respect to version-id's. - - :type delimiter: string - :param delimiter: If this optional, Unicode string parameter - is included with your request, then keys that - contain the same string between the prefix and - the first occurrence of the delimiter will be - rolled up into a single result element in the - CommonPrefixes collection. These rolled-up keys - are not returned elsewhere in the response. - - :rtype: ResultSet - :return: The result from S3 listing the keys requested - - """ - return self._get_all([('Version', self.key_class), - ('CommonPrefixes', Prefix), - ('DeleteMarker', DeleteMarker)], - 'versions', headers, **params) - def new_key(self, key_name=None): """ Creates a new key @@ -328,49 +225,23 @@ def new_key(self, key_name=None): """ return self.key_class(self, key_name) - def generate_url(self, expires_in, method='GET', - headers=None, force_http=False): - return self.connection.generate_url(expires_in, method, self.name, - headers=headers, + def generate_url(self, expires_in, method='GET', headers=None, force_http=False): + return self.connection.generate_url(expires_in, method, self.name, headers=headers, force_http=force_http) - def delete_key(self, key_name, headers=None, - version_id=None, mfa_token=None): + def delete_key(self, key_name, headers=None): """ - Deletes a key from the bucket. If a version_id is provided, - only that version of the key will be deleted. + Deletes a key from the bucket. :type key_name: string :param key_name: The key name to delete - - :type version_id: string - :param version_id: The version ID (optional) - - :type mfa_token: tuple or list of strings - :param mfa_token: A tuple or list consisting of the serial number - from the MFA device and the current value of - the six-digit token associated with the device. - This value is required anytime you are - deleting versioned objects from a bucket - that has the MFADelete option on the bucket. """ - if version_id: - query_args = 'versionId=%s' % version_id - else: - query_args = None - if mfa_token: - if not headers: - headers = {} - headers['x-amz-mfa'] = ' '.join(mfa_token) - response = self.connection.make_request('DELETE', self.name, key_name, - headers=headers, - query_args=query_args) + response = self.connection.make_request('DELETE', self.name, key_name, headers=headers) body = response.read() if response.status != 204: raise S3ResponseError(response.status, response.reason, body) - def copy_key(self, new_key_name, src_bucket_name, - src_key_name, metadata=None, src_version_id=None): + def copy_key(self, new_key_name, src_bucket_name, src_key_name, metadata=None): """ Create a new key in the bucket by copying another existing key. @@ -383,11 +254,6 @@ def copy_key(self, new_key_name, src_bucket_name, :type src_key_name: string :param src_key_name: The name of the source key - :type src_version_id: string - :param src_version_id: The version id for the key. This param - is optional. If not specified, the newest - version of the key will be copied. - :type metadata: dict :param metadata: Metadata to be associated with new key. If metadata is supplied, it will replace the @@ -399,8 +265,6 @@ def copy_key(self, new_key_name, src_bucket_name, :returns: An instance of the newly created key object """ src = '%s/%s' % (src_bucket_name, urllib.quote(src_key_name)) - if src_version_id: - src += '?version_id=%s' % src_version_id if metadata: headers = {'x-amz-copy-source' : src, 'x-amz-metadata-directive' : 'REPLACE'} @@ -417,13 +281,11 @@ def copy_key(self, new_key_name, src_bucket_name, xml.sax.parseString(body, h) if hasattr(key, 'Error'): raise S3CopyError(key.Code, key.Message, body) - key.handle_version_headers(response) return key else: raise S3ResponseError(response.status, response.reason, body) - def set_canned_acl(self, acl_str, key_name='', headers=None, - version_id=None): + def set_canned_acl(self, acl_str, key_name='', headers=None): assert acl_str in CannedACLStrings if headers: @@ -431,54 +293,36 @@ def set_canned_acl(self, acl_str, key_name='', headers=None, else: headers={'x-amz-acl': acl_str} - query_args='acl' - if version_id: - query_args += '&versionId=%s' % version_id response = self.connection.make_request('PUT', self.name, key_name, - headers=headers, query_args=query_args) + headers=headers, query_args='acl') body = response.read() if response.status != 200: raise S3ResponseError(response.status, response.reason, body) - def get_xml_acl(self, key_name='', headers=None, version_id=None): - query_args = 'acl' - if version_id: - query_args += '&versionId=%s' % version_id + def get_xml_acl(self, key_name='', headers=None): response = self.connection.make_request('GET', self.name, key_name, - query_args=query_args, - headers=headers) + query_args='acl', headers=headers) body = response.read() if response.status != 200: raise S3ResponseError(response.status, response.reason, body) return body - def set_xml_acl(self, acl_str, key_name='', headers=None, version_id=None): - query_args = 'acl' - if version_id: - query_args += '&versionId=%s' % version_id + def set_xml_acl(self, acl_str, key_name='', headers=None): response = self.connection.make_request('PUT', self.name, key_name, - data=acl_str, - query_args=query_args, - headers=headers) + data=acl_str, query_args='acl', headers=headers) body = response.read() if response.status != 200: raise S3ResponseError(response.status, response.reason, body) - def set_acl(self, acl_or_str, key_name='', headers=None, version_id=None): + def set_acl(self, acl_or_str, key_name='', headers=None): if isinstance(acl_or_str, Policy): - self.set_xml_acl(acl_or_str.to_xml(), key_name, - headers, version_id) + self.set_xml_acl(acl_or_str.to_xml(), key_name, headers=headers) else: - self.set_canned_acl(acl_or_str, key_name, - headers, version_id) + self.set_canned_acl(acl_or_str, key_name, headers=headers) - def get_acl(self, key_name='', headers=None, version_id=None): - query_args = 'acl' - if version_id: - query_args += '&versionId=%s' % version_id + def get_acl(self, key_name='', headers=None): response = self.connection.make_request('GET', self.name, key_name, - query_args=query_args, - headers=headers) + query_args='acl', headers=headers) body = response.read() if response.status == 200: policy = Policy(self) @@ -494,30 +338,26 @@ def make_public(self, recursive=False, headers=None): for key in self: self.set_canned_acl('public-read', key.name, headers=headers) - def add_email_grant(self, permission, email_address, - recursive=False, headers=None): + def add_email_grant(self, permission, email_address, recursive=False, headers=None): """ - Convenience method that provides a quick way to add an email grant - to a bucket. This method retrieves the current ACL, creates a new - grant based on the parameters passed in, adds that grant to the ACL - and then PUT's the new ACL back to S3. + Convenience method that provides a quick way to add an email grant to a bucket. + This method retrieves the current ACL, creates a new grant based on the parameters + passed in, adds that grant to the ACL and then PUT's the new ACL back to S3. + :param permission: The permission being granted. Should be one of: (READ, WRITE, READ_ACP, WRITE_ACP, FULL_CONTROL). + See http://docs.amazonwebservices.com/AmazonS3/2006-03-01/UsingAuthAccess.html for more details on permissions. :type permission: string - :param permission: The permission being granted. Should be one of: - (READ, WRITE, READ_ACP, WRITE_ACP, FULL_CONTROL). + :param email_address: The email address associated with the AWS account your are granting + the permission to. :type email_address: string - :param email_address: The email address associated with the AWS - account your are granting the permission to. + :param recursive: A boolean value to controls whether the command will apply the + grant to all keys within the bucket or not. The default value is False. + By passing a True value, the call will iterate through all keys in the + bucket and apply the same grant to each key. + CAUTION: If you have a lot of keys, this could take a long time! :type recursive: boolean - :param recursive: A boolean value to controls whether the command - will apply the grant to all keys within the bucket - or not. The default value is False. By passing a - True value, the call will iterate through all keys - in the bucket and apply the same grant to each key. - CAUTION: If you have a lot of keys, this could take - a long time! """ if permission not in S3Permissions: raise S3PermissionsError('Unknown Permission: %s' % permission) @@ -535,21 +375,21 @@ def add_user_grant(self, permission, user_id, recursive=False, headers=None): passed in, adds that grant to the ACL and then PUT's the new ACL back to S3. :type permission: string - :param permission: The permission being granted. Should be one of: - (READ, WRITE, READ_ACP, WRITE_ACP, FULL_CONTROL). - + :param permission: The permission being granted. Should be one of: + READ|WRITE|READ_ACP|WRITE_ACP|FULL_CONTROL + See http://docs.amazonwebservices.com/AmazonS3/2006-03-01/UsingAuthAccess.html + for more details on permissions. + :type user_id: string :param user_id: The canonical user id associated with the AWS account your are granting the permission to. - :type recursive: boolean - :param recursive: A boolean value to controls whether the command - will apply the grant to all keys within the bucket - or not. The default value is False. By passing a - True value, the call will iterate through all keys - in the bucket and apply the same grant to each key. - CAUTION: If you have a lot of keys, this could take - a long time! + :type recursive: bool + :param recursive: A boolean value that controls whether the command will apply the + grant to all keys within the bucket or not. The default value is False. + By passing a True value, the call will iterate through all keys in the + bucket and apply the same grant to each key. + CAUTION: If you have a lot of keys, this could take a long time! """ if permission not in S3Permissions: raise S3PermissionsError('Unknown Permission: %s' % permission) @@ -622,6 +462,16 @@ def set_as_logging_target(self, headers=None): policy.acl.add_grant(g2) self.set_acl(policy, headers=headers) + def disable_logging(self, headers=None): + body = self.EmptyBucketLoggingBody + response = self.connection.make_request('PUT', self.name, data=body, + query_args='logging', headers=headers) + body = response.read() + if response.status == 200: + return True + else: + raise S3ResponseError(response.status, response.reason, body) + def get_request_payment(self, headers=None): response = self.connection.make_request('GET', self.name, query_args='requestPayment', headers=headers) @@ -641,81 +491,5 @@ def set_request_payment(self, payer='BucketOwner', headers=None): else: raise S3ResponseError(response.status, response.reason, body) - def configure_versioning(self, versioning, mfa_delete=False, - mfa_token=None, headers=None): - """ - Configure versioning for this bucket. - Note: This feature is currently in beta release and is available - only in the Northern California region. - - :type versioning: bool - :param versioning: A boolean indicating whether version is - enabled (True) or disabled (False). - - :type mfa_delete: bool - :param mfa_delete: A boolean indicating whether the Multi-Factor - Authentication Delete feature is enabled (True) - or disabled (False). If mfa_delete is enabled - then all Delete operations will require the - token from your MFA device to be passed in - the request. - - :type mfa_token: tuple or list of strings - :param mfa_token: A tuple or list consisting of the serial number - from the MFA device and the current value of - the six-digit token associated with the device. - This value is required when you are changing - the status of the MfaDelete property of - the bucket. - """ - if versioning: - ver = 'Enabled' - else: - ver = 'Disabled' - if mfa_delete: - mfa = 'Enabled' - else: - mfa = 'Disabled' - body = self.VersioningBody % (ver, mfa) - if mfa_token: - if not headers: - headers = {} - headers['x-amz-mfa'] = ' '.join(mfa_token) - response = self.connection.make_request('PUT', self.name, data=body, - query_args='versioning', headers=headers) - body = response.read() - if response.status == 200: - return True - else: - raise S3ResponseError(response.status, response.reason, body) - - def get_versioning_status(self, headers=None): - """ - Returns the current status of versioning on the bucket. - - :rtype: dict - :returns: A dictionary containing a key named 'Versioning' - that can have a value of either Enabled, Disabled, - or Suspended. Also, if MFADelete has ever been enabled - on the bucket, the dictionary will contain a key - named 'MFADelete' which will have a value of either - Enabled or Suspended. - """ - response = self.connection.make_request('GET', self.name, - query_args='versioning', headers=headers) - body = response.read() - boto.log.debug(body) - if response.status == 200: - d = {} - ver = re.search(self.VersionRE, body) - if ver: - d['Versioning'] = ver.group(1) - mfa = re.search(self.MFADeleteRE, body) - if mfa: - d['MfaDelete'] = mfa.group(1) - return d - else: - raise S3ResponseError(response.status, response.reason, body) - def delete(self, headers=None): return self.connection.delete_bucket(self.name, headers=headers) diff --git a/boto/s3/bucketlistresultset.py b/boto/s3/bucketlistresultset.py index 9fc79bd..66ed4ee 100644 --- a/boto/s3/bucketlistresultset.py +++ b/boto/s3/bucketlistresultset.py @@ -54,46 +54,4 @@ def __iter__(self): return bucket_lister(self.bucket, prefix=self.prefix, delimiter=self.delimiter, marker=self.marker, headers=self.headers) -def versioned_bucket_lister(bucket, prefix='', delimiter='', - key_marker='', version_id_marker='', headers=None): - """ - A generator function for listing versions in a bucket. - """ - more_results = True - k = None - while more_results: - rs = bucket.get_all_versions(prefix=prefix, key_marker=key_marker, - version_id_marker=version_id_marker, - delimiter=delimiter, headers=headers) - for k in rs: - yield k - key_marker = rs.key_marker - version_id_marker = rs.version_id_marker - more_results= rs.is_truncated - -class VersionedBucketListResultSet: - """ - A resultset for listing versions within a bucket. Uses the bucket_lister - generator function and implements the iterator interface. This - transparently handles the results paging from S3 so even if you have - many thousands of keys within the bucket you can iterate over all - keys in a reasonably efficient manner. - """ - - def __init__(self, bucket=None, prefix='', delimiter='', key_marker='', - version_id_marker='', headers=None): - self.bucket = bucket - self.prefix = prefix - self.delimiter = delimiter - self.key_marker = key_marker - self.version_id_marker = version_id_marker - self.headers = headers - - def __iter__(self): - return versioned_bucket_lister(self.bucket, prefix=self.prefix, - delimiter=self.delimiter, - key_marker=self.key_marker, - version_id_marker=self.version_id_marker, - headers=self.headers) - diff --git a/boto/s3/connection.py b/boto/s3/connection.py index 614de0b..56c9204 100644 --- a/boto/s3/connection.py +++ b/boto/s3/connection.py @@ -23,6 +23,7 @@ import urllib, base64 import time import boto.utils +import types from boto.connection import AWSAuthConnection from boto import handler from boto.s3.bucket import Bucket @@ -84,10 +85,7 @@ def build_path_base(self, bucket, key=''): class Location: DEFAULT = '' EU = 'EU' - USWest = 'us-west-1' -#boto.set_stream_logger('s3') - class S3Connection(AWSAuthConnection): DefaultHost = 's3.amazonaws.com' @@ -115,8 +113,8 @@ def build_post_policy(self, expiration_time, conditions): """ Taken from the AWS book Python examples and modified for use with boto """ - if type(expiration_time) != time.struct_time: - raise 'Policy document must include a valid expiration Time object' + assert type(expiration_time) == time.struct_time, \ + 'Policy document must include a valid expiration Time object' # Convert conditions object mappings to condition statements @@ -230,7 +228,7 @@ def generate_url(self, expires_in, method, bucket='', key='', hmac_copy.update(canonical_str) b64_hmac = base64.encodestring(hmac_copy.digest()).strip() encoded_canonical = urllib.quote_plus(b64_hmac) - self.calling_format.build_path_base(bucket, key) + path = self.calling_format.build_path_base(bucket, key) if query_auth: query_part = '?' + self.QueryString % (encoded_canonical, expires, self.aws_access_key_id) @@ -274,7 +272,7 @@ def get_canonical_user_id(self, headers=None): def get_bucket(self, bucket_name, validate=True, headers=None): bucket = Bucket(self, bucket_name) if validate: - bucket.get_all_keys(headers, maxkeys=0) + rs = bucket.get_all_keys(headers, maxkeys=0) return bucket def lookup(self, bucket_name, validate=True, headers=None): @@ -284,8 +282,7 @@ def lookup(self, bucket_name, validate=True, headers=None): bucket = None return bucket - def create_bucket(self, bucket_name, headers=None, - location=Location.DEFAULT, policy=None): + def create_bucket(self, bucket_name, headers=None, location=Location.DEFAULT, policy=None): """ Creates a new located bucket. By default it's in the USA. You can pass Location.EU to create an European bucket. @@ -303,10 +300,6 @@ def create_bucket(self, bucket_name, headers=None, :param policy: A canned ACL policy that will be applied to the new key in S3. """ - # TODO: Not sure what Exception Type from boto.exception to use. - if not bucket_name.islower(): - raise Exception("Bucket names must be lower case.") - if policy: if headers: headers['x-amz-acl'] = policy diff --git a/boto/s3/key.py b/boto/s3/key.py index a0bf840..ada4352 100644 --- a/boto/s3/key.py +++ b/boto/s3/key.py @@ -27,14 +27,12 @@ import boto.utils from boto.exception import S3ResponseError, S3DataError, BotoClientError from boto.s3.user import User -from boto import UserAgent - +from boto import UserAgent, config try: from hashlib import md5 except ImportError: from md5 import md5 - class Key(object): DefaultContentType = 'application/octet-stream' @@ -58,9 +56,6 @@ def __init__(self, bucket=None, name=None): self.resp = None self.mode = None self.size = None - self.version_id = None - self.source_version_id = None - self.delete_marker = False def __repr__(self): if self.bucket: @@ -83,14 +78,6 @@ def __setattr__(self, name, value): def __iter__(self): return self - def handle_version_headers(self, resp): - self.version_id = resp.getheader('x-amz-version-id', None) - self.source_version_id = resp.getheader('x-amz-copy-source-version-id', None) - if resp.getheader('x-amz-delete-marker', 'false') == 'true': - self.delete_marker = True - else: - self.delete_marker = False - def open_read(self, headers=None, query_args=None): """ Open this key for reading @@ -104,13 +91,9 @@ def open_read(self, headers=None, query_args=None): if self.resp == None: self.mode = 'r' - self.resp = self.bucket.connection.make_request('GET', - self.bucket.name, - self.name, headers, - query_args=query_args) + self.resp = self.bucket.connection.make_request('GET', self.bucket.name, self.name, headers, query_args=query_args) if self.resp.status < 199 or self.resp.status > 299: - body = self.resp.read() - raise S3ResponseError(self.resp.status, self.resp.reason, body) + raise S3ResponseError(self.resp.status, self.resp.reason) response_headers = self.resp.msg self.metadata = boto.utils.get_aws_metadata(response_headers) for name,value in response_headers.items(): @@ -124,7 +107,6 @@ def open_read(self, headers=None, query_args=None): self.content_encoding = value elif name.lower() == 'last-modified': self.last_modified = value - self.handle_version_headers(self.resp) def open_write(self, headers=None): """ @@ -223,8 +205,6 @@ def endElement(self, name, value, connection): self.storage_class = value elif name == 'Owner': pass - elif name == 'VersionId': - self.version_id = value else: setattr(self, name, value) @@ -383,10 +363,8 @@ def sender(http_conn, method, path, data, headers): headers['Content-Length'] = str(self.size) headers['Expect'] = '100-Continue' headers = boto.utils.merge_meta(headers, self.metadata) - resp = self.bucket.connection.make_request('PUT', self.bucket.name, - self.name, headers, - sender=sender) - self.handle_version_headers(resp) + return self.bucket.connection.make_request('PUT', self.bucket.name, + self.name, headers, sender=sender) def compute_md5(self, fp): """ @@ -558,12 +536,10 @@ def set_contents_from_string(self, s, headers=None, replace=True, cb=None, num_c used as the MD5 values of the file. Otherwise, the checksum will be computed. """ fp = StringIO.StringIO(s) - r = self.set_contents_from_file(fp, headers, replace, cb, num_cb, policy) + self.set_contents_from_file(fp, headers, replace, cb, num_cb, policy) fp.close() - return r - def get_file(self, fp, headers=None, cb=None, num_cb=10, - torrent=False, version_id=None): + def get_file(self, fp, headers=None, cb=None, num_cb=10, torrent=False): """ Retrieves a file from an S3 Key @@ -599,13 +575,9 @@ def get_file(self, fp, headers=None, cb=None, num_cb=10, save_debug = self.bucket.connection.debug if self.bucket.connection.debug == 1: self.bucket.connection.debug = 0 - - query_args = '' - if torrent: - query_args = 'torrent' - elif version_id: - query_args = 'versionId=%s' % version_id - self.open('r', headers, query_args=query_args) + + if torrent: torrent = "torrent" + self.open('r', headers, query_args=torrent) for bytes in self: fp.write(bytes) if cb: @@ -640,10 +612,7 @@ def get_torrent_file(self, fp, headers=None, cb=None, num_cb=10): """ return self.get_file(fp, headers, cb, num_cb, torrent=True) - def get_contents_to_file(self, fp, headers=None, - cb=None, num_cb=10, - torrent=False, - version_id=None): + def get_contents_to_file(self, fp, headers=None, cb=None, num_cb=10, torrent=False): """ Retrieve an object from S3 using the name of the Key object as the key in S3. Write the contents of the object to the file pointed @@ -673,13 +642,9 @@ def get_contents_to_file(self, fp, headers=None, """ if self.bucket != None: - self.get_file(fp, headers, cb, num_cb, torrent=torrent, - version_id=version_id) + self.get_file(fp, headers, cb, num_cb, torrent=torrent) - def get_contents_to_filename(self, filename, headers=None, - cb=None, num_cb=10, - torrent=False, - version_id=None): + def get_contents_to_filename(self, filename, headers=None, cb=None, num_cb=10, torrent=False): """ Retrieve an object from S3 using the name of the Key object as the key in S3. Store contents of the object to a file named by 'filename'. @@ -710,8 +675,7 @@ def get_contents_to_filename(self, filename, headers=None, """ fp = open(filename, 'wb') - self.get_contents_to_file(fp, headers, cb, num_cb, torrent=torrent, - version_id=version_id) + self.get_contents_to_file(fp, headers, cb, num_cb, torrent=torrent) fp.close() # if last_modified date was sent from s3, try to set file's timestamp if self.last_modified != None: @@ -721,10 +685,7 @@ def get_contents_to_filename(self, filename, headers=None, os.utime(fp.name, (modified_stamp, modified_stamp)) except Exception: pass - def get_contents_as_string(self, headers=None, - cb=None, num_cb=10, - torrent=False, - version_id=None): + def get_contents_as_string(self, headers=None, cb=None, num_cb=10, torrent=False): """ Retrieve an object from S3 using the name of the Key object as the key in S3. Return the contents of the object as a string. @@ -759,8 +720,7 @@ def get_contents_as_string(self, headers=None, :returns: The contents of the file as a string """ fp = StringIO.StringIO() - self.get_contents_to_file(fp, headers, cb, num_cb, torrent=torrent, - version_id=version_id) + self.get_contents_to_file(fp, headers, cb, num_cb, torrent=torrent) return fp.getvalue() def add_email_grant(self, permission, email_address): diff --git a/boto/sdb/__init__.py b/boto/sdb/__init__.py index df1f95b..42af6a9 100644 --- a/boto/sdb/__init__.py +++ b/boto/sdb/__init__.py @@ -20,9 +20,9 @@ # IN THE SOFTWARE. # +import boto from regioninfo import SDBRegionInfo - def regions(): """ Get all available regions for the SDB service. diff --git a/boto/sdb/connection.py b/boto/sdb/connection.py index 1863b83..28e130a 100644 --- a/boto/sdb/connection.py +++ b/boto/sdb/connection.py @@ -19,8 +19,10 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. +import urllib import xml.sax import threading +import boto from boto import handler from boto.connection import AWSQueryConnection from boto.sdb.domain import Domain, DomainMetaData @@ -30,7 +32,6 @@ from boto.resultset import ResultSet import warnings - class ItemThread(threading.Thread): def __init__(self, name, domain_name, item_names): diff --git a/boto/sdb/db/blob.py b/boto/sdb/db/blob.py index 8c0b66e..d92eb65 100644 --- a/boto/sdb/db/blob.py +++ b/boto/sdb/db/blob.py @@ -49,7 +49,7 @@ def readline(self): return self.file.readline() def next(self): - return self.file.next() + return sefl.file.next() def __iter__(self): return iter(self.file) diff --git a/boto/sdb/db/manager/__init__.py b/boto/sdb/db/manager/__init__.py index 0777796..1d75549 100644 --- a/boto/sdb/db/manager/__init__.py +++ b/boto/sdb/db/manager/__init__.py @@ -66,8 +66,6 @@ def get_manager(cls): db_port = boto.config.getint(db_section, 'db_port', db_port) enable_ssl = boto.config.getint(db_section, 'enable_ssl', enable_ssl) debug = boto.config.getint(db_section, 'debug', debug) - elif hasattr(cls.__bases__[0], "_manager"): - return cls.__bases__[0]._manager if db_type == 'SimpleDB': from sdbmanager import SDBManager return SDBManager(cls, db_name, db_user, db_passwd, diff --git a/boto/sdb/db/manager/pgmanager.py b/boto/sdb/db/manager/pgmanager.py index 73a93f0..4c7e3ad 100644 --- a/boto/sdb/db/manager/pgmanager.py +++ b/boto/sdb/db/manager/pgmanager.py @@ -22,10 +22,8 @@ from boto.sdb.db.model import Model import psycopg2 import psycopg2.extensions -import uuid -import os -import string -from boto.exception import SDBPersistenceError +import uuid, sys, os, string +from boto.exception import * psycopg2.extensions.register_type(psycopg2.extensions.UNICODE) @@ -288,7 +286,7 @@ def lookup(self, cls, name, value): qs += "%s=" % name qs += "%s" if not found: - raise SDBPersistenceError('%s is not a valid field' % name) + raise SDBPersistenceError('%s is not a valid field' % key) qs += ';' print qs self.cursor.execute(qs, values) @@ -315,7 +313,7 @@ def query(self, cls, filters, limit=None, order_by=None): value = self.encode_value(property, value) parts.append(""""%s"%s'%s'""" % (name, op, value)) if not found: - raise SDBPersistenceError('%s is not a valid field' % name) + raise SDBPersistenceError('%s is not a valid field' % key) qs += ','.join(parts) qs += ';' print qs diff --git a/boto/sdb/db/manager/sdbmanager.py b/boto/sdb/db/manager/sdbmanager.py index 1bfa0b0..2bb2440 100644 --- a/boto/sdb/db/manager/sdbmanager.py +++ b/boto/sdb/db/manager/sdbmanager.py @@ -28,10 +28,10 @@ from boto.sdb.db.property import ListProperty, MapProperty from datetime import datetime from boto.exception import SDBPersistenceError +from tempfile import TemporaryFile ISO8601 = '%Y-%m-%dT%H:%M:%SZ' - class SDBConverter: """ Responsible for converting base Python types to format compatible with underlying @@ -290,32 +290,19 @@ def __init__(self, cls, db_name, db_user, db_passwd, self.s3 = None self.bucket = None self.converter = SDBConverter(self) - self._sdb = None - self._domain = None - - @property - def sdb(self): - if self._sdb is None: - self._connect() - return self._sdb - - @property - def domain(self): - if self._domain is None: - self._connect() - return self._domain + self._connect() def _connect(self): - self._sdb = boto.connect_sdb(aws_access_key_id=self.db_user, + self.sdb = boto.connect_sdb(aws_access_key_id=self.db_user, aws_secret_access_key=self.db_passwd, is_secure=self.enable_ssl) # This assumes that the domain has already been created # It's much more efficient to do it this way rather than # having this make a roundtrip each time to validate. # The downside is that if the domain doesn't exist, it breaks - self._domain = self._sdb.lookup(self.db_name, validate=False) - if not self._domain: - self._domain = self._sdb.create_domain(self.db_name) + self.domain = self.sdb.lookup(self.db_name, validate=False) + if not self.domain: + self.domain = self.sdb.create_domain(self.db_name) def _object_lister(self, cls, query_lister): for item in query_lister: @@ -444,8 +431,8 @@ def _build_filter_part(self, cls, filters, order_by=None): query_parts.append("`%s` %s '%s'" % (name, op, val.replace("'", "''"))) type_query = "(`__type__` = '%s'" % cls.__name__ - for subclass in self._get_all_decendents(cls).keys(): - type_query += " or `__type__` = '%s'" % subclass + for subclass in cls.__sub_classes__: + type_query += " or `__type__` = '%s'" % subclass.__name__ type_query +=")" query_parts.append(type_query) @@ -461,14 +448,6 @@ def _build_filter_part(self, cls, filters, order_by=None): return "" - def _get_all_decendents(self, cls): - """Get all decendents for a given class""" - decendents = {} - for sc in cls.__sub_classes__: - decendents[sc.__name__] = sc - decendents.update(self._get_all_decendents(sc)) - return decendents - def query_gql(self, query_string, *args, **kwds): raise NotImplementedError, "GQL queries not supported in SimpleDB" diff --git a/boto/sdb/db/manager/xmlmanager.py b/boto/sdb/db/manager/xmlmanager.py index a3f8074..b12f5df 100644 --- a/boto/sdb/db/manager/xmlmanager.py +++ b/boto/sdb/db/manager/xmlmanager.py @@ -20,9 +20,11 @@ # IN THE SOFTWARE. import boto from boto.utils import find_class, Password +import uuid from boto.sdb.db.key import Key from boto.sdb.db.model import Model from datetime import datetime +from boto.exception import SDBPersistenceError from xml.dom.minidom import getDOMImplementation, parse, parseString, Node ISO8601 = '%Y-%m-%dT%H:%M:%SZ' diff --git a/boto/sdb/db/model.py b/boto/sdb/db/model.py index 3b8e1b6..dc142e8 100644 --- a/boto/sdb/db/model.py +++ b/boto/sdb/db/model.py @@ -20,7 +20,7 @@ # IN THE SOFTWARE. from boto.sdb.db.manager import get_manager -from boto.sdb.db.property import Property +from boto.sdb.db.property import * from boto.sdb.db.key import Key from boto.sdb.db.query import Query import boto @@ -55,7 +55,6 @@ def __init__(cls, name, bases, dict): class Model(object): __metaclass__ = ModelMeta - id = None @classmethod def get_lineage(cls): diff --git a/boto/sdb/db/property.py b/boto/sdb/db/property.py index a48fb9f..61d424a 100644 --- a/boto/sdb/db/property.py +++ b/boto/sdb/db/property.py @@ -23,6 +23,8 @@ from key import Key from boto.utils import Password from boto.sdb.db.query import Query +from tempfile import TemporaryFile + import re import boto import boto.s3.key @@ -61,7 +63,7 @@ def __set__(self, obj, value): if obj._loaded and hasattr(obj, "on_set_%s" % self.name): fnc = getattr(obj, "on_set_%s" % self.name) value = fnc(value) - except Exception: + except Exception, e: boto.log.exception("Exception running on_set_%s" % self.name) setattr(obj, self.slot_name, value) @@ -361,7 +363,14 @@ def __get__(self, obj, objtype): # the object now that is the attribute has actually been accessed. This lazy # instantiation saves unnecessary roundtrips to SimpleDB if isinstance(value, str) or isinstance(value, unicode): - value = self.reference_class(value) + # This is some minor handling to allow us to use the base "Model" class + # as our reference class. If we do so, we're going to assume we're using + # our own class's manager to fetch objects + if hasattr(self.reference_class, "_manager"): + manager = self.reference_class._manager + else: + manager = obj._manager + value = manager.get_object(self.reference_class, value) setattr(obj, self.name, value) return value @@ -405,7 +414,6 @@ class _ReverseReferenceProperty(Property): def __init__(self, model, prop, name): self.__model = model self.__property = prop - self.collection_name = prop self.name = name self.item_type = model @@ -506,8 +514,6 @@ def __set__(self, obj, value): item_type = self.item_type if isinstance(value, item_type): value = [value] - elif value == None: # Override to allow them to set this to "None" to remove everything - value = [] return super(ListProperty, self).__set__(obj,value) diff --git a/boto/sdb/db/test_db.py b/boto/sdb/db/test_db.py index 0c345ab..b790b9e 100644 --- a/boto/sdb/db/test_db.py +++ b/boto/sdb/db/test_db.py @@ -1,7 +1,6 @@ from boto.sdb.db.model import Model -from boto.sdb.db.property import StringProperty, IntegerProperty, BooleanProperty -from boto.sdb.db.property import DateTimeProperty, FloatProperty, ReferenceProperty -from boto.sdb.db.property import PasswordProperty, ListProperty, MapProperty +from boto.sdb.db.property import * +from boto.sdb.db.manager import get_manager from datetime import datetime import time from boto.exception import SDBPersistenceError diff --git a/boto/sdb/domain.py b/boto/sdb/domain.py index 410a88f..3c0def6 100644 --- a/boto/sdb/domain.py +++ b/boto/sdb/domain.py @@ -270,7 +270,7 @@ class DomainDumpParser(ContentHandler): """ def __init__(self, domain): - self.uploader = UploaderThread(domain) + self.uploader = UploaderThread(domain.name) self.item_id = None self.attrs = {} self.attribute = None @@ -300,10 +300,10 @@ def endElement(self, name): self.attrs[attr_name] = [value] elif name == "Item": self.uploader.items[self.item_id] = self.attrs - # Every 20 items we spawn off the uploader - if len(self.uploader.items) >= 20: + # Every 40 items we spawn off the uploader + if len(self.uploader.items) >= 40: self.uploader.start() - self.uploader = UploaderThread(self.domain) + self.uploader = UploaderThread(self.domain.name) elif name == "Domain": # If we're done, spawn off our last Uploader Thread self.uploader.start() @@ -312,8 +312,10 @@ def endElement(self, name): class UploaderThread(Thread): """Uploader Thread""" - def __init__(self, domain): - self.db = domain + def __init__(self, domain_name): + import boto + self.sdb = boto.connect_sdb() + self.db = self.sdb.get_domain(domain_name) self.items = {} Thread.__init__(self) diff --git a/boto/sdb/item.py b/boto/sdb/item.py index d6a56a9..b81e715 100644 --- a/boto/sdb/item.py +++ b/boto/sdb/item.py @@ -89,14 +89,6 @@ def load(self): def save(self, replace=True): self.domain.put_attributes(self.name, self, replace) - def add_value(self, key, value): - if key in self: - if not isinstance(self[key], list): - self[key] = [self[key]] - self[key].append(value) - else: - self[key] = value - def delete(self): self.domain.delete_item(self) diff --git a/boto/sdb/persist/checker.py b/boto/sdb/persist/checker.py index e2146c9..147ea47 100644 --- a/boto/sdb/persist/checker.py +++ b/boto/sdb/persist/checker.py @@ -20,6 +20,7 @@ # IN THE SOFTWARE. from datetime import datetime +import boto from boto.s3.key import Key from boto.s3.bucket import Bucket from boto.sdb.persist import revive_object_from_id diff --git a/boto/sdb/persist/object.py b/boto/sdb/persist/object.py index 993df1e..3646d43 100644 --- a/boto/sdb/persist/object.py +++ b/boto/sdb/persist/object.py @@ -21,7 +21,7 @@ from boto.exception import SDBPersistenceError from boto.sdb.persist import get_manager, object_lister -from boto.sdb.persist.property import Property, ScalarProperty +from boto.sdb.persist.property import * import uuid class SDBBase(type): diff --git a/boto/sdb/persist/property.py b/boto/sdb/persist/property.py index 4776d35..6eea765 100644 --- a/boto/sdb/persist/property.py +++ b/boto/sdb/persist/property.py @@ -20,8 +20,7 @@ # IN THE SOFTWARE. from boto.exception import SDBPersistenceError -from boto.sdb.persist.checker import StringChecker, PasswordChecker, IntegerChecker, BooleanChecker -from boto.sdb.persist.checker import DateTimeChecker, ObjectChecker, S3KeyChecker, S3BucketChecker +from boto.sdb.persist.checker import * from boto.utils import Password class Property(object): diff --git a/boto/sdb/persist/test_persist.py b/boto/sdb/persist/test_persist.py index 080935d..3207e58 100644 --- a/boto/sdb/persist/test_persist.py +++ b/boto/sdb/persist/test_persist.py @@ -1,8 +1,5 @@ from boto.sdb.persist.object import SDBObject -from boto.sdb.persist.property import StringProperty, PositiveIntegerProperty, IntegerProperty -from boto.sdb.persist.property import BooleanProperty, DateTimeProperty, S3KeyProperty -from boto.sdb.persist.property import ObjectProperty, StringListProperty -from boto.sdb.persist.property import PositiveIntegerListProperty, BooleanListProperty, ObjectListProperty +from boto.sdb.persist.property import * from boto.sdb.persist import Manager from datetime import datetime import time diff --git a/boto/sdb/queryresultset.py b/boto/sdb/queryresultset.py index 285a541..a9430f4 100644 --- a/boto/sdb/queryresultset.py +++ b/boto/sdb/queryresultset.py @@ -19,6 +19,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. +from boto.sdb.item import Item def query_lister(domain, query='', max_items=None, attr_names=None): more_results = True diff --git a/boto/services/bs.py b/boto/services/bs.py index 3d70031..aafe867 100755 --- a/boto/services/bs.py +++ b/boto/services/bs.py @@ -21,6 +21,7 @@ # IN THE SOFTWARE. from optparse import OptionParser from boto.services.servicedef import ServiceDef +from boto.services.message import ServiceMessage from boto.services.submit import Submitter from boto.services.result import ResultProcessor import boto diff --git a/boto/services/message.py b/boto/services/message.py index 79f6d19..6bb2e58 100644 --- a/boto/services/message.py +++ b/boto/services/message.py @@ -19,6 +19,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. +import boto from boto.sqs.message import MHMessage from boto.utils import get_ts from socket import gethostname diff --git a/boto/services/result.py b/boto/services/result.py index f6c4407..240085b 100644 --- a/boto/services/result.py +++ b/boto/services/result.py @@ -20,8 +20,9 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. -import os +import getopt, sys, os, time, mimetypes from datetime import datetime, timedelta +from boto.services.servicedef import ServiceDef from boto.utils import parse_ts import boto @@ -82,7 +83,9 @@ def process_record(self, record, path, get_file=True): bucket = boto.lookup('s3', record['Bucket']) for output in outputs: if get_file: - key_name = output.split(';')[0] + key_name, type = output.split(';') + if type: + mimetype = type.split('=')[1] key = bucket.lookup(key_name) file_name = os.path.join(path, key_name) print 'retrieving file: %s to %s' % (key_name, file_name) @@ -108,8 +111,8 @@ def get_results_from_bucket(self, path): if bucket: print 'No output queue or domain, just retrieving files from output_bucket' for key in bucket: - file_name = os.path.join(path, key) - print 'retrieving file: %s to %s' % (key, file_name) + file_name = os.path.join(path, key_name) + print 'retrieving file: %s to %s' % (key_name, file_name) key.get_contents_to_filename(file_name) self.num_files + 1 diff --git a/boto/services/service.py b/boto/services/service.py index 8ee1a8b..942c47f 100644 --- a/boto/services/service.py +++ b/boto/services/service.py @@ -23,12 +23,14 @@ from boto.services.message import ServiceMessage from boto.services.servicedef import ServiceDef from boto.pyami.scriptbase import ScriptBase +from boto.exception import S3ResponseError from boto.utils import get_ts +import StringIO import time import os +import sys, traceback import mimetypes - class Service(ScriptBase): # Time required to process a transaction @@ -153,7 +155,7 @@ def main(self, notify=False): else: empty_reads += 1 time.sleep(self.loop_delay) - except Exception: + except Exception, e: boto.log.exception('Service Failed') empty_reads += 1 self.notify('Service: %s Shutting Down' % self.name) diff --git a/boto/services/sonofmmm.py b/boto/services/sonofmmm.py index acb7e61..5b94f90 100644 --- a/boto/services/sonofmmm.py +++ b/boto/services/sonofmmm.py @@ -22,8 +22,7 @@ import boto from boto.services.service import Service from boto.services.message import ServiceMessage -import os -import mimetypes +import os, time, mimetypes class SonOfMMM(Service): diff --git a/boto/services/submit.py b/boto/services/submit.py index 89c439c..dfa71f2 100644 --- a/boto/services/submit.py +++ b/boto/services/submit.py @@ -19,9 +19,8 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. -import time -import os - +import boto +import time, os class Submitter: diff --git a/boto/sqs/__init__.py b/boto/sqs/__init__.py index 303611f..0b3924c 100644 --- a/boto/sqs/__init__.py +++ b/boto/sqs/__init__.py @@ -24,9 +24,10 @@ boto.check_extensions(__name__, __path__) +from queue import Queue +from message import Message, MHMessage, EncodedMHMessage from regioninfo import SQSRegionInfo - def regions(): """ Get all available regions for the SQS service. diff --git a/boto/sqs/connection.py b/boto/sqs/connection.py index 42a3e21..fd13d2a 100644 --- a/boto/sqs/connection.py +++ b/boto/sqs/connection.py @@ -20,13 +20,15 @@ # IN THE SOFTWARE. from boto.connection import AWSQueryConnection +import xml.sax from boto.sqs.regioninfo import SQSRegionInfo from boto.sqs.queue import Queue from boto.sqs.message import Message from boto.sqs.attributes import Attributes +from boto import handler +from boto.resultset import ResultSet from boto.exception import SQSError - class SQSConnection(AWSQueryConnection): """ A Connection to the SQS Service. @@ -155,37 +157,9 @@ def receive_message(self, queue, number_messages=1, visibility_timeout=None, queue.id, queue) def delete_message(self, queue, message): - """ - Delete a message from a queue. - - :type queue: A :class:`boto.sqs.queue.Queue` object - :param queue: The Queue from which messages are read. - - :type message: A :class:`boto.sqs.message.Message` object - :param message: The Message to be deleted - - :rtype: bool - :return: True if successful, False otherwise. - """ params = {'ReceiptHandle' : message.receipt_handle} return self.get_status('DeleteMessage', params, queue.id) - def delete_message_from_handle(self, queue, receipt_handle): - """ - Delete a message from a queue, given a receipt handle. - - :type queue: A :class:`boto.sqs.queue.Queue` object - :param queue: The Queue from which messages are read. - - :type receipt_handle: str - :param receipt_handle: The receipt handle for the message - - :rtype: bool - :return: True if successful, False otherwise. - """ - params = {'ReceiptHandle' : receipt_handle} - return self.get_status('DeleteMessage', params, queue.id) - def send_message(self, queue, message_content): params = {'MessageBody' : message_content} return self.get_object('SendMessage', params, Message, queue.id, verb='POST') diff --git a/boto/sqs/queue.py b/boto/sqs/queue.py index 82d2cef..48b6115 100644 --- a/boto/sqs/queue.py +++ b/boto/sqs/queue.py @@ -23,9 +23,12 @@ Represents an SQS Queue """ +import xml.sax import urlparse +from boto.exception import SQSError +from boto.handler import XmlHandler from boto.sqs.message import Message - +from boto.resultset import ResultSet class Queue: diff --git a/boto/tests/test.py b/boto/tests/test.py index e3c3ce7..c6175ca 100755 --- a/boto/tests/test.py +++ b/boto/tests/test.py @@ -30,7 +30,6 @@ from boto.tests.test_sqsconnection import SQSConnectionTest from boto.tests.test_s3connection import S3ConnectionTest -from boto.tests.test_s3versioning import S3VersionTest from boto.tests.test_ec2connection import EC2ConnectionTest from boto.tests.test_sdbconnection import SDBConnectionTest @@ -67,9 +66,6 @@ def main(): suite.addTest(unittest.makeSuite(SDBConnectionTest)) elif testsuite == 's3': suite.addTest(unittest.makeSuite(S3ConnectionTest)) - suite.addTest(unittest.makeSuite(S3VersionTest)) - elif testsuite == 's3ver': - suite.addTest(unittest.makeSuite(S3VersionTest)) elif testsuite == 'sqs': suite.addTest(unittest.makeSuite(SQSConnectionTest)) elif testsuite == 'ec2': diff --git a/boto/tests/test_ec2connection.py b/boto/tests/test_ec2connection.py index db6e2af..8f1fb59 100644 --- a/boto/tests/test_ec2connection.py +++ b/boto/tests/test_ec2connection.py @@ -37,7 +37,7 @@ class EC2ConnectionTest (unittest.TestCase): def test_1_basic(self): # this is my user_id, if you want to run these tests you should # replace this with yours or they won't work - user_id = '963068290131' + user_id = '084307701560' print '--- running EC2Connection tests ---' c = EC2Connection() # get list of private AMI's diff --git a/boto/tests/test_s3connection.py b/boto/tests/test_s3connection.py index a952d65..7afc8d2 100644 --- a/boto/tests/test_s3connection.py +++ b/boto/tests/test_s3connection.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- + # Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ # # Permission is hereby granted, free of charge, to any person obtaining a @@ -113,21 +113,15 @@ def test_1_basic(self): mdkey2 = 'meta2' mdval2 = 'This is the second metadata value' k.set_metadata(mdkey2, mdval2) - # try a unicode metadata value - mdval3 = u'föö' - mdkey3 = 'meta3' - k.set_metadata(mdkey3, mdval3) k.set_contents_from_string(s1) k = bucket.lookup('has_metadata') assert k.get_metadata(mdkey1) == mdval1 assert k.get_metadata(mdkey2) == mdval2 - assert k.get_metadata(mdkey3) == mdval3 k = bucket.new_key() k.name = 'has_metadata' k.get_contents_as_string() assert k.get_metadata(mdkey1) == mdval1 assert k.get_metadata(mdkey2) == mdval2 - assert k.get_metadata(mdkey3) == mdval3 bucket.delete_key(k) # test list and iterator rs1 = bucket.list() diff --git a/boto/tests/test_sqsconnection.py b/boto/tests/test_sqsconnection.py index 0fbd1f1..f24ad32 100644 --- a/boto/tests/test_sqsconnection.py +++ b/boto/tests/test_sqsconnection.py @@ -98,8 +98,8 @@ def test_1_basic(self): message = queue.read() assert message == None - # now wait 30 seconds and try again - time.sleep(30) + # now wait 10 seconds and try again + time.sleep(10) message = queue.read() assert message diff --git a/boto/utils.py b/boto/utils.py index 255d42f..db16d30 100644 --- a/boto/utils.py +++ b/boto/utils.py @@ -36,12 +36,13 @@ Some handy utility functions used by several classes. """ +import base64 +import hmac import re -import urllib -import urllib2 -import subprocess -import StringIO -import time +import urllib, urllib2 +import imp +import subprocess, os, StringIO +import time, datetime import logging.handlers import boto import tempfile @@ -91,11 +92,10 @@ def canonical_string(method, path, headers, expires=None): buf = "%s\n" % method for key in sorted_header_keys: - val = interesting_headers[key] if key.startswith(AMAZON_HEADER_PREFIX): - buf += "%s:%s\n" % (key, val) + buf += "%s:%s\n" % (key, interesting_headers[key]) else: - buf += "%s\n" % val + buf += "%s\n" % interesting_headers[key] # don't include anything after the first ? in the resource... buf += "%s" % path.split('?')[0] @@ -111,14 +111,6 @@ def canonical_string(method, path, headers, expires=None): buf += "?location" elif re.search("[&?]requestPayment($|=|&)", path): buf += "?requestPayment" - elif re.search("[&?]versions($|=|&)", path): - buf += "?versions" - elif re.search("[&?]versioning($|=|&)", path): - buf += "?versioning" - else: - m = re.search("[&?]versionId=([^&]+)($|=|&)", path) - if m: - buf += '?versionId=' + m.group(1) return buf @@ -138,8 +130,7 @@ def get_aws_metadata(headers): metadata = {} for hkey in headers.keys(): if hkey.lower().startswith(METADATA_PREFIX): - val = urllib.unquote_plus(headers[hkey]) - metadata[hkey[len(METADATA_PREFIX):]] = unicode(val, 'utf-8') + metadata[hkey[len(METADATA_PREFIX):]] = headers[hkey] del headers[hkey] return metadata @@ -223,6 +214,7 @@ def find_class(module_name, class_name=None): if class_name: module_name = "%s.%s" % (module_name, class_name) modules = module_name.split('.') + path = None c = None try: @@ -255,6 +247,7 @@ def fetch_file(uri, file=None, username=None, password=None): if file == None: file = tempfile.NamedTemporaryFile() try: + working_dir = boto.config.get("General", "working_dir") if uri.startswith('s3://'): bucket_name, key_name = uri[len('s3://'):].split('/', 1) c = boto.connect_s3() @@ -347,6 +340,11 @@ def emit(self, record): without having to resort to cut and paste inheritance but, no. """ try: + import smtplib + try: + from email.Utils import formatdate + except: + formatdate = self.date_time port = self.mailport if not port: port = smtplib.SMTP_PORT @@ -355,7 +353,7 @@ def emit(self, record): msg = self.format(record) msg = "From: %s\r\nTo: %s\r\nSubject: %s\r\nDate: %s\r\n\r\n%s" % ( self.fromaddr, - ','.join(self.toaddrs), + string.join(self.toaddrs, ","), self.getSubject(record), formatdate(), msg) smtp.sendmail(self.fromaddr, self.toaddrs, msg) @@ -462,6 +460,7 @@ def _insert_item(self, item): def _manage_size(self): while len(self._dict) > self.capacity: + olditem = self._dict[self.tail.key] del self._dict[self.tail.key] if self.tail != self.head: self.tail = self.tail.previous diff --git a/boto/vpc/__init__.py b/boto/vpc/__init__.py index 16c420d..80b0073 100644 --- a/boto/vpc/__init__.py +++ b/boto/vpc/__init__.py @@ -23,7 +23,12 @@ Represents a connection to the EC2 service. """ +import urllib +import base64 +import boto +from boto import config from boto.ec2.connection import EC2Connection +from boto.resultset import ResultSet from boto.vpc.vpc import VPC from boto.vpc.customergateway import CustomerGateway from boto.vpc.vpngateway import VpnGateway, Attachment @@ -376,7 +381,7 @@ def delete_dhcp_options(self, dhcp_options_id): :rtype: bool :return: True if successful """ - params = {'DhcpOptionsId': dhcp_options_id} + params = {'DhcpOptionsId': subnet_id} return self.get_status('DeleteDhcpOptions', params) def associate_dhcp_options(self, dhcp_options_id, vpc_id): @@ -392,7 +397,7 @@ def associate_dhcp_options(self, dhcp_options_id, vpc_id): :rtype: bool :return: True if successful """ - params = {'DhcpOptionsId': dhcp_options_id, + params = {'DhcpOptionsId': dhcp_option, 'VpcId' : vpc_id} return self.get_status('AssociateDhcpOptions', params) @@ -433,7 +438,7 @@ def get_all_vpn_connections(self, vpn_connection_ids=None, filters=None): params[('Filter.%d.Key' % i)] = filter[0] params[('Filter.%d.Value.1')] = filter[1] i += 1 - return self.get_list('DescribeVpnConnections', params, [('item', VpnConnection)]) + return self.get_list('DescribeVpnConnections', params, [('item', VPNConnection)]) def create_vpn_connection(self, type, customer_gateway_id, vpn_gateway_id): """ diff --git a/boto/vpc/vpnconnection.py b/boto/vpc/vpnconnection.py index c02789b..42739d9 100644 --- a/boto/vpc/vpnconnection.py +++ b/boto/vpc/vpnconnection.py @@ -34,7 +34,7 @@ def __init__(self, connection=None): self.customer_gateway_configuration = None self.type = None self.customer_gateway_id = None - self.vpn_gateway_id = None + self.vpn_gateway_id = Nonen def __repr__(self): return 'VpnConnection:%s' % self.id diff --git a/docs/fabfile.py b/docs/fabfile.py new file mode 100644 index 0000000..9f6a265 --- /dev/null +++ b/docs/fabfile.py @@ -0,0 +1,33 @@ +from fabric.operations import sudo, local, put + +def deploy(**kwargs): + remote_path = '/var/www' + if kwargs.get('remote_path', None): + remote_path = kwargs['remote_path'] + + # Update + local("svn up ../") + rev = local("svn info | grep Revision") + rev = rev.replace("Revision: ", "").strip() + conf = open('source/conf-orig.py', 'r+b').read() + open('source/conf.py', 'w+b').write(conf.replace('release = "HEAD"', 'release = "%s"' % rev)) + tmp_folder_name = 'boto-docs.r%s' % rev + archive_name = '%s.tar.gz' % tmp_folder_name + + # Clean + local("rm -rf %s" % tmp_folder_name) + local("rm -f %s" % archive_name) + + # Build + local("make html") + local("mv build/html %s" % tmp_folder_name) + local("tar zcf %s %s" % (archive_name, tmp_folder_name)) + + # Deploy + put(archive_name, '~/') + sudo("rm -f %s/%s && mv ~/%s %s/%s" % (remote_path, archive_name, archive_name, remote_path, archive_name)) + sudo("cd %s && rm -rf %s && tar zxf %s" % (remote_path, tmp_folder_name, archive_name)) + sudo("cd %s && rm -f boto-docs && ln -s %s boto-docs" % (remote_path, tmp_folder_name)) + + # Validate + sudo("ls -al %s" % remote_path) \ No newline at end of file diff --git a/docs/source/conf-orig.py b/docs/source/conf-orig.py new file mode 100644 index 0000000..cf08240 --- /dev/null +++ b/docs/source/conf-orig.py @@ -0,0 +1,206 @@ +# -*- coding: utf-8 -*- +# +# boto documentation build configuration file, created by +# sphinx-quickstart on Tue Sep 15 13:34:43 2009. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys, os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.append(os.path.abspath('.')) + +# -- General configuration ----------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx.ext.todo'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'boto' +copyright = u'2009, Mitch Garnaat' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '1.8' +# The full version, including alpha/beta/rc tags. +release = "HEAD" #'1.8d' +try: + import subprocess + p = subprocess.Popen(["svn info ../../boto | grep Revision | awk '{print $2}'"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + release = p.stdout.read().strip() + print p.stderr.read() +except: + pass + + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of documents that shouldn't be included in the build. +#unused_docs = [] + +# List of directories, relative to source directory, that shouldn't be searched +# for source files. +exclude_trees = [] + +# The reST default role (used for this markup: `text`) to use for all documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +html_theme = 'sphinxdoc' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +html_title = "boto v%s (r%s)" % (version, release) + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_use_modindex = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = '' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'botodoc' + + +# -- Options for LaTeX output -------------------------------------------------- + +# The paper size ('letter' or 'a4'). +#latex_paper_size = 'letter' + +# The font size ('10pt', '11pt' or '12pt'). +#latex_font_size = '10pt' + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'boto.tex', u'boto Documentation', + u'Mitch Garnaat', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# Additional stuff for the LaTeX preamble. +#latex_preamble = '' + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_use_modindex = True + + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = {'http://docs.python.org/': None} diff --git a/docs/source/index.rst b/docs/source/index.rst index 24b6ba0..f36df7d 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,34 +1,10 @@ .. _index: -=============================================== -boto: A Python interface to Amazon Web Services -=============================================== - -An integrated interface to current and future infrastructural services -offered by Amazon Web Services. - -Currently, this includes: - -- Simple Storage Service (S3) -- Simple Queue Service (SQS) -- Elastic Compute Cloud (EC2) - - * Elastic Load Balancer (ELB) - * CloudWatch - * AutoScale - -- Mechanical Turk -- SimpleDB (SDB) - See SimpleDbPage for details -- CloudFront -- Virtual Private Cloud (VPC) - -Follow project updates on Twitter (http://twitter.com/pythonboto). - -Follow Mitch on Twitter (http://twitter.com/garnaat). - +================== +boto Documentation +================== -Documentation Contents ----------------------- +Contents: .. toctree:: :maxdepth: 2 diff --git a/docs/source/ref/index.rst b/docs/source/ref/index.rst index ca1c930..cd57eca 100644 --- a/docs/source/ref/index.rst +++ b/docs/source/ref/index.rst @@ -17,7 +17,6 @@ API Reference mashups mturk pyami - rds s3 sdb services diff --git a/docs/source/vpc_tut.rst b/docs/source/vpc_tut.rst index 0040866..498daed 100644 --- a/docs/source/vpc_tut.rst +++ b/docs/source/vpc_tut.rst @@ -1,4 +1,4 @@ -.. _vpc_tut: +.. _ec2_tut: ======================================= An Introduction to boto's VPC interface diff --git a/setup.py b/setup.py index f70e532..0a93426 100644 --- a/setup.py +++ b/setup.py @@ -34,17 +34,14 @@ long_description="Python interface to Amazon's Web Services.", author = "Mitch Garnaat", author_email = "mitch@garnaat.com", - scripts = ["bin/sdbadmin", "bin/elbadmin", "bin/cfadmin", - "bin/s3put", "bin/fetch_file", "bin/launch_instance", - "bin/list_instances", "bin/taskadmin", "bin/kill_instance", - "bin/bundle_image"], + scripts = ["bin/sdbadmin", "bin/elbadmin", "bin/s3put", "bin/fetch_file", "bin/launch_instance", 'bin/list_instances', "bin/taskadmin"], url = "http://code.google.com/p/boto/", packages = [ 'boto', 'boto.sqs', 'boto.s3', 'boto.ec2', 'boto.ec2.cloudwatch', 'boto.ec2.autoscale', 'boto.ec2.elb', 'boto.sdb', 'boto.sdb.persist', 'boto.sdb.db', 'boto.sdb.db.manager', 'boto.mturk', 'boto.pyami', 'boto.mashups', 'boto.contrib', 'boto.manage', 'boto.services', 'boto.tests', 'boto.cloudfront', 'boto.rds', 'boto.vpc', - 'boto.fps', 'boto.emr'], + 'boto.fps'], license = 'MIT', platforms = 'Posix; MacOS X; Windows', classifiers = [ 'Development Status :: 3 - Alpha',