Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

* S3/S3.py: Support for buckets stored in Europe, access now

	  goes via <bucket>.s3.amazonaws.com where possible.



git-svn-id: https://s3tools.svn.sourceforge.net/svnroot/s3tools/s3cmd/trunk@155 830e0280-6d2a-0410-9c65-932aecc39d9d
  • Loading branch information...
commit dc758146996ea3be1cce6c3fc7a8137a0b972861 1 parent 8215784
@mludvig mludvig authored
View
5 ChangeLog
@@ -1,3 +1,8 @@
+2007-11-13 Michal Ludvig <michal@logix.cz>
+
+ * S3/S3.py: Support for buckets stored in Europe, access now
+ goes via <bucket>.s3.amazonaws.com where possible.
+
2007-11-12 Michal Ludvig <michal@logix.cz>
* s3cmd: Support for storing file attributes (like ownership,
View
2  NEWS
@@ -1,5 +1,7 @@
s3cmd 0.9.5 - ???
===========
+* Support for buckets created in Europe
+* Initial 'sync' support, for now local to s3 direction only
* Much better handling of multiple args to put, get and del
* Tries to use ElementTree from any available module
* Support for buckets with over 1000 objects.
View
4 S3/Config.py
@@ -13,7 +13,8 @@ class Config(object):
_doc = {}
access_key = ""
secret_key = ""
- host = "s3.amazonaws.com"
+ host_base = "s3.amazonaws.com"
+ host_bucket = "%(bucket)s.s3.amazonaws.com"
verbosity = logging.WARNING
send_chunk = 4096
recv_chunk = 4096
@@ -42,6 +43,7 @@ class Config(object):
gpg_encrypt = "%(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s"
gpg_decrypt = "%(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s"
use_https = False
+ bucket_location = "US"
## Creating a singleton
def __new__(self, configfile = None):
View
78 S3/S3.py
@@ -39,7 +39,7 @@ def __str__(self):
retval = "%d (%s)" % (self.status, self.reason)
try:
retval += (": %s" % self.info["Code"])
- except AttributeError:
+ except (AttributeError, KeyError):
pass
return retval
@@ -83,19 +83,30 @@ class S3(object):
def __init__(self, config):
self.config = config
- def get_connection(self):
- if self.config.use_https:
- return httplib.HTTPSConnection(self.config.host)
+ def get_connection(self, bucket):
if self.config.proxy_host != "":
return httplib.HTTPConnection(self.config.proxy_host, self.config.proxy_port)
else:
- return httplib.HTTPConnection(self.config.host)
+ if self.config.use_https:
+ return httplib.HTTPSConnection(self.get_hostname(bucket))
+ else:
+ return httplib.HTTPConnection(self.get_hostname(bucket))
- def format_resource(self, resource):
- if self.config.proxy_host != "":
- resource = "http://%s%s" % (self.config.host, resource)
+ def get_hostname(self, bucket):
+ if bucket:
+ host = self.config.host_bucket % { 'bucket' : bucket }
+ else:
+ host = self.config.host_base
+ debug('get_hostname(): ' + host)
+ return host
- return resource
+ def format_uri(self, resource):
+ if self.config.proxy_host != "":
+ uri = "http://%s%s" % (self.get_hostname(resource['bucket']), resource['uri'])
+ else:
+ uri = resource['uri']
+ debug('format_uri(): ' + uri)
+ return uri
## Commands / Actions
def list_all_buckets(self):
@@ -126,12 +137,18 @@ def _get_contents(data):
response['list'] = list
return response
- def bucket_create(self, bucket):
+ def bucket_create(self, bucket, bucket_location = None):
self.check_bucket_name(bucket)
headers = SortedDict()
- headers["content-length"] = 0
+ body = ""
+ if bucket_location and bucket_location.strip().upper() != "US":
+ body = "<CreateBucketConfiguration><LocationConstraint>"
+ body += bucket_location.strip().upper()
+ body += "</LocationConstraint></CreateBucketConfiguration>"
+ debug("bucket_location: " + body)
+ headers["content-length"] = len(body)
request = self.create_request("BUCKET_CREATE", bucket = bucket, headers = headers)
- response = self.send_request(request)
+ response = self.send_request(request, body)
return response
def bucket_delete(self, bucket):
@@ -140,9 +157,9 @@ def bucket_delete(self, bucket):
return response
def bucket_info(self, bucket):
- request = self.create_request("BUCKET_LIST", bucket = bucket + "?location")
+ request = self.create_request("BUCKET_LIST", bucket = bucket, extra = "?location")
response = self.send_request(request)
- response['bucket-location'] = getTextFromXml(response['data'], ".//LocationConstraint") or "any"
+ response['bucket-location'] = getTextFromXml(response['data'], "LocationConstraint") or "any"
return response
def object_put(self, filename, bucket, object, extra_headers = None):
@@ -237,12 +254,14 @@ def urlencode_string(self, string):
debug("String '%s' encoded to '%s'" % (string, encoded))
return encoded
- def create_request(self, operation, bucket = None, object = None, headers = None, **params):
- resource = "/"
+ def create_request(self, operation, bucket = None, object = None, headers = None, extra = None, **params):
+ resource = { 'bucket' : None, 'uri' : "/" }
if bucket:
- resource += str(bucket)
+ resource['bucket'] = str(bucket)
if object:
- resource += "/" + self.urlencode_string(object)
+ resource['uri'] = "/" + self.urlencode_string(object)
+ if extra:
+ resource['uri'] += extra
if not headers:
headers = SortedDict()
@@ -265,21 +284,22 @@ def create_request(self, operation, bucket = None, object = None, headers = None
else:
param_str += "&%s" % param
if param_str != "":
- resource += "?" + param_str[1:]
- debug("CreateRequest: resource=" + resource)
+ resource['uri'] += "?" + param_str[1:]
+ debug("CreateRequest: resource[uri]=" + resource['uri'])
return (method_string, resource, headers)
- def send_request(self, request):
+ def send_request(self, request, body = None):
method_string, resource, headers = request
info("Processing request, please wait...")
- conn = self.get_connection()
- conn.request(method_string, self.format_resource(resource), {}, headers)
+ conn = self.get_connection(resource['bucket'])
+ conn.request(method_string, self.format_uri(resource), body, headers)
response = {}
http_response = conn.getresponse()
response["status"] = http_response.status
response["reason"] = http_response.reason
response["headers"] = convertTupleListToDict(http_response.getheaders())
response["data"] = http_response.read()
+ debug("Response: " + str(response))
conn.close()
if response["status"] < 200 or response["status"] > 299:
raise S3Error(response)
@@ -288,9 +308,9 @@ def send_request(self, request):
def send_file(self, request, file):
method_string, resource, headers = request
info("Sending file '%s', please wait..." % file.name)
- conn = self.get_connection()
+ conn = self.get_connection(resource['bucket'])
conn.connect()
- conn.putrequest(method_string, self.format_resource(resource))
+ conn.putrequest(method_string, self.format_uri(resource))
for header in headers.keys():
conn.putheader(header, str(headers[header]))
conn.endheaders()
@@ -319,9 +339,9 @@ def send_file(self, request, file):
def recv_file(self, request, stream):
method_string, resource, headers = request
info("Receiving file '%s', please wait..." % stream.name)
- conn = self.get_connection()
+ conn = self.get_connection(resource['bucket'])
conn.connect()
- conn.putrequest(method_string, self.format_resource(resource))
+ conn.putrequest(method_string, self.format_uri(resource))
for header in headers.keys():
conn.putheader(header, str(headers[header]))
conn.endheaders()
@@ -369,7 +389,9 @@ def sign_headers(self, method, resource, headers):
for header in headers.keys():
if header.startswith("x-amz-"):
h += header+":"+str(headers[header])+"\n"
- h += resource
+ if resource['bucket']:
+ h += "/" + resource['bucket']
+ h += resource['uri']
debug("SignHeaders: " + repr(h))
return base64.encodestring(hmac.new(self.config.secret_key, h, sha).digest()).strip()
View
5 S3/Utils.py
@@ -59,7 +59,10 @@ def getListFromXml(xml, node):
def getTextFromXml(xml, xpath):
tree = ET.fromstring(xml)
xmlns = getNameSpace(tree)
- return tree.findtext(fixupXPath(xmlns, xpath))
+ if tree.tag.endswith(xpath):
+ return tree.text
+ else:
+ return tree.findtext(fixupXPath(xmlns, xpath))
def dateS3toPython(date):
date = re.compile("\.\d\d\dZ").sub(".000Z", date)
View
4 s3cmd
@@ -133,10 +133,9 @@ def cmd_bucket_create(args):
uri = S3Uri(args[0])
if not uri.type == "s3" or not uri.has_bucket() or uri.has_object():
raise ParameterError("Expecting S3 URI with just the bucket name set instead of '%s'" % args[0])
-
try:
s3 = S3(Config())
- response = s3.bucket_create(uri.bucket())
+ response = s3.bucket_create(uri.bucket(), cfg.bucket_location)
except S3Error, e:
if S3.codes.has_key(e.info["Code"]):
error(S3.codes[e.info["Code"]] % uri.bucket())
@@ -648,6 +647,7 @@ if __name__ == '__main__':
optparser.add_option( "--no-delete-removed", dest="delete_removed", action="store_false", help="Don't delete remote objects.")
optparser.add_option("-p", "--preserve", dest="preserve_attrs", action="store_true", help="Preserve filesystem attributes (mode, ownership, timestamps). Default for [sync] command.")
optparser.add_option( "--no-preserve", dest="preserve_attrs", action="store_false", help="Don't store FS attributes")
+ optparser.add_option( "--bucket-location", dest="bucket_location", help="Datacentre to create bucket in. Either EU or US (default)")
optparser.add_option("-m", "--mime-type", dest="default_mime_type", type="mimetype", metavar="MIME/TYPE", help="Default MIME-type to be set for objects stored.")
optparser.add_option("-M", "--guess-mime-type", dest="guess_mime_type", action="store_true", help="Guess MIME-type of files by their extension. Falls back to default MIME-Type as specified by --mime-type option")
Please sign in to comment.
Something went wrong with that request. Please try again.