Skip to content
Newer
Older
100644 521 lines (470 sloc) 18.6 KB
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
1 ## Amazon S3 manager
2 ## Author: Michal Ludvig <michal@logix.cz>
3 ## http://www.logix.cz/michal
4 ## License: GPL Version 2
5
a580bd1 - New feature - allow "get" to stdout
mludvig authored Feb 19, 2007
6 import sys
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
7 import os, os.path
8 import base64
9 import md5
10 import sha
11 import hmac
12 import httplib
13 import logging
1202afb 2008-02-11 Michal Ludvig <michal@logix.cz>
ludvigm authored Feb 11, 2008
14 import mimetypes
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
15 from logging import debug, info, warning, error
16 from stat import ST_SIZE
17
18 from Utils import *
19 from SortedDict import SortedDict
20 from BidirMap import BidirMap
21 from Config import Config
5415fba 2008-04-29 Michal Ludvig <michal@logix.cz>
ludvigm authored Apr 29, 2008
22 from Exceptions import *
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
23
24 class S3(object):
25 http_methods = BidirMap(
26 GET = 0x01,
27 PUT = 0x02,
28 HEAD = 0x04,
29 DELETE = 0x08,
30 MASK = 0x0F,
31 )
32
33 targets = BidirMap(
34 SERVICE = 0x0100,
35 BUCKET = 0x0200,
36 OBJECT = 0x0400,
37 MASK = 0x0700,
38 )
39
40 operations = BidirMap(
41 UNDFINED = 0x0000,
42 LIST_ALL_BUCKETS = targets["SERVICE"] | http_methods["GET"],
43 BUCKET_CREATE = targets["BUCKET"] | http_methods["PUT"],
44 BUCKET_LIST = targets["BUCKET"] | http_methods["GET"],
45 BUCKET_DELETE = targets["BUCKET"] | http_methods["DELETE"],
46 OBJECT_PUT = targets["OBJECT"] | http_methods["PUT"],
47 OBJECT_GET = targets["OBJECT"] | http_methods["GET"],
48 OBJECT_HEAD = targets["OBJECT"] | http_methods["HEAD"],
49 OBJECT_DELETE = targets["OBJECT"] | http_methods["DELETE"],
50 )
51
52 codes = {
53 "NoSuchBucket" : "Bucket '%s' does not exist",
54 "AccessDenied" : "Access to bucket '%s' was denied",
55 "BucketAlreadyExists" : "Bucket '%s' already exists",
56 }
57
3ecbb36 2008-02-11 Michal Ludvig <michal@logix.cz>
ludvigm authored Feb 11, 2008
58 ## S3 sometimes sends HTTP-307 response
59 redir_map = {}
60
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
61 def __init__(self, config):
62 self.config = config
63
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored Nov 13, 2007
64 def get_connection(self, bucket):
1fb8944 2007-07-05 Michal Ludvig <michal@logix.cz>
ludvigm authored Jul 5, 2007
65 if self.config.proxy_host != "":
66 return httplib.HTTPConnection(self.config.proxy_host, self.config.proxy_port)
67 else:
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored Nov 13, 2007
68 if self.config.use_https:
69 return httplib.HTTPSConnection(self.get_hostname(bucket))
70 else:
71 return httplib.HTTPConnection(self.get_hostname(bucket))
1fb8944 2007-07-05 Michal Ludvig <michal@logix.cz>
ludvigm authored Jul 5, 2007
72
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored Nov 13, 2007
73 def get_hostname(self, bucket):
e353d9a * s3cmd, S3/S3.py, S3/Config.py: Removed --use-old-connect-method
ludvigm authored Sep 3, 2008
74 if bucket and self.check_bucket_name_dns_conformity(bucket):
3ecbb36 2008-02-11 Michal Ludvig <michal@logix.cz>
ludvigm authored Feb 11, 2008
75 if self.redir_map.has_key(bucket):
76 host = self.redir_map[bucket]
77 else:
78 host = self.config.host_bucket % { 'bucket' : bucket }
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored Nov 13, 2007
79 else:
80 host = self.config.host_base
3ecbb36 2008-02-11 Michal Ludvig <michal@logix.cz>
ludvigm authored Feb 11, 2008
81 debug('get_hostname(%s): %s' % (bucket, host))
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored Nov 13, 2007
82 return host
1fb8944 2007-07-05 Michal Ludvig <michal@logix.cz>
ludvigm authored Jul 5, 2007
83
3ecbb36 2008-02-11 Michal Ludvig <michal@logix.cz>
ludvigm authored Feb 11, 2008
84 def set_hostname(self, bucket, redir_hostname):
85 self.redir_map[bucket] = redir_hostname
86
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored Nov 13, 2007
87 def format_uri(self, resource):
e353d9a * s3cmd, S3/S3.py, S3/Config.py: Removed --use-old-connect-method
ludvigm authored Sep 3, 2008
88 if resource['bucket'] and not self.check_bucket_name_dns_conformity(resource['bucket']):
c301244 * s3cmd, S3/S3.py, S3/Config.py: Allow access to upper-case
ludvigm authored Sep 1, 2008
89 uri = "/%s%s" % (resource['bucket'], resource['uri'])
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored Nov 13, 2007
90 else:
91 uri = resource['uri']
c301244 * s3cmd, S3/S3.py, S3/Config.py: Allow access to upper-case
ludvigm authored Sep 1, 2008
92 if self.config.proxy_host != "":
93 uri = "http://%s%s" % (self.get_hostname(resource['bucket']), uri)
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored Nov 13, 2007
94 debug('format_uri(): ' + uri)
95 return uri
1fb8944 2007-07-05 Michal Ludvig <michal@logix.cz>
ludvigm authored Jul 5, 2007
96
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
97 ## Commands / Actions
98 def list_all_buckets(self):
99 request = self.create_request("LIST_ALL_BUCKETS")
100 response = self.send_request(request)
101 response["list"] = getListFromXml(response["data"], "Bucket")
102 return response
103
104 def bucket_list(self, bucket, prefix = None):
169af58 * S3/S3.py: Support for buckets with over 1000 objects.
ludvigm authored Sep 13, 2007
105 def _list_truncated(data):
925f2de 2007-09-25 Michal Ludvig <michal@logix.cz>
ludvigm authored Sep 25, 2007
106 ## <IsTruncated> can either be "true" or "false" or be missing completely
107 is_truncated = getTextFromXml(data, ".//IsTruncated") or "false"
108 return is_truncated.lower() != "false"
169af58 * S3/S3.py: Support for buckets with over 1000 objects.
ludvigm authored Sep 13, 2007
109
110 def _get_contents(data):
111 return getListFromXml(data, "Contents")
112
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
113 request = self.create_request("BUCKET_LIST", bucket = bucket, prefix = prefix)
114 response = self.send_request(request)
7fda7c7 2007-09-02 Michal Ludvig <michal@logix.cz>
ludvigm authored Sep 1, 2007
115 #debug(response)
169af58 * S3/S3.py: Support for buckets with over 1000 objects.
ludvigm authored Sep 13, 2007
116 list = _get_contents(response["data"])
117 while _list_truncated(response["data"]):
118 marker = list[-1]["Key"]
20cb0e0 * s3cmd, S3/S3.py: Make --verbose mode more useful and default
ludvigm authored Sep 3, 2008
119 debug("Listing continues after '%s'" % marker)
a878770 2008-02-11 Michal Ludvig <michal@logix.cz>
ludvigm authored Feb 11, 2008
120 request = self.create_request("BUCKET_LIST", bucket = bucket,
121 prefix = prefix,
122 marker = self.urlencode_string(marker))
169af58 * S3/S3.py: Support for buckets with over 1000 objects.
ludvigm authored Sep 13, 2007
123 response = self.send_request(request)
124 list += _get_contents(response["data"])
125 response['list'] = list
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
126 return response
127
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored Nov 13, 2007
128 def bucket_create(self, bucket, bucket_location = None):
1fb8944 2007-07-05 Michal Ludvig <michal@logix.cz>
ludvigm authored Jul 5, 2007
129 headers = SortedDict()
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored Nov 13, 2007
130 body = ""
131 if bucket_location and bucket_location.strip().upper() != "US":
132 body = "<CreateBucketConfiguration><LocationConstraint>"
133 body += bucket_location.strip().upper()
134 body += "</LocationConstraint></CreateBucketConfiguration>"
135 debug("bucket_location: " + body)
306a4a7 * S3/S3.py: "s3cmd mb" can create upper-case buckets again
ludvigm authored Sep 15, 2008
136 self.check_bucket_name(bucket, dns_strict = True)
137 else:
138 self.check_bucket_name(bucket, dns_strict = False)
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored Nov 13, 2007
139 headers["content-length"] = len(body)
a878770 2008-02-11 Michal Ludvig <michal@logix.cz>
ludvigm authored Feb 11, 2008
140 if self.config.acl_public:
141 headers["x-amz-acl"] = "public-read"
1fb8944 2007-07-05 Michal Ludvig <michal@logix.cz>
ludvigm authored Jul 5, 2007
142 request = self.create_request("BUCKET_CREATE", bucket = bucket, headers = headers)
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored Nov 13, 2007
143 response = self.send_request(request, body)
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
144 return response
145
146 def bucket_delete(self, bucket):
147 request = self.create_request("BUCKET_DELETE", bucket = bucket)
148 response = self.send_request(request)
149 return response
150
d8a573e 2008-02-27 Michal Ludvig <michal@logix.cz>
ludvigm authored Feb 27, 2008
151 def bucket_info(self, uri):
152 request = self.create_request("BUCKET_LIST", bucket = uri.bucket(), extra = "?location")
889b7c1 * s3cmd, S3/S3.py: New command 'ib' to get information about
ludvigm authored Nov 12, 2007
153 response = self.send_request(request)
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored Nov 13, 2007
154 response['bucket-location'] = getTextFromXml(response['data'], "LocationConstraint") or "any"
889b7c1 * s3cmd, S3/S3.py: New command 'ib' to get information about
ludvigm authored Nov 12, 2007
155 return response
156
26f22ff * S3/S3.py: removed object_{get,put,delete}_uri() functions
ludvigm authored Sep 1, 2008
157 def object_put(self, filename, uri, extra_headers = None):
158 # TODO TODO
159 # Make it consistent with stream-oriented object_get()
160 if uri.type != "s3":
161 raise ValueError("Expected URI type 's3', got '%s'" % uri.type)
162
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
163 if not os.path.isfile(filename):
164 raise ParameterError("%s is not a regular file" % filename)
165 try:
cf51d50 * S3/S3.py, S3/Utils.py: open files in binary mode (otherwise windows
ludvigm authored Feb 27, 2008
166 file = open(filename, "rb")
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
167 size = os.stat(filename)[ST_SIZE]
168 except IOError, e:
169 raise ParameterError("%s: %s" % (filename, e.strerror))
170 headers = SortedDict()
fd56bd5 2007-05-27 Michal Ludvig <michal@logix.cz>
ludvigm authored May 27, 2007
171 if extra_headers:
172 headers.update(extra_headers)
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
173 headers["content-length"] = size
1202afb 2008-02-11 Michal Ludvig <michal@logix.cz>
ludvigm authored Feb 11, 2008
174 content_type = None
175 if self.config.guess_mime_type:
176 content_type = mimetypes.guess_type(filename)[0]
177 if not content_type:
178 content_type = self.config.default_mime_type
179 debug("Content-Type set to '%s'" % content_type)
180 headers["content-type"] = content_type
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
181 if self.config.acl_public:
182 headers["x-amz-acl"] = "public-read"
26f22ff * S3/S3.py: removed object_{get,put,delete}_uri() functions
ludvigm authored Sep 1, 2008
183 request = self.create_request("OBJECT_PUT", uri = uri, headers = headers)
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
184 response = self.send_file(request, file)
185 return response
186
26f22ff * S3/S3.py: removed object_{get,put,delete}_uri() functions
ludvigm authored Sep 1, 2008
187 def object_get(self, uri, stream):
0f4094f Implemented S3->local sync
ludvigm authored Jun 4, 2008
188 if uri.type != "s3":
189 raise ValueError("Expected URI type 's3', got '%s'" % uri.type)
26f22ff * S3/S3.py: removed object_{get,put,delete}_uri() functions
ludvigm authored Sep 1, 2008
190 request = self.create_request("OBJECT_GET", uri = uri)
a580bd1 - New feature - allow "get" to stdout
mludvig authored Feb 19, 2007
191 response = self.recv_file(request, stream)
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
192 return response
0f4094f Implemented S3->local sync
ludvigm authored Jun 4, 2008
193
26f22ff * S3/S3.py: removed object_{get,put,delete}_uri() functions
ludvigm authored Sep 1, 2008
194 def object_delete(self, uri):
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
195 if uri.type != "s3":
196 raise ValueError("Expected URI type 's3', got '%s'" % uri.type)
26f22ff * S3/S3.py: removed object_{get,put,delete}_uri() functions
ludvigm authored Sep 1, 2008
197 request = self.create_request("OBJECT_DELETE", uri = uri)
198 response = self.send_request(request)
199 return response
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
200
d8a573e 2008-02-27 Michal Ludvig <michal@logix.cz>
ludvigm authored Feb 27, 2008
201 def object_info(self, uri):
26f22ff * S3/S3.py: removed object_{get,put,delete}_uri() functions
ludvigm authored Sep 1, 2008
202 request = self.create_request("OBJECT_HEAD", uri = uri)
d8a573e 2008-02-27 Michal Ludvig <michal@logix.cz>
ludvigm authored Feb 27, 2008
203 response = self.send_request(request)
204 return response
205
206 def get_acl(self, uri):
207 if uri.has_object():
26f22ff * S3/S3.py: removed object_{get,put,delete}_uri() functions
ludvigm authored Sep 1, 2008
208 request = self.create_request("OBJECT_GET", uri = uri, extra = "?acl")
d8a573e 2008-02-27 Michal Ludvig <michal@logix.cz>
ludvigm authored Feb 27, 2008
209 else:
210 request = self.create_request("BUCKET_LIST", bucket = uri.bucket(), extra = "?acl")
211 acl = {}
212 response = self.send_request(request)
213 grants = getListFromXml(response['data'], "Grant")
214 for grant in grants:
215 if grant['Grantee'][0].has_key('DisplayName'):
216 user = grant['Grantee'][0]['DisplayName']
217 if grant['Grantee'][0].has_key('URI'):
218 user = grant['Grantee'][0]['URI']
219 if user == 'http://acs.amazonaws.com/groups/global/AllUsers':
220 user = "*anon*"
221 perm = grant['Permission']
222 acl[user] = perm
223 return acl
224
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
225 ## Low level methods
8192aed 2007-08-13 Michal Ludvig <michal@logix.cz>
ludvigm authored Aug 13, 2007
226 def urlencode_string(self, string):
8b940d1 * s3cmd, S3/S3Uri.py, S3/S3.py: All internal representations of
ludvigm authored Sep 9, 2008
227 if type(string) == unicode:
228 string = string.encode("utf-8")
8192aed 2007-08-13 Michal Ludvig <michal@logix.cz>
ludvigm authored Aug 13, 2007
229 encoded = ""
230 ## List of characters that must be escaped for S3
231 ## Haven't found this in any official docs
232 ## but my tests show it's more less correct.
233 ## If you start getting InvalidSignature errors
234 ## from S3 check the error headers returned
235 ## from S3 to see whether the list hasn't
236 ## changed.
237 for c in string: # I'm not sure how to know in what encoding
238 # 'object' is. Apparently "type(object)==str"
239 # but the contents is a string of unicode
240 # bytes, e.g. '\xc4\x8d\xc5\xafr\xc3\xa1k'
241 # Don't know what it will do on non-utf8
242 # systems.
243 # [hope that sounds reassuring ;-)]
244 o = ord(c)
245 if (o <= 32 or # Space and below
246 o == 0x22 or # "
247 o == 0x23 or # #
248 o == 0x25 or # %
249 o == 0x2B or # + (or it would become <space>)
250 o == 0x3C or # <
251 o == 0x3E or # >
252 o == 0x3F or # ?
253 o == 0x5B or # [
254 o == 0x5C or # \
255 o == 0x5D or # ]
256 o == 0x5E or # ^
257 o == 0x60 or # `
258 o >= 123): # { and above, including >= 128 for UTF-8
259 encoded += "%%%02X" % o
260 else:
261 encoded += c
262 debug("String '%s' encoded to '%s'" % (string, encoded))
263 return encoded
264
26f22ff * S3/S3.py: removed object_{get,put,delete}_uri() functions
ludvigm authored Sep 1, 2008
265 def create_request(self, operation, uri = None, bucket = None, object = None, headers = None, extra = None, **params):
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored Nov 13, 2007
266 resource = { 'bucket' : None, 'uri' : "/" }
26f22ff * S3/S3.py: removed object_{get,put,delete}_uri() functions
ludvigm authored Sep 1, 2008
267
268 if uri and (bucket or object):
269 raise ValueError("Both 'uri' and either 'bucket' or 'object' parameters supplied")
270 ## If URI is given use that instead of bucket/object parameters
271 if uri:
272 bucket = uri.bucket()
273 object = uri.has_object() and uri.object() or None
274
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
275 if bucket:
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored Nov 13, 2007
276 resource['bucket'] = str(bucket)
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
277 if object:
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored Nov 13, 2007
278 resource['uri'] = "/" + self.urlencode_string(object)
279 if extra:
280 resource['uri'] += extra
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
281
282 if not headers:
283 headers = SortedDict()
284
285 if headers.has_key("date"):
286 if not headers.has_key("x-amz-date"):
287 headers["x-amz-date"] = headers["date"]
288 del(headers["date"])
289
290 if not headers.has_key("x-amz-date"):
3946150 * S3/S3.py: modify 'x-amz-date' format (problems reported on MacOS X).
ludvigm authored Feb 27, 2008
291 headers["x-amz-date"] = time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime())
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
292
293 method_string = S3.http_methods.getkey(S3.operations[operation] & S3.http_methods["MASK"])
294 signature = self.sign_headers(method_string, resource, headers)
295 headers["Authorization"] = "AWS "+self.config.access_key+":"+signature
296 param_str = ""
297 for param in params:
298 if params[param] not in (None, ""):
299 param_str += "&%s=%s" % (param, params[param])
889b7c1 * s3cmd, S3/S3.py: New command 'ib' to get information about
ludvigm authored Nov 12, 2007
300 else:
301 param_str += "&%s" % param
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
302 if param_str != "":
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored Nov 13, 2007
303 resource['uri'] += "?" + param_str[1:]
304 debug("CreateRequest: resource[uri]=" + resource['uri'])
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
305 return (method_string, resource, headers)
306
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored Nov 13, 2007
307 def send_request(self, request, body = None):
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
308 method_string, resource, headers = request
20cb0e0 * s3cmd, S3/S3.py: Make --verbose mode more useful and default
ludvigm authored Sep 3, 2008
309 debug("Processing request, please wait...")
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored Nov 13, 2007
310 conn = self.get_connection(resource['bucket'])
311 conn.request(method_string, self.format_uri(resource), body, headers)
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
312 response = {}
313 http_response = conn.getresponse()
314 response["status"] = http_response.status
315 response["reason"] = http_response.reason
316 response["headers"] = convertTupleListToDict(http_response.getheaders())
317 response["data"] = http_response.read()
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored Nov 13, 2007
318 debug("Response: " + str(response))
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
319 conn.close()
3ecbb36 2008-02-11 Michal Ludvig <michal@logix.cz>
ludvigm authored Feb 11, 2008
320
321 if response["status"] == 307:
322 ## RedirectPermanent
323 redir_bucket = getTextFromXml(response['data'], ".//Bucket")
324 redir_hostname = getTextFromXml(response['data'], ".//Endpoint")
325 self.set_hostname(redir_bucket, redir_hostname)
20cb0e0 * s3cmd, S3/S3.py: Make --verbose mode more useful and default
ludvigm authored Sep 3, 2008
326 warning("Redirected to: %s" % (redir_hostname))
3ecbb36 2008-02-11 Michal Ludvig <michal@logix.cz>
ludvigm authored Feb 11, 2008
327 return self.send_request(request, body)
328
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
329 if response["status"] < 200 or response["status"] > 299:
330 raise S3Error(response)
331 return response
332
b1eaa42 * s3cmd, S3/S3.py, S3/Utils.py: Throttle upload speed and retry
ludvigm authored Mar 4, 2008
333 def send_file(self, request, file, throttle = 0, retries = 3):
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
334 method_string, resource, headers = request
335 info("Sending file '%s', please wait..." % file.name)
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored Nov 13, 2007
336 conn = self.get_connection(resource['bucket'])
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
337 conn.connect()
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored Nov 13, 2007
338 conn.putrequest(method_string, self.format_uri(resource))
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
339 for header in headers.keys():
340 conn.putheader(header, str(headers[header]))
341 conn.endheaders()
3ecbb36 2008-02-11 Michal Ludvig <michal@logix.cz>
ludvigm authored Feb 11, 2008
342 file.seek(0)
b1eaa42 * s3cmd, S3/S3.py, S3/Utils.py: Throttle upload speed and retry
ludvigm authored Mar 4, 2008
343 timestamp_start = time.time()
ab13aa8 * S3/S3.py: send_file() now computes MD5 sum of the file
ludvigm authored Apr 28, 2008
344 md5_hash = md5.new()
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
345 size_left = size_total = headers.get("content-length")
346 while (size_left > 0):
347 debug("SendFile: Reading up to %d bytes from '%s'" % (self.config.send_chunk, file.name))
348 data = file.read(self.config.send_chunk)
ab13aa8 * S3/S3.py: send_file() now computes MD5 sum of the file
ludvigm authored Apr 28, 2008
349 md5_hash.update(data)
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
350 debug("SendFile: Sending %d bytes to the server" % len(data))
b1eaa42 * s3cmd, S3/S3.py, S3/Utils.py: Throttle upload speed and retry
ludvigm authored Mar 4, 2008
351 try:
352 conn.send(data)
353 except Exception, e:
354 ## When an exception occurs insert a
355 if retries:
356 conn.close()
357 warning("Upload of '%s' failed %s " % (file.name, e))
358 throttle = throttle and throttle * 5 or 0.01
359 warning("Retrying on lower speed (throttle=%0.2f)" % throttle)
360 return self.send_file(request, file, throttle, retries - 1)
361 else:
362 debug("Giving up on '%s' %s" % (file.name, e))
363 raise S3UploadError
364
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
365 size_left -= len(data)
b1eaa42 * s3cmd, S3/S3.py, S3/Utils.py: Throttle upload speed and retry
ludvigm authored Mar 4, 2008
366 if throttle:
367 time.sleep(throttle)
20cb0e0 * s3cmd, S3/S3.py: Make --verbose mode more useful and default
ludvigm authored Sep 3, 2008
368 ## Call progress meter from here
369 debug("Sent %d bytes (%d %% of %d)" % (
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
370 (size_total - size_left),
371 (size_total - size_left) * 100 / size_total,
372 size_total))
b1eaa42 * s3cmd, S3/S3.py, S3/Utils.py: Throttle upload speed and retry
ludvigm authored Mar 4, 2008
373 timestamp_end = time.time()
ab13aa8 * S3/S3.py: send_file() now computes MD5 sum of the file
ludvigm authored Apr 28, 2008
374 md5_computed = md5_hash.hexdigest()
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
375 response = {}
376 http_response = conn.getresponse()
377 response["status"] = http_response.status
378 response["reason"] = http_response.reason
379 response["headers"] = convertTupleListToDict(http_response.getheaders())
b1eaa42 * s3cmd, S3/S3.py, S3/Utils.py: Throttle upload speed and retry
ludvigm authored Mar 4, 2008
380 response["data"] = http_response.read()
381 response["elapsed"] = timestamp_end - timestamp_start
382 response["size"] = size_total
2644cd7 * S3/S3.py: Don't run into ZeroDivisionError when speed counter
ludvigm authored Sep 15, 2008
383 response["speed"] = response["elapsed"] and float(response["size"]) / response["elapsed"] or float(-1)
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
384 conn.close()
3ecbb36 2008-02-11 Michal Ludvig <michal@logix.cz>
ludvigm authored Feb 11, 2008
385
386 if response["status"] == 307:
387 ## RedirectPermanent
388 redir_bucket = getTextFromXml(response['data'], ".//Bucket")
389 redir_hostname = getTextFromXml(response['data'], ".//Endpoint")
390 self.set_hostname(redir_bucket, redir_hostname)
20cb0e0 * s3cmd, S3/S3.py: Make --verbose mode more useful and default
ludvigm authored Sep 3, 2008
391 warning("Redirected to: %s" % (redir_hostname))
3ecbb36 2008-02-11 Michal Ludvig <michal@logix.cz>
ludvigm authored Feb 11, 2008
392 return self.send_file(request, file)
393
8696851 * S3/S3.py: Re-upload when Amazon doesn't send ETag
ludvigm authored Jun 30, 2008
394 # S3 from time to time doesn't send ETag back in a response :-(
395 # Force re-upload here.
396 if not response['headers'].has_key('etag'):
397 response['headers']['etag'] = ''
398
ab13aa8 * S3/S3.py: send_file() now computes MD5 sum of the file
ludvigm authored Apr 28, 2008
399 debug("MD5 sums: computed=%s, received=%s" % (md5_computed, response["headers"]["etag"]))
400 if response["headers"]["etag"].strip('"\'') != md5_hash.hexdigest():
401 warning("MD5 Sums don't match!")
402 if retries:
20cb0e0 * s3cmd, S3/S3.py: Make --verbose mode more useful and default
ludvigm authored Sep 3, 2008
403 warning("Retrying upload of %s" % (file.name))
ab13aa8 * S3/S3.py: send_file() now computes MD5 sum of the file
ludvigm authored Apr 28, 2008
404 return self.send_file(request, file, throttle, retries - 1)
405 else:
20cb0e0 * s3cmd, S3/S3.py: Make --verbose mode more useful and default
ludvigm authored Sep 3, 2008
406 warning("Too many failures. Giving up on '%s'" % (file.name))
ab13aa8 * S3/S3.py: send_file() now computes MD5 sum of the file
ludvigm authored Apr 28, 2008
407 raise S3UploadError
408
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
409 if response["status"] < 200 or response["status"] > 299:
410 raise S3Error(response)
411 return response
412
a580bd1 - New feature - allow "get" to stdout
mludvig authored Feb 19, 2007
413 def recv_file(self, request, stream):
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
414 method_string, resource, headers = request
a580bd1 - New feature - allow "get" to stdout
mludvig authored Feb 19, 2007
415 info("Receiving file '%s', please wait..." % stream.name)
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored Nov 13, 2007
416 conn = self.get_connection(resource['bucket'])
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
417 conn.connect()
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored Nov 13, 2007
418 conn.putrequest(method_string, self.format_uri(resource))
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
419 for header in headers.keys():
420 conn.putheader(header, str(headers[header]))
421 conn.endheaders()
422 response = {}
423 http_response = conn.getresponse()
424 response["status"] = http_response.status
425 response["reason"] = http_response.reason
426 response["headers"] = convertTupleListToDict(http_response.getheaders())
3ecbb36 2008-02-11 Michal Ludvig <michal@logix.cz>
ludvigm authored Feb 11, 2008
427
428 if response["status"] == 307:
429 ## RedirectPermanent
430 response['data'] = http_response.read()
431 redir_bucket = getTextFromXml(response['data'], ".//Bucket")
432 redir_hostname = getTextFromXml(response['data'], ".//Endpoint")
433 self.set_hostname(redir_bucket, redir_hostname)
20cb0e0 * s3cmd, S3/S3.py: Make --verbose mode more useful and default
ludvigm authored Sep 3, 2008
434 warning("Redirected to: %s" % (redir_hostname))
3ecbb36 2008-02-11 Michal Ludvig <michal@logix.cz>
ludvigm authored Feb 11, 2008
435 return self.recv_file(request, stream)
436
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
437 if response["status"] < 200 or response["status"] > 299:
438 raise S3Error(response)
439
440 md5_hash = md5.new()
441 size_left = size_total = int(response["headers"]["content-length"])
a580bd1 - New feature - allow "get" to stdout
mludvig authored Feb 19, 2007
442 size_recvd = 0
b1eaa42 * s3cmd, S3/S3.py, S3/Utils.py: Throttle upload speed and retry
ludvigm authored Mar 4, 2008
443 timestamp_start = time.time()
a580bd1 - New feature - allow "get" to stdout
mludvig authored Feb 19, 2007
444 while (size_recvd < size_total):
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
445 this_chunk = size_left > self.config.recv_chunk and self.config.recv_chunk or size_left
446 debug("ReceiveFile: Receiving up to %d bytes from the server" % this_chunk)
447 data = http_response.read(this_chunk)
a580bd1 - New feature - allow "get" to stdout
mludvig authored Feb 19, 2007
448 debug("ReceiveFile: Writing %d bytes to file '%s'" % (len(data), stream.name))
449 stream.write(data)
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
450 md5_hash.update(data)
a580bd1 - New feature - allow "get" to stdout
mludvig authored Feb 19, 2007
451 size_recvd += len(data)
20cb0e0 * s3cmd, S3/S3.py: Make --verbose mode more useful and default
ludvigm authored Sep 3, 2008
452 ## Call progress meter from here...
453 debug("Received %d bytes (%d %% of %d)" % (
a580bd1 - New feature - allow "get" to stdout
mludvig authored Feb 19, 2007
454 size_recvd,
455 size_recvd * 100 / size_total,
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
456 size_total))
457 conn.close()
b1eaa42 * s3cmd, S3/S3.py, S3/Utils.py: Throttle upload speed and retry
ludvigm authored Mar 4, 2008
458 timestamp_end = time.time()
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
459 response["md5"] = md5_hash.hexdigest()
460 response["md5match"] = response["headers"]["etag"].find(response["md5"]) >= 0
b1eaa42 * s3cmd, S3/S3.py, S3/Utils.py: Throttle upload speed and retry
ludvigm authored Mar 4, 2008
461 response["elapsed"] = timestamp_end - timestamp_start
a580bd1 - New feature - allow "get" to stdout
mludvig authored Feb 19, 2007
462 response["size"] = size_recvd
2644cd7 * S3/S3.py: Don't run into ZeroDivisionError when speed counter
ludvigm authored Sep 15, 2008
463 response["speed"] = response["elapsed"] and float(response["size"]) / response["elapsed"] or float(-1)
a580bd1 - New feature - allow "get" to stdout
mludvig authored Feb 19, 2007
464 if response["size"] != long(response["headers"]["content-length"]):
465 warning("Reported size (%s) does not match received size (%s)" % (
466 response["headers"]["content-length"], response["size"]))
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
467 debug("ReceiveFile: Computed MD5 = %s" % response["md5"])
468 if not response["md5match"]:
469 warning("MD5 signatures do not match: computed=%s, received=%s" % (
470 response["md5"], response["headers"]["etag"]))
471 return response
472
473 def sign_headers(self, method, resource, headers):
474 h = method+"\n"
475 h += headers.get("content-md5", "")+"\n"
476 h += headers.get("content-type", "")+"\n"
477 h += headers.get("date", "")+"\n"
478 for header in headers.keys():
479 if header.startswith("x-amz-"):
480 h += header+":"+str(headers[header])+"\n"
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored Nov 13, 2007
481 if resource['bucket']:
482 h += "/" + resource['bucket']
483 h += resource['uri']
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
484 debug("SignHeaders: " + repr(h))
485 return base64.encodestring(hmac.new(self.config.secret_key, h, sha).digest()).strip()
486
ad4fd61 * S3/S3Uri.py: Display public URLs correctly for non-DNS buckets.
ludvigm authored Sep 15, 2008
487 @staticmethod
488 def check_bucket_name(bucket, dns_strict = True):
e353d9a * s3cmd, S3/S3.py, S3/Config.py: Removed --use-old-connect-method
ludvigm authored Sep 3, 2008
489 if dns_strict:
490 invalid = re.search("([^a-z0-9\.-])", bucket)
491 if invalid:
492 raise ParameterError("Bucket name '%s' contains disallowed character '%s'. The only supported ones are: lowercase us-ascii letters (a-z), digits (0-9), dot (.) and hyphen (-)." % (bucket, invalid.groups()[0]))
493 else:
494 invalid = re.search("([^A-Za-z0-9\._-])", bucket)
495 if invalid:
496 raise ParameterError("Bucket name '%s' contains disallowed character '%s'. The only supported ones are: us-ascii letters (a-z, A-Z), digits (0-9), dot (.), hyphen (-) and underscore (_)." % (bucket, invalid.groups()[0]))
497
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
498 if len(bucket) < 3:
499 raise ParameterError("Bucket name '%s' is too short (min 3 characters)" % bucket)
500 if len(bucket) > 255:
501 raise ParameterError("Bucket name '%s' is too long (max 255 characters)" % bucket)
e353d9a * s3cmd, S3/S3.py, S3/Config.py: Removed --use-old-connect-method
ludvigm authored Sep 3, 2008
502 if dns_strict:
503 if len(bucket) > 63:
504 raise ParameterError("Bucket name '%s' is too long (max 63 characters)" % bucket)
505 if re.search("-\.", bucket):
506 raise ParameterError("Bucket name '%s' must not contain sequence '-.' for DNS compatibility" % bucket)
507 if re.search("\.\.", bucket):
508 raise ParameterError("Bucket name '%s' must not contain sequence '..' for DNS compatibility" % bucket)
509 if not re.search("^[0-9a-z]", bucket):
510 raise ParameterError("Bucket name '%s' must start with a letter or a digit" % bucket)
511 if not re.search("[0-9a-z]$", bucket):
512 raise ParameterError("Bucket name '%s' must end with a letter or a digit" % bucket)
45d65cf Renamed s3py to s3cmd
mludvig authored Feb 7, 2007
513 return True
514
ad4fd61 * S3/S3Uri.py: Display public URLs correctly for non-DNS buckets.
ludvigm authored Sep 15, 2008
515 @staticmethod
516 def check_bucket_name_dns_conformity(bucket):
e353d9a * s3cmd, S3/S3.py, S3/Config.py: Removed --use-old-connect-method
ludvigm authored Sep 3, 2008
517 try:
ad4fd61 * S3/S3Uri.py: Display public URLs correctly for non-DNS buckets.
ludvigm authored Sep 15, 2008
518 return S3.check_bucket_name(bucket, dns_strict = True)
e353d9a * s3cmd, S3/S3.py, S3/Config.py: Removed --use-old-connect-method
ludvigm authored Sep 3, 2008
519 except ParameterError:
520 return False
Something went wrong with that request. Please try again.